def test_columns_task_creates_columns_only_when_run():
task = TestColumnsTask()
with session_scope() as session:
assert_equals(session.query(OBSColumn).count(), 0)
runtask(task)
with session_scope() as session:
assert_equals(session.query(OBSColumn).count(), 2)
assert_equals(session.query(OBSColumnToColumn).count(), 1)
assert_equals(task.output()['pop'].get(session).id, '"test_util".population')
assert_equals(task.output()['foobar'].get(session).id, '"test_util".foobar')
pop = session.query(OBSColumn).get('"test_util".population')
foobar = session.query(OBSColumn).get('"test_util".foobar')
assert_equals(len(pop.sources), 1)
assert_equals(len(foobar.targets), 1)
assert_equals(pop.sources.keys()[0].id, foobar.id)
assert_equals(foobar.targets.keys()[0].id, pop.id)
assert_equals(True, task.complete())
table = OBSTable(id='table', tablename='tablename')
with session_scope() as session:
coltable = OBSColumnTable(column=task.output()['pop'].get(session),
table=table, colname='colnamaste')
session.add(table)
session.add(coltable)
with session_scope() as session:
assert_equals(session.query(OBSColumnTable).count(), 1)
def test_table_task_qualifies_table_name_schema():
task = TestTableTask()
runtask(task)
assert_equals(task.table.schema, 'test_util')
assert_equals(task.table.name, 'test_table_task_alpha_1996_beta_5000')
def test_columns_task_fails_no_columns():
class TestColumnsTask(ColumnsTask):
pass
task = TestColumnsTask()
with assert_raises(NotImplementedError):
runtask(task)
def test_columns_task_creates_columns_only_when_run():
task = TestColumnsTask()
with session_scope() as session:
assert_equals(session.query(OBSColumn).count(), 0)
runtask(task)
with session_scope() as session:
assert_equals(session.query(OBSColumn).count(), 2)
assert_equals(session.query(OBSColumnToColumn).count(), 1)
assert_equals(task.output()['pop'].get(session).id,
'test_util.population')
assert_equals(task.output()['foobar'].get(session).id,
'test_util.foobar')
pop = session.query(OBSColumn).get('test_util.population')
foobar = session.query(OBSColumn).get('test_util.foobar')
assert_equals(len(pop.sources), 1)
assert_equals(len(foobar.targets), 1)
assert_equals(list(pop.sources.keys())[0].id, foobar.id)
assert_equals(list(foobar.targets.keys())[0].id, pop.id)
assert_equals(True, task.complete())
table = OBSTable(id='table', tablename='tablename')
with session_scope() as session:
coltable = OBSColumnTable(column=task.output()['pop'].get(session),
table=table,
colname='colnamaste')
session.add(table)
session.add(coltable)
with session_scope() as session:
assert_equals(session.query(OBSColumnTable).count(), 1)
def test_empty_obs_meta_to_local():
'''
OBSMetaToLocal should work even if tables are empty. Should result in
creation of blank obs_meta, obs_meta_numer, obs_meta_denom, obs_meta_geom,
obs_meta_timespan tables.
'''
imp.reload(tasks.carto)
task = tasks.carto.OBSMetaToLocal()
runtask(task)
session = current_session()
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta').fetchone()[0], 0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_numer').fetchone()[0],
0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_denom').fetchone()[0],
0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom').fetchone()[0], 0)
assert_equals(
session.execute('SELECT COUNT(*) FROM observatory.obs_meta_timespan').
fetchone()[0], 0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom_numer_timespan').
fetchone()[0], 0)
def test_table_task(klass, params):
'''
Test {} task with {}.
This does not clear out all database artifacts between runs, for
performance reasons. The order of decorators could be switched to enforce
cleaner behavior, but running all necessary ColumnsTasks repeatedly tends
to be very slow.
'''.format(klass, params)
params.update(get_custom_parameters(klass.__name__))
task = klass(**params)
runtask(task, superclasses=[TagsTask, ColumnsTask, TableTask])
imp.reload(tasks.carto)
obs_meta_to_local = tasks.carto.OBSMetaToLocal()
runtask(obs_meta_to_local)
session = current_session()
assert_greater(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta').fetchone()[0], 0)
session.execute('DROP TABLE observatory.obs_meta')
session.execute('DELETE FROM observatory.obs_table')
session.commit()
def test_columns_task_fails_no_columns():
class TestColumnsTask(ColumnsTask):
pass
task = TestColumnsTask()
with assert_raises(NotImplementedError):
runtask(task)
def test_obs_meta_to_local_works_twice():
'''
Should be possible to run OBSMetaToLocal twice in a row.
'''
imp.reload(tasks.carto)
task = tasks.carto.OBSMetaToLocal()
runtask(task)
runtask(task)
session = current_session()
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta').fetchone()[0], 0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_numer').fetchone()[0],
0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_denom').fetchone()[0],
0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom').fetchone()[0], 0)
assert_equals(
session.execute('SELECT COUNT(*) FROM observatory.obs_meta_timespan').
fetchone()[0], 0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom_numer_timespan').
fetchone()[0], 0)
def test_column_task(klass):
# Ensure every column task runs and produces some kind of independent
# metadata.
# TODO: test columnstasks that have params
if klass.get_param_names():
raise SkipTest("Cannot test ColumnsTask with params")
task = klass()
runtask(task)
assert_greater(current_session().query(OBSColumn).filter(OBSColumn.id.startswith(classpath(task))).count(), 0)
def test_table_task_table():
task = TestTableTask()
runtask(task)
with session_scope() as session:
assert_equals('"{schema}".{name}'.format(schema=task.table.schema,
name=task.table.name),
task.output().get(session).id)
def test_download_unzip_task():
'''
Download unzip task should download remote assets and unzip them locally.
'''
task = TestDownloadUnzipTask()
if task.output().exists():
shell('rm -r {}'.format(task.output().path))
assert_false(task.output().exists())
runtask(task)
assert_true(task.output().exists())
def test_download_unzip_task():
'''
Download unzip task should download remote assets and unzip them locally.
'''
task = TestRepoFileUnzipTask()
if task.output().exists():
shell('rm -r {}'.format(task.output().path))
assert_false(task.output().exists())
runtask(task)
assert_true(task.output().exists())
def test_column_task(klass):
# Ensure every column task runs and produces some kind of independent
# metadata.
# TODO: test columnstasks that have params
if klass.get_param_names():
raise SkipTest('Cannot test ColumnsTask with params')
task = klass()
runtask(task)
assert_greater(
current_session().query(OBSColumn).filter(
OBSColumn.id.startswith(classpath(task))).count(), 0)
def test_table_task_creates_columns_when_run():
task = TestTableTask()
assert_equals(False, task.complete())
runtask(task)
assert_equals(True, task.complete())
with session_scope() as session:
assert_equals(session.query(OBSColumn).count(), 2)
assert_equals(session.query(OBSColumnTable).count(), 2)
assert_equals(session.query(OBSTable).count(), 1)
assert_in(task.output().table, metadata.tables)
def test_table_task_creates_columns_when_run():
task = TestTableTask()
assert_equals(False, task.complete())
runtask(task)
assert_equals(True, task.complete())
with session_scope() as session:
assert_equals(session.query(OBSColumn).count(), 2)
assert_equals(session.query(OBSColumnTable).count(), 2)
assert_equals(session.query(OBSTable).count(), 1)
assert_in(task.table.fullname, metadata.tables)
def test_table_task():
'''
Very simple table task should run and make an entry in OBSTable, as
well as a physical table with rows in observatory schema.
'''
task = TestTableTask()
runtask(task)
table = task.output().get(current_session())
assert_is_not_none(table)
assert_is_none(table.the_geom)
assert_equal(current_session().execute(
'SELECT COUNT(*) FROM observatory.{}'.format(
table.tablename)).fetchone()[0], 1)
def test_table_task():
'''
Very simple table task should run and make an entry in OBSTable, as
well as a physical table with rows in observatory schema.
'''
task = TestTableTask()
runtask(task)
table = task.output().get(current_session())
assert_is_not_none(table)
assert_is_none(table.the_geom)
assert_equal(
current_session().execute('SELECT COUNT(*) FROM observatory.{}'.format(
table.tablename)).fetchone()[0], 1)
def test_geom_table_task_update():
'''
Should be possible to generate a new version of a table by incrementing
the version number.
'''
task = TestGeomTableTask()
runtask(task)
assert_true(task.complete())
task = TestGeomTableTask()
task.version = lambda: 10
assert_false(task.complete())
current_session().commit()
runtask(task)
assert_true(task.complete())
def test_geom_table_task_update():
'''
Should be possible to generate a new version of a table by incrementing
the version number.
'''
task = TestGeomTableTask()
runtask(task)
assert_true(task.complete())
task = TestGeomTableTask()
task.version = lambda: 10
assert_false(task.complete())
current_session().commit()
runtask(task)
assert_true(task.complete())
def test_table_task_increment_version_runs_again():
task = TestTableTask()
runtask(task)
output = task.output()
assert_true(output.exists())
task = TestTableTask()
task.version = lambda: 10
output = task.output()
assert_false(output.exists())
current_session().rollback()
runtask(task)
assert_true(output.exists())
def test_table_task_increment_version_runs_again():
task = TestTableTask()
runtask(task)
output = task.output()
assert_true(output.exists())
task = TestTableTask()
task.version = lambda: 10
output = task.output()
assert_false(output.exists())
current_session().rollback()
runtask(task)
assert_true(output.exists())
def test_geom_table_task():
'''
Table task with geoms should run and generate an entry in OBSTable with
a summary geom, plus a physical table with rows in observatory schema.
'''
with session_scope() as session:
task = TestGeomTableTask()
runtask(task)
table = task.output().get(session)
assert_is_not_none(table)
assert_is_not_none(table.the_geom)
assert_equal(session.execute(
'SELECT COUNT(*) FROM observatory.{}'.format(
table.tablename)).fetchone()[0], 1)
def test_carto_2_temp_table_task():
'''
Convert a table on CARTO to a temp table.
'''
task = TestCSV2TempTableTask()
before_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
runtask(task)
table = task.output()
assert_equal(
current_session().execute('SELECT COUNT(*) FROM {}'.format(
table.table)).fetchone()[0], 10)
after_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
assert_equal(before_table_count, after_table_count)
def test_carto_2_temp_table_task():
'''
Convert a table on CARTO to a temp table.
'''
task = TestCSV2TempTableTask()
before_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
runtask(task)
table = task.output()
assert_equal(current_session().execute(
'SELECT COUNT(*) FROM {}'.format(
table.table)).fetchone()[0], 10)
after_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
assert_equal(before_table_count, after_table_count)
def test_geom_table_task():
'''
Table task with geoms should run and generate an entry in OBSTable with
a summary geom, plus a physical table with rows in observatory schema.
It should also generate a raster tile summary.
'''
task = TestGeomTableTask()
runtask(task)
table = task.output().get(current_session())
assert_is_not_none(table)
assert_is_not_none(table.the_geom)
assert_equal(current_session().execute(
'SELECT COUNT(*) FROM observatory.{}'.format(
table.tablename)).fetchone()[0], 1)
assert_greater(current_session().query(OBSColumnTableTile).count(), 0)
def test_csv_2_temp_table_task():
'''
CSV to temp table task should run and generate no entry in OBSTable, but
a physical table in the right schema.
'''
task = TestCSV2TempTableTask()
before_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
runtask(task)
table = task.output()
assert_equal(
current_session().execute('SELECT COUNT(*) FROM {}'.format(
table.table)).fetchone()[0], 10)
after_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
assert_equal(before_table_count, after_table_count)
def test_geom_table_task():
'''
Table task with geoms should run and generate an entry in OBSTable with
a summary geom, plus a physical table with rows in observatory schema.
It should also generate a raster tile summary.
'''
task = TestGeomTableTask()
runtask(task)
table = task.output().get(current_session())
assert_is_not_none(table)
assert_is_not_none(table.the_geom)
assert_equal(
current_session().execute('SELECT COUNT(*) FROM observatory.{}'.format(
table.tablename)).fetchone()[0], 1)
assert_greater(current_session().query(OBSColumnTableTile).count(), 0)
def test_csv_2_temp_table_task():
'''
CSV to temp table task should run and generate no entry in OBSTable, but
a physical table in the right schema.
'''
task = TestCSV2TempTableTask()
before_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
runtask(task)
table = task.output()
assert_equal(current_session().execute(
'SELECT COUNT(*) FROM {}'.format(
table.table)).fetchone()[0], 10)
after_table_count = current_session().execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
assert_equal(before_table_count, after_table_count)
def test_table_task_replaces_data():
task = TestTableTask()
runtask(task)
with session_scope() as session:
assert_equals(session.query(task.table).count(), 0)
session.execute('INSERT INTO "{schema}"."{tablename}" VALUES (100, 100)'.format(
schema=task.table.schema,
tablename=task.table.name))
assert_equals(session.query(task.table).count(), 1)
runtask(task)
with session_scope() as session:
assert_equals(session.query(task.table).count(), 1)
def test_tabletask_without_timespan():
class TestTableTaskWithoutTimespan(TableTask):
def requires(self):
return {'meta': TestColumnsTask()}
def columns(self):
return {'foobar': self.input()['meta']['foobar']}
def populate(self):
session = current_session()
session.execute('INSERT INTO {output} VALUES (100)'.format(
output=self.output().table))
task = TestTableTaskWithoutTimespan()
with assert_raises(NotImplementedError):
runtask(task)
def test_table_task_replaces_data():
task = TestTableTask()
runtask(task)
table = task.output().table
with session_scope() as session:
assert_equals(session.execute(
'select count(*) from ' + table).fetchone()[0], 1)
runtask(task)
with session_scope() as session:
assert_equals(session.execute(
'select count(*) from ' + table).fetchone()[0], 1)
def test_table_task_replaces_data():
task = TestTableTask()
runtask(task)
table = task.output().table
with session_scope() as session:
assert_equals(
session.execute('select count(*) from ' + table).fetchone()[0], 1)
runtask(task)
with session_scope() as session:
assert_equals(
session.execute('select count(*) from ' + table).fetchone()[0], 1)
def test_columns_task_with_tags_def_one_tag():
class TestColumnsTaskWithOneTag(TestColumnsTask):
def requires(self):
return {
'tags': TestTagsTask(),
'section': SectionTagsTask(),
}
def tags(self, input_, col_key, col):
return input_['section']['section']
task = TestColumnsTaskWithOneTag()
runtask(task)
output = task.output()
for coltarget in list(output.values()):
assert_in('section', [t.name for t in coltarget._column.tags],
'Section tag not added from tags method')
def test_shp_2_temp_table_task():
'''
Shp to temp table task should run and generate no entry in OBSTable, but
a physical table in the right schema.
'''
with session_scope() as session:
task = TestShp2TempTableTask()
before_table_count = session.execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
runtask(task)
table = task.output()
assert_equal(session.execute(
'SELECT COUNT(*) FROM {}'.format(
table.table)).fetchone()[0], 10)
after_table_count = session.execute(
'SELECT COUNT(*) FROM observatory.obs_table').fetchone()[0]
assert_equal(before_table_count, after_table_count)
def test_acs_columns_run():
task = Columns()
assert_equals(False, task.complete())
assert_equals(0, len(current_session().dirty))
worker = runtask(task)
assert_equals(True, task.complete())
assert_equals(True, worker.run_succeeded)
assert_equals(0, len(current_session().dirty))
def test_catalog():
meta_wrappers = collect_meta_wrappers(
test_module=os.environ.get('TEST_MODULE',
'').replace('.', os.path.sep),
test_all=os.environ.get('TEST_ALL', '') != '')
for klass, params in meta_wrappers:
task = klass(**params)
runtask(task, superclasses=[TagsTask, ColumnsTask, TableTask])
runtask(OBSMetaToLocal(force=True))
runtask(Catalog(force=True))
def test_geoid_columns_run():
runtask(GeoidColumns(year='2015'))
def test_columns_task_fails_no_universe_target():
task = TestTableTaskWithoutUniverseTarget()
with assert_raises(ValueError):
runtask(task)
def test_obs_meta_to_local_overwrites_changes():
'''
If OBSMetaToLocal is run when obs_meta* already exists, it replaces the
existing data.
'''
imp.reload(tasks.carto)
task = tasks.carto.OBSMetaToLocal()
runtask(task)
session = current_session()
session.commit()
session.execute('''
INSERT INTO observatory.obs_meta (numer_id, geom_id, denom_id, numer_timespan)
VALUES ('foo', 'bar', 'baz', 'boo')
''')
session.execute(
"INSERT INTO observatory.obs_meta_numer (numer_id) VALUES ('foo')")
session.execute(
"INSERT INTO observatory.obs_meta_geom (geom_id) VALUES ('bar')")
session.execute(
"INSERT INTO observatory.obs_meta_denom (denom_id) VALUES ('baz')")
session.execute(
"INSERT INTO observatory.obs_meta_timespan (timespan_id) VALUES ('boo')"
)
session.execute(
"INSERT INTO observatory.obs_meta_geom_numer_timespan (geom_id, numer_id) VALUES ('bar', 'foo')"
)
session.commit()
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta').fetchone()[0], 1)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_numer').fetchone()[0],
1)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_denom').fetchone()[0],
1)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom').fetchone()[0], 1)
assert_equals(
session.execute('SELECT COUNT(*) FROM observatory.obs_meta_timespan').
fetchone()[0], 1)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom_numer_timespan').
fetchone()[0], 1)
imp.reload(tasks.carto)
task = tasks.carto.OBSMetaToLocal(force=True)
runtask(task)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta').fetchone()[0], 0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta').fetchone()[0], 0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_numer').fetchone()[0],
0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_denom').fetchone()[0],
0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom').fetchone()[0], 0)
assert_equals(
session.execute('SELECT COUNT(*) FROM observatory.obs_meta_timespan').
fetchone()[0], 0)
assert_equals(
session.execute(
'SELECT COUNT(*) FROM observatory.obs_meta_geom_numer_timespan').
fetchone()[0], 0)
def test_wac_columns_run():
runtask(WorkplaceAreaCharacteristicsColumns())
def test_geoid_columns_run():
runtask(GeoidColumns())
def test_geom_columns_run():
runtask(GeomColumns())
