def test_insert_data(self):
etl = PlenarioETL(self.existing_meta, source_path=self.dog_path)
etl.update()
existing = self.existing_table
all_rows = postgres_session.execute(existing.select()).fetchall()
self.assertEqual(len(all_rows), 5)
def ingest_from_fixture(fixture_meta, fname):
md = MetaTable(**fixture_meta)
session.add(md)
session.commit()
path = os.path.join(fixtures_path, fname)
point_etl = PlenarioETL(md, source_path=path)
point_etl.add()
def ingest_from_fixture(fixture_meta, fname):
md = MetaTable(**fixture_meta)
session.add(md)
session.commit()
path = os.path.join(fixtures_path, fname)
point_etl = PlenarioETL(md, source_path=path)
point_etl.add()
def test_insert_data(self):
etl = PlenarioETL(self.existing_meta, source_path=self.dog_path)
etl.update()
existing = self.existing_table
all_rows = session.execute(existing.select()).fetchall()
self.assertEqual(len(all_rows), 5)
def update_dataset(self, source_url_hash):
md = session.query(MetaTable).get(source_url_hash)
if md.result_ids:
ids = md.result_ids
ids.append(self.request.id)
else:
ids = [self.request.id]
with engine.begin() as c:
c.execute(MetaTable.__table__.update()\
.where(MetaTable.source_url_hash == source_url_hash)\
.values(result_ids=ids))
etl = PlenarioETL(md)
etl.update()
return 'Finished updating {0} ({1})'.format(md.human_name, md.source_url_hash)
def test_location_col_add(self):
drop_if_exists(self.opera_meta.dataset_name)
etl = PlenarioETL(self.opera_meta, source_path=self.opera_path)
new_table = etl.add()
all_rows = session.execute(new_table.select()).fetchall()
self.assertEqual(len(all_rows), 5)
session.close()
new_table.drop(app_engine, checkfirst=True)
# Did we add a bbox?
bbox = MetaTable.get_by_dataset_name('public_opera_performances').bbox
self.assertIsNotNone(bbox)
def test_new_table(self):
drop_if_exists(self.unloaded_meta.dataset_name)
etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path)
new_table = etl.add()
all_rows = session.execute(new_table.select()).fetchall()
self.assertEqual(len(all_rows), 5)
session.close()
new_table.drop(app_engine, checkfirst=True)
# Did we add a bbox?
bbox = MetaTable.get_by_dataset_name('community_radio_events').bbox
self.assertIsNotNone(bbox)
def test_update_no_change(self):
etl = PlenarioETL(self.existing_meta, source_path=self.dog_path)
etl.update()
# We're just checking that an exception doesn't get thrown.
etl = PlenarioETL(self.existing_meta, source_path=self.dog_path)
etl.update()
def test_location_col_add(self):
drop_if_exists(self.opera_meta.dataset_name)
etl = PlenarioETL(self.opera_meta, source_path=self.opera_path)
new_table = etl.add()
all_rows = postgres_session.execute(new_table.select()).fetchall()
self.assertEqual(len(all_rows), 5)
postgres_session.close()
new_table.drop(postgres_engine, checkfirst=True)
# Did we add a bbox?
bbox = MetaTable.get_by_dataset_name('public_opera_performances').bbox
self.assertIsNotNone(bbox)
def test_new_table_has_correct_column_names_in_meta(self):
drop_if_exists(self.unloaded_meta.dataset_name)
etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path)
new_table = etl.add()
columns = postgres_session.query(MetaTable.column_names)
columns = columns.filter(MetaTable.dataset_name == self.unloaded_meta.dataset_name)
columns = columns.first()[0]
self.assertEqual(len(columns), 4)
postgres_session.close()
new_table.drop(postgres_engine, checkfirst=True)
def test_new_table(self):
drop_if_exists(self.unloaded_meta.dataset_name)
etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path)
new_table = etl.add()
all_rows = postgres_session.execute(new_table.select()).fetchall()
self.assertEqual(len(all_rows), 5)
postgres_session.close()
new_table.drop(postgres_engine, checkfirst=True)
# Did we add a bbox?
bbox = MetaTable.get_by_dataset_name('community_radio_events').bbox
self.assertIsNotNone(bbox)
def update_dataset(name: str) -> bool:
"""Update the row information for an approved point dataset.
"""
logger.info('Begin. (name: "{}")'.format(name))
meta = get_meta(name)
PlenarioETL(meta).update()
logger.info('End.')
return True
def test_update_with_delete(self):
etl = PlenarioETL(self.existing_meta, source_path=self.dog_path)
etl.update()
# The same source CSV, but with one less record
deleted_path = os.path.join(fixtures_path, 'dog_park_permits_deleted.csv')
etl = PlenarioETL(self.existing_meta, source_path=deleted_path)
etl.update()
all_rows = postgres_session.execute(self.existing_table.select()).fetchall()
self.assertEqual(len(all_rows), 4)
def test_update_no_change(self):
etl = PlenarioETL(self.existing_meta, source_path=self.dog_path)
etl.update()
# We're just checking that an exception doesn't get thrown.
etl = PlenarioETL(self.existing_meta, source_path=self.dog_path)
etl.update()
def test_update_with_change(self):
drop_if_exists(self.unloaded_meta.dataset_name)
etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path)
table = etl.add()
changed_path = os.path.join(fixtures_path, 'community_radio_events_changed.csv')
etl = PlenarioETL(self.unloaded_meta, source_path=changed_path)
etl.update()
sel = sa.select([table.c.date]).where(table.c.event_name == 'baz')
changed_date = postgres_engine.execute(sel).fetchone()[0]
self.assertEqual(changed_date, date(1993, 11, 10))