def ingest_from_fixture(fixture_meta, fname): md = MetaTable(**fixture_meta) session.add(md) session.commit() path = os.path.join(fixtures_path, fname) point_etl = PlenarioETL(md, source_path=path) point_etl.add()
def ingest_from_fixture(fixture_meta, fname): md = MetaTable(**fixture_meta) session.add(md) session.commit() path = os.path.join(fixtures_path, fname) point_etl = PlenarioETL(md, source_path=path) point_etl.add()
def add_dataset(self, source_url_hash, data_types=None): md = session.query(MetaTable).get(source_url_hash) session.close() if md.result_ids: ids = md.result_ids ids.append(self.request.id) else: ids = [self.request.id] with engine.begin() as c: c.execute(MetaTable.__table__.update()\ .where(MetaTable.source_url_hash == source_url_hash)\ .values(result_ids=ids)) etl = PlenarioETL(md) etl.add() return 'Finished adding {0} ({1})'.format(md.human_name, md.source_url_hash)
def test_update_with_change(self): drop_if_exists(self.unloaded_meta.dataset_name) etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path) table = etl.add() changed_path = os.path.join(fixtures_path, 'community_radio_events_changed.csv') etl = PlenarioETL(self.unloaded_meta, source_path=changed_path) etl.update() sel = sa.select([table.c.date]).where(table.c.event_name == 'baz') changed_date = postgres_engine.execute(sel).fetchone()[0] self.assertEqual(changed_date, date(1993, 11, 10))
def test_location_col_add(self): drop_if_exists(self.opera_meta.dataset_name) etl = PlenarioETL(self.opera_meta, source_path=self.opera_path) new_table = etl.add() all_rows = session.execute(new_table.select()).fetchall() self.assertEqual(len(all_rows), 5) session.close() new_table.drop(app_engine, checkfirst=True) # Did we add a bbox? bbox = MetaTable.get_by_dataset_name('public_opera_performances').bbox self.assertIsNotNone(bbox)
def test_new_table(self): drop_if_exists(self.unloaded_meta.dataset_name) etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path) new_table = etl.add() all_rows = session.execute(new_table.select()).fetchall() self.assertEqual(len(all_rows), 5) session.close() new_table.drop(app_engine, checkfirst=True) # Did we add a bbox? bbox = MetaTable.get_by_dataset_name('community_radio_events').bbox self.assertIsNotNone(bbox)
def test_location_col_add(self): drop_if_exists(self.opera_meta.dataset_name) etl = PlenarioETL(self.opera_meta, source_path=self.opera_path) new_table = etl.add() all_rows = postgres_session.execute(new_table.select()).fetchall() self.assertEqual(len(all_rows), 5) postgres_session.close() new_table.drop(postgres_engine, checkfirst=True) # Did we add a bbox? bbox = MetaTable.get_by_dataset_name('public_opera_performances').bbox self.assertIsNotNone(bbox)
def test_new_table_has_correct_column_names_in_meta(self): drop_if_exists(self.unloaded_meta.dataset_name) etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path) new_table = etl.add() columns = postgres_session.query(MetaTable.column_names) columns = columns.filter(MetaTable.dataset_name == self.unloaded_meta.dataset_name) columns = columns.first()[0] self.assertEqual(len(columns), 4) postgres_session.close() new_table.drop(postgres_engine, checkfirst=True)
def test_new_table(self): drop_if_exists(self.unloaded_meta.dataset_name) etl = PlenarioETL(self.unloaded_meta, source_path=self.radio_path) new_table = etl.add() all_rows = postgres_session.execute(new_table.select()).fetchall() self.assertEqual(len(all_rows), 5) postgres_session.close() new_table.drop(postgres_engine, checkfirst=True) # Did we add a bbox? bbox = MetaTable.get_by_dataset_name('community_radio_events').bbox self.assertIsNotNone(bbox)