def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(indicator_config_id) else: config = DataSourceConfiguration.get(indicator_config_id) # Save the start time now in case anything goes wrong. This way we'll be # able to see if the rebuild started a long time ago without finishing. config.meta.build.initiated = datetime.datetime.utcnow() config.save() adapter = IndicatorSqlAdapter(config, engine=create_engine()) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) except DataError as e: logging.exception("problem saving document {} to table. {}".format(doc["_id"], e)) adapter.engine.dispose() if not is_static: config.meta.build.finished = True config.save()
def test_table_population(self): engine = create_engine() adapter = IndicatorSqlAdapter(self.config) # Delete and create table adapter.rebuild_table() # Create a doc now = datetime.datetime.now() one_hour = datetime.timedelta(hours=1) logs = [ {"start_time": now, "end_time": now + one_hour, "person": "al"}, {"start_time": now + one_hour, "end_time": now + (one_hour * 2), "person": "chris"}, {"start_time": now + (one_hour * 2), "end_time": now + (one_hour * 3), "person": "katie"}, ] doc = _test_doc(form={'time_logs': logs}) # Save this document into the table adapter.save(doc) # Get rows from the table with engine.connect() as connection: rows = connection.execute(adapter.get_table().select()) retrieved_logs = [ { 'start_time': r[3], 'end_time': r[4], 'person': r[5], } for r in rows ] # Check those rows against the expected result self.assertItemsEqual( retrieved_logs, logs, "The repeat data saved in the data source table did not match the expected data!" )
def get_sql_engine(cls): # todo: switch to connection_manager engine = getattr(cls, '_engine', None) if not engine: cls._engine = create_engine() return cls._engine