def test_simple(conn): with closing(conn.cursor()) as cursor: attribute_names = ['CellID', 'CCR', 'Drops'] datasource = name_to_datasource(cursor, "integration-test") entitytype = name_to_entitytype(cursor, "UtranCell") timestamp = pytz.utc.localize(datetime.utcnow()) data_rows = [(10023, timestamp, ('10023', '0.9919', '17'))] datapackage = DataPackage(attribute_names, data_rows) attributes = datapackage.deduce_attributes() attributestore = AttributeStore(datasource, entitytype, attributes) attributestore.create(cursor) attributestore.store_txn(datapackage).run(conn) query = ( "SELECT attribute_directory.materialize_curr_ptr(attributestore) " "FROM attribute_directory.attributestore " "WHERE id = %s") cursor.execute(query, (attributestore.id,)) query = ( "SELECT timestamp " "FROM {0} " "LIMIT 1").format(attributestore.table.render()) cursor.execute(query) timestamp, = cursor.fetchone() eq_(timestamp.toordinal(), timestamp.toordinal())
def test_retrieve(conn): with closing(conn.cursor()) as cursor: time1 = pytz.utc.localize(datetime.utcnow()) trend_names = ['CellID', 'CCR', 'Drops'] data_rows = [ (10023, time1, ('10023', '0.9919', '17')), (10047, time1, ('10047', '0.9963', '18')) ] datapackage = DataPackage(trend_names, data_rows) entitytype = name_to_entitytype(cursor, "UtranCell") datasource = name_to_datasource(cursor, "integration-test") data_types = ["text", "real", "smallint"] attributes = [Attribute(name, datatype) for name, datatype in zip(trend_names, data_types)] attributestore = AttributeStore(datasource, entitytype, attributes) attributestore.create(cursor) attributestore.store_txn(datapackage).run(conn) time.sleep(5) time2 = pytz.utc.localize(datetime.utcnow()) update_data_rows = [(10023, time2, ('10023', '0.9919', '18'))] update_datapackage = DataPackage(trend_names, update_data_rows) attributestore.store_txn(update_datapackage).run(conn) conn.commit() data = retrieve(conn, attributestore.table, trend_names, [10023]) assert_not_equal(data, None)
def test_changing_datatype(conn): with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "storagetest") entitytype = name_to_entitytype(cursor, "UtranCell") timestamp = pytz.utc.localize(datetime.utcnow()) attribute_names = ['site_nr', 'height'] datapackage_a = DataPackage( attribute_names=attribute_names, rows=[ (10023, timestamp, ('10023', '15')) ] ) datapackage_b = DataPackage( attribute_names=attribute_names, rows=[ (10023, timestamp, ('10023', '25.6')) ] ) attributes = datapackage_a.deduce_attributes() attributestore = AttributeStore(datasource, entitytype, attributes) attributestore.create(cursor) conn.commit() attributestore.store_txn(datapackage_a).run(conn) attributestore.store_txn(datapackage_b).run(conn) conn.commit() column_names = get_column_names(conn, "attribute_history", attributestore.table_name()) eq_(len(column_names), 6)
def test_extra_column(conn): with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "storagetest") entitytype = name_to_entitytype(cursor, "UtranCell") timestamp = pytz.utc.localize(datetime.utcnow()) datapackage_a = DataPackage( attribute_names=['test0', 'test1'], rows=[ (10023, timestamp, ('10023', '0.9919')) ] ) datapackage_b = DataPackage( attribute_names=['test0', 'test1', "test2"], rows=[ (10023, timestamp, ('10023', '0.9919', '17')) ] ) attributes = datapackage_a.deduce_attributes() attributestore = AttributeStore(datasource, entitytype, attributes) attributestore.create(cursor) conn.commit() attributestore.store_txn(datapackage_a).run(conn) attributestore.store_txn(datapackage_b).run(conn) conn.commit() column_names = get_column_names(conn, 'attribute_history', attributestore.table_name()) eq_(len(column_names), 7)
def test_store_copy_from_2(conn): trend_names = ['CCR', 'CCRatts', 'Drops'] data_rows = [ (10023, ('0.9919', '2105', '17')) ] data_types = ['integer', 'smallint', 'smallint'] curr_timezone = timezone("Europe/Amsterdam") timestamp = curr_timezone.localize(datetime(2013, 1, 2, 10, 45, 0)) modified = curr_timezone.localize(datetime.now()) granularity = create_granularity("900") with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "test-src010") entitytype = name_to_entitytype(cursor, "test-type002") trendstore = TrendStore(datasource, entitytype, granularity, 86400, "table").create(cursor) partition = trendstore.partition(timestamp) partition.create(cursor) partition.check_columns_exist(trend_names, data_types)(cursor) table = partition.table() store_copy_from(conn, SCHEMA, table.name, trend_names, timestamp, modified, data_rows) conn.commit() eq_(row_count(cursor, table), 1) table.select(Call("max", Column("modified"))).execute(cursor) max_modified = first(cursor.fetchone()) eq_(max_modified, modified)
def test_get_aggregate_shard(self): awacs_query = [{"type": "C", "value": ["dummy_type"]}] granularity = create_granularity("900") formula = "SUM(Drops)" shard_indexes = [15680] with closing(self.conn.cursor()) as cursor: entitytype_cell = name_to_entitytype(cursor, 'dummy_type') sql, args, entity_id_column = compile_sql(awacs_query, None) select_statement = "SELECT {} AS id {}".format(entity_id_column, sql) entities_query = cursor.mogrify(select_statement, args) get_shard = partial(get_aggregate_shard, self.conn, entities_query, entitytype_cell.id, granularity, formula) shards = map(get_shard, shard_indexes) for shard in shards: logging.debug("{} - {}".format(shard[0], shard[-1])) eq_(len(shards), len(shard_indexes))
def setup(self): self.conn = connect() clear_database(self.conn) with closing(self.conn.cursor()) as cursor: self.datasource = name_to_datasource(cursor, "test-source") self.entitytype = name_to_entitytype(cursor, "test_type") self.conn.commit()
def load(self, cursor): self.entitytype = name_to_entitytype(cursor, self.entitytype_name) self.entities = map(partial(dn_to_entity, cursor), self.dns) granularity = create_granularity("900") # Data a self.datasource_a = name_to_datasource(cursor, "test-source-a") self.trendstore_a = TrendStore(self.datasource_a, self.entitytype, granularity, partition_size=86400, type="table").create(cursor) datapackage = generate_datapackage_a(granularity, self.timestamp_1, self.entities) self.partition_a = store_datapackage(cursor, self.trendstore_a, datapackage, self.modified) # Data b self.datasource_b = name_to_datasource(cursor, "test-source-b") self.trendstore_b = TrendStore(self.datasource_b, self.entitytype, granularity, partition_size=86400, type="table").create(cursor) datapackage = generate_datapackage_b(granularity, self.timestamp_1, self.entities) self.partition_b = store_datapackage(cursor, self.trendstore_b, datapackage, self.modified) # Data c self.datasource_c = name_to_datasource(cursor, "test-source-c") self.trendstore_c = TrendStore(self.datasource_c, self.entitytype, granularity, partition_size=86400, type="table").create(cursor) datapackage = generate_datapackage_c(granularity, self.timestamp_1, self.entities) self.partition_c = store_datapackage(cursor, self.trendstore_c, datapackage, self.modified) # Data d self.datasource_d = name_to_datasource(cursor, "test-source-d") self.trendstore_d = TrendStore(self.datasource_d, self.entitytype, granularity, partition_size=86400, type="table").create(cursor) datapackage_1 = generate_datapackage_d(granularity, self.timestamp_1, self.entities) self.partition_d_1 = store_datapackage(cursor, self.trendstore_d, datapackage_1, self.modified) datapackage_2 = generate_datapackage_d(granularity, self.timestamp_2, self.entities) self.partition_d_2 = store_datapackage(cursor, self.trendstore_d, datapackage_2, self.modified)
def test_update_modified_column(conn): curr_timezone = timezone("Europe/Amsterdam") trend_names = ['CellID', 'CCR', 'Drops'] data_rows = [ (10023, ('10023', '0.9919', '17')), (10047, ('10047', '0.9963', '18')) ] data_types = extract_data_types(data_rows) update_data_rows = [(10023, ('10023', '0.9919', '17'))] timestamp = curr_timezone.localize(datetime.now()) granularity = create_granularity("900") with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "test-src009") entitytype = name_to_entitytype(cursor, "test-type001") trendstore = TrendStore(datasource, entitytype, granularity, 86400, "table").create(cursor) partition = trendstore.partition(timestamp) table = partition.table() partition.create(cursor) partition.check_columns_exist(trend_names, data_types)(cursor) store(conn, SCHEMA, table.name, trend_names, timestamp, data_rows) time.sleep(1) store(conn, SCHEMA, table.name, trend_names, timestamp, update_data_rows) conn.commit() query = table.select([Column("modified")]) query.execute(cursor) modified_list = [modified for modified in cursor.fetchall()] assert_not_equal(modified_list[0], modified_list[1]) table.select(Call("max", Column("modified"))).execute(cursor) max_modified = first(cursor.fetchone()) modified_table.select(Column("end")).where_( Eq(Column("table_name"), table.name)).execute(cursor) end = first(cursor.fetchone()) eq_(end, max_modified)
def test_store_copy_from_1(conn): trend_names = ['CellID', 'CCR', 'CCRatts', 'Drops'] data_rows = [ (10023, ('10023', '0.9919', '2105', '17')), (10047, ('10047', '0.9963', '4906', '18')), (10048, ('10048', '0.9935', '2448', '16')), (10049, ('10049', '0.9939', '5271', '32')), (10050, ('10050', '0.9940', '3693', '22')), (10051, ('10051', '0.9944', '3753', '21')), (10052, ('10052', '0.9889', '2168', '24')), (10053, ('10053', '0.9920', '2372', '19')), (10085, ('10085', '0.9987', '2282', '3')), (10086, ('10086', '0.9972', '1763', '5')), (10087, ('10087', '0.9931', '1453', '10')) ] curr_timezone = timezone("Europe/Amsterdam") data_types = extract_data_types(data_rows) timestamp = curr_timezone.localize(datetime(2013, 1, 2, 10, 45, 0)) granularity = create_granularity("900") modified = curr_timezone.localize(datetime.now()) with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "test-src009") entitytype = name_to_entitytype(cursor, "test-type001") trendstore = TrendStore(datasource, entitytype, granularity, 86400, "table").create(cursor) partition = trendstore.partition(timestamp) table = partition.table() partition.create(cursor) partition.check_columns_exist(trend_names, data_types)(cursor) store_copy_from(conn, SCHEMA, table.name, trend_names, timestamp, modified, data_rows) conn.commit() eq_(row_count(cursor, table), 11) table.select(Call("max", Column("modified"))).execute(cursor) max_modified = first(cursor.fetchone()) eq_(max_modified, modified)
def test_update_and_modify_columns_fractured(conn): curr_timezone = timezone("Europe/Amsterdam") granularity = create_granularity("900") timestamp = curr_timezone.localize(datetime(2013, 1, 2, 10, 45, 0)) entity_ids = range(1023, 1023 + 100) trend_names_a = ["CellID", "CCR", "Drops"] data_rows_a = [(i, ("10023", "0.9919", "17")) for i in entity_ids] data_types_a = extract_data_types(data_rows_a) trend_names_b = ["CellID", "Drops"] data_rows_b = [(i, ("10023", "19")) for i in entity_ids] data_types_b = extract_data_types(data_rows_b) with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "test-src009") entitytype = name_to_entitytype(cursor, "test-type001") trendstore = TrendStore(datasource, entitytype, granularity, 86400, "table").create(cursor) partition = trendstore.partition(timestamp) table = partition.table() partition.create(cursor) partition.check_columns_exist(trend_names_a, data_types_a)(cursor) conn.commit() store(conn, SCHEMA, table.name, trend_names_a, timestamp, data_rows_a) time.sleep(0.2) check_columns = map(Column, ["modified", "Drops"]) query = table.select(check_columns) with closing(conn.cursor()) as cursor: query.execute(cursor) row_before = cursor.fetchone() store(conn, SCHEMA, table.name, trend_names_b, timestamp, data_rows_b) query = table.select(check_columns) with closing(conn.cursor()) as cursor: query.execute(cursor) row_after = cursor.fetchone() assert_not_equal(row_before[0], row_after[0]) assert_not_equal(row_before[1], row_after[1])
def test_update_modified_column(conn): attribute_names = ['CCR', 'Drops'] with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "integration-test") entitytype = name_to_entitytype(cursor, "UtranCell") timestamp = datasource.tzinfo.localize(datetime.now()) rows = [ (10023, timestamp, ('0.9919', '17')), (10047, timestamp, ('0.9963', '18')) ] datapackage_a = DataPackage( attribute_names=attribute_names, rows=rows) datapackage_b = DataPackage( attribute_names=attribute_names, rows=rows[:1]) attributes = datapackage_a.deduce_attributes() attributestore = AttributeStore(datasource, entitytype, attributes) attributestore.create(cursor) conn.commit() query = Query(( "SELECT modified, hash " "FROM {0} " "WHERE entity_id = 10023").format(attributestore.history_table.render())) attributestore.store_txn(datapackage_a).run(conn) with closing(conn.cursor()) as cursor: modified_a, hash_a = query.execute(cursor).fetchone() attributestore.store_txn(datapackage_b).run(conn) with closing(conn.cursor()) as cursor: modified_b, hash_b = query.execute(cursor).fetchone() # modified should be updated when same data is delivered again ok_(modified_a < modified_b) # hash should remain the same when then same data is delivered again eq_(hash_a, hash_b)
def test_get_trendstore(conn): plugin = get_plugin("trend") instance = plugin(conn, api_version=4) granularity = create_granularity("900") with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "test-src") entitytype = name_to_entitytype(cursor, "test-type") instance.TrendStore(datasource, entitytype, granularity, 86400, "table").create(cursor) trendstore = instance.get_trendstore(datasource, entitytype, granularity) assert_not_equal(trendstore, None)
def load(self, cursor): entitytype = name_to_entitytype(cursor, "materialize_dummytype001") self.datasource = name_to_datasource(cursor, "materialize_src_normal001") view_datasource = name_to_datasource(cursor, "vmaterialize_normal001") granularity = create_granularity('900') self.timestamp = self.datasource.tzinfo.localize( datetime.datetime(2013, 8, 26, 22, 0, 0)) trend_names = ["cntr"] rows_small = [ (1234, (55,)), (1235, (56,))] self.small_datapackage = DataPackage(granularity, self.timestamp, trend_names, rows_small) rows_large = [ (1234, (55243444334,)), (1235, (56242343242,))] self.large_datapackage = DataPackage(granularity, self.timestamp, trend_names, rows_large) self.trendstore = TrendStore(self.datasource, entitytype, granularity, 86400, 'table') self.trendstore.create(cursor) partition = self.trendstore.partition(self.timestamp) partition.create(cursor) self.trendstore.check_columns_exist(trend_names, ["smallint"])(cursor) modified = self.datasource.tzinfo.localize(datetime.datetime.now()) store_copy_from(cursor, partition.table(), self.small_datapackage, modified) mark_modified(cursor, partition.table(), self.timestamp, modified) view_trendstore = TrendStore(view_datasource, entitytype, granularity, 0, 'view').create(cursor) sql = ( "SELECT " "entity_id, " "timestamp, " 'cntr FROM {}').format(self.trendstore.base_table().render()) self.view = View(view_trendstore, sql).define(cursor).create(cursor)
def load(self, cursor): self.datasource = name_to_datasource(cursor, "testset1") self.entitytype = name_to_entitytype(cursor, self.entitytype_name) self.entities = map(partial(dn_to_entity, cursor), self.dns) datapackage = generate_datapackage_a(self.granularity, self.timestamp, self.entities) self.trendstore = TrendStore.get(cursor, self.datasource, self.entitytype, self.granularity) if not self.trendstore: self.trendstore = TrendStore(self.datasource, self.entitytype, self.granularity, partition_size=86400, type="table").create(cursor) self.partition = store_datapackage(cursor, self.trendstore, datapackage, self.modified)
def test_retrieve_aggregate(self): granularity = create_granularity("900") with closing(self.conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "test") entitytype = name_to_entitytype(cursor, "Cell") TrendStore(datasource, entitytype, granularity, 86400, "table").create(cursor) column_expressions = ["COUNT(entity_id)"] start = datasource.tzinfo.localize(datetime(2012, 12, 6, 14, 15)) end = datasource.tzinfo.localize(datetime(2012, 12, 6, 14, 15)) interval = start, end retrieve_aggregated(self.conn, datasource, granularity, entitytype, column_expressions, interval, group_by="entity_id")
def test_array(conn): with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "integration-test") entitytype = name_to_entitytype(cursor, "UtranCell") timestamp = pytz.utc.localize(datetime.utcnow()) datapackage = DataPackage( attribute_names=['channel', 'pwr'], rows=[ (10023, timestamp, ('7', '0,0,0,2,5,12,87,34,5,0,0')), (10024, timestamp, ('9', '0,0,0,1,11,15,95,41,9,0,0')) ] ) attributes = datapackage.deduce_attributes() attributestore = AttributeStore(datasource, entitytype, attributes) attributestore.create(cursor) conn.commit() attributestore.store_txn(datapackage).run(conn)
def test_update(conn): with closing(conn.cursor()) as cursor: attribute_names = ['CellID', 'CCR', 'Drops'] datasource = name_to_datasource(cursor, "integration-test") entitytype = name_to_entitytype(cursor, "UtranCell") time1 = datasource.tzinfo.localize(datetime.now()) data_rows = [ (10023, time1, ('10023', '0.9919', '17')), (10047, time1, ('10047', '0.9963', '18')) ] update_data_rows = [ (10023, time1, ('10023', '0.5555', '17')) ] datapackage = DataPackage(attribute_names, data_rows) attributes = datapackage.deduce_attributes() attributestore = AttributeStore(datasource, entitytype, attributes) attributestore.create(cursor) attributestore.store_txn(datapackage).run(conn) time.sleep(1) datapackage = DataPackage(attribute_names, update_data_rows) attributestore.store_txn(datapackage).run(conn) conn.commit() query = ( 'SELECT modified, "CCR" ' 'FROM {0}').format(attributestore.history_table.render()) cursor.execute(query) test_list = [(modified, ccr) for modified, ccr in cursor.fetchall()] assert_not_equal(test_list[0][0], test_list[1][0]) assert_not_equal(test_list[0][1], test_list[1][1])
def test_update(conn): trend_names = ["CellID", "CCR", "Drops"] data_rows = [ (10023, ("10023", "0.9919", "17")), (10047, ("10047", "0.9963", "18")) ] data_types = extract_data_types(data_rows) update_data_rows = [(10023, ("10023", "0.5555", "17"))] timestamp = datetime.now() granularity = create_granularity("900") with closing(conn.cursor()) as cursor: datasource = name_to_datasource(cursor, "test-src009") entitytype = name_to_entitytype(cursor, "test-type001") trendstore = TrendStore(datasource, entitytype, granularity, 86400, "table").create(cursor) partition = trendstore.partition(timestamp) table = partition.table() partition.create(cursor) partition.check_columns_exist(trend_names, data_types)(cursor) store(conn, SCHEMA, table.name, trend_names, timestamp, data_rows) store(conn, SCHEMA, table.name, trend_names, timestamp, update_data_rows) conn.commit() query = table.select([Column("modified"), Column("CCR")]) with closing(conn.cursor()) as cursor: query.execute(cursor) rows = cursor.fetchall() assert_not_equal(rows[0][0], rows[1][0]) assert_not_equal(rows[0][1], rows[1][1])
def test_name_to_entitytype(conn): with closing(conn.cursor()) as cursor: entitytype = helpers_v4.name_to_entitytype(cursor, "test_name_to_entitytype") assert not entitytype is None assert entitytype.name == "test_name_to_entitytype"
def load(self, cursor): self.datasource = name_to_datasource(cursor, "test-source") self.entitytype = name_to_entitytype(cursor, "test_type")