def test_retrieve(self):
table_a = self.data.partition_a.table()
table_b = self.data.partition_b.table()
table_c = self.data.partition_c.table()
table_names = [
table_a.name,
table_b.name,
table_c.name]
start = self.data.timestamp_1
end = self.data.timestamp_1
table_a_cols = [
Column("CellID"),
Column("CCR"),
Column("CCRatts"),
Column("Drops")]
table_b_cols = [
Column("counter_a"),
Column("counter_b")]
table_c_cols = [
Column("counter_x"),
Column("counter_y")]
system_columns = [Column("entity_id"), Column("timestamp")]
columns = table_a_cols + table_b_cols + table_c_cols
with closing(self.conn.cursor()) as cursor:
table_a.select(system_columns + table_a_cols).execute(cursor)
logging.debug(unlines(render_result(cursor)))
table_b.select(system_columns + table_b_cols).execute(cursor)
logging.debug(unlines(render_result(cursor)))
table_c.select(system_columns + table_c_cols).execute(cursor)
logging.debug(unlines(render_result(cursor)))
r = retrieve(self.conn, SCHEMA, table_names, columns, None, start, end,
entitytype=self.data.entitytype)
data = [["entity_id", "timestamp"] + [c.name for c in columns]] + r
logging.debug(unlines(render_source(data)))
eq_(len(r), 5)
def test_run_hr(conn):
with closing(conn.cursor()) as cursor:
reset_db(cursor)
conn.commit()
source_granularity = create_granularity("900")
dest_granularity = create_granularity("3600")
with closing(conn.cursor()) as cursor:
source_datasource = get_dummy_datasource(cursor, "dummy-src-5")
dest_datasource = get_dummy_datasource(cursor, "dummy-transform-src")
dest_entitytype = get_dummy_entitytype(cursor, "dummy_type_aggregate")
function_mapping = add_function_mapping(cursor, "sum", ["counter_a"], "sum_a")
partition_size = 86400
trendstore_1 = TrendStore3(source_datasource, dest_entitytype,
source_granularity, partition_size, "table")
trendstore_1.create(cursor)
dest_trendstore = TrendStore3(dest_datasource, dest_entitytype,
dest_granularity, partition_size, "table")
dest_trendstore.create(cursor)
function_set_qtr = add_function_set(cursor, "test_set_agg", "",
[function_mapping.id], [source_datasource.id], dest_entitytype.id,
source_granularity.name, dest_datasource.id, dest_entitytype.id,
dest_granularity.name, None, ["entity_id"], None, True)
conn.commit()
transaction = store_modified_at(trendstore_1, source_1_1, modified_a)
transaction.run(conn)
transaction = store_modified_at(trendstore_1, source_1_2, modified_b)
transaction.run(conn)
transaction = store_modified_at(trendstore_1, source_1_3, modified_a)
transaction.run(conn)
transaction = store_modified_at(trendstore_1, source_1_4, modified_a)
transaction.run(conn)
logging.debug("source_1")
logging.debug(unlines(render_datapackage(source_1_1)))
processed_max_modified = None
minerva_context = MinervaContext(conn, conn)
transformation = Transformation(function_set_qtr, dest_timestamp)
transformation.execute(minerva_context)
columns = map(Column, ["entity_id", "sum_a"])
dest_partition = dest_trendstore.partition(dest_timestamp)
result_table = dest_partition.table()
query = result_table.select(columns)
with closing(conn.cursor()) as cursor:
query.execute(cursor)
logging.debug(unlines(render_result(cursor)))
query.execute(cursor)
row = cursor.fetchone()
eq_(row[1], 21)
def test_run(conn):
with closing(conn.cursor()) as cursor:
reset_db(cursor)
conn.commit()
minerva_context = MinervaContext(conn, conn)
source_granularity = create_granularity("900")
dest_granularity = create_granularity("900")
with closing(conn.cursor()) as cursor:
source_datasource_1 = get_dummy_datasource(cursor, "dummy-src-1")
source_datasource_2 = get_dummy_datasource(cursor, "dummy-src-2")
dest_datasource = get_dummy_datasource(cursor, "dummy-transform-src")
entitytype = get_dummy_entitytype(cursor, dummy_type_name)
partition_size = 86400
trendstore_1 = TrendStore(
source_datasource_1, entitytype, source_granularity,
partition_size, "table")
trendstore_1.create(cursor)
trendstore_2 = TrendStore(
source_datasource_2, entitytype, source_granularity,
partition_size, "table")
trendstore_2.create(cursor)
result_trendstore = TrendStore(
dest_datasource, entitytype, dest_granularity, partition_size,
"table")
result_trendstore.create(cursor)
function_mappings = [
add_function_mapping(cursor, None, ["counter_a"], "identity_a"),
add_function_mapping(cursor, None, ["counter_b"], "identity_b"),
add_function_mapping(cursor, None, ["counter_c"], "identity_c"),
add_function_mapping(cursor, "add", ["counter_a", "counter_b"], "add_a_b"),
add_function_mapping(cursor, "multiply", ["counter_a", "300"], "a_times_300")]
function_mapping_ids = [fm.id for fm in function_mappings]
function_set_qtr = add_function_set(cursor, "test_set", "", function_mapping_ids,
[source_datasource_1.id, source_datasource_2.id], entitytype.id, source_granularity.name, dest_datasource.id, entitytype.id,
dest_granularity.name, None, [], None, True)
entities = map(partial(get_or_create_entity, cursor), dns)
conn.commit()
source_1 = create_source_1(source_granularity, entities)
def store_modified_at(trendstore, datapackage, modified):
def set_modified(state):
state["modified"] = modified
partition = trendstore.partition(datapackage.timestamp)
set_modified_action = UpdateState(set_modified)
copy_from = CopyFrom(k(partition), k(datapackage))
return DbTransaction(set_modified_action, copy_from)
transaction = store_modified_at(trendstore_1, source_1, modified_a)
transaction.run(conn)
source_2 = create_source_2(source_granularity, entities)
transaction = store_modified_at(trendstore_2, source_2, modified_a)
transaction.run(conn)
result_partition = result_trendstore.partition(timestamp)
result_table = result_partition.table()
conn.commit()
logging.debug("source_1")
logging.debug(unlines(render_datapackage(source_1)))
logging.debug("source_2")
logging.debug(unlines(render_datapackage(source_2)))
dest_timestamp = timestamp
transformation = Transformation(function_set_qtr, dest_timestamp)
transformation.execute(minerva_context)
columns = map(Column, ["entity_id", "identity_a", "identity_b", "add_a_b", "a_times_300"])
query = result_table.select(columns)
with closing(conn.cursor()) as cursor:
query.execute(cursor)
logging.debug(unlines(render_result(cursor)))
src_table_1 = trendstore_1.partition(timestamp).table()
query = src_table_1.select(Call("max", Column("modified")))
query.execute(cursor)
src1_max_modified = first(cursor.fetchone())
src_table_2 = trendstore_2.partition(timestamp).table()
query = src_table_2.select(Call("max", Column("modified")))
query.execute(cursor)
src2_max_modified = first(cursor.fetchone())
query = modified_table.select(Column("end")).where_(Eq(Column("table_name"), result_table.name))
query.execute(cursor)
query = state_table.select(Column("processed_max_modified")).where_(Eq(Column("function_set_id")))
query.execute(cursor, (function_set_qtr.id,))
processed_max_modified = first(cursor.fetchone())
eq_(max(src1_max_modified, src2_max_modified), processed_max_modified)