def test_table_drop(conn):
table = Table("test_table")
create_query = table.create()
drop_query = table.drop()
with closing(conn.cursor()) as cursor:
create_query.execute(cursor)
eq_(table_exists(cursor, table), True)
drop_query.execute(cursor)
eq_(table_exists(cursor, table), False)
def test_table_create(conn):
table = Table("test_table")
query = table.create()
with_cursor(conn, query.execute)
with closing(conn.cursor()) as cursor:
eq_(table_exists(cursor, table), True)
def test_run(conn):
clear_database(conn)
plugin = load_plugin()
minerva_context = MinervaContext(conn, conn)
instance = plugin(minerva_context)
job_id = 67
description = {
"function_set_id": 42,
"dest_timestamp": timestamp.isoformat(),
"processed_max_modified": "2012-12-11 14:03:29+01:00"}
config = {}
job = instance.create_job(job_id, description, config)
assert_not_equal(job, None)
dest_granularity = 900
function_mapping_table = Table("transform", "function_mapping")
with closing(conn.cursor()) as cursor:
state_table.truncate().execute(cursor)
function_set_table.truncate(cascade=True).execute(cursor)
source_datasource_1 = get_dummy_datasource(cursor, "dummy-src-1")
source_datasource_2 = get_dummy_datasource(cursor, "dummy-src-2")
dest_datasource = get_dummy_datasource(cursor, "dummy-transform-src")
dest_entitytype = get_dummy_entitytype(cursor, "dummy_type_standard")
function_mapping_table.truncate().execute(cursor)
get_function_mapping(cursor, 1, None, ["counter_a"], "identity_a")
get_function_mapping(cursor, 2, None, ["counter_b"], "identity_b")
get_function_mapping(cursor, 3, None, ["counter_c"], "identity_c")
get_function_mapping(cursor, 4, "add", ["counter_a", "counter_b"], "add_a_b")
get_function_mapping(cursor, 5, "multiply", ["counter_a", "300"], "a_times_300")
get_function_set(cursor, 42, "test_set", [1, 2, 3, 4, 5], [3, 4], 42, 900, 6,
dest_entitytype.id, dest_granularity, None, [], True)
args = 1, "unittest", "transform", ""
add_job_source(cursor, *args)
size = 233
job_source_id = 1
args = job_id, "transform", "", size, "2012-12-11 14:34:00", None, None, None, job_source_id, "running"
add_job(cursor, *args)
args = 42, description["dest_timestamp"], description["processed_max_modified"], "2012-12-11 13:03:00", job_id
add_state(cursor, *args)
table_name = "dummy-src-1_dummy_type_standard_qtr_20121211"
columns = [
Column("entity_id"),
Column("timestamp", type_=SqlType("timestamp with time zone")),
Column("modified", type_=SqlType("timestamp with time zone")),
Column("counter_a"),
Column("counter_b")]
src_table_1 = Table("trend", table_name, columns=columns)
if table_exists(cursor, src_table_1):
src_table_1.drop().execute(cursor)
src_table_1.create().execute(cursor)
entities = map(partial(get_or_create_entity, cursor), dns)
source_1 = create_source_1(entities)
store(cursor, src_table_1, source_1)
table_name = "dummy-src-2_dummy_type_standard_qtr_20121211"
columns = [
Column("entity_id"),
Column("timestamp", type_=SqlType("timestamp with time zone")),
Column("modified", type_=SqlType("timestamp with time zone")),
Column("counter_c")]
src_table_2 = Table("trend", table_name, columns=columns)
if table_exists(cursor, src_table_2):
src_table_2.drop().execute(cursor)
src_table_2.create().execute(cursor)
source_2 = create_source_2(entities)
store(cursor, src_table_2, source_2)
result_table = Table("trend", "dummy-transform-src_dummy_type_standard_qtr_20121211")
if table_exists(cursor, result_table):
result_table.truncate().execute(cursor)
conn.commit()
print("source_1")
print(unlines(render_source(source_1)))
print("source_2")
print(unlines(render_source(source_2)))
job.execute()
columns = map(Column, ["entity_id", "identity_a", "identity_b", "add_a_b", "a_times_300"])
query = result_table.select(columns)
with closing(conn.cursor()) as cursor:
query.execute(cursor, args)
print(unlines(render_result(cursor)))
query = src_table_1.select(Call("max", Column("modified")))
query.execute(cursor)
src1_max_modified = first(cursor.fetchone())
query = src_table_2.select(Call("max", Column("modified")))
query.execute(cursor)
src2_max_modified = first(cursor.fetchone())
query = modified_table.select(Column("end")).where_(Eq(Column("table_name"), result_table.name))
query.execute(cursor)
query = state_table.select(Column("processed_max_modified")).where_(Eq(Column("function_set_id")))
query.execute(cursor, (42,))
processed_max_modified = first(cursor.fetchone())
eq_(max(src1_max_modified, src2_max_modified), processed_max_modified)
def test_run(conn):
clear_database(conn)
plugin = load_plugin()
minerva_context = MinervaContext(conn, conn)
instance = plugin(minerva_context)
job_id = 67
description = {
"function_set_id": 43,
"dest_timestamp": "2012-12-11 13:00:00",
"processed_max_modified": "2012-12-11 13:03:29",
}
config = {}
job = instance.create_job(job_id, description, config)
assert_not_equal(job, None)
dest_granularity = 3600
with closing(conn.cursor()) as cursor:
source_datasource_1 = get_dummy_datasource(cursor, 5, "dummy-src-5")
dest_datasource = get_dummy_datasource(cursor, 6, "dummy-transform-src")
dest_entitytype = get_dummy_entitytype(cursor, 45, "dummy_type_aggregate")
get_function_mapping(cursor, 11, "sum", ["counter_a"], "sum_a")
get_function_set(
cursor, 43, "test_set_agg", [11], [5], 45, 900, 6, dest_entitytype.id, dest_granularity, None, ["entity_id"]
)
args = 1, "unittest", "transform", ""
add_job_source(cursor, *args)
size = 233
job_source_id = 1
args = job_id, "transform", "", size, "2012-12-11 14:34:00", None, None, None, job_source_id, "running"
add_job(cursor, *args)
args = 43, description["dest_timestamp"], description["processed_max_modified"], "2012-12-11 13:03:00", job_id
add_state(cursor, *args)
table_name = "dummy-src-5_dummy_type_aggregate_qtr_20121211"
columns = [
Column("entity_id"),
Column("timestamp", type_=SqlType("timestamp with time zone")),
Column("modified", type_=SqlType("timestamp with time zone")),
Column("counter_a"),
Column("counter_b"),
]
table = Table("trend", table_name, columns=columns)
if table_exists(cursor, table):
table.drop().execute(cursor)
table.create().execute(cursor)
store(cursor, table, source_1)
result_table = Table("trend", "dummy-transform-src_dummy_type_aggregate_hr_20121207")
if table_exists(cursor, result_table):
result_table.truncate().execute(cursor)
conn.commit()
print("source_1")
print("\n".join(render_source(source_1)))
job.execute()
columns = map(Column, ["entity_id", "sum_a"])
query = result_table.select(columns)
with closing(conn.cursor()) as cursor:
query.execute(cursor, args)
print("\n".join(render_result(cursor)))
query.execute(cursor, args)
row = cursor.fetchone()
eq_(row[1], 21)