def framework_ref_test(path):
conn = sqlite3.connect(path)
wrapper = pygrametl.ConnectionWrapper(connection=conn)
dim1 = Dimension(
name='dim1',
key='key1',
attributes=['attr1', 'attr2']
)
dim2 = Dimension(
name='dim2',
key='key2',
attributes=['attr3', 'attr4'],
)
ft1 = FactTable(
name='ft1',
keyrefs=['key1', 'key2'],
measures=['measure']
)
dim1rep = DimRepresentation(dim1, conn)
dim2rep = DimRepresentation(dim2, conn)
ft1rep = FTRepresentation(ft1, conn)
dw_rep = DWRepresentation([dim1rep, dim2rep], conn, [ft1rep])
not_null = ReferentialIntegrityPredicate()
case = Case(dw_rep, [not_null])
start = time.monotonic() # the instantiations take almost no time
case.run() # but we may as well measure the time for
conn.close() # executing the case
end = time.monotonic()
elapsed = end - start
print('{}{}'.format(round(elapsed, 3), 's'))
def framework_dup_test(dbname):
dw_rep = make_dw_rep(dbname)
dup = NoDuplicateRowPredicate('ft1', ['key1', 'key2'])
case = Case(dw_rep, [dup])
start = time.monotonic() # the instantiations take almost no time
case.run()
end = time.monotonic()
elapsed = end - start
print('{}{}'.format(round(elapsed, 3), 's'))
def framework_row_count_test(path):
dw_rep = make_dw_rep(path)
count = RowCountPredicate('ft1', 1000000)
case = Case(dw_rep, [count])
start = time.monotonic() # the instantiations take almost no time
case.run() # but we may as well measure the time for
end = time.monotonic() # executing the case
elapsed = end - start
print('{}{}'.format(round(elapsed, 3), 's'))
def framework_not_null_test(dbname):
dw_rep = make_dw_rep(dbname)
not_null = ColumnNotNullPredicate('ft1', ['key1', 'key2'])
case = Case(dw_rep, [not_null])
start = time.monotonic() # the instantiations take almost no time
case.run()
end = time.monotonic()
elapsed = end - start
print('{}{}'.format(round(elapsed, 3), 's'))
def framework_fd_test(path):
dw_rep = make_dw_rep(path)
print("Running Case")
fd = FunctionalDependencyPredicate(["ft1"], ((("key1",), ("key2",)),))
case = Case(dw_rep, [fd])
start = time.monotonic()
case.run()
end = time.monotonic()
elapsed = end - start
print("Done: {}{}".format(round(elapsed, 3), "s"))
def framework_scd_test(dbname):
dw_rep = make_scd_rep(dbname)
print('Running Case')
scd = SCDVersionPredicate('scd', {'attr2': 1}, 1)
case = Case(dw_rep, [scd])
start = time.monotonic()
case.run()
end = time.monotonic()
elapsed = end - start
print('Done: {}{}'.format(round(elapsed, 3), 's'))
def framework_compare_test(path, number):
dw_rep = make_dw_rep(path)
expected_rows = []
counter = 0
print('Generating expected table')
start = time.monotonic()
for i in range(1, number + 1):
for j in range(1, number + 1):
counter += 10
expected_rows.append({'key1': i, 'key2': j, 'measure': counter})
end = time.monotonic()
elapsed = end - start
print('Done: {}{}'.format(round(elapsed, 3), 's'))
print('Running Case')
compare = CompareTablePredicate('ft1', expected_rows)
case = Case(dw_rep, [compare])
start = time.monotonic()
case.run()
end = time.monotonic()
elapsed = end - start
print('Done: {}{}'.format(round(elapsed, 3), 's'))
import sqlite3
path = "test_program_1.py"
notnull = ColumnNotNullPredicate('dim1', 'att1')
input_conn = sqlite3.connect('input.db')
input2_conn = sqlite3.connect('input2.db')
dw_conn = sqlite3.connect('dw.db')
dw_conn2 = sqlite3.connect('dw2.db')
try:
# Exact amount of sources
c = Case(path, [input_conn, input2_conn], dw_conn, [notnull], True)
c.run()
except Exception as e:
print(e)
try:
# Exact amount of sources
c = Case(path, [input_conn, input2_conn], dw_conn2, [notnull], True)
c.run()
except Exception as e:
print(e)
rip_test = ReferentialIntegrityPredicate()
pred_list = [ndrp_test, e_ndrp_test, fdp_test, e_fdp_test, rip_test]
dw_path = './dw.db'
pygrametl_program_path = './etl.py'
dwp = DWPopulator(pygrametl_program_path,
pep249_module=sqlite3,
program_is_path=True,
database=dw_path) # Arg til sqlite3
dw_rep = dwp.run()
# Checking how long it took.
time_before_test = time_passed(start)
case = Case(dw_rep, pred_list)
case.run()
time_after_test = time_passed(start)
# Checking how long it took.
print(" TIME BEFORE TEST " + time_before_test)
print(" TIME AFTER TEST " + time_after_test)
