def test_etl():
ctx = reload.context()
with copy_file("redcap/record.json", ctx["dataInputFilePath"]):
with copy_file("redcap/metadata.json", ctx["dataDictionaryInputFilePath"]):
with datatables(lambda: reload.etl(ctx)) as ret:
assert ret
assert os.path.isfile("/data/tables/Proposal")
with open("/data/tables/Proposal") as f:
assert sum(1 for _ in f) == 2
def do_test_auxiliary(aux1, exp):
aux0 = os.environ.get("AUXILIARY_PATH")
os.environ["AUXILIARY_PATH"] = aux1
ctx = reload.context()
shutil.copy("redcap/record.json", ctx["dataInputFilePath"])
shutil.copy("redcap/metadata.json", ctx["dataDictionaryInputFilePath"])
assert reload.etl(ctx)
with open("/data/tables/ProposalFunding") as f:
i = f.readline().split(",").index("totalBudgetInt")
assert f.readline().split(",")[i] == exp
os.remove(ctx["dataInputFilePath"])
os.remove(ctx["dataDictionaryInputFilePath"])
shutil.rmtree("/data/tables")
if aux0 is None:
del os.environ["AUXILIARY_PATH"]
else:
os.environ["AUXILIARY_PATH"] = aux0
def do_test_blocklist2(blocklist1, exp):
blocklist0 = os.environ.get("BLOCK_PATH")
os.environ["BLOCK_PATH"] = blocklist1
ctx = reload.context()
shutil.copy("redcap/record2.json", ctx["dataInputFilePath"])
shutil.copy("redcap/metadata.json", ctx["dataDictionaryInputFilePath"])
assert reload.etl(ctx)
with open("/data/tables/Proposal", newline="") as f:
reader = csv.reader(f)
headers = next(reader)
i = sum(1 for row in reader)
assert i == exp
os.remove(ctx["dataInputFilePath"])
os.remove(ctx["dataDictionaryInputFilePath"])
shutil.rmtree("/data/tables")
if blocklist0 is None:
del os.environ["BLOCK_PATH"]
else:
os.environ["BLOCK_PATH"] = blocklist0