def test_sync():
try:
local_test_data_prefix = tempfile.mkdtemp()
print(f"Created temp directory '{local_test_data_prefix}'")
print(f"Loading manifest: {TEST_MANIFEST_FNAME}")
manifest = DataManifest(TEST_MANIFEST_FNAME,
remote_prefix=DEFAULT_REMOTE_PREFIX)
print(LOCAL_TEST_DATA_PREFIX)
manifest.sync(LOCAL_TEST_DATA_PREFIX)
print(f"Synced data to '{LOCAL_TEST_DATA_PREFIX}'")
finally:
reset_test_manifest()
shutil.rmtree(local_test_data_prefix)
get_gcs_blob(DEFAULT_REMOTE_PREFIX, TEST_REMOTE_RELATIVE_PATH).delete()
def test_verify_success():
"""test that verify works when everything matches."""
try:
local_test_data_prefix = tempfile.mkdtemp()
print(f"Created temp directory '{local_test_data_prefix}'")
os.mkdir(os.path.join(local_test_data_prefix, "./hg38"))
shutil.copy(
TEST_DATA_FILE_2,
os.path.join(local_test_data_prefix, TEST_2_LOCAL_RELATIVE_PATH))
print(f"Loading manifest: {TEST_MANIFEST_FNAME}")
manifest = DataManifest(TEST_MANIFEST_FNAME,
remote_prefix=DEFAULT_REMOTE_PREFIX)
print(local_test_data_prefix)
manifest.sync(local_test_data_prefix)
print(f"Synced data to '{local_test_data_prefix}'")
finally:
reset_test_manifest()
print(local_test_data_prefix)
# shutil.rmtree(local_test_data_prefix)
get_gcs_blob(DEFAULT_REMOTE_PREFIX, TEST_REMOTE_RELATIVE_PATH).delete()
def test_verify_md5_fail():
"""test that verify failes when the sizes match but the md5sum's dont."""
try:
local_test_data_prefix = tempfile.mkdtemp()
print(f"Created temp directory '{local_test_data_prefix}'")
os.mkdir(os.path.join(local_test_data_prefix, "./hg38"))
shutil.copy(
# we are copying the wrong file to induce an error
TEST_DATA_FILE,
os.path.join(local_test_data_prefix, TEST_2_LOCAL_RELATIVE_PATH))
print(f"Loading manifest: {TEST_MANIFEST_FNAME}")
manifest = DataManifest(TEST_MANIFEST_FNAME,
remote_prefix=DEFAULT_REMOTE_PREFIX)
print(local_test_data_prefix)
manifest.sync(local_test_data_prefix)
print(f"Synced data to '{local_test_data_prefix}'")
finally:
reset_test_manifest()
print(local_test_data_prefix)
# shutil.rmtree(local_test_data_prefix)
get_gcs_blob(DEFAULT_REMOTE_PREFIX, TEST_REMOTE_RELATIVE_PATH).delete()
def test_removing_file_works():
try:
print(f"Loading manifest: {TEST_MANIFEST_FNAME}")
manifest = DataManifest(TEST_MANIFEST_FNAME,
remote_prefix=DEFAULT_REMOTE_PREFIX)
print(f"Adding file to manifest.")
manifest.add_file('eight_As', TEST_DATA_FILE, TEST_LOCAL_RELATIVE_PATH,
TEST_REMOTE_RELATIVE_PATH)
# make sure that the file is in the manifest
manifest_2 = DataManifest(TEST_MANIFEST_FNAME,
remote_prefix=DEFAULT_REMOTE_PREFIX)
assert 'eight_As' in manifest_2
print(f"Removing file from manifest.")
manifest_2.remove_file('eight_As')
manifest_3 = DataManifest(TEST_MANIFEST_FNAME,
remote_prefix=DEFAULT_REMOTE_PREFIX)
assert 'eight_As' not in manifest_3
finally:
reset_test_manifest()
get_gcs_blob(DEFAULT_REMOTE_PREFIX, TEST_REMOTE_RELATIVE_PATH).delete()
def test_add_duplicate_key():
# test that we get an error if we try to add this file again with the same key
try:
manifest = DataManifest(TEST_MANIFEST_FNAME, DEFAULT_REMOTE_PREFIX)
manifest.add_file('eight_As', TEST_DATA_FILE, TEST_LOCAL_RELATIVE_PATH,
TEST_REMOTE_RELATIVE_PATH)
manifest.add_file('eight_As', TEST_DATA_FILE, TEST_LOCAL_RELATIVE_PATH,
TEST_REMOTE_RELATIVE_PATH)
# this is what we expect. Any other exception will be propogated.
except KeyAlreadyExistsError:
pass
# no expection is enexpected
else:
assert False, "Expected to see a 'KeyAlreadyExistsError'"
finally:
reset_test_manifest()
def test_adding_two_files_works():
try:
test_data_fname = TEST_DATA_FILE
remote_relative_path = TEST_REMOTE_RELATIVE_PATH
local_relative_path = TEST_LOCAL_RELATIVE_PATH
print(f"Loading manifest: {TEST_MANIFEST_FNAME}")
manifest = DataManifest(TEST_MANIFEST_FNAME,
remote_prefix=DEFAULT_REMOTE_PREFIX)
print(f"Adding file 1 to manifest.")
manifest.add_file('eight_As', test_data_fname, local_relative_path,
remote_relative_path)
print(f"Adding file 2 to manifest.")
manifest.add_file('eight_As_v2', test_data_fname,
local_relative_path + '.v2',
remote_relative_path + '.v2')
finally:
reset_test_manifest()
get_gcs_blob(DEFAULT_REMOTE_PREFIX, TEST_REMOTE_RELATIVE_PATH).delete()