def test_redownload(self):
test_config.copy_corrupt_data()
local_files = utils.ls(config.CONFIG['GENERAL']['DATA_DIR'])
scihub.redownload(local_files)
new_local_files = utils.ls(config.CONFIG['GENERAL']['DATA_DIR'])
self.assertEqual(set(local_files), set(new_local_files))
for f in local_files:
with self.subTest(file=f):
_, healthy, msg = check.check_file(f, mode='file')
utils.eprint(msg)
self.assertTrue(healthy)
def test_etag_small_files(self):
for f in utils.ls(config.CONFIG['GENERAL']['DATA_DIR']):
with self.subTest(file=f):
#
# Assert that the computed etag is equal to the md5
# checksum for files smaller than the chunksize.
#
size_mb = max(10, int(os.path.getsize(f) / 1024**2))
self.assertEqual(checksum.md5(f),
checksum.etag(f, chunksize=2 * size_mb))
def test_doctor_delete(self):
test_config.copy_corrupt_data()
corrupt_files = utils.ls(test_config.TEST_DATA_DIR_CORRUPT, path=False)
healthy_files = utils.ls(test_config.TEST_DATA_DIR_ORIGINAL,
path=False)
main.doctor(delete=True)
#
# Assert that the corrupt files have been deleted.
#
for f in corrupt_files:
self.assertFalse(
os.path.isfile(
os.path.join(config.CONFIG['GENERAL']['DATA_DIR'], f)))
#
# Assert that the healthy files have not been deleted.
#
for f in healthy_files:
self.assertTrue(
os.path.isfile(
os.path.join(config.CONFIG['GENERAL']['DATA_DIR'], f)))
def test_check_file_zip_healthy(self):
for f in utils.ls(config.CONFIG['GENERAL']['DATA_DIR']):
with self.subTest(file=f):
#
# Assert that the files check out in `file` mode.
#
try:
file_path, healthy, message = \
check.check_file(f, mode='file')
self.assertTrue(healthy)
except Exception as e:
self.fail('File check failed: {}'.format(e))
def test_md5_from_file(self):
for f in utils.ls(config.CONFIG['GENERAL']['DATA_DIR']):
with self.subTest(file=f):
#
# Assert that the md5 sum computed from the local file is equal
# to the md5 sum obtained from the remote server.
#
try:
remote_md5 = scihub.md5(f)
self.assertEqual(checksum.md5(f), remote_md5)
except Exception as e:
self.fail('Remote MD5 could not be obtained: {}'.format(e))
def test_check_file_zip_corrupt(self):
test_config.clear_test_data()
test_config.copy_corrupt_data()
for f in utils.ls(config.CONFIG['GENERAL']['DATA_DIR']):
with self.subTest(file=f):
#
# Assert that the files are detected as corrupt in `file` mode.
#
try:
file_path, healthy, message = \
check.check_file(f, mode='file')
self.assertFalse(healthy)
except Exception as e:
self.fail('File check failed: {}'.format(e))
def test_doctor_repair(self):
test_config.copy_corrupt_data()
corrupt_files = utils.ls(test_config.TEST_DATA_DIR_CORRUPT, path=False)
# healthy_files = utils.ls(test_config.TEST_DATA_DIR_ORIGINAL,
# path=False)
main.doctor(repair=True)
for f in corrupt_files:
repaired_f = os.path.join(config.CONFIG['GENERAL']['DATA_DIR'], f)
with self.subTest(file=repaired_f):
#
# Assert that each corrupt file has been repaired.
#
_, healthy, msg = check.check_file(repaired_f, mode='file')
utils.eprint(msg)
self.assertTrue(healthy)
def test_md5(self):
for f in utils.ls(config.CONFIG['GENERAL']['DATA_DIR']):
with self.subTest(file=f):
#
# Assert that the md5 checksum returned by checksum.md5() is
# equal to the md5 sum returned by bash md5 or md5sum tool.
#
for exe in ['md5', 'md5sum']:
if utils._which(exe) is not None:
bash_output = subprocess.check_output([exe, f])
if not PY2:
bash_output = bash_output.decode()
bash_md5 = re.search('[a-zA-Z0-9]{32}',
bash_output).group()
break
self.assertEqual(checksum.md5(f), bash_md5)
def test_download_many(self):
file_list = scihub.search({'query': SMALL_SIZE_QUERY}, limit=2)
scihub.download(file_list)
#
# Assert that all downloads were successful.
#
local_files = utils.ls(config.CONFIG['GENERAL']['DATA_DIR'])
local_files_identifiers = [
os.path.splitext(os.path.split(_)[1])[0] for _ in local_files
]
for f in file_list:
self.assertIn(f['filename'], local_files_identifiers)
for f in local_files:
with self.subTest(file=f):
_, healthy, msg = check.check_file(f, mode='file')
utils.eprint(msg)
self.assertTrue(healthy)
def test_doctor(self):
test_config.copy_corrupt_data()
corrupt_files = utils.ls(test_config.TEST_DATA_DIR_CORRUPT, path=False)
# healthy_files = utils.ls(test_config.TEST_DATA_DIR_ORIGINAL,
# path=False)
result = main.doctor()
bad_files = [
os.path.split(status[0])[1] for status in result
if status[1] is False
]
#
# Assert that the number of healthy/corrupt files detected are correct
#
self.assertEqual(len(bad_files), len(corrupt_files))
for corrupt_file in corrupt_files:
#
# Assert that each corrupt file has been registered.
#
self.assertIn(corrupt_file, bad_files)
def copy_corrupt_data():
for f in utils.ls(TEST_DATA_DIR_CORRUPT):
target = os.path.join(config.CONFIG['GENERAL']['DATA_DIR'],
os.path.split(f)[1])
shutil.copy2(f, target)
def copy_test_data():
for f in utils.ls(TEST_DATA_DIR_ORIGINAL):
target = os.path.join(config.CONFIG['GENERAL']['DATA_DIR'],
os.path.split(f)[1])
shutil.copy2(f, target)