def check_dataset(dataset):
os.chdir(os.path.join(file_location))
md5 = None
status = None
reason = None
diff = None
try:
try:
dataset_detail = json.load(open('dataset_details.json', 'r'))
except FileNotFoundError:
with open("dataset_details.json", 'w') as json_file:
dataset_detail = dict()
json.dump(dataset_detail, json_file)
if dataset_type(dataset) == 'spatial':
workdir = None
try:
workdir = mkdtemp(dir=file_location)
download(dataset.name, path=workdir)
md5 = getmd5(workdir, data_type='dir')
except Exception:
raise
finally:
if workdir:
rmtree(workdir)
else:
md5 = get_dataset_md5(dataset)
if dataset.name not in dataset_detail \
or md5 != dataset_detail[dataset.name]['md5']:
diff = diff_generator(dataset)
status = True
except Exception as e:
reason = str(e)
status = False
finally:
os.chdir(os.path.join(file_location))
with FileLock('dataset_details.json.lock'):
dataset_details_read = open('dataset_details.json', 'r')
json_file_details = json.load(dataset_details_read)
json_file_details[dataset.name] = {
"md5": md5,
"status": status,
"reason": reason,
"diff": diff
}
dataset_details_write = open('dataset_details.json', 'w')
json.dump(json_file_details,
dataset_details_write,
sort_keys=True,
indent=4)
def check_dataset(dataset):
md5 = None
status = None
reason = None
diff = None
dataset_detail = None
try:
try:
with open(os.path.join(file_location, "dataset_details.json"), 'r') as json_file:
dataset_detail = json.load(json_file)
except (OSError, JSONDecodeError):
dataset_detail = dict()
dataset_detail['dataset_details'] = {}
if dataset_type(dataset) == 'spatial':
workdir = None
try:
workdir = mkdtemp(dir=file_location)
download(dataset.name, path=workdir)
md5 = getmd5(workdir, data_type='dir')
except Exception:
raise
finally:
if workdir:
rmtree(workdir)
else:
md5 = get_dataset_md5(dataset)
if dataset.name not in dataset_detail \
or md5 != dataset_detail[dataset.name]['md5']:
diff = diff_generator(dataset)
status = True
except Exception as e:
reason = str(e)
status = False
finally:
json_file_details = dataset_detail
json_file_details["dataset_details"][dataset.name] = {
"md5": md5,
"status": status,
"reason": reason,
"diff": diff}
json_file_details["last_checked_on"] = datetime.now(timezone.utc).strftime("%d %b %Y")
dataset_details_write = open(os.path.join(file_location, 'dataset_details.json'), 'w')
json.dump(json_file_details, dataset_details_write,
sort_keys=True, indent=4)
dataset_details_write.close()
def test_download_regression(dataset, expected):
"""Test download regression."""
os.chdir(retriever_root_dir)
base_path = 'test_raw_data'
path = os.path.join(base_path, dataset)
data_dir = "test_temp"
data = os.path.normpath(os.path.join(retriever_root_dir, path, data_dir))
rt.download(dataset, path=path)
current_md5 = getmd5(data=path, data_type='dir')
assert current_md5 == expected
shutil.rmtree(base_path)
# download using path and sub_dir
os.chdir(retriever_root_dir)
rt.download(dataset, path=path, sub_dir=data_dir)
current_md5 = getmd5(data=data, data_type='dir')
assert current_md5 == expected
shutil.rmtree(base_path)
def test_download_regression(dataset, expected):
"""Test download regression."""
os.chdir(retriever_root_dir)
base_path = 'test_raw_data'
path = os.path.join(base_path, dataset)
data_dir = "test_temp"
data = os.path.normpath(os.path.join(retriever_root_dir, path, data_dir))
rt.download(dataset, path=path)
current_md5 = getmd5(data=path, data_type='dir')
assert current_md5 == expected
shutil.rmtree(base_path)
# download using path and sub_dir
os.chdir(retriever_root_dir)
rt.download(dataset, path=path, sub_dir=data_dir)
current_md5 = getmd5(data=data, data_type='dir')
assert current_md5 == expected
shutil.rmtree(base_path)
def test_download_regression(dataset, expected):
"""Test download regression."""
os.chdir(retriever_root_dir)
download(dataset, "raw_data/{0}".format(dataset))
current_md5 = getmd5(data="raw_data/{0}".format(dataset), data_type='dir')
assert current_md5 == expected
def test_download_regression(dataset, expected):
"""Test download regression."""
os.chdir(retriever_root_dir)
download(dataset, "raw_data/{0}".format(dataset))
current_md5 = getmd5(data="raw_data/{0}".format(dataset), data_type='dir')
assert current_md5 == expected