def test_reset_retriever(tmpdir):
"""Test the dataset reset function."""
pwd_name = os.getcwd()
workdir = tmpdir.mkdtemp()
workdir.chdir()
offline_datasets = rt.dataset_names()['offline']
offline_datasets = [
dataset for dataset in offline_datasets
if not dataset.startswith('test-')
]
if not offline_datasets:
return
dataset = random.choice(offline_datasets)
rt.reset_retriever(dataset)
rt.reload_scripts()
assert os.path.exists(
os.path.join(HOME_DIR,
dataset.replace("-", "_") + ".json")) == False
assert os.path.exists(
os.path.join(HOME_DIR,
dataset.replace("-", "_") + ".py")) == False
if dataset in RETRIEVER_DATASETS:
rt.get_script_upstream(dataset, repo=RETRIEVER_REPOSITORY)
else:
rt.get_script_upstream(dataset)
rt.reload_scripts()
assert dataset in rt.dataset_names()['offline']
os.chdir(pwd_name)
def setup_module():
"""Update retriever scripts and cd to test directory to find data."""
os.chdir(retriever_root_dir)
subprocess.call(['cp', '-r', 'test/raw_data', retriever_root_dir])
src = os.path.join(retriever_root_dir, 'scripts')
copy_tree(src, script_home)
# Add spatail scripts
spatial_src = os.path.join(retriever_root_dir, 'test/raw_data_gis/scripts/')
copy_tree(spatial_src, script_home)
# Use reload to include the new test scripts
rt.reload_scripts()
def run():
create_dirs()
datasets_to_check = []
if (os.environ.get("RETR_TEST") == "true"):
datasets_to_check = [
script for script in reload_scripts() if script.name in DEV_LIST
]
else:
datasets_to_check = reload_scripts()
for dataset in datasets_to_check:
print("Checking dataset {}:".format(dataset.name))
check_dataset(dataset)
def test_reset_retriever(tmpdir):
"""Test the dataset reset function."""
pwd_name = os.getcwd()
workdir = tmpdir.mkdtemp()
workdir.chdir()
dataset = random.choice(rt.dataset_names())
rt.reset_retriever(dataset)
rt.reload_scripts()
assert dataset not in rt.dataset_names()
rt.check_for_updates()
rt.reload_scripts()
assert dataset in rt.dataset_names()
os.chdir(pwd_name)
def setup_module():
"""Update retriever scripts and cd to test directory to find data."""
os.chdir(retriever_root_dir)
subprocess.call(['cp', '-r', 'test/raw_data', retriever_root_dir])
src = os.path.join(retriever_root_dir, 'scripts')
copy_tree(src, script_home)
# Add spatail scripts
spatial_src = os.path.join(retriever_root_dir, 'test/raw_data_gis/scripts/')
copy_tree(spatial_src, script_home)
# Use reload to include the new test scripts
rt.reload_scripts()
def create_json(path="dataset_details.json"):
"""This function creates a json file with md5 values
of all(currently those in example_datasets) datasets"""
data = {}
for dataset in reload_scripts():
if dataset.name in example_datasets:
data[dataset.name] = {"md5": get_dataset_md5(dataset)}
with open(path, 'w') as json_file:
json.dump(data, json_file, sort_keys=True, indent=4)
def set_retriever_res(resource_up=True):
"""Create or tear down retriever data and scripts
If resource_up, set up retriever else tear down.
Data directory names use "-",
Data file names use "_"
Script file names use "_"
"""
if not os.path.exists(RETRIEVER_SCRIPT_DIR):
os.makedirs(RETRIEVER_SCRIPT_DIR)
for file_names in RETRIEVER_TESTS_DATA:
data_dir_path = os.path.join(RETRIEVER_DATA_DIR, file_names["name"])
data_dir_path = os.path.normpath(data_dir_path)
data_file_name = file_names["name"].replace("-", "_") + ".txt"
data_file_path = os.path.normpath(os.path.join(data_dir_path, data_file_name))
script_name = file_names["script"]["name"] + ".json"
script_file_path = os.path.normpath(
os.path.join(RETRIEVER_SCRIPT_DIR, script_name.replace("-", "_"))
)
# Set or tear down raw data files
# in '~/.retriever/raw_data/data_dir_path/data_file_name'
if resource_up:
if not os.path.exists(data_dir_path):
os.makedirs(data_dir_path)
create_file(file_names["raw_data"], data_file_path)
with open(script_file_path, "w") as js:
json.dump(file_names["script"], js, indent=2)
else:
shutil.rmtree(data_dir_path)
os.remove(script_file_path)
for script_name in RETRIEVER_SPATIAL_DATA:
file_name = script_name.replace("-", "_")
script_file_path = os.path.normpath(
os.path.join(RETRIEVER_SCRIPT_DIR, script_name.replace("-", "_") + ".json")
)
if resource_up:
url = RETRIEVER_GIS_REPO.format(script_names=file_name)
urlretrieve(url, script_file_path)
else:
os.remove(script_file_path)
rt.reload_scripts()
def run():
create_dirs()
datasets_to_check = [script for script in reload_scripts() if
script.name not in IGNORE_LIST]
for dataset in datasets_to_check:
check_dataset(dataset)
