def test_reset_retriever(tmpdir): """Test the dataset reset function.""" pwd_name = os.getcwd() workdir = tmpdir.mkdtemp() workdir.chdir() offline_datasets = rt.dataset_names()['offline'] offline_datasets = [ dataset for dataset in offline_datasets if not dataset.startswith('test-') ] if not offline_datasets: return dataset = random.choice(offline_datasets) rt.reset_retriever(dataset) rt.reload_scripts() assert os.path.exists( os.path.join(HOME_DIR, dataset.replace("-", "_") + ".json")) == False assert os.path.exists( os.path.join(HOME_DIR, dataset.replace("-", "_") + ".py")) == False if dataset in RETRIEVER_DATASETS: rt.get_script_upstream(dataset, repo=RETRIEVER_REPOSITORY) else: rt.get_script_upstream(dataset) rt.reload_scripts() assert dataset in rt.dataset_names()['offline'] os.chdir(pwd_name)
def setup_module(): """Update retriever scripts and cd to test directory to find data.""" os.chdir(retriever_root_dir) subprocess.call(['cp', '-r', 'test/raw_data', retriever_root_dir]) src = os.path.join(retriever_root_dir, 'scripts') copy_tree(src, script_home) # Add spatail scripts spatial_src = os.path.join(retriever_root_dir, 'test/raw_data_gis/scripts/') copy_tree(spatial_src, script_home) # Use reload to include the new test scripts rt.reload_scripts()
def run(): create_dirs() datasets_to_check = [] if (os.environ.get("RETR_TEST") == "true"): datasets_to_check = [ script for script in reload_scripts() if script.name in DEV_LIST ] else: datasets_to_check = reload_scripts() for dataset in datasets_to_check: print("Checking dataset {}:".format(dataset.name)) check_dataset(dataset)
def test_reset_retriever(tmpdir): """Test the dataset reset function.""" pwd_name = os.getcwd() workdir = tmpdir.mkdtemp() workdir.chdir() dataset = random.choice(rt.dataset_names()) rt.reset_retriever(dataset) rt.reload_scripts() assert dataset not in rt.dataset_names() rt.check_for_updates() rt.reload_scripts() assert dataset in rt.dataset_names() os.chdir(pwd_name)
def setup_module(): """Update retriever scripts and cd to test directory to find data.""" os.chdir(retriever_root_dir) subprocess.call(['cp', '-r', 'test/raw_data', retriever_root_dir]) src = os.path.join(retriever_root_dir, 'scripts') copy_tree(src, script_home) # Add spatail scripts spatial_src = os.path.join(retriever_root_dir, 'test/raw_data_gis/scripts/') copy_tree(spatial_src, script_home) # Use reload to include the new test scripts rt.reload_scripts()
def create_json(path="dataset_details.json"): """This function creates a json file with md5 values of all(currently those in example_datasets) datasets""" data = {} for dataset in reload_scripts(): if dataset.name in example_datasets: data[dataset.name] = {"md5": get_dataset_md5(dataset)} with open(path, 'w') as json_file: json.dump(data, json_file, sort_keys=True, indent=4)
def set_retriever_res(resource_up=True): """Create or tear down retriever data and scripts If resource_up, set up retriever else tear down. Data directory names use "-", Data file names use "_" Script file names use "_" """ if not os.path.exists(RETRIEVER_SCRIPT_DIR): os.makedirs(RETRIEVER_SCRIPT_DIR) for file_names in RETRIEVER_TESTS_DATA: data_dir_path = os.path.join(RETRIEVER_DATA_DIR, file_names["name"]) data_dir_path = os.path.normpath(data_dir_path) data_file_name = file_names["name"].replace("-", "_") + ".txt" data_file_path = os.path.normpath(os.path.join(data_dir_path, data_file_name)) script_name = file_names["script"]["name"] + ".json" script_file_path = os.path.normpath( os.path.join(RETRIEVER_SCRIPT_DIR, script_name.replace("-", "_")) ) # Set or tear down raw data files # in '~/.retriever/raw_data/data_dir_path/data_file_name' if resource_up: if not os.path.exists(data_dir_path): os.makedirs(data_dir_path) create_file(file_names["raw_data"], data_file_path) with open(script_file_path, "w") as js: json.dump(file_names["script"], js, indent=2) else: shutil.rmtree(data_dir_path) os.remove(script_file_path) for script_name in RETRIEVER_SPATIAL_DATA: file_name = script_name.replace("-", "_") script_file_path = os.path.normpath( os.path.join(RETRIEVER_SCRIPT_DIR, script_name.replace("-", "_") + ".json") ) if resource_up: url = RETRIEVER_GIS_REPO.format(script_names=file_name) urlretrieve(url, script_file_path) else: os.remove(script_file_path) rt.reload_scripts()
def run(): create_dirs() datasets_to_check = [script for script in reload_scripts() if script.name not in IGNORE_LIST] for dataset in datasets_to_check: check_dataset(dataset)