def setup_module(): """Put raw data and scripts in appropriate .retriever directories.""" for test in tests: if not os.path.exists(os.path.join(HOME_DIR, "raw_data", test['name'])): os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name'])) rd_path = os.path.join(HOME_DIR, "raw_data", test['name'], test['name'] + '.txt') create_file(test['raw_data'], rd_path) path_js = os.path.join(HOME_DIR, "scripts", test['name'] + '.json') with open(path_js, 'w') as js: json.dump(test['script'], js, indent=2) read_json(os.path.join(HOME_DIR, "scripts", test['name']))
def get_script_module(script_name): """Load a script module""" if script_name in python_files: file, pathname, desc = imp.find_module(script_name, [working_script_dir]) return imp.load_module(script_name + '.py', file, pathname, desc) return read_json(os.path.join(retriever_root_dir, 'scripts', script_name))
def get_script_upstream(dataset, repo=REPOSITORY): """Return the upstream script for a named dataset.""" is_json = True script = dataset.replace('-', '_') script_name = script + ".json" filepath = "scripts/" + script_name newpath = os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)) r = get_data_upstream(repo + filepath) if not r: is_json = False script_name = script + ".py" filepath = "scripts/" + script_name newpath = os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)) r = get_data_upstream(repo + filepath) if not r: return None with open(newpath, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): f.write(chunk) r.close() if is_json: read_script = read_json(join(SCRIPT_WRITE_PATH, script)) setattr(read_script, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name)) setattr(read_script, "_name", script) return read_script file, pathname, desc = imp.find_module(script, [SCRIPT_WRITE_PATH]) new_module = imp.load_module(script, file, pathname, desc) setattr(new_module.SCRIPT, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name)) setattr(new_module.SCRIPT, "_name", script) return new_module.SCRIPT
def get_script(path_to_archive): """ Reads script from archive. """ with ZipFile(os.path.normpath(path_to_archive), 'r') as archive: try: commit_details = get_metadata(path_to_archive=path_to_archive) workdir = mkdtemp(dir=os.path.dirname(path_to_archive)) archive.extract( '/'.join(('script', commit_details['script_name'])), workdir) if commit_details['script_name'].endswith('.json'): script_object = read_json( os.path.join(workdir, 'script', commit_details['script_name'].split('.')[0])) elif commit_details['script_name'].endswith('.py'): spec = util.spec_from_file_location( "script_module", os.path.join(workdir, 'script', commit_details['script_name']), ) script_module = util.module_from_spec(spec) spec.loader.exec_module(script_module) script_object = script_module.SCRIPT rmtree(workdir) except Exception as e: print(e) return return script_object
def reload_scripts(): """Load scripts from scripts directory and return list of modules.""" modules = [] loaded_files = [] loaded_scripts = [] if not os.path.isdir(SCRIPT_WRITE_PATH): os.makedirs(SCRIPT_WRITE_PATH) for search_path in [search_path for search_path in SCRIPT_SEARCH_PATHS if exists(search_path)]: data_packages = [file_i for file_i in os.listdir(search_path) if file_i.endswith(".json")] for script in data_packages: script_name = '.'.join(script.split('.')[:-1]) if script_name not in loaded_files: read_script = read_json(join(search_path, script_name)) if read_script and read_script.name.lower() not in loaded_scripts: if not check_retriever_minimum_version(read_script): continue setattr(read_script, "_file", os.path.join(search_path, script)) setattr(read_script, "_name", script_name) modules.append(read_script) loaded_files.append(script_name) loaded_scripts.append(read_script.name.lower()) files = [file for file in os.listdir(search_path) if file[-3:] == ".py" and file[0] != "_" and ('#retriever' in ' '.join(open_fr(join(search_path, file), encoding=ENCODING).readlines()[:2]).lower()) ] for script in files: script_name = '.'.join(script.split('.')[:-1]) if script_name not in loaded_files: loaded_files.append(script_name) file, pathname, desc = imp.find_module(script_name, [search_path]) try: new_module = imp.load_module(script_name, file, pathname, desc) if hasattr(new_module.SCRIPT, "retriever_minimum_version"): # a script with retriever_minimum_version should be loaded # only if its compliant with the version of the retriever if not check_retriever_minimum_version(new_module.SCRIPT): continue # if the script wasn't found in an early search path # make sure it works and then add it new_module.SCRIPT.download setattr(new_module.SCRIPT, "_file", os.path.join(search_path, script)) setattr(new_module.SCRIPT, "_name", script_name) modules.append(new_module.SCRIPT) except Exception as e: sys.stderr.write("Failed to load script: {} ({})\n" "Exception: {} \n" .format(script_name, search_path, str(e))) if global_script_list: global_script_list.set_scripts(modules) return modules
def get_script_module(script_name): """Load a script module.""" return read_json(os.path.join(HOME_DIR, "scripts", script_name))
def get_script_module(script_name): """Load a script module.""" return read_json(os.path.join(file_location, script_name))
def get_script_module(script_name): """Load a script module""" return read_json(os.path.join(retriever_root_dir, "scripts", script_name))
def get_script_module(script_name): """Load a script module.""" return read_json(join_path([file_location, script_name]))