def setup_module():
"""Put raw data and scripts in appropriate .retriever directories."""
for test in tests:
if not os.path.exists(os.path.join(HOME_DIR, "raw_data", test['name'])):
os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name']))
rd_path = os.path.join(HOME_DIR,
"raw_data", test['name'], test['name'] + '.txt')
create_file(test['raw_data'], rd_path)
path_js = os.path.join(HOME_DIR, "scripts", test['name'] + '.json')
with open(path_js, 'w') as js:
json.dump(test['script'], js, indent=2)
read_json(os.path.join(HOME_DIR, "scripts", test['name']))
def get_script_module(script_name):
"""Load a script module"""
if script_name in python_files:
file, pathname, desc = imp.find_module(script_name,
[working_script_dir])
return imp.load_module(script_name + '.py', file, pathname, desc)
return read_json(os.path.join(retriever_root_dir, 'scripts', script_name))
def get_script_module(script_name):
"""Load a script module"""
if script_name in python_files:
file, pathname, desc = imp.find_module(script_name,
[working_script_dir])
return imp.load_module(script_name + '.py', file, pathname, desc)
return read_json(os.path.join(retriever_root_dir, 'scripts', script_name))
def get_script_upstream(dataset, repo=REPOSITORY):
"""Return the upstream script for a named dataset."""
is_json = True
script = dataset.replace('-', '_')
script_name = script + ".json"
filepath = "scripts/" + script_name
newpath = os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name))
r = get_data_upstream(repo + filepath)
if not r:
is_json = False
script_name = script + ".py"
filepath = "scripts/" + script_name
newpath = os.path.normpath(os.path.join(SCRIPT_WRITE_PATH,
script_name))
r = get_data_upstream(repo + filepath)
if not r:
return None
with open(newpath, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
r.close()
if is_json:
read_script = read_json(join(SCRIPT_WRITE_PATH, script))
setattr(read_script, "_file",
os.path.join(SCRIPT_WRITE_PATH, script_name))
setattr(read_script, "_name", script)
return read_script
file, pathname, desc = imp.find_module(script, [SCRIPT_WRITE_PATH])
new_module = imp.load_module(script, file, pathname, desc)
setattr(new_module.SCRIPT, "_file",
os.path.join(SCRIPT_WRITE_PATH, script_name))
setattr(new_module.SCRIPT, "_name", script)
return new_module.SCRIPT
def get_script(path_to_archive):
"""
Reads script from archive.
"""
with ZipFile(os.path.normpath(path_to_archive), 'r') as archive:
try:
commit_details = get_metadata(path_to_archive=path_to_archive)
workdir = mkdtemp(dir=os.path.dirname(path_to_archive))
archive.extract(
'/'.join(('script', commit_details['script_name'])), workdir)
if commit_details['script_name'].endswith('.json'):
script_object = read_json(
os.path.join(workdir, 'script',
commit_details['script_name'].split('.')[0]))
elif commit_details['script_name'].endswith('.py'):
spec = util.spec_from_file_location(
"script_module",
os.path.join(workdir, 'script',
commit_details['script_name']),
)
script_module = util.module_from_spec(spec)
spec.loader.exec_module(script_module)
script_object = script_module.SCRIPT
rmtree(workdir)
except Exception as e:
print(e)
return
return script_object
def reload_scripts():
"""Load scripts from scripts directory and return list of modules."""
modules = []
loaded_files = []
loaded_scripts = []
if not os.path.isdir(SCRIPT_WRITE_PATH):
os.makedirs(SCRIPT_WRITE_PATH)
for search_path in [search_path for search_path in SCRIPT_SEARCH_PATHS if exists(search_path)]:
data_packages = [file_i for file_i in os.listdir(search_path) if file_i.endswith(".json")]
for script in data_packages:
script_name = '.'.join(script.split('.')[:-1])
if script_name not in loaded_files:
read_script = read_json(join(search_path, script_name))
if read_script and read_script.name.lower() not in loaded_scripts:
if not check_retriever_minimum_version(read_script):
continue
setattr(read_script, "_file", os.path.join(search_path, script))
setattr(read_script, "_name", script_name)
modules.append(read_script)
loaded_files.append(script_name)
loaded_scripts.append(read_script.name.lower())
files = [file for file in os.listdir(search_path)
if file[-3:] == ".py" and file[0] != "_" and
('#retriever' in
' '.join(open_fr(join(search_path, file), encoding=ENCODING).readlines()[:2]).lower())
]
for script in files:
script_name = '.'.join(script.split('.')[:-1])
if script_name not in loaded_files:
loaded_files.append(script_name)
file, pathname, desc = imp.find_module(script_name, [search_path])
try:
new_module = imp.load_module(script_name, file, pathname, desc)
if hasattr(new_module.SCRIPT, "retriever_minimum_version"):
# a script with retriever_minimum_version should be loaded
# only if its compliant with the version of the retriever
if not check_retriever_minimum_version(new_module.SCRIPT):
continue
# if the script wasn't found in an early search path
# make sure it works and then add it
new_module.SCRIPT.download
setattr(new_module.SCRIPT, "_file", os.path.join(search_path, script))
setattr(new_module.SCRIPT, "_name", script_name)
modules.append(new_module.SCRIPT)
except Exception as e:
sys.stderr.write("Failed to load script: {} ({})\n"
"Exception: {} \n"
.format(script_name, search_path, str(e)))
if global_script_list:
global_script_list.set_scripts(modules)
return modules
def get_script_module(script_name):
"""Load a script module."""
return read_json(os.path.join(HOME_DIR, "scripts", script_name))
def get_script_module(script_name):
"""Load a script module."""
return read_json(os.path.join(file_location, script_name))
def get_script_module(script_name):
"""Load a script module."""
return read_json(os.path.join(HOME_DIR, "scripts", script_name))
def get_script_module(script_name):
"""Load a script module"""
return read_json(os.path.join(retriever_root_dir, "scripts", script_name))
def get_script_module(script_name):
"""Load a script module"""
return read_json(os.path.join(retriever_root_dir, "scripts", script_name))
def get_script_module(script_name):
"""Load a script module."""
return read_json(join_path([file_location, script_name]))
