def pull(*params):
parser = parser_pull()
results = parser.parse_args(params)
from boutiques.puller import Puller
puller = Puller(results.zids, results.verbose, results.sandbox)
return puller.pull()
def pull(*params):
parser = ArgumentParser("Ensures that Zenodo descriptors are locally "
"cached, downloading them if needed.")
parser.add_argument("zids",
nargs="+",
action="store",
help="One or "
"more Zenodo IDs for the descriptor(s) to pull, "
"prefixed by 'zenodo.', e.g. zenodo.123456 "
"zenodo.123457")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Print information messages")
parser.add_argument("--sandbox",
action="store_true",
help="pull from Zenodo's sandbox instead of "
"production server. Recommended for tests.")
result = parser.parse_args(params)
from boutiques.puller import Puller
puller = Puller(result.zids, result.verbose, result.sandbox)
return puller.pull()
def run(self):
try:
boutique_cache_dir = os.path.join(os.path.expanduser('~'),
".cache", "boutiques",
"production")
# first search for all descriptors
searcher = Searcher(query=None, max_results=100, no_trunc=True)
all_descriptors = searcher.search()
# then pull every single descriptor
all_descriptor_ids = list(map(lambda x: x["ID"], all_descriptors))
files = Puller(all_descriptor_ids).pull()
# fetch every single descriptor into one file
detailed_all_descriptors = list(
map(lambda f: json.load(open(f, 'r')), files))
# store data in cache
with open(os.path.join(boutique_cache_dir, "all_descriptors.json"),
"w") as f:
json.dump(all_descriptors, f, indent=4)
with open(
os.path.join(boutique_cache_dir,
"detailed_all_descriptors.json"), "w") as f:
json.dump(detailed_all_descriptors, f, indent=4)
except Exception as e:
logging.exception(
"An exception occurred in the thread:{0}.".format(e))
def run(self):
try:
# if cache directory doesn't exist then create it
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
# first search for all descriptors
searcher = Searcher(query="", max_results=100, no_trunc=True)
all_descriptors = searcher.search()
# then pull every single descriptor
all_descriptor_ids = list(map(lambda x: x["ID"], all_descriptors))
Puller(all_descriptor_ids).pull()
# fetch every single descriptor into one file
detailed_all_descriptors = [
json.load(open(os.path.join(self.cache_dir,
descriptor["ID"].replace(".", "-") + ".json"),
"r"))
for descriptor in all_descriptors
]
# store data in cache
with open(os.path.join(self.cache_dir,
"all_descriptors.json"),
"w") as f:
json.dump(all_descriptors, f, indent=4)
with open(os.path.join(self.cache_dir,
"detailed_all_descriptors.json"),
"w") as f:
json.dump(detailed_all_descriptors, f, indent=4)
except Exception as e:
logging.exception("An exception occurred in the thread.")
def loadJson(userInput, verbose=False):
# Check for JSON file (local or from Zenodo)
json_file = None
if os.path.isfile(userInput):
json_file = userInput
elif userInput.split(".")[0].lower() == "zenodo":
from boutiques.puller import Puller
puller = Puller([userInput], verbose)
json_file = puller.pull()[0]
if json_file is not None:
with open(json_file, 'r') as f:
return json.loads(f.read())
# JSON file not found, so try to parse JSON object
e = ("Cannot parse input {}: file not found, "
"invalid Zenodo ID, or invalid JSON object").format(userInput)
if userInput.isdigit():
raise_error(LoadError, e)
try:
return json.loads(userInput)
except ValueError:
raise_error(LoadError, e)
def pull(*params):
parser = ArgumentParser("Download a descriptor from Zenodo.")
parser.add_argument("zid",
action="store",
help="Zenodo ID "
"of the descriptor to pull, prefixed by "
"'zenodo.', e.g. zenodo.123456")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Print information messages")
parser.add_argument("--sandbox",
action="store_true",
help="pull from Zenodo's sandbox instead of "
"production server. Recommended for tests.")
result = parser.parse_args(params)
from boutiques.puller import Puller
puller = Puller(result.zid, result.verbose, result.sandbox)
return puller.pull()
def deprecate(zenodo_id,
by_zenodo_id=None,
sandbox=False,
verbose=False,
zenodo_token=None,
download_function=urlretrieve):
# Get the descriptor and Zenodo id
puller = Puller([zenodo_id], verbose=verbose, sandbox=sandbox)
descriptor_fname = puller.pull()[0]
descriptor_json = loadJson(descriptor_fname,
sandbox=sandbox,
verbose=verbose)
# Return if tool is already deprecated
deprecated = descriptor_json.get('deprecated-by-doi')
if deprecated is not None:
if isinstance(deprecated, str):
print_info('Tool {0} is already deprecated by {1} '.format(
zenodo_id, deprecated))
if by_zenodo_id is not None:
prompt = ("Tool {0} will be deprecated by {1}, "
"this cannot be undone. Are you sure? (Y/n) ")\
.format(zenodo_id, by_zenodo_id)
ret = input(prompt)
if ret.upper() != "Y":
return
else:
print_warning('Tool {0} is already deprecated'.format(zenodo_id))
return
# Set record id and Zenodo id
zhelper = ZenodoHelper(sandbox=sandbox, no_int=True, verbose=verbose)
zid = zhelper.get_zid_from_filename(descriptor_fname)
record_id = zhelper.get_record_id_from_zid(zid)
# Return if tool has a newer version
record = zhelper.zenodo_get_record(record_id)
if not record['metadata']['relations']['version'][0]['is_last']:
new_version = (record['metadata']['relations']['version'][0]
['last_child']['pid_value'])
raise_error(
DeprecateError, 'Tool {0} has a newer version '
'(zenodo.{1}), it cannot be deprecated.'.format(
zenodo_id, new_version))
return
# Add deprecated property
if by_zenodo_id is None:
descriptor_json['deprecated-by-doi'] = True
else:
# Check that by_zenodo_id exists
by_record_id = zhelper.get_record_id_from_zid(by_zenodo_id)
if zhelper.record_exists(by_record_id) is False:
raise_error(DeprecateError,
"Tool does not exist: {0}".format(by_zenodo_id))
# Assign deprecated-by property
by_doi_id = zhelper.get_doi_from_zid(by_zenodo_id)
descriptor_json['deprecated-by-doi'] = by_doi_id
# Add doi to descriptor (mandatory for update)
if descriptor_json.get('doi') is None:
descriptor_json['doi'] = zhelper.get_doi_from_zid(zid)
# Save descriptor in temp file
tmp = tempfile.NamedTemporaryFile(delete=False, mode='w', suffix=".json")
content = json.dumps(descriptor_json, indent=4, sort_keys=True)
tmp.write(content)
tmp.close()
# Publish updated descriptor
publisher = Publisher(tmp.name,
zenodo_token,
replace=True,
sandbox=sandbox,
no_int=True,
id="zenodo." + zid,
verbose=verbose)
return publisher.publish()