def pull(self):
# return cached file if it exists
if os.path.isfile(self.cached_fname):
if (self.verbose):
print_info("Found cached file at %s" % self.cached_fname)
return self.cached_fname
from boutiques.searcher import Searcher
searcher = Searcher(self.zid,
self.verbose,
self.sandbox,
exact_match=True)
r = searcher.zenodo_search()
for hit in r.json()["hits"]["hits"]:
file_path = hit["files"][0]["links"]["self"]
file_name = file_path.split(os.sep)[-1]
if hit["id"] == int(self.zid):
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
if (self.verbose):
print_info("Downloading descriptor %s" % file_name)
downloaded = urlretrieve(file_path, self.cached_fname)
print("Downloaded descriptor to " + downloaded[0])
return downloaded[0]
raise_error(ZenodoError, "Descriptor not found")
def search(*params):
parser = ArgumentParser("Search Zenodo for Boutiques descriptors. "
"When no term is supplied, will search for "
"all descriptors.")
parser.add_argument("query", nargs="?", default="boutiques",
action="store", help="Search query")
parser.add_argument("-v", "--verbose", action="store_true",
help="Print information messages")
parser.add_argument("--sandbox", action="store_true",
help="search Zenodo's sandbox instead of "
"production server. Recommended for tests.")
parser.add_argument("-m", "--max", action="store", type=int,
help="Specify the maximum number of results "
"to be returned. Default is 10.")
parser.add_argument("-nt", "--no-trunc", action="store_true",
help="Do not truncate long tool descriptions.")
parser.add_argument("-e", "--exact", action="store_true",
help="Only return results containing the exact query.")
result = parser.parse_args(params)
from boutiques.searcher import Searcher
searcher = Searcher(result.query, result.verbose, result.sandbox,
result.max, result.no_trunc, result.exact)
return searcher.search()
def run(self):
try:
boutique_cache_dir = os.path.join(os.path.expanduser('~'),
".cache", "boutiques",
"production")
# first search for all descriptors
searcher = Searcher(query=None, max_results=100, no_trunc=True)
all_descriptors = searcher.search()
# then pull every single descriptor
all_descriptor_ids = list(map(lambda x: x["ID"], all_descriptors))
files = Puller(all_descriptor_ids).pull()
# fetch every single descriptor into one file
detailed_all_descriptors = list(
map(lambda f: json.load(open(f, 'r')), files))
# store data in cache
with open(os.path.join(boutique_cache_dir, "all_descriptors.json"),
"w") as f:
json.dump(all_descriptors, f, indent=4)
with open(
os.path.join(boutique_cache_dir,
"detailed_all_descriptors.json"), "w") as f:
json.dump(detailed_all_descriptors, f, indent=4)
except Exception as e:
logging.exception(
"An exception occurred in the thread:{0}.".format(e))
def publish(self):
if (not self.no_int):
prompt = ("The descriptor will be published to Zenodo, "
"this cannot be undone. Are you sure? (Y/n) ")
try:
ret = raw_input(prompt) # Python 2
except NameError:
ret = input(prompt) # Python 3
if ret.upper() != "Y":
return
if self.id_to_update is not None:
publish_update = True
else:
# perform a search to check if descriptor is an updated version
# of an existing one
from boutiques.searcher import Searcher
searcher = Searcher(self.descriptor.get("name"),
self.verbose,
self.sandbox,
exact_match=True)
r = searcher.zenodo_search()
publish_update = False
for hit in r.json()["hits"]["hits"]:
title = hit["metadata"]["title"]
if title == self.descriptor.get("name"):
self.id_to_update = hit["id"]
break
if self.id_to_update is not None:
if (not self.no_int):
prompt = ("Found an existing record with the same name, "
"would you like to update it? "
"(Y:Update existing / n:Publish new entry with "
"name {}) ".format(self.descriptor.get("name")))
try:
ret = raw_input(prompt) # Python 2
except NameError:
ret = input(prompt) # Python 3
if ret.upper() == "Y":
publish_update = True
else:
publish_update = True
if publish_update:
deposition_id = self.zenodo_helper.zenodo_deposit_updated_version(
self.create_metadata(), self.zenodo_access_token,
self.id_to_update)
else:
deposition_id = self.zenodo_helper.zenodo_deposit(
self.create_metadata(), self.zenodo_access_token)
self.zenodo_upload_descriptor(deposition_id)
self.doi = self.zenodo_helper.zenodo_publish(self.zenodo_access_token,
deposition_id,
"Descriptor")
self.descriptor['doi'] = self.doi
with open(self.descriptor_file_name, "w") as f:
f.write(json.dumps(self.descriptor, indent=4, sort_keys=True))
def run(self):
try:
# if cache directory doesn't exist then create it
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
# first search for all descriptors
searcher = Searcher(query="", max_results=100, no_trunc=True)
all_descriptors = searcher.search()
# then pull every single descriptor
all_descriptor_ids = list(map(lambda x: x["ID"], all_descriptors))
Puller(all_descriptor_ids).pull()
# fetch every single descriptor into one file
detailed_all_descriptors = [
json.load(open(os.path.join(self.cache_dir,
descriptor["ID"].replace(".", "-") + ".json"),
"r"))
for descriptor in all_descriptors
]
# store data in cache
with open(os.path.join(self.cache_dir,
"all_descriptors.json"),
"w") as f:
json.dump(all_descriptors, f, indent=4)
with open(os.path.join(self.cache_dir,
"detailed_all_descriptors.json"),
"w") as f:
json.dump(detailed_all_descriptors, f, indent=4)
except Exception as e:
logging.exception("An exception occurred in the thread.")
def pull(self):
from boutiques.searcher import Searcher
searcher = Searcher(self.zid, self.verbose, self.sandbox, None)
r = searcher.zenodo_search()
for hit in r.json()["hits"]["hits"]:
file_path = hit["files"][0]["links"]["self"]
file_name = file_path.split(os.sep)[-1]
if hit["id"] == int(self.zid):
if self.download:
cache_dir = os.path.join(os.path.expanduser('~'), ".cache",
"boutiques")
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
if (self.verbose):
print_info("Downloading descriptor %s" % file_name)
downloaded = urlretrieve(
file_path, os.path.join(cache_dir, file_name))
print_info("Downloaded descriptor to " + cache_dir)
return downloaded
if (self.verbose):
print_info("Opening descriptor %s" % file_name)
return urlopen(file_path)
raise_error(ZenodoError, "Descriptor not found")
def search(*params):
parser = parser_search()
results = parser.parse_args(params)
from boutiques.searcher import Searcher
searcher = Searcher(results.query, results.verbose, results.sandbox,
results.max, results.no_trunc, results.exact)
return searcher.search()
def publish(self):
if (not self.no_int):
prompt = ("The descriptor will be published to Zenodo, "
"this cannot be undone. Are you sure? (Y/n) ")
try:
ret = raw_input(prompt) # Python 2
except NameError:
ret = input(prompt) # Python 3
if ret.upper() != "Y":
return
self.zenodo_test_api()
# perform a search to check if descriptor is an updated version
# of an existing one
from boutiques.searcher import Searcher
searcher = Searcher(self.descriptor.get("name"), self.verbose,
self.sandbox, None)
r = searcher.zenodo_search()
id_to_update = 0
publish_update = False
for hit in r.json()["hits"]["hits"]:
title = hit["metadata"]["title"]
if title == self.descriptor.get("name"):
id_to_update = hit["id"]
break
if id_to_update:
if (not self.no_int):
prompt = ("Found an existing record with the same name, "
"would you like to update it? "
"(Y:Update existing / n:Publish new entry with "
"name {}) ".format(self.descriptor.get("name")))
try:
ret = raw_input(prompt) # Python 2
except NameError:
ret = input(prompt) # Python 3
if ret.upper() == "Y":
publish_update = True
else:
publish_update = True
if publish_update:
self.publish_updated_version(id_to_update)
else:
self.publish_new_entry()
def _download_spec(self, zenodo_id):
"""
usind boutiques Searcher to find url of zenodo file for a specific id,
and download the file to self.cache_dir
"""
from boutiques.searcher import Searcher
searcher = Searcher(zenodo_id, exact_match=True)
hits = searcher.zenodo_search().json()["hits"]["hits"]
if len(hits) == 0:
raise Exception(f"can't find zenodo spec for {zenodo_id}")
elif len(hits) > 1:
raise Exception(f"too many hits for {zenodo_id}")
else:
zenodo_url = hits[0]["files"][0]["links"]["self"]
zenodo_file = self.cache_dir / f"zenodo.{zenodo_id}.json"
urlretrieve(zenodo_url, zenodo_file)
return zenodo_file
def pull(self):
# return cached file if it exists
json_files = []
for entry in self.zenodo_entries:
if os.path.isfile(entry["fname"]):
if (self.verbose):
print_info("Found cached file at %s" % entry["fname"])
json_files.append(entry["fname"])
continue
searcher = Searcher(entry["zid"],
self.verbose,
self.sandbox,
exact_match=True)
r = searcher.zenodo_search()
if not len(r.json()["hits"]["hits"]):
raise_error(
ZenodoError, "Descriptor \"{0}\" "
"not found".format(entry["zid"]))
for hit in r.json()["hits"]["hits"]:
file_path = hit["files"][0]["links"]["self"]
file_name = file_path.split(os.sep)[-1]
if hit["id"] == int(entry["zid"]):
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
if (self.verbose):
print_info("Downloading descriptor %s" % file_name)
downloaded = urlretrieve(file_path, entry["fname"])
if (self.verbose):
print_info("Downloaded descriptor to " + downloaded[0])
json_files.append(downloaded[0])
else:
raise_error(
ZenodoError, "Searched-for descriptor \"{0}\" "
"does not match descriptor \"{1}\" returned "
"from Zenodo".format(entry["zid"], hit["id"]))
return json_files