def get_nexus_endpoint(self):
# change once nexus is setup
endpoint = "https://sandbox.bluebrainnexus.io/v1" if self.sandbox \
else "https://sandbox.bluebrainnexus.io/v1"
if self.verbose:
print_info("Using Nexus endpoint {0}".format(endpoint))
return endpoint
def zenodo_deposit_updated_version(self, metadata, access_token,
deposition_id):
r = requests.post(
self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/newversion' % deposition_id,
params={'access_token': access_token})
if r.status_code == 403:
raise_error(
ZenodoError, "You do not have permission to access "
"this resource. Note that you cannot "
"publish an update to a tool belonging "
"to someone else.", r)
elif (r.status_code != 201):
raise_error(
ZenodoError, "Deposition of new version failed. Check "
"that the Zenodo ID is correct (if one "
"was provided).", r)
if (self.verbose):
print_info("Deposition of new version succeeded", r)
new_url = r.json()['links']['latest_draft']
new_zid = new_url.split("/")[-1]
self.zenodo_update_metadata(new_zid,
r.json()['doi'], metadata, access_token)
self.zenodo_delete_files(new_zid, r.json()["files"], access_token)
return new_zid
def nexus_test_api(self, access_token, org, project):
# Test connection to endpoint without token
self.nexus.config.set_environment(self.nexus_endpoint)
try:
self.nexus.permissions.fetch()
except ConnectionError as e:
raise_error(NexusError, "Cannot access Nexus endpoint", e.response)
if self.verbose:
print_info("Nexus endpoint is accessible")
# Test authentication with token, org and project
self.nexus.config.set_token(access_token)
try:
self.nexus.projects.fetch(org, project)
if self.verbose:
print_info("Authentication to Nexus successful")
except HTTPError as e:
if 404 == e.response.status_code:
raise_error(
NexusError, "No project '{}' in organization '{}' "
"in Nexus repository".format(project, org), e.response)
elif 401 == e.response.status_code:
raise_error(
NexusError, "Cannot authenticate to Nexus API, check "
"your access token", e.response)
else:
raise_error(NexusError, "Something went wrong", e.response)
def __init__(self, zids, verbose=False, sandbox=False):
# remove zenodo prefix
self.zenodo_entries = []
self.cache_dir = os.path.join(os.path.expanduser('~'), ".cache",
"boutiques")
discarded_zids = zids
# This removes duplicates, should maintain order
zids = list(dict.fromkeys(zids))
for zid in zids:
discarded_zids.remove(zid)
try:
# Zenodo returns the full DOI, but for the purposes of
# Boutiques we just use the Zenodo-specific portion (as its the
# unique part). If the API updates on Zenodo to no longer
# provide the full DOI, this still works because it just grabs
# the last thing after the split.
zid = zid.split('/')[-1]
newzid = zid.split(".", 1)[1]
newfname = os.path.join(self.cache_dir,
"zenodo-{0}.json".format(newzid))
self.zenodo_entries.append({"zid": newzid, "fname": newfname})
except IndexError:
raise_error(ZenodoError, "Zenodo ID must be prefixed by "
"'zenodo', e.g. zenodo.123456")
self.verbose = verbose
self.sandbox = sandbox
if(self.verbose):
for zid in discarded_zids:
print_info("Discarded duplicate id {0}".format(zid))
def search(self):
results = self.zenodo_search()
print_info("Showing %d of %d results." % (len(
results.json()["hits"]["hits"]), results.json()["hits"]["total"]))
if self.verbose:
return self.create_results_list_verbose(results.json())
return self.create_results_list(results.json())
def pull(self):
# return cached file if it exists
if os.path.isfile(self.cached_fname):
if (self.verbose):
print_info("Found cached file at %s" % self.cached_fname)
return self.cached_fname
from boutiques.searcher import Searcher
searcher = Searcher(self.zid,
self.verbose,
self.sandbox,
exact_match=True)
r = searcher.zenodo_search()
for hit in r.json()["hits"]["hits"]:
file_path = hit["files"][0]["links"]["self"]
file_name = file_path.split(os.sep)[-1]
if hit["id"] == int(self.zid):
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
if (self.verbose):
print_info("Downloading descriptor %s" % file_name)
downloaded = urlretrieve(file_path, self.cached_fname)
print("Downloaded descriptor to " + downloaded[0])
return downloaded[0]
raise_error(ZenodoError, "Descriptor not found")
def delete(self, file=None, no_int=False):
self.filename = extractFileName(file)
self.no_int = no_int
# Verify deletion
if not self.no_int:
prompt = self._get_delete_prompt()
try:
ret = raw_input(prompt) # Python 2
except NameError:
ret = input(prompt) # Python 3
if ret.upper() != "Y":
return
# Remove the file specified by the file option
if file is not None:
# Check file exists in cache
self._file_exists_in_cache(file)
# Remove file from cache
file_path = os.path.join(self.cache_dir, file)
os.remove(file_path)
print_info(
"File {} has been removed from the data cache".format(file))
# Remove all files in the data cache
else:
[
os.remove(os.path.join(self.cache_dir, f))
for f in self.cache_files
]
print_info("All files have been removed from the data cache")
def __init__(self, query, verbose=False, sandbox=False, max_results=None,
no_trunc=False, exact_match=False):
if query is not None:
self.query = query
else:
self.query = 'boutiques'
if not exact_match:
self.query = '*' + self.query + '*'
self.verbose = verbose
self.sandbox = sandbox
self.no_trunc = no_trunc
self.max_results = max_results
# Display top 10 results by default
if max_results is None:
self.max_results = 10
# Zenodo will error if asked for more than 9999 results
if self.max_results > 9999:
self.max_results = 9999
# Set Zenodo endpoint
self.zenodo_endpoint = "https://sandbox.zenodo.org" if\
self.sandbox else "https://zenodo.org"
if(self.verbose):
print_info("Using Zenodo endpoint {0}".
format(self.zenodo_endpoint))
def save_zenodo_access_token(self, access_token):
json_creds = self.read_credentials()
json_creds[self.config_token_property_name()] = access_token
with open(self.config_file, 'w') as f:
f.write(json.dumps(json_creds, indent=4, sort_keys=True))
if (self.verbose):
print_info("Zenodo access token saved in {0}".format(
self.config_file))
def validate_bids(descriptor, valid=False):
if not valid:
msg = "Please provide a Boutiques descriptor that has been validated."
raise_error(DescriptorValidationError, msg)
errors = []
# TODO: verify not only that all fields/keys exist, their properties, too
# Ensure the command-line conforms to the BIDS app spec
msg_template = " CLIError: command-line doesn't match template: {}"
cltemp = r"mkdir -p \[OUTPUT_DIR\]; (.*) \[BIDS_DIR\] \[OUTPUT_DIR\]"\
r" \[ANALYSIS_LEVEL\] \[PARTICIPANT_LABEL\] \[SESSION_LABEL\]"\
r"[\\s]*(.*)"
cmdline = descriptor["command-line"]
if len(re.findall(cltemp, cmdline)) < 1:
errors += [msg_template.format(cltemp)]
# Verify IDs are present which link to the OUTPUT_DIR
# key bot as File and String
ftypes = set(["File", "String"])
msg_template = " OutError: \"{}\" types for outdir do not match \"{}\""
outtypes = set([
inp["type"] for inp in descriptor["inputs"]
if inp["value-key"] == "[OUTPUT_DIR]"
])
if outtypes != ftypes:
errors += [msg_template.format(", ".join(outtypes), ", ".join(ftypes))]
# Verify that analysis levels is an enumerable with some
# subset of "paricipant", "session", and "group"
choices = ["session", "participant", "group"]
msg_template = " LevelError: \"{}\" is not a valid analysis level"
alevels = [
inp["value-choices"] for inp in descriptor["inputs"]
if inp["value-key"] == "[ANALYSIS_LEVEL]"
][0]
errors += [msg_template.format(lv) for lv in alevels if lv not in choices]
# Verify there is only a single output defined (the directory)
msg_template = "OutputError: 0 or multiple outputs defined"
if len(descriptor["output-files"]) != 1:
errors += [msg_template]
else:
# Verify that the output shows up as an output
msg_template = "OutputError: OUTPUT_DIR is not represented as an output"
if descriptor["output-files"][0]["path-template"] != "[OUTPUT_DIR]":
errors += [msg_template]
errors = None if errors == [] else errors
if errors is None:
print_info("BIDS validation OK")
else:
raise_error(DescriptorValidationError, "Invalid BIDS app descriptor:"
"\n" + "\n".join(errors))
def zenodo_search(self):
r = requests.get(self.zenodo_endpoint + '/api/records/?q=%s&'
'keywords=boutiques&keywords=schema&'
'keywords=version&file_type=json&type=software'
'&page=1&size=%s' % (self.query, self.max_results))
if (r.status_code != 200):
raise_error(ZenodoError, "Error searching Zenodo", r)
if (self.verbose):
print_info("Search successful.", r)
return r
def search(self):
results = self.zenodo_search()
num_results = len(results.json()["hits"]["hits"])
total_results = results.json()["hits"]["total"]
print_info("Showing %d of %d results."
% (num_results if num_results < self.max_results
else self.max_results, total_results))
if self.verbose:
return self.create_results_list_verbose(results.json())
return self.create_results_list(results.json())
def save_nexus_inputs(self, access_token, org, project):
json_creds = self.read_credentials()
json_creds[self.config_token_property_name()] = access_token
json_creds["nexus-organization"] = org
json_creds["nexus-project"] = project
with open(self.config_file, 'w') as f:
f.write(json.dumps(json_creds, indent=4, sort_keys=True))
if self.verbose:
print_info("Nexus access token, organization and project"
" saved in {0}".format(self.config_file))
def zenodo_delete_files(self, new_deposition_id, files, access_token):
for file in files:
file_id = file["id"]
r = requests.delete(self.zenodo_endpoint +
'/api/deposit/depositions/%s/files/%s' %
(new_deposition_id, file_id),
params={'access_token': access_token})
if (r.status_code != 204):
raise_error(ZenodoError, "Could not delete old file", r)
if (self.verbose):
print_info("Deleted old file", r)
def zenodo_publish(self, deposition_id):
r = requests.post(self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/publish'
% deposition_id,
params={'access_token': self.zenodo_access_token})
if(r.status_code != 202):
raise_error(ZenodoError, "Cannot publish descriptor", r)
if(self.verbose):
print_info("Descriptor published to Zenodo, doi is {0}".
format(r.json()['doi']), r)
return r.json()['doi']
def zenodo_update_metadata(self, deposition_id):
data = self.create_metadata()
headers = {"Content-Type": "application/json"}
r = requests.put(self.zenodo_endpoint +
'/api/deposit/depositions/%s' % deposition_id,
params={'access_token': self.zenodo_access_token},
data=json.dumps(data),
headers=headers)
if (r.status_code != 200):
raise_error(ZenodoError, "Cannot update metadata of new version",
r)
if (self.verbose):
print_info("Updated metadata of new version", r)
def zenodo_test_api(self, access_token):
r = requests.get(self.zenodo_endpoint + '/api/deposit/depositions')
if (r.status_code != 401):
raise_error(ZenodoError, "Cannot access Zenodo", r)
if (self.verbose):
print_info("Zenodo is accessible", r)
r = requests.get(self.zenodo_endpoint + '/api/deposit/depositions',
params={'access_token': access_token})
message = "Cannot authenticate to Zenodo API, check your access token"
if (r.status_code != 200):
raise_error(ZenodoError, message, r)
if (self.verbose):
print_info("Authentication to Zenodo successful", r)
def zenodo_search(self):
# Get all results
r = requests.get(self.zenodo_endpoint + '/api/records/?q='
'keywords:(/Boutiques/) AND '
'keywords:(/schema-version.*/)'
'%s'
'&file_type=json&type=software&'
'page=1&size=%s' % (self.query_line, 9999))
if(r.status_code != 200):
raise_error(ZenodoError, "Error searching Zenodo", r)
if(self.verbose):
print_info("Search successful for query \"%s\"" % self.query, r)
return r
def zenodo_publish(self, access_token, deposition_id, msg_obj):
r = requests.post(
self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/publish' % deposition_id,
params={'access_token': access_token})
if (r.status_code != 202):
raise_error(ZenodoError, "Cannot publish {}".format(msg_obj), r)
if (self.verbose):
print_info(
"{0} published to Zenodo, doi is {1}".format(
msg_obj,
r.json()['doi']), r)
return r.json()['doi']
def zenodo_deposit_updated_version(self, deposition_id):
r = requests.post(
self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/newversion' % deposition_id,
params={'access_token': self.zenodo_access_token})
if (r.status_code != 201):
raise_error(ZenodoError, "Deposition of new version failed", r)
if (self.verbose):
print_info("Deposition of new version succeeded", r)
new_url = r.json()['links']['latest_draft']
new_zid = new_url.split("/")[-1]
self.zenodo_update_metadata(new_zid)
self.zenodo_delete_files(new_zid, r.json()["files"])
return new_zid
def validateSchema(s, d=None, **kwargs):
# Check schema wrt meta-schema
try:
jsonschema.Draft4Validator.check_schema(s)
except jsonschema.SchemaError as se:
errExit("Invocation schema is invalid.\n" + str(se.message), False)
# Check data instance against schema
if d:
try:
jsonschema.validate(d, s)
except ValidationError as e:
raise_error(InvocationValidationError, e)
if kwargs.get("verbose"):
print_info("Invocation Schema validation OK")
def _zenodo_upload_dataset(self, deposition_id, file):
file_path = os.path.join(self.cache_dir, file)
data = {'filename': file}
files = {'file': open(file_path, 'rb')}
r = requests.post(self.zenodo_endpoint +
'/api/deposit/depositions/%s/files' % deposition_id,
params={'access_token': self.zenodo_access_token},
data=data,
files=files)
if (r.status_code != 201):
raise_error(ZenodoError, "Cannot upload record", r)
if (self.verbose):
print_info("Record uploaded to Zenodo", r)
def zenodo_upload_descriptor(self, deposition_id):
data = {'filename': os.path.basename(self.descriptor_file_name)}
files = {'file': open(self.descriptor_file_name, 'rb')}
r = requests.post(self.zenodo_endpoint +
'/api/deposit/depositions/%s/files' % deposition_id,
params={'access_token': self.zenodo_access_token},
data=data,
files=files)
# Status code is inconsistent with Zenodo documentation
if (r.status_code != 201):
raise_error(ZenodoError, "Cannot upload descriptor", r)
if (self.verbose):
print_info("Descriptor uploaded to Zenodo", r)
def __init__(self,
query,
verbose=False,
sandbox=False,
max_results=None,
no_trunc=False,
exact_match=False):
if query is not None:
self.query = query
if not exact_match:
terms = self.query.split(" ")
self.query_line = ''
for t in terms:
uncased_term = ''
for ch in t:
uncased_term = uncased_term + "[" + ch.upper() +\
ch.lower() + "]"
uncased_term = quote(uncased_term)
self.query_line = self.query_line + \
' AND (/.*%s.*/)' % uncased_term
else:
self.query_line =\
' AND (/%s/)' % self.query
else:
self.query_line = ''
self.query = ''
if (verbose):
print_info("Using Query Line: " + self.query_line)
self.verbose = verbose
self.sandbox = sandbox
self.no_trunc = no_trunc
self.max_results = max_results
# Display top 10 results by default
if max_results is None:
self.max_results = 10
# Zenodo will error if asked for more than 9999 results
if self.max_results > 9999:
self.max_results = 9999
# Set Zenodo endpoint
self.zenodo_endpoint = "https://sandbox.zenodo.org" if\
self.sandbox else "https://zenodo.org"
if (self.verbose):
print_info("Using Zenodo endpoint {0}".format(
self.zenodo_endpoint))
def zenodo_deposit(self, metadata, access_token):
headers = {"Content-Type": "application/json"}
data = metadata
r = requests.post(self.zenodo_endpoint + '/api/deposit/depositions',
params={'access_token': access_token},
json={},
data=json.dumps(data),
headers=headers)
if (r.status_code != 201):
raise_error(ZenodoError, "Deposition failed", r)
zid = r.json()['id']
if (self.verbose):
print_info("Deposition succeeded, id is {0}".format(zid), r)
return zid
def __init__(self, descriptor_file_name, verbose, sandbox, no_int,
auth_token):
# Straightforward assignments
self.verbose = verbose
self.sandbox = sandbox
self.descriptor_file_name = descriptor_file_name
self.no_int = no_int
self.zenodo_access_token = auth_token
# Validate and load descriptor
validate_descriptor(descriptor_file_name)
self.descriptor = json.loads(open(self.descriptor_file_name).read())
# Get relevant descriptor propertis
self.url = self.descriptor.get('url')
self.tool_doi = self.descriptor.get('tool-doi')
self.descriptor_url = self.descriptor.get('descriptor-url')
# Get tool author and check that it's defined
if self.descriptor.get("author") is None:
raise_error(
ZenodoError, "Tool must have an author to be "
"published. Add an 'author' property to your "
"descriptor.")
self.creator = self.descriptor['author']
# Get descriptor doi and check that it's not defined
if self.descriptor.get('doi') is not None:
raise_error(
ZenodoError, "Descriptor already has a DOI. Please "
"remove it from the descriptor before publishing it "
"again. A new DOI will be generated.")
self.config_file = os.path.join(os.path.expanduser('~'), ".boutiques")
# Fix Zenodo access token
if self.zenodo_access_token is None:
self.zenodo_access_token = self.get_zenodo_access_token()
self.save_zenodo_access_token()
# Set Zenodo endpoint
self.zenodo_endpoint = "https://sandbox.zenodo.org" if\
self.sandbox else "https://zenodo.org"
if (self.verbose):
print_info("Using Zenodo endpoint {0}".format(
self.zenodo_endpoint))
def zenodo_update_metadata(self, new_deposition_id, old_doi):
data = self.create_metadata()
# Add the new DOI to the metadata
old_doi_split = old_doi.split(".")
old_doi_split[-1] = new_deposition_id
new_doi = '.'.join(old_doi_split)
data['metadata']['doi'] = new_doi
headers = {"Content-Type": "application/json"}
r = requests.put(self.zenodo_endpoint+'/api/deposit/depositions/%s'
% new_deposition_id,
params={'access_token': self.zenodo_access_token},
data=json.dumps(data),
headers=headers)
if(r.status_code != 200):
raise_error(ZenodoError, "Cannot update metadata of new version", r)
if(self.verbose):
print_info("Updated metadata of new version", r)
def zenodo_upload_descriptor(self, deposition_id):
# If in replace mode, remove the old DOI
if self.descriptor.get('doi'):
del self.descriptor['doi']
with open(self.descriptor_file_name, 'w') as fhandle:
fhandle.write(json.dumps(self.descriptor, indent=4))
data = {'filename': os.path.basename(self.descriptor_file_name)}
files = {'file': open(self.descriptor_file_name, 'rb')}
r = requests.post(self.zenodo_endpoint +
'/api/deposit/depositions/%s/files' % deposition_id,
params={'access_token': self.zenodo_access_token},
data=data,
files=files)
# Status code is inconsistent with Zenodo documentation
if (r.status_code != 201):
raise_error(ZenodoError, "Cannot upload descriptor", r)
if (self.verbose):
print_info("Descriptor uploaded to Zenodo", r)
def __init__(self, query, verbose, sandbox, max_results):
if query is not None:
self.query = query
else:
self.query = 'boutiques'
self.verbose = verbose
self.sandbox = sandbox
# Return max 10 results by default
if max_results is not None:
self.max_results = max_results
else:
self.max_results = 10
# Set Zenodo endpoint
self.zenodo_endpoint = "https://sandbox.zenodo.org" if\
self.sandbox else "https://zenodo.org"
if (self.verbose):
print_info("Using Zenodo endpoint {0}".format(
self.zenodo_endpoint))
def search(self):
results = self.zenodo_search()
total_results = results.json()["hits"]["total"]
total_deprecated = len([h['metadata']['keywords'] for h in
results.json()['hits']['hits'] if
'metadata' in h and
'keywords' in h['metadata'] and
'deprecated' in h['metadata']['keywords']])
results_list = self.create_results_list_verbose(results.json()) if\
self.verbose else\
self.create_results_list(results.json())
num_results = len(results_list)
print_info("Showing %d of %d result(s)%s"
% (num_results if num_results < self.max_results
else self.max_results,
total_results if self.verbose
else total_results - total_deprecated,
"." if self.verbose
else ", exluding %d deprecated result(s)."
% total_deprecated))
return results_list
