def pull(self):
from boutiques.searcher import Searcher
searcher = Searcher(self.zid, self.verbose, self.sandbox, None)
r = searcher.zenodo_search()
for hit in r.json()["hits"]["hits"]:
file_path = hit["files"][0]["links"]["self"]
file_name = file_path.split(os.sep)[-1]
if hit["id"] == int(self.zid):
if self.download:
cache_dir = os.path.join(os.path.expanduser('~'), ".cache",
"boutiques")
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
if (self.verbose):
print_info("Downloading descriptor %s" % file_name)
downloaded = urlretrieve(
file_path, os.path.join(cache_dir, file_name))
print_info("Downloaded descriptor to " + cache_dir)
return downloaded
if (self.verbose):
print_info("Opening descriptor %s" % file_name)
return urlopen(file_path)
raise_error(ZenodoError, "Descriptor not found")
def pull(self):
# return cached file if it exists
if os.path.isfile(self.cached_fname):
if (self.verbose):
print_info("Found cached file at %s" % self.cached_fname)
return self.cached_fname
from boutiques.searcher import Searcher
searcher = Searcher(self.zid,
self.verbose,
self.sandbox,
exact_match=True)
r = searcher.zenodo_search()
for hit in r.json()["hits"]["hits"]:
file_path = hit["files"][0]["links"]["self"]
file_name = file_path.split(os.sep)[-1]
if hit["id"] == int(self.zid):
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
if (self.verbose):
print_info("Downloading descriptor %s" % file_name)
downloaded = urlretrieve(file_path, self.cached_fname)
print("Downloaded descriptor to " + downloaded[0])
return downloaded[0]
raise_error(ZenodoError, "Descriptor not found")
def zenodo_deposit_updated_version(self, metadata, access_token,
deposition_id):
r = requests.post(
self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/newversion' % deposition_id,
params={'access_token': access_token})
if r.status_code == 403:
raise_error(
ZenodoError, "You do not have permission to access "
"this resource. Note that you cannot "
"publish an update to a tool belonging "
"to someone else.", r)
elif (r.status_code != 201):
raise_error(
ZenodoError, "Deposition of new version failed. Check "
"that the Zenodo ID is correct (if one "
"was provided).", r)
if (self.verbose):
print_info("Deposition of new version succeeded", r)
new_url = r.json()['links']['latest_draft']
new_zid = new_url.split("/")[-1]
self.zenodo_update_metadata(new_zid,
r.json()['doi'], metadata, access_token)
self.zenodo_delete_files(new_zid, r.json()["files"], access_token)
return new_zid
def zenodo_get_record(self, zenodo_id):
r = requests.get(self.zenodo_endpoint +
'/api/records/{}'.format(zenodo_id))
if r.status_code != 200:
raise_error(ZenodoError,
"Descriptor \"{}\" not found".format(zenodo_id), r)
return r.json()
def __init__(self, zids, verbose=False, sandbox=False):
# remove zenodo prefix
self.zenodo_entries = []
self.cache_dir = os.path.join(os.path.expanduser('~'), ".cache",
"boutiques")
discarded_zids = zids
# This removes duplicates, should maintain order
zids = list(dict.fromkeys(zids))
for zid in zids:
discarded_zids.remove(zid)
try:
# Zenodo returns the full DOI, but for the purposes of
# Boutiques we just use the Zenodo-specific portion (as its the
# unique part). If the API updates on Zenodo to no longer
# provide the full DOI, this still works because it just grabs
# the last thing after the split.
zid = zid.split('/')[-1]
newzid = zid.split(".", 1)[1]
newfname = os.path.join(self.cache_dir,
"zenodo-{0}.json".format(newzid))
self.zenodo_entries.append({"zid": newzid, "fname": newfname})
except IndexError:
raise_error(ZenodoError, "Zenodo ID must be prefixed by "
"'zenodo', e.g. zenodo.123456")
self.verbose = verbose
self.sandbox = sandbox
if(self.verbose):
for zid in discarded_zids:
print_info("Discarded duplicate id {0}".format(zid))
def __init__(self, docopt_str, base_descriptor):
with open(base_descriptor, "r") as base_desc:
self.descriptor = collections.OrderedDict(json.load(base_desc))
del self.descriptor['groups']
del self.descriptor['inputs']
del self.descriptor['output-files']
self.docopt_str = docopt_str
self.dependencies = collections.OrderedDict()
self.all_desc_and_type = collections.OrderedDict()
self.unique_ids = []
try:
# docopt code snippet to extract args tree (pattern)
# should run if docopt script is valid
options = parse_defaults(docopt_str)
self.pattern = parse_pattern(
formal_usage(self._parse_section('usage:', docopt_str)[0]),
options)
argv = parse_argv(TokenStream(sys.argv[1:], DocoptLanguageError),
list(options), False)
pattern_options = set(self.pattern.flat(Option))
for options_shortcut in self.pattern.flat(AnyOptions):
doc_options = parse_defaults(docopt_str)
options_shortcut.children = list(
set(doc_options) - pattern_options)
matched, left, collected = self.pattern.fix().match(argv)
except Exception:
os.remove(base_descriptor)
raise_error(ImportError, "Invalid docopt script")
def _loadInputsFromUsage(self, usage):
ancestors = []
for arg in usage.children:
# Traverse usage args and add them to dependencies tree
arg_type = type(arg).__name__
if hasattr(arg, "children"):
fchild_type = type(arg.children[0]).__name__
# Has sub-arguments, maybe recurse into _loadRtrctnsFrmUsg
# but have to deal with children in subtype
if arg_type == "Optional" and fchild_type == "AnyOptions":
for option in arg.children[0].children:
self._addArgumentToDependencies(option,
ancestors=ancestors,
optional=True)
elif arg_type == "OneOrMore":
list_name = "<list_of_{0}>".format(
self._getParamName(arg.children[0].name))
list_arg = Argument(list_name)
list_arg.parse(list_name)
self.all_desc_and_type[list_name] = {
'desc':
"List of {0}".format(
self._getParamName(arg.children[0].name))
}
self._addArgumentToDependencies(list_arg,
ancestors=ancestors,
isList=True)
ancestors.append(list_name)
elif arg_type == "Optional" and fchild_type == "Option":
for option in arg.children:
self._addArgumentToDependencies(option,
ancestors=ancestors,
optional=True)
elif arg_type == "Optional" and fchild_type == "Either":
# Mutex choices group, add group to dependencies tree
# add choices args to group
ancestors = self._addGroupArgumentToDependencies(
arg, ancestors, optional=True)
elif arg_type == "Required" and fchild_type == "Either":
# Mutex choices group, add group to dependencies tree
# add choices args to group
ancestors = self._addGroupArgumentToDependencies(
arg, ancestors)
elif arg_type == "Command":
self._addArgumentToDependencies(arg, ancestors=ancestors)
ancestors.append(arg.name)
elif arg_type == "Argument":
self._addArgumentToDependencies(arg, ancestors=ancestors)
ancestors.append(arg.name)
elif arg_type == "Option":
self._addArgumentToDependencies(arg,
ancestors=ancestors,
optional=True)
ancestors.append(arg.name)
else:
raise_error(
ImportError,
"Non implemented docopt arg.type: {0}".format(arg_type))
def record_exists(self, record_id):
r = requests.get(self.zenodo_endpoint +
'/api/records/{}'.format(record_id))
if r.status_code == 200:
return True
if r.status_code == 404:
return False
raise_error(ZenodoError,
"Cannot test existence of record {}".format(record_id), r)
def get_nexus_access_token(self):
json_creds = self.read_credentials()
if json_creds.get(self.config_token_property_name()):
return json_creds.get(self.config_token_property_name())
if self.no_int:
raise_error(NexusError, "Cannot find Nexus credentials.")
prompt = ("Please enter your Nexus access token (it will be "
"saved in {0} for future use): ".format(self.config_file))
return self.prompt(prompt)
def validate_bids(descriptor, valid=False):
if not valid:
msg = "Please provide a Boutiques descriptor that has been validated."
raise_error(DescriptorValidationError, msg)
errors = []
# TODO: verify not only that all fields/keys exist, their properties, too
# Ensure the command-line conforms to the BIDS app spec
msg_template = " CLIError: command-line doesn't match template: {}"
cltemp = r"mkdir -p \[OUTPUT_DIR\]; (.*) \[BIDS_DIR\] \[OUTPUT_DIR\]"\
r" \[ANALYSIS_LEVEL\] \[PARTICIPANT_LABEL\] \[SESSION_LABEL\]"\
r"[\\s]*(.*)"
cmdline = descriptor["command-line"]
if len(re.findall(cltemp, cmdline)) < 1:
errors += [msg_template.format(cltemp)]
# Verify IDs are present which link to the OUTPUT_DIR
# key bot as File and String
ftypes = set(["File", "String"])
msg_template = " OutError: \"{}\" types for outdir do not match \"{}\""
outtypes = set([
inp["type"] for inp in descriptor["inputs"]
if inp["value-key"] == "[OUTPUT_DIR]"
])
if outtypes != ftypes:
errors += [msg_template.format(", ".join(outtypes), ", ".join(ftypes))]
# Verify that analysis levels is an enumerable with some
# subset of "paricipant", "session", and "group"
choices = ["session", "participant", "group"]
msg_template = " LevelError: \"{}\" is not a valid analysis level"
alevels = [
inp["value-choices"] for inp in descriptor["inputs"]
if inp["value-key"] == "[ANALYSIS_LEVEL]"
][0]
errors += [msg_template.format(lv) for lv in alevels if lv not in choices]
# Verify there is only a single output defined (the directory)
msg_template = "OutputError: 0 or multiple outputs defined"
if len(descriptor["output-files"]) != 1:
errors += [msg_template]
else:
# Verify that the output shows up as an output
msg_template = "OutputError: OUTPUT_DIR is not represented as an output"
if descriptor["output-files"][0]["path-template"] != "[OUTPUT_DIR]":
errors += [msg_template]
errors = None if errors == [] else errors
if errors is None:
print_info("BIDS validation OK")
else:
raise_error(DescriptorValidationError, "Invalid BIDS app descriptor:"
"\n" + "\n".join(errors))
def get_zid_from_filename(self, filename):
# Filename must be in the form /a/b/c/zenodo-1234.json
# where zenodo.1234 is the record id.
basename = os.path.basename(filename)
if not re.match(r'zenodo-[0-9]*\.json', basename):
raise_error(
ZenodoError,
'This does not look like a valid file name: {}'.format(
filename))
return basename.replace('.json', '').replace('-', '.')
def zenodo_search(self):
r = requests.get(self.zenodo_endpoint + '/api/records/?q=%s&'
'keywords=boutiques&keywords=schema&'
'keywords=version&file_type=json&type=software'
'&page=1&size=%s' % (self.query, self.max_results))
if (r.status_code != 200):
raise_error(ZenodoError, "Error searching Zenodo", r)
if (self.verbose):
print_info("Search successful.", r)
return r
def get_nexus_project(self):
json_creds = self.read_credentials()
if json_creds.get("nexus-project"):
return json_creds.get("nexus-project")
if self.no_int:
raise_error(NexusError, "Cannot find Nexus project.")
prompt = ("Please enter the Nexus project you want to publish to"
" (it will be saved in {0} for future use): ".format(
self.config_file))
return self.prompt(prompt)
def zenodo_publish(self, deposition_id):
r = requests.post(self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/publish'
% deposition_id,
params={'access_token': self.zenodo_access_token})
if(r.status_code != 202):
raise_error(ZenodoError, "Cannot publish descriptor", r)
if(self.verbose):
print_info("Descriptor published to Zenodo, doi is {0}".
format(r.json()['doi']), r)
return r.json()['doi']
def zenodo_delete_files(self, new_deposition_id, files, access_token):
for file in files:
file_id = file["id"]
r = requests.delete(self.zenodo_endpoint +
'/api/deposit/depositions/%s/files/%s' %
(new_deposition_id, file_id),
params={'access_token': access_token})
if (r.status_code != 204):
raise_error(ZenodoError, "Could not delete old file", r)
if (self.verbose):
print_info("Deleted old file", r)
def resolve_glob(glob, boutiques_inputs):
if not glob.startswith("$"):
return glob
if not glob.startswith("$(inputs."):
raise_error(ImportError, "Unsupported reference: " + glob)
input_id = glob.replace("$(inputs.", "").replace(")", "")
for i in boutiques_inputs:
if i['id'] == input_id:
return i['value-key']
raise_error(ImportError, "Unresolved reference"
" in glob: " + glob)
def __init__(self, zid, verbose, download, sandbox):
# remove zenodo prefix
try:
self.zid = zid.split(".", 1)[1]
except IndexError:
raise_error(
ZenodoError, "Zenodo ID must be prefixed by "
"'zenodo', e.g. zenodo.123456")
self.verbose = verbose
self.download = download
self.sandbox = sandbox
def get_record_id_from_zid(self, zenodo_id):
'''
zenodo_id is in the form zenodo.1234567
record id is 1234567
'''
if not re.match(r'zenodo\.[0-9]', zenodo_id):
raise_error(
ZenodoError, 'This does not look like a valid Zenodo ID: {}.'
'Zenodo ids must be in the form zenodo.1234567'.format(
zenodo_id))
parts = zenodo_id.split('.')
return parts[1]
def get_zenodo_access_token(self):
json_creds = self.read_credentials()
if json_creds.get(self.config_token_property_name()):
return json_creds.get(self.config_token_property_name())
if (self.no_int):
raise_error(ZenodoError, "Cannot find Zenodo credentials.")
prompt = ("Please enter your Zenodo access token (it will be "
"saved in {0} for future use): ".format(self.config_file))
try:
return raw_input(prompt) # Python 2
except NameError:
return input(prompt) # Python 3
def zenodo_search(self):
# Get all results
r = requests.get(self.zenodo_endpoint + '/api/records/?q='
'keywords:(/Boutiques/) AND '
'keywords:(/schema-version.*/)'
'%s'
'&file_type=json&type=software&'
'page=1&size=%s' % (self.query_line, 9999))
if(r.status_code != 200):
raise_error(ZenodoError, "Error searching Zenodo", r)
if(self.verbose):
print_info("Search successful for query \"%s\"" % self.query, r)
return r
def zenodo_update_metadata(self, deposition_id):
data = self.create_metadata()
headers = {"Content-Type": "application/json"}
r = requests.put(self.zenodo_endpoint +
'/api/deposit/depositions/%s' % deposition_id,
params={'access_token': self.zenodo_access_token},
data=json.dumps(data),
headers=headers)
if (r.status_code != 200):
raise_error(ZenodoError, "Cannot update metadata of new version",
r)
if (self.verbose):
print_info("Updated metadata of new version", r)
def zenodo_test_api(self, access_token):
r = requests.get(self.zenodo_endpoint + '/api/deposit/depositions')
if (r.status_code != 401):
raise_error(ZenodoError, "Cannot access Zenodo", r)
if (self.verbose):
print_info("Zenodo is accessible", r)
r = requests.get(self.zenodo_endpoint + '/api/deposit/depositions',
params={'access_token': access_token})
message = "Cannot authenticate to Zenodo API, check your access token"
if (r.status_code != 200):
raise_error(ZenodoError, message, r)
if (self.verbose):
print_info("Authentication to Zenodo successful", r)
def zenodo_publish(self, access_token, deposition_id, msg_obj):
r = requests.post(
self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/publish' % deposition_id,
params={'access_token': access_token})
if (r.status_code != 202):
raise_error(ZenodoError, "Cannot publish {}".format(msg_obj), r)
if (self.verbose):
print_info(
"{0} published to Zenodo, doi is {1}".format(
msg_obj,
r.json()['doi']), r)
return r.json()['doi']
def parse_req(req, req_type, bout_desc):
# We could support InitialWorkDirRequiment, through config files
if req_type == 'DockerRequirement':
container_image = {}
container_image['type'] = 'docker'
container_image['index'] = 'index.docker.io'
container_image['image'] = req['dockerPull']
bout_desc['container-image'] = container_image
return
if req_type == 'EnvVarRequirement':
bout_envars = []
for env_var in req['envDef']:
bout_env_var = {}
bout_env_var['name'] = env_var
bout_env_var['value'] = resolve_glob(
req['envDef'][env_var],
boutiques_inputs)
bout_envars.append(bout_env_var)
bout_desc['environment-variables'] = bout_envars
return
if req_type == 'ResourceRequirement':
suggested_resources = {}
if req.get('ramMin'):
suggested_resources['ram'] = req['ramMin']
if req.get('coresMin'):
suggeseted_resources['cpu-cores'] = req['coresMin']
bout_desc['suggested-resources'] = suggested_resources
return
if req_type == 'InitialWorkDirRequirement':
listing = req.get('listing')
for entry in listing:
file_name = entry.get('entryname')
assert(file_name is not None)
template = entry.get('entry')
for i in boutiques_inputs:
if i.get("value-key"):
template = template.replace("$(inputs."+i['id']+")",
i.get("value-key"))
template = template.split(os.linesep)
assert(template is not None)
name = op.splitext(file_name)[0]
boutiques_outputs.append(
{
'id': name,
'name': name,
'path-template': file_name,
'file-template': template
})
return
raise_error(ImportError, 'Unsupported requirement: '+str(req))
def validateSchema(s, d=None, **kwargs):
# Check schema wrt meta-schema
try:
jsonschema.Draft4Validator.check_schema(s)
except jsonschema.SchemaError as se:
errExit("Invocation schema is invalid.\n" + str(se.message), False)
# Check data instance against schema
if d:
try:
jsonschema.validate(d, s)
except ValidationError as e:
raise_error(InvocationValidationError, e)
if kwargs.get("verbose"):
print_info("Invocation Schema validation OK")
def _zenodo_upload_dataset(self, deposition_id, file):
file_path = os.path.join(self.cache_dir, file)
data = {'filename': file}
files = {'file': open(file_path, 'rb')}
r = requests.post(self.zenodo_endpoint +
'/api/deposit/depositions/%s/files' % deposition_id,
params={'access_token': self.zenodo_access_token},
data=data,
files=files)
if (r.status_code != 201):
raise_error(ZenodoError, "Cannot upload record", r)
if (self.verbose):
print_info("Record uploaded to Zenodo", r)
def zenodo_deposit_updated_version(self, deposition_id):
r = requests.post(
self.zenodo_endpoint +
'/api/deposit/depositions/%s/actions/newversion' % deposition_id,
params={'access_token': self.zenodo_access_token})
if (r.status_code != 201):
raise_error(ZenodoError, "Deposition of new version failed", r)
if (self.verbose):
print_info("Deposition of new version succeeded", r)
new_url = r.json()['links']['latest_draft']
new_zid = new_url.split("/")[-1]
self.zenodo_update_metadata(new_zid)
self.zenodo_delete_files(new_zid, r.json()["files"])
return new_zid
def zenodo_upload_descriptor(self, deposition_id):
data = {'filename': os.path.basename(self.descriptor_file_name)}
files = {'file': open(self.descriptor_file_name, 'rb')}
r = requests.post(self.zenodo_endpoint +
'/api/deposit/depositions/%s/files' % deposition_id,
params={'access_token': self.zenodo_access_token},
data=data,
files=files)
# Status code is inconsistent with Zenodo documentation
if (r.status_code != 201):
raise_error(ZenodoError, "Cannot upload descriptor", r)
if (self.verbose):
print_info("Descriptor uploaded to Zenodo", r)
def __init__(self, zid, verbose=False, sandbox=False):
# remove zenodo prefix
try:
self.zid = zid.split(".", 1)[1]
except IndexError:
raise_error(
ZenodoError, "Zenodo ID must be prefixed by "
"'zenodo', e.g. zenodo.123456")
self.verbose = verbose
self.sandbox = sandbox
self.cache_dir = os.path.join(os.path.expanduser('~'), ".cache",
"boutiques")
self.cached_fname = os.path.join(self.cache_dir,
"zenodo-{0}.json".format(self.zid))
def zenodo_deposit(self, metadata, access_token):
headers = {"Content-Type": "application/json"}
data = metadata
r = requests.post(self.zenodo_endpoint + '/api/deposit/depositions',
params={'access_token': access_token},
json={},
data=json.dumps(data),
headers=headers)
if (r.status_code != 201):
raise_error(ZenodoError, "Deposition failed", r)
zid = r.json()['id']
if (self.verbose):
print_info("Deposition succeeded, id is {0}".format(zid), r)
return zid