def get_nexus_endpoint(self): # change once nexus is setup endpoint = "https://sandbox.bluebrainnexus.io/v1" if self.sandbox \ else "https://sandbox.bluebrainnexus.io/v1" if self.verbose: print_info("Using Nexus endpoint {0}".format(endpoint)) return endpoint
def zenodo_deposit_updated_version(self, metadata, access_token, deposition_id): r = requests.post( self.zenodo_endpoint + '/api/deposit/depositions/%s/actions/newversion' % deposition_id, params={'access_token': access_token}) if r.status_code == 403: raise_error( ZenodoError, "You do not have permission to access " "this resource. Note that you cannot " "publish an update to a tool belonging " "to someone else.", r) elif (r.status_code != 201): raise_error( ZenodoError, "Deposition of new version failed. Check " "that the Zenodo ID is correct (if one " "was provided).", r) if (self.verbose): print_info("Deposition of new version succeeded", r) new_url = r.json()['links']['latest_draft'] new_zid = new_url.split("/")[-1] self.zenodo_update_metadata(new_zid, r.json()['doi'], metadata, access_token) self.zenodo_delete_files(new_zid, r.json()["files"], access_token) return new_zid
def nexus_test_api(self, access_token, org, project): # Test connection to endpoint without token self.nexus.config.set_environment(self.nexus_endpoint) try: self.nexus.permissions.fetch() except ConnectionError as e: raise_error(NexusError, "Cannot access Nexus endpoint", e.response) if self.verbose: print_info("Nexus endpoint is accessible") # Test authentication with token, org and project self.nexus.config.set_token(access_token) try: self.nexus.projects.fetch(org, project) if self.verbose: print_info("Authentication to Nexus successful") except HTTPError as e: if 404 == e.response.status_code: raise_error( NexusError, "No project '{}' in organization '{}' " "in Nexus repository".format(project, org), e.response) elif 401 == e.response.status_code: raise_error( NexusError, "Cannot authenticate to Nexus API, check " "your access token", e.response) else: raise_error(NexusError, "Something went wrong", e.response)
def __init__(self, zids, verbose=False, sandbox=False): # remove zenodo prefix self.zenodo_entries = [] self.cache_dir = os.path.join(os.path.expanduser('~'), ".cache", "boutiques") discarded_zids = zids # This removes duplicates, should maintain order zids = list(dict.fromkeys(zids)) for zid in zids: discarded_zids.remove(zid) try: # Zenodo returns the full DOI, but for the purposes of # Boutiques we just use the Zenodo-specific portion (as its the # unique part). If the API updates on Zenodo to no longer # provide the full DOI, this still works because it just grabs # the last thing after the split. zid = zid.split('/')[-1] newzid = zid.split(".", 1)[1] newfname = os.path.join(self.cache_dir, "zenodo-{0}.json".format(newzid)) self.zenodo_entries.append({"zid": newzid, "fname": newfname}) except IndexError: raise_error(ZenodoError, "Zenodo ID must be prefixed by " "'zenodo', e.g. zenodo.123456") self.verbose = verbose self.sandbox = sandbox if(self.verbose): for zid in discarded_zids: print_info("Discarded duplicate id {0}".format(zid))
def search(self): results = self.zenodo_search() print_info("Showing %d of %d results." % (len( results.json()["hits"]["hits"]), results.json()["hits"]["total"])) if self.verbose: return self.create_results_list_verbose(results.json()) return self.create_results_list(results.json())
def pull(self): # return cached file if it exists if os.path.isfile(self.cached_fname): if (self.verbose): print_info("Found cached file at %s" % self.cached_fname) return self.cached_fname from boutiques.searcher import Searcher searcher = Searcher(self.zid, self.verbose, self.sandbox, exact_match=True) r = searcher.zenodo_search() for hit in r.json()["hits"]["hits"]: file_path = hit["files"][0]["links"]["self"] file_name = file_path.split(os.sep)[-1] if hit["id"] == int(self.zid): if not os.path.exists(self.cache_dir): os.makedirs(self.cache_dir) if (self.verbose): print_info("Downloading descriptor %s" % file_name) downloaded = urlretrieve(file_path, self.cached_fname) print("Downloaded descriptor to " + downloaded[0]) return downloaded[0] raise_error(ZenodoError, "Descriptor not found")
def delete(self, file=None, no_int=False): self.filename = extractFileName(file) self.no_int = no_int # Verify deletion if not self.no_int: prompt = self._get_delete_prompt() try: ret = raw_input(prompt) # Python 2 except NameError: ret = input(prompt) # Python 3 if ret.upper() != "Y": return # Remove the file specified by the file option if file is not None: # Check file exists in cache self._file_exists_in_cache(file) # Remove file from cache file_path = os.path.join(self.cache_dir, file) os.remove(file_path) print_info( "File {} has been removed from the data cache".format(file)) # Remove all files in the data cache else: [ os.remove(os.path.join(self.cache_dir, f)) for f in self.cache_files ] print_info("All files have been removed from the data cache")
def __init__(self, query, verbose=False, sandbox=False, max_results=None, no_trunc=False, exact_match=False): if query is not None: self.query = query else: self.query = 'boutiques' if not exact_match: self.query = '*' + self.query + '*' self.verbose = verbose self.sandbox = sandbox self.no_trunc = no_trunc self.max_results = max_results # Display top 10 results by default if max_results is None: self.max_results = 10 # Zenodo will error if asked for more than 9999 results if self.max_results > 9999: self.max_results = 9999 # Set Zenodo endpoint self.zenodo_endpoint = "https://sandbox.zenodo.org" if\ self.sandbox else "https://zenodo.org" if(self.verbose): print_info("Using Zenodo endpoint {0}". format(self.zenodo_endpoint))
def save_zenodo_access_token(self, access_token): json_creds = self.read_credentials() json_creds[self.config_token_property_name()] = access_token with open(self.config_file, 'w') as f: f.write(json.dumps(json_creds, indent=4, sort_keys=True)) if (self.verbose): print_info("Zenodo access token saved in {0}".format( self.config_file))
def validate_bids(descriptor, valid=False): if not valid: msg = "Please provide a Boutiques descriptor that has been validated." raise_error(DescriptorValidationError, msg) errors = [] # TODO: verify not only that all fields/keys exist, their properties, too # Ensure the command-line conforms to the BIDS app spec msg_template = " CLIError: command-line doesn't match template: {}" cltemp = r"mkdir -p \[OUTPUT_DIR\]; (.*) \[BIDS_DIR\] \[OUTPUT_DIR\]"\ r" \[ANALYSIS_LEVEL\] \[PARTICIPANT_LABEL\] \[SESSION_LABEL\]"\ r"[\\s]*(.*)" cmdline = descriptor["command-line"] if len(re.findall(cltemp, cmdline)) < 1: errors += [msg_template.format(cltemp)] # Verify IDs are present which link to the OUTPUT_DIR # key bot as File and String ftypes = set(["File", "String"]) msg_template = " OutError: \"{}\" types for outdir do not match \"{}\"" outtypes = set([ inp["type"] for inp in descriptor["inputs"] if inp["value-key"] == "[OUTPUT_DIR]" ]) if outtypes != ftypes: errors += [msg_template.format(", ".join(outtypes), ", ".join(ftypes))] # Verify that analysis levels is an enumerable with some # subset of "paricipant", "session", and "group" choices = ["session", "participant", "group"] msg_template = " LevelError: \"{}\" is not a valid analysis level" alevels = [ inp["value-choices"] for inp in descriptor["inputs"] if inp["value-key"] == "[ANALYSIS_LEVEL]" ][0] errors += [msg_template.format(lv) for lv in alevels if lv not in choices] # Verify there is only a single output defined (the directory) msg_template = "OutputError: 0 or multiple outputs defined" if len(descriptor["output-files"]) != 1: errors += [msg_template] else: # Verify that the output shows up as an output msg_template = "OutputError: OUTPUT_DIR is not represented as an output" if descriptor["output-files"][0]["path-template"] != "[OUTPUT_DIR]": errors += [msg_template] errors = None if errors == [] else errors if errors is None: print_info("BIDS validation OK") else: raise_error(DescriptorValidationError, "Invalid BIDS app descriptor:" "\n" + "\n".join(errors))
def zenodo_search(self): r = requests.get(self.zenodo_endpoint + '/api/records/?q=%s&' 'keywords=boutiques&keywords=schema&' 'keywords=version&file_type=json&type=software' '&page=1&size=%s' % (self.query, self.max_results)) if (r.status_code != 200): raise_error(ZenodoError, "Error searching Zenodo", r) if (self.verbose): print_info("Search successful.", r) return r
def search(self): results = self.zenodo_search() num_results = len(results.json()["hits"]["hits"]) total_results = results.json()["hits"]["total"] print_info("Showing %d of %d results." % (num_results if num_results < self.max_results else self.max_results, total_results)) if self.verbose: return self.create_results_list_verbose(results.json()) return self.create_results_list(results.json())
def save_nexus_inputs(self, access_token, org, project): json_creds = self.read_credentials() json_creds[self.config_token_property_name()] = access_token json_creds["nexus-organization"] = org json_creds["nexus-project"] = project with open(self.config_file, 'w') as f: f.write(json.dumps(json_creds, indent=4, sort_keys=True)) if self.verbose: print_info("Nexus access token, organization and project" " saved in {0}".format(self.config_file))
def zenodo_delete_files(self, new_deposition_id, files, access_token): for file in files: file_id = file["id"] r = requests.delete(self.zenodo_endpoint + '/api/deposit/depositions/%s/files/%s' % (new_deposition_id, file_id), params={'access_token': access_token}) if (r.status_code != 204): raise_error(ZenodoError, "Could not delete old file", r) if (self.verbose): print_info("Deleted old file", r)
def zenodo_publish(self, deposition_id): r = requests.post(self.zenodo_endpoint + '/api/deposit/depositions/%s/actions/publish' % deposition_id, params={'access_token': self.zenodo_access_token}) if(r.status_code != 202): raise_error(ZenodoError, "Cannot publish descriptor", r) if(self.verbose): print_info("Descriptor published to Zenodo, doi is {0}". format(r.json()['doi']), r) return r.json()['doi']
def zenodo_update_metadata(self, deposition_id): data = self.create_metadata() headers = {"Content-Type": "application/json"} r = requests.put(self.zenodo_endpoint + '/api/deposit/depositions/%s' % deposition_id, params={'access_token': self.zenodo_access_token}, data=json.dumps(data), headers=headers) if (r.status_code != 200): raise_error(ZenodoError, "Cannot update metadata of new version", r) if (self.verbose): print_info("Updated metadata of new version", r)
def zenodo_test_api(self, access_token): r = requests.get(self.zenodo_endpoint + '/api/deposit/depositions') if (r.status_code != 401): raise_error(ZenodoError, "Cannot access Zenodo", r) if (self.verbose): print_info("Zenodo is accessible", r) r = requests.get(self.zenodo_endpoint + '/api/deposit/depositions', params={'access_token': access_token}) message = "Cannot authenticate to Zenodo API, check your access token" if (r.status_code != 200): raise_error(ZenodoError, message, r) if (self.verbose): print_info("Authentication to Zenodo successful", r)
def zenodo_search(self): # Get all results r = requests.get(self.zenodo_endpoint + '/api/records/?q=' 'keywords:(/Boutiques/) AND ' 'keywords:(/schema-version.*/)' '%s' '&file_type=json&type=software&' 'page=1&size=%s' % (self.query_line, 9999)) if(r.status_code != 200): raise_error(ZenodoError, "Error searching Zenodo", r) if(self.verbose): print_info("Search successful for query \"%s\"" % self.query, r) return r
def zenodo_publish(self, access_token, deposition_id, msg_obj): r = requests.post( self.zenodo_endpoint + '/api/deposit/depositions/%s/actions/publish' % deposition_id, params={'access_token': access_token}) if (r.status_code != 202): raise_error(ZenodoError, "Cannot publish {}".format(msg_obj), r) if (self.verbose): print_info( "{0} published to Zenodo, doi is {1}".format( msg_obj, r.json()['doi']), r) return r.json()['doi']
def zenodo_deposit_updated_version(self, deposition_id): r = requests.post( self.zenodo_endpoint + '/api/deposit/depositions/%s/actions/newversion' % deposition_id, params={'access_token': self.zenodo_access_token}) if (r.status_code != 201): raise_error(ZenodoError, "Deposition of new version failed", r) if (self.verbose): print_info("Deposition of new version succeeded", r) new_url = r.json()['links']['latest_draft'] new_zid = new_url.split("/")[-1] self.zenodo_update_metadata(new_zid) self.zenodo_delete_files(new_zid, r.json()["files"]) return new_zid
def validateSchema(s, d=None, **kwargs): # Check schema wrt meta-schema try: jsonschema.Draft4Validator.check_schema(s) except jsonschema.SchemaError as se: errExit("Invocation schema is invalid.\n" + str(se.message), False) # Check data instance against schema if d: try: jsonschema.validate(d, s) except ValidationError as e: raise_error(InvocationValidationError, e) if kwargs.get("verbose"): print_info("Invocation Schema validation OK")
def _zenodo_upload_dataset(self, deposition_id, file): file_path = os.path.join(self.cache_dir, file) data = {'filename': file} files = {'file': open(file_path, 'rb')} r = requests.post(self.zenodo_endpoint + '/api/deposit/depositions/%s/files' % deposition_id, params={'access_token': self.zenodo_access_token}, data=data, files=files) if (r.status_code != 201): raise_error(ZenodoError, "Cannot upload record", r) if (self.verbose): print_info("Record uploaded to Zenodo", r)
def zenodo_upload_descriptor(self, deposition_id): data = {'filename': os.path.basename(self.descriptor_file_name)} files = {'file': open(self.descriptor_file_name, 'rb')} r = requests.post(self.zenodo_endpoint + '/api/deposit/depositions/%s/files' % deposition_id, params={'access_token': self.zenodo_access_token}, data=data, files=files) # Status code is inconsistent with Zenodo documentation if (r.status_code != 201): raise_error(ZenodoError, "Cannot upload descriptor", r) if (self.verbose): print_info("Descriptor uploaded to Zenodo", r)
def __init__(self, query, verbose=False, sandbox=False, max_results=None, no_trunc=False, exact_match=False): if query is not None: self.query = query if not exact_match: terms = self.query.split(" ") self.query_line = '' for t in terms: uncased_term = '' for ch in t: uncased_term = uncased_term + "[" + ch.upper() +\ ch.lower() + "]" uncased_term = quote(uncased_term) self.query_line = self.query_line + \ ' AND (/.*%s.*/)' % uncased_term else: self.query_line =\ ' AND (/%s/)' % self.query else: self.query_line = '' self.query = '' if (verbose): print_info("Using Query Line: " + self.query_line) self.verbose = verbose self.sandbox = sandbox self.no_trunc = no_trunc self.max_results = max_results # Display top 10 results by default if max_results is None: self.max_results = 10 # Zenodo will error if asked for more than 9999 results if self.max_results > 9999: self.max_results = 9999 # Set Zenodo endpoint self.zenodo_endpoint = "https://sandbox.zenodo.org" if\ self.sandbox else "https://zenodo.org" if (self.verbose): print_info("Using Zenodo endpoint {0}".format( self.zenodo_endpoint))
def zenodo_deposit(self, metadata, access_token): headers = {"Content-Type": "application/json"} data = metadata r = requests.post(self.zenodo_endpoint + '/api/deposit/depositions', params={'access_token': access_token}, json={}, data=json.dumps(data), headers=headers) if (r.status_code != 201): raise_error(ZenodoError, "Deposition failed", r) zid = r.json()['id'] if (self.verbose): print_info("Deposition succeeded, id is {0}".format(zid), r) return zid
def __init__(self, descriptor_file_name, verbose, sandbox, no_int, auth_token): # Straightforward assignments self.verbose = verbose self.sandbox = sandbox self.descriptor_file_name = descriptor_file_name self.no_int = no_int self.zenodo_access_token = auth_token # Validate and load descriptor validate_descriptor(descriptor_file_name) self.descriptor = json.loads(open(self.descriptor_file_name).read()) # Get relevant descriptor propertis self.url = self.descriptor.get('url') self.tool_doi = self.descriptor.get('tool-doi') self.descriptor_url = self.descriptor.get('descriptor-url') # Get tool author and check that it's defined if self.descriptor.get("author") is None: raise_error( ZenodoError, "Tool must have an author to be " "published. Add an 'author' property to your " "descriptor.") self.creator = self.descriptor['author'] # Get descriptor doi and check that it's not defined if self.descriptor.get('doi') is not None: raise_error( ZenodoError, "Descriptor already has a DOI. Please " "remove it from the descriptor before publishing it " "again. A new DOI will be generated.") self.config_file = os.path.join(os.path.expanduser('~'), ".boutiques") # Fix Zenodo access token if self.zenodo_access_token is None: self.zenodo_access_token = self.get_zenodo_access_token() self.save_zenodo_access_token() # Set Zenodo endpoint self.zenodo_endpoint = "https://sandbox.zenodo.org" if\ self.sandbox else "https://zenodo.org" if (self.verbose): print_info("Using Zenodo endpoint {0}".format( self.zenodo_endpoint))
def zenodo_update_metadata(self, new_deposition_id, old_doi): data = self.create_metadata() # Add the new DOI to the metadata old_doi_split = old_doi.split(".") old_doi_split[-1] = new_deposition_id new_doi = '.'.join(old_doi_split) data['metadata']['doi'] = new_doi headers = {"Content-Type": "application/json"} r = requests.put(self.zenodo_endpoint+'/api/deposit/depositions/%s' % new_deposition_id, params={'access_token': self.zenodo_access_token}, data=json.dumps(data), headers=headers) if(r.status_code != 200): raise_error(ZenodoError, "Cannot update metadata of new version", r) if(self.verbose): print_info("Updated metadata of new version", r)
def zenodo_upload_descriptor(self, deposition_id): # If in replace mode, remove the old DOI if self.descriptor.get('doi'): del self.descriptor['doi'] with open(self.descriptor_file_name, 'w') as fhandle: fhandle.write(json.dumps(self.descriptor, indent=4)) data = {'filename': os.path.basename(self.descriptor_file_name)} files = {'file': open(self.descriptor_file_name, 'rb')} r = requests.post(self.zenodo_endpoint + '/api/deposit/depositions/%s/files' % deposition_id, params={'access_token': self.zenodo_access_token}, data=data, files=files) # Status code is inconsistent with Zenodo documentation if (r.status_code != 201): raise_error(ZenodoError, "Cannot upload descriptor", r) if (self.verbose): print_info("Descriptor uploaded to Zenodo", r)
def __init__(self, query, verbose, sandbox, max_results): if query is not None: self.query = query else: self.query = 'boutiques' self.verbose = verbose self.sandbox = sandbox # Return max 10 results by default if max_results is not None: self.max_results = max_results else: self.max_results = 10 # Set Zenodo endpoint self.zenodo_endpoint = "https://sandbox.zenodo.org" if\ self.sandbox else "https://zenodo.org" if (self.verbose): print_info("Using Zenodo endpoint {0}".format( self.zenodo_endpoint))
def search(self): results = self.zenodo_search() total_results = results.json()["hits"]["total"] total_deprecated = len([h['metadata']['keywords'] for h in results.json()['hits']['hits'] if 'metadata' in h and 'keywords' in h['metadata'] and 'deprecated' in h['metadata']['keywords']]) results_list = self.create_results_list_verbose(results.json()) if\ self.verbose else\ self.create_results_list(results.json()) num_results = len(results_list) print_info("Showing %d of %d result(s)%s" % (num_results if num_results < self.max_results else self.max_results, total_results if self.verbose else total_results - total_deprecated, "." if self.verbose else ", exluding %d deprecated result(s)." % total_deprecated)) return results_list