def _get_sorteddict(self, object, dictwithhash=False, hash_method=None, hash_files=True): if isinstance(object, dict): out = {} for key, val in sorted(object.items()): if isdefined(val): out[key] = self._get_sorteddict(val, dictwithhash, hash_method=hash_method, hash_files=hash_files) elif isinstance(object, (list, tuple)): out = [] for val in object: if isdefined(val): out.append(self._get_sorteddict(val, dictwithhash, hash_method=hash_method, hash_files=hash_files)) if isinstance(object, tuple): out = tuple(out) else: if isdefined(object): if hash_files and isinstance(object, str) and os.path.isfile(object): if hash_method == None: hash_method = config.get("execution", "hash_method") if hash_method.lower() == "timestamp": hash = hash_timestamp(object) elif hash_method.lower() == "content": hash = hash_infile(object) else: raise Exception("Unknown hash method: %s" % hash_method) if dictwithhash: out = (object, hash) else: out = hash elif isinstance(object, float): out = "%.10f" % object else: out = object return out
def _add_provenance(in_file, settings, air_msk, rot_msk): from mriqc import __version__ as version from nipype.utils.filemanip import hash_infile import nibabel as nb import numpy as np air_msk_size = nb.load(air_msk).get_data().astype(np.uint8).sum() rot_msk_size = nb.load(rot_msk).get_data().astype(np.uint8).sum() out_prov = { 'md5sum': hash_infile(in_file), 'version': version, 'software': 'mriqc', 'warnings': { 'small_air_mask': bool(air_msk_size < 5e5), 'large_rot_frame': bool(rot_msk_size > 500), }, 'webapi_url': settings.pop('webapi_url'), 'webapi_port': settings.pop('webapi_port'), } if settings: out_prov['settings'] = settings return out_prov
def _run_interface(self, runtime): from nipype.utils.filemanip import hash_infile self._results["out_prov"] = { "md5sum": hash_infile(self.inputs.in_file), "version": config.environment.version, "software": "mriqc", "webapi_url": config.execution.webapi_url, "webapi_port": config.execution.webapi_port, "settings": { "testing": config.execution.debug, }, } if self.inputs.modality in ("T1w", "T2w"): air_msk_size = (np.asanyarray( nb.load(self.inputs.air_msk).dataobj).astype(bool).sum()) rot_msk_size = (np.asanyarray( nb.load(self.inputs.rot_msk).dataobj).astype(bool).sum()) self._results["out_prov"]["warnings"] = { "small_air_mask": bool(air_msk_size < 5e5), "large_rot_frame": bool(rot_msk_size > 500), } if self.inputs.modality == "bold": self._results["out_prov"]["settings"].update({ "fd_thres": config.workflow.fd_thres, }) return runtime
def _get_sorteddict(self, object, dictwithhash=False): if isinstance(object, dict): out = {} for key, val in sorted(object.items()): if isdefined(val): out[key] = self._get_sorteddict(val, dictwithhash) elif isinstance(object, (list,tuple)): out = [] for val in object: if isdefined(val): out.append(self._get_sorteddict(val, dictwithhash)) if isinstance(object, tuple): out = tuple(out) else: if isdefined(object): if isinstance(object, str) and os.path.isfile(object): if config.get('execution', 'hash_method').lower() == 'timestamp': hash = hash_timestamp(object) elif config.get('execution', 'hash_method').lower() == 'content': hash = hash_infile(object) else: raise Exception("Unknown hash method: %s" % config.get('execution', 'hash_method')) if dictwithhash: out = (object, hash) else: out = hash elif isinstance(object, float): out = '%.10f'%object else: out = object return out
def bet(record): import nipype from nipype.interfaces.fsl import BET from nipype.utils.filemanip import hash_infile nipype.config.enable_provenance() in_file_uri = record['t1_uri'] os.chdir('/tmp') fname = 'anatomy.nii.gz' with open(fname, 'wb') as fd: response = requests.get(in_file_uri, stream=True) if not response.ok: response.raise_for_status() for chunk in response.iter_content(1024): fd.write(chunk) # Check if interface has run with this input before sha = hash_infile(os.path.abspath(fname), crypto=hashlib.sha512) select = SelectQuery(config=app.config) res = select.execute_select('E0921842-1EDB-49F8-A4B3-BA51B85AD407') sha_recs = res[res.sha512.str.contains(sha)] bet_recs = res[res.interface.str.contains('BET')] results = dict() if sha_recs.empty or \ (not sha_recs.empty and bet_recs.empty): better = BET() better.inputs.in_file = os.path.abspath(fname) result = better.run() prov = result.provenance.rdf().serialize(format='json-ld') results.update({'prov': prov}) else: results.update({'prov': res.to_json(orient='records')}) return results
def main(): """Entry point""" from ..niworkflows.func.util import init_bold_reference_wf opts = get_parser().parse_args() wf = init_bold_reference_wf(1, name=hash_infile(opts.input_file), gen_report=True) wf.inputs.inputnode.bold_file = opts.input_file wf.base_dir = os.getcwd() wf.run()
def main(): """Entry point""" from fmriprep.workflows.bold.util import init_bold_reference_wf opts = get_parser().parse_args() wf = init_bold_reference_wf(1, name=hash_infile(opts.input_file), gen_report=True) wf.inputs.inputnode.bold_file = opts.input_file wf.base_dir = os.getcwd() wf.run()
def _add_provenance(in_file, settings): from mriqc import __version__ as version from nipype.utils.filemanip import hash_infile out_prov = { 'md5sum': hash_infile(in_file), 'version': version, 'software': 'mriqc', 'webapi_url': settings.get('webapi_url'), 'webapi_port': settings.get('webapi_port'), } if settings: out_prov['settings'] = settings return out_prov
def _add_provenance(in_file, settings): from mriqc import __version__ as version from nipype.utils.filemanip import hash_infile out_prov = { 'md5sum': hash_infile(in_file), 'version': version, 'software': 'mriqc', 'webapi_url': settings.pop('webapi_url'), 'webapi_port': settings.pop('webapi_port'), } if settings: out_prov['settings'] = settings return out_prov
def file(self, file_uri=None): print "File: ", file_uri parsed_object = urlparse(file_uri) fullpath = os.path.realpath(parsed_object.path) print fullpath if not os.path.exists(fullpath) or \ not fullpath.startswith('/mindhive/xnat/surfaces/adhd200'): raise cherrypy.HTTPError("403 Forbidden", "You are not allowed to access this resource.") file_hash = hash_infile(fullpath) object_hash = md5.md5(file_uri + file_hash).hexdigest() link_path = os.path.join(FILE_DIR, object_hash) if not os.path.exists(link_path): os.symlink(fullpath, link_path) cherrypy.response.headers['Content-Type'] = 'application/json' return json.dumps({'uri': cherrypy.url('/files/%s' % object_hash), 'md5sum': file_hash})
def _hash_infile(self, adict, key): """ Inject file hashes into adict[key]""" stuff = adict[key] if not is_container(stuff): stuff = [stuff] file_list = [] for afile in stuff: if is_container(afile): hashlist = self._hash_infile({"infiles": afile}, "infiles") hash = [val[1] for val in hashlist] else: if config.get("execution", "hash_method").lower() == "timestamp": hash = hash_timestamp(afile) elif config.get("execution", "hash_method").lower() == "content": hash = hash_infile(afile) else: raise Exception("Unknown hash method: %s" % config.get("execution", "hash_method")) file_list.append((afile, hash)) return file_list
def create_entity(graph, fullpath, root, subject_id, basedir): """ Create a PROV entity for a file in a FreeSurfer directory """ # identify FreeSurfer terms based on directory and file names relpath = fullpath.replace(basedir, '').replace(root, '').lstrip('/') fstypes = relpath.split('/')[:-1] additional_types = relpath.split('/')[-1].split('.') file_hash = hash_infile(fullpath) if file_hash is None: print fullpath # build a NI-DM object obj_attr = [(prov.PROV["type"], fs[fstype]) for fstype in fstypes] + \ [(prov.PROV["label"], "%s:%s" % ('.'.join(fstypes), '.'.join(additional_types))), (fs["relative_path"], "%s" % relpath), (nidm["file"], "file://%s%s" % (getfqdn(), fullpath)), (nidm["md5sum"], "%s" % file_hash), ] # append approprate FreeSurfer terms if 'lh' in additional_types: obj_attr.append((fs["hemisphere"], "left")) if 'rh' in additional_types: obj_attr.append((fs["hemisphere"], "right")) if 'aparc' in additional_types: obj_attr.append((prov.PROV["type"], fs["aparc"])) if 'a2005s' in additional_types: obj_attr.append((prov.PROV["type"], fs["a2005s"])) if 'a2009s' in additional_types: obj_attr.append((prov.PROV["type"], fs["a2009s"])) if 'exvivo' in relpath: obj_attr.append((prov.PROV["type"], fs["exvivo"])) if 'aseg' in additional_types: obj_attr.append((prov.PROV["type"], fs["aseg"])) if 'aparc.stats' in relpath: obj_attr.append((prov.PROV["type"], fs["desikan_killiany"])) if 'stats' in fstypes and 'stats' in additional_types: obj_attr.append((prov.PROV["type"], fs["statistics"])) id = fs[md5.new(subject_id + relpath).hexdigest()] graph.entity(id, obj_attr) return id
def main(args=None): """Entry point.""" from nipype.utils.filemanip import hash_infile from ..func.util import init_bold_reference_wf opts = get_parser().parse_args(args=args) wf = init_bold_reference_wf( opts.omp_nthreads, gen_report=True, name=hash_infile(opts.input_file), ) wf.inputs.inputnode.bold_file = opts.input_file wf.base_dir = os.getcwd() plugin = { 'plugin': 'MultiProc', 'plugin_args': { 'nprocs': opts.nprocs }, } if opts.nprocs < 2: plugin = {'plugin': 'Linear'} wf.run(**plugin)
def _run_interface(self, runtime): metadata = get_metadata_for_nifti(self.inputs.in_file) output_keys = [ key for key in list(self.output_spec().get().keys()) if key.endswith('_id') ] outputs = self.expr.search(op.basename( self.inputs.in_file)).groupdict() metadata['md5sum'] = hash_infile(self.inputs.in_file) for key in output_keys: id_value = outputs.get(key) if id_value is not None: self._results[key] = outputs.get(key) if isdefined(self.inputs.fields) and self.inputs.fields: for fname in self.inputs.fields: self._results[fname] = metadata[fname] else: self._results['out_dict'] = metadata return runtime