def _get_sorteddict(self, object, dictwithhash=False, hash_method=None, hash_files=True):
if isinstance(object, dict):
out = {}
for key, val in sorted(object.items()):
if isdefined(val):
out[key] = self._get_sorteddict(val, dictwithhash, hash_method=hash_method, hash_files=hash_files)
elif isinstance(object, (list, tuple)):
out = []
for val in object:
if isdefined(val):
out.append(self._get_sorteddict(val, dictwithhash, hash_method=hash_method, hash_files=hash_files))
if isinstance(object, tuple):
out = tuple(out)
else:
if isdefined(object):
if hash_files and isinstance(object, str) and os.path.isfile(object):
if hash_method == None:
hash_method = config.get("execution", "hash_method")
if hash_method.lower() == "timestamp":
hash = hash_timestamp(object)
elif hash_method.lower() == "content":
hash = hash_infile(object)
else:
raise Exception("Unknown hash method: %s" % hash_method)
if dictwithhash:
out = (object, hash)
else:
out = hash
elif isinstance(object, float):
out = "%.10f" % object
else:
out = object
return out
def _add_provenance(in_file, settings, air_msk, rot_msk):
from mriqc import __version__ as version
from nipype.utils.filemanip import hash_infile
import nibabel as nb
import numpy as np
air_msk_size = nb.load(air_msk).get_data().astype(np.uint8).sum()
rot_msk_size = nb.load(rot_msk).get_data().astype(np.uint8).sum()
out_prov = {
'md5sum': hash_infile(in_file),
'version': version,
'software': 'mriqc',
'warnings': {
'small_air_mask': bool(air_msk_size < 5e5),
'large_rot_frame': bool(rot_msk_size > 500),
},
'webapi_url': settings.pop('webapi_url'),
'webapi_port': settings.pop('webapi_port'),
}
if settings:
out_prov['settings'] = settings
return out_prov
def _run_interface(self, runtime):
from nipype.utils.filemanip import hash_infile
self._results["out_prov"] = {
"md5sum": hash_infile(self.inputs.in_file),
"version": config.environment.version,
"software": "mriqc",
"webapi_url": config.execution.webapi_url,
"webapi_port": config.execution.webapi_port,
"settings": {
"testing": config.execution.debug,
},
}
if self.inputs.modality in ("T1w", "T2w"):
air_msk_size = (np.asanyarray(
nb.load(self.inputs.air_msk).dataobj).astype(bool).sum())
rot_msk_size = (np.asanyarray(
nb.load(self.inputs.rot_msk).dataobj).astype(bool).sum())
self._results["out_prov"]["warnings"] = {
"small_air_mask": bool(air_msk_size < 5e5),
"large_rot_frame": bool(rot_msk_size > 500),
}
if self.inputs.modality == "bold":
self._results["out_prov"]["settings"].update({
"fd_thres":
config.workflow.fd_thres,
})
return runtime
def _get_sorteddict(self, object, dictwithhash=False):
if isinstance(object, dict):
out = {}
for key, val in sorted(object.items()):
if isdefined(val):
out[key] = self._get_sorteddict(val, dictwithhash)
elif isinstance(object, (list,tuple)):
out = []
for val in object:
if isdefined(val):
out.append(self._get_sorteddict(val, dictwithhash))
if isinstance(object, tuple):
out = tuple(out)
else:
if isdefined(object):
if isinstance(object, str) and os.path.isfile(object):
if config.get('execution', 'hash_method').lower() == 'timestamp':
hash = hash_timestamp(object)
elif config.get('execution', 'hash_method').lower() == 'content':
hash = hash_infile(object)
else:
raise Exception("Unknown hash method: %s" % config.get('execution', 'hash_method'))
if dictwithhash:
out = (object, hash)
else:
out = hash
elif isinstance(object, float):
out = '%.10f'%object
else:
out = object
return out
def bet(record):
import nipype
from nipype.interfaces.fsl import BET
from nipype.utils.filemanip import hash_infile
nipype.config.enable_provenance()
in_file_uri = record['t1_uri']
os.chdir('/tmp')
fname = 'anatomy.nii.gz'
with open(fname, 'wb') as fd:
response = requests.get(in_file_uri, stream=True)
if not response.ok:
response.raise_for_status()
for chunk in response.iter_content(1024):
fd.write(chunk)
# Check if interface has run with this input before
sha = hash_infile(os.path.abspath(fname), crypto=hashlib.sha512)
select = SelectQuery(config=app.config)
res = select.execute_select('E0921842-1EDB-49F8-A4B3-BA51B85AD407')
sha_recs = res[res.sha512.str.contains(sha)]
bet_recs = res[res.interface.str.contains('BET')]
results = dict()
if sha_recs.empty or \
(not sha_recs.empty and bet_recs.empty):
better = BET()
better.inputs.in_file = os.path.abspath(fname)
result = better.run()
prov = result.provenance.rdf().serialize(format='json-ld')
results.update({'prov': prov})
else:
results.update({'prov': res.to_json(orient='records')})
return results
def main():
"""Entry point"""
from ..niworkflows.func.util import init_bold_reference_wf
opts = get_parser().parse_args()
wf = init_bold_reference_wf(1, name=hash_infile(opts.input_file), gen_report=True)
wf.inputs.inputnode.bold_file = opts.input_file
wf.base_dir = os.getcwd()
wf.run()
def main():
"""Entry point"""
from fmriprep.workflows.bold.util import init_bold_reference_wf
opts = get_parser().parse_args()
wf = init_bold_reference_wf(1, name=hash_infile(opts.input_file), gen_report=True)
wf.inputs.inputnode.bold_file = opts.input_file
wf.base_dir = os.getcwd()
wf.run()
def _add_provenance(in_file, settings):
from mriqc import __version__ as version
from nipype.utils.filemanip import hash_infile
out_prov = {
'md5sum': hash_infile(in_file),
'version': version,
'software': 'mriqc',
'webapi_url': settings.get('webapi_url'),
'webapi_port': settings.get('webapi_port'),
}
if settings:
out_prov['settings'] = settings
return out_prov
def _add_provenance(in_file, settings):
from mriqc import __version__ as version
from nipype.utils.filemanip import hash_infile
out_prov = {
'md5sum': hash_infile(in_file),
'version': version,
'software': 'mriqc',
'webapi_url': settings.pop('webapi_url'),
'webapi_port': settings.pop('webapi_port'),
}
if settings:
out_prov['settings'] = settings
return out_prov
def file(self, file_uri=None):
print "File: ", file_uri
parsed_object = urlparse(file_uri)
fullpath = os.path.realpath(parsed_object.path)
print fullpath
if not os.path.exists(fullpath) or \
not fullpath.startswith('/mindhive/xnat/surfaces/adhd200'):
raise cherrypy.HTTPError("403 Forbidden", "You are not allowed to access this resource.")
file_hash = hash_infile(fullpath)
object_hash = md5.md5(file_uri + file_hash).hexdigest()
link_path = os.path.join(FILE_DIR, object_hash)
if not os.path.exists(link_path):
os.symlink(fullpath, link_path)
cherrypy.response.headers['Content-Type'] = 'application/json'
return json.dumps({'uri': cherrypy.url('/files/%s' % object_hash),
'md5sum': file_hash})
def _hash_infile(self, adict, key):
""" Inject file hashes into adict[key]"""
stuff = adict[key]
if not is_container(stuff):
stuff = [stuff]
file_list = []
for afile in stuff:
if is_container(afile):
hashlist = self._hash_infile({"infiles": afile}, "infiles")
hash = [val[1] for val in hashlist]
else:
if config.get("execution", "hash_method").lower() == "timestamp":
hash = hash_timestamp(afile)
elif config.get("execution", "hash_method").lower() == "content":
hash = hash_infile(afile)
else:
raise Exception("Unknown hash method: %s" % config.get("execution", "hash_method"))
file_list.append((afile, hash))
return file_list
def create_entity(graph, fullpath, root, subject_id, basedir):
""" Create a PROV entity for a file in a FreeSurfer directory
"""
# identify FreeSurfer terms based on directory and file names
relpath = fullpath.replace(basedir, '').replace(root, '').lstrip('/')
fstypes = relpath.split('/')[:-1]
additional_types = relpath.split('/')[-1].split('.')
file_hash = hash_infile(fullpath)
if file_hash is None:
print fullpath
# build a NI-DM object
obj_attr = [(prov.PROV["type"], fs[fstype]) for fstype in fstypes] + \
[(prov.PROV["label"], "%s:%s" % ('.'.join(fstypes), '.'.join(additional_types))),
(fs["relative_path"], "%s" % relpath),
(nidm["file"], "file://%s%s" % (getfqdn(), fullpath)),
(nidm["md5sum"], "%s" % file_hash),
]
# append approprate FreeSurfer terms
if 'lh' in additional_types:
obj_attr.append((fs["hemisphere"], "left"))
if 'rh' in additional_types:
obj_attr.append((fs["hemisphere"], "right"))
if 'aparc' in additional_types:
obj_attr.append((prov.PROV["type"], fs["aparc"]))
if 'a2005s' in additional_types:
obj_attr.append((prov.PROV["type"], fs["a2005s"]))
if 'a2009s' in additional_types:
obj_attr.append((prov.PROV["type"], fs["a2009s"]))
if 'exvivo' in relpath:
obj_attr.append((prov.PROV["type"], fs["exvivo"]))
if 'aseg' in additional_types:
obj_attr.append((prov.PROV["type"], fs["aseg"]))
if 'aparc.stats' in relpath:
obj_attr.append((prov.PROV["type"], fs["desikan_killiany"]))
if 'stats' in fstypes and 'stats' in additional_types:
obj_attr.append((prov.PROV["type"], fs["statistics"]))
id = fs[md5.new(subject_id + relpath).hexdigest()]
graph.entity(id, obj_attr)
return id
def main(args=None):
"""Entry point."""
from nipype.utils.filemanip import hash_infile
from ..func.util import init_bold_reference_wf
opts = get_parser().parse_args(args=args)
wf = init_bold_reference_wf(
opts.omp_nthreads,
gen_report=True,
name=hash_infile(opts.input_file),
)
wf.inputs.inputnode.bold_file = opts.input_file
wf.base_dir = os.getcwd()
plugin = {
'plugin': 'MultiProc',
'plugin_args': {
'nprocs': opts.nprocs
},
}
if opts.nprocs < 2:
plugin = {'plugin': 'Linear'}
wf.run(**plugin)
def _run_interface(self, runtime):
metadata = get_metadata_for_nifti(self.inputs.in_file)
output_keys = [
key for key in list(self.output_spec().get().keys())
if key.endswith('_id')
]
outputs = self.expr.search(op.basename(
self.inputs.in_file)).groupdict()
metadata['md5sum'] = hash_infile(self.inputs.in_file)
for key in output_keys:
id_value = outputs.get(key)
if id_value is not None:
self._results[key] = outputs.get(key)
if isdefined(self.inputs.fields) and self.inputs.fields:
for fname in self.inputs.fields:
self._results[fname] = metadata[fname]
else:
self._results['out_dict'] = metadata
return runtime
