def put(self):
"""Receive a sortable reaper or user upload."""
#if not self.uid and not self.drone_request:
# self.abort(402, 'uploads must be from an authorized user or drone')
if 'Content-MD5' not in self.request.headers:
self.abort(400, 'Request must contain a valid "Content-MD5" header.')
filename = self.request.headers.get('Content-Disposition', '').partition('filename=')[2].strip('"')
if not filename:
self.abort(400, 'Request must contain a valid "Content-Disposition" header.')
with tempfile.TemporaryDirectory(prefix='.tmp', dir=self.app.config['upload_path']) as tempdir_path:
filepath = os.path.join(tempdir_path, filename)
success, digest, filesize, duration = util.receive_stream_and_validate(self.request.body_file, filepath, self.request.headers['Content-MD5'])
if not success:
self.abort(400, 'Content-MD5 mismatch.')
if not tarfile.is_tarfile(filepath):
self.abort(415, 'Only tar files are accepted.')
log.info('Received %s [%s] from %s' % (filename, util.hrsize(self.request.content_length), self.request.user_agent))
datainfo = util.parse_file(filepath, digest)
if datainfo is None:
util.quarantine_file(filepath, self.app.config['quarantine_path'])
self.abort(202, 'Quarantining %s (unparsable)' % filename)
util.commit_file(self.app.db.acquisitions, None, datainfo, filepath, self.app.config['data_path'])
util.create_job(self.app.db.acquisitions, datainfo) # FIXME we should only mark files as new and let engine take it from there
throughput = filesize / duration.total_seconds()
log.info('Received %s [%s, %s/s] from %s' % (filename, util.hrsize(filesize), util.hrsize(throughput), self.request.client_addr))
def store_file(fd, filename, md5, arcpath, arcname):
with tempfile.TemporaryDirectory(prefix='.tmp', dir=self.app.config['upload_path']) as tempdir_path:
filepath = os.path.join(tempdir_path, filename)
success, _, _, _ = util.receive_stream_and_validate(fd, filepath, md5)
if not success:
self.abort(400, 'Content-MD5 mismatch.')
with lockfile.LockFile(arcpath):
with tarfile.open(arcpath, 'a') as archive:
archive.add(filepath, os.path.join(arcname, filename))
def run(self):
while self.alive:
stage_items = [
os.path.join(self.stage_path, si)
for si in os.listdir(self.stage_path) if not si.startswith('.')
] # ignore dot files
if stage_items:
for stage_item in sorted(stage_items,
key=os.path.getmtime): # oldest first
if os.path.islink(stage_item):
os.remove(stage_item)
elif 'gephysio' in os.path.basename(
stage_item
): # HACK !!!!!!!!!!!!!!!! NIMS 1.0 cannot sort gephysio
log.info('Unpacking %s' %
os.path.basename(stage_item))
with tempfile.TemporaryDirectory() as tempdir_path:
with tarfile.open(stage_item) as archive:
archive.extractall(path=tempdir_path)
physiodir_path = os.listdir(tempdir_path)[0]
for f in os.listdir(
os.path.join(tempdir_path,
physiodir_path)):
shutil.copy(
os.path.join(tempdir_path, physiodir_path,
f),
os.path.join(self.nims_path, 'physio'))
log.info('Done %s' %
os.path.basename(stage_item))
os.remove(stage_item)
elif os.path.isfile(stage_item):
self.sort(stage_item)
else:
for subpath in [
os.path.join(dirpath, fn)
for (dirpath, _,
filenames) in os.walk(stage_item)
for fn in filenames
]:
if not os.path.islink(
subpath) and not subpath.startswith('.'):
self.sort(subpath)
shutil.rmtree(stage_item)
else:
log.debug('Waiting for data...')
time.sleep(self.sleep_time)
def post(self):
"""Create a new App."""
# this handles receive and writing the file
# but the the json validation and database is handled by util.
apps_path = self.app.config['apps_path']
if not apps_path:
self.abort(503, 'POST api/apps unavailable. apps_path not defined')
if self.public_request: # TODO: how to handle auth during bootstrap?
self.abort(403, 'must be logged in to upload apps')
app_meta = None
with tempfile.TemporaryDirectory(prefix='.tmp',
dir=apps_path) as tempdir_path:
hash_ = hashlib.sha1()
app_temp = os.path.join(tempdir_path, 'temp')
with open(app_temp, 'wb') as fd:
for chunk in iter(lambda: self.request.body_file.read(2**20),
''):
hash_.update(chunk)
fd.write(chunk)
if hash_.hexdigest() != self.request.headers['Content-MD5']:
self.abort(400, 'Content-MD5 mismatch.') # sha1
if not tarfile.is_tarfile(app_temp):
self.abort(415, 'Only tar files are accepted.')
with tarfile.open(app_temp) as tf:
for ti in tf:
if ti.name.endswith('description.json'):
app_meta = json.load(tf.extractfile(ti))
break
if not app_meta:
self.abort(
415, 'application tar does not contain description.json')
try:
jsonschema.validate(app_meta, APP_SCHEMA)
except (ValueError, jsonschema.ValidationError) as e:
self.abort(400, str(e))
util.insert_app(
self.app.db, app_temp, apps_path,
app_meta=app_meta) # pass meta info, prevent re-reading
log.debug('Recieved App: %s' % app_meta.get('_id'))
def recon_spirec(self, tempdir, num_jobs):
"""Do spiral image reconstruction and populate self.imagedata."""
with tempfile.TemporaryDirectory(dir=tempdir) as temp_dirpath:
if self.compressed:
pfile_path = os.path.join(temp_dirpath, self.basename)
with open(pfile_path, 'wb') as fd:
with gzip.open(self.filepath, 'rb') as gzfile:
fd.writelines(gzfile)
else:
pfile_path = self.filepath
basepath = os.path.join(temp_dirpath, 'recon')
cmd = 'spirec -l --rotate -90 --magfile --savefmap2 --b0navigator -r %s -t %s' % (
pfile_path, 'recon')
log.debug(cmd)
subprocess.call(
shlex.split(cmd),
cwd=temp_dirpath,
stdout=open(
'/dev/null',
'w')) # run spirec to generate .mag and fieldmap files
self.imagedata = np.fromfile(
file=basepath + '.mag_float',
dtype=np.float32).reshape([
self.size[0], self.size[1], self.num_timepoints,
self.num_echos, self.num_slices
],
order='F').transpose((0, 1, 4, 2, 3))
if os.path.exists(basepath +
'.B0freq2') and os.path.getsize(basepath +
'.B0freq2') > 0:
self.fm_data = np.fromfile(
file=basepath + '.B0freq2',
dtype=np.float32).reshape([
self.size[0], self.size[1], self.num_echos,
self.num_slices
],
order='F').transpose(
(0, 1, 3, 2))
def sort(self, filepath):
"""
Revised sorter to handle multiple pfile acquisitions from a single series.
Expects tgz file to contain METADATA.json and DIGEST.txt as the first files
in the archive.
"""
filename = os.path.basename(filepath)
if 'pfile' in filename:
log.info('Parsing %s' % filename)
with tempfile.TemporaryDirectory(dir=None) as tempdir_path:
with tarfile.open(filepath) as archive:
archive.extractall(path=tempdir_path)
newdata_dir = os.path.join(tempdir_path,
os.listdir(tempdir_path)[0])
try:
new_digest = open(os.path.join(newdata_dir,
'DIGEST.txt')).read()
except IOError:
log.debug('%s has no digest' % filepath)
new_digest = None
pfile = glob.glob(os.path.join(newdata_dir, 'P?????.7'))[0]
try:
mrfile = nimsdata.parse(pfile,
filetype='pfile',
full_parse=True)
except nimsdata.NIMSDataError:
self.preserve(filepath)
else:
log.info('Sorting %s' % filename)
filename = '_'.join(filename.rsplit('_')[-4:])
dataset = nimsgears.model.Dataset.from_mrfile(
mrfile, self.nims_path)
existing_pf = glob.glob(
os.path.join(self.nims_path, dataset.relpath,
'*pfile.tgz'))
if not existing_pf:
shutil.move(
filepath,
os.path.join(self.nims_path, dataset.relpath,
filename))
else:
orig_pf = existing_pf[0]
with tarfile.open(orig_pf) as orig_archive:
for ti in orig_archive:
if 'DIGEST.txt' in ti.name:
orig_digest = orig_archive.extractfile(
ti).read()
break
else:
log.debug('no digest')
orig_digest = None
if (new_digest is None or orig_digest is None) or (
new_digest != orig_digest):
log.debug('repacking')
with tempfile.TemporaryDirectory(
dir=tempdir_path) as combined_dir:
with tarfile.open(orig_pf) as orig_archive:
orig_archive.extractall(path=combined_dir)
combineddata_dir = os.path.join(
combined_dir,
os.listdir(combined_dir)[0])
for f in glob.glob(
os.path.join(newdata_dir, '*')):
fn = os.path.basename(f)
log.debug(
'MOVING %s into %s' %
(f, os.path.join(combineddata_dir,
fn)))
shutil.move(
f, os.path.join(combineddata_dir, fn))
log.debug(os.listdir(combineddata_dir))
outpath = os.path.join(tempdir_path, filename)
create_archive(
outpath,
combineddata_dir,
os.path.basename(combineddata_dir),
compresslevel=6)
shutil.move(
outpath,
os.path.join(self.nims_path,
dataset.relpath, filename))
os.remove(filepath)
else:
shutil.move(
filepath,
os.path.join(self.nims_path, dataset.relpath,
filename))
log.debug('file sorted into to %s' % os.path.join(
self.nims_path, dataset.relpath, filename))
dataset.container.num_mux_cal_cycle = mrfile.num_mux_cal_cycle
dataset.filenames = [filename]
dataset.updatetime = datetime.datetime.now()
dataset.untrash()
transaction.commit()
else:
try:
mrfile = nimsdata.parse(filepath)
except nimsdata.NIMSDataError:
self.preserve(filepath)
else:
mrfile.num_mux_cal_cycle = None # dcms will never have num_mux_cal_cycles
if mrfile.is_screenshot:
mrfile.acq_no = 0
mrfile.timestamp = datetime.datetime.strptime(
datetime.datetime.strftime(mrfile.timestamp, '%Y%m%d')
+ '235959', '%Y%m%d%H%M%S')
log.info('Sorting %s' % filename)
filename = '_'.join(filename.rsplit('_')[-4:])
dataset = nimsgears.model.Dataset.from_mrfile(
mrfile, self.nims_path)
shutil.move(
filepath,
os.path.join(self.nims_path, dataset.relpath, filename))
dataset.filenames = [filename]
dataset.updatetime = datetime.datetime.now()
dataset.untrash()
transaction.commit()
log.info('Done %s' % filename)
def _put_file(self, _id, container, filename):
"""Receive a targeted processor or user upload."""
tags = []
metadata = {}
if self.request.content_type == 'multipart/form-data':
filestream = None
# use cgi lib to parse multipart data without loading all into memory; use tempfile instead
# FIXME avoid using tempfile; processs incoming stream on the fly
fs_environ = self.request.environ.copy()
fs_environ.setdefault('CONTENT_LENGTH', '0')
fs_environ['QUERY_STRING'] = ''
form = cgi.FieldStorage(fp=self.request.body_file,
environ=fs_environ,
keep_blank_values=True)
for fieldname in form:
field = form[fieldname]
if fieldname == 'file':
filestream = field.file
filename = field.filename
elif fieldname == 'tags':
try:
tags = json.loads(field.value)
except ValueError:
self.abort(400, 'non-JSON value in "tags" parameter')
elif fieldname == 'metadata':
try:
metadata = json.loads(field.value)
except ValueError:
self.abort(400,
'non-JSON value in "metadata" parameter')
if filestream is None:
self.abort(400,
'multipart/form-data must contain a "file" field')
elif filename is None:
self.abort(400, 'Request must contain a filename parameter.')
else:
if 'Content-MD5' not in self.request.headers:
self.abort(
400, 'Request must contain a valid "Content-MD5" header.')
try:
tags = json.loads(self.request.get('tags', '[]'))
except ValueError:
self.abort(400, 'invalid "tags" parameter')
try:
metadata = json.loads(self.request.get('metadata', '{}'))
except ValueError:
self.abort(400, 'invalid "metadata" parameter')
filestream = self.request.body_file
flavor = self.request.GET.get('flavor',
'data') # TODO: flavor should go away
if flavor not in ['data', 'attachment']:
self.abort(
400,
'Query must contain flavor parameter: "data" or "attachment".')
with tempfile.TemporaryDirectory(
prefix='.tmp',
dir=self.app.config['upload_path']) as tempdir_path:
filepath = os.path.join(tempdir_path, filename)
md5 = self.request.headers.get('Content-MD5')
success, digest, _, duration = util.receive_stream_and_validate(
filestream, filepath, md5)
if not success:
self.abort(400, 'Content-MD5 mismatch.')
filesize = os.path.getsize(filepath)
mimetype = util.guess_mimetype(filepath)
filetype = util.guess_filetype(filepath, mimetype)
datainfo = {
'fileinfo': {
'filename': filename,
'filesize': filesize,
'filehash': digest,
'filetype': filetype,
'flavor': flavor,
'mimetype': mimetype,
'tags': tags,
'metadata': metadata,
},
}
throughput = filesize / duration.total_seconds()
log.info('Received %s [%s, %s/s] from %s' %
(filename, util.hrsize(filesize), util.hrsize(throughput),
self.request.client_addr))
util.commit_file(self.dbc, _id, datainfo, filepath,
self.app.config['data_path'])
def recon_mux_epi(self,
tempdir,
num_jobs,
timepoints=[],
octave_bin='octave'):
start_sec = time.time()
"""Do mux_epi image reconstruction and populate self.imagedata."""
ref_file = os.path.join(self.dirpath, '_' + self.basename + '_ref.dat')
vrgf_file = os.path.join(self.dirpath,
'_' + self.basename + '_vrgf.dat')
# See if external calibration data files are needed:
cal_file, cal_ref_file, cal_vrgf_file, cal_compressed = self.find_mux_cal_file(
)
# The dat files might be missing or empty if the vendor recon was disabled. If so, try to use the cal dat file.
# FIXME: if the p-file is not compressed, the cal dat file will not be used! We should refactor the recon
# code so that the dat files are always explicitly specified.
if not os.path.isfile(ref_file) or os.path.getsize(ref_file) < 64:
if cal_ref_file:
ref_file = cal_ref_file
else:
raise NIMSPFileError('ref.dat file not found')
if not os.path.isfile(vrgf_file) or os.path.getsize(vrgf_file) < 64:
if cal_vrgf_file:
vrgf_file = cal_vrgf_file
else:
raise NIMSPFileError('vrgf.dat file not found')
if self.recon_type == None:
# set the recon type automatically
# Scans with mux>1, arc>1, caipi
if self.is_dwi and self.num_bands > 1 and self.phase_encode_undersample < 1. and self.caipi:
sense_recon = 1
else:
sense_recon = 0
elif self.recon_type == 'sense':
sense_recon = 1
else:
sense_recon = 0
fermi_filt = 1
with tempfile.TemporaryDirectory(dir=tempdir) as temp_dirpath:
log.info(
'Running %d v-coil mux recon on %s in tempdir %s with %d jobs (sense=%d, fermi=%d, notch=%f).'
% (self.num_vcoils, self.filepath, tempdir, num_jobs,
sense_recon, fermi_filt, self.notch_thresh))
if cal_file != '':
log.info('Using calibration file: %s.' % cal_file)
if self.compressed:
shutil.copy(
ref_file,
os.path.join(temp_dirpath, os.path.basename(ref_file)))
shutil.copy(
vrgf_file,
os.path.join(temp_dirpath, os.path.basename(vrgf_file)))
pfile_path = uncompress(self.filepath, temp_dirpath)
else:
pfile_path = self.filepath
if cal_file and cal_compressed:
shutil.copy(
cal_ref_file,
os.path.join(temp_dirpath, os.path.basename(cal_ref_file)))
shutil.copy(
vrgf_file,
os.path.join(temp_dirpath,
os.path.basename(cal_vrgf_file)))
cal_file = uncompress(cal_file, temp_dirpath)
recon_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'mux_epi_recon'))
outname = os.path.join(temp_dirpath, 'sl')
# Spawn the desired number of subprocesses until all slices have been spawned
mux_recon_jobs = []
slice_num = 0
while slice_num < self.num_slices:
num_running_jobs = sum(
[job.poll() == None for job in mux_recon_jobs])
if num_running_jobs < num_jobs:
# Recon each slice separately. Note the slice_num+1 to deal with matlab's 1-indexing.
# Use 'str' on timepoints so that an empty array will produce '[]'
cmd = (
'%s --no-window-system -p %s --eval \'mux_epi_main("%s", "%s_%03d.mat", "%s", %d, %s, %d, 0, %s, %s, %s);\''
% (octave_bin, recon_path, pfile_path,
outname, slice_num, cal_file, slice_num + 1,
str(timepoints), self.num_vcoils, str(sense_recon),
str(fermi_filt), str(self.notch_thresh)))
log.debug(cmd)
mux_recon_jobs.append(
subprocess.Popen(args=shlex.split(cmd),
stdout=open('/dev/null', 'w')))
slice_num += 1
else:
time.sleep(1.)
# Now wait for all the jobs to finish
for job in mux_recon_jobs:
job.wait()
# Load the first slice to initialize the image array
img = self.load_imagedata_from_file("%s_%03d.mat" % (outname, 0))
for slice_num in range(1, self.num_slices):
new_img = self.load_imagedata_from_file("%s_%03d.mat" %
(outname, slice_num))
# Allow for a partial last timepoint. This sometimes happens when the user aborts.
t = min(img.shape[-1], new_img.shape[-1])
img[..., 0:t] += new_img[..., 0:t]
img = img.astype(np.float32)
self.update_imagedata(img)
elapsed = time.time() - start_sec
log.info(
'Mux recon of %s with %d v-coils finished in %0.2f minutes using %d jobs.'
% (self.filepath, self.num_vcoils, elapsed / 60.,
min(num_jobs, self.num_slices)))