def engine(self):
"""
URL format: api/engine?level=<container_type>&id=<container_id>
It expects a multipart/form-data request with a "metadata" field (json valid against api/schemas/input/enginemetadata)
and 0 or more file fields with a non null filename property (filename is null for the "metadata").
"""
level = self.get_param("level")
if level != "acquisition":
self.abort(404, "engine uploads are supported only at the acquisition level")
acquisition_id = self.get_param("id")
if not acquisition_id:
self.abort(404, "container id is required")
else:
acquisition_id = bson.ObjectId(acquisition_id)
if not self.superuser_request:
self.abort(402, "uploads must be from an authorized drone")
with tempfile.TemporaryDirectory(prefix=".tmp", dir=config.get_item("persistent", "data_path")) as tempdir_path:
try:
file_store = files.MultiFileStore(self.request, tempdir_path)
except files.FileStoreException as e:
self.abort(400, str(e))
if not file_store.metadata:
self.abort(400, "metadata is missing")
metadata_validator = validators.payload_from_schema_file(self, "enginemetadata.json")
metadata_validator(file_store.metadata, "POST")
file_infos = file_store.metadata["acquisition"].pop("files", [])
try:
acquisition_obj = reaperutil.update_container_hierarchy(file_store.metadata, acquisition_id, level)
except APIStorageException as e:
self.abort(400, e.message)
# move the files before updating the database
for name, fileinfo in file_store.files.items():
path = fileinfo["path"]
target_path = os.path.join(
config.get_item("persistent", "data_path"), util.path_from_hash(fileinfo["hash"])
)
files.move_file(path, target_path)
self._merge_fileinfos(file_store.files, file_infos)
# update the fileinfo in mongo if a file already exists
for f in acquisition_obj["files"]:
fileinfo = file_store.files.get(f["name"])
if fileinfo:
fileinfo.pop("path", None)
reaperutil.update_fileinfo("acquisitions", acquisition_obj["_id"], fileinfo)
fileinfo["existing"] = True
# create the missing fileinfo in mongo
for name, fileinfo in file_store.files.items():
# if the file exists we don't need to create it
# skip update fileinfo for files that doesn't have a path
if not fileinfo.get("existing") and fileinfo.get("path"):
del fileinfo["path"]
reaperutil.add_fileinfo("acquisitions", acquisition_obj["_id"], fileinfo)
return [{"filename": k, "hash": v["hash"], "size": v["size"]} for k, v in file_store.files.items()]
def download(self):
"""
In downloads we use filters in the payload to exclude/include files.
To pass a single filter, each of its conditions should be satisfied.
If a file pass at least one filter, it is included in the targets.
For example:
download_payload = {
'optional': True,
'nodes': [{'level':'project', '_id':project_id}],
'filters':[{
'tags':{'+':['incomplete']}
},
{
'types':{'-':['dicom']}
}]
}
will download files with tag 'incomplete' OR type different from 'dicom'
download_payload = {
'optional': True,
'nodes': [{'level':'project', '_id':project_id}],
'filters':[{
'tags':{'+':['incomplete']},
'types':{'+':['dicom']}
}]
}
will download only files with tag 'incomplete' AND type different from 'dicom'
"""
ticket_id = self.get_param("ticket")
if ticket_id:
ticket = config.db.downloads.find_one({"_id": ticket_id})
if not ticket:
self.abort(404, "no such ticket")
if ticket["ip"] != self.request.client_addr:
self.abort(400, "ticket not for this source IP")
if self.get_param("symlinks"):
self.response.app_iter = self._symlinkarchivestream(ticket, config.get_item("persistent", "data_path"))
else:
self.response.app_iter = self._archivestream(ticket)
self.response.headers["Content-Type"] = "application/octet-stream"
self.response.headers["Content-Disposition"] = "attachment; filename=" + str(ticket["filename"])
for project_id in ticket["projects"]:
config.db.projects.update_one({"_id": project_id}, {"$inc": {"counter": 1}})
else:
req_spec = self.request.json_body
validator = validators.payload_from_schema_file(self, "download.json")
validator(req_spec, "POST")
log.debug(json.dumps(req_spec, sort_keys=True, indent=4, separators=(",", ": ")))
if self.get_param("format") == "bids":
return self._preflight_archivestream_bids(req_spec)
else:
return self._preflight_archivestream(req_spec)
def engine(self):
"""
URL format: api/engine?level=<container_type>&id=<container_id>
It expects a multipart/form-data request with a "metadata" field (json valid against api/schemas/input/enginemetadata)
and 0 or more file fields with a non null filename property (filename is null for the "metadata").
"""
level = self.get_param('level')
if level is None:
self.abort(404, 'container level is required')
if level != 'acquisition':
self.abort(
404,
'engine uploads are supported only at the acquisition level')
acquisition_id = self.get_param('id')
if not acquisition_id:
self.abort(404, 'container id is required')
else:
acquisition_id = util.ObjectId(acquisition_id)
if not self.superuser_request:
self.abort(402, 'uploads must be from an authorized drone')
with tempfile.TemporaryDirectory(prefix='.tmp',
dir=config.get_item(
'persistent',
'data_path')) as tempdir_path:
try:
file_store = files.MultiFileStore(self.request, tempdir_path)
except files.FileStoreException as e:
self.abort(400, str(e))
if not file_store.metadata:
self.abort(400, 'metadata is missing')
metadata_validator = validators.payload_from_schema_file(
self, 'enginemetadata.json')
metadata_validator(file_store.metadata, 'POST')
file_infos = file_store.metadata['acquisition'].pop('files', [])
now = datetime.datetime.utcnow()
try:
acquisition_obj = reaperutil.update_container_hierarchy(
file_store.metadata, acquisition_id, level)
except APIStorageException as e:
self.abort(400, e.message)
# move the files before updating the database
for name, fileinfo in file_store.files.items():
path = fileinfo['path']
target_path = os.path.join(
config.get_item('persistent', 'data_path'),
util.path_from_hash(fileinfo['hash']))
files.move_file(path, target_path)
# merge infos from the actual file and from the metadata
merged_infos = self._merge_fileinfos(file_store.files, file_infos)
# update the fileinfo in mongo if a file already exists
for f in acquisition_obj['files']:
fileinfo = merged_infos.get(f['name'])
if fileinfo:
fileinfo.pop('path', None)
fileinfo['modified'] = now
acquisition_obj = reaperutil.update_fileinfo(
'acquisitions', acquisition_obj['_id'], fileinfo)
fileinfo['existing'] = True
# create the missing fileinfo in mongo
for name, fileinfo in merged_infos.items():
# if the file exists we don't need to create it
# skip update fileinfo for files that doesn't have a path
if not fileinfo.get('existing') and fileinfo.get('path'):
del fileinfo['path']
fileinfo['created'] = now
fileinfo['modified'] = now
acquisition_obj = reaperutil.add_fileinfo(
'acquisitions', acquisition_obj['_id'], fileinfo)
for f in acquisition_obj['files']:
if f['name'] in file_store.files:
file_ = {
'name': f['name'],
'hash': f['hash'],
'type': f.get('type'),
'measurements': f.get('measurements', [])
}
rules.create_jobs(config.db, acquisition_obj,
'acquisition', file_)
return [{
'name': k,
'hash': v['hash'],
'size': v['size']
} for k, v in merged_infos.items()]
