def engine(self): """ URL format: api/engine?level=<container_type>&id=<container_id> It expects a multipart/form-data request with a "metadata" field (json valid against api/schemas/input/enginemetadata) and 0 or more file fields with a non null filename property (filename is null for the "metadata"). """ level = self.get_param("level") if level != "acquisition": self.abort(404, "engine uploads are supported only at the acquisition level") acquisition_id = self.get_param("id") if not acquisition_id: self.abort(404, "container id is required") else: acquisition_id = bson.ObjectId(acquisition_id) if not self.superuser_request: self.abort(402, "uploads must be from an authorized drone") with tempfile.TemporaryDirectory(prefix=".tmp", dir=config.get_item("persistent", "data_path")) as tempdir_path: try: file_store = files.MultiFileStore(self.request, tempdir_path) except files.FileStoreException as e: self.abort(400, str(e)) if not file_store.metadata: self.abort(400, "metadata is missing") metadata_validator = validators.payload_from_schema_file(self, "enginemetadata.json") metadata_validator(file_store.metadata, "POST") file_infos = file_store.metadata["acquisition"].pop("files", []) try: acquisition_obj = reaperutil.update_container_hierarchy(file_store.metadata, acquisition_id, level) except APIStorageException as e: self.abort(400, e.message) # move the files before updating the database for name, fileinfo in file_store.files.items(): path = fileinfo["path"] target_path = os.path.join( config.get_item("persistent", "data_path"), util.path_from_hash(fileinfo["hash"]) ) files.move_file(path, target_path) self._merge_fileinfos(file_store.files, file_infos) # update the fileinfo in mongo if a file already exists for f in acquisition_obj["files"]: fileinfo = file_store.files.get(f["name"]) if fileinfo: fileinfo.pop("path", None) reaperutil.update_fileinfo("acquisitions", acquisition_obj["_id"], fileinfo) fileinfo["existing"] = True # create the missing fileinfo in mongo for name, fileinfo in file_store.files.items(): # if the file exists we don't need to create it # skip update fileinfo for files that doesn't have a path if not fileinfo.get("existing") and fileinfo.get("path"): del fileinfo["path"] reaperutil.add_fileinfo("acquisitions", acquisition_obj["_id"], fileinfo) return [{"filename": k, "hash": v["hash"], "size": v["size"]} for k, v in file_store.files.items()]
def download(self): """ In downloads we use filters in the payload to exclude/include files. To pass a single filter, each of its conditions should be satisfied. If a file pass at least one filter, it is included in the targets. For example: download_payload = { 'optional': True, 'nodes': [{'level':'project', '_id':project_id}], 'filters':[{ 'tags':{'+':['incomplete']} }, { 'types':{'-':['dicom']} }] } will download files with tag 'incomplete' OR type different from 'dicom' download_payload = { 'optional': True, 'nodes': [{'level':'project', '_id':project_id}], 'filters':[{ 'tags':{'+':['incomplete']}, 'types':{'+':['dicom']} }] } will download only files with tag 'incomplete' AND type different from 'dicom' """ ticket_id = self.get_param("ticket") if ticket_id: ticket = config.db.downloads.find_one({"_id": ticket_id}) if not ticket: self.abort(404, "no such ticket") if ticket["ip"] != self.request.client_addr: self.abort(400, "ticket not for this source IP") if self.get_param("symlinks"): self.response.app_iter = self._symlinkarchivestream(ticket, config.get_item("persistent", "data_path")) else: self.response.app_iter = self._archivestream(ticket) self.response.headers["Content-Type"] = "application/octet-stream" self.response.headers["Content-Disposition"] = "attachment; filename=" + str(ticket["filename"]) for project_id in ticket["projects"]: config.db.projects.update_one({"_id": project_id}, {"$inc": {"counter": 1}}) else: req_spec = self.request.json_body validator = validators.payload_from_schema_file(self, "download.json") validator(req_spec, "POST") log.debug(json.dumps(req_spec, sort_keys=True, indent=4, separators=(",", ": "))) if self.get_param("format") == "bids": return self._preflight_archivestream_bids(req_spec) else: return self._preflight_archivestream(req_spec)
def engine(self): """ URL format: api/engine?level=<container_type>&id=<container_id> It expects a multipart/form-data request with a "metadata" field (json valid against api/schemas/input/enginemetadata) and 0 or more file fields with a non null filename property (filename is null for the "metadata"). """ level = self.get_param('level') if level is None: self.abort(404, 'container level is required') if level != 'acquisition': self.abort( 404, 'engine uploads are supported only at the acquisition level') acquisition_id = self.get_param('id') if not acquisition_id: self.abort(404, 'container id is required') else: acquisition_id = util.ObjectId(acquisition_id) if not self.superuser_request: self.abort(402, 'uploads must be from an authorized drone') with tempfile.TemporaryDirectory(prefix='.tmp', dir=config.get_item( 'persistent', 'data_path')) as tempdir_path: try: file_store = files.MultiFileStore(self.request, tempdir_path) except files.FileStoreException as e: self.abort(400, str(e)) if not file_store.metadata: self.abort(400, 'metadata is missing') metadata_validator = validators.payload_from_schema_file( self, 'enginemetadata.json') metadata_validator(file_store.metadata, 'POST') file_infos = file_store.metadata['acquisition'].pop('files', []) now = datetime.datetime.utcnow() try: acquisition_obj = reaperutil.update_container_hierarchy( file_store.metadata, acquisition_id, level) except APIStorageException as e: self.abort(400, e.message) # move the files before updating the database for name, fileinfo in file_store.files.items(): path = fileinfo['path'] target_path = os.path.join( config.get_item('persistent', 'data_path'), util.path_from_hash(fileinfo['hash'])) files.move_file(path, target_path) # merge infos from the actual file and from the metadata merged_infos = self._merge_fileinfos(file_store.files, file_infos) # update the fileinfo in mongo if a file already exists for f in acquisition_obj['files']: fileinfo = merged_infos.get(f['name']) if fileinfo: fileinfo.pop('path', None) fileinfo['modified'] = now acquisition_obj = reaperutil.update_fileinfo( 'acquisitions', acquisition_obj['_id'], fileinfo) fileinfo['existing'] = True # create the missing fileinfo in mongo for name, fileinfo in merged_infos.items(): # if the file exists we don't need to create it # skip update fileinfo for files that doesn't have a path if not fileinfo.get('existing') and fileinfo.get('path'): del fileinfo['path'] fileinfo['created'] = now fileinfo['modified'] = now acquisition_obj = reaperutil.add_fileinfo( 'acquisitions', acquisition_obj['_id'], fileinfo) for f in acquisition_obj['files']: if f['name'] in file_store.files: file_ = { 'name': f['name'], 'hash': f['hash'], 'type': f.get('type'), 'measurements': f.get('measurements', []) } rules.create_jobs(config.db, acquisition_obj, 'acquisition', file_) return [{ 'name': k, 'hash': v['hash'], 'size': v['size'] } for k, v in merged_infos.items()]