def discardPartialUploads(self, uploadId, userId, parentId, assetstoreId, minimumAge, includeUntracked, params): filters = {} if uploadId is not None: filters['uploadId'] = uploadId if userId is not None: filters['userId'] = userId if assetstoreId is not None: filters['assetstoreId'] = assetstoreId if parentId is not None: filters['parentId'] = parentId if minimumAge is not None: filters['minimumAge'] = minimumAge uploadList = list(self.model('upload').list(filters=filters)) # Move the results to list that isn't a cursor so we don't have to have # the cursor sitting around while we work on the data. for upload in uploadList: try: self.model('upload').cancelUpload(upload) except OSError as exc: if exc.errno == errno.EACCES: raise GirderException( 'Failed to delete upload.', 'girder.api.v1.system.delete-upload-failed') raise if includeUntracked: uploadList += self.model('upload').untrackedUploads('delete', assetstoreId) return uploadList
def getApiUrl(url=None, preferReferer=False): """ In a request thread, call this to get the path to the root of the REST API. The returned path does *not* end in a forward slash. :param url: URL from which to extract the base URL. If not specified, uses the server root system setting. If that is not specified, uses `cherrypy.url()` :param preferReferer: if no url is specified, this is true, and this is in a cherrypy request that has a referer header that contains the api string, use that referer as the url. """ apiStr = '/api/v1' if not url: if preferReferer and apiStr in cherrypy.request.headers.get( 'referer', ''): url = cherrypy.request.headers['referer'] else: root = ModelImporter.model('setting').get(SettingKey.SERVER_ROOT) if root: return posixpath.join( root, config.getConfig()['server']['api_root'].lstrip('/')) url = url or cherrypy.url() idx = url.find(apiStr) if idx < 0: raise GirderException('Could not determine API root in %s.' % url) return url[:idx + len(apiStr)]
def doSegmentation(self, image, seedCoord, tolerance): """ Run a lesion segmentation. :param image: A Girder Image item. :param seedCoord: (X, Y) coordinates of the segmentation seed point. :type seedCoord: tuple[int] :param tolerance: The intensity tolerance value for the segmentation. :type tolerance: int :return: The lesion segmentation, as a mask. :rtype: numpy.ndarray """ imageData = Image().imageData(image) if not( # The imageData has a shape of (rows, cols), the seed is (x, y) 0.0 <= seedCoord[0] <= imageData.shape[1] and 0.0 <= seedCoord[1] <= imageData.shape[0] ): raise GirderException('seedCoord is out of bounds') # OpenCV is significantly faster at segmentation right now mask = OpenCVSegmentationHelper.segment( imageData, seedCoord, tolerance) return mask
def downloadFile(self, file, offset=0, headers=True): """ Returns a generator function that will be used to stream the file from disk to the response. """ path = os.path.join(self.assetstore['root'], file['path']) if not os.path.isfile(path): raise GirderException( 'File %s does not exist.' % path, 'girder.utility.filesystem_assetstore_adapter.' 'file-does-not-exist') if headers: mimeType = file.get('mimeType', 'application/octet-stream') if not mimeType: mimeType = 'application/octet-stream' cherrypy.response.headers['Content-Type'] = mimeType cherrypy.response.headers['Content-Length'] = file['size'] - offset cherrypy.response.headers['Content-Disposition'] = \ 'attachment; filename="%s"' % file['name'] def stream(): with open(path, 'rb') as f: if offset > 0: f.seek(offset) while True: data = f.read(BUF_SIZE) if not data: break yield data return stream
def _matchRoute(self, method, path): """ Helper function that attempts to match the requested ``method`` and ``path`` with a registered route specification. :param method: The requested HTTP method, in lowercase. :type method: str :param path: The requested path. :type path: tuple[str] :returns: A tuple of ``(route, handler, wildcards)``, where ``route`` is the registered `list` of route components, ``handler`` is the route handler `function`, and ``wildcards`` is a `dict` of kwargs that should be passed to the underlying handler, based on the wildcard tokens of the route. :raises: `GirderException`, when no routes are defined on this resource. :raises: `RestException`, when no route can be matched. """ if not self._routes: raise GirderException('No routes defined for resource') for route, handler in self._routes[method][len(path)]: wildcards = {} for routeComponent, pathComponent in six.moves.zip(route, path): if routeComponent[0] == ':': # Wildcard token wildcards[routeComponent[1:]] = pathComponent elif routeComponent != pathComponent: # Exact match token break else: return route, handler, wildcards raise RestException('No matching route for "%s %s"' % (method.upper(), '/'.join(path)))
def downloadFile(self, file, offset=0, headers=True, endByte=None, **kwargs): if 'path' not in file: raise Exception('Missing path property') full_path = file['path'] url = '%s/file/%s/%s?view=read' % (self.newt_base_url, self.machine, full_path) if headers: raise cherrypy.HTTPRedirect(url) else: session_id = parse('newt.sessionId').find(getCurrentUser()) if len(session_id) > 0: session_id = session_id[0].value if session_id is None: raise GirderException('Missing NEWT session id') def stream(): cookies = dict(newt_sessionid=session_id) r = requests.get(url, cookies=cookies, stream=True) for chunk in r.iter_content(chunk_size=BUF_LEN): if chunk: yield chunk return stream
def getUploadAssetstore(self, event): """ Handle the model.upload.assetstore event. This event passes a dictionary consisting of a model type and resource document. If the base document has an assetstore policy, then set an assetstore key of this dictionary to an assetstore document that should be used or prevent the default action if no appropriate assetstores are allowed. :param event: event record. """ model, resource = self._getBaseResource(event.info['model'], event.info['resource']) if resource is None: return policy = resource[QUOTA_FIELD] assetstore = self._checkAssetstore( policy.get('preferredAssetstore', None)) if assetstore is False: assetstore = self._checkAssetstore( policy.get('fallbackAssetstore', None)) if assetstore is not False: logger.info('preferredAssetstore not available for %s %s, ' 'using fallbackAssetstore', model, resource['_id']) if assetstore is False: raise GirderException('Required assetstore is unavailable') if assetstore: event.addResponse(assetstore)
def _getIdValue(self, kwargs, idParam): if idParam in kwargs: return kwargs.pop(idParam) elif idParam in kwargs['params']: return kwargs['params'].pop(idParam) else: raise GirderException('No ID parameter passed: ' + idParam, 'girder.api.rest.no-id')
def initUpload(self, params): """ Before any bytes of the actual file are sent, a request should be made to initialize the upload. This creates the temporary record of the forthcoming upload that will be passed in chunks to the readChunk method. If you pass a "linkUrl" parameter, it will make a link file in the designated parent. """ self.requireParams(('name', 'parentId', 'parentType'), params) user = self.getCurrentUser() mimeType = params.get('mimeType', 'application/octet-stream') parentType = params['parentType'].lower() if parentType not in ('folder', 'item'): raise RestException('The parentType must be "folder" or "item".') parent = self.model(parentType).load(id=params['parentId'], user=user, level=AccessType.WRITE, exc=True) if 'linkUrl' in params: return self.model('file').filter( self.model('file').createLinkFile(url=params['linkUrl'], parent=parent, name=params['name'], parentType=parentType, creator=user, size=params.get('size')), user) else: self.requireParams('size', params) assetstore = None if params.get('assetstoreId'): assetstore = self.model('assetstore').load( params['assetstoreId']) try: upload = self.model('upload').createUpload( user=user, name=params['name'], parentType=parentType, parent=parent, size=int(params['size']), mimeType=mimeType, reference=params.get('reference'), assetstore=assetstore) except OSError as exc: if exc.errno == errno.EACCES: raise GirderException( 'Failed to create upload.', 'girder.api.v1.file.create-upload-failed') raise if upload['size'] > 0: return upload else: return self.model('file').filter( self.model('upload').finalizeUpload(upload), user)
def initUpload(self, parentType, parentId, name, size, mimeType, linkUrl, reference, assetstoreId): """ Before any bytes of the actual file are sent, a request should be made to initialize the upload. This creates the temporary record of the forthcoming upload that will be passed in chunks to the readChunk method. If you pass a "linkUrl" parameter, it will make a link file in the designated parent. """ user = self.getCurrentUser() parent = self.model(parentType).load( id=parentId, user=user, level=AccessType.WRITE, exc=True) if linkUrl is not None: return self.model('file').filter( self.model('file').createLinkFile( url=linkUrl, parent=parent, name=name, parentType=parentType, creator=user, size=size, mimeType=mimeType), user) else: self.requireParams({'size': size}) assetstore = None if assetstoreId: self.requireAdmin( user, message='You must be an admin to select a destination assetstore.') assetstore = self.model('assetstore').load(assetstoreId) chunk = None if size > 0 and cherrypy.request.headers.get('Content-Length'): ct = cherrypy.request.body.content_type.value if (ct not in cherrypy.request.body.processors and ct.split('/', 1)[0] not in cherrypy.request.body.processors): chunk = RequestBodyStream(cherrypy.request.body) if chunk is not None and chunk.getSize() <= 0: chunk = None try: # TODO: This can be made more efficient by adding # save=chunk is None # to the createUpload call parameters. However, since this is # a breaking change, that should be deferred until a major # version upgrade. upload = self.model('upload').createUpload( user=user, name=name, parentType=parentType, parent=parent, size=size, mimeType=mimeType, reference=reference, assetstore=assetstore) except OSError as exc: if exc.errno == errno.EACCES: raise GirderException( 'Failed to create upload.', 'girder.api.v1.file.create-upload-failed') raise if upload['size'] > 0: if chunk: return self.model('upload').handleChunk(upload, chunk, filter=True, user=user) return upload else: return self.model('file').filter( self.model('upload').finalizeUpload(upload), user)
def initUpload(self, parentType, parentId, name, size, mimeType, linkUrl, reference, assetstoreId, params): """ Before any bytes of the actual file are sent, a request should be made to initialize the upload. This creates the temporary record of the forthcoming upload that will be passed in chunks to the readChunk method. If you pass a "linkUrl" parameter, it will make a link file in the designated parent. """ user = self.getCurrentUser() parent = self.model(parentType).load(id=parentId, user=user, level=AccessType.WRITE, exc=True) if linkUrl is not None: return self.model('file').filter( self.model('file').createLinkFile(url=linkUrl, parent=parent, name=name, parentType=parentType, creator=user, size=size, mimeType=mimeType), user) else: self.requireParams({'size': size}) assetstore = None if assetstoreId: self.requireAdmin( user, message= 'You must be an admin to select a destination assetstore.') assetstore = self.model('assetstore').load(assetstoreId) try: upload = self.model('upload').createUpload( user=user, name=name, parentType=parentType, parent=parent, size=size, mimeType=mimeType, reference=reference, assetstore=assetstore) except OSError as exc: if exc.errno == errno.EACCES: raise GirderException( 'Failed to create upload.', 'girder.api.v1.file.create-upload-failed') raise if upload['size'] > 0: return upload else: return self.model('file').filter( self.model('upload').finalizeUpload(upload), user)
def computeDescriptor(self, params): # @todo Naively assuming we will always be able to retrieve the URL image, _type = base64FromUrl(params['url']) r = requests.post('%(base_url)s/compute/base64://%(image)s?content_type=%(type)s' % { 'base_url': self.search_url, 'image': image, 'type': _type}) if not r.ok: logger.error('Failed to compute SMQTK descriptor for image %s.' % params['url']) raise GirderException('Failed to compute descriptor', 'girder.plugins.imagespace_smqtk.smqtk.computeDescriptor') else: return r.json()
def _getImageMasks(self, annotation, featureId, image=None): if self.getState(annotation) != Study().State.COMPLETE: raise GirderException('Annotation is incomplete.') featureValues = annotation['meta']['annotations'].get(featureId, []) if not isinstance(featureValues, list): raise GirderException( 'Feature %s is not a superpixel annotation.' % featureId) possibleSuperpixelNums = numpy.array([ superpixelNum for superpixelNum, featureValue in enumerate(featureValues) if featureValue == 0.5 ]) definiteSuperpixelNums = numpy.array([ superpixelNum for superpixelNum, featureValue in enumerate(featureValues) if featureValue == 1.0 ]) if not image: image = Image().load(annotation['meta']['imageId'], force=True, exc=True) superpixelsData = Image().superpixelsData(image) possibleMask = numpy.in1d(superpixelsData.flat, possibleSuperpixelNums).reshape( superpixelsData.shape) possibleMask = possibleMask.astype(numpy.bool_) definiteMask = numpy.in1d(superpixelsData.flat, definiteSuperpixelNums).reshape( superpixelsData.shape) definiteMask = definiteMask.astype(numpy.bool_) return possibleMask, definiteMask
def getApiUrl(url=None): """ In a request thread, call this to get the path to the root of the REST API. The returned path does *not* end in a forward slash. :param url: URL from which to extract the base URL. If not specified, uses `cherrypy.url()` """ url = url or cherrypy.url() idx = url.find('/api/v1') if idx < 0: raise GirderException('Could not determine API root in %s.' % url) return url[:idx + 7]
def _import_path(self, parent, user, path, parent_type='folder'): url = '%s/file/%s/%s' % (self.newt_base_url, self.machine, path) if 'newt_sessionid' not in cherrypy.request.cookie: raise GirderException('Missing newt_sessionid') newt_sessionid = cherrypy.request.cookie['newt_sessionid'].value cookies = dict(newt_sessionid=newt_sessionid) r = requests.get(url, cookies=cookies) r.raise_for_status() paths = r.json() for p in paths: perms = p['perms'] name = p['name'] size = int(p['size']) full_path = os.path.join(path, name) if name in ['.', '..']: continue if perms.startswith('d'): print folder = self.model('folder').createFolder( parent=parent, name=name, parentType=parent_type, creator=user, reuseExisting=True) self._import_path(folder, user, full_path) else: item = self.model('item').createItem(name=name, creator=user, folder=parent, reuseExisting=True) file = self.model('file').createFile( name=name, creator=user, item=item, reuseExisting=True, assetstore=self.assetstore, mimeType=None, size=size) file['imported'] = True file['path'] = full_path self.model('file').save(file)
def downloadFile(self, file, offset=0, headers=True, endByte=None, contentDisposition=None, extraParameters=None, **kwargs): """ Returns a generator function that will be used to stream the file from disk to the response. """ if endByte is None or endByte > file['size']: endByte = file['size'] path = self.fullPath(file) if not os.path.isfile(path): raise GirderException( 'File %s does not exist.' % path, 'girder.utility.filesystem_assetstore_adapter.' 'file-does-not-exist') if headers: cherrypy.response.headers['Accept-Ranges'] = 'bytes' self.setContentHeaders(file, offset, endByte, contentDisposition) def stream(): bytesRead = offset with open(path, 'rb') as f: if offset > 0: f.seek(offset) while True: readLen = min(BUF_SIZE, endByte - bytesRead) if readLen <= 0: break data = f.read(readLen) bytesRead += readLen if not data: break yield data return stream
def discardPartialUploads(self, params): uploadList = list(self.model('upload').list(filters=params)) # Move the results to list that isn't a cursor so we don't have to have # the cursor sitting around while we work on the data. for upload in uploadList: try: self.model('upload').cancelUpload(upload) except OSError as exc: if exc.errno == errno.EACCES: raise GirderException( 'Failed to delete upload.', 'girder.api.v1.system.delete-upload-failed') raise untracked = self.boolParam('includeUntracked', params, default=True) if untracked: assetstoreId = params.get('assetstoreId', None) uploadList += self.model('upload').untrackedUploads( 'delete', assetstoreId) return uploadList
def checkUploadFinalize(self, event): """ Check if an upload will fit within a quota restriction before finalizing it. If it doesn't, discard it. :param event: event record. """ upload = event.info quotaInfo = self._checkUploadSize(upload) if quotaInfo: # Delete the upload self.model('upload').cancelUpload(upload) raise GirderException( 'Upload exceeded file storage quota (need %s, only %s ' 'available - used %s out of %s)' % (formatSize(quotaInfo['sizeNeeded']), formatSize(quotaInfo['quotaLeft']), formatSize(quotaInfo['quotaUsed']), formatSize(quotaInfo['fileSizeQuota'])), 'user_quota.upload-exceeds-quota')
def checkUploadStart(self, event): """ Check if an upload will fit within a quota restriction. This is before the upload occurs, but since multiple uploads can be started concurrently, we also have to check when the upload is being completed. :param event: event record. """ if '_id' in event.info: return quotaInfo = self._checkUploadSize(event.info) if quotaInfo: raise GirderException( 'Upload would exceed file storage quota (need %s, only %s ' 'available - used %s out of %s)' % (formatSize(quotaInfo['sizeNeeded']), formatSize(quotaInfo['quotaLeft']), formatSize(quotaInfo['quotaUsed']), formatSize(quotaInfo['fileSizeQuota'])), 'user_quota.upload-exceeds-quota')
def read(self, size=None): """ Read *size* bytes from the file data. :param size: The number of bytes to read from the current position. The actual number returned could be less than this if the end of the file is reached. An empty response indicates that the file has been completely consumed. If None or negative, read to the end of the file. :type size: int :rtype: bytes """ if size is None or size < 0: size = self._file['size'] - self._pos if size > self._maximumReadSize: raise GirderException('Read exceeds maximum allowed size.') data = six.BytesIO() length = 0 for chunk in itertools.chain(self._prev, self._stream): chunkLen = len(chunk) if chunkLen == 0: break if length + chunkLen <= size: data.write(chunk) self._prev = [] length += chunkLen if length == size: break else: chunkLen = min(size - length, chunkLen) data.write(chunk[:chunkLen]) self._prev = [chunk[chunkLen:]] length += chunkLen break self._pos += length return data.getvalue()
def reviewImages(self, dataset, acceptedImages, flaggedImages, user): # Avoid circular import from .image import Image # Verify that all images are pending review prereviewFolder = self.prereviewFolder(dataset) if not prereviewFolder: raise GirderException('No Pre-review folder for this dataset.') for image in itertools.chain(acceptedImages, flaggedImages): if image['folderId'] != prereviewFolder['_id']: raise ValidationException('Image %s is not in Pre-review.' % image['_id']) now = datetime.datetime.utcnow() for image in acceptedImages: image = Image().setMetadata(image, { 'reviewed': { 'userId': user['_id'], 'time': now, 'accepted': True } }) Image().move(image, dataset) if flaggedImages: flaggedCollection = Collection().findOne( {'name': 'Flagged Images'}) flaggedFolder = Folder().findOne({ 'name': dataset['name'], 'parentId': flaggedCollection['_id'], 'parentCollection': 'collection' }) if not flaggedFolder: flaggedFolder = Folder().createFolder(parent=flaggedCollection, name=dataset['name'], parentType='collection', public=None, creator=user, allowRename=False, reuseExisting=False) flaggedFolder = Folder().copyAccessPolicies(prereviewFolder, flaggedFolder, save=True) for image in flaggedImages: image = Image().setMetadata( image, { 'reviewed': { 'userId': user['_id'], 'time': now, 'accepted': False } }) Image().move(image, flaggedFolder) # Remove an empty Pre-review folder if (Folder().countItems(prereviewFolder) + Folder().countFolders(prereviewFolder)) == 0: Folder().remove(prereviewFolder)
def _proxiedUploadChunk(self, upload, chunk): """ Clients that do not support direct-to-S3 upload behavior will go through this method by sending the chunk data as they normally would for other assetstore types. Girder will send the data to S3 on behalf of the client. """ bucket = self._getBucket() if upload['s3']['chunked']: if 'uploadId' not in upload['s3']: # Initiate a new multipart upload mp = bucket.initiate_multipart_upload( upload['s3']['key'], headers=self._getRequestHeaders(upload)) upload['s3']['uploadId'] = mp.id upload['s3']['keyName'] = mp.key_name upload['s3']['partNumber'] = 0 upload['s3']['partNumber'] += 1 s3Info = upload['s3'] size = chunk.getSize() queryParams = { 'partNumber': s3Info['partNumber'], 'uploadId': s3Info['uploadId'] } headers = {'Content-Length': str(size)} url = self._botoGenerateUrl(method='PUT', key=s3Info['key'], queryParams=queryParams, headers=headers) resp = requests.request(method='PUT', url=url, data=chunk, headers=headers) if resp.status_code not in (200, 201): logger.error( 'S3 multipart upload failure %d (uploadId=%s):\n%s' % (resp.status_code, upload['_id'], resp.text)) raise GirderException('Upload failed (bad gateway)') upload['received'] += size else: size = chunk.getSize() if size < upload['size']: raise ValidationException( 'Uploads of this length must be sent in a single chunk.') reqInfo = upload['s3']['request'] resp = requests.request(method=reqInfo['method'], url=reqInfo['url'], data=chunk, headers=dict( reqInfo['headers'], **{'Content-Length': str(size)})) if resp.status_code not in (200, 201): logger.error('S3 upload failure %d (uploadId=%s):\n%s' % (resp.status_code, upload['_id'], resp.text)) raise GirderException('Upload failed (bad gateway)') upload['received'] = size return upload
def _proxiedUploadChunk(self, upload, chunk): """ Clients that do not support direct-to-S3 upload behavior will go through this method by sending the chunk data as they normally would for other assetstore types. Girder will send the data to S3 on behalf of the client. """ if upload['s3']['chunked']: if 'uploadId' not in upload['s3']: # Initiate a new multipart upload if this is the first chunk disp = 'attachment; filename="%s"' % upload['name'] mime = upload.get('mimeType', '') mp = self.client.create_multipart_upload( Bucket=self.assetstore['bucket'], Key=upload['s3']['key'], ACL='private', ContentDisposition=disp, ContentType=mime, Metadata={ 'uploader-id': str(upload['userId']), 'uploader-ip': str(cherrypy.request.remote.ip) }) upload['s3']['uploadId'] = mp['UploadId'] upload['s3']['keyName'] = mp['Key'] upload['s3']['partNumber'] = 0 upload['s3']['partNumber'] += 1 size = chunk.getSize() headers = {'Content-Length': str(size)} # We can't just call upload_part directly because they require a # seekable file object, and ours isn't. url = self.client.generate_presigned_url( ClientMethod='upload_part', Params={ 'Bucket': self.assetstore['bucket'], 'Key': upload['s3']['key'], 'ContentLength': size, 'UploadId': upload['s3']['uploadId'], 'PartNumber': upload['s3']['partNumber'] }) resp = requests.request(method='PUT', url=url, data=chunk, headers=headers) if resp.status_code not in (200, 201): logger.error( 'S3 multipart upload failure %d (uploadId=%s):\n%s' % (resp.status_code, upload['_id'], resp.text)) raise GirderException('Upload failed (bad gateway)') upload['received'] += size else: size = chunk.getSize() if size < upload['size']: raise ValidationException( 'Uploads of this length must be sent in a single chunk.') reqInfo = upload['s3']['request'] resp = requests.request(method=reqInfo['method'], url=reqInfo['url'], data=chunk, headers=dict( reqInfo['headers'], **{'Content-Length': str(size)})) if resp.status_code not in (200, 201): logger.error('S3 upload failure %d (uploadId=%s):\n%s' % (resp.status_code, upload['_id'], resp.text)) raise GirderException('Upload failed (bad gateway)') upload['received'] = size return upload
def downloadFile(self, file, offset=0, headers=True, endByte=None, contentDisposition=None, extraParameters=None, **kwargs): dataOffset = 0 dataLimit = None if extraParameters is not None: extraParameters = json.loads(extraParameters) extraParameters['format'] = 'csv' dataOffset = extraParameters.get('offset', 0) dataLimit = extraParameters.get('limit', None) extraParameters['offset'] = 0 extraParameters['limit'] = None # Get the parent class's stream. base_stream = super(NewCls, self).downloadFile(file, offset, headers, endByte, contentDisposition, json.dumps(extraParameters), **kwargs) # Fall back to base class when no special parameters are specified. if extraParameters is None: return base_stream # Construct and return our own stream that implements the special # behaviors requested in extraParameters on top of the base class's # stream. outputType = extraParameters.get('outputType') if outputType is None: outputType = 'json' if outputType not in allowed_outputtypes: print 'outputType = %s is not allowed' % (outputType) raise GirderException( '"outputType" must be one of: %s' % (', '.join(allowed_outputtypes)), '%s.illegal-argument' % (module)) filterFunc = extraParameters.get('filter') if filterFunc is None: def filterFunc(x): return True else: filterFunc = astToFunction(filterFunc) # Set content-length header to zero and clear content-range. if 'Content-Length' in cherrypy.response.headers: del cherrypy.response.headers['Content-Length'] if 'Content-Range' in cherrypy.response.headers: del cherrypy.response.headers['Content-Range'] fileType = extraParameters.get('fileType', 'csv') if fileType == 'csv': return csv_stream(base_stream, dataOffset, dataLimit, filterFunc, outputType) elif fileType == 'json': return json_stream(base_stream, dataOffset, dataLimit, filterFunc, outputType) else: raise RuntimeError('illegal fileType: %s' % (fileType))