def read_chunk(self, event, path, root, user=None): params = event.info["params"] if "chunk" in params: chunk = params["chunk"] if isinstance(chunk, cherrypy._cpreqbody.Part): # Seek is the only obvious way to get the length of the part chunk.file.seek(0, os.SEEK_END) size = chunk.file.tell() chunk.file.seek(0, os.SEEK_SET) chunk = RequestBodyStream(chunk.file, size=size) else: chunk = RequestBodyStream(cherrypy.request.body) if not user: user = self.getCurrentUser() offset = int(params.get("offset", 0)) upload = Upload().load(params["uploadId"]) if upload["userId"] != user["_id"]: raise AccessException("You did not initiate this upload.") if upload["received"] != offset: raise RestException( "Server has received %s bytes, but client sent offset %s." % (upload["received"], offset) ) try: fobj = self._handle_chunk(upload, chunk, filter=True, user=user) event.preventDefault().addResponse(fobj) except IOError as exc: if exc.errno == errno.EACCES: raise Exception("Failed to store upload.") raise
def initUpload(self, parentType, parentId, name, size, mimeType, linkUrl, reference, assetstoreId): """ Before any bytes of the actual file are sent, a request should be made to initialize the upload. This creates the temporary record of the forthcoming upload that will be passed in chunks to the readChunk method. If you pass a "linkUrl" parameter, it will make a link file in the designated parent. """ user = self.getCurrentUser() parent = self.model(parentType).load( id=parentId, user=user, level=AccessType.WRITE, exc=True) if linkUrl is not None: return self.model('file').filter( self.model('file').createLinkFile( url=linkUrl, parent=parent, name=name, parentType=parentType, creator=user, size=size, mimeType=mimeType), user) else: self.requireParams({'size': size}) assetstore = None if assetstoreId: self.requireAdmin( user, message='You must be an admin to select a destination assetstore.') assetstore = self.model('assetstore').load(assetstoreId) chunk = None if size > 0 and cherrypy.request.headers.get('Content-Length'): ct = cherrypy.request.body.content_type.value if (ct not in cherrypy.request.body.processors and ct.split('/', 1)[0] not in cherrypy.request.body.processors): chunk = RequestBodyStream(cherrypy.request.body) if chunk is not None and chunk.getSize() <= 0: chunk = None try: # TODO: This can be made more efficient by adding # save=chunk is None # to the createUpload call parameters. However, since this is # a breaking change, that should be deferred until a major # version upgrade. upload = self.model('upload').createUpload( user=user, name=name, parentType=parentType, parent=parent, size=size, mimeType=mimeType, reference=reference, assetstore=assetstore) except OSError as exc: if exc.errno == errno.EACCES: raise GirderException( 'Failed to create upload.', 'girder.api.v1.file.create-upload-failed') raise if upload['size'] > 0: if chunk: return self.model('upload').handleChunk(upload, chunk, filter=True, user=user) return upload else: return self.model('file').filter( self.model('upload').finalizeUpload(upload), user)
def moveFileToAssetstore(self, file, user, assetstore, progress=noProgress): """ Move a file from whatever assetstore it is located in to a different assetstore. This is done by downloading and re-uploading the file. :param file: the file to move. :param user: the user that is authorizing the move. :param assetstore: the destination assetstore. :param progress: optional progress context. :returns: the original file if it is not moved, or the newly 'uploaded' file if it is. """ from .file import File if file['assetstoreId'] == assetstore['_id']: return file # Allow an event to cancel the move. This could be done, for instance, # on files that could change dynamically. event = events.trigger('model.upload.movefile', { 'file': file, 'assetstore': assetstore }) if event.defaultPrevented: raise GirderException( 'The file %s could not be moved to assetstore %s' % (file['_id'], assetstore['_id'])) # Create a new upload record into the existing file upload = self.createUploadToFile(file=file, user=user, size=int(file['size']), assetstore=assetstore) if file['size'] == 0: return File().filter(self.finalizeUpload(upload), user) # Uploads need to be chunked for some assetstores chunkSize = self._getChunkSize() chunk = None for data in File().download(file, headers=False)(): if chunk is not None: chunk += data else: chunk = data if len(chunk) >= chunkSize: upload = self.handleChunk( upload, RequestBodyStream(six.BytesIO(chunk), len(chunk))) progress.update(increment=len(chunk)) chunk = None if chunk is not None: upload = self.handleChunk( upload, RequestBodyStream(six.BytesIO(chunk), len(chunk))) progress.update(increment=len(chunk)) return upload
def addImage(self, dataset, params): params = self._decodeParams(params) self.requireParams(['filename', 'signature'], params) user = self.getCurrentUser() User().requireCreateDataset(user) filename = params['filename'].strip() if not filename: raise ValidationException('Filename must be specified.', 'filename') signature = params['signature'].strip() if not signature: raise ValidationException('Signature must be specified.', 'signature') imageDataStream = RequestBodyStream(cherrypy.request.body) imageDataSize = len(imageDataStream) if not imageDataSize: raise RestException('No data provided in request body.') image = Dataset().addImage( dataset=dataset, imageDataStream=imageDataStream, imageDataSize=imageDataSize, filename=filename, signature=signature, user=user) return Image().filter(image, user=user)
def readChunk(self, upload, offset, params): """ After the temporary upload record has been created (see initUpload), the bytes themselves should be passed up in ordered chunks. The user must remain logged in when passing each chunk, to authenticate that the writer of the chunk is the same as the person who initiated the upload. The passed offset is a verification mechanism for ensuring the server and client agree on the number of bytes sent/received. """ if 'chunk' in params: # If we see the undocumented "chunk" query string parameter, then we abort trying to # read the body, use the query string value as chunk, and pass it through to # Upload().handleChunk. This case is used by the direct S3 upload process. chunk = params['chunk'] else: chunk = RequestBodyStream(cherrypy.request.body) user = self.getCurrentUser() if upload['userId'] != user['_id']: raise AccessException('You did not initiate this upload.') if upload['received'] != offset: raise RestException( 'Server has received %s bytes, but client sent offset %s.' % ( upload['received'], offset)) try: return Upload().handleChunk(upload, chunk, filter=True, user=user) except IOError as exc: if exc.errno == errno.EACCES: raise Exception('Failed to store upload.') raise
def uploadFromFile(self, obj, size, name, parentType=None, parent=None, user=None, mimeType=None, reference=None, assetstore=None, attachParent=False): """ This method wraps the entire upload process into a single function to facilitate "internal" uploads from a file-like object. Example: .. code-block:: python size = os.path.getsize(filename) with open(filename, 'rb') as f: Upload().uploadFromFile(f, size, filename, 'item', parentItem, user) :param obj: The object representing the content to upload. :type obj: file-like :param size: The total size of the file. :type size: int :param name: The name of the file to create. :type name: str :param parentType: The type of the parent: "folder" or "item". :type parentType: str :param parent: The parent (item or folder) to upload into. :type parent: dict :param user: The user who is creating the file. :type user: dict :param mimeType: MIME type of the file. :type mimeType: str :param reference: An optional reference string that will be sent to the data.process event. :param assetstore: An optional assetstore to use to store the file. If unspecified, the current assetstore is used. :type reference: str :param attachParent: if True, instead of creating an item within the parent or giving the file an itemId, set itemId to None and set attachedToType and attachedToId instead (using the values passed in parentType and parent). This is intended for files that shouldn't appear as direct children of the parent, but are still associated with it. :type attachParent: boolean """ upload = self.createUpload( user=user, name=name, parentType=parentType, parent=parent, size=size, mimeType=mimeType, reference=reference, assetstore=assetstore, attachParent=attachParent) if size == 0: return self.finalizeUpload(upload) # The greater of 32 MB or the the upload minimum chunk size. chunkSize = self._getChunkSize() while True: data = obj.read(chunkSize) if not data: break upload = self.handleChunk(upload, RequestBodyStream(io.BytesIO(data), len(data))) return upload
def addImage(self, dataset, params): params = self._decodeParams(params) self.requireParams(['filename', 'signature'], params) user = self.getCurrentUser() User().requireCreateDataset(user) filename = params['filename'].strip() if not filename: raise ValidationException('Filename must be specified.', 'filename') signature = params['signature'].strip() if not signature: raise ValidationException('Signature must be specified.', 'signature') imageDataStream = RequestBodyStream(cherrypy.request.body) imageDataSize = len(imageDataStream) if not imageDataSize: raise RestException('No data provided in request body.') image = Dataset().addImage(dataset=dataset, imageDataStream=imageDataStream, imageDataSize=imageDataSize, filename=filename, signature=signature, user=user) # avoid circular imports from models.__init__ from isic_archive.tasks import ingestImage ingestImage.delay(image['_id']) return Image().filter(image, user=user)
def readChunk(self, upload, offset, params): """ After the temporary upload record has been created (see initUpload), the bytes themselves should be passed up in ordered chunks. The user must remain logged in when passing each chunk, to authenticate that the writer of the chunk is the same as the person who initiated the upload. The passed offset is a verification mechanism for ensuring the server and client agree on the number of bytes sent/received. This method accepts both the legacy multipart content encoding, as well as passing offset and uploadId as query parameters and passing the chunk as the body, which is the recommended method. .. deprecated :: 2.2.0 """ if 'chunk' in params: chunk = params['chunk'] if isinstance(chunk, cherrypy._cpreqbody.Part): # Seek is the only obvious way to get the length of the part chunk.file.seek(0, os.SEEK_END) size = chunk.file.tell() chunk.file.seek(0, os.SEEK_SET) chunk = RequestBodyStream(chunk.file, size=size) else: chunk = RequestBodyStream(cherrypy.request.body) user = self.getCurrentUser() if upload['userId'] != user['_id']: raise AccessException('You did not initiate this upload.') if upload['received'] != offset: raise RestException( 'Server has received %s bytes, but client sent offset %s.' % (upload['received'], offset)) try: return self.model('upload').handleChunk(upload, chunk, filter=True, user=user) except IOError as exc: if exc.errno == errno.EACCES: raise Exception('Failed to store upload.') raise
def create_file(self, event, path, root, user=None): params = event.info["params"] self.is_dir(path, root["_id"]) name = params["name"] parent = Folder().filter(self.vFolder(path, root), user=user) file_path = path / name try: with file_path.open(mode="a"): os.utime(file_path.as_posix()) except PermissionError: raise GirderException( "Insufficient perms to write on {}".format(path.as_posix()), "girder.api.v1.file.create-upload-failed", ) except Exception: raise size = int(params["size"]) chunk = None if size > 0 and cherrypy.request.headers.get("Content-Length"): ct = cherrypy.request.body.content_type.value if ( ct not in cherrypy.request.body.processors and ct.split("/", 1)[0] not in cherrypy.request.body.processors ): chunk = RequestBodyStream(cherrypy.request.body) if chunk is not None and chunk.getSize() <= 0: chunk = None upload = Upload().createUpload( user=user, name=name, parentType="folder", parent=parent, size=size ) if upload["size"] > 0: if chunk: fobj = self._handle_chunk(upload, chunk, filter=True, user=user) event.preventDefault().addResponse(fobj) return event.preventDefault().addResponse(upload) else: event.preventDefault().addResponse( File().filter(self.vFile(file_path, root), user=user) )
def registerMetadata(self, dataset, params): params = self._decodeParams(params) self.requireParams(['filename'], params) user = self.getCurrentUser() filename = params['filename'].strip() if not filename: raise ValidationException('Filename must be specified.', 'filename') metadataDataStream = RequestBodyStream(cherrypy.request.body) if not len(metadataDataStream): raise RestException('No data provided in request body.') Dataset().registerMetadata( dataset=dataset, user=user, metadataDataStream=metadataDataStream, filename=filename, sendMail=True) # TODO: return value? return {'status': 'success'}
def writeData(currentUser, file, data): stream = BytesIO(str.encode(data)) chunk = RequestBodyStream(stream, size=len(data)) upload = Upload().createUploadToFile(file, currentUser, len(data)) Upload().handleChunk(upload, chunk, filter=True, user=currentUser) return upload
def copy_file(src_file, dest_file, user): upload = Upload().createUploadToFile(dest_file, user, src_file['size']) Upload().handleChunk(upload=upload, chunk=RequestBodyStream(File().open(src_file), size=dest_file['size']), user=user) return upload
def writeBytes(currentUser, file, bytes): stream = BytesIO(bytes) chunk = RequestBodyStream(stream, size=len(bytes)) upload = Upload().createUploadToFile(file, currentUser, len(bytes)) Upload().handleChunk(upload, chunk, filter=True, user=currentUser) return upload
def postScore(self, submission, score, params): # Ensure admin access on the containing challenge phase phase = self.model('phase', 'covalic').load(submission['phaseId'], user=self.getCurrentUser(), exc=True, level=AccessType.ADMIN) # Record whether submission is being re-scored rescoring = 'overallScore' in submission # Save document to trigger computing overall score submission.pop('overallScore', None) submission['score'] = score submission = self.model('submission', 'covalic').save(submission) ######################################################### step1 ############################################ ######### 第一次嘗試 from girder.models.upload import Upload from girder.api.v1.file import File from girder.utility import RequestBodyStream import six dest = dict(submission) # for i in range(5000): # dest.setdefault(i, i + 20) dest1 = str(dest) size = len(dest1) user = self.model('user').load(submission['creatorId'], force=True) #str1 = submission['title'] # str2 = '.txt' # name = "".join((str1, str2)) name = submission['title'] parentType = 'folder' parentId = submission['folderId'] parent = self.model(parentType).load(id=parentId, user=user, level=AccessType.ADMIN, exc=True) mimeType = u'image/png' reference = None File().requireParams({'size': size}) assetstore = None chunk = None if size > 0 and cherrypy.request.headers.get('Content-Length'): ct = cherrypy.request.body.content_type.value #if (ct not in cherrypy.request.body.processors and # ct.split('/', 1)[0] not in cherrypy.request.body.processors): # chunk = RequestBodyStream(cherrypy.request.body) if chunk is not None and chunk.getSize() <= 0: chunk = None try: # TODO: This can be made more efficient by adding # save=chunk is None # to the createUpload call parameters. However, since this is # a breaking change, that should be deferred until a major # version upgrade. upload = Upload().createUpload(user=user, name=name, parentType=parentType, parent=parent, size=size, mimeType=mimeType, reference=reference, assetstore=assetstore) except OSError as exc: if exc.errno == errno.EACCES: raise GirderException( 'Failed to create upload.', 'girder.api.v1.file.create-upload-failed') raise if upload['size'] > 0: if chunk: return Upload().handleChunk(upload, chunk, filter=True, user=user) else: return self._model.filter(Upload().finalizeUpload(upload), user) offset = 0 if 'chunk' in params: chunk = params['chunk'] if isinstance(chunk, cherrypy._cpreqbody.Part): # Seek is the only obvious way to get the length of the part chunk.file.seek(0, os.SEEK_END) size = chunk.file.tell() chunk.file.seek(0, os.SEEK_SET) chunk = RequestBodyStream(chunk.file, size=size) else: chunk = RequestBodyStream(six.BytesIO(dest1), len(dest1)) #chunk = FileHandle(dest1) if upload['userId'] != user['_id']: raise AccessException('You did not initiate this upload.') if upload['received'] != offset: raise RestException( 'Server has received %s bytes, but client sent offset %s.' % (upload['received'], offset)) try: Upload().handleChunk(upload, chunk, filter=True, user=user) except IOError as exc: if exc.errno == errno.EACCES: raise Exception('Failed to store upload.') raise ########################################################################## # Delete the scoring user's job token since the job is now complete. token = self.getCurrentToken() self.model('token').remove(token) return submission