def read_chunk(self, event, path, root, user=None):
params = event.info["params"]
if "chunk" in params:
chunk = params["chunk"]
if isinstance(chunk, cherrypy._cpreqbody.Part):
# Seek is the only obvious way to get the length of the part
chunk.file.seek(0, os.SEEK_END)
size = chunk.file.tell()
chunk.file.seek(0, os.SEEK_SET)
chunk = RequestBodyStream(chunk.file, size=size)
else:
chunk = RequestBodyStream(cherrypy.request.body)
if not user:
user = self.getCurrentUser()
offset = int(params.get("offset", 0))
upload = Upload().load(params["uploadId"])
if upload["userId"] != user["_id"]:
raise AccessException("You did not initiate this upload.")
if upload["received"] != offset:
raise RestException(
"Server has received %s bytes, but client sent offset %s."
% (upload["received"], offset)
)
try:
fobj = self._handle_chunk(upload, chunk, filter=True, user=user)
event.preventDefault().addResponse(fobj)
except IOError as exc:
if exc.errno == errno.EACCES:
raise Exception("Failed to store upload.")
raise
def initUpload(self, parentType, parentId, name, size, mimeType, linkUrl, reference,
assetstoreId):
"""
Before any bytes of the actual file are sent, a request should be made
to initialize the upload. This creates the temporary record of the
forthcoming upload that will be passed in chunks to the readChunk
method. If you pass a "linkUrl" parameter, it will make a link file
in the designated parent.
"""
user = self.getCurrentUser()
parent = self.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
if linkUrl is not None:
return self.model('file').filter(
self.model('file').createLinkFile(
url=linkUrl, parent=parent, name=name, parentType=parentType, creator=user,
size=size, mimeType=mimeType), user)
else:
self.requireParams({'size': size})
assetstore = None
if assetstoreId:
self.requireAdmin(
user, message='You must be an admin to select a destination assetstore.')
assetstore = self.model('assetstore').load(assetstoreId)
chunk = None
if size > 0 and cherrypy.request.headers.get('Content-Length'):
ct = cherrypy.request.body.content_type.value
if (ct not in cherrypy.request.body.processors and
ct.split('/', 1)[0] not in cherrypy.request.body.processors):
chunk = RequestBodyStream(cherrypy.request.body)
if chunk is not None and chunk.getSize() <= 0:
chunk = None
try:
# TODO: This can be made more efficient by adding
# save=chunk is None
# to the createUpload call parameters. However, since this is
# a breaking change, that should be deferred until a major
# version upgrade.
upload = self.model('upload').createUpload(
user=user, name=name, parentType=parentType, parent=parent, size=size,
mimeType=mimeType, reference=reference, assetstore=assetstore)
except OSError as exc:
if exc.errno == errno.EACCES:
raise GirderException(
'Failed to create upload.', 'girder.api.v1.file.create-upload-failed')
raise
if upload['size'] > 0:
if chunk:
return self.model('upload').handleChunk(upload, chunk, filter=True, user=user)
return upload
else:
return self.model('file').filter(
self.model('upload').finalizeUpload(upload), user)
def moveFileToAssetstore(self,
file,
user,
assetstore,
progress=noProgress):
"""
Move a file from whatever assetstore it is located in to a different
assetstore. This is done by downloading and re-uploading the file.
:param file: the file to move.
:param user: the user that is authorizing the move.
:param assetstore: the destination assetstore.
:param progress: optional progress context.
:returns: the original file if it is not moved, or the newly 'uploaded'
file if it is.
"""
from .file import File
if file['assetstoreId'] == assetstore['_id']:
return file
# Allow an event to cancel the move. This could be done, for instance,
# on files that could change dynamically.
event = events.trigger('model.upload.movefile', {
'file': file,
'assetstore': assetstore
})
if event.defaultPrevented:
raise GirderException(
'The file %s could not be moved to assetstore %s' %
(file['_id'], assetstore['_id']))
# Create a new upload record into the existing file
upload = self.createUploadToFile(file=file,
user=user,
size=int(file['size']),
assetstore=assetstore)
if file['size'] == 0:
return File().filter(self.finalizeUpload(upload), user)
# Uploads need to be chunked for some assetstores
chunkSize = self._getChunkSize()
chunk = None
for data in File().download(file, headers=False)():
if chunk is not None:
chunk += data
else:
chunk = data
if len(chunk) >= chunkSize:
upload = self.handleChunk(
upload, RequestBodyStream(six.BytesIO(chunk), len(chunk)))
progress.update(increment=len(chunk))
chunk = None
if chunk is not None:
upload = self.handleChunk(
upload, RequestBodyStream(six.BytesIO(chunk), len(chunk)))
progress.update(increment=len(chunk))
return upload
def addImage(self, dataset, params):
params = self._decodeParams(params)
self.requireParams(['filename', 'signature'], params)
user = self.getCurrentUser()
User().requireCreateDataset(user)
filename = params['filename'].strip()
if not filename:
raise ValidationException('Filename must be specified.', 'filename')
signature = params['signature'].strip()
if not signature:
raise ValidationException('Signature must be specified.', 'signature')
imageDataStream = RequestBodyStream(cherrypy.request.body)
imageDataSize = len(imageDataStream)
if not imageDataSize:
raise RestException('No data provided in request body.')
image = Dataset().addImage(
dataset=dataset,
imageDataStream=imageDataStream,
imageDataSize=imageDataSize,
filename=filename,
signature=signature,
user=user)
return Image().filter(image, user=user)
def readChunk(self, upload, offset, params):
"""
After the temporary upload record has been created (see initUpload),
the bytes themselves should be passed up in ordered chunks. The user
must remain logged in when passing each chunk, to authenticate that
the writer of the chunk is the same as the person who initiated the
upload. The passed offset is a verification mechanism for ensuring the
server and client agree on the number of bytes sent/received.
"""
if 'chunk' in params:
# If we see the undocumented "chunk" query string parameter, then we abort trying to
# read the body, use the query string value as chunk, and pass it through to
# Upload().handleChunk. This case is used by the direct S3 upload process.
chunk = params['chunk']
else:
chunk = RequestBodyStream(cherrypy.request.body)
user = self.getCurrentUser()
if upload['userId'] != user['_id']:
raise AccessException('You did not initiate this upload.')
if upload['received'] != offset:
raise RestException(
'Server has received %s bytes, but client sent offset %s.' % (
upload['received'], offset))
try:
return Upload().handleChunk(upload, chunk, filter=True, user=user)
except IOError as exc:
if exc.errno == errno.EACCES:
raise Exception('Failed to store upload.')
raise
def uploadFromFile(self, obj, size, name, parentType=None, parent=None,
user=None, mimeType=None, reference=None,
assetstore=None, attachParent=False):
"""
This method wraps the entire upload process into a single function to
facilitate "internal" uploads from a file-like object. Example:
.. code-block:: python
size = os.path.getsize(filename)
with open(filename, 'rb') as f:
Upload().uploadFromFile(f, size, filename, 'item', parentItem, user)
:param obj: The object representing the content to upload.
:type obj: file-like
:param size: The total size of the file.
:type size: int
:param name: The name of the file to create.
:type name: str
:param parentType: The type of the parent: "folder" or "item".
:type parentType: str
:param parent: The parent (item or folder) to upload into.
:type parent: dict
:param user: The user who is creating the file.
:type user: dict
:param mimeType: MIME type of the file.
:type mimeType: str
:param reference: An optional reference string that will be sent to the
data.process event.
:param assetstore: An optional assetstore to use to store the file. If
unspecified, the current assetstore is used.
:type reference: str
:param attachParent: if True, instead of creating an item within the
parent or giving the file an itemId, set itemId to None and set
attachedToType and attachedToId instead (using the values passed in
parentType and parent). This is intended for files that shouldn't
appear as direct children of the parent, but are still associated
with it.
:type attachParent: boolean
"""
upload = self.createUpload(
user=user, name=name, parentType=parentType, parent=parent,
size=size, mimeType=mimeType, reference=reference,
assetstore=assetstore, attachParent=attachParent)
if size == 0:
return self.finalizeUpload(upload)
# The greater of 32 MB or the the upload minimum chunk size.
chunkSize = self._getChunkSize()
while True:
data = obj.read(chunkSize)
if not data:
break
upload = self.handleChunk(upload, RequestBodyStream(io.BytesIO(data), len(data)))
return upload
def addImage(self, dataset, params):
params = self._decodeParams(params)
self.requireParams(['filename', 'signature'], params)
user = self.getCurrentUser()
User().requireCreateDataset(user)
filename = params['filename'].strip()
if not filename:
raise ValidationException('Filename must be specified.',
'filename')
signature = params['signature'].strip()
if not signature:
raise ValidationException('Signature must be specified.',
'signature')
imageDataStream = RequestBodyStream(cherrypy.request.body)
imageDataSize = len(imageDataStream)
if not imageDataSize:
raise RestException('No data provided in request body.')
image = Dataset().addImage(dataset=dataset,
imageDataStream=imageDataStream,
imageDataSize=imageDataSize,
filename=filename,
signature=signature,
user=user)
# avoid circular imports from models.__init__
from isic_archive.tasks import ingestImage
ingestImage.delay(image['_id'])
return Image().filter(image, user=user)
def readChunk(self, upload, offset, params):
"""
After the temporary upload record has been created (see initUpload),
the bytes themselves should be passed up in ordered chunks. The user
must remain logged in when passing each chunk, to authenticate that
the writer of the chunk is the same as the person who initiated the
upload. The passed offset is a verification mechanism for ensuring the
server and client agree on the number of bytes sent/received.
This method accepts both the legacy multipart content encoding, as
well as passing offset and uploadId as query parameters and passing
the chunk as the body, which is the recommended method.
.. deprecated :: 2.2.0
"""
if 'chunk' in params:
chunk = params['chunk']
if isinstance(chunk, cherrypy._cpreqbody.Part):
# Seek is the only obvious way to get the length of the part
chunk.file.seek(0, os.SEEK_END)
size = chunk.file.tell()
chunk.file.seek(0, os.SEEK_SET)
chunk = RequestBodyStream(chunk.file, size=size)
else:
chunk = RequestBodyStream(cherrypy.request.body)
user = self.getCurrentUser()
if upload['userId'] != user['_id']:
raise AccessException('You did not initiate this upload.')
if upload['received'] != offset:
raise RestException(
'Server has received %s bytes, but client sent offset %s.' %
(upload['received'], offset))
try:
return self.model('upload').handleChunk(upload,
chunk,
filter=True,
user=user)
except IOError as exc:
if exc.errno == errno.EACCES:
raise Exception('Failed to store upload.')
raise
def create_file(self, event, path, root, user=None):
params = event.info["params"]
self.is_dir(path, root["_id"])
name = params["name"]
parent = Folder().filter(self.vFolder(path, root), user=user)
file_path = path / name
try:
with file_path.open(mode="a"):
os.utime(file_path.as_posix())
except PermissionError:
raise GirderException(
"Insufficient perms to write on {}".format(path.as_posix()),
"girder.api.v1.file.create-upload-failed",
)
except Exception:
raise
size = int(params["size"])
chunk = None
if size > 0 and cherrypy.request.headers.get("Content-Length"):
ct = cherrypy.request.body.content_type.value
if (
ct not in cherrypy.request.body.processors
and ct.split("/", 1)[0] not in cherrypy.request.body.processors
):
chunk = RequestBodyStream(cherrypy.request.body)
if chunk is not None and chunk.getSize() <= 0:
chunk = None
upload = Upload().createUpload(
user=user, name=name, parentType="folder", parent=parent, size=size
)
if upload["size"] > 0:
if chunk:
fobj = self._handle_chunk(upload, chunk, filter=True, user=user)
event.preventDefault().addResponse(fobj)
return
event.preventDefault().addResponse(upload)
else:
event.preventDefault().addResponse(
File().filter(self.vFile(file_path, root), user=user)
)
def registerMetadata(self, dataset, params):
params = self._decodeParams(params)
self.requireParams(['filename'], params)
user = self.getCurrentUser()
filename = params['filename'].strip()
if not filename:
raise ValidationException('Filename must be specified.', 'filename')
metadataDataStream = RequestBodyStream(cherrypy.request.body)
if not len(metadataDataStream):
raise RestException('No data provided in request body.')
Dataset().registerMetadata(
dataset=dataset, user=user, metadataDataStream=metadataDataStream, filename=filename,
sendMail=True)
# TODO: return value?
return {'status': 'success'}
def writeData(currentUser, file, data):
stream = BytesIO(str.encode(data))
chunk = RequestBodyStream(stream, size=len(data))
upload = Upload().createUploadToFile(file, currentUser, len(data))
Upload().handleChunk(upload, chunk, filter=True, user=currentUser)
return upload
def copy_file(src_file, dest_file, user):
upload = Upload().createUploadToFile(dest_file, user, src_file['size'])
Upload().handleChunk(upload=upload,
chunk=RequestBodyStream(File().open(src_file), size=dest_file['size']),
user=user)
return upload
def writeBytes(currentUser, file, bytes):
stream = BytesIO(bytes)
chunk = RequestBodyStream(stream, size=len(bytes))
upload = Upload().createUploadToFile(file, currentUser, len(bytes))
Upload().handleChunk(upload, chunk, filter=True, user=currentUser)
return upload
def postScore(self, submission, score, params):
# Ensure admin access on the containing challenge phase
phase = self.model('phase', 'covalic').load(submission['phaseId'],
user=self.getCurrentUser(),
exc=True,
level=AccessType.ADMIN)
# Record whether submission is being re-scored
rescoring = 'overallScore' in submission
# Save document to trigger computing overall score
submission.pop('overallScore', None)
submission['score'] = score
submission = self.model('submission', 'covalic').save(submission)
######################################################### step1
############################################
######### 第一次嘗試
from girder.models.upload import Upload
from girder.api.v1.file import File
from girder.utility import RequestBodyStream
import six
dest = dict(submission)
# for i in range(5000):
# dest.setdefault(i, i + 20)
dest1 = str(dest)
size = len(dest1)
user = self.model('user').load(submission['creatorId'], force=True)
#str1 = submission['title']
# str2 = '.txt'
# name = "".join((str1, str2))
name = submission['title']
parentType = 'folder'
parentId = submission['folderId']
parent = self.model(parentType).load(id=parentId,
user=user,
level=AccessType.ADMIN,
exc=True)
mimeType = u'image/png'
reference = None
File().requireParams({'size': size})
assetstore = None
chunk = None
if size > 0 and cherrypy.request.headers.get('Content-Length'):
ct = cherrypy.request.body.content_type.value
#if (ct not in cherrypy.request.body.processors and
# ct.split('/', 1)[0] not in cherrypy.request.body.processors):
# chunk = RequestBodyStream(cherrypy.request.body)
if chunk is not None and chunk.getSize() <= 0:
chunk = None
try:
# TODO: This can be made more efficient by adding
# save=chunk is None
# to the createUpload call parameters. However, since this is
# a breaking change, that should be deferred until a major
# version upgrade.
upload = Upload().createUpload(user=user,
name=name,
parentType=parentType,
parent=parent,
size=size,
mimeType=mimeType,
reference=reference,
assetstore=assetstore)
except OSError as exc:
if exc.errno == errno.EACCES:
raise GirderException(
'Failed to create upload.',
'girder.api.v1.file.create-upload-failed')
raise
if upload['size'] > 0:
if chunk:
return Upload().handleChunk(upload,
chunk,
filter=True,
user=user)
else:
return self._model.filter(Upload().finalizeUpload(upload), user)
offset = 0
if 'chunk' in params:
chunk = params['chunk']
if isinstance(chunk, cherrypy._cpreqbody.Part):
# Seek is the only obvious way to get the length of the part
chunk.file.seek(0, os.SEEK_END)
size = chunk.file.tell()
chunk.file.seek(0, os.SEEK_SET)
chunk = RequestBodyStream(chunk.file, size=size)
else:
chunk = RequestBodyStream(six.BytesIO(dest1), len(dest1))
#chunk = FileHandle(dest1)
if upload['userId'] != user['_id']:
raise AccessException('You did not initiate this upload.')
if upload['received'] != offset:
raise RestException(
'Server has received %s bytes, but client sent offset %s.' %
(upload['received'], offset))
try:
Upload().handleChunk(upload, chunk, filter=True, user=user)
except IOError as exc:
if exc.errno == errno.EACCES:
raise Exception('Failed to store upload.')
raise
##########################################################################
# Delete the scoring user's job token since the job is now complete.
token = self.getCurrentToken()
self.model('token').remove(token)
return submission