def _simple_upload(self): """One-shot upload, less than 5MB""" self.buffer.seek(0) data = self.buffer.read() path = ('https://www.googleapis.com/upload/storage/v1/b/%s/o' % quote_plus(self.bucket)) metadata = {'name': self.key} if self.metadata is not None: metadata['metadata'] = self.metadata metadata = json.dumps(metadata) data = ('--==0==' '\nContent-Type: application/json; charset=UTF-8' '\n\n' + metadata + '\n--==0==' '\nContent-Type: application/octet-stream' '\n\n').encode() + data + b'\n--==0==--' r = self.gcsfs._call( 'POST', path, uploadType='multipart', headers={'Content-Type': 'multipart/related; boundary="==0=="'}, data=data) size, md5 = int(r.json()['size']), r.json()['md5Hash'] if self.consistency == 'size': assert size == self.buffer.tell(), "Size mismatch" if self.consistency == 'md5': self.md5.update(data) assert b64encode( self.md5.digest()) == md5.encode(), "MD5 checksum failed"
def _simple_upload(self): """One-shot upload, less than 5MB""" self.buffer.seek(0) data = self.buffer.read() path = "https://www.googleapis.com/upload/storage/v1/b/%s/o" % quote_plus( self.bucket) metadata = {"name": self.key} if self.metadata is not None: metadata["metadata"] = self.metadata metadata = json.dumps(metadata) data = (("--==0==" "\nContent-Type: application/json; charset=UTF-8" "\n\n" + metadata + "\n--==0==" "\nContent-Type: {0}" "\n\n").format(self.content_type).encode() + data + b"\n--==0==--") r = self.gcsfs._call( "POST", path, uploadType="multipart", headers={"Content-Type": 'multipart/related; boundary="==0=="'}, data=data, ) size, md5 = int(r.json()["size"]), r.json()["md5Hash"] if self.consistency == "size": assert size == self.buffer.tell(), "Size mismatch" if self.consistency == "md5": self.md5.update(data) assert b64encode( self.md5.digest()) == md5.encode(), "MD5 checksum failed"
def _upload_chunk(self, final=False): """ Write one part of a multi-block file upload Parameters ---------- final: bool Complete and commit upload """ self.buffer.seek(0) data = self.buffer.getvalue() head = {} l = len(data) if final and self.autocommit: if l: head['Content-Range'] = 'bytes %i-%i/%i' % ( self.offset, self.offset + l - 1, self.offset + l) else: # closing when buffer is empty head['Content-Range'] = 'bytes */%i' % self.offset data = None else: if l < GCS_MIN_BLOCK_SIZE: if not self.autocommit: return elif not final: raise ValueError("Non-final chunk write below min size.") head['Content-Range'] = 'bytes %i-%i/*' % ( self.offset, self.offset + l - 1) head.update({'Content-Type': 'application/octet-stream', 'Content-Length': str(l)}) r = self.gcsfs._call('POST', self.location, uploadType='resumable', headers=head, data=data) if 'Range' in r.headers: end = int(r.headers['Range'].split('-')[1]) shortfall = (self.offset + l - 1) - end if shortfall: if self.consistency == 'md5': self.md5.update(data[:-shortfall]) self.buffer = io.BytesIO(data[-shortfall:]) self.buffer.seek(shortfall) self.offset += l - shortfall return False else: if self.consistency == 'md5': self.md5.update(data) elif l: # assert final, "Response looks like upload is over" size, md5 = int(r.json()['size']), r.json()['md5Hash'] if self.consistency == 'size': assert size == self.buffer.tell() + self.offset, "Size mismatch" if self.consistency == 'md5': assert b64encode( self.md5.digest()) == md5.encode(), "MD5 checksum failed" else: assert final, "Response looks like upload is over" return True
def _upload_chunk(self, final=False): self.buffer.seek(0) data = self.buffer.read() head = self.gcsfs.header.copy() l = self.buffer.tell() if final: if l: head['Content-Range'] = 'bytes %i-%i/%i' % ( self.offset, self.offset + l - 1, self.offset + l) else: # closing when buffer is empty head['Content-Range'] = 'bytes */%i' % self.offset data = None else: head['Content-Range'] = 'bytes %i-%i/*' % (self.offset, self.offset + l - 1) head.update({ 'Content-Type': 'application/octet-stream', 'Content-Length': str(l) }) r = requests.post(self.location, params={'uploadType': 'resumable'}, headers=head, data=data) validate_response(r, self.location) if 'Range' in r.headers: assert not final, "Response looks like upload is partial" shortfall = (self.offset + l - 1) - int( r.headers['Range'].split('-')[1]) if shortfall: if self.consistency == 'md5': self.md5.update(data[:-shortfall]) self.buffer = io.BytesIO(data[-shortfall:]) self.buffer.seek(shortfall) else: if self.consistency == 'md5': self.md5.update(data) self.buffer = io.BytesIO() self.offset += l - shortfall else: assert final, "Response looks like upload is over" size, md5 = int(r.json()['size']), r.json()['md5Hash'] if self.consistency == 'size': assert size == self.buffer.tell( ) + self.offset, "Size mismatch" if self.consistency == 'md5': assert b64encode( self.md5.digest()) == md5.encode(), "MD5 checksum failed" self.buffer = io.BytesIO() self.offset += l
def _simple_upload(self): """One-shot upload, less than 5MB""" self.buffer.seek(0) data = self.buffer.read() path = ('https://www.googleapis.com/upload/storage/v1/b/%s/o' % quote_plus(self.bucket)) r = self.gcsfs.session.post(path, params={ 'uploadType': 'media', 'name': self.key }, data=data) validate_response(r, path) size, md5 = int(r.json()['size']), r.json()['md5Hash'] if self.consistency == 'size': assert size == self.buffer.tell(), "Size mismatch" if self.consistency == 'md5': self.md5.update(data) assert b64encode( self.md5.digest()) == md5.encode(), "MD5 checksum failed"
def _upload_chunk(self, final=False): """ Write one part of a multi-block file upload Parameters ---------- final: bool Complete and commit upload """ self.buffer.seek(0) data = self.buffer.getvalue() head = {} l = len(data) if final and self.autocommit: if l: head["Content-Range"] = "bytes %i-%i/%i" % ( self.offset, self.offset + l - 1, self.offset + l, ) else: # closing when buffer is empty head["Content-Range"] = "bytes */%i" % self.offset data = None else: if l < GCS_MIN_BLOCK_SIZE: if not self.autocommit: return False elif not final: raise ValueError("Non-final chunk write below min size.") head["Content-Range"] = "bytes %i-%i/*" % (self.offset, self.offset + l - 1) head.update({"Content-Type": self.content_type, "Content-Length": str(l)}) r = self.gcsfs._call( "POST", self.location, uploadType="resumable", headers=head, data=data ) if "Range" in r.headers: end = int(r.headers["Range"].split("-")[1]) shortfall = (self.offset + l - 1) - end if shortfall: if self.consistency == "md5": self.md5.update(data[:-shortfall]) self.buffer = io.BytesIO(data[-shortfall:]) self.buffer.seek(shortfall) self.offset += l - shortfall return False else: if self.consistency == "md5": self.md5.update(data) elif l: # assert final, "Response looks like upload is over" size, md5 = int(r.json()["size"]), r.json()["md5Hash"] if self.consistency == "size": # update offset with final chunk of data self.offset += l assert size == self.buffer.tell() + self.offset, "Size mismatch" if self.consistency == "md5": # update md5 with final chunk of data self.md5.update(data) assert ( b64encode(self.md5.digest()) == md5.encode() ), "MD5 checksum failed" else: assert final, "Response looks like upload is over" return True