def test_media_io_base_upload_serializable(self): f = open(datafile('small.png'), 'r') upload = MediaIoBaseUpload(fd=f, mimetype='image/png') try: json = upload.to_json() self.fail('MediaIoBaseUpload should not be serializable.') except NotImplementedError: pass
def test_media_io_base_upload_streamable(self): try: import io fd = io.BytesIO('stuff') upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) self.assertEqual(True, upload.has_stream()) self.assertEqual(fd, upload.stream()) except ImportError: pass
def test_media_io_base_upload_from_file_io(self): fd = FileIO(datafile('small.png'), 'r') upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) self.assertEqual('image/png', upload.mimetype()) self.assertEqual(190, upload.size()) self.assertEqual(True, upload.resumable()) self.assertEqual(500, upload.chunksize()) self.assertEqual(b'PNG', upload.getbytes(1, 3))
def test_media_io_base_upload_from_string_io(self): f = open(datafile('small.png'), 'r') fd = StringIO.StringIO(f.read()) f.close() upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) self.assertEqual('image/png', upload.mimetype()) self.assertEqual(190, upload.size()) self.assertEqual(True, upload.resumable()) self.assertEqual(500, upload.chunksize()) self.assertEqual('PNG', upload.getbytes(1, 3)) f.close()
def test_media_io_base_upload_from_bytes(self): try: import io f = open(datafile('small.png'), 'r') fd = io.BytesIO(f.read()) upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) self.assertEqual('image/png', upload.mimetype()) self.assertEqual(190, upload.size()) self.assertEqual(True, upload.resumable()) self.assertEqual(500, upload.chunksize()) self.assertEqual('PNG', upload.getbytes(1, 3)) except ImportError: pass
def test_media_io_base_upload_streamable(self): fd = BytesIO(b'stuff') upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) self.assertEqual(True, upload.has_stream()) self.assertEqual(fd, upload.stream())
def close(self): super().close() # close the file so that it's readable for upload if self.parsedMode.writing: # google doesn't accept the fractional second part now = datetime.utcnow().replace(microsecond=0).isoformat() + "Z" onlineMetadata = {"modifiedTime": now} with open(self.localPath, "rb") as f: dataToWrite = f.read() debug(f"About to upload data: {dataToWrite}") if len(dataToWrite) > 0: upload = MediaFileUpload(self.localPath, resumable=True) if self.thisMetadata is None: debug("Creating new file") onlineMetadata.update({ "name": basename(self.path), "parents": [self.parentMetadata["id"]], "createdTime": now }) request = self.fs.drive.files().create(body=onlineMetadata, media_body=upload) else: debug("Updating existing file") request = self.fs.drive.files().update( fileId=self.thisMetadata["id"], body={}, media_body=upload) response = None while response is None: status, response = request.next_chunk() debug(f"{status}: {response}") # MediaFileUpload doesn't close it's file handle, so we have to workaround it (https://github.com/googleapis/google-api-python-client/issues/575) upload._fd.close() # pylint: disable=protected-access else: fh = BytesIO(b"") media = MediaIoBaseUpload(fh, mimetype="application/octet-stream", chunksize=-1, resumable=False) if self.thisMetadata is None: onlineMetadata.update({ "name": basename(self.path), "parents": [self.parentMetadata["id"]], "createdTime": now }) createdFile = self.fs.drive.files().create( body=onlineMetadata, media_body=media).execute( num_retries=self.fs.retryCount) debug(f"Created empty file: {createdFile}") else: updatedFile = self.fs.drive.files().update( fileId=self.thisMetadata["id"], body={}, media_body=media).execute( num_retries=self.fs.retryCount) debug(f"Updated file to empty: {updatedFile}") remove(self.localPath)
def file_create(auth, name, filename, data, parent=None): """ Checks if file with name already exists ( outside of trash ) and if not, uploads the file. Determines filetype based on filename extension and attempts to map to Google native such as Docs, Sheets, Slides, etc... For example: - ```file_create('user', 'Sample Document', 'sample.txt', BytesIO('File contents'))``` - Creates a Google Document object in the user's drive. - ```file_Create('user', 'Sample Sheet', 'sample.csv', BytesIO('col1,col2\nrow1a,row1b\n'))```` - Creates a Google Sheet object in the user's drive. See: https://developers.google.com/drive/api/v3/manage-uploads ### Args: - * auth: (string) specify 'service' or 'user' to toggle between credentials used to access - * name: (string) name of file to create, used as key to check if file exists - * filename: ( string) specified as "file.extension" only to automate detection of mime type. - * data: (BytesIO) any file like object that can be read from - * parent: (string) the Google Drive to upload the file to ### Returns: - * JSON specification of file created or existing. """ # attempt to find the file by name ( not in trash ) drive_file = file_find(auth, name, parent) # if file exists, return it, prevents obliterating user changes if drive_file: if project.verbose: print('Drive: File exists.') # if file does not exist, create it else: if project.verbose: print('Drive: Creating file.') # file mime is used for uplaod / fallback # drive mime attempts to map to a native Google format file_mime = mimetypes.guess_type(filename, strict=False)[0] drive_mime = about('importFormats')['importFormats'].get(file_mime, file_mime)[0] if project.verbose: print('Drive Mimes:', file_mime, drive_mime) # construct upload object, and stream upload in chunks body = { 'name':name, 'parents' : [parent] if parent else [], 'mimeType': drive_mime, } media = MediaIoBaseUpload( BytesIO(data or ' '), # if data is empty BAD REQUEST error occurs mimetype=file_mime, chunksize=CHUNKSIZE, resumable=True ) drive_file = API_Drive(auth).files().create( body=body, media_body=media, fields='id' ).execute() return drive_file
def _photo_file_upload(self, f_io): media = MediaIoBaseUpload(f_io, mimetype='image/jpeg') return media
def upload_docx(drive, doc_id: str, body: bytes): drive.files().update( fileId=doc_id, media_body=MediaIoBaseUpload(io.BytesIO(body), mimetype=DOCX_MIMETYPE), ).execute()
def io_to_table(auth, project_id, dataset_id, table_id, data_bytes, source_format='CSV', schema=None, skip_rows=0, disposition='WRITE_TRUNCATE', wait=True): # if data exists, write data to table data_bytes.seek(0, 2) if data_bytes.tell() > 0: data_bytes.seek(0) media = MediaIoBaseUpload(data_bytes, mimetype='application/octet-stream', resumable=True, chunksize=BIGQUERY_CHUNKSIZE) body = { 'configuration': { 'load': { 'destinationTable': { 'projectId': project_id, 'datasetId': dataset_id, 'tableId': table_id, }, 'sourceFormat': source_format, # CSV, NEWLINE_DELIMITED_JSON 'writeDisposition': disposition, # WRITE_TRUNCATE, WRITE_APPEND, WRITE_EMPTY 'autodetect': True, 'allowJaggedRows': True, 'allowQuotedNewlines': True, 'ignoreUnknownValues': True, } } } if schema: body['configuration']['load']['schema'] = {'fields': schema} body['configuration']['load']['autodetect'] = False if disposition == 'WRITE_APPEND': body['configuration']['load']['autodetect'] = False if source_format == 'CSV': body['configuration']['load']['skipLeadingRows'] = skip_rows job = API_BigQuery(auth).jobs().insert( projectId=project.id, body=body, media_body=media).execute(run=False) execution = job.execute() response = None while response is None: status, response = job.next_chunk() if project.verbose and status: print('Uploaded %d%%.' % int(status.progress() * 100)) if project.verbose: print('Uploaded 100%') if wait: job_wait(auth, execution) else: return execution # if it does not exist and write, clear the table elif disposition == 'WRITE_TRUNCATE': if project.verbose: print('BIGQUERY: No data, clearing table.') table_create(auth, project_id, dataset_id, table_id, schema)
def io_to_table(auth, project_id, dataset_id, table_id, data_bytes, source_format='CSV', schema=None, skip_rows=0, disposition='WRITE_TRUNCATE', wait=True): # if data exists, write data to table data_bytes.seek(0, 2) if data_bytes.tell() > 0: data_bytes.seek(0) media = MediaIoBaseUpload(data_bytes, mimetype='application/octet-stream', resumable=True, chunksize=BIGQUERY_CHUNKSIZE) body = { 'configuration': { 'load': { 'destinationTable': { 'projectId': project_id, 'datasetId': dataset_id, 'tableId': table_id, }, 'sourceFormat': source_format, 'writeDisposition': disposition, 'autodetect': True, 'allowJaggedRows': True, 'allowQuotedNewlines': True, 'ignoreUnknownValues': True, } } } if schema: body['configuration']['load']['schema'] = {'fields': schema} body['configuration']['load']['autodetect'] = False if disposition == 'WRITE_APPEND': body['configuration']['load']['autodetect'] = False if source_format == 'CSV': body['configuration']['load']['skipLeadingRows'] = skip_rows job = API_BigQuery(auth).jobs().insert( projectId=project.id, body=body, media_body=media).execute(run=False) execution = job.execute() response = None while response is None: status, response = job.next_chunk() if project.verbose and status: print("Uploaded %d%%." % int(status.progress() * 100)) if project.verbose: print("Uploaded 100%") if wait: job_wait(auth, job.execute()) else: return job # if it does not exist and write, clear the table elif disposition == 'WRITE_TRUNCATE': if project.verbose: print("BIGQUERY: No data, clearing table.") body = { "tableReference": { "projectId": project_id, "datasetId": dataset_id, "tableId": table_id }, "schema": { "fields": schema } } # change project_id to be project.id, better yet project.cloud_id from JSON API_BigQuery(auth).tables().insert(projectId=project.id, datasetId=dataset_id, body=body).execute()
def close(self): super().close() # close the file so that it's readable for upload if self.parsedMode.writing: # google doesn't accept the fractional second part now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' uploadMetadata = {'modifiedTime': now} if self.thisMetadata is None: uploadMetadata.update({ 'name': basename(self.path), 'parents': [self.parentMetadata['id']], 'createdTime': now }) if 'createdDateTime' in self.options: uploadMetadata.update({ 'createdTime': self.options['createdDateTime'].replace( microsecond=0).isoformat() + 'Z' }) with open(self.localPath, 'rb') as f: dataToWrite = f.read() _log.debug(f'About to upload data: {dataToWrite}') if len(dataToWrite) > 0: upload = MediaFileUpload(self.localPath, resumable=True) if self.thisMetadata is None: _log.debug('Creating new file') request = self.fs.google_resource().files().create( body=uploadMetadata, media_body=upload, **self.fs._file_kwargs, # pylint: disable=protected-access ) else: _log.debug('Updating existing file') request = self.fs.google_resource().files().update( fileId=self.thisMetadata['id'], body={}, media_body=upload, **self.fs._file_kwargs, # pylint: disable=protected-access ) response = None while response is None: status, response = request.next_chunk() _log.debug(f'{status}: {response}') else: fh = BytesIO(b'') media = MediaIoBaseUpload(fh, mimetype='application/octet-stream', chunksize=-1, resumable=False) if self.thisMetadata is None: createdFile = self.fs.google_resource().files().create( body=uploadMetadata, media_body=media, **self.fs._file_kwargs, # pylint: disable=protected-access ).execute(num_retries=self.fs.retryCount) _log.debug(f'Created empty file: {createdFile}') else: updatedFile = self.fs.google_resource().files().update( fileId=self.thisMetadata['id'], body={}, media_body=media, **self.fs._file_kwargs, # pylint: disable=protected-access ).execute(num_retries=self.fs.retryCount) _log.debug(f'Updated file to empty: {updatedFile}') remove(self.localPath)
def store_file_from_memory(self, key, memstring, metadata=None, extra_props=None, # pylint: disable=arguments-differ cache_control=None, mimetype=None): upload = MediaIoBaseUpload( BytesIO(memstring), mimetype or "application/octet-stream", chunksize=UPLOAD_CHUNK_SIZE, resumable=True ) return self._upload(upload, key, self.sanitize_metadata(metadata), extra_props, cache_control=cache_control)
def upload_file(self, io_stream, mime_type, name): file_metadata = {"name": name} media = MediaIoBaseUpload(io_stream, mime_type, -1) return self.service.files().\ create(body=file_metadata, media_body=media, fields='id').execute().get('id')
def _video_file_upload(self, f_io): media = MediaIoBaseUpload(f_io, mimetype='video/mp4') return media
with open(shop_config["json_path"], "wb+") as f: f.write(shop_json) if "tinfoil_path" in shop_config and shop_tinfoil: print(" Tinfoil:\n Location: {}\n MD5: {}".format( shop_config["tinfoil_path"], md5(shop_tinfoil).hexdigest())) with open(shop_config["tinfoil_path"], "wb+") as f: f.write(shop_tinfoil) if "file_id" in shop_config or args.c is not None: mimetype = "application/json" if shop_tinfoil: mimetype = "application/tinfoil" media = MediaIoBaseUpload(BytesIO(shop_bytes), mimetype=mimetype) if args.c is not None: body = { "name": "index.json", "mimeType": mimetype, "contentHints": { "thumbnail": { "image": urlsafe_b64encode(open("tinfoil.png", "rb").read()).decode("utf8"), "mimeType": "image/png", } }, }