def upload_file(service, title, description, parent_id, mime_type, filename): """Insert new file. Args: service: Drive API service instance. title: Title of the file to insert, including the extension. description: Description of the file to insert. parent_id: Parent folder's ID. mime_type: MIME type of the file to insert. filename: Filename of the file to insert. Returns: Inserted file metadata if successful, None otherwise. """ media_body = http.MediaFileUpload(filename, mimetype=mime_type, resumable=True) body = {'title': title, 'description': description, 'mimeType': mime_type} # Set the parent folder. if parent_id: body['parents'] = [{'id': parent_id}] try: file = service.files().insert(body=body, media_body=media_body).execute() # Uncomment the following line to print the File ID print 'File ID: %s' % file['id'] except errors.HttpError, error: print 'An error occured: %s' % error return INT_FAILURE_UPLOAD, None
def push(self, name): body = {'name': name, 'parents': [self.folder_id]} media_body = http.MediaFileUpload(name) file = self.files.create(body=body, media_body=media_body, fields='id,parents').execute() self.file_id = file.get('id')
def upload_video(self, video: str, properties: dict, progress: callable = None, *args) -> dict: self.progress = progress self.progress_args = args self.video = video self.properties = properties body = dict( snippet=dict( title=self.properties.get("title"), description=self.properties.get("description"), categoryId=self.properties.get("category"), ), status=dict(privacyStatus=self.properties.get("privacyStatus")), ) media_body = http.MediaFileUpload( self.video, chunksize=self.chunksize, resumable=True, ) self.request = self.youtube.videos().insert(part=",".join(body.keys()), body=body, media_body=media_body) self._resumable_upload() return self.response
def send_to_drive(self, processed_file): """Send a processed file and its content to the drive""" folder_id = self.find_or_create_out_folder() for sub_file in processed_file.ProcessedFile.Files: name = processed_file.Name + "_" + sub_file.Name + ".csv" self.delete_file_if_exists(folder_id, name) if not os.path.exists(self.temp_directory): os.makedirs(self.temp_directory) tmp_file_path = os.path.join(self.temp_directory, name) file_metadata = {'name': name, 'parents': [folder_id]} # Generates a file in the disk to upload to the drive sub_file.generate_csv_content_file(tmp_file_path) media_body = http.MediaFileUpload(tmp_file_path, "text/csv") try: self.drive_service.files().create(body=file_metadata, media_body=media_body, fields='id').execute() media_body.stream().close() except Exception as e: print("Error while sending the CSV file: " + e) finally: os.remove(tmp_file_path)
def initialize_upload(youtube, options): tags = None if options.keywords: tags = options.keywords.split(",") body = dict(snippet=dict(title=options.title, description=options.description, tags=tags, categoryId=options.category), status=dict(privacyStatus=options.privacyStatus)) # Call the API's videos.insert method to create and upload the video. insert_request = youtube.videos().insert( part=",".join(body.keys()), body=body, # The chunksize parameter specifies the size of each chunk of data, in # bytes, that will be uploaded at a time. Set a higher value for # reliable connections as fewer chunks lead to faster uploads. Set a lower # value for better recovery on less reliable connections. # # Setting "chunksize" equal to -1 in the code below means that the entire # file will be uploaded in a single HTTP request. (If the upload fails, # it will still be retried where it left off.) This is usually a best # practice, but if you're using Python older than 2.6 or if you're # running on App Engine, you should set the chunksize to something like # 1024 * 1024 (1 megabyte). media_body=http.MediaFileUpload(options.file, chunksize=-1, resumable=True)) resumable_upload(insert_request)
def upload_file(self, file_path, parent_folder_id=None): """ Upload a file to Google Drive in a "chunked" manner :param file_path: the local path to the file being uploaded :param parent_folder_id: the id of the folder where this file should be placed :returns: the API's response object """ self.logger.log_verbose("Attempting upload:" + file_path + " to " + str(parent_folder_id)) media = http.MediaFileUpload(file_path, resumable=True) file_metadata = {'name': os.path.basename(file_path)} if parent_folder_id is not None: file_metadata['parents'] = [parent_folder_id] request = self.DRIVE_SERVICE.files().create(body=file_metadata, media_body=media) response = None while response is None: status, response = request.next_chunk() if status: self.logger.log_verbose("Uploaded {:d}%.".format( int(status.progress() * 100))) self.logger.log_verbose("Upload complete: " + file_path) return response
def publish_video(self, video, title='', description='', tags=None, private=True, **kwargs): try: logger.info('trying to publish youtube video, for user: %s' % self.user) insert_request = self.youtube.videos().insert( part='snippet,status', body=dict( snippet=dict( title=title, description=description, tags=tags ), status=dict( privacyStatus='private' if private else 'public' ) ), media_body=http.MediaFileUpload(video.path, chunksize=-1, resumable=True) ) result = self._resumable_upload(insert_request, title) logger.info(str(result)) return result except Exception as e: logger.error(e) raise e
def upload(self, content, name, location, mime='application/pdf'): print(content, name, location) """Upload file to google drive""" file_metadata = {'name': name, 'parents': [location]} media = http.MediaFileUpload(content, mimetype=mime) file = service.files().create(body=file_metadata, media_body=media, fields='id').execute() return file.get('id')
def upload_report_to_drive(self): file_metadata = { 'name': self.start_datetime.strftime('%Y%m%d_%H%M%S'), 'parents': [self.drive_dir_id] } media = http.MediaFileUpload(self.report_local_path, mimetype='application/pdf') self.drive_service.files().create(body=file_metadata, media_body=media).execute()
def putFile(service, folderID, filePath, verbose=False): """Creates a File on a google drive folder. Args: service: google drive service instance to use folderID: Parent Folder's ID in which to create the new file filePath: Path to the file that needs to be put on google drive verbose: print debugging information Returns: File name and ID of the newly created File or error if error occured To Do: Check if File and folder exist Possible check if File is not 0 length before putting it up on google drive """ drive_service = service f = os.path.basename(filePath) mime_type = '' # Getting Env info to use for metadata tagging atc = os.getenv('ATC_EXTERNAL_URL', 'DEFAULT_ATC') pipe = os.getenv('BUILD_PIPELINE_NAME', 'DEFAULT_PIPELINE') job = os.getenv('BUILD_JOB_NAME', 'DEFAULT_JOB') build = os.getenv('BUILD_NAME', 'DEFAULT_BUILD_NUM') # Creteing a link URL to put in metadata link = atc + '/pipelines/' + pipe + '/jobs/' + job + '/builds/' + build media_body = http.MediaFileUpload(filePath, mimetype=mime_type, resumable=True) body = { 'title': f, 'description': 'Uploaded by concourse task: \n\n' + link + '', 'mimeType': mime_type } # Set the parent Folder if folderID: body['parents'] = [{'id': folderID}] #Temporarily adding perms here #body['permissions'] = perms if verbose: #print('Body: ' .format(dir(body))) print('Body: {0}'.format(body), file=sys.stderr) try: file = drive_service.files().insert(body=body, media_body=media_body).execute() #Printing for debug only #print ('File ID {0} ' .format(file.get('id'))) return file except errors.HttpError, error: #print 'An error occured: %s' % error return error
def upload_file(service, filename, mimeType): body = {'title': filename, 'description': "DESC OF " + filename} media_body = http.MediaFileUpload(filename, mimetype=mimeType, resumable=True) request = service.files().insert(body=body, media_body=media_body) response = None while response is None: status, response = request.next_chunk() if status: print("Uploaded %d%%." % int(status.progress() * 100)) print("Upload Complete!")
def upload(self, filepath, folder, filename=None): """Upload a file to a specific folder.""" if filename is None: filename = os.path.basename(filepath) mime_type, _ = mimetypes.guess_type(filename) media = http.MediaFileUpload(filepath, mimetype=mime_type, resumable=True) metadata = { 'name': filename, 'parents': [folder], } self.service.files().create(body=metadata, media_body=media).execute()
def _insert(self, new_item, feed_item): """Handles the upload of creative assets to DCM and the creation of the associated entity. This method makes a call to the DCM API to create a new entity. Args: new_item: The item to insert into DCM. feed_item: The feed item representing the creative asset from the Bulkdozer feed. Returns: The newly created item in DCM. """ local_file = os.path.join( '/tmp', feed_item.get(FieldMap.CREATIVE_ASSET_FILE_NAME, None)) self._download_from_gcs( feed_item.get(FieldMap.CREATIVE_ASSET_BUCKET_NAME, None), feed_item.get(FieldMap.CREATIVE_ASSET_FILE_NAME, None), local_file, auth=self.auth) media = http.MediaFileUpload(local_file) if not media.mimetype(): mimetype = 'application/zip' if asset_type == 'HTML' else 'application/octet-stream' media = http.MediaFileUpload(asset_file, mimetype) result = self._retry( self._service.insert(profileId=self.profile_id, advertiserId=feed_item.get( FieldMap.ADVERTISER_ID, None), media_body=media, body=new_item)) os.remove(local_file) return result
def upload_image(): service = get_service('drive', 'v3', DRIVE_SCOPES) media = http.MediaFileUpload(config['final_image']['save_path'], mimetype='image/png', resumable=True) metadata = {'name': add_timestamp(config['google_service']['portrait_name']), 'parents': [config['google_service']['portrait_folder_id']]} file_id = service.files().create(body=metadata, media_body=media, fields='id').execute()['id'] logger.info('Uploaded image to Drive') data['endo_map_url'] = get_shareable_link(service, file_id)
def upload(self, filepath, folder): """Upload a file to a specific folder.""" filename = os.path.basename(filepath) mime_type = 'application/pdf' media = http.MediaFileUpload(filepath, mimetype=mime_type, resumable=True) metadata = { 'name': filename, 'parents': [folder], } self.service.files().create(body=metadata, media_body=media).execute()
def upload_file(self, file_name, file_path, mime_type, team_drive=False, team_drive_id=None): file_metadata = {'name': file_name} media = http.MediaFileUpload(file_path, mimetype=mime_type) fl = None if not team_drive: fl = self.files.create(body=file_metadata, media_body=media, fields='id').execute() elif team_drive and team_drive_id is not None: payload = { 'parents': [team_drive_id], 'name': file_name } fl = self.files.create(supportsAllDrives=True, media_body=media, body=payload).execute() elif team_drive and team_drive_id is None: raise ValueError("team_drive_id should have value if this is a team drive upload") return fl
def upload_file(self, file_path, folder_id=''): """ Upload a file to a specified directory. :param file_path: Absolute or relative path to the file :param folder_id: The id of the directory to which the file is to be uploaded :return: """ (directory, filename) = os.path.split(file_path) (mime, encoding) = mimetypes.guess_type(file_path) if mime is None: mime = "application/octet-stream" media = http.MediaFileUpload(file_path, mimetype=mime, resumable=True) body = { 'name': filename, 'parents': [folder_id] } self.service.files().create(body=body, media_body=media).execute()
def process(self, element): import tensorflow as tf from google.oauth2 import service_account from googleapiclient.discovery import build from apiclient import http import csv import os elements = list(csv.reader([element.encode("utf-8")], delimiter=","))[0] gcs_file_path, gdrive_file_name = elements # gcs_file_path, gdrive_file_name = element.split(",") # gcs_file_path, gdrive_file_name = gcs_file_path.encode("utf-8"), gdrive_file_name.encode("utf-8") # Read file in Cloud Storage with tf.gfile.Open(gcs_file_path, "rb") as f: file_content = f.read() # Save file to local disk print(gcs_file_path) print(gdrive_file_name) tf.gfile.MakeDirs("tmp") with tf.gfile.Open("tmp/{}".format(gdrive_file_name), "w") as f: f.write(file_content) # Download service account file print(self.service_account_file.get()) with tf.gfile.Open(self.service_account_file.get(), "rb") as f: file_content = f.read() with tf.gfile.Open("service_account.json", "w") as f: f.write(file_content) # Upload file to Google Drive scopes = ["https://www.googleapis.com/auth/drive.file"] credentials = service_account.Credentials.from_service_account_file( "service_account.json", scopes=scopes) service = build("drive", "v3", credentials=credentials, cache_discovery=False) media_body = http.MediaFileUpload("tmp/{}".format(gdrive_file_name)) body = { "name": gdrive_file_name, "parents": [self.gdrive_directory_id.get()], } print(body) file_ = service.files().create(body=body, fields="id", media_body=media_body).execute()
def upload_backup(self, filename): """ Upload backup file to GDrive """ print_log('Uploading backup file is started') metadata = dict(name=filename.split('/').pop(), parents=[self.drive_directory]) file_body = http.MediaFileUpload(filename, 'application/octet-stream', chunksize=1024 * 1024 * 2, resumable=True) try: self.service.files().create(body=metadata, media_body=file_body).execute() except http.HttpError: print_log('Uploading backup file is failed') return False else: print_log('Uploading backup file is finished successfully') return True
def upload_file(self, from_filename, to_paths, to_filename, mime_type="text/csv", parent_id='root', max_retry=10, retry_interval=10, overwrite=True): """ Upload a file. :param from_filename: :param to_paths: A list of directory [[path, to, dir], [path, to, dir], ...] :param to_filename: Filename :param mime_type: :param parent_id: Parent folder ID :param max_retry: Max number of retry :param retry_interval: Interval second between retries :param overwrite: Bool :return: """ # validations if not from_filename: raise Exception("Invalid from_filename: %s" % from_filename) if not isinstance(to_paths, list): raise Exception("Invalid to_paths: %s" % to_paths) if not to_filename: raise Exception("filename not found: %s" % to_filename) # media media = http.MediaFileUpload(from_filename, mimetype=mime_type, chunksize=1024 * 1024, resumable=True) return call_with_retry( self._upload, (media, to_paths, to_filename, mime_type, parent_id, overwrite), max_retry, retry_interval)
def upload_file(self, __name: str, __path: Path, __folder_id: str) -> str: """ Upload a file to the Drive. MimeType gets automatically. :param __name: string, name, the file will have in Drive. :param __path: Path, file path. :param __folder_id: string, folder's ID, to which the file will be uploaded. :return: string, uploaded file's ID. :exception Trouble: if the file does not exist. """ if not __path.exists(): raise FileNotFoundError("File to upload not found") file_metadata = {'name': __name, 'parents': [__folder_id]} # mime type getting m_type = comm_func.mime_type(__path) media = http.MediaFileUpload(__path, mimetype=m_type) _file = self.__drive.files().create(body=file_metadata, media_body=media, fields='id').execute() return _file.get('id')
def upload_file(self, file_path, file_name, parent_id=None): """ Create a file in your Google Drive. :param file_path: A path of the file being created :param file_name: A name of the file being created :param parent_id: Default: None. If None, the file being created will be at My Drive of your Google Drive. Otherwise, the file being created will go under the parent folder id passed as a parameter. :return: an id of the created file """ file_metadata = { 'name': file_name, 'copyRequiresWriterPermission': True } if parent_id is not None: file_metadata['parents'] = [parent_id] media = http.MediaFileUpload(file_path, resumable=True) f = self.drive_service.files().create(body=file_metadata, media_body=media, fields='id').execute() print('{} File ID: {}'.format(file_name, f.get('id'))) return f.get('id')
def _upload_one(item): """The engine behind |BigQuery.upload|. Args: item: An _UploadItem to upload. """ context = item.context logging.debug('Uploading data to (project:%s, datatset:%s, table:%s) ' 'from %s with schema %s' % (context.project_id, context.dataset_id, context.table_id, item.data_path, context.schema_path)) size = os.stat(item.data_path).st_size if size == 0: logging.info('No data found. Skipping upload.') return if size > 10 * 1024 * 1024: logging.error('File to upload too large (%s MB). ' 'BigQuery limit is 10 MB. Upload will likely fail.' % (size,)) # Post to the jobs resource using the client's media upload interface. # See: # http://developers.google.com/api-client-library/ # python/guide/media_upload # Provide a configuration object. See: # https://cloud.google.com/bigquery/docs/reference/v2/jobs#resource body = { 'configuration': { 'load': { 'schema': { 'fields': json.load(open(context.schema_path, 'r')) }, 'destinationTable': { 'projectId': context.project_id, 'datasetId': context.dataset_id, 'tableId': context.table_id }, 'sourceFormat': context.data_format, } } } media_body = http.MediaFileUpload( item.data_path, mimetype='application/octet-stream') with context.lock: insert_request = context.bigquery.jobs().insert( projectId=context.project_id, body=body, media_body=media_body) job = insert_request.execute() status_request = context.bigquery.jobs().get( projectId=job['jobReference']['projectId'], jobId=job['jobReference']['jobId']) # Poll the job until it finishes. while True: with context.lock: result = status_request.execute(num_retries=2) status = result['status'] if status['state'] != 'DONE': time.sleep(1) continue if 'errorResult' not in status: break msg = ('Error when updating table %s with rows from %s using ' 'schema %s' % (context.table_id, item.data_path, context.schema_path)) error = status['errorResult'] if 'debugInfo' in error: del error['debugInfo'] logging.error(pprint.pformat(error)) raise BigQueryException(msg)
def upload_file(service, input_file, output_name=None, folder_name=None, show_progress=False, chunksize=1048576): """ Upload file to gdrive using MediaFileUpload that support large file upload and resumable Display upload speed and progress when call this function :param service: gdrive api service :param input_file: full path or relative path to source file to upload :param output_name: target filename on gdrive, if none, use same name as source filename :param folder_name: target folder name to store file. this folder will create under root folder on gdrive :return: File instance from google api or None if error occur """ # filename, ext = os.path.splitext(input_file) mime_type = 'application/octet-stream' if os.path.isfile(input_file): file_size = os.path.getsize(input_file) else: logger.error("Error " + input_file + " : raw file not found") return if not output_name: output_name = os.path.basename(input_file) body = { 'name': output_name, } if folder_name: parent_id = get_or_create_folder(service, folder_name) if parent_id: body['parents'] = [parent_id] else: logger.error('No value from get_or_create_folder: %s', folder_name) return logger.debug('Prepare file to upload [%s] mime[%s] chunk[%s] body[%s]', input_file, mime_type, chunksize, body) media = api_http.MediaFileUpload(input_file, mimetype=mime_type, chunksize=chunksize, resumable=True) request = service.files().create(media_body=media, body=body) logger.info("Upload : %s, Size : %s", input_file, file_size) start_time = time.time() status = None response = None last_progress = 0 idle_count = 0 while response is None: if idle_count >= 10: logger.error('Max idle retry for upload') break try: status, response = request.next_chunk() except api_errors.HttpError as error: if error.resp.status in [404]: # Start the upload all over again. logger.error( 'Upload [%s] fail http 404 retry all over again: %s' % (input_file, error)) break elif error.resp.status in [403, 500, 502, 503, 504]: idle_count += 1 # Call next_chunk() again, but use an exponential backoff for repeated errors. logger.warn( 'Upload [%s] fail http [%s] retry [%s] next_chunk: %s' % (input_file, error.resp.status, idle_count, error)) time.sleep(idle_count * idle_count * 3) continue else: # Do not retry. Log the error and fail. logger.error( 'Upload [%s] fail http [%s] error in next_chunk: %s', (input_file, error.resp.status, error)) break except socket.error as error: idle_count += 1 logger.warn( 'Upload [%s] fail [socket.error] retry [%s] next_chunk: %s' % (input_file, idle_count, error)) time.sleep(idle_count * idle_count * 3) continue except: logger.error( 'Upload [%s] fail An error occurred in next_chunk: [%s] %s ', input_file, sys.exc_info()[0], sys.exc_info()[1]) break if status: if last_progress >= status.resumable_progress: logger.warn( 'Upload [%s] retry [%s] next_chunk: No progress from last chunk' % (input_file, idle_count)) else: idle_count = 0 if show_progress: upload_speed = int(status.resumable_progress / (1024 * (time.time() - start_time))) logger.debug( "Uploaded {}% - ({:,}/{:,}) - Avg {:,} Kps".format( int(status.progress() * 100), status.resumable_progress, status.total_size, upload_speed)) print("Uploaded {}% - ({:,}/{:,}) - Avg {:,} Kps".format( int(status.progress() * 100), status.resumable_progress, status.total_size, upload_speed), end='\r') last_progress = status.resumable_progress else: idle_count += 1 logger.warn( 'Upload [%s] retry [%s] next_chunk: Not receive status from status, response = request.next_chunk()' % (input_file, idle_count)) if response: logger.info("Upload {} Complete! -- {:,} seconds".format( input_file, time.time() - start_time)) return response else: logger.error("Upload {} Error! -- {:,} seconds".format( input_file, time.time() - start_time)) return False
from httplib2 import Http from oauth2client import file, client, tools import sys from time import gmtime, strftime SCOPES = ('https://www.googleapis.com/auth/drive', 'https://www.googleapis.com/auth/drive.appdata', 'https://www.googleapis.com/auth/drive.file', 'https://www.googleapis.com/auth/drive.readonly.metadata') store = file.Storage('/home/mpiuser/blocktime/datafiles/storage.json') creds = store.get() if not creds or creds.invalid: flow = client.flow_from_clientsecrets( '/home/mpiuser/blocktime/datafiles/client_id.json', SCOPES) creds = tools.run_flow(flow, store) drive = discovery.build('drive', 'v3', http=creds.authorize(Http())) files = drive.files().list().execute().get('files', []) for i in files: if (i["name"] == "BlockTimeFiles"): dirId = i["id"] break file_metadata = {'name': strftime("%m-%d %H:%M") + '.csv', "parents": [dirId]} media = http.MediaFileUpload('/home/mpiuser/blocktime/datafiles/timesdata.csv', mimetype='text/csv') file = drive.files().create(body=file_metadata, media_body=media, fields='id') file.execute()
def post(self, request): auth = request.data.pop('auth') title = request.data.pop('title') dct = request.data date = datetime.datetime.now().strftime("%y/%m/%d") AUTH = auth SCOPES = [ 'https://www.googleapis.com/auth/drive.file', 'https://www.googleapis.com/auth/documents' ] CLIENT_SECRET_FILE = 'DocsAPI/client_secrets.json' creds = client.credentials_from_clientsecrets_and_code( CLIENT_SECRET_FILE, SCOPES, AUTH) #import template drive_service = discovery.build('drive', 'v3', credentials=creds) file_metadata = { 'name': title, 'mimeType': 'application/vnd.google-apps.document' } media = http.MediaFileUpload('DocsAPI/template#1.html', mimetype='text/html', resumable=True) file = drive_service.files().create(body=file_metadata, media_body=media).execute() service = discovery.build('docs', 'v1', credentials=creds) requests = [] for i in dct: replace = dct[i] requests.append({ 'replaceAllText': { 'containsText': { 'text': '{{' + i + '}}', 'matchCase': 'true' }, 'replaceText': replace, } }) requests.append({ 'replaceAllText': { 'containsText': { 'text': '{{date}}', 'matchCase': 'true' }, 'replaceText': str(date), } }) fileId = file.get('id') result = service.documents().batchUpdate(documentId=fileId, body={ 'requests': requests }).execute() return Response( { "detail": "https://docs.google.com/document/d/" + file.get("id") + "/edit" }, status=HTTP_200_OK)