def save_pdf(self,name_on_disk,name_in_drive): """ Saves a PDF from the disk to the Google Drive :param name_on_disk: the name of the PDF to upload :type name_on_disk: str :param name_in_drive: the PDFs name on the Drive :type name_in_drive: str """ tfid = None for key in self.subitem_dict: if key == name_in_drive: tfid = self.subitem_dict[key] break if tfid is None: media_body = media.MediaFileUpload(name_on_disk,resumable=True,mimetype="application/pdf") metadata={ "name": name_in_drive, "parents": [self.folder_id] } self.service.files().create( body = metadata, media_body = media_body).execute() self.refresh_dict() else: media_body = media.MediaFileUpload(name_on_disk,resumable=True) self.service.files().update( fileId = tfid, media_body = media_body).execute()
def _upload_file_to_object(self, local_file_path: str, bucket_name: str, object_name: str): """Upload the contents of a local file to an object in a GCS bucket.""" media_body = http.MediaFileUpload(local_file_path) body = {'name': object_name} request = self._storage_service.objects().insert(bucket=bucket_name, body=body, media_body=media_body) try: response = request.execute(num_retries=5) if 'name' not in response: raise CloudStorageError( 'Unexpected responses when uploading file "{}" to ' 'bucket "{}"'.format(local_file_path, bucket_name)) except errors.HttpError as e: if e.resp.status == 403: raise CloudStorageError( 'You do not have permission to upload files to ' 'bucket "{}"'.format(bucket_name)) elif e.resp.status == 404: raise CloudStorageError( 'Bucket "{}" not found.'.format(bucket_name)) else: raise CloudStorageError( 'Unexpected error when uploading file "{}" to ' 'bucket "{}"'.format(local_file_path, bucket_name)) from e # http.MediaFileUpload opens a file but never closes it. So we # need to manually close the file to avoid "ResourceWarning: # unclosed file". # TODO: Remove this line when # https://github.com/googleapis/google-api-python-client/issues/575 # is resolved. media_body.stream().close()
def UploadFile(self, local_file_name, gcs_file_name, mimetype='application/octet-stream'): """Upload a file to GCS. Args: local_file_name: Path to the file on the client's machine. gcs_file_name: Name with which to store the file on GCS. It can be a full path (e.g., gs://makani/file), or a path relative to the preset default bucket. mimetype: MIME type of the object. Returns: Response to the Google Client API's download request. """ resumable = os.stat(local_file_name).st_size > 0 media = gapi_http.MediaFileUpload(local_file_name, mimetype=mimetype, resumable=resumable) # gsutil's code suggests that 404s and 410s are retryable for resumable # uploads (see ResumableUploadStartOverException). def _ErrorMatcher(error): return (self._CommonErrorMatcher(error) or (isinstance(error, gapi_errors.HttpError) and error.resp.status in (404, 410))) return self._RunWithRetries( lambda: self._UploadWithProgress(media, gcs_file_name), _ErrorMatcher)
def google_drive_write(bucket: str, path: str, file_name: str, creds: str): """ Upload file to Google Drive. File is stored in a parent folder which name is that of 'path' without the '/'. Parameters bucket: Unused parameter. path (str): path = '{prefix}/{exchange}/{data_type}/{pair}/ {exchange}-{data_type}-{pair}-{int(timestamp)}.parquet' String from which is derived folder name into which is written the file, as well as the root folder that will contain all these folders. file_name (str): File name preceded by path on local disk. creds (str): Path to credential file. """ # Retrieve folder ID to be used to write the file into, and upload. # If not existing, folder will be created. drive = _get_drive(creds) folder_id, folder_name = _get_folder_in_parent(drive, path) media = gah.MediaFileUpload(file_name, resumable=True) file_metadata = {'name': path.split('/')[-1], 'parents': [folder_id]} request = drive.files().create(body=file_metadata, media_body=media, fields='id') response = None while response is None: status, response = request.next_chunk(num_retries=4) return
def insert(self, file_path, mime_type='application/csv'): """Stores a new object and metadata. For tips on uploading to Cloud Storage, see this link : - https://cloud.google.com/storage/docs/best-practices?hl=fr#uploading For examples of performing object uploads with different Cloud Storage tools and client libraries, see the Uploading Objects guide. This method supports an /upload URI and accepts uploaded media with the following characteristics: - Maximum file size: 5 TB - Accepted Media MIME types: */* - The authenticated user must have WRITER permissions on the bucket. Note: Metadata-only requests are not allowed. To change an object's metadata, use either the update or patch methods. To provide a customer-supplied encryption key along with the object upload, use the headers listed on the Encryption page in your request. Args: self : Authorized Cloud Storage API service instance. file_path: Path of your File mime_type: Global description about your File Returns: """ try: media = http.MediaFileUpload(file_path, mimetype=mime_type, resumable=True) response = self.__client.objects().insert(bucket=self.__bucket_id, name=file_path, media_body=media ).execute() return response except Exception as exception: print(exception)
def _upload_file(self, local_abs_file_path, remote_parent_id, remote_name): """ :param local_abs_file_path pathlib.Path | str: absolute path pointing to the file. :param remote_parent_id str: remote id of the folder where the file will reside. :param remote_name str: the new name the file will have on the server. :return str | None: remote id of the uploaded file. None if failed. """ media = http.MediaFileUpload(local_abs_file_path, resumable=True) meta = {"name": remote_name, "parents": [remote_parent_id]} request = self._file_service.create(body=meta, media_body=media, fields="id") try: # execute response = None while response is None: # keep executing chunk by chunk until response means its done. response = request.next_chunk()[1] except Exception as error: self.app.logger.error( "upload_file: {name} : {error_type} : {error_description}, ignoring..." .format( name=remote_name, error_type=type(error).__name__, error_description=error, )) return None self.app.logger.log( 9, "file uploaded, remote id: {}".format(response["id"])) return response["id"]
def upload_file(storage_service, bucket, bucket_path, file_name, verbose=True): '''get_folder will return the folder with folder_name, and if create=True, will create it if not found. If folder is found or created, the metadata is returned, otherwise None is returned :param storage_service: the drive_service created from get_storage_service :param bucket: the bucket object from get_bucket :param file_name: the name of the file to upload :param bucket_path: the path to upload to ''' # Set up path on bucket upload_path = "%s/%s" % (bucket['id'], bucket_path) if upload_path[-1] != '/': upload_path = "%s/" % (upload_path) upload_path = "%s%s" % (upload_path, os.path.basename(file_name)) body = {'name': upload_path} # Create media object with correct mimetype if os.path.exists(file_name): mimetype = sniff_extension(file_name, verbose=verbose) media = http.MediaFileUpload(file_name, mimetype=mimetype, resumable=True) request = storage_service.objects().insert(bucket=bucket['id'], body=body, predefinedAcl="publicRead", media_body=media) result = request.execute() return result bot.warning('%s requested for upload does not exist, skipping' % file_name)
def put(self, filename, dest_path, mimetype=None, chunksize=None): chunksize = chunksize or self.chunksize resumable = os.path.getsize(filename) > 0 mimetype = mimetype or mimetypes.guess_type(dest_path)[0] or DEFAULT_MIMETYPE media = http.MediaFileUpload(filename, mimetype, chunksize=chunksize, resumable=resumable) self._do_put(media, dest_path)
def InsertData(self, table_id, fd, schema, job_id): """Insert data into a bigquery table. If the table specified doesn't exist, it will be created with the specified schema. Args: table_id: string table id fd: open file descriptor containing the newline separated JSON schema: BigQuery schema dict job_id: string job id Returns: API response object on success, None on failure """ configuration = { "schema": { "fields": schema }, "destinationTable": { "projectId": self.project_id, "tableId": table_id, "datasetId": self.dataset_id }, "sourceFormat": "NEWLINE_DELIMITED_JSON", } body = { "configuration": { "load": configuration }, "jobReference": { "projectId": self.project_id, "jobId": job_id } } # File content can be gzipped for bandwidth efficiency. The server handles # it correctly without any changes to the request. mediafile = http.MediaFileUpload(fd.name, mimetype="application/octet-stream") job = self.service.jobs().insert(projectId=self.project_id, body=body, media_body=mediafile) try: response = job.execute() return response except errors.HttpError as e: if self.GetDataset(self.dataset_id): logging.exception("Error with job: %s", job_id) else: # If this is our first export ever, we need to create the dataset. logging.info("Attempting to create dataset: %s", self.dataset_id) self.CreateDataset() return self.RetryUpload(job, job_id, e)
def upload(self): file_metadata = {'name': self.fileName, 'parents' : [self.gSettings['folderID']]} media = http.MediaFileUpload(self.fullPathForFileToUpload, mimetype='audio/jpeg') file = self.service.files().create(body=file_metadata, media_body=media, fields='id').execute() file_id = file.get('id') logger.debug('File creation successful -- ID: %s' % file_id) return file_id
def save_song(gDrive_info, song_name, song_path): SCOPES = ['https://www.googleapis.com/auth/drive'] SERVICE_ACCOUNT_FILE = os.path.join(sys.path[0], 'credentials.json') credentials = service_account.Credentials.from_service_account_file( SERVICE_ACCOUNT_FILE, scopes=SCOPES) service = build('drive', 'v3', credentials=credentials) file_metadata = {'name': song_name, 'parents': [gDrive_info['folder_id']]} media = http.MediaFileUpload(song_path, mimetype='audio/jpeg') file = service.files().create(body=file_metadata, media_body=media, fields='id').execute() file_id = file.get('id') print('File creation successful -- ID: %s' % file_id) return file_id
def add_thumbnails(youtube: d.Resource, video_ids: list, env: str): df_videos = lib.youtube_videos.get_videos(env=env) for video_id in video_ids: print(f'Adding thumbnail for video #{video_id}') thumbnail_img = f"input/thumbnails/{video_id}.jpg" link = df_videos.loc[df_videos.Id == video_id, 'NewYoutubeLink'].values[0] if not pd.isna(link): youtube_id = link.split('/watch?v=')[1] request = youtube.thumbnails().set( videoId=youtube_id, media_body=h.MediaFileUpload(thumbnail_img)) request.execute() else: raise ValueError(f'Youtube-link is missing!')
def uploadSheet(myDrive, file): # myDrive = googledrive('/Users/aciuffo/.config/folderAudit/credentials/credentials.json') file = Path(file) file_metadata = { 'name': file.name, 'mimeType': 'application/vnd.google-apps.spreadsheet' } media = http.MediaFileUpload(file, mimetype='text/csv', resumable=True) file = myDrive.service.files().create(body=file_metadata, media_body=media, fields='id, webViewLink').execute() if file: print('File uploaded to: {}'.format(file.get('webViewLink'))) print( f'Use the Audit Template here to help review the data: {constants.audit_url}' ) return (file)
def upload(filePath, parentID=None): """recursively makes 1:1 copy maintaining file hierarchy in root, if single file simply copies to root. ignore parentid, for recursion""" filePath = os.path.abspath(filePath) if os.path.isfile(filePath): file_metadata = {'name': filePath.split("/")[-1]} if parentID: file_metadata.update({"parents":[parentID]}) media = http.MediaFileUpload(filePath) f = DRIVE.files().create(body=file_metadata, media_body=media, fields='id').execute() return else: #create empty folder folder_metadata = { 'name': filePath.split("/")[-1], 'mimeType': 'application/vnd.google-apps.folder', } if parentID: folder_metadata.update({"parents":[parentID]}) folderID = DRIVE.files().create(body=folder_metadata, fields='id').execute().get("id") for f in os.listdir(filePath): print(f) upload(os.path.abspath(filePath+"/"+f), parentID = folderID) return
def _CopyFileToBucket(self, client, filename, object_name): """Copies file to Google Cloud storage bucket. Args: client: Google Cloud service client object. filename: String path to local file to copy. object_name: String name of file to copy. Returns: Boolean indicating success. """ WriteToStdOut(u'Uploading file: {0:s}'.format(filename)) media = storage_http.MediaFileUpload(filename, resumable=True, chunksize=1024000) request = client.objects().insert(bucket=config.BUCKET_NAME, name=object_name, media_body=media, body=filename) response = None while response is None: error_count = 0 try: status, response = request.next_chunk() except (google_errors.HttpError, google_errors.ResumableUploadError) as e: if e.resp.status in [404]: return False elif e.resp.status in [500, 502, 503, 504]: time.sleep(5) # Call next_chunk() again, but use an exponential backoff for repeated # errors. elif e.resp.status in [400]: error_count += 1 if error_count > 5: raise time.sleep(5) else: raise return True
def upload_file(storage_service, bucket, bucket_path, file_path, verbose=True, mimetype=None, permission=None): '''get_folder will return the folder with folder_name, and if create=True, will create it if not found. If folder is found or created, the metadata is returned, otherwise None is returned :param storage_service: the drive_service created from get_storage_service :param bucket: the bucket object from get_bucket :param file_path: the path to the file to upload :param bucket_path: the path to upload to ''' if bucket_path[-1] != '/': bucket_path = "%s/" % (bucket_path) bucket_path = "%s%s" % (bucket_path, os.path.basename(file_path)) body = {'name': bucket_path} if permission == None: permission = "publicRead" if mimetype == None: mimetype = sniff_extension(file_path, verbose=verbose) media = http.MediaFileUpload(file_path, mimetype=mimetype, resumable=True) try: request = storage_service.objects().insert(bucket=bucket['id'], body=body, predefinedAcl=permission, media_body=media) result = request.execute() except HttpError: result = None pass return result
def upload_content(self, bucket_name: str, static_content_dir: str, folder_root: str = None): """Upload content in the given directory to a GCS bucket. Args: bucket_name: Name of the bucket you want to upload static content to. static_content_dir: Absolute path of the directory containing static files of the Django app. folder_root: Name of root folder for files in GCS bucket. Raises: StaticContentServeError: When failed to upload files. """ prefix_length = len(static_content_dir) # The api only supports uploading a single file. So we need to iterate # all files in the given directory. for root, _, files in os.walk(static_content_dir): relative_dir = root[prefix_length + 1:] for relative_file_path in files: folder_root = folder_root or self.GCS_ROOT # Path of the file in GCS bucket gcs_file_path = os.path.join(folder_root, relative_dir, relative_file_path) # Local absolute path of the file absolute_file_path = os.path.join(root, relative_file_path) media_body = http.MediaFileUpload(absolute_file_path) body = {'name': gcs_file_path} request = self._storage_service.objects().insert( bucket=bucket_name, body=body, media_body=media_body) try: response = request.execute() if 'name' not in response: raise StaticContentServeError( 'Unexpected responses when uploading file "{}" to ' 'bucket "{}"'.format(absolute_file_path, bucket_name)) except errors.HttpError as e: if e.resp.status == 403: raise StaticContentServeError( 'You do not have permission to upload files to ' 'bucket "{}"'.format(bucket_name)) elif e.resp.status == 404: raise StaticContentServeError( 'Bucket "{}" not found.'.format(bucket_name)) else: raise StaticContentServeError( 'Unexpected error when uploading file "{}" to ' 'bucket "{}"'.format(absolute_file_path, bucket_name)) from e # http.MediaFileUpload opens a file but never closes it. So we # need to manually close the file to avoid "ResourceWarning: # unclosed file". # TODO: Remove this line when # https://github.com/googleapis/google-api-python-client/issues/575 # is resolved. media_body.stream().close()
body = { 'name': bucket, } req = gcs_service.buckets().insert(project=project_id, body=body) resp = req.execute() else: raise hell click.echo('Uploading %s to %s...' % (os.path.basename(archive.name), bucket)) try: body = { 'name': os.path.basename(archive.name), } media = http.MediaFileUpload(archive.name, mimetype='application/x-gzip', chunksize=4194304, resumable=True) req = gcs_service.objects().insert( bucket=bucket, name=os.path.basename(archive.name), media_body=media, body={"cacheControl": "public,max-age=31536000"}) resp = None while resp is None: status, resp = req.next_chunk() if status: print "Uploaded %d%%." % int(status.progress() * 100) click.echo('...done!') except HttpError, amy: if amy.resp.status == 403: raise click.UsageError(
def generate_pdf(): df, drive = get_briefs() # Make data latex safe cols = ["Title"] + list(df.columns[6:11]) for col in cols: df[col] = df[col].apply(lambda x: unicode_to_latex(x)) # Generate category list categories = [ n for n in pd.unique(df[df.columns[6:12]].values.ravel("K")) if isinstance(n, str) and n != "nan" ] briefs_by_category = {} for category in sorted(categories): briefs = df[(df[df.columns[6:12]] == category).any(axis=1)] briefs_by_category[category] = [{ "index": index, "title": brief["Title"] } for index, brief in briefs.iterrows()] # Generate flat list briefs_sorted = [] for index, brief in df.sort_values("Title").iterrows(): file_path = os.path.join(cache_path, brief["file_id"] + ".pdf").replace("\\", "/") briefs_sorted.append({ "index": index, "title": brief["Title"], "path": file_path, "id": brief["file_id"] }) briefs = sorted(briefs_sorted, key=lambda item: int(item["index"])) latex_string = latex_jinja_env.get_template("template.tex") with open(os.path.join(path, "indexed_briefs.tex"), "w") as f: f.write( latex_string.render(briefs_by_category=briefs_by_category, briefs_sorted=briefs_sorted, briefs=briefs)) os.system(r"pdflatex -interaction=nonstopmode " + os.path.join(path, "indexed_briefs.tex")) for file in os.listdir(path): if file.startswith("indexed_briefs") and not (file.endswith(".pdf") or file.endswith(".tex")): os.remove(file) # Get list of all files in Google (F**k Google) Drive files = get_pdf(drive_=drive) if files: drive.files().delete(fileId=files['id']).execute() # Upload new file file_metadata = { 'name': f'Indexed Briefs ({str(date.today())})', 'description': 'GlckOayFQgdIdOqRBOL8', "parents": ['1PSgntCxfM-2YidrIjS8hzfzdzoDGv0ze'] } media = http.MediaFileUpload('indexed_briefs.pdf', mimetype='application/pdf') drive.files().create(body=file_metadata, media_body=media).execute()
includeItemsFromAllDrives=True, driveId='<driveId>', spaces='drive', fields='files(id, name, parents, teamDriveId, driveId)', pageSize=1000).execute() print(folder_list) # Download files drive_list = service.files().get_media(fileId='<fileId>') fh = io.FileIO('<filename>', mode='wb') downloader = http.MediaIoBaseDownload(fh, drive_list) done = False while done is False: status, done = downloader.next_chunk() print("Download %d%%." % int(status.progress() * 100)) # Upload file file_metadata = { 'name': '<filename_to_upload>', 'mimeType': '*/*', 'parents': [ '<parentId(s)>' ] } media = http.MediaFileUpload('<filename_to_upload>', mimetype='*/*', resumable=True) file = service.files().create(body=file_metadata, media_body=media, fields='id').execute() print ('File ID: ' + file.get('id'))