def save_pdf(self,name_on_disk,name_in_drive):
""" Saves a PDF from the disk to the Google Drive
:param name_on_disk: the name of the PDF to upload
:type name_on_disk: str
:param name_in_drive: the PDFs name on the Drive
:type name_in_drive: str
"""
tfid = None
for key in self.subitem_dict:
if key == name_in_drive:
tfid = self.subitem_dict[key]
break
if tfid is None:
media_body = media.MediaFileUpload(name_on_disk,resumable=True,mimetype="application/pdf")
metadata={
"name": name_in_drive,
"parents": [self.folder_id]
}
self.service.files().create(
body = metadata,
media_body = media_body).execute()
self.refresh_dict()
else:
media_body = media.MediaFileUpload(name_on_disk,resumable=True)
self.service.files().update(
fileId = tfid,
media_body = media_body).execute()
def _upload_file_to_object(self, local_file_path: str, bucket_name: str,
object_name: str):
"""Upload the contents of a local file to an object in a GCS bucket."""
media_body = http.MediaFileUpload(local_file_path)
body = {'name': object_name}
request = self._storage_service.objects().insert(bucket=bucket_name,
body=body,
media_body=media_body)
try:
response = request.execute(num_retries=5)
if 'name' not in response:
raise CloudStorageError(
'Unexpected responses when uploading file "{}" to '
'bucket "{}"'.format(local_file_path, bucket_name))
except errors.HttpError as e:
if e.resp.status == 403:
raise CloudStorageError(
'You do not have permission to upload files to '
'bucket "{}"'.format(bucket_name))
elif e.resp.status == 404:
raise CloudStorageError(
'Bucket "{}" not found.'.format(bucket_name))
else:
raise CloudStorageError(
'Unexpected error when uploading file "{}" to '
'bucket "{}"'.format(local_file_path, bucket_name)) from e
# http.MediaFileUpload opens a file but never closes it. So we
# need to manually close the file to avoid "ResourceWarning:
# unclosed file".
# TODO: Remove this line when
# https://github.com/googleapis/google-api-python-client/issues/575
# is resolved.
media_body.stream().close()
def UploadFile(self,
local_file_name,
gcs_file_name,
mimetype='application/octet-stream'):
"""Upload a file to GCS.
Args:
local_file_name: Path to the file on the client's machine.
gcs_file_name: Name with which to store the file on GCS.
It can be a full path (e.g., gs://makani/file), or a path
relative to the preset default bucket.
mimetype: MIME type of the object.
Returns:
Response to the Google Client API's download request.
"""
resumable = os.stat(local_file_name).st_size > 0
media = gapi_http.MediaFileUpload(local_file_name,
mimetype=mimetype,
resumable=resumable)
# gsutil's code suggests that 404s and 410s are retryable for resumable
# uploads (see ResumableUploadStartOverException).
def _ErrorMatcher(error):
return (self._CommonErrorMatcher(error)
or (isinstance(error, gapi_errors.HttpError)
and error.resp.status in (404, 410)))
return self._RunWithRetries(
lambda: self._UploadWithProgress(media, gcs_file_name),
_ErrorMatcher)
def google_drive_write(bucket: str, path: str, file_name: str, creds: str):
"""
Upload file to Google Drive.
File is stored in a parent folder which name is that of 'path' without the
'/'.
Parameters
bucket:
Unused parameter.
path (str):
path = '{prefix}/{exchange}/{data_type}/{pair}/
{exchange}-{data_type}-{pair}-{int(timestamp)}.parquet'
String from which is derived folder name into which is written the
file, as well as the root folder that will contain all these
folders.
file_name (str):
File name preceded by path on local disk.
creds (str):
Path to credential file.
"""
# Retrieve folder ID to be used to write the file into, and upload.
# If not existing, folder will be created.
drive = _get_drive(creds)
folder_id, folder_name = _get_folder_in_parent(drive, path)
media = gah.MediaFileUpload(file_name, resumable=True)
file_metadata = {'name': path.split('/')[-1], 'parents': [folder_id]}
request = drive.files().create(body=file_metadata, media_body=media,
fields='id')
response = None
while response is None:
status, response = request.next_chunk(num_retries=4)
return
def insert(self, file_path, mime_type='application/csv'):
"""Stores a new object and metadata. For tips on uploading to Cloud Storage, see this link :
- https://cloud.google.com/storage/docs/best-practices?hl=fr#uploading
For examples of performing object uploads with different Cloud Storage tools and client libraries,
see the Uploading Objects guide.
This method supports an /upload URI and accepts uploaded media with the following characteristics:
- Maximum file size: 5 TB
- Accepted Media MIME types: */*
- The authenticated user must have WRITER permissions on the bucket.
Note: Metadata-only requests are not allowed. To change an object's metadata, use either the update or patch methods.
To provide a customer-supplied encryption key along with the object upload,
use the headers listed on the Encryption page in your request.
Args:
self : Authorized Cloud Storage API service instance.
file_path: Path of your File
mime_type: Global description about your File
Returns:
"""
try:
media = http.MediaFileUpload(file_path,
mimetype=mime_type,
resumable=True)
response = self.__client.objects().insert(bucket=self.__bucket_id,
name=file_path,
media_body=media
).execute()
return response
except Exception as exception:
print(exception)
def _upload_file(self, local_abs_file_path, remote_parent_id, remote_name):
"""
:param local_abs_file_path pathlib.Path | str: absolute path pointing to the file.
:param remote_parent_id str: remote id of the folder where the file will reside.
:param remote_name str: the new name the file will have on the server.
:return str | None: remote id of the uploaded file. None if failed.
"""
media = http.MediaFileUpload(local_abs_file_path, resumable=True)
meta = {"name": remote_name, "parents": [remote_parent_id]}
request = self._file_service.create(body=meta,
media_body=media,
fields="id")
try:
# execute
response = None
while response is None:
# keep executing chunk by chunk until response means its done.
response = request.next_chunk()[1]
except Exception as error:
self.app.logger.error(
"upload_file: {name} : {error_type} : {error_description}, ignoring..."
.format(
name=remote_name,
error_type=type(error).__name__,
error_description=error,
))
return None
self.app.logger.log(
9, "file uploaded, remote id: {}".format(response["id"]))
return response["id"]
def upload_file(storage_service, bucket, bucket_path, file_name, verbose=True):
'''get_folder will return the folder with folder_name, and if create=True,
will create it if not found. If folder is found or created, the metadata is
returned, otherwise None is returned
:param storage_service: the drive_service created from get_storage_service
:param bucket: the bucket object from get_bucket
:param file_name: the name of the file to upload
:param bucket_path: the path to upload to
'''
# Set up path on bucket
upload_path = "%s/%s" % (bucket['id'], bucket_path)
if upload_path[-1] != '/':
upload_path = "%s/" % (upload_path)
upload_path = "%s%s" % (upload_path, os.path.basename(file_name))
body = {'name': upload_path}
# Create media object with correct mimetype
if os.path.exists(file_name):
mimetype = sniff_extension(file_name, verbose=verbose)
media = http.MediaFileUpload(file_name,
mimetype=mimetype,
resumable=True)
request = storage_service.objects().insert(bucket=bucket['id'],
body=body,
predefinedAcl="publicRead",
media_body=media)
result = request.execute()
return result
bot.warning('%s requested for upload does not exist, skipping' % file_name)
def put(self, filename, dest_path, mimetype=None, chunksize=None):
chunksize = chunksize or self.chunksize
resumable = os.path.getsize(filename) > 0
mimetype = mimetype or mimetypes.guess_type(dest_path)[0] or DEFAULT_MIMETYPE
media = http.MediaFileUpload(filename, mimetype, chunksize=chunksize, resumable=resumable)
self._do_put(media, dest_path)
def InsertData(self, table_id, fd, schema, job_id):
"""Insert data into a bigquery table.
If the table specified doesn't exist, it will be created with the specified
schema.
Args:
table_id: string table id
fd: open file descriptor containing the newline separated JSON
schema: BigQuery schema dict
job_id: string job id
Returns:
API response object on success, None on failure
"""
configuration = {
"schema": {
"fields": schema
},
"destinationTable": {
"projectId": self.project_id,
"tableId": table_id,
"datasetId": self.dataset_id
},
"sourceFormat": "NEWLINE_DELIMITED_JSON",
}
body = {
"configuration": {
"load": configuration
},
"jobReference": {
"projectId": self.project_id,
"jobId": job_id
}
}
# File content can be gzipped for bandwidth efficiency. The server handles
# it correctly without any changes to the request.
mediafile = http.MediaFileUpload(fd.name,
mimetype="application/octet-stream")
job = self.service.jobs().insert(projectId=self.project_id,
body=body,
media_body=mediafile)
try:
response = job.execute()
return response
except errors.HttpError as e:
if self.GetDataset(self.dataset_id):
logging.exception("Error with job: %s", job_id)
else:
# If this is our first export ever, we need to create the dataset.
logging.info("Attempting to create dataset: %s",
self.dataset_id)
self.CreateDataset()
return self.RetryUpload(job, job_id, e)
def upload(self):
file_metadata = {'name': self.fileName,
'parents' : [self.gSettings['folderID']]}
media = http.MediaFileUpload(self.fullPathForFileToUpload,
mimetype='audio/jpeg')
file = self.service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
file_id = file.get('id')
logger.debug('File creation successful -- ID: %s' % file_id)
return file_id
def save_song(gDrive_info, song_name, song_path):
SCOPES = ['https://www.googleapis.com/auth/drive']
SERVICE_ACCOUNT_FILE = os.path.join(sys.path[0], 'credentials.json')
credentials = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE, scopes=SCOPES)
service = build('drive', 'v3', credentials=credentials)
file_metadata = {'name': song_name, 'parents': [gDrive_info['folder_id']]}
media = http.MediaFileUpload(song_path, mimetype='audio/jpeg')
file = service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
file_id = file.get('id')
print('File creation successful -- ID: %s' % file_id)
return file_id
def add_thumbnails(youtube: d.Resource, video_ids: list, env: str):
df_videos = lib.youtube_videos.get_videos(env=env)
for video_id in video_ids:
print(f'Adding thumbnail for video #{video_id}')
thumbnail_img = f"input/thumbnails/{video_id}.jpg"
link = df_videos.loc[df_videos.Id == video_id,
'NewYoutubeLink'].values[0]
if not pd.isna(link):
youtube_id = link.split('/watch?v=')[1]
request = youtube.thumbnails().set(
videoId=youtube_id,
media_body=h.MediaFileUpload(thumbnail_img))
request.execute()
else:
raise ValueError(f'Youtube-link is missing!')
def uploadSheet(myDrive, file):
# myDrive = googledrive('/Users/aciuffo/.config/folderAudit/credentials/credentials.json')
file = Path(file)
file_metadata = {
'name': file.name,
'mimeType': 'application/vnd.google-apps.spreadsheet'
}
media = http.MediaFileUpload(file, mimetype='text/csv', resumable=True)
file = myDrive.service.files().create(body=file_metadata,
media_body=media,
fields='id, webViewLink').execute()
if file:
print('File uploaded to: {}'.format(file.get('webViewLink')))
print(
f'Use the Audit Template here to help review the data: {constants.audit_url}'
)
return (file)
def upload(filePath, parentID=None):
"""recursively makes 1:1 copy maintaining file hierarchy in root, if single file simply copies to root. ignore parentid, for recursion"""
filePath = os.path.abspath(filePath)
if os.path.isfile(filePath):
file_metadata = {'name': filePath.split("/")[-1]}
if parentID: file_metadata.update({"parents":[parentID]})
media = http.MediaFileUpload(filePath)
f = DRIVE.files().create(body=file_metadata, media_body=media, fields='id').execute()
return
else:
#create empty folder
folder_metadata = {
'name': filePath.split("/")[-1],
'mimeType': 'application/vnd.google-apps.folder',
}
if parentID: folder_metadata.update({"parents":[parentID]})
folderID = DRIVE.files().create(body=folder_metadata, fields='id').execute().get("id")
for f in os.listdir(filePath):
print(f)
upload(os.path.abspath(filePath+"/"+f), parentID = folderID)
return
def _CopyFileToBucket(self, client, filename, object_name):
"""Copies file to Google Cloud storage bucket.
Args:
client: Google Cloud service client object.
filename: String path to local file to copy.
object_name: String name of file to copy.
Returns:
Boolean indicating success.
"""
WriteToStdOut(u'Uploading file: {0:s}'.format(filename))
media = storage_http.MediaFileUpload(filename,
resumable=True,
chunksize=1024000)
request = client.objects().insert(bucket=config.BUCKET_NAME,
name=object_name,
media_body=media,
body=filename)
response = None
while response is None:
error_count = 0
try:
status, response = request.next_chunk()
except (google_errors.HttpError,
google_errors.ResumableUploadError) as e:
if e.resp.status in [404]:
return False
elif e.resp.status in [500, 502, 503, 504]:
time.sleep(5)
# Call next_chunk() again, but use an exponential backoff for repeated
# errors.
elif e.resp.status in [400]:
error_count += 1
if error_count > 5:
raise
time.sleep(5)
else:
raise
return True
def upload_file(storage_service,
bucket,
bucket_path,
file_path,
verbose=True,
mimetype=None,
permission=None):
'''get_folder will return the folder with folder_name, and if create=True,
will create it if not found. If folder is found or created, the metadata is
returned, otherwise None is returned
:param storage_service: the drive_service created from get_storage_service
:param bucket: the bucket object from get_bucket
:param file_path: the path to the file to upload
:param bucket_path: the path to upload to
'''
if bucket_path[-1] != '/':
bucket_path = "%s/" % (bucket_path)
bucket_path = "%s%s" % (bucket_path, os.path.basename(file_path))
body = {'name': bucket_path}
if permission == None:
permission = "publicRead"
if mimetype == None:
mimetype = sniff_extension(file_path, verbose=verbose)
media = http.MediaFileUpload(file_path, mimetype=mimetype, resumable=True)
try:
request = storage_service.objects().insert(bucket=bucket['id'],
body=body,
predefinedAcl=permission,
media_body=media)
result = request.execute()
except HttpError:
result = None
pass
return result
def upload_content(self,
bucket_name: str,
static_content_dir: str,
folder_root: str = None):
"""Upload content in the given directory to a GCS bucket.
Args:
bucket_name: Name of the bucket you want to upload static content
to.
static_content_dir: Absolute path of the directory containing
static files of the Django app.
folder_root: Name of root folder for files in GCS bucket.
Raises:
StaticContentServeError: When failed to upload files.
"""
prefix_length = len(static_content_dir)
# The api only supports uploading a single file. So we need to iterate
# all files in the given directory.
for root, _, files in os.walk(static_content_dir):
relative_dir = root[prefix_length + 1:]
for relative_file_path in files:
folder_root = folder_root or self.GCS_ROOT
# Path of the file in GCS bucket
gcs_file_path = os.path.join(folder_root, relative_dir,
relative_file_path)
# Local absolute path of the file
absolute_file_path = os.path.join(root, relative_file_path)
media_body = http.MediaFileUpload(absolute_file_path)
body = {'name': gcs_file_path}
request = self._storage_service.objects().insert(
bucket=bucket_name, body=body, media_body=media_body)
try:
response = request.execute()
if 'name' not in response:
raise StaticContentServeError(
'Unexpected responses when uploading file "{}" to '
'bucket "{}"'.format(absolute_file_path,
bucket_name))
except errors.HttpError as e:
if e.resp.status == 403:
raise StaticContentServeError(
'You do not have permission to upload files to '
'bucket "{}"'.format(bucket_name))
elif e.resp.status == 404:
raise StaticContentServeError(
'Bucket "{}" not found.'.format(bucket_name))
else:
raise StaticContentServeError(
'Unexpected error when uploading file "{}" to '
'bucket "{}"'.format(absolute_file_path,
bucket_name)) from e
# http.MediaFileUpload opens a file but never closes it. So we
# need to manually close the file to avoid "ResourceWarning:
# unclosed file".
# TODO: Remove this line when
# https://github.com/googleapis/google-api-python-client/issues/575
# is resolved.
media_body.stream().close()
body = {
'name': bucket,
}
req = gcs_service.buckets().insert(project=project_id, body=body)
resp = req.execute()
else:
raise hell
click.echo('Uploading %s to %s...' %
(os.path.basename(archive.name), bucket))
try:
body = {
'name': os.path.basename(archive.name),
}
media = http.MediaFileUpload(archive.name,
mimetype='application/x-gzip',
chunksize=4194304,
resumable=True)
req = gcs_service.objects().insert(
bucket=bucket,
name=os.path.basename(archive.name),
media_body=media,
body={"cacheControl": "public,max-age=31536000"})
resp = None
while resp is None:
status, resp = req.next_chunk()
if status:
print "Uploaded %d%%." % int(status.progress() * 100)
click.echo('...done!')
except HttpError, amy:
if amy.resp.status == 403:
raise click.UsageError(
def generate_pdf():
df, drive = get_briefs()
# Make data latex safe
cols = ["Title"] + list(df.columns[6:11])
for col in cols:
df[col] = df[col].apply(lambda x: unicode_to_latex(x))
# Generate category list
categories = [
n for n in pd.unique(df[df.columns[6:12]].values.ravel("K"))
if isinstance(n, str) and n != "nan"
]
briefs_by_category = {}
for category in sorted(categories):
briefs = df[(df[df.columns[6:12]] == category).any(axis=1)]
briefs_by_category[category] = [{
"index": index,
"title": brief["Title"]
} for index, brief in briefs.iterrows()]
# Generate flat list
briefs_sorted = []
for index, brief in df.sort_values("Title").iterrows():
file_path = os.path.join(cache_path,
brief["file_id"] + ".pdf").replace("\\", "/")
briefs_sorted.append({
"index": index,
"title": brief["Title"],
"path": file_path,
"id": brief["file_id"]
})
briefs = sorted(briefs_sorted, key=lambda item: int(item["index"]))
latex_string = latex_jinja_env.get_template("template.tex")
with open(os.path.join(path, "indexed_briefs.tex"), "w") as f:
f.write(
latex_string.render(briefs_by_category=briefs_by_category,
briefs_sorted=briefs_sorted,
briefs=briefs))
os.system(r"pdflatex -interaction=nonstopmode " +
os.path.join(path, "indexed_briefs.tex"))
for file in os.listdir(path):
if file.startswith("indexed_briefs") and not (file.endswith(".pdf") or
file.endswith(".tex")):
os.remove(file)
# Get list of all files in Google (F**k Google) Drive
files = get_pdf(drive_=drive)
if files:
drive.files().delete(fileId=files['id']).execute()
# Upload new file
file_metadata = {
'name': f'Indexed Briefs ({str(date.today())})',
'description': 'GlckOayFQgdIdOqRBOL8',
"parents": ['1PSgntCxfM-2YidrIjS8hzfzdzoDGv0ze']
}
media = http.MediaFileUpload('indexed_briefs.pdf',
mimetype='application/pdf')
drive.files().create(body=file_metadata, media_body=media).execute()
includeItemsFromAllDrives=True,
driveId='<driveId>',
spaces='drive',
fields='files(id, name, parents, teamDriveId, driveId)',
pageSize=1000).execute()
print(folder_list)
# Download files
drive_list = service.files().get_media(fileId='<fileId>')
fh = io.FileIO('<filename>', mode='wb')
downloader = http.MediaIoBaseDownload(fh, drive_list)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
# Upload file
file_metadata = {
'name': '<filename_to_upload>',
'mimeType': '*/*',
'parents':
[
'<parentId(s)>'
]
}
media = http.MediaFileUpload('<filename_to_upload>',
mimetype='*/*',
resumable=True)
file = service.files().create(body=file_metadata, media_body=media, fields='id').execute()
print ('File ID: ' + file.get('id'))
