def upload_file(service, title, description, parent_id, mime_type, filename):
"""Insert new file.
Args:
service: Drive API service instance.
title: Title of the file to insert, including the extension.
description: Description of the file to insert.
parent_id: Parent folder's ID.
mime_type: MIME type of the file to insert.
filename: Filename of the file to insert.
Returns:
Inserted file metadata if successful, None otherwise.
"""
media_body = http.MediaFileUpload(filename,
mimetype=mime_type,
resumable=True)
body = {'title': title, 'description': description, 'mimeType': mime_type}
# Set the parent folder.
if parent_id:
body['parents'] = [{'id': parent_id}]
try:
file = service.files().insert(body=body,
media_body=media_body).execute()
# Uncomment the following line to print the File ID
print 'File ID: %s' % file['id']
except errors.HttpError, error:
print 'An error occured: %s' % error
return INT_FAILURE_UPLOAD, None
def push(self, name):
body = {'name': name, 'parents': [self.folder_id]}
media_body = http.MediaFileUpload(name)
file = self.files.create(body=body,
media_body=media_body,
fields='id,parents').execute()
self.file_id = file.get('id')
def upload_video(self,
video: str,
properties: dict,
progress: callable = None,
*args) -> dict:
self.progress = progress
self.progress_args = args
self.video = video
self.properties = properties
body = dict(
snippet=dict(
title=self.properties.get("title"),
description=self.properties.get("description"),
categoryId=self.properties.get("category"),
),
status=dict(privacyStatus=self.properties.get("privacyStatus")),
)
media_body = http.MediaFileUpload(
self.video,
chunksize=self.chunksize,
resumable=True,
)
self.request = self.youtube.videos().insert(part=",".join(body.keys()),
body=body,
media_body=media_body)
self._resumable_upload()
return self.response
def send_to_drive(self, processed_file):
"""Send a processed file and its content to the drive"""
folder_id = self.find_or_create_out_folder()
for sub_file in processed_file.ProcessedFile.Files:
name = processed_file.Name + "_" + sub_file.Name + ".csv"
self.delete_file_if_exists(folder_id, name)
if not os.path.exists(self.temp_directory):
os.makedirs(self.temp_directory)
tmp_file_path = os.path.join(self.temp_directory, name)
file_metadata = {'name': name, 'parents': [folder_id]}
# Generates a file in the disk to upload to the drive
sub_file.generate_csv_content_file(tmp_file_path)
media_body = http.MediaFileUpload(tmp_file_path, "text/csv")
try:
self.drive_service.files().create(body=file_metadata,
media_body=media_body,
fields='id').execute()
media_body.stream().close()
except Exception as e:
print("Error while sending the CSV file: " + e)
finally:
os.remove(tmp_file_path)
def initialize_upload(youtube, options):
tags = None
if options.keywords:
tags = options.keywords.split(",")
body = dict(snippet=dict(title=options.title,
description=options.description,
tags=tags,
categoryId=options.category),
status=dict(privacyStatus=options.privacyStatus))
# Call the API's videos.insert method to create and upload the video.
insert_request = youtube.videos().insert(
part=",".join(body.keys()),
body=body,
# The chunksize parameter specifies the size of each chunk of data, in
# bytes, that will be uploaded at a time. Set a higher value for
# reliable connections as fewer chunks lead to faster uploads. Set a lower
# value for better recovery on less reliable connections.
#
# Setting "chunksize" equal to -1 in the code below means that the entire
# file will be uploaded in a single HTTP request. (If the upload fails,
# it will still be retried where it left off.) This is usually a best
# practice, but if you're using Python older than 2.6 or if you're
# running on App Engine, you should set the chunksize to something like
# 1024 * 1024 (1 megabyte).
media_body=http.MediaFileUpload(options.file,
chunksize=-1,
resumable=True))
resumable_upload(insert_request)
def upload_file(self, file_path, parent_folder_id=None):
"""
Upload a file to Google Drive in a "chunked" manner
:param file_path: the local path to the file being uploaded
:param parent_folder_id: the id of the folder where this file should be
placed
:returns: the API's response object
"""
self.logger.log_verbose("Attempting upload:" + file_path + " to " +
str(parent_folder_id))
media = http.MediaFileUpload(file_path, resumable=True)
file_metadata = {'name': os.path.basename(file_path)}
if parent_folder_id is not None:
file_metadata['parents'] = [parent_folder_id]
request = self.DRIVE_SERVICE.files().create(body=file_metadata,
media_body=media)
response = None
while response is None:
status, response = request.next_chunk()
if status:
self.logger.log_verbose("Uploaded {:d}%.".format(
int(status.progress() * 100)))
self.logger.log_verbose("Upload complete: " + file_path)
return response
def publish_video(self, video, title='', description='', tags=None, private=True, **kwargs):
try:
logger.info('trying to publish youtube video, for user: %s' % self.user)
insert_request = self.youtube.videos().insert(
part='snippet,status',
body=dict(
snippet=dict(
title=title,
description=description,
tags=tags
),
status=dict(
privacyStatus='private' if private else 'public'
)
),
media_body=http.MediaFileUpload(video.path, chunksize=-1, resumable=True)
)
result = self._resumable_upload(insert_request, title)
logger.info(str(result))
return result
except Exception as e:
logger.error(e)
raise e
def upload(self, content, name, location, mime='application/pdf'):
print(content, name, location)
"""Upload file to google drive"""
file_metadata = {'name': name, 'parents': [location]}
media = http.MediaFileUpload(content, mimetype=mime)
file = service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
return file.get('id')
def upload_report_to_drive(self):
file_metadata = {
'name': self.start_datetime.strftime('%Y%m%d_%H%M%S'),
'parents': [self.drive_dir_id]
}
media = http.MediaFileUpload(self.report_local_path,
mimetype='application/pdf')
self.drive_service.files().create(body=file_metadata,
media_body=media).execute()
def putFile(service, folderID, filePath, verbose=False):
"""Creates a File on a google drive folder.
Args:
service: google drive service instance to use
folderID: Parent Folder's ID in which to create the new file
filePath: Path to the file that needs to be put on google drive
verbose: print debugging information
Returns:
File name and ID of the newly created File or error if error occured
To Do:
Check if File and folder exist
Possible check if File is not 0 length before putting it up on google drive
"""
drive_service = service
f = os.path.basename(filePath)
mime_type = ''
# Getting Env info to use for metadata tagging
atc = os.getenv('ATC_EXTERNAL_URL', 'DEFAULT_ATC')
pipe = os.getenv('BUILD_PIPELINE_NAME', 'DEFAULT_PIPELINE')
job = os.getenv('BUILD_JOB_NAME', 'DEFAULT_JOB')
build = os.getenv('BUILD_NAME', 'DEFAULT_BUILD_NUM')
# Creteing a link URL to put in metadata
link = atc + '/pipelines/' + pipe + '/jobs/' + job + '/builds/' + build
media_body = http.MediaFileUpload(filePath,
mimetype=mime_type,
resumable=True)
body = {
'title': f,
'description': 'Uploaded by concourse task: \n\n' + link + '',
'mimeType': mime_type
}
# Set the parent Folder
if folderID:
body['parents'] = [{'id': folderID}]
#Temporarily adding perms here
#body['permissions'] = perms
if verbose:
#print('Body: ' .format(dir(body)))
print('Body: {0}'.format(body), file=sys.stderr)
try:
file = drive_service.files().insert(body=body,
media_body=media_body).execute()
#Printing for debug only
#print ('File ID {0} ' .format(file.get('id')))
return file
except errors.HttpError, error:
#print 'An error occured: %s' % error
return error
def upload_file(service, filename, mimeType):
body = {'title': filename, 'description': "DESC OF " + filename}
media_body = http.MediaFileUpload(filename,
mimetype=mimeType,
resumable=True)
request = service.files().insert(body=body, media_body=media_body)
response = None
while response is None:
status, response = request.next_chunk()
if status:
print("Uploaded %d%%." % int(status.progress() * 100))
print("Upload Complete!")
def upload(self, filepath, folder, filename=None):
"""Upload a file to a specific folder."""
if filename is None:
filename = os.path.basename(filepath)
mime_type, _ = mimetypes.guess_type(filename)
media = http.MediaFileUpload(filepath, mimetype=mime_type, resumable=True)
metadata = {
'name': filename,
'parents': [folder],
}
self.service.files().create(body=metadata, media_body=media).execute()
def _insert(self, new_item, feed_item):
"""Handles the upload of creative assets to DCM and the creation of the associated entity.
This method makes a call to the DCM API to create a new entity.
Args:
new_item: The item to insert into DCM.
feed_item: The feed item representing the creative asset from the
Bulkdozer feed.
Returns:
The newly created item in DCM.
"""
local_file = os.path.join(
'/tmp', feed_item.get(FieldMap.CREATIVE_ASSET_FILE_NAME, None))
self._download_from_gcs(
feed_item.get(FieldMap.CREATIVE_ASSET_BUCKET_NAME, None),
feed_item.get(FieldMap.CREATIVE_ASSET_FILE_NAME, None),
local_file,
auth=self.auth)
media = http.MediaFileUpload(local_file)
if not media.mimetype():
mimetype = 'application/zip' if asset_type == 'HTML' else 'application/octet-stream'
media = http.MediaFileUpload(asset_file, mimetype)
result = self._retry(
self._service.insert(profileId=self.profile_id,
advertiserId=feed_item.get(
FieldMap.ADVERTISER_ID, None),
media_body=media,
body=new_item))
os.remove(local_file)
return result
def upload_image():
service = get_service('drive', 'v3', DRIVE_SCOPES)
media = http.MediaFileUpload(config['final_image']['save_path'],
mimetype='image/png', resumable=True)
metadata = {'name': add_timestamp(config['google_service']['portrait_name']),
'parents': [config['google_service']['portrait_folder_id']]}
file_id = service.files().create(body=metadata, media_body=media,
fields='id').execute()['id']
logger.info('Uploaded image to Drive')
data['endo_map_url'] = get_shareable_link(service, file_id)
def upload(self, filepath, folder):
"""Upload a file to a specific folder."""
filename = os.path.basename(filepath)
mime_type = 'application/pdf'
media = http.MediaFileUpload(filepath,
mimetype=mime_type,
resumable=True)
metadata = {
'name': filename,
'parents': [folder],
}
self.service.files().create(body=metadata, media_body=media).execute()
def upload_file(self, file_name, file_path, mime_type, team_drive=False, team_drive_id=None):
file_metadata = {'name': file_name}
media = http.MediaFileUpload(file_path, mimetype=mime_type)
fl = None
if not team_drive:
fl = self.files.create(body=file_metadata, media_body=media, fields='id').execute()
elif team_drive and team_drive_id is not None:
payload = {
'parents': [team_drive_id],
'name': file_name
}
fl = self.files.create(supportsAllDrives=True, media_body=media, body=payload).execute()
elif team_drive and team_drive_id is None:
raise ValueError("team_drive_id should have value if this is a team drive upload")
return fl
def upload_file(self, file_path, folder_id=''):
"""
Upload a file to a specified directory.
:param file_path: Absolute or relative path to the file
:param folder_id: The id of the directory to which the file is to be uploaded
:return:
"""
(directory, filename) = os.path.split(file_path)
(mime, encoding) = mimetypes.guess_type(file_path)
if mime is None:
mime = "application/octet-stream"
media = http.MediaFileUpload(file_path, mimetype=mime, resumable=True)
body = {
'name': filename,
'parents': [folder_id]
}
self.service.files().create(body=body, media_body=media).execute()
def process(self, element):
import tensorflow as tf
from google.oauth2 import service_account
from googleapiclient.discovery import build
from apiclient import http
import csv
import os
elements = list(csv.reader([element.encode("utf-8")],
delimiter=","))[0]
gcs_file_path, gdrive_file_name = elements
# gcs_file_path, gdrive_file_name = element.split(",")
# gcs_file_path, gdrive_file_name = gcs_file_path.encode("utf-8"), gdrive_file_name.encode("utf-8")
# Read file in Cloud Storage
with tf.gfile.Open(gcs_file_path, "rb") as f:
file_content = f.read()
# Save file to local disk
print(gcs_file_path)
print(gdrive_file_name)
tf.gfile.MakeDirs("tmp")
with tf.gfile.Open("tmp/{}".format(gdrive_file_name), "w") as f:
f.write(file_content)
# Download service account file
print(self.service_account_file.get())
with tf.gfile.Open(self.service_account_file.get(), "rb") as f:
file_content = f.read()
with tf.gfile.Open("service_account.json", "w") as f:
f.write(file_content)
# Upload file to Google Drive
scopes = ["https://www.googleapis.com/auth/drive.file"]
credentials = service_account.Credentials.from_service_account_file(
"service_account.json", scopes=scopes)
service = build("drive",
"v3",
credentials=credentials,
cache_discovery=False)
media_body = http.MediaFileUpload("tmp/{}".format(gdrive_file_name))
body = {
"name": gdrive_file_name,
"parents": [self.gdrive_directory_id.get()],
}
print(body)
file_ = service.files().create(body=body,
fields="id",
media_body=media_body).execute()
def upload_backup(self, filename):
""" Upload backup file to GDrive """
print_log('Uploading backup file is started')
metadata = dict(name=filename.split('/').pop(),
parents=[self.drive_directory])
file_body = http.MediaFileUpload(filename,
'application/octet-stream',
chunksize=1024 * 1024 * 2,
resumable=True)
try:
self.service.files().create(body=metadata,
media_body=file_body).execute()
except http.HttpError:
print_log('Uploading backup file is failed')
return False
else:
print_log('Uploading backup file is finished successfully')
return True
def upload_file(self,
from_filename,
to_paths,
to_filename,
mime_type="text/csv",
parent_id='root',
max_retry=10,
retry_interval=10,
overwrite=True):
"""
Upload a file.
:param from_filename:
:param to_paths: A list of directory
[[path, to, dir], [path, to, dir], ...]
:param to_filename: Filename
:param mime_type:
:param parent_id: Parent folder ID
:param max_retry: Max number of retry
:param retry_interval: Interval second between retries
:param overwrite: Bool
:return:
"""
# validations
if not from_filename:
raise Exception("Invalid from_filename: %s" % from_filename)
if not isinstance(to_paths, list):
raise Exception("Invalid to_paths: %s" % to_paths)
if not to_filename:
raise Exception("filename not found: %s" % to_filename)
# media
media = http.MediaFileUpload(from_filename,
mimetype=mime_type,
chunksize=1024 * 1024,
resumable=True)
return call_with_retry(
self._upload,
(media, to_paths, to_filename, mime_type, parent_id, overwrite),
max_retry, retry_interval)
def upload_file(self, __name: str, __path: Path, __folder_id: str) -> str:
""" Upload a file to the Drive. MimeType gets automatically.
:param __name: string, name, the file will have in Drive.
:param __path: Path, file path.
:param __folder_id: string, folder's ID, to which the file will be uploaded.
:return: string, uploaded file's ID.
:exception Trouble: if the file does not exist.
"""
if not __path.exists():
raise FileNotFoundError("File to upload not found")
file_metadata = {'name': __name, 'parents': [__folder_id]}
# mime type getting
m_type = comm_func.mime_type(__path)
media = http.MediaFileUpload(__path, mimetype=m_type)
_file = self.__drive.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
return _file.get('id')
def upload_file(self, file_path, file_name, parent_id=None):
"""
Create a file in your Google Drive.
:param file_path: A path of the file being created
:param file_name: A name of the file being created
:param parent_id: Default: None. If None, the file being created will be at
My Drive of your Google Drive. Otherwise, the file being created will go under
the parent folder id passed as a parameter.
:return: an id of the created file
"""
file_metadata = {
'name': file_name,
'copyRequiresWriterPermission': True
}
if parent_id is not None:
file_metadata['parents'] = [parent_id]
media = http.MediaFileUpload(file_path,
resumable=True)
f = self.drive_service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
print('{} File ID: {}'.format(file_name, f.get('id')))
return f.get('id')
def _upload_one(item):
"""The engine behind |BigQuery.upload|.
Args:
item: An _UploadItem to upload.
"""
context = item.context
logging.debug('Uploading data to (project:%s, datatset:%s, table:%s) '
'from %s with schema %s' %
(context.project_id, context.dataset_id, context.table_id,
item.data_path, context.schema_path))
size = os.stat(item.data_path).st_size
if size == 0:
logging.info('No data found. Skipping upload.')
return
if size > 10 * 1024 * 1024:
logging.error('File to upload too large (%s MB). '
'BigQuery limit is 10 MB. Upload will likely fail.' %
(size,))
# Post to the jobs resource using the client's media upload interface.
# See:
# http://developers.google.com/api-client-library/
# python/guide/media_upload
# Provide a configuration object. See:
# https://cloud.google.com/bigquery/docs/reference/v2/jobs#resource
body = {
'configuration': {
'load': {
'schema': {
'fields': json.load(open(context.schema_path, 'r'))
},
'destinationTable': {
'projectId': context.project_id,
'datasetId': context.dataset_id,
'tableId': context.table_id
},
'sourceFormat': context.data_format,
}
}
}
media_body = http.MediaFileUpload(
item.data_path,
mimetype='application/octet-stream')
with context.lock:
insert_request = context.bigquery.jobs().insert(
projectId=context.project_id,
body=body,
media_body=media_body)
job = insert_request.execute()
status_request = context.bigquery.jobs().get(
projectId=job['jobReference']['projectId'],
jobId=job['jobReference']['jobId'])
# Poll the job until it finishes.
while True:
with context.lock:
result = status_request.execute(num_retries=2)
status = result['status']
if status['state'] != 'DONE':
time.sleep(1)
continue
if 'errorResult' not in status:
break
msg = ('Error when updating table %s with rows from %s using '
'schema %s' % (context.table_id, item.data_path,
context.schema_path))
error = status['errorResult']
if 'debugInfo' in error:
del error['debugInfo']
logging.error(pprint.pformat(error))
raise BigQueryException(msg)
def upload_file(service,
input_file,
output_name=None,
folder_name=None,
show_progress=False,
chunksize=1048576):
"""
Upload file to gdrive using MediaFileUpload that support large file upload and resumable
Display upload speed and progress when call this function
:param service: gdrive api service
:param input_file: full path or relative path to source file to upload
:param output_name: target filename on gdrive, if none, use same name as source filename
:param folder_name: target folder name to store file. this folder will create under root folder on gdrive
:return:
File instance from google api or None if error occur
"""
# filename, ext = os.path.splitext(input_file)
mime_type = 'application/octet-stream'
if os.path.isfile(input_file):
file_size = os.path.getsize(input_file)
else:
logger.error("Error " + input_file + " : raw file not found")
return
if not output_name:
output_name = os.path.basename(input_file)
body = {
'name': output_name,
}
if folder_name:
parent_id = get_or_create_folder(service, folder_name)
if parent_id:
body['parents'] = [parent_id]
else:
logger.error('No value from get_or_create_folder: %s', folder_name)
return
logger.debug('Prepare file to upload [%s] mime[%s] chunk[%s] body[%s]',
input_file, mime_type, chunksize, body)
media = api_http.MediaFileUpload(input_file,
mimetype=mime_type,
chunksize=chunksize,
resumable=True)
request = service.files().create(media_body=media, body=body)
logger.info("Upload : %s, Size : %s", input_file, file_size)
start_time = time.time()
status = None
response = None
last_progress = 0
idle_count = 0
while response is None:
if idle_count >= 10:
logger.error('Max idle retry for upload')
break
try:
status, response = request.next_chunk()
except api_errors.HttpError as error:
if error.resp.status in [404]:
# Start the upload all over again.
logger.error(
'Upload [%s] fail http 404 retry all over again: %s' %
(input_file, error))
break
elif error.resp.status in [403, 500, 502, 503, 504]:
idle_count += 1
# Call next_chunk() again, but use an exponential backoff for repeated errors.
logger.warn(
'Upload [%s] fail http [%s] retry [%s] next_chunk: %s' %
(input_file, error.resp.status, idle_count, error))
time.sleep(idle_count * idle_count * 3)
continue
else:
# Do not retry. Log the error and fail.
logger.error(
'Upload [%s] fail http [%s] error in next_chunk: %s',
(input_file, error.resp.status, error))
break
except socket.error as error:
idle_count += 1
logger.warn(
'Upload [%s] fail [socket.error] retry [%s] next_chunk: %s' %
(input_file, idle_count, error))
time.sleep(idle_count * idle_count * 3)
continue
except:
logger.error(
'Upload [%s] fail An error occurred in next_chunk: [%s] %s ',
input_file,
sys.exc_info()[0],
sys.exc_info()[1])
break
if status:
if last_progress >= status.resumable_progress:
logger.warn(
'Upload [%s] retry [%s] next_chunk: No progress from last chunk'
% (input_file, idle_count))
else:
idle_count = 0
if show_progress:
upload_speed = int(status.resumable_progress /
(1024 * (time.time() - start_time)))
logger.debug(
"Uploaded {}% - ({:,}/{:,}) - Avg {:,} Kps".format(
int(status.progress() * 100),
status.resumable_progress, status.total_size,
upload_speed))
print("Uploaded {}% - ({:,}/{:,}) - Avg {:,} Kps".format(
int(status.progress() * 100), status.resumable_progress,
status.total_size, upload_speed),
end='\r')
last_progress = status.resumable_progress
else:
idle_count += 1
logger.warn(
'Upload [%s] retry [%s] next_chunk: Not receive status from status, response = request.next_chunk()'
% (input_file, idle_count))
if response:
logger.info("Upload {} Complete! -- {:,} seconds".format(
input_file,
time.time() - start_time))
return response
else:
logger.error("Upload {} Error! -- {:,} seconds".format(
input_file,
time.time() - start_time))
return False
from httplib2 import Http
from oauth2client import file, client, tools
import sys
from time import gmtime, strftime
SCOPES = ('https://www.googleapis.com/auth/drive',
'https://www.googleapis.com/auth/drive.appdata',
'https://www.googleapis.com/auth/drive.file',
'https://www.googleapis.com/auth/drive.readonly.metadata')
store = file.Storage('/home/mpiuser/blocktime/datafiles/storage.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets(
'/home/mpiuser/blocktime/datafiles/client_id.json', SCOPES)
creds = tools.run_flow(flow, store)
drive = discovery.build('drive', 'v3', http=creds.authorize(Http()))
files = drive.files().list().execute().get('files', [])
for i in files:
if (i["name"] == "BlockTimeFiles"):
dirId = i["id"]
break
file_metadata = {'name': strftime("%m-%d %H:%M") + '.csv', "parents": [dirId]}
media = http.MediaFileUpload('/home/mpiuser/blocktime/datafiles/timesdata.csv',
mimetype='text/csv')
file = drive.files().create(body=file_metadata, media_body=media, fields='id')
file.execute()
def post(self, request):
auth = request.data.pop('auth')
title = request.data.pop('title')
dct = request.data
date = datetime.datetime.now().strftime("%y/%m/%d")
AUTH = auth
SCOPES = [
'https://www.googleapis.com/auth/drive.file',
'https://www.googleapis.com/auth/documents'
]
CLIENT_SECRET_FILE = 'DocsAPI/client_secrets.json'
creds = client.credentials_from_clientsecrets_and_code(
CLIENT_SECRET_FILE, SCOPES, AUTH)
#import template
drive_service = discovery.build('drive', 'v3', credentials=creds)
file_metadata = {
'name': title,
'mimeType': 'application/vnd.google-apps.document'
}
media = http.MediaFileUpload('DocsAPI/template#1.html',
mimetype='text/html',
resumable=True)
file = drive_service.files().create(body=file_metadata,
media_body=media).execute()
service = discovery.build('docs', 'v1', credentials=creds)
requests = []
for i in dct:
replace = dct[i]
requests.append({
'replaceAllText': {
'containsText': {
'text': '{{' + i + '}}',
'matchCase': 'true'
},
'replaceText': replace,
}
})
requests.append({
'replaceAllText': {
'containsText': {
'text': '{{date}}',
'matchCase': 'true'
},
'replaceText': str(date),
}
})
fileId = file.get('id')
result = service.documents().batchUpdate(documentId=fileId,
body={
'requests': requests
}).execute()
return Response(
{
"detail":
"https://docs.google.com/document/d/" + file.get("id") +
"/edit"
},
status=HTTP_200_OK)
