def test_media_io_base_download(self):
self.request.http = HttpMockSequence([
({'status': '200',
'content-range': '0-2/5'}, '123'),
({'status': '200',
'content-range': '3-4/5'}, '45'),
])
download = MediaIoBaseDownload(
fd=self.fd, request=self.request, chunksize=3)
self.assertEqual(self.fd, download._fd)
self.assertEqual(3, download._chunksize)
self.assertEqual(0, download._progress)
self.assertEqual(None, download._total_size)
self.assertEqual(False, download._done)
self.assertEqual(self.request.uri, download._uri)
status, done = download.next_chunk()
self.assertEqual(self.fd.getvalue(), '123')
self.assertEqual(False, done)
self.assertEqual(3, download._progress)
self.assertEqual(5, download._total_size)
self.assertEqual(3, status.resumable_progress)
status, done = download.next_chunk()
self.assertEqual(self.fd.getvalue(), '12345')
self.assertEqual(True, done)
self.assertEqual(5, download._progress)
self.assertEqual(5, download._total_size)
def download(self, folder_service_callback):
uploadFolder = self.helper.get_fileid_by_name(helper.UPLOADFOLDER)
# get newest file download url
downloadInfo = self.helper.get_newest_file_down_info(uploadFolder)
progressless_iters = 0
# download file
print('Downloading latest backup ...')
filename = folder_service_callback.getTempFolder() + downloadInfo['title']
fh = io.FileIO(filename, 'wb')
downloader = MediaIoBaseDownload(fh, downloadInfo['request'])
done = False
while not done:
error = None
try:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100), end="\r")
except HttpError as err:
error = err
if err.resp.status < 500:
raise
except self.RETRYABLE_ERRORS as err:
error = err
if error:
progressless_iters += 1
self.handle_progressless_iter(error, progressless_iters)
else:
progressless_iters = 0
return filename
def scan_item_contents(item):
if item.get('mimType') == self.FOLDER_MIME:
return
shared = item.get('shared', None)
if shared is None or shared:
return
# only process files
file_id = item.get('id', None)
name = item.get('name', None)
available = True
if file_id and name:
f_path = create_temp_name(temp_dir, name)
logger.warning("Processing file {}...{}".format(name, f_path))
f = open(f_path, 'wb')
try:
request = self.client.files().get_media(fileId=file_id)
downloader = MediaIoBaseDownload(f, request)
done = False
while done is False:
status, done = downloader.next_chunk()
except Exception as ex:
logger.error("Unable to download file {}. {}".format(name, ex))
available = False
f.close()
if available:
try:
matches.extend(search_content(f_path, expressions))
except Exception as ex:
logger.error("Unable to parse content in file {}. {}".format(name, ex))
try:
os.remove(f_path)
except Exception as ex:
logger.error("Unable to clean up temprary file {}. {}".format(f_path, ex))
def download_file(self,file_id, file_name):
"""Download a Drive file's content to the local filesystem.
Args:
service: Drive API Service instance.
file_id: ID of the Drive file that will downloaded.
file_name: used as name for to write content in.
"""
fd = open(file_name,'w+')
request = self._service.files().get_media(fileId=file_id)
media_request = MediaIoBaseDownload(fd, request)
while True:
try:
download_progress, done = media_request.next_chunk()
except:
print('An error occurred')
fd.close()
return
if download_progress:
print('Download Progress: %d%%' % int(download_progress.progress() * 100))
if done:
print('Download Complete')
fd.close()
return
def get(service):
try:
file_name = raw_input('Which filename to download from cloud?\n')
req = service.objects().get(bucket=_BUCKET_NAME,object=file_name,)
resp = req.execute()
print json.dumps(resp, indent=2)
req = service.objects().get_media(bucket=_BUCKET_NAME,object=file_name,)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024)
done = False
while not done:
status, done = downloader.next_chunk()
if status:
print 'Download %d%%.' % int(status.progress() * 100)
print 'Download Complete!'
reader = csv.reader(open('dict_pw.csv', 'rb'))
newD = dict(x for x in reader)
key= newD[file_name]
print key
decrypt_file(fh.getvalue(),key,file_name)
except client.AccessTokenRefreshError:
print ("Error in the credentials")
def processObject(client, itemname):
try:
#try to access json object in Google Compute Storage
# Get Payload Data
req = client.objects().get_media(
bucket = errorsBucket,
object=itemname)
#store info whether a json-object exists in the bucket or not
fileExists = True
try:
resp = req.execute()
except:
fileExists = False
pass
#continue only when the object exists
if (fileExists):
# The BytesIO object may be replaced with any io.Base instance.
fh = io.BytesIO()
#prepare for reading a json-object
downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024)
done = False
while not done:
status, done = downloader.next_chunk()
#print (fh.getvalue())
#load accessed json-object into dictionary
jsonFile = fh.getvalue()
#print(jsonFile)
jf = open("../" + errorsBucket + "/" + itemname.encode(), 'w')
jf.write(jsonFile)
jf.close()
#store error message into respective errors bucket
except oauth2_client.AccessTokenRefreshError:
pass
def download(argv):
bucket_name, object_name = argv[1][5:].split('/', 1)
filename = argv[2]
assert bucket_name and object_name
service = get_authenticated_service(RO_SCOPE)
print 'Building download request...'
f = file(filename, 'w')
request = service.objects().get_media(bucket=bucket_name,
object=object_name)
media = MediaIoBaseDownload(f, request, chunksize=CHUNKSIZE)
print 'Downloading bucket: %s object: %s to file: %s' % (bucket_name,
object_name,
filename)
progressless_iters = 0
done = False
while not done:
error = None
try:
progress, done = media.next_chunk()
if progress:
print_with_carriage_return(
'Download %d%%.' % int(progress.progress() * 100))
except HttpError, err:
error = err
if err.resp.status < 500:
raise
except RETRYABLE_ERRORS, err:
error = err
def processObject(client, itemname):
try:
#try to access json object in Google Compute Storage
# Get Payload Data
req = client.objects().get_media(
bucket = comm.jsonsDir,
object=itemname)
#store info whether a json-object exists in the bucket or not
fileExists = True
try:
resp = req.execute()
except:
fileExists = False
pass
#continue only when the object exists
if (fileExists):
# The BytesIO object may be replaced with any io.Base instance.
fh = io.BytesIO()
#prepare for reading a json-object
downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024)
done = False
while not done:
status, done = downloader.next_chunk()
#load accessed json-object into dictionary
jsonToDict = json.loads(fh.getvalue())#json.loads(fh.getvalue())#return value
#print ("RETURNED VALUE: " + jsonToDict)
doDownloadJob(jsonToDict, itemname)
#store error message into respective errors bucket
except oauth2_client.AccessTokenRefreshError:
comm.printException(comm.pathToSaveDownloadErrors, errString="False credentials")
pass
def get(self, remote_path, local_path, report_to=None):
LOG.debug("Downloading %s from cloud storage (local path: %s)", remote_path, local_path)
bucket, name = self._parse_url(remote_path)
local_path = os.path.join(local_path, os.path.basename(remote_path))
request = self.cloudstorage.objects().get_media(bucket=bucket, object=name)
f = open(local_path, "w")
try:
media = MediaIoBaseDownload(f, request, chunksize=self.chunk_size)
last_progress = 0
done = False
while not done:
status, done = media.next_chunk()
if status:
percentage = int(status.progress() * 100)
if percentage - last_progress >= self.report_interval:
if report_to:
report_to(status.resumable_progress, status.total_size)
last_progress = percentage
finally:
f.close()
LOG.debug("Finished downloading %s", os.path.basename(local_path))
return local_path
def download_file(service, filename): result = [] result = retrieve_all_files(service) for i in result : if i['title'] == filename: tmp = i['id'] break f = open (filename, 'wb') request = service.files().get_media(fileId=tmp) media_request = MediaIoBaseDownload(f, request) while True: try: download_progress, done = media_request.next_chunk() except errors.HttpError, error: print 'An error occurred: %s' % error return #if download_progress: # print 'Download Progress: %d%%' % int(download_progress.progress() * 100) if done: print 'Google download complete' return
def main():
"""Shows basic usage of the Google Drive API.
Creates a Google Drive API service object and outputs the names and IDs
for up to 10 files.
"""
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('drive', 'v3', http=http)
#file_id = '1cxgbJZKnysKOKBDg-ZbV1E3S4B-iAG7XY-1x7U8Yfsg' # For the grocery doc
file_id = '18r3cUWKbMaWVYtNKLJjxZFHB2m7y1QJdkSPlrU197PA' # For the test doc
request = service.files().export_media(fileId=file_id, mimeType='text/plain')
response = request.execute()
fh = open(FILE_LOC, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %.2f%%." % (status.progress() * 100.0))
fh.close()
# Now read doc.txt to get information
find_date()
# Upload the file back
update_request = service.files().update(fileId=file_id, media_body=FILE_LOC)
update_response = update_request.execute()
def test_media_io_base_download(self):
self.request.http = HttpMockSequence(
[({"status": "200", "content-range": "0-2/5"}, "123"), ({"status": "200", "content-range": "3-4/5"}, "45")]
)
download = MediaIoBaseDownload(fh=self.fh, request=self.request, chunksize=3)
self.assertEqual(self.fh, download.fh_)
self.assertEqual(3, download.chunksize_)
self.assertEqual(0, download.progress_)
self.assertEqual(None, download.total_size_)
self.assertEqual(False, download.done_)
self.assertEqual(self.request.uri, download.uri_)
status, done = download.next_chunk()
self.assertEqual(self.fh.getvalue(), "123")
self.assertEqual(False, done)
self.assertEqual(3, download.progress_)
self.assertEqual(5, download.total_size_)
self.assertEqual(3, status.resumable_progress)
status, done = download.next_chunk()
self.assertEqual(self.fh.getvalue(), "12345")
self.assertEqual(True, done)
self.assertEqual(5, download.progress_)
self.assertEqual(5, download.total_size_)
def retrieve_content(
self, bucket, path, transform=None, generation=None, **kwargs):
"""Retrieves the content at the specified path.
Args:
bucket: [string] The bucket to retrieve front.
path: [string] The path to the content to retrieve from the bucket.
generation: [long] Specifies version of object (or None for current).
transform: [callable(string)] transform the downloaded bytes into
something else (e.g. a JSON object). If None then the identity.
Returns:
transformed object.
"""
self.logger.info('Retrieving path=%s from bucket=%s [generation=%s]',
path, bucket, generation)
# Get Payload Data
request = self.service.objects().get_media(
bucket=bucket,
object=path,
generation=generation,
**kwargs)
data = io.BytesIO()
downloader = MediaIoBaseDownload(data, request, chunksize=1024*1024)
done = False
while not done:
status, done = downloader.next_chunk()
if status:
self.logger.debug('Download %d%%', int(status.progress() * 100))
result = data.getvalue()
return result if transform is None else transform(result)
def File(gs_path, chunk_size=CHUNK_SIZE):
"""Download a file from the cloud, and return a file that's readable.
Args:
gs_path: Fully qualified gfs path, eg, 'gfs://bucket/path/to/FILE'.
chunk_size: The chunk_size to download, defaults to CHUNK_SIZE.
Returns:
An IO stream to be read.
"""
bucket_name, object_name = gs_path[5:].split('/', 1)
logging.info('Downloading file: %s/%s', bucket_name, object_name)
credentials = GoogleCredentials.get_application_default()
service = discovery.build('storage', 'v1', credentials=credentials)
request = service.objects().get_media(bucket=bucket_name, object=object_name)
output = StringIO.StringIO()
media = MediaIoBaseDownload(output, request, chunksize=chunk_size)
done = False
while not done:
try:
_, done = media.next_chunk()
except HttpError, err:
if err.resp.status < 500:
raise
except RETRYABLE_ERRORS, err:
pass
def main(argv):
#print (argv)
#print(argv["object"])
#print(argv["bucket"])
_BUCKET_NAME = argv["bucket"]
_FILE1_NAME = argv["object"]
http = httplib2.Http()
token_uri = '%s/%s/token' % (METADATA_SERVER, SERVICE_ACCOUNT)
resp, content = http.request(token_uri, method='GET',
body=None,
headers={'Metadata-Flavor': 'Google'})
if resp.status == 200:
d = json.loads(content)
access_token = d['access_token'] # Save the access token
credentials = oauth2_client.AccessTokenCredentials(access_token, 'my-user-agent/1.0')
client = api_discovery.build('storage', _API_VERSION, http=credentials.authorize(http))
try:
# Get Metadata
req = client.objects().get(
bucket=_BUCKET_NAME,
object=_FILE1_NAME) # optional
fileExists = True
try:
resp = req.execute()
except HttpError:
fileExists = False
print (str(fileExists))
except:
fileExists = False
print (str(fileExists))
if (fileExists):
# Get Payload Data
req = client.objects().get_media(
bucket=_BUCKET_NAME,
object=_FILE1_NAME) # optional
# The BytesIO object may be replaced with any io.Base instance.
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024)
done = False
while not done:
status, done = downloader.next_chunk()
returnValue = json.loads(fh.getvalue())#return value
#print ("RETURNED VALUE: " + returnValue)
''''''
print ("STR ")
print (type(returnValue) is str)
print ("DICT ")
print (type(returnValue) is dict)
print ("LIST ")
print (type(returnValue) is list)
except oauth2_client.AccessTokenRefreshError:
print ("False credentials")
else:
print (str(False) + str(resp.status))
def download_file_as(service, file_id, media_type, file_name):
request = service.files().export_media(fileId=file_id, mimeType=media_type)
fh = io.FileIO(file_name, mode='wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
def download_file(self,file_name,file_id):
request = self._service.files().get_media(fileId=file_id)
fh = io.FileIO(file_name, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
logger.debug("Download %d%%." % int(status.progress() * 100))
def download(self, filename, bucket_name, object_name):
with open(filename, 'wb') as f:
req = self.cloud.objects().get_media(
bucket=bucket_name, object=object_name)
downloader = MediaIoBaseDownload(f, req)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download {}%.".format(int(status.progress() * 100)))
def gdrive_get_file(drive, file_id):
request = drive.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download {}%%.".format(int(status.progress() * 100)))
return fh
def run(self, bucket: str, obj: str, local_path: str):
archive = self.api.objects.get_media(bucket=bucket, object=obj)
filename = os.path.join(local_path, os.path.basename(obj))
with FileIO(filename, "wb") as fh:
downloader = MediaIoBaseDownload(fh, archive, chunksize=1024 * 1024)
complete = False
while not complete:
_, complete = downloader.next_chunk()
return filename
def download(self, filename, object_id):
logging.debug("Starting download of object %s to %s" % (object_id, filename))
with open(filename, "wb") as fd:
request = self.service.files().get_media(fileId=object_id)
downloader = MediaIoBaseDownload(fd, request, chunksize=CHUNKSIZE)
done = False
while done is False:
status, done = downloader.next_chunk()
logging.info("Download %d%%." % int(status.progress() * 100))
logging.info("Object %s has been downloaded as %s" % (object_id, filename))
def download_file(service, file_id, file_name):
request = service.files().export_media(fileId=file_id,
mimeType='text/csv')
fh = io.FileIO(os.path.join(OUT_PATH, file_name), 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
def pull_file(service, file_id, src_path, dst_path):
request = service.files().get_media(fileId=file_id)
with open(dst_path, 'w') as f:
downloader = MediaIoBaseDownload(f, request)
(status, done) = downloader.next_chunk()
while done is False:
logging.info('downloading... {:.2f}%'.format(
status.progress() * 100))
(status, done) = downloader.next_chunk()
logging.info('{} <- {}'.format(dst_path, src_path))
def Download(self, LocalPath="", progressCallback=None):
LocalPath = LocalPath or self.name
request = self.__ds.files().get_media(fileId=self.__id)
#fh = io.BytesIO()
with io.open(LocalPath, "wb") as fh:
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
if progressCallback:
progressCallback(status.progress() * 100)
def _download_file(self, item, filename): """Downloads item to filename.""" id = item['id'] request = self.service.files().export_media(fileId=id, mimeType='application/pdf') fh = io.BytesIO() downloader = MediaIoBaseDownload(fh, request) done = False while done is False: status, done = downloader.next_chunk() open(filename, 'wb').write(fh.getvalue())
def pullCreditFromDrive(self):
filePrefix = CREDIT_FILENAME.rsplit(".", 1)[0]
creditFile = self.findSheet(filePrefix)
self.creditFileId = creditFile.get('id')
if creditFile == None:
return
request = self.driveService.files().export_media(fileId=creditFile.get('id'), mimeType='text/csv')
fh = io.FileIO(CREDIT_FILENAME, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
def get_file(service, file_id, file_name, location, exported):
if exported:
request = service.files().export_media(fileId=file_id, mimeType='application/pdf')
file_name += '.pdf'
else:
request = service.files().get_media(fileId=file_id)
fh = io.FileIO(os.path.join(location, file_name), mode='wb')
print('downloading -' + str(os.path.join(location, file_name)))
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
def test_media_io_base_download_retries_5xx(self):
self.request.http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '200',
'content-range': '0-2/5'}, '123'),
({'status': '503'}, ''),
({'status': '503'}, ''),
({'status': '503'}, ''),
({'status': '200',
'content-range': '3-4/5'}, '45'),
])
download = MediaIoBaseDownload(
fd=self.fd, request=self.request, chunksize=3)
self.assertEqual(self.fd, download._fd)
self.assertEqual(3, download._chunksize)
self.assertEqual(0, download._progress)
self.assertEqual(None, download._total_size)
self.assertEqual(False, download._done)
self.assertEqual(self.request.uri, download._uri)
# Set time.sleep and random.random stubs.
sleeptimes = []
download._sleep = lambda x: sleeptimes.append(x)
download._rand = lambda: 10
status, done = download.next_chunk(num_retries=3)
# Check for exponential backoff using the rand function above.
self.assertEqual([20, 40, 80], sleeptimes)
self.assertEqual(self.fd.getvalue(), '123')
self.assertEqual(False, done)
self.assertEqual(3, download._progress)
self.assertEqual(5, download._total_size)
self.assertEqual(3, status.resumable_progress)
# Reset time.sleep stub.
del sleeptimes[0:len(sleeptimes)]
status, done = download.next_chunk(num_retries=3)
# Check for exponential backoff using the rand function above.
self.assertEqual([20, 40, 80], sleeptimes)
self.assertEqual(self.fd.getvalue(), '12345')
self.assertEqual(True, done)
self.assertEqual(5, download._progress)
self.assertEqual(5, download._total_size)
def download_file(filename,file_id):
#request = DRIVE.files().get(fileId=file_id)
request = DRIVE.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request,chunksize=-1)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
fh.seek(0)
f=open(filename,'wb')
f.write(fh.read())
f.close()
def print_file(service, file_id):
location = DEFAULT_TEMP_DOWNLOAD_LOCATION
request = service.files().get_media(fileId=file_id)
file_name = os.path.join(location, "." + file_id)
fh = io.FileIO(file_name, mode='wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
with open(file_name, 'r') as f:
for line in f:
print(line, end='')
os.remove(file_name)
def downloadFile(file_id, filepath):
request = drive_service.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
print("downloading")
status, done = downloader.next_chunk()
print(int(status.progress() * 100))
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
def download(doc_id):
service = getservice()
request = service.files().export_media(fileId=doc_id, mimeType='text/plain')
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
sys.stderr.write("Download Google Doc %d%%.\n" % int(status.progress() * 100))
fh.seek(0)
text_obj = fh.read().decode('UTF-8')
return text_obj.encode('utf-8')
def downloadFile(file, mimeType, filepath):
"""Downloads the itinerary as a plain text file.
"""
request = service.create_service().files().export_media(
fileId=file.get('id'), mimeType=mimeType)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
def download_file(service, file_id, location, filename):
request = service.files().get_media(fileId=file_id)
fh = io.FileIO('{}{}'.format(location, filename), 'wb')
downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)
done = False
while done is False:
status, done = downloader.next_chunk()
if status:
#print '\rDownload {}%.'.format(int(status.progress() * 100)),
print(int(status.progress() * 100),
" percent complete \r"),
#sys.stdout.flush()
print("")
print('%s downloaded!' % filename)
def download_file(self, source, file_id, file_name, mime_type):
filepath = source + '/' + file_name + mimetypes.guess_extension(
mime_type)
request = self.driveService.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
return filepath
def googledriveapi_csv2pdf(service,file_id,filename='D:\\temp.pdf',filetype='application/pdf'):
import io
from apiclient.http import MediaIoBaseDownload
import pandas as pd
request = service.files().export_media(fileId=file_id, mimeType=filetype)
fh = io.FileIO(filename, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d" % int(status.progress() * 100))
return None
def download(self, file_id, filepath):
request = self.service.files().export_media(fileId=file_id,
mimeType='text/plain')
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False ##################################################
while done is False: # FINALLY DOWNLOADING ONLY THE OC RECOGNISED FILE#
status, done = downloader.next_chunk(
) ##################################################
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
def downloadFile(file_id, filename, user, folder_id):
request = drive.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
filepath = "upload/" + filename
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
lsb_hiding.lsb_watermark(filepath, user, filepath)
uploadFile(filename, user, folder_id)
os.remove(filepath)
def get(service):
#User can be prompted to input file name(using raw_input) that needs to be be downloaded,
#as an example file name is hardcoded for this function.
print "List of files on cloud"
listobj(service)
fileName = raw_input("Please enter fileName: ")
try:
# Get Metadata
try:
req = service.objects().get(
bucket=_BUCKET_NAME,
object=fileName,
fields='bucket,name,metadata(my-key)',
)
resp = req.execute()
print json.dumps(resp, indent=2)
# Get Payload Data
req = service.objects().get_media(
bucket=_BUCKET_NAME ,
object=fileName,
)
# The BytesIO object may be replaced with any io.Base instance.
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024) #show progress at download
done = False
while not done:
status, done = downloader.next_chunk()
if status:
print 'Download %d%%.' % int(status.progress() * 100)
print 'Download Complete!'
password = raw_input("Please enter Password for decryption: ")
#Calling decrypt function to decrypt data retrieve from cloud
dec = decrypt(fh.getvalue(),createKey(password))
with open(decryptedFileFolderPath+fileName, 'wb') as fo:
fo.write(dec)
print json.dumps(resp, indent=2)
print "File is Succeesfully Downloaded"
print "File is decrypted using the password you provided"
except:
print "File Not Found"
except (client.AccessTokenRefreshError):
print ("Error in the credentials")
def download_file(self, file_id, localpath, overwrite=False):
if os.path.exists(localpath) and not overwrite:
logger.info('File {} exists, skipping'.format(localpath))
return
logger.info('Downloading {} from drive to {}'.format(
file_id, localpath))
request = self.service.files().get_media(fileId=file_id)
fh = io.FileIO(localpath, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
logging.info("{}%%. downloaded".format(int(status.progress() *
100)))
def downloadFiles(service, file_id,filepath):
'''Using Drive API download all the files. Please call in listAllFiles fuction to download or take File_id and
call saperataly'''
request = service.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print ("Download %d%%." % int(status.progress() * 100))
if filepath.endswith('.JPG') or filepath.endswith('.PNG'):
print ("File type is images ")
else:
print ("Doc files ",filepath.endswith('.docx'))
def download_file(file_id, output_file):
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('drive', 'v3', http=http)
#file_id = '0BwwA4oUTeiV1UVNwOHItT0xfa2M'
request = service.files().export_media(fileId=file_id,mimeType='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
#request = service.files().get_media(fileId=file_id)
fh = open(output_file,'wb') #io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
#print ("Download %d%%." % int(status.progress() * 100))
fh.close()
def get_xml(file_id, file_name):
request = service.files().get_media(fileId=file_id)
fh = io.FileIO(file_name, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
renamed_file = 'OLD' + file_name
os.rename(file_name, renamed_file)
xml = parse_xml(renamed_file, file_name)
file_metadata = {'name': xml, 'parents': ['1rnETUjOevkE1ZEFdo6yxdspV2XrFzHID']}
media = MediaFileUpload(xml,mimetype='text/xml')
file = service.files().create(body=file_metadata,media_body=media,fields='id').execute()
os.remove(renamed_file)
os.remove(file_name)
def download_other_files(self, remote_file_id, file_name, file_path):
req = self.drive_service.files().get_media(fileId=remote_file_id)
with open(file_path, "wb") as fh:
downloader = MediaIoBaseDownload(fh, req)
download_complete = False
log(f"Downloading {file_name}")
progress_bar.start_progress()
while not download_complete:
status, download_complete = downloader.next_chunk()
progress_bar.progress(status.progress())
progress_bar.end_progress()
def download_file(self, filename):
"""Create a file with the content downloaded from the user's Google Drive account,
in case file is not found, write the message to the stdout"""
try:
request = self.__drive.files().get_media(
fileId=self.search(filename))
except:
print("The file wasn't found in Google Drive.")
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
download_succeeded = False
while not download_succeeded:
download_succeeded = downloader.next_chunk()
with open(filename, 'wb') as file:
file.write(fh.getvalue())
def download_file(file_id, output_file):
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('drive', 'v3', http=http)
request = service.files().export_media(
fileId=file_id,
mimeType=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
fh = open(output_file, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
fh.close()
def downloadSingleFile(self, fileId, destinationPath) :
print("Destination Path:", destinationPath)
try :
request = self.drive_service.files().get_media(fileId=fileId)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print ("Download %d%%." % int(status.progress() * 100))
with io.open(destinationPath, 'wb') as f :
fh.seek(0)
f.write(fh.read())
except Exception as e:
logging.error("Error in downloading file:"+str(e))
def download(file_id, fileName): #Take a file name and id
try:
global service
request = service.files().get_media(fileId=file_id)
fileRequest = io.BytesIO()
downloader = MediaIoBaseDownload(fileRequest, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(fileName, 'w+') as file:
fileRequest.seek(0)
file.write(fileRequest.read().decode())
except:
print("Downlaod Failed")
def download_file(self, file_id, output_file):
try:
credentials = self.credentials
http = credentials.authorize(httplib2.Http())
service = discovery.build("drive", "v3", http=http)
request = service.files().get_media(fileId=file_id)
fh = open(os.path.join("Data", "raw", output_file), "wb")
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
fh.close()
except Exception as e:
print(e)
def download_report(youtube_reporting, report_url):
request = youtube_reporting.media().download(
resourceName=""
)
request.uri = report_url
fh = FileIO('report', mode='wb')
# Stream/download the report in a single request.
downloader = MediaIoBaseDownload(fh, request, chunksize=-1)
done = False
while done is False:
status, done = downloader.next_chunk()
if status:
print "Download %d%%." % int(status.progress() * 100)
print "Download Complete!"
def cloudstorage_download(service,
project_id,
bucket,
source_file,
dest_file,
show_status_messages=True):
"""Download a file from a Cloud Storage bucket to a local file.
Args:
service: BigQuery service object that is authenticated. Example: service = build('bigquery','v2', http=http)
project_id: string, Name of Google project to download from
bucket: string, Name of Cloud Storage bucket (exclude the "gs://" prefix)
source_file: string, Path to the file to download on Cloud Storage
dest_file: string, Name to give the downloaded file
Returns:
None
"""
# Starting code for this function is a combination from these sources:
# https://code.google.com/p/google-cloud-platform-samples/source/browse/file-transfer-json/chunked_transfer.py?repo=storage
filename = dest_file
bucket_name = bucket
object_name = source_file
assert bucket_name and object_name
if show_status_messages:
log.info('Download request for {0}'.format(source_file))
# media = MediaFileUpload(filename, chunksize=CHUNKSIZE, resumable=True)
# if not media.mimetype():
# media = MediaFileUpload(filename, DEFAULT_MIMETYPE, resumable=True)
f = file(filename, 'w')
request = service.objects().get_media(bucket=bucket_name,
object=object_name)
# response = request.execute()
media = MediaIoBaseDownload(f, request, chunksize=CHUNKSIZE)
progressless_iters = 0
done = False
while not done:
error = None
try:
p, done = media.next_chunk()
except HttpError, err:
error = err
if err.resp.status < 500:
raise
except RETRYABLE_ERRORS, err:
error = err
def ocr(imgfile):
# print("... wating OCR")
# Image with texts (png, jpg, bmp, gif, pdf)
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token.pickle', 'wb') as token:
pickle.dump(creds, token)
service = build('drive', 'v3', credentials=creds)
txtfile = 'output.txt' # Text file outputted by OCR
mime = 'application/vnd.google-apps.document'
res = service.files().create(
body={
'name': imgfile,
'mimeType': mime
},
media_body=MediaFileUpload(imgfile, mimetype=mime,
resumable=True)).execute()
downloader = MediaIoBaseDownload(
io.FileIO(txtfile, 'wb'),
service.files().export_media(fileId=res['id'], mimeType="text/plain"))
done = False
while done is False:
status, done = downloader.next_chunk()
service.files().delete(fileId=res['id']).execute()
# print("Done.")
with open(txtfile) as r:
return ' '.join(r.read().split())
def main():
"""Shows basic usage of the Drive v3 API.
Prints the names and ids of the first 10 files the user has access to.
"""
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'/media/kasem/Happy_place/University stuff/PlayGround/Google Drive sync app/credentials/client_secret.json',
SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token.pickle', 'wb') as token:
pickle.dump(creds, token)
service = build('drive', 'v3', credentials=creds)
# Call the Drive v3 API
results = service.files().list(
pageSize=10, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
if not items:
print('No files found.')
else:
print('Files:')
for item in items:
pass
print(u'{0} ({1})'.format(item['name'], item['id']))
#download a file
file_id = '1y6U1Edt8zMAzIkDSGL-iqU1ju9wex2Pz'
request = service.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
def download_file(service, filename, filename_new=None):
results = service.files().list(
pageSize=10, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
file_id, filename = get_file_id(items, filename)
if file_id == None:
return
request = service.files().get_media(fileId=file_id)
if filename_new == None:
filename_new = filename
fh = io.FileIO(filename_new, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
def download(self,
file_id: str,
writer,
mime_type: Optional[str] = None) -> None:
"""
Download a file and write its content using the binary writer ``writer``.
Example:
with open("my_file.ext", "wb") as f:
client.download(file_id, f)
:param file_id:
:param writer: binary writer
:param mime_type:
:return:
"""
kw = dict(fileId=file_id)
fn = self._files.get_media
if mime_type:
kw["mimeType"] = mime_type
fn = self._files.export_media
downloader = MediaIoBaseDownload(writer, fn(**kw))
# bypass the downloader; there appear to be a bug for large files
writer.write(downloader._request.execute())
def downLoadFile(id, filepath, service=service):
file_id = id
request = service.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
anyKey = input("Press any key to continue!")
os.system('clear')
menuCLI()
def _load_file_from_gcs(gcs_file_path, credentials=None):
"""Load context from a text file in gcs.
Args:
gcs_file_path: The target file path; should have the 'gs://' prefix.
credentials: Optional credential to be used to load the file from gcs.
Returns:
The content of the text file as a string.
"""
gcs_service = _get_storage_service(credentials)
bucket_name, object_name = gcs_file_path[len('gs://'):].split('/', 1)
request = gcs_service.objects().get_media(bucket=bucket_name,
object=object_name)
file_handle = io.BytesIO()
downloader = MediaIoBaseDownload(file_handle,
request,
chunksize=1024 * 1024)
done = False
while not done:
_, done = _downloader_next_chunk(downloader)
filevalue = file_handle.getvalue()
if not isinstance(filevalue, six.string_types):
filevalue = filevalue.decode()
return six.StringIO(filevalue)
def download(service, fileId, filename):
"""Download file with given fileId to filename"""
try:
#Request downloading
request = service.files().get_media(fileId=fileId)
fh = io.BytesIO() #Buffer
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
(status, done) = downloader.next_chunk()
file = open(filename, 'wb') #Writing buffer to file
file.write(fh.getvalue())
fh.close()
print('Download %d%%.' % int(status.progress() * 100))
except:
raise DError("Download error")
def main():
store = file.Storage('token.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets('credentials.json', SCOPES)
creds = tools.run_flow(flow, store)
service = build('drive', 'v3', http=creds.authorize(Http()))
file_id = '***'
request = service.files().export_media(fileId=file_id,
mimeType='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
fh = io.FileIO('Feedback_Form.xlsx', 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print('Feedback_Form.xlsx updated.')
def download_csv(self, file_id, dir_path, filename):
"""
Download file by id.
"""
request = self.service.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
filepath = os.path.join(dir_path, filename)
with open(filepath, 'wb') as output:
output.write(fh.getvalue())
return filepath