def process_gdrive_file(file_id,
folder_id,
postfix,
separator,
is_uploaded=False):
"""Process gdrive file to new folder with renaming"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(API_SERVICE_NAME,
API_VERSION,
http=http_auth)
file_meta = drive_service.files().get(fileId=file_id).execute()
new_file_name = generate_file_name(file_meta['name'], postfix,
separator)
if is_uploaded:
# if file was uploaded from a local folder, FE put it into
# a gdrive folder, we just need to rename file.
response = rename_file_request(drive_service,
file_id,
body={'name': new_file_name})
else:
body = _build_request_body(folder_id, new_file_name)
response = copy_file_request(drive_service, file_id, body)
validate_response(response)
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError('Processing of the file failed due to'
' internal server error.')
return response
def get_gdrive_file_data(file_data):
"""Get text/csv data from gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(API_SERVICE_NAME,
API_VERSION,
http=http_auth)
# check file type
file_meta = drive_service.files().get(fileId=file_data['id']).execute()
if file_meta.get('mimeType') == 'text/csv':
file_data = drive_service.files().get_media(
fileId=file_data['id']).execute()
else:
file_data = drive_service.files().export_media(
fileId=file_data['id'], mimeType='text/csv').execute()
csv_data = read_csv_file(StringIO(file_data))
except AttributeError:
# when file_data has no splitlines() method
raise BadRequest('Wrong file format.')
except HttpAccessTokenRefreshError:
handle_token_error('Try to reload /import page')
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(
'Import failed due to internal server error.')
return csv_data, file_data, file_meta.get('name')
def get_gdrive_file_data(file_data):
"""Get text/csv data from gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(API_SERVICE_NAME, API_VERSION,
http=http_auth)
# check file type
file_meta = drive_service.files().get(fileId=file_data['id']).execute()
if file_meta.get('mimeType') == 'text/csv':
file_data = drive_service.files().get_media(
fileId=file_data['id']).execute()
else:
file_data = drive_service.files().export_media(
fileId=file_data['id'], mimeType='text/csv').execute()
csv_data = read_csv_file(StringIO(file_data))
except AttributeError:
# when file_data has no splitlines() method
raise BadRequest(errors.WRONG_FILE_FORMAT)
except HttpAccessTokenRefreshError:
handle_token_error('Try to reload /import page')
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(errors.INTERNAL_SERVER_ERROR)
return csv_data, file_data, file_meta.get('name')
def process_gdrive_file(file_id, folder_id, is_uploaded=False):
"""Process gdrive file to new folder"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(API_SERVICE_NAME,
API_VERSION,
http=http_auth)
file_meta = drive_service.files().get(
fileId=file_id, fields='id,webViewLink,name').execute()
if is_uploaded:
response = file_meta
else:
response = drive_service.files().list(
q="name contains '{}'".format(file_meta['name']),
spaces='drive',
fields='files(parents, name)').execute()
file_name = _generate_new_file_name(response.get('files', []),
file_meta['name'], folder_id)
body = _build_request_body(folder_id, file_name)
response = copy_file_request(drive_service, file_id, body)
validate_response(response)
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
handle_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(errors.INTERNAL_SERVER_ERROR)
return response
def process_gdrive_file(file_id, folder_id, postfix, separator,
is_uploaded=False):
"""Process gdrive file to new folder with renaming"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(
API_SERVICE_NAME, API_VERSION, http=http_auth)
file_meta = drive_service.files().get(fileId=file_id).execute()
new_file_name = generate_file_name(file_meta['name'], postfix, separator)
if is_uploaded:
# if file was uploaded from a local folder, FE put it into
# a gdrive folder, we just need to rename file.
response = rename_file_request(drive_service, file_id,
body={'name': new_file_name})
else:
body = _build_request_body(folder_id, new_file_name)
response = copy_file_request(drive_service, file_id, body)
validate_response(response)
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(errors.INTERNAL_SERVER_ERROR)
return response
def get_gdrive_file(file_data):
"""Get text/csv data from gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(API_SERVICE_NAME,
API_VERSION,
http=http_auth)
# check file type
file_meta = drive_service.files().get(fileId=file_data['id']).execute()
if file_meta.get("mimeType") == "text/csv":
file_data = drive_service.files().get_media(
fileId=file_data['id']).execute()
else:
file_data = drive_service.files().export_media(
fileId=file_data['id'], mimeType='text/csv').execute()
csv_data = read_csv_file(StringIO(file_data))
except AttributeError:
# when file_data has no splitlines() method
raise BadRequest("Wrong file format.")
except Unauthorized as ex:
raise Unauthorized("{} Try to reload /import page".format(ex.message))
except HttpAccessTokenRefreshError:
# drive_service can rise this exception in case of invalid token, that why
# we need to handle it here
del flask.session['credentials']
raise Unauthorized('Unable to get valid credentials.'
' Try to reload /import page')
except HttpError as ex:
hande_http_error(ex)
except: # pylint: disable=bare-except
raise InternalServerError(
"Import failed due to internal server error.")
return csv_data
def get_gdrive_file(file_data):
"""Get text/csv data from gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build('drive', 'v3', http=http_auth)
# check file type
file_meta = drive_service.files().get(fileId=file_data['id']).execute()
if file_meta.get("mimeType") == "text/csv":
file_data = drive_service.files().get_media(
fileId=file_data['id']).execute()
else:
file_data = drive_service.files().export_media(
fileId=file_data['id'], mimeType='text/csv').execute()
csv_data = read_csv_file(StringIO(file_data))
except AttributeError:
# when file_data has no splitlines() method
raise BadRequest("Wrong file format.")
except HttpError as e:
message = json.loads(e.content).get("error").get("message")
if e.resp.status == 404:
raise NotFound(message)
if e.resp.status == 401:
raise Unauthorized("{} Try to reload /import page".format(message))
if e.resp.status == 400:
raise BadRequest(message + " Probably the file is of a wrong type.")
raise InternalServerError(message)
except: # pylint: disable=bare-except
raise InternalServerError("Import failed due to internal server error.")
return csv_data
def run_bulk_complete():
"""Call bulk complete job"""
data = flask.request.json
parameters = {"data": data}
if _detect_files(data):
try:
gdrive.get_http_auth()
except gdrive.GdriveUnauthorized:
response = app.make_response(
("auth", 401, [("Content-Type", "text/html")]))
return response
parameters["credentials"] = flask.session.get('credentials')
bg_task = background_task.create_task(name="bulk_complete",
url=flask.url_for(
bulk_complete.__name__),
queued_callback=bulk_complete,
parameters=parameters)
db.session.commit()
return bg_task.make_response(
app.make_response(
(utils.as_json(bg_task), 200, [('Content-Type', "text/json")])))
def add_gdrive_file_folder(file_id, folder_id):
"""Add folder to gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(
API_SERVICE_NAME, API_VERSION, http=http_auth)
response = add_folder_request(drive_service, file_id, folder_id)
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(errors.INTERNAL_SERVER_ERROR)
return response['webViewLink']
def add_gdrive_file_folder(file_id, folder_id):
"""Add folder to gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(
API_SERVICE_NAME, API_VERSION, http=http_auth)
response = add_folder_request(drive_service, file_id, folder_id)
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(errors.INTERNAL_SERVER_ERROR)
return response['webViewLink']
def create_gdrive_file(csv_string, filename):
"""Post text/csv data to a gdrive file"""
http_auth = get_http_auth()
drive_service = discovery.build('drive', 'v3', http=http_auth)
# make export to sheets
file_metadata = {
'name': filename,
'mimeType': 'application/vnd.google-apps.spreadsheet'
}
media = http.MediaInMemoryUpload(csv_string,
mimetype='text/csv',
resumable=True)
return drive_service.files().create(body=file_metadata,
media_body=media,
fields='id, name, parents').execute()
def get_gdrive_file_link(file_id):
"""Returns file link"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(
API_SERVICE_NAME, API_VERSION, http=http_auth)
file_meta = drive_service.files().get(fileId=file_id,
fields='webViewLink').execute()
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(errors.INTERNAL_SERVER_ERROR)
return file_meta['webViewLink']
def get_gdrive_file_link(file_id):
"""Returns file link"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(
API_SERVICE_NAME, API_VERSION, http=http_auth)
file_meta = drive_service.files().get(fileId=file_id,
fields='webViewLink').execute()
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
except Exception as ex:
logger.error(ex.message)
raise InternalServerError(errors.INTERNAL_SERVER_ERROR)
return file_meta['webViewLink']
def create_gdrive_file(csv_string, filename):
"""Post text/csv data to a gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(API_SERVICE_NAME, API_VERSION,
http=http_auth)
# make export to sheets
file_metadata = {
'name': filename,
'mimeType': 'application/vnd.google-apps.spreadsheet'
}
media = http.MediaInMemoryUpload(csv_string,
mimetype='text/csv',
resumable=True)
result = drive_service.files().create(body=file_metadata,
media_body=media,
fields='id, name, parents').execute()
return result
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
def create_gdrive_file(csv_string, filename):
"""Post text/csv data to a gdrive file"""
http_auth = get_http_auth()
try:
drive_service = discovery.build(API_SERVICE_NAME, API_VERSION,
http=http_auth)
# make export to sheets
file_metadata = {
'name': filename,
'mimeType': 'application/vnd.google-apps.spreadsheet'
}
media = http.MediaInMemoryUpload(csv_string,
mimetype='text/csv',
resumable=True)
result = drive_service.files().create(body=file_metadata,
media_body=media,
fields='id, name, parents').execute()
return result
except HttpAccessTokenRefreshError:
handle_token_error()
except HttpError as ex:
hande_http_error(ex)
