def _refresh_credentials(self, project_credentials: Dict[str, str],
user_token: Dict[str, str]) -> Dict[str, str]:
# Remove top-level element
secrets = project_credentials[
'web'] if 'web' in project_credentials else project_credentials[
'installed']
# Init credentials
creds = google.oauth2.credentials.Credentials(
None,
refresh_token=user_token['refresh_token'],
token_uri="https://accounts.google.com/o/oauth2/token",
client_id=secrets['client_id'],
client_secret=secrets['client_secret'])
# Force Refresh token
creds.refresh(google.auth.transport.requests.Request())
refresh_token_details = {
'access_token': creds.token,
'refresh_token': creds.refresh_token
}
Cloud_Storage.write_file(
bucket=self.bucket,
file=self.client_token,
data=json.dumps(refresh_token_details).encode('utf-8'))
return creds
def upload_report(self,
bucket: str,
report_details: Dict[str, Any],
input_buffer: BytesIO = None):
output_buffer = StringIO() #BytesIO()
try:
if not input_buffer:
input_buffer = BytesIO()
request = requests.Download(report_details['url'],
stream=input_buffer)
request.consume(transport=self.transport)
logging.info('Report data size: {bytes}'.format(bytes=0))
input_buffer.seek(0)
soup = self._soupify(input_buffer)
# del input_buffer
headers = soup.find('thead').find_all('th')
fieldnames = []
for header in headers:
fieldnames.append(CSVHelpers.sanitize_string(header.string))
rows = soup.find('tbody').find_all('tr')
report_data = []
for row in rows:
data = []
for col in row.contents:
data.append(col.string)
report_data.append(dict(zip(fieldnames, data)))
writer = csv.DictWriter(output_buffer, fieldnames=fieldnames)
writer.writeheader()
for row in report_data:
writer.writerow(row)
output_buffer.seek(0)
Cloud_Storage.write_file(bucket=bucket,
file=f"{report_details['id']}.csv",
data=output_buffer.getvalue())
report_details['schema'] = CSVHelpers.create_table_schema(
fieldnames)
except Exception as e:
logging.error(e)
def store_credentials(self, creds: credentials.Credentials) -> None:
"""Stores the credentials.
This function uses the datastore to store the user credentials for later.
Args:
creds (credentials.Credentials): the user credentials."""
# TODO: Remove the GCS write when fully migrated to Firestore.
if self._email:
key = self.encode_key(self._email)
refresh_token_details = {
'access_token': creds.token,
'refresh_token': creds.refresh_token,
'_key': key
}
self.datastore.update_document(
type=Type._ADMIN,
id='auth',
new_data={key: json.loads(creds.to_json())})
Cloud_Storage.write_file(
bucket=self.bucket,
file=self.client_token,
data=json.dumps(refresh_token_details).encode('utf-8'))
def oauth_complete(self, request: Request):
logging.info(request.args)
state = request.args.get('state', type=str)
firestore = Firestore()
email, project = firestore.get_oauth_state(state)
project_credentials = json.loads(Files.fetch_file(
'{project}-report2bq-tokens'.format(project=project),
'client_secrets.json'),
encoding='utf-8')
_flow = flow.Flow.from_client_config(client_config=project_credentials,
scopes=self.SCOPES)
_flow.redirect_uri = f"https://{os.environ.get('FUNCTION_REGION')}-{os.environ.get('GCP_PROJECT')}.cloudfunctions.net/OAuthComplete"
r = urlparse(request.url)
auth_response = urlunparse(
['https', r.netloc, r.path, r.params, r.query, r.fragment])
_flow.fetch_token(authorization_response=auth_response)
logging.info(_flow.credentials)
token_details = {
'access_token': _flow.credentials.token,
'refresh_token': _flow.credentials.refresh_token
}
Cloud_Storage.write_file(
'{project}-report2bq-tokens'.format(project=project),
'{email}_user_token.json'.format(email=email),
json.dumps(token_details).encode('utf-8'))
firestore.delete_oauth_state(state=state)
return 'Ok'