def send_message(self, message: str, credentials: Credentials):
gmail = discovery.get_service(service=Service.GMAIL,
credentials=credentials)
request = gmail.users().messages().send(userId='me', body=message.create_message())
response = request.execute()
logging.info(response)
def validate(self, config: ManagerConfiguration, **unused) -> None:
sa360_report_definitions = \
self.firestore.get_document(self.report_type, '_reports')
validation_results = []
sa360_objects = self._read_json(config)
for sa360_object in sa360_objects:
if sa360_object == '_reports':
continue
creds = Credentials(project=config.project,
email=sa360_object['email'])
sa360_service = \
discovery.get_service(service=Service.SA360, credentials=creds)
(valid, validation) = \
self._report_validation(sa360_report_definitions,
sa360_object, sa360_service)
validation_results.append(validation)
if validation_results:
if config.type == ManagerType.BIG_QUERY:
results = [json.loads(r.to_json()) for r in validation_results]
# write to BQ
client = bigquery.Client(project=config.project)
table = client.dataset(
config.dataset).table('sa360_validation')
job_config = bigquery.LoadJobConfig(
write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON)
client.load_table_from_json(results,
table,
job_config=job_config)
else:
csv_output = f'{config.email}-<now>-validation.csv'
if config.gcs_stored:
csv_bytes = io.StringIO()
writer = csv.DictWriter(csv_bytes,
fieldnames=Validation.keys(),
quoting=csv.QUOTE_ALL)
writer.writeheader()
writer.writerows([r.to_dict() for r in validation_results])
Cloud_Storage(project=config.project,
email=config.email).write_file(
bucket=self.bucket,
file=csv_output,
data=csv_bytes.getvalue())
else:
with open(csv_output, 'w') as csv_file:
writer = csv.DictWriter(csv_file,
fieldnames=Validation.keys(),
quoting=csv.QUOTE_ALL)
writer.writeheader()
writer.writerows(
[r.to_dict() for r in validation_results])
def service(self):
"""Creates the Scheduler service.
Returns:
googleapiclient.discovert.Resource: the service
"""
return discovery.get_service(service=Service.SCHEDULER,
credentials=self.credentials,
api_key=os.environ['API_KEY'])
def service(self) -> Resource:
"""Creates the API service for the product.
Returns:
Resource: the service definition
"""
return discovery.get_service(service=self.service_definition,
credentials=credentials.Credentials(
email=self.email,
project=self.project))
def adh(self) -> Resource:
"""Create the ADH Service
Use the discovery API to create the ADH service
Returns:
Resource: ADH service
"""
adh_service = \
discovery.get_service(service=Service.ADH,
credentials=self.credentials, api_key=self.api_key)
return adh_service
def send_message(message: GMailMessage,
credentials: credentials.Credentials,
**unused) -> None:
"""Sends a message via the Gmail API.
Args:
message (GMailMessage): the message.
credentials (Credentials): credentials valid for the gmail scope.
"""
gmail = discovery.get_service(service=services.Service.GMAIL,
credentials=credentials)
request = gmail.users().messages().send(userId='me',
body=message.create_message())
response = request.execute()
logging.info(response)
def install(self, config: report_manager.ManagerConfiguration,
**unused) -> None:
if not self.scheduler:
logging.warn(
'No scheduler is available: jobs will be stored but not scheduled.'
)
results = []
random.seed(uuid.uuid4())
runners = self._read_json(config)
for runner in runners:
id = f'{runner["report"]}_{runner["view_id"]}'
creds = Credentials(project=config.project,
email=runner.get('email'))
service = discovery.get_service(service=Service.GA360,
credentials=creds)
self.firestore.update_document(type=self.report_type,
id=id,
new_data=runner)
# Now schedule.
if self.scheduler:
if not (description := runner.get('description')):
if title := runner.get('title'):
description = title
else:
description = (
f'Runner: report {runner.get("report")}, '
f'view_id {runner.get("view_id")}.')
runner['description'] = description
runner['hour'] = runner.get('hour') or '1'
results.append(
self._schedule_job(project=config.project,
runner=runner,
id=id))
def service(self) -> Resource:
return discovery.get_service(service=Service.SA360,
credentials=self.creds)
class SA360Manager(ReportManager):
report_type = Type.SA360_RPT
sa360 = None
sa360_service = None
saved_column_names = {}
actions = {
'list',
'show',
'add',
'delete',
'validate',
'install',
}
def manage(self, **kwargs) -> Any:
project = kwargs['project']
email = kwargs.get('email')
self.bucket = f'{project}-report2bq-sa360-manager'
if kwargs.get('api_key') is not None:
os.environ['API_KEY'] = kwargs['API_KEY']
if 'name' in kwargs:
report_name = kwargs['name']
elif 'file' in kwargs:
report_name = kwargs['file'].split('/')[-1].split('.')[0]
else:
report_name = None
source = None
if kwargs.get('file') and kwargs.get('gcs_stored'):
source = ManagerType.FILE_GCS
elif kwargs.get('file'):
source = ManagerType.FILE_LOCAL
else:
source = ManagerType.BIG_QUERY
config: ManagerConfiguration = ManagerConfiguration(
type=source,
project=project,
email=email,
file=kwargs.get('file'),
table='updated_sa_inputs')
args = {
'report': report_name,
'config': config,
**kwargs,
}
return self._get_action(kwargs.get('action'))(**args)
def validate(self, config: ManagerConfiguration, **unused) -> None:
sa360_report_definitions = \
self.firestore.get_document(self.report_type, '_reports')
validation_results = []
sa360_objects = self._read_json(config)
for sa360_object in sa360_objects:
if sa360_object == '_reports':
continue
creds = Credentials(project=config.project,
email=sa360_object['email'])
sa360_service = \
discovery.get_service(service=Service.SA360, credentials=creds)
(valid, validation) = \
self._report_validation(sa360_report_definitions,
sa360_object, sa360_service)
validation_results.append(validation)
if validation_results:
if config.type == ManagerType.BIG_QUERY:
results = [json.loads(r.to_json()) for r in validation_results]
# write to BQ
client = bigquery.Client(project=config.project)
table = client.dataset(
config.dataset).table('sa360_validation')
job_config = bigquery.LoadJobConfig(
write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON)
client.load_table_from_json(results,
table,
job_config=job_config)
else:
csv_output = f'{config.email}-<now>-validation.csv'
if config.gcs_stored:
csv_bytes = io.StringIO()
writer = csv.DictWriter(csv_bytes,
fieldnames=Validation.keys(),
quoting=csv.QUOTE_ALL)
writer.writeheader()
writer.writerows([r.to_dict() for r in validation_results])
Cloud_Storage(project=config.project,
email=config.email).write_file(
bucket=self.bucket,
file=csv_output,
data=csv_bytes.getvalue())
else:
with open(csv_output, 'w') as csv_file:
writer = csv.DictWriter(csv_file,
fieldnames=Validation.keys(),
quoting=csv.QUOTE_ALL)
writer.writeheader()
writer.writerows(
[r.to_dict() for r in validation_results])
def _report_validation(self,
sa360_report_definitions: Dict[str, Any],
report: Dict[str, Any],
sa360_service: gdiscovery.Resource) -> \
Tuple[bool, Dict[str, Any]]:
logging.info('Validating %s (%s/%s) on report %s',
report.get("agencyName", "-"), report["AgencyId"],
report["AdvertiserId"], report["report"])
target_report = sa360_report_definitions[report['report']]
validator = \
sa360_validator_factory.SA360ValidatorFactory().get_validator(
report_type=target_report['report']['reportType'],
sa360_service=sa360_service,
agency=report['AgencyId'], advertiser=report['AdvertiserId'])
report_custom_columns = \
[column['name'] for column in target_report['parameters']
if 'is_list' in column]
valid = True
validation = Validation(agency=report['AgencyId'],
advertiser=report['AdvertiserId'],
customColumns=validator.saved_column_names)
for report_custom_column in report_custom_columns:
if report[report_custom_column]:
(valid_column,
name) = validator.validate(report[report_custom_column])
valid = valid and valid_column
validity = Validity.UNDEFINED
if not valid:
validity = Validity.INVALID
elif report[report_custom_column]['value']:
validity = Validity.VALID
setattr(validation, stringcase.camelcase(report_custom_column),
validity)
if not valid_column and name:
logging.info(f' Field {report_custom_column} - '
f'{report[report_custom_column]}: '
f'{valid_column}, did you mean "{name}"')
else:
logging.info(f' Field {report_custom_column} - '
f'{report[report_custom_column]}: '
f'{valid_column}')
if len(set(report_custom_columns)) != len(report_custom_columns):
valid = False
return (valid, validation)
def install(self, config: ManagerConfiguration, **unused) -> None:
if not self.scheduler:
logging.warn(
'No scheduler is available: jobs will be stored but not scheduled.'
)
results = []
random.seed(uuid.uuid4())
runners = self._read_json(config)
sa360_report_definitions = \
self.firestore.get_document(self.report_type, '_reports')
credentials = {}
services = {}
for runner in runners:
id = f"{runner['report']}_{runner['AgencyId']}_{runner['AdvertiserId']}"
if not runner['dest_dataset']:
runner['dest_dataset'] = \
f'sa360_hourly_depleted_{runner["country_code"].lower()}'
if not (description := runner.get('description')):
description = (
f'[{runner["country_code"]}] '
f'{runner["title"] if "title" in runner else runner["report"]}: '
f'{runner["agencyName"]}/{runner["advertiserName"]}')
runner['description'] = description
if not (creds := credentials.get(runner['email'])):
creds = Credentials(project=config.project,
email=runner['email'])
credentials[runner['email']] = creds
if not (sa360_service := services.get(runner['email'])):
sa360_service = \
discovery.get_service(service=Service.SA360, credentials=creds)
services[runner['email']] = sa360_service
ga360_report.GA360ReportDefinition.from_json(
json.dumps(report_config))
else:
raise NotImplementedError(f'No such runner: {self._report_id}')
definition.view_id = runner.get('view_id')
ranges = []
for date_range in runner.get('date_ranges'):
range = \
ga360_report.GA360DateRange(start_date=date_range.get('start_date'),
end_date=date_range.get('end_date'))
ranges.append(range)
definition.date_ranges = ranges
request_body = {'reportRequests': [definition.report_request]}
ga360_service = discovery.get_service(Service.GA360,
self.credentials)
request = ga360_service.reports().batchGet(body=request_body)
response = request.execute()
# TODO: Right now, process just the first result, we're not allowing for
# multiple reports in the same request.
# Also, we are assuming that this whole report can be processed in
# a single chunk in memory. If this is not the case, we have a much,
# much larger issue to handle a json file in pieces. This is not
# supported currently.
# -- davidharcombe@, 2021/04/09
if report := response.get('reports'):
report_json = json.dumps(report[0])
result = \
ga360_report_response.GA360ReportResponse.from_json(report_json)