def create_job(self, credentials: Credentials=None, project: str=None, location: str=None, job: Dict[str, Any]=None):
service = DiscoverService.get_service(Service.SCHEDULER, credentials, api_key=os.environ['API_KEY'])
_method = service.projects().locations().jobs().create
if not location:
locations = self.list_locations(credentials=credentials, project=project)
location = locations[-1]
_parent = scheduler.CloudSchedulerClient.location_path(project=project, location=location)
_target = {
'topicName': f"projects/{project}/topics/{job.get('topic', '')}",
# 'data': base64.b64encode(b'RUN'),
'attributes': job.get('attributes', ''),
}
body: dict = {
"name": Scheduler.job_path(project=project, location=location, job=job.get('name', '')),
"description": job.get('description', ''),
"schedule": job.get('schedule', ''),
"timeZone": job.get('timezone', ''),
'pubsubTarget': _target
}
_args = {
'parent': _parent,
'body': body
}
request = _method(**_args)
request.execute()
def send_message(self, message: str, credentials: Credentials):
gmail = DiscoverService.get_service(Service.GMAIL,
credentials=credentials)
request = gmail.users().messages().send(userId='me',
body=message.create_message())
response = request.execute()
logging.info(response)
def list_locations(self, credentials: Credentials=None, project: str=None):
service = DiscoverService.get_service(Service.SCHEDULER, credentials, api_key=os.environ['API_KEY'])
locations_response = self.fetch(
method=service.projects().locations().list,
**{'name': Scheduler.project_path(project)}
)
locations = list([ location['locationId'] for location in locations_response['locations'] ])
return locations
def _get_adh_service(self) -> Resource:
"""Create the ADH Service
Use the discovery API to create the ADH service
Returns:
Resource -- ADH service
"""
adh_service = DiscoverService.get_service(Service.ADH, self.credentials, self.api_key)
return adh_service
def get_job(self, job_id: str=None, credentials: Credentials=None, project: str=None, location: str=None) -> Tuple[bool, Dict[str, Any]]:
service = DiscoverService.get_service(Service.SCHEDULER, credentials, api_key=os.environ['API_KEY'])
method = service.projects().locations().jobs().get
try:
job = method(
name=Scheduler.job_path(
project=project, location=location, job=job_id
)).execute()
return (True, job)
except HttpError as error:
e = json.loads(error.content)
return (False, e)
def delete_job(self, job_id: str=None, credentials: Credentials=None, project: str=None, location: str=None) -> Tuple[bool, Dict[str, Any]]:
service = DiscoverService.get_service(Service.SCHEDULER, credentials, api_key=os.environ['API_KEY'])
method = service.projects().locations().jobs().delete
if not location:
locations = self.list_locations(credentials=credentials, project=project)
location = locations[-1]
try:
method(name=scheduler.CloudSchedulerClient.job_path(project=project, location=location, job=job_id)).execute()
return (True, None)
except HttpError as error:
e = json.loads(error.content)
return (False, e)
def handle_offline_report(self, run_config: Dict[str, Any]) -> bool:
sa360_service = DiscoverService.get_service(Service.SA360, self.creds)
request = sa360_service.reports().get(reportId=run_config['file_id'])
try:
report = request.execute()
if report['isReportReady']:
report_config = self.firestore.get_report_config(
type=Type.SA360_RPT, id=run_config['report_id'])
csv_header, csv_types = self.read_header(report)
schema = CSVHelpers.create_table_schema(
csv_header, csv_types if self.infer_schema else None)
report_config['schema'] = schema
report_config['files'] = report['files']
if 'dest_project' in run_config:
report_config['dest_project'] = run_config['dest_project']
if 'dest_dataset' in run_config:
report_config['dest_dataset'] = run_config['dest_dataset']
if 'notify_topic' in run_config:
report_config['notifier'] = {
'topic': run_config['notify_topic'],
}
if 'notify_message' in run_config:
report_config['notifier']['message'] = run_config[
'notify_message']
# update the report details please...
self.firestore.update_document(Type.SA360_RPT,
run_config['report_id'],
report_config)
# ... then stream the file to GCS a la DV360/CM
self._stream_report_to_gcs(report_details=report_config,
run_config=run_config)
return report['isReportReady']
except Exception as e:
logging.error(
f'Report fetch error: Run {run_config["file_id"]} for report {run_config["report_id"]}'
)
return False
def list_jobs(self, credentials: Credentials=None, project: str=None, location: str=None, email: str=None) -> List[Dict[str, Any]]:
"""copy from one bucket to another
This is a copy from the bucket defined in the report definition (as DV360 stores it's
reports in GCS) into the monitored bucket for upload. It's BLAZING fast, to the extent
that there is essentially no limit on the maximum size of a DV360 report we can handle.
The destination file name is the report's id.
Arguments:
bucket_name {str} -- destination bucket name
report {Dict[str, Any]} -- report definition
"""
service = DiscoverService.get_service(Service.SCHEDULER, credentials, api_key=os.environ['API_KEY'])
token = None
method = service.projects().locations().jobs().list
jobs = []
if not location:
locations = self.list_locations(credentials=credentials, project=project)
location = locations[-1]
while True:
_kwargs = {
'parent': Scheduler.location_path(project, location),
'pageToken': token
}
_jobs = self.fetch(method, **_kwargs)
jobs.extend(_jobs['jobs'] if 'jobs' in _jobs else [])
if 'nextPageToken' not in _jobs:
break
token = _jobs['nextPageToken']
if email and jobs:
jobs = filter(
lambda j: j.get('pubsubTarget', {}).get('attributes', {}).get('email', '') == email,
jobs
)
return list(jobs)
def list_jobs(self, credentials: Credentials=None, project: str=None, location: str=None, email: str=None) -> List[Dict[str, Any]]:
"""[summary]
Args:
credentials (Credentials, optional): [description]. Defaults to None.
project (str, optional): [description]. Defaults to None.
location (str, optional): [description]. Defaults to None.
email (str, optional): [description]. Defaults to None.
Returns:
List[Dict[str, Any]]: [description]
"""
service = DiscoverService.get_service(Service.SCHEDULER, credentials, api_key=os.environ['API_KEY'])
token = None
method = service.projects().locations().jobs().list
jobs = []
if not location:
locations = self.list_locations(credentials=credentials, project=project)
location = locations[-1]
while True:
_kwargs = {
'parent': scheduler.CloudSchedulerClient.location_path(project, location),
'pageToken': token
}
_jobs = self.fetch(method, **_kwargs)
jobs.extend(_jobs['jobs'] if 'jobs' in _jobs else [])
if 'nextPageToken' not in _jobs:
break
token = _jobs['nextPageToken']
if email and jobs:
jobs = filter(
lambda j: j.get('pubsubTarget', {}).get('attributes', {}).get('email', '') == email,
jobs
)
return list(jobs)
def _unattended_run(self, sa360: SA360) -> Dict[str, Any]:
runner = None
report_config = None
try:
report_config = self.firestore.get_document(type=Type.SA360_RPT, id=self.report_id)
if not report_config:
raise NotImplementedError(f'No such runner: {self.report_id}')
_tz = pytz.timezone(report_config.get('timezone') or self.timezone or 'America/Toronto')
_today = datetime.now(_tz)
report_config['StartDate'] = (_today - timedelta(days=(report_config.get('offset') or 0))).strftime('%Y-%m-%d')
report_config['EndDate'] = (_today - timedelta(days=(report_config.get('lookback') or 0))).strftime('%Y-%m-%d')
template = self.firestore.get_document(Type.SA360_RPT, '_reports').get(report_config['report'])
request_body = SA360ReportTemplate().prepare(template=template, values=report_config)
sa360_service = DiscoverService.get_service(Service.SA360, sa360.creds)
request = sa360_service.reports().request(body=request_body)
response = request.execute()
logging.info(response)
runner = {
'type': Type.SA360_RPT.value,
'project': self.project,
'report_id': self.report_id,
'email': self.email,
'file_id': response['id']
}
self.firestore.store_report_runner(runner)
except Exception as e:
self._email_error(email=self.email, error=e, report_config=report_config,
message=f'Error in SA360 Report Runner for report {self.report_id}')
finally:
return runner
def service(self) -> Resource:
return DiscoverService.get_service(
Service.CM, Credentials(email=self.email, project=self.project))