def _check_cm_report(self, job_config: Dict[str, Any],
run_config: Dict[str, Any]):
"""Check a running CM report for completion
Arguments:
report {Dict[str, Any]} -- The report data structure from Firestore
"""
job_attributes = job_config['pubsubTarget']['attributes']
dcm = DCM(email=job_attributes['email'],
project=self.project,
profile=job_attributes['profile_id'])
append = job_attributes[
'append'] if job_attributes and 'append' in job_attributes else False
# TODO: Add report_file.id to run_config
response = dcm.report_state(report_id=job_attributes['report_id'],
file_id=run_config['report_file']['id'])
status = response[
'status'] if response and 'status' in response else 'UNKNOWN'
logging.info('Report {report} status: {status}'.format(
report=job_attributes['report_id'], status=status))
if status == 'REPORT_AVAILABLE':
# Remove job from running
self.firestore.remove_report_runner(job_attributes['report_id'])
# Send pubsub to trigger report2bq now
topic = 'projects/{project}/topics/report2bq-trigger'.format(
project=self.project)
self.PS.publish(topic=topic, data=b'RUN', **job_attributes)
elif status == 'FAILED' or status == 'CANCELLED':
# Remove job from running
logging.error('Report {report} failed!'.format(
report=job_attributes['report_id']))
self.firestore.remove_report_runner(job_attributes['report_id'])
def _attended_run(self, dcm: DCM) -> None:
"""_attended_run.
Run the report and wait for it to finish.
Args:
dcm (DCM): The CM controller.
"""
successful = []
response = dcm.run_report(report_id=self.cm_id, synchronous=True)
if response:
buffer = StringIO()
pprint.pprint(response, stream=buffer)
logging.info(buffer.getvalue())
while response['status'] == 'PROCESSING':
time.sleep(60 * 0.5)
response = dcm.report_state(report_id=self.cm_id, file_id=response['id'])
buffer = StringIO()
pprint.pprint(response, stream=buffer)
logging.info(buffer.getvalue())
report2bq = Report2BQ(
cm=True, cm_id=self.cm_id, email=self.email, project=self.project,
profile=self.cm_profile
)
report2bq.handle_report_fetcher(fetcher=dcm, report_id=self.cm_id)
def _check_cm_report(self, job_config: Dict[str, Any],
run_config: Dict[str, Any]) -> None:
"""Check a running CM360 report for completion.
Args:
job_config (Dict[str, Any]): job configuration
run_config (Dict[str, Any]): current run configuration
"""
job_attributes = job_config['pubsubTarget']['attributes']
dcm = DCM(email=job_attributes['email'],
project=job_attributes['project'],
profile=job_attributes['profile'])
response = dcm.report_state(report_id=run_config['report_id'],
file_id=run_config['file_id'])
status = \
response['status'] if response and 'status' in response else 'UNKNOWN'
logging.info('Report %s status: %s.', run_config['report_id'], status)
if status == 'REPORT_AVAILABLE':
# Send pubsub to trigger report2bq now
topic = f'projects/{self.project}/topics/report2bq-fetcher'
self.pubsub_client.publish(topic=topic,
data=b'RUN',
**job_attributes)
# Remove job from running
self.remove_report_runner(run_config['report_id'])
elif status == 'FAILED' or status == 'CANCELLED':
# Remove job from running
logging.error('Report %s failed!', run_config['report_id'])
self.remove_report_runner(run_config['report_id'])
else:
self.update_report_runner(run_config['report_id'], status)
def _unattended_run(self, dcm: DCM) -> None:
"""_unattended_run.
Start the report running and store the run configuration in Firestore. This
will then be monitored for completion and import by the run-monitor.
Args:
dcm (DCM): The CM controller.
"""
response = dcm.run_report(report_id=self.cm_id, synchronous=False)
if response:
buffer = StringIO()
pprint.pprint(response, stream=buffer)
logging.info(buffer.getvalue())
runner = {
'type': Type.CM.value,
'project': self.project,
'report_id': self.cm_id,
'email': self.email,
'profile': self.cm_profile,
'file_id': response['id']
}
self.firestore.store_document(type=Type._RUNNING,
id=runner['report_id'], document=runner)
def run(self, unattended: bool=True) -> None:
dcm = DCM(email=self.email, project=self.project, profile=self.cm_profile)
if unattended:
self._unattended_run(dcm)
else:
self._attended_run(dcm)
def run(self, unattended: bool=True) -> None:
"""Perform the report run
Args:
unattended (bool, optional): Is this a fire and forget (True) or wait
for the report to complete (False). Defaults to True.
"""
dcm = DCM(email=self.email, project=self.project, profile=self.cm_profile)
if unattended:
self._unattended_run(dcm)
else:
self._attended_run(dcm)
def create_fetcher(product: Type, **kwargs) -> ReportFetcher:
fetcher = None
if product == Type.DV360:
fetcher = DBM(**kwargs)
elif product == Type.CM:
fetcher = DCM(**kwargs)
elif product == Type.SA360:
fetcher = SA360(**kwargs)
else:
raise Exception(f'Cannot create fetcher for {product}')
return fetcher
def _unattended_run(self, dcm: DCM) -> None:
response = dcm.run_report(report_id=self.cm_id, synchronous=False)
if response:
buffer = StringIO()
pprint.pprint(response, stream=buffer)
logging.info(buffer.getvalue())
runner = {
'type': Type.CM.value,
'project': self.project,
'report_id': self.cm_id,
'email': self.email,
'profile': self.cm_profile,
'file_id': response['id']
}
self.firestore.store_report_runner(runner)