def get(self):
aborted = []
incomplete = []
today = datetime.date.today()
compare_date = datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=1)
query = MetaData.query(MetaData.name == 'ETL mapreduce job', MetaData.created > compare_date) # Retrieve all ETL crons from previous day.
for result in query:
if result.created.date() == datetime.date.today():
continue
pipeline_id = result.values[0]
pipeline = mapreduce_pipeline.MapreducePipeline.from_id(pipeline_id) #@UndefinedVariable
if pipeline.has_finalized:
if pipeline.was_aborted:
aborted.append(pipeline_id)
else:
incomplete.append(result.values[0])
if len(aborted) > 0 or len(incomplete) > 0:
template_values = {'aborted': aborted, 'incomplete': incomplete, 'host': os.environ['HTTP_HOST']}
sender_address = DataStoreBase.ADMIN_EMAIL
to_address = DataStoreBase.SUPPORT_EMAIL
template_name = 'views/email/crons/etl_status_alert.html'
jinja_environment = self.request.registry['jinja_environment']
html_template = jinja_environment.get_template(template_name)
html_body = html_template.render(template_values)
template_name = template_name[:-5]
plaintext_template = jinja_environment.get_template('{template_name}_plaintext.html'.format(template_name=template_name))
plaintext_body = plaintext_template.render(template_values)
task_name = "_".join(["admin", "etl_status", str(int(time.time()))])
email_task(task_name, sender_address, to_address, 'ETL error alert.', plaintext_body, html_body=html_body)
def get(self):
if gs_bucket_name is not None:
end_time = self.request.get("end_time", None)
if end_time is None:
now = time.time()
# Get the last end_time
try:
last_run = MetaData.query(MetaData.name == "Last ETL run").fetch(1)[0]
start_time = last_run.values[0]
except IndexError:
# No previous last run, get the last hour
# Should only ever occur once.
start_time = now - 60 * 60
last_run = MetaData(name="Last ETL run", keys=["end_time", "pipeline_id"], values=[now, ""])
else:
now = float(end_time)
start_time = float(self.request.get("start_time", None))
version = self.request.get("version", None)
if version is None:
major = os.environ["CURRENT_VERSION_ID"].split(".")[0]
else:
major = version
p = Log2Bq(start_time, now, [major])
p.start()
meta_data = MetaData(name="ETL mapreduce job", keys=["pipeline_id"], values=[p.root_pipeline_id])
meta_data.put()
if end_time is None:
# Update the last_run meta data with the new end_time (now).
last_run.values[0] = now
last_run.values[1] = p.root_pipeline_id
last_run.put()
else:
try:
manual_run = MetaData.query(MetaData.name == "Manual ETL run").fetch(1)[0]
except IndexError:
manual_run = MetaData(name="Manual ETL run", keys=["pipeline_id"], values=[""])
manual_run.values[0] = p.root_pipeline_id
manual_run.put()
# self.redirect('/mapreduce/pipeline/status?root=%s' % p.root_pipeline_id)
else:
logging.error("GS Bucket name was None")