def send_mails(self, title, dryrun=False):
if not self.send_nag_mail:
return
env = Environment(loader=FileSystemLoader("templates"))
common = env.get_template("common.html")
login_info = utils.get_login_info()
From = Nag.get_from()
Default_Cc = self.get_cc()
mails = self.prepare_mails()
for m in mails:
Cc = Default_Cc.copy()
if m["manager"]:
Cc.add(m["manager"])
body = common.render(message=m["body"], query_url=None)
receivers = set(m["to"]) | set(Cc)
status = "Success"
try:
mail.send(
From,
sorted(m["to"]),
title,
body,
Cc=sorted(Cc),
html=True,
login=login_info,
dryrun=dryrun,
)
except Exception:
logger.exception("Tool {}".format(self.name()))
status = "Failure"
db.Email.add(self.name(), receivers, "individual", status)
def send_mails(self, title, dryrun=False):
if not self.send_nag_mail:
return
env = Environment(loader=FileSystemLoader('templates'))
common = env.get_template('common.html')
login_info = utils.get_login_info()
From = Nag.get_from()
Default_Cc = self.get_cc()
mails = self.prepare_mails()
for m in mails:
Cc = Default_Cc.copy()
if m['manager']:
Cc.add(m['manager'])
body = common.render(message=m['body'], query_url=None)
receivers = set(m['to']) | set(Cc)
status = 'Success'
try:
mail.send(
From,
sorted(m['to']),
title,
body,
Cc=sorted(Cc),
html=True,
login=login_info,
dryrun=dryrun,
)
except: # NOQA
logger.exception('Tool {}'.format(self.name()))
status = 'Failure'
db.Email.add(self.name(), receivers, 'individual', status)
def retrieve_model(self):
os.makedirs('models', exist_ok=True)
file_name = f'{self.name()}model' # noqa: E999
file_path = os.path.join('models', file_name)
model_url = f'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_train.latest/artifacts/public/{file_name}.xz' # noqa
r = requests.head(model_url, allow_redirects=True)
new_etag = r.headers['ETag']
try:
with open(f'{file_path}.etag', 'r') as f: # noqa
old_etag = f.read()
except IOError:
old_etag = None
if old_etag != new_etag:
try:
urlretrieve(model_url, f'{file_path}.xz')
except HTTPError:
logger.exception('Tool {}'.format(self.name()))
return file_path
with lzma.open(f'{file_path}.xz', 'rb') as input_f: # noqa
with open(file_path, 'wb') as output_f:
shutil.copyfileobj(input_f, output_f)
with open(f'{file_path}.etag', 'w') as f: # noqa
f.write(new_etag)
return file_path
def send_mails(self, title, dryrun=False):
if not self.send_nag_mail:
return
env = Environment(loader=FileSystemLoader('templates'))
common = env.get_template('common.html')
login_info = utils.get_login_info()
From = Nag.get_from()
Default_Cc = Nag.get_cc()
mails = self.prepare_mails()
for m in mails:
Cc = Default_Cc.copy()
if m['manager']:
Cc.add(m['manager'])
body = common.render(message=m['body'], query_url=None)
receivers = set(m['to']) | set(Cc)
status = 'Success'
try:
mail.send(
From,
sorted(m['to']),
title,
body,
Cc=sorted(Cc),
html=True,
login=login_info,
dryrun=dryrun,
)
except: # NOQA
logger.exception('Tool {}'.format(self.name()))
status = 'Failure'
db.Email.add(self.name(), receivers, 'individual', status)
def retrieve_model(self):
os.makedirs("models", exist_ok=True)
file_name = f"{self.name()}model"
file_path = os.path.join("models", file_name)
model_url = f"https://index.taskcluster.net/v1/task/project.relman.bugbug.train_{self.name()}.latest/artifacts/public/{file_name}.zst"
r = requests.head(model_url, allow_redirects=True)
new_etag = r.headers["ETag"]
try:
with open(f"{file_path}.etag", "r") as f:
old_etag = f.read()
except IOError:
old_etag = None
if old_etag != new_etag:
try:
urlretrieve(model_url, f"{file_path}.zst")
except HTTPError:
logger.exception("Tool {}".format(self.name()))
return file_path
dctx = zstandard.ZstdDecompressor()
with open(f"{file_path}.zst", "rb") as input_f:
with open(file_path, "wb") as output_f:
dctx.copy_stream(input_f, output_f)
with open(f"{file_path}.etag", "w") as f:
f.write(new_etag)
return file_path
def run(self):
"""Run the tool"""
args = self.get_args_parser().parse_args()
self.parse_custom_arguments(args)
date = '' if self.ignore_date() else args.date
try:
self.send_email(date=date, dryrun=args.dryrun)
except Exception:
logger.exception('Tool {}'.format(self.name()))
def run(self):
"""Run the tool"""
args = self.get_args_parser().parse_args()
self.parse_custom_arguments(args)
date = '' if self.ignore_date() else args.date
self.dryrun = args.dryrun
self.cache.set_dry_run(self.dryrun)
try:
self.send_email(date=date)
except Exception:
logger.exception('Tool {}'.format(self.name()))
def run(self):
"""Run the tool"""
args = self.get_args_parser().parse_args()
self.parse_custom_arguments(args)
date = "" if self.ignore_date() else args.date
self.dryrun = args.dryrun
self.cache.set_dry_run(self.dryrun)
try:
self.send_email(date=date)
self.terminate()
logger.info("Tool {} has finished.".format(self.get_tool_path()))
except Exception:
logger.exception("Tool {}".format(self.name()))
def send_email(self, date="today"):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info("The tool {} hasn't enough data to run".format(
self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(date)
if title:
receivers = self.get_receivers()
status = "Success"
try:
mail.send(
login_info["ldap_username"],
receivers,
title,
body,
html=True,
login=login_info,
dryrun=self.dryrun,
)
except Exception:
logger.exception("Tool {}".format(self.name()))
status = "Failure"
db.Email.add(self.name(), receivers, "global", status)
if isinstance(self, Nag):
self.send_mails(title, dryrun=self.dryrun)
else:
name = self.name().upper()
if date:
logger.info("{}: No data for {}".format(name, date))
else:
logger.info("{}: No data".format(name))
logger.info("Query: {}".format(self.query_url))
def send_email(self, date='today'):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info('The tool {} hasn\'t enough data to run'.format(
self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(date)
if title:
receivers = self.get_receivers()
status = 'Success'
try:
mail.send(
login_info['ldap_username'],
receivers,
title,
body,
html=True,
login=login_info,
dryrun=self.dryrun,
)
except: # NOQA
logger.exception('Tool {}'.format(self.name()))
status = 'Failure'
db.Email.add(self.name(), receivers, 'global', status)
if isinstance(self, Nag):
self.send_mails(title, dryrun=self.dryrun)
else:
name = self.name().upper()
if date:
logger.info('{}: No data for {}'.format(name, date))
else:
logger.info('{}: No data'.format(name))
logger.info('Query: {}'.format(self.query_url))
def send_email(self, date='today'):
"""Send the email"""
if date:
date = lmdutils.get_date(date)
d = lmdutils.get_date_ymd(date)
if isinstance(self, Nag):
self.nag_date = d
if not self.must_run(d):
return
if not self.has_enough_data():
logger.info('The tool {} hasn\'t enough data to run'.format(self.name()))
return
login_info = utils.get_login_info()
title, body = self.get_email(date)
if title:
receivers = self.get_receivers()
status = 'Success'
try:
mail.send(
login_info['ldap_username'],
receivers,
title,
body,
html=True,
login=login_info,
dryrun=self.dryrun,
)
except: # NOQA
logger.exception('Tool {}'.format(self.name()))
status = 'Failure'
db.Email.add(self.name(), receivers, 'global', status)
if isinstance(self, Nag):
self.send_mails(title, dryrun=self.dryrun)
else:
name = self.name().upper()
if date:
logger.info('{}: No data for {}'.format(name, date))
else:
logger.info('{}: No data'.format(name))
logger.info('Query: {}'.format(self.query_url))
