def test_cache(self): cache = Cache('test_cache', 7) cache.set_dry_run(False) bugids = [123, 456, 789] cache.add(bugids) for bugid in bugids: assert bugid in cache assert str(bugid) in cache assert 101112 not in cache assert '101112' not in cache with open(cache.get_path(), 'r') as In: data = json.load(In) for bugid in ['123', '456']: date = data[bugid] date = lmdutils.get_date_ymd(date) - relativedelta(days=8) data[bugid] = lmdutils.get_date_str(date) with open(cache.get_path(), 'w') as Out: json.dump(data, Out) cache = Cache('test_cache', 7) cache.set_dry_run(False) assert 123 not in cache assert 456 not in cache assert 789 in cache
def __init__(self): super(BzCleaner, self).__init__() self._set_tool_name() self.has_autofix = False self.no_manager = set() self.auto_needinfo = {} self.has_flags = False self.cache = Cache(self.name(), self.max_days_in_cache()) self.test_mode = utils.get_config('common', 'test', False)
def __init__(self): super(BzCleaner, self).__init__() self._set_tool_name() self.has_autofix = False self.no_manager = set() self.auto_needinfo = {} self.has_flags = False self.cache = Cache(self.name(), self.max_days_in_cache()) self.test_mode = utils.get_config("common", "test", False) self.versions = None logger.info("Run tool {}".format(self.get_tool_path()))
def test_cache(self): cache = Cache("test_cache", 7) cache.set_dry_run(False) bugids = [123, 456, 789] cache.add(bugids) for bugid in bugids: assert bugid in cache assert str(bugid) in cache assert 101112 not in cache assert "101112" not in cache with open(cache.get_path(), "r") as In: data = json.load(In) for bugid in ["123", "456"]: date = data[bugid] date = lmdutils.get_date_ymd(date) - relativedelta(days=8) data[bugid] = lmdutils.get_date_str(date) with open(cache.get_path(), "w") as Out: json.dump(data, Out) cache = Cache("test_cache", 7) cache.set_dry_run(False) assert 123 not in cache assert 456 not in cache assert 789 in cache
class BzCleaner(object): def __init__(self): super(BzCleaner, self).__init__() self._set_tool_name() self.has_autofix = False self.no_manager = set() self.auto_needinfo = {} self.has_flags = False self.cache = Cache(self.name(), self.max_days_in_cache()) self.test_mode = utils.get_config("common", "test", False) self.versions = None logger.info("Run tool {}".format(self.get_tool_path())) def _set_tool_name(self): module = sys.modules[self.__class__.__module__] base = os.path.dirname(__file__) scripts = os.path.join(base, "scripts") self.__tool_path__ = os.path.relpath(module.__file__, scripts) name = os.path.basename(module.__file__) name = os.path.splitext(name)[0] self.__tool_name__ = name def init_versions(self): self.versions = utils.get_checked_versions() return bool(self.versions) def max_days_in_cache(self): """Get the max number of days the data must be kept in cache""" return self.get_config("max_days_in_cache", -1) def preamble(self): return None def description(self): """Get the description for the help""" return "" def name(self): """Get the tool name""" return self.__tool_name__ def get_tool_path(self): """Get the tool path""" return self.__tool_path__ def needinfo_template(self): """Get the txt template filename""" return self.name() + "_needinfo.txt" def template(self): """Get the html template filename""" return self.name() + ".html" def subject(self): """Get the partial email subject""" return self.description() def get_email_subject(self, date): """Get the email subject with a date or not""" af = "[autofix]" if self.has_autofix else "" if date: return "[autonag]{} {} for the {}".format(af, self.subject(), date) return "[autonag]{} {}".format(af, self.subject()) def ignore_date(self): """Should we ignore the date ?""" return False def must_run(self, date): """Check if the tool must run for this date""" days = self.get_config("must_run", None) if not days: return True weekday = date.weekday() week = utils.get_weekdays() for day in days: if week[day] == weekday: return True return False def has_enough_data(self): """Check if the tool has enough data to run""" if self.versions is None: # init_versions() has never been called return True return bool(self.versions) def filter_no_nag_keyword(self): """If True, then remove the bugs with [no-nag] in whiteboard from the bug list""" return True def add_no_manager(self, bugid): self.no_manager.add(str(bugid)) def has_assignee(self): return False def has_needinfo(self): return False def get_mail_to_auto_ni(self, bug): return None def all_include_fields(self): return False def get_max_ni(self): return -1 def ignore_meta(self): return False def columns(self): """The fields to get for the columns in email report""" return ["id", "summary"] def sort_columns(self): """Returns the key to sort columns""" return None def get_dates(self, date): """Get the dates for the bugzilla query (changedafter and changedbefore fields)""" date = lmdutils.get_date_ymd(date) lookup = self.get_config("days_lookup", 7) start_date = date - relativedelta(days=lookup) end_date = date + relativedelta(days=1) return start_date, end_date def get_extra_for_template(self): """Get extra data to put in the template""" return {} def get_extra_for_needinfo_template(self): """Get extra data to put in the needinfo template""" return {} def get_config(self, entry, default=None): return utils.get_config(self.name(), entry, default=default) def get_bz_params(self, date): """Get the Bugzilla parameters for the search query""" return {} def get_data(self): """Get the data structure to use in the bughandler""" return {} def get_summary(self, bug): return "..." if bug["groups"] else bug["summary"] def has_default_products(self): return True def has_product_component(self): return False def get_product_component(self): return self.prod_comp def get_max_years(self): return self.get_config("max-years", -1) def has_access_to_sec_bugs(self): return self.get_config("sec", True) def handle_bug(self, bug, data): """Implement this function to get all the bugs from the query""" return bug def get_db_extra(self): """Get extra information required for db insertion""" return { bugid: ni_mail for ni_mail, v in self.auto_needinfo.items() for bugid in v["bugids"] } def get_auto_ni_skiplist(self): return set() def add_auto_ni(self, bugid, data): if not data: return ni_mail = data["mail"] if ni_mail in self.get_auto_ni_skiplist(): return if ni_mail in self.auto_needinfo: max_ni = self.get_max_ni() info = self.auto_needinfo[ni_mail] if max_ni <= 0 or len(info["bugids"]) < max_ni: info["bugids"].append(str(bugid)) else: self.auto_needinfo[ni_mail] = { "nickname": data["nickname"], "bugids": [str(bugid)], } def get_receivers(self): receivers = self.get_config("receivers") if isinstance(receivers, six.string_types): receivers = utils.get_config("common", "receiver_list", default={})[ receivers ] return receivers def bughandler(self, bug, data): """bug handler for the Bugzilla query""" if bug["id"] in self.cache: return if self.handle_bug(bug, data) is None: return bugid = str(bug["id"]) res = {"id": bugid} auto_ni = self.get_mail_to_auto_ni(bug) self.add_auto_ni(bugid, auto_ni) res["summary"] = self.get_summary(bug) if self.has_assignee(): real = bug["assigned_to_detail"]["real_name"] if utils.is_no_assignee(bug["assigned_to"]): real = "nobody" if real.strip() == "": real = bug["assigned_to_detail"]["name"] if real.strip() == "": real = bug["assigned_to_detail"]["email"] res["assignee"] = real if self.has_needinfo(): s = set() for flag in utils.get_needinfo(bug): s.add(flag["requestee"]) res["needinfos"] = sorted(s) if self.has_product_component(): for k in ["product", "component"]: res[k] = bug[k] if isinstance(self, Nag): bug = self.set_people_to_nag(bug, res) if not bug: return if bugid in data: data[bugid].update(res) else: data[bugid] = res def amend_bzparams(self, params, bug_ids): """Amend the Bugzilla params""" if not self.all_include_fields(): if "include_fields" in params: fields = params["include_fields"] if isinstance(fields, list): if "id" not in fields: fields.append("id") elif isinstance(fields, six.string_types): if fields != "id": params["include_fields"] = [fields, "id"] else: params["include_fields"] = [fields, "id"] else: params["include_fields"] = ["id"] params["include_fields"] += ["summary", "groups"] if self.has_assignee() and "assigned_to" not in params["include_fields"]: params["include_fields"].append("assigned_to") if self.has_product_component(): if "product" not in params["include_fields"]: params["include_fields"].append("product") if "component" not in params["include_fields"]: params["include_fields"].append("component") if self.has_needinfo() and "flags" not in params["include_fields"]: params["include_fields"].append("flags") if bug_ids: params["bug_id"] = bug_ids if self.filter_no_nag_keyword(): n = utils.get_last_field_num(params) params.update( { "f" + n: "status_whiteboard", "o" + n: "notsubstring", "v" + n: "[no-nag]", } ) if self.ignore_meta(): n = utils.get_last_field_num(params) params.update({"f" + n: "keywords", "o" + n: "nowords", "v" + n: "meta"}) # Limit the checkers to X years. Unlimited if max_years = -1 max_years = self.get_max_years() if max_years > 0: n = utils.get_last_field_num(params) params.update( { f"f{n}": "creation_ts", f"o{n}": "greaterthan", f"v{n}": f"-{max_years}y", } ) if self.has_default_products(): params["product"] = self.get_config("products") + self.get_config( "additional_products", [] ) if not self.has_access_to_sec_bugs(): n = utils.get_last_field_num(params) params.update({"f" + n: "bug_group", "o" + n: "isempty"}) self.has_flags = "flags" in params.get("include_fields", []) def get_bugs(self, date="today", bug_ids=[], chunk_size=None): """Get the bugs""" bugs = self.get_data() params = self.get_bz_params(date) self.amend_bzparams(params, bug_ids) self.query_url = utils.get_bz_search_url(params) if isinstance(self, Nag): self.query_params = params old_CHUNK_SIZE = Bugzilla.BUGZILLA_CHUNK_SIZE try: if chunk_size: Bugzilla.BUGZILLA_CHUNK_SIZE = chunk_size Bugzilla( params, bughandler=self.bughandler, bugdata=bugs, timeout=self.get_config("bz_query_timeout"), ).get_data().wait() finally: Bugzilla.BUGZILLA_CHUNK_SIZE = old_CHUNK_SIZE self.get_comments(bugs) return bugs def commenthandler(self, bug, bugid, data): return def _commenthandler(self, bug, bugid, data): comments = bug["comments"] bugid = str(bugid) if self.has_last_comment_time(): if comments: data[bugid]["last_comment"] = utils.get_human_lag(comments[-1]["time"]) else: data[bugid]["last_comment"] = "" self.commenthandler(bug, bugid, data) def get_comments(self, bugs): """Get the bugs comments""" if self.has_last_comment_time(): bugids = self.get_list_bugs(bugs) Bugzilla( bugids=bugids, commenthandler=self._commenthandler, commentdata=bugs ).get_data().wait() return bugs def has_last_comment_time(self): return False def get_list_bugs(self, bugs): return [x["id"] for x in bugs.values()] def get_documentation(self): return "For more information, please visit [auto_nag documentation](https://wiki.mozilla.org/Release_Management/autonag#{}).".format( self.get_tool_path().replace("/", ".2F") ) def has_bot_set_ni(self, bug): if not self.has_flags: raise Exception return utils.has_bot_set_ni(bug) def set_needinfo(self): if not self.auto_needinfo: return {} template_name = self.needinfo_template() assert bool(template_name) env = Environment(loader=FileSystemLoader("templates")) template = env.get_template(template_name) res = {} doc = self.get_documentation() for ni_mail, info in self.auto_needinfo.items(): nick = info["nickname"] for bugid in info["bugids"]: data = { "comment": {"body": ""}, "flags": [ { "name": "needinfo", "requestee": ni_mail, "status": "?", "new": "true", } ], } comment = None if nick: comment = template.render( nickname=nick, extra=self.get_extra_for_needinfo_template(), plural=utils.plural, bugid=bugid, documentation=doc, ) comment = comment.strip() + "\n" data["comment"]["body"] = comment if bugid not in res: res[bugid] = data else: res[bugid]["flags"] += data["flags"] if comment: res[bugid]["comment"]["body"] = comment return res def has_individual_autofix(self, changes): # check if we have a dictionary with bug numbers as keys # return True if all the keys are bug number # (which means that each bug has its own autofix) return changes and all( isinstance(bugid, six.integer_types) or bugid.isdigit() for bugid in changes ) def get_autofix_change(self): """Get the change to do to autofix the bugs""" return {} def autofix(self, bugs): """Autofix the bugs according to what is returned by get_autofix_change""" ni_changes = self.set_needinfo() change = self.get_autofix_change() if not ni_changes and not change: return bugs self.has_autofix = True new_changes = {} if not self.has_individual_autofix(change): bugids = self.get_list_bugs(bugs) for bugid in bugids: new_changes[bugid] = utils.merge_bz_changes( change, ni_changes.get(bugid, {}) ) else: change = {str(k): v for k, v in change.items()} bugids = set(change.keys()) | set(ni_changes.keys()) for bugid in bugids: mrg = utils.merge_bz_changes( change.get(bugid, {}), ni_changes.get(bugid, {}) ) if mrg: new_changes[bugid] = mrg if self.dryrun or self.test_mode: for bugid, ch in new_changes.items(): logger.info( "The bugs: {}\n will be autofixed with:\n{}".format(bugid, ch) ) else: extra = self.get_db_extra() max_retries = utils.get_config("common", "bugzilla_max_retries", 3) for bugid, ch in new_changes.items(): added = False for _ in range(max_retries): failures = Bugzilla([str(bugid)]).put(ch) if failures: time.sleep(1) else: added = True db.BugChange.add(self.name(), bugid, extra=extra.get(bugid, "")) break if not added: self.failure_callback(bugid) logger.error( "{}: Cannot put data for bug {} (change => {}).".format( self.name(), bugid, ch ) ) return bugs def failure_callback(self, bugid): """Called on Bugzilla.put failures""" return def terminate(self): """Called when everything is done""" return def organize(self, bugs): return utils.organize(bugs, self.columns(), key=self.sort_columns()) def add_to_cache(self, bugs): """Add the bug keys to cache""" if isinstance(bugs, dict): self.cache.add(bugs.keys()) else: self.cache.add(bugs) def get_email(self, date, bug_ids=[]): """Get title and body for the email""" bugs = self.get_bugs(date=date, bug_ids=bug_ids) bugs = self.autofix(bugs) self.add_to_cache(bugs) if bugs: bugs = self.organize(bugs) extra = self.get_extra_for_template() env = Environment(loader=FileSystemLoader("templates")) template = env.get_template(self.template()) message = template.render( date=date, data=bugs, extra=extra, str=str, enumerate=enumerate, plural=utils.plural, no_manager=self.no_manager, table_attrs=self.get_config("table_attrs"), preamble=self.preamble(), ) common = env.get_template("common.html") body = common.render( message=message, query_url=utils.split_long_url(self.query_url) ) return self.get_email_subject(date), body return None, None def send_email(self, date="today"): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info("The tool {} hasn't enough data to run".format(self.name())) return login_info = utils.get_login_info() title, body = self.get_email(date) if title: receivers = self.get_receivers() status = "Success" try: mail.send( login_info["ldap_username"], receivers, title, body, html=True, login=login_info, dryrun=self.dryrun, ) except Exception: logger.exception("Tool {}".format(self.name())) status = "Failure" db.Email.add(self.name(), receivers, "global", status) if isinstance(self, Nag): self.send_mails(title, dryrun=self.dryrun) else: name = self.name().upper() if date: logger.info("{}: No data for {}".format(name, date)) else: logger.info("{}: No data".format(name)) logger.info("Query: {}".format(self.query_url)) def add_custom_arguments(self, parser): pass def parse_custom_arguments(self, args): pass def get_args_parser(self): """Get the argumends from the command line""" parser = argparse.ArgumentParser(description=self.description()) parser.add_argument( "-d", "--dryrun", dest="dryrun", action="store_true", help="Just do the query, and print emails to console without emailing anyone", ) if not self.ignore_date(): parser.add_argument( "-D", "--date", dest="date", action="store", default="today", help="Date for the query", ) self.add_custom_arguments(parser) return parser def run(self): """Run the tool""" args = self.get_args_parser().parse_args() self.parse_custom_arguments(args) date = "" if self.ignore_date() else args.date self.dryrun = args.dryrun self.cache.set_dry_run(self.dryrun) try: self.send_email(date=date) self.terminate() logger.info("Tool {} has finished.".format(self.get_tool_path())) except Exception: logger.exception("Tool {}".format(self.name()))
class BzCleaner(object): def __init__(self): super(BzCleaner, self).__init__() self._set_tool_name() self.has_autofix = False self.no_manager = set() self.auto_needinfo = {} self.has_flags = False self.cache = Cache(self.name(), self.max_days_in_cache()) self.test_mode = utils.get_config('common', 'test', False) def _is_a_bzcleaner_init(self, info): if info[3] == '__init__': frame = info[0] args = inspect.getargvalues(frame) if 'self' in args.locals: zelf = args.locals['self'] return isinstance(zelf, BzCleaner) return False def _set_tool_name(self): stack = inspect.stack() init = [s for s in stack if self._is_a_bzcleaner_init(s)] last = init[-1] info = inspect.getframeinfo(last[0]) base = os.path.dirname(__file__) scripts = os.path.join(base, 'scripts') self.__tool_path__ = os.path.relpath(info.filename, scripts) name = os.path.basename(info.filename) name = os.path.splitext(name)[0] self.__tool_name__ = name def max_days_in_cache(self): """Get the max number of days the data must be kept in cache""" return self.get_config('max_days_in_cache', -1) def description(self): """Get the description for the help""" return '' def name(self): """Get the tool name""" return self.__tool_name__ def get_tool_path(self): """Get the tool path""" return self.__tool_path__ def needinfo_template(self): """Get the txt template filename""" return self.name() + '_needinfo.txt' def template(self): """Get the html template filename""" return self.name() + '.html' def subject(self): """Get the partial email subject""" return self.description() def get_email_subject(self, date): """Get the email subject with a date or not""" af = '[autofix]' if self.has_autofix else '' if date: return '[autonag]{} {} for the {}'.format(af, self.subject(), date) return '[autonag]{} {}'.format(af, self.subject()) def ignore_date(self): """Should we ignore the date ?""" return False def must_run(self, date): """Check if the tool must run for this date""" return True def has_enough_data(self): """Check if the tool has enough data to run""" return True def filter_no_nag_keyword(self): """If True, then remove the bugs with [no-nag] in whiteboard from the bug list""" return True def add_no_manager(self, bugid): self.no_manager.add(str(bugid)) def has_assignee(self): return False def has_needinfo(self): return False def get_mail_to_auto_ni(self, bug): return None def all_include_fields(self): return False def get_max_ni(self): return -1 def ignore_meta(self): return False def columns(self): """The fields to get for the columns in email report""" return ['id', 'summary'] def sort_columns(self): """Returns the key to sort columns""" return None def get_dates(self, date): """Get the dates for the bugzilla query (changedafter and changedbefore fields)""" date = lmdutils.get_date_ymd(date) lookup = self.get_config('days_lookup', 7) start_date = date - relativedelta(days=lookup) end_date = date + relativedelta(days=1) return start_date, end_date def get_extra_for_template(self): """Get extra data to put in the template""" return {} def get_extra_for_needinfo_template(self): """Get extra data to put in the needinfo template""" return {} def get_config(self, entry, default=None): return utils.get_config(self.name(), entry, default=default) def get_bz_params(self, date): """Get the Bugzilla parameters for the search query""" return {} def get_data(self): """Get the data structure to use in the bughandler""" return {} def get_summary(self, bug): return '...' if bug['groups'] else bug['summary'] def has_default_products(self): return True def has_product_component(self): return False def get_product_component(self): return self.prod_comp def get_max_years(self): return self.get_config('max-years', -1) def has_access_to_sec_bugs(self): return self.get_config('sec', True) def handle_bug(self, bug, data): """Implement this function to get all the bugs from the query""" return bug def get_db_extra(self): """Get extra information required for db insertion""" return { bugid: ni_mail for ni_mail, v in self.auto_needinfo.items() for bugid in v['bugids'] } def get_auto_ni_blacklist(self): return set() def add_auto_ni(self, bugid, data): if not data: return ni_mail = data['mail'] if ni_mail in self.get_auto_ni_blacklist(): return if ni_mail in self.auto_needinfo: max_ni = self.get_max_ni() info = self.auto_needinfo[ni_mail] if max_ni <= 0 or len(info['bugids']) < max_ni: info['bugids'].append(str(bugid)) else: self.auto_needinfo[ni_mail] = { 'nickname': data['nickname'], 'bugids': [str(bugid)], } def get_receivers(self): receivers = self.get_config('receivers') if isinstance(receivers, six.string_types): receivers = utils.get_config('common', 'receiver_list', default={})[ receivers ] return receivers def bughandler(self, bug, data): """bug handler for the Bugzilla query""" if bug['id'] in self.cache: return if self.handle_bug(bug, data) is None: return bugid = str(bug['id']) res = {'id': bugid} auto_ni = self.get_mail_to_auto_ni(bug) self.add_auto_ni(bugid, auto_ni) res['summary'] = self.get_summary(bug) if self.has_assignee(): real = bug['assigned_to_detail']['real_name'] if utils.is_no_assignee(bug['assigned_to']): real = 'nobody' if real.strip() == '': real = bug['assigned_to_detail']['name'] if real.strip() == '': real = bug['assigned_to_detail']['email'] res['assignee'] = real if self.has_needinfo(): s = set() for flag in utils.get_needinfo(bug): s.add(flag['requestee']) res['needinfos'] = sorted(s) if self.has_product_component(): for k in ['product', 'component']: res[k] = bug[k] if isinstance(self, Nag): bug = self.set_people_to_nag(bug, res) if not bug: return if bugid in data: data[bugid].update(res) else: data[bugid] = res def amend_bzparams(self, params, bug_ids): """Amend the Bugzilla params""" if not self.all_include_fields(): if 'include_fields' in params: fields = params['include_fields'] if isinstance(fields, list): if 'id' not in fields: fields.append('id') elif isinstance(fields, six.string_types): if fields != 'id': params['include_fields'] = [fields, 'id'] else: params['include_fields'] = [fields, 'id'] else: params['include_fields'] = ['id'] params['include_fields'] += ['summary', 'groups'] if self.has_assignee() and 'assigned_to' not in params['include_fields']: params['include_fields'].append('assigned_to') if self.has_product_component(): if 'product' not in params['include_fields']: params['include_fields'].append('product') if 'component' not in params['include_fields']: params['include_fields'].append('component') if self.has_needinfo() and 'flags' not in params['include_fields']: params['include_fields'].append('flags') if bug_ids: params['bug_id'] = bug_ids if self.filter_no_nag_keyword(): n = utils.get_last_field_num(params) params.update( { 'f' + n: 'status_whiteboard', 'o' + n: 'notsubstring', 'v' + n: '[no-nag]', } ) if self.ignore_meta(): n = utils.get_last_field_num(params) params.update({'f' + n: 'keywords', 'o' + n: 'nowords', 'v' + n: 'meta'}) # Limit the checkers to X years. Unlimited if max_years = -1 max_years = self.get_max_years() if max_years > 0: n = utils.get_last_field_num(params) today = lmdutils.get_date_ymd('today') few_years_ago = today - relativedelta(years=max_years) params.update( {'f' + n: 'creation_ts', 'o' + n: 'greaterthan', 'v' + n: few_years_ago} ) if self.has_default_products(): params['product'] = self.get_config('products') if not self.has_access_to_sec_bugs(): n = utils.get_last_field_num(params) params.update({'f' + n: 'bug_group', 'o' + n: 'isempty'}) self.has_flags = 'flags' in params.get('include_fields', []) def get_bugs(self, date='today', bug_ids=[]): """Get the bugs""" bugs = self.get_data() params = self.get_bz_params(date) self.amend_bzparams(params, bug_ids) self.query_url = utils.get_bz_search_url(params) if isinstance(self, Nag): self.query_params = params Bugzilla( params, bughandler=self.bughandler, bugdata=bugs, timeout=self.get_config('bz_query_timeout'), ).get_data().wait() self.get_comments(bugs) return bugs def commenthandler(self, bug, bugid, data): return def _commenthandler(self, bug, bugid, data): comments = bug['comments'] bugid = str(bugid) if self.has_last_comment_time(): if comments: # get the timestamp of the last comment today = pytz.utc.localize(datetime.utcnow()) dt = dateutil.parser.parse(comments[-1]['time']) data[bugid]['last_comment'] = humanize.naturaldelta(today - dt) else: data[bugid]['last_comment'] = '' self.commenthandler(bug, bugid, data) def get_comments(self, bugs): """Get the bugs comments""" if self.has_last_comment_time(): bugids = self.get_list_bugs(bugs) Bugzilla( bugids=bugids, commenthandler=self._commenthandler, commentdata=bugs ).get_data().wait() return bugs def has_last_comment_time(self): return False def get_list_bugs(self, bugs): return [x['id'] for x in bugs.values()] def has_bot_set_ni(self, bug): if not self.has_flags: raise Exception return utils.has_bot_set_ni(bug) def set_needinfo(self): if not self.auto_needinfo: return {} template_name = self.needinfo_template() assert bool(template_name) env = Environment(loader=FileSystemLoader('templates')) template = env.get_template(template_name) res = {} doc = 'For more information, please visit [auto_nag documentation](https://wiki.mozilla.org/Release_Management/autonag#{}).'.format( self.get_tool_path().replace('/', '.2F') ) for ni_mail, info in self.auto_needinfo.items(): nick = info['nickname'] for bugid in info['bugids']: comment = template.render( nickname=nick, extra=self.get_extra_for_needinfo_template(), plural=utils.plural, bugid=bugid, documentation=doc, ) comment = comment.strip() + '\n' data = { 'comment': {'body': comment}, 'flags': [ { 'name': 'needinfo', 'requestee': ni_mail, 'status': '?', 'new': 'true', } ], } res[bugid] = data return res def has_individual_autofix(self, changes): # check if we have a dictionary with bug numbers as keys # return True if all the keys are bug number # (which means that each bug has its own autofix) return changes and all( isinstance(bugid, six.integer_types) or bugid.isdigit() for bugid in changes ) def get_autofix_change(self): """Get the change to do to autofix the bugs""" return {} def autofix(self, bugs): """Autofix the bugs according to what is returned by get_autofix_change""" ni_changes = self.set_needinfo() change = self.get_autofix_change() if not ni_changes and not change: return bugs self.has_autofix = True new_changes = {} if not self.has_individual_autofix(change): bugids = self.get_list_bugs(bugs) for bugid in bugids: new_changes[bugid] = utils.merge_bz_changes( change, ni_changes.get(bugid, {}) ) else: change = {str(k): v for k, v in change.items()} bugids = set(change.keys()) | set(ni_changes.keys()) for bugid in bugids: mrg = utils.merge_bz_changes( change.get(bugid, {}), ni_changes.get(bugid, {}) ) if mrg: new_changes[bugid] = mrg if self.dryrun or self.test_mode: for bugid, ch in new_changes.items(): logger.info( 'The bugs: {}\n will be autofixed with:\n{}'.format(bugid, ch) ) else: extra = self.get_db_extra() for bugid, ch in new_changes.items(): Bugzilla([str(bugid)]).put(ch) db.BugChange.add(self.name(), bugid, extra=extra.get(bugid, '')) return bugs def organize(self, bugs): return utils.organize(bugs, self.columns(), key=self.sort_columns()) def add_to_cache(self, bugs): """Add the bug keys to cache""" if isinstance(bugs, dict): self.cache.add(bugs.keys()) else: self.cache.add(bugs) def get_email(self, date, bug_ids=[]): """Get title and body for the email""" bugs = self.get_bugs(date=date, bug_ids=bug_ids) bugs = self.autofix(bugs) self.add_to_cache(bugs) if bugs: bugs = self.organize(bugs) extra = self.get_extra_for_template() env = Environment(loader=FileSystemLoader('templates')) template = env.get_template(self.template()) message = template.render( date=date, data=bugs, extra=extra, str=str, enumerate=enumerate, plural=utils.plural, no_manager=self.no_manager, table_attrs=self.get_config('table_attrs'), ) common = env.get_template('common.html') body = common.render(message=message, query_url=self.query_url) return self.get_email_subject(date), body return None, None def send_email(self, date='today'): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info('The tool {} hasn\'t enough data to run'.format(self.name())) return login_info = utils.get_login_info() title, body = self.get_email(date) if title: receivers = self.get_receivers() status = 'Success' try: mail.send( login_info['ldap_username'], receivers, title, body, html=True, login=login_info, dryrun=self.dryrun, ) except: # NOQA logger.exception('Tool {}'.format(self.name())) status = 'Failure' db.Email.add(self.name(), receivers, 'global', status) if isinstance(self, Nag): self.send_mails(title, dryrun=self.dryrun) else: name = self.name().upper() if date: logger.info('{}: No data for {}'.format(name, date)) else: logger.info('{}: No data'.format(name)) logger.info('Query: {}'.format(self.query_url)) def add_custom_arguments(self, parser): pass def parse_custom_arguments(self, args): pass def get_args_parser(self): """Get the argumends from the command line""" parser = argparse.ArgumentParser(description=self.description()) parser.add_argument( '-d', '--dryrun', dest='dryrun', action='store_true', help='Just do the query, and print emails to console without emailing anyone', ) if not self.ignore_date(): parser.add_argument( '-D', '--date', dest='date', action='store', default='today', help='Date for the query', ) self.add_custom_arguments(parser) return parser def run(self): """Run the tool""" args = self.get_args_parser().parse_args() self.parse_custom_arguments(args) date = '' if self.ignore_date() else args.date self.dryrun = args.dryrun self.cache.set_dry_run(self.dryrun) try: self.send_email(date=date) except Exception: logger.exception('Tool {}'.format(self.name()))
class BzCleaner(object): def __init__(self): super(BzCleaner, self).__init__() self._set_tool_name() self.has_autofix = False self.no_manager = set() self.auto_needinfo = {} self.has_flags = False self.cache = Cache(self.name(), self.max_days_in_cache()) self.test_mode = utils.get_config('common', 'test', False) def _is_a_bzcleaner_init(self, info): if info[3] == '__init__': frame = info[0] args = inspect.getargvalues(frame) if 'self' in args.locals: zelf = args.locals['self'] return isinstance(zelf, BzCleaner) return False def _set_tool_name(self): stack = inspect.stack() init = [s for s in stack if self._is_a_bzcleaner_init(s)] last = init[-1] info = inspect.getframeinfo(last[0]) base = os.path.dirname(__file__) scripts = os.path.join(base, 'scripts') self.__tool_path__ = os.path.relpath(info.filename, scripts) name = os.path.basename(info.filename) name = os.path.splitext(name)[0] self.__tool_name__ = name def max_days_in_cache(self): """Get the max number of days the data must be kept in cache""" return self.get_config('max_days_in_cache', -1) def description(self): """Get the description for the help""" return '' def name(self): """Get the tool name""" return self.__tool_name__ def get_tool_path(self): """Get the tool path""" return self.__tool_path__ def needinfo_template(self): """Get the txt template filename""" return self.name() + '_needinfo.txt' def template(self): """Get the html template filename""" return self.name() + '.html' def subject(self): """Get the partial email subject""" return self.description() def get_email_subject(self, date): """Get the email subject with a date or not""" af = '[autofix]' if self.has_autofix else '' if date: return '[autonag]{} {} for the {}'.format(af, self.subject(), date) return '[autonag]{} {}'.format(af, self.subject()) def ignore_date(self): """Should we ignore the date ?""" return False def must_run(self, date): """Check if the tool must run for this date""" return True def has_enough_data(self): """Check if the tool has enough data to run""" return True def filter_no_nag_keyword(self): """If True, then remove the bugs with [no-nag] in whiteboard from the bug list""" return True def add_no_manager(self, bugid): self.no_manager.add(str(bugid)) def has_assignee(self): return False def has_needinfo(self): return False def get_mail_to_auto_ni(self, bug): return None def all_include_fields(self): return False def get_max_ni(self): return -1 def ignore_meta(self): return False def columns(self): """The fields to get for the columns in email report""" return ['id', 'summary'] def sort_columns(self): """Returns the key to sort columns""" return None def get_dates(self, date): """Get the dates for the bugzilla query (changedafter and changedbefore fields)""" date = lmdutils.get_date_ymd(date) lookup = self.get_config('days_lookup', 7) start_date = date - relativedelta(days=lookup) end_date = date + relativedelta(days=1) return start_date, end_date def get_extra_for_template(self): """Get extra data to put in the template""" return {} def get_extra_for_needinfo_template(self): """Get extra data to put in the needinfo template""" return {} def get_config(self, entry, default=None): return utils.get_config(self.name(), entry, default=default) def get_bz_params(self, date): """Get the Bugzilla parameters for the search query""" return {} def get_data(self): """Get the data structure to use in the bughandler""" return {} def get_summary(self, bug): return '...' if bug['groups'] else bug['summary'] def has_default_products(self): return True def has_product_component(self): return False def get_product_component(self): return self.prod_comp def get_max_years(self): return self.get_config('max-years', -1) def has_access_to_sec_bugs(self): return self.get_config('sec', True) def handle_bug(self, bug, data): """Implement this function to get all the bugs from the query""" return bug def get_db_extra(self): """Get extra information required for db insertion""" return { bugid: ni_mail for ni_mail, v in self.auto_needinfo.items() for bugid in v['bugids'] } def get_auto_ni_skiplist(self): return set() def add_auto_ni(self, bugid, data): if not data: return ni_mail = data['mail'] if ni_mail in self.get_auto_ni_skiplist(): return if ni_mail in self.auto_needinfo: max_ni = self.get_max_ni() info = self.auto_needinfo[ni_mail] if max_ni <= 0 or len(info['bugids']) < max_ni: info['bugids'].append(str(bugid)) else: self.auto_needinfo[ni_mail] = { 'nickname': data['nickname'], 'bugids': [str(bugid)], } def get_receivers(self): receivers = self.get_config('receivers') if isinstance(receivers, six.string_types): receivers = utils.get_config('common', 'receiver_list', default={})[receivers] return receivers def bughandler(self, bug, data): """bug handler for the Bugzilla query""" if bug['id'] in self.cache: return if self.handle_bug(bug, data) is None: return bugid = str(bug['id']) res = {'id': bugid} auto_ni = self.get_mail_to_auto_ni(bug) self.add_auto_ni(bugid, auto_ni) res['summary'] = self.get_summary(bug) if self.has_assignee(): real = bug['assigned_to_detail']['real_name'] if utils.is_no_assignee(bug['assigned_to']): real = 'nobody' if real.strip() == '': real = bug['assigned_to_detail']['name'] if real.strip() == '': real = bug['assigned_to_detail']['email'] res['assignee'] = real if self.has_needinfo(): s = set() for flag in utils.get_needinfo(bug): s.add(flag['requestee']) res['needinfos'] = sorted(s) if self.has_product_component(): for k in ['product', 'component']: res[k] = bug[k] if isinstance(self, Nag): bug = self.set_people_to_nag(bug, res) if not bug: return if bugid in data: data[bugid].update(res) else: data[bugid] = res def amend_bzparams(self, params, bug_ids): """Amend the Bugzilla params""" if not self.all_include_fields(): if 'include_fields' in params: fields = params['include_fields'] if isinstance(fields, list): if 'id' not in fields: fields.append('id') elif isinstance(fields, six.string_types): if fields != 'id': params['include_fields'] = [fields, 'id'] else: params['include_fields'] = [fields, 'id'] else: params['include_fields'] = ['id'] params['include_fields'] += ['summary', 'groups'] if self.has_assignee( ) and 'assigned_to' not in params['include_fields']: params['include_fields'].append('assigned_to') if self.has_product_component(): if 'product' not in params['include_fields']: params['include_fields'].append('product') if 'component' not in params['include_fields']: params['include_fields'].append('component') if self.has_needinfo() and 'flags' not in params['include_fields']: params['include_fields'].append('flags') if bug_ids: params['bug_id'] = bug_ids if self.filter_no_nag_keyword(): n = utils.get_last_field_num(params) params.update({ 'f' + n: 'status_whiteboard', 'o' + n: 'notsubstring', 'v' + n: '[no-nag]', }) if self.ignore_meta(): n = utils.get_last_field_num(params) params.update({ 'f' + n: 'keywords', 'o' + n: 'nowords', 'v' + n: 'meta' }) # Limit the checkers to X years. Unlimited if max_years = -1 max_years = self.get_max_years() if max_years > 0: n = utils.get_last_field_num(params) today = lmdutils.get_date_ymd('today') few_years_ago = today - relativedelta(years=max_years) params.update({ 'f' + n: 'creation_ts', 'o' + n: 'greaterthan', 'v' + n: few_years_ago }) if self.has_default_products(): params['product'] = self.get_config('products') if not self.has_access_to_sec_bugs(): n = utils.get_last_field_num(params) params.update({'f' + n: 'bug_group', 'o' + n: 'isempty'}) self.has_flags = 'flags' in params.get('include_fields', []) def get_bugs(self, date='today', bug_ids=[]): """Get the bugs""" bugs = self.get_data() params = self.get_bz_params(date) self.amend_bzparams(params, bug_ids) self.query_url = utils.get_bz_search_url(params) if isinstance(self, Nag): self.query_params = params Bugzilla( params, bughandler=self.bughandler, bugdata=bugs, timeout=self.get_config('bz_query_timeout'), ).get_data().wait() self.get_comments(bugs) return bugs def commenthandler(self, bug, bugid, data): return def _commenthandler(self, bug, bugid, data): comments = bug['comments'] bugid = str(bugid) if self.has_last_comment_time(): if comments: data[bugid]['last_comment'] = utils.get_human_lag( comments[-1]['time']) else: data[bugid]['last_comment'] = '' self.commenthandler(bug, bugid, data) def get_comments(self, bugs): """Get the bugs comments""" if self.has_last_comment_time(): bugids = self.get_list_bugs(bugs) Bugzilla(bugids=bugids, commenthandler=self._commenthandler, commentdata=bugs).get_data().wait() return bugs def has_last_comment_time(self): return False def get_list_bugs(self, bugs): return [x['id'] for x in bugs.values()] def get_documentation(self): return 'For more information, please visit [auto_nag documentation](https://wiki.mozilla.org/Release_Management/autonag#{}).'.format( self.get_tool_path().replace('/', '.2F')) def has_bot_set_ni(self, bug): if not self.has_flags: raise Exception return utils.has_bot_set_ni(bug) def set_needinfo(self): if not self.auto_needinfo: return {} template_name = self.needinfo_template() assert bool(template_name) env = Environment(loader=FileSystemLoader('templates')) template = env.get_template(template_name) res = {} doc = self.get_documentation() for ni_mail, info in self.auto_needinfo.items(): nick = info['nickname'] for bugid in info['bugids']: comment = template.render( nickname=nick, extra=self.get_extra_for_needinfo_template(), plural=utils.plural, bugid=bugid, documentation=doc, ) comment = comment.strip() + '\n' data = { 'comment': { 'body': comment }, 'flags': [{ 'name': 'needinfo', 'requestee': ni_mail, 'status': '?', 'new': 'true', }], } res[bugid] = data return res def has_individual_autofix(self, changes): # check if we have a dictionary with bug numbers as keys # return True if all the keys are bug number # (which means that each bug has its own autofix) return changes and all( isinstance(bugid, six.integer_types) or bugid.isdigit() for bugid in changes) def get_autofix_change(self): """Get the change to do to autofix the bugs""" return {} def autofix(self, bugs): """Autofix the bugs according to what is returned by get_autofix_change""" ni_changes = self.set_needinfo() change = self.get_autofix_change() if not ni_changes and not change: return bugs self.has_autofix = True new_changes = {} if not self.has_individual_autofix(change): bugids = self.get_list_bugs(bugs) for bugid in bugids: new_changes[bugid] = utils.merge_bz_changes( change, ni_changes.get(bugid, {})) else: change = {str(k): v for k, v in change.items()} bugids = set(change.keys()) | set(ni_changes.keys()) for bugid in bugids: mrg = utils.merge_bz_changes(change.get(bugid, {}), ni_changes.get(bugid, {})) if mrg: new_changes[bugid] = mrg if self.dryrun or self.test_mode: for bugid, ch in new_changes.items(): logger.info( 'The bugs: {}\n will be autofixed with:\n{}'.format( bugid, ch)) else: extra = self.get_db_extra() for bugid, ch in new_changes.items(): Bugzilla([str(bugid)]).put(ch) db.BugChange.add(self.name(), bugid, extra=extra.get(bugid, '')) return bugs def organize(self, bugs): return utils.organize(bugs, self.columns(), key=self.sort_columns()) def add_to_cache(self, bugs): """Add the bug keys to cache""" if isinstance(bugs, dict): self.cache.add(bugs.keys()) else: self.cache.add(bugs) def get_email(self, date, bug_ids=[]): """Get title and body for the email""" bugs = self.get_bugs(date=date, bug_ids=bug_ids) bugs = self.autofix(bugs) self.add_to_cache(bugs) if bugs: bugs = self.organize(bugs) extra = self.get_extra_for_template() env = Environment(loader=FileSystemLoader('templates')) template = env.get_template(self.template()) message = template.render( date=date, data=bugs, extra=extra, str=str, enumerate=enumerate, plural=utils.plural, no_manager=self.no_manager, table_attrs=self.get_config('table_attrs'), ) common = env.get_template('common.html') body = common.render(message=message, query_url=self.query_url) return self.get_email_subject(date), body return None, None def send_email(self, date='today'): """Send the email""" if date: date = lmdutils.get_date(date) d = lmdutils.get_date_ymd(date) if isinstance(self, Nag): self.nag_date = d if not self.must_run(d): return if not self.has_enough_data(): logger.info('The tool {} hasn\'t enough data to run'.format( self.name())) return login_info = utils.get_login_info() title, body = self.get_email(date) if title: receivers = self.get_receivers() status = 'Success' try: mail.send( login_info['ldap_username'], receivers, title, body, html=True, login=login_info, dryrun=self.dryrun, ) except: # NOQA logger.exception('Tool {}'.format(self.name())) status = 'Failure' db.Email.add(self.name(), receivers, 'global', status) if isinstance(self, Nag): self.send_mails(title, dryrun=self.dryrun) else: name = self.name().upper() if date: logger.info('{}: No data for {}'.format(name, date)) else: logger.info('{}: No data'.format(name)) logger.info('Query: {}'.format(self.query_url)) def add_custom_arguments(self, parser): pass def parse_custom_arguments(self, args): pass def get_args_parser(self): """Get the argumends from the command line""" parser = argparse.ArgumentParser(description=self.description()) parser.add_argument( '-d', '--dryrun', dest='dryrun', action='store_true', help= 'Just do the query, and print emails to console without emailing anyone', ) if not self.ignore_date(): parser.add_argument( '-D', '--date', dest='date', action='store', default='today', help='Date for the query', ) self.add_custom_arguments(parser) return parser def run(self): """Run the tool""" args = self.get_args_parser().parse_args() self.parse_custom_arguments(args) date = '' if self.ignore_date() else args.date self.dryrun = args.dryrun self.cache.set_dry_run(self.dryrun) try: self.send_email(date=date) except Exception: logger.exception('Tool {}'.format(self.name()))