def main():
workercount = 8
ts1 = datetime.datetime.now()
# Run the triager ...
#AnsibleTriage(args=sys.argv[1:]).start()
# init just creates all the tools ...
parent = AnsibleTriage(args=sys.argv[1:])
# collect_repos() gets all the issues to be triaged ...
parent.collect_repos()
# get the issue numbers
numbers = parent.repos['ansible/ansible']['issues'].numbers[:]
# make a range of numbers for each worker
chunks = grouper(int(len(numbers) / (workercount - 1)), numbers)
chunks = list(chunks)
# start each worker with it's numbers
pids = []
for chunk in chunks:
p = Process(target=run_triage_worker, args=(chunk, ))
pids.append(p)
[x.start() for x in pids]
[x.join() for x in pids]
ts2 = datetime.datetime.now()
td = (ts2 - ts1).total_seconds()
logger.info('COMPLETED MP TRIAGE IN %s SECONDS' % td)
def test_module_matching(self):
print('')
AT = AnsibleTriage(args={})
jfile = 'tests/fixtures/issue_template_meta.json'
with open(jfile, 'rb') as f:
jdata = json.load(f)
keys = sorted([int(x) for x in jdata.keys()])
for key in keys:
k = str(key)
v = jdata[k]
if '/pull/' in v['html_url']:
continue
print(v['html_url'])
# extract fields from the body
td = extract_template_data(v['body'],
issue_number=key,
issue_class=None)
# schema tests
assert isinstance(td, dict)
assert 'component_raw' in td
assert 'component name' in td
# confirm the raw converted to the component name
assert td['component_raw'] == v['component_raw']
assert td['component name'] == v['component_name']
# confirm module matching works.
mm = AT.find_module_match(v['title'], td)
if v['module_match']:
if mm is None:
import epdb
epdb.st()
elif mm['filepath'] != v['module_match'] and \
mm['name'] != v['module_match']:
import epdb
epdb.st()
elif mm is not None:
import epdb
epdb.st()
def test_module_matching(self):
print('')
AT = AnsibleTriage(args={})
jfile = 'tests/fixtures/issue_template_meta.json'
with open(jfile, 'rb') as f:
jdata = json.load(f)
keys = sorted([int(x) for x in jdata.keys()])
for key in keys:
k = to_text(key)
v = jdata[k]
if '/pull/' in v['html_url']:
continue
print(v['html_url'])
# extract fields from the body
td = extract_template_data(
v['body'],
issue_number=key,
issue_class=None
)
# schema tests
assert isinstance(td, dict)
assert 'component_raw' in td
assert 'component name' in td
# confirm the raw converted to the component name
assert td['component_raw'] == v['component_raw']
assert td['component name'] == v['component_name']
# confirm module matching works.
mm = AT.find_module_match(v['title'], td)
if v['module_match']:
if mm is None:
import epdb; epdb.st()
elif mm['filepath'] != v['module_match'] and \
mm['name'] != v['module_match']:
import epdb; epdb.st()
elif mm is not None:
import epdb; epdb.st()
def create_parser(cls):
parser = AnsibleTriage.create_parser()
# report specific
parser.add_argument("--dest", required=True,
help="store the results in this directory")
return parser
def run_triage_worker(numbers):
thispid = os.getpid()
logger.info('%s started with %s numbers' % (str(thispid), len(numbers)))
tfh, tfn = tempfile.mkstemp(suffix='.json')
#logger.info('%s %s' % (thispid, tfh))
logger.info('%s %s' % (thispid, tfn))
with open(tfn, 'w') as f:
f.write(json.dumps(numbers))
args = sys.argv[1:]
args.append('--id=%s' % tfn)
logger.info(args)
triager = AnsibleTriage(args=args, update_checkouts=False)
triager.run()
os.remove(tfn)
return (tfn)
def test_component_matching(self):
print('')
AT = AnsibleTriage(args={})
AT.file_indexer.get_files()
jfile = 'tests/fixtures/issue_template_meta.json'
with open(jfile, 'rb') as f:
jdata = json.load(f)
keys = sorted([int(x) for x in jdata.keys()])
for key in keys:
k = str(key)
v = jdata[k]
if '/pull/' in v['html_url']:
continue
if not v.get('labels'):
continue
if 'module' in v['labels']:
continue
clabels = [x for x in v['labels'] if x.startswith('c:')]
#if not clabels:
# continue
print(v['html_url'])
# extract fields from the body
td = extract_template_data(v['body'],
issue_number=key,
issue_class=None)
components = AT.file_indexer.find_component_match(
v['title'], v['body'], td)
if components and clabels:
comp_labels = AT.file_indexer.get_component_labels(
AT.valid_labels, components)
print('\t' + str(comp_labels))
def test_no_actions_on_second_run(self, *args, **kwargs):
'''Verify no actions were taken on a subsequent run'''
with BotMockManager() as mm:
print(type(mm))
print(dir(mm))
print(hasattr(mm, 'issuedb'))
mm.issuedb.debug = True
bot_args = [
#'--debug',
'--verbose',
#'--only_issues',
'--ignore_module_commits',
'--cachedir=%s' % mm.cachedir,
'--logfile=%s' % os.path.join(mm.cachedir, 'bot.log'),
'--no_since',
#'--id=2',
#'--id=1',
'--force',
'--ignore_galaxy',
'--ci=shippable',
]
# create a bug report
body = [
'#### ISSUE TYPE', 'bug report', '#### SUMMARY',
'does not work.', '#### COMPONENT NAME', 'vmware_guest',
'#### ANSIBLE VERSION', '2.9.0'
]
mm.issuedb.add_issue(body='\n'.join(body), login='******')
# create a PR that fixes #1
pbody = body[:]
pbody[3] = 'fixes #1'
mm.issuedb.add_issue(body='\n'.join(pbody),
itype='pull',
login='******')
mm.issuedb.add_cross_reference(number=1, reference=2)
mm.issuedb.add_reaction('+1', login='******', number=2)
# add more random issues
for x in range(0, 5):
mm.issuedb.add_issue(body='\n'.join(body), login='******')
# add needs info issue
mm.issuedb.add_issue(body="I don't like issue templates!",
login='******')
AT = AnsibleTriage(args=bot_args)
for x in range(0, 2):
print(
'################################################################'
)
print(' START RUN')
print(
'################################################################'
)
AT.run()
print(
'################################################################'
)
print(' STOP RUN')
print(
'################################################################'
)
print('# issuedb %s' % id(mm.issuedb))
# /tmp/ansibot.test.isxYlS/ansible/ansible/issues/1/meta.json
metafiles = glob.glob(
'%s/*/meta.json' %
os.path.join(mm.cachedir, 'ansible', 'ansible', 'issues'))
metafiles = sorted(metafiles)
for mf in metafiles:
with open(mf, 'r') as f:
meta = json.loads(f.read())
print('checking %s' % mf)
# ensure no actions were created on the last run
for k, v in meta['actions'].items():
#if v:
# import epdb; epdb.st()
assert not v
logging.basicConfig()
def test_supershipit(self, *args, **kwargs):
with BotMockManager() as mm:
botmeta = {
'automerge': True,
'files': {
'lib/ansible/modules/foo/bar.py': {
'support': 'community',
'maintainers': 'zippy',
'supershipit': 'jiffy'
}
},
'macros': {
}
}
bmfile = os.path.join(mm.cachedir, 'BOTMETA.yml')
with open(bmfile, 'w') as f:
f.write(yaml.dump(botmeta))
bot_args = [
'--debug',
'--verbose',
'--ignore_module_commits',
'--cachedir=%s' % mm.cachedir,
'--logfile=%s' % os.path.join(mm.cachedir, 'bot.log'),
'--no_since',
'--force',
'--botmetafile=%s' % bmfile,
'--ignore_galaxy',
'--ci=shippable',
]
# create a bug report
body = [
'#### ISSUE TYPE',
'bugfix pullrequest',
'',
'#### SUMMARY',
'removing some files from ignore.txt',
'',
'#### COMPONENT NAME',
'vmware_guest',
'',
'#### ANSIBLE VERSION',
'2.9.0'
]
ts = get_custom_timestamp()
# this one should get automerged because it only DELETEs from ignore.txt
mm.issuedb.add_issue(body='\n'.join(body), number=1, itype='pull', login='******', created_at=ts)
mm.issuedb.add_issue_file('test/sanity/ignore.txt', number=1, deletions=1, created_at=ts)
mm.issuedb.add_issue_comment('shipit', login='******', number=1)
# this one should NOT get automerged because it ADDs to ignore.txt
mm.issuedb.add_issue(body='\n'.join(body), number=2, itype='pull', login='******', created_at=ts)
mm.issuedb.add_issue_file('test/sanity/ignore.txt', number=2, additions=1, created_at=ts)
mm.issuedb.add_issue_comment('shipit', login='******', number=2)
AT = AnsibleTriage(args=bot_args)
AT.run()
# /tmp/ansibot.test.isxYlS/ansible/ansible/issues/1/meta.json
metafiles = glob.glob('%s/*/meta.json' % os.path.join(mm.cachedir, 'ansible', 'ansible', 'issues'))
metafiles = sorted(metafiles)
for mf in metafiles:
number = int(mf.split('/')[-2])
with open(mf) as f:
meta = json.loads(f.read())
print(mf)
print('shipit: %s' % ('shipit' in meta['actions']['newlabel']))
print('automerge: %s' % ('automerge' in meta['actions']['newlabel']))
print('merge: %s' % meta['actions']['merge'])
print('mergeable: %s' % meta['mergeable'])
print('mergeable_state: %s' % meta['mergeable_state'])
print('automege: %s' % meta['automerge'])
print('automerge_status: %s' % meta['automerge_status'])
if number == 1:
assert meta['actions']['merge']
if number == 2:
assert not meta['actions']['merge']
def main():
initialize_sentry()
# Run the triager ...
AnsibleTriage().start()
def main():
# Run the triager ...
AnsibleTriage().start()
def main():
initialize_sentry()
# Run the triager ...
AnsibleTriage(args=sys.argv[1:]).start()
def main():
description = "Triage issue and pullrequest queues for Ansible.\n"
description += " (NOTE: only useful if you have commit access to"
description += " the repo in question.)"
parser = argparse.ArgumentParser(description=description)
parser.add_argument("--skip_no_update",
action="store_true",
help="skip processing if updated_at hasn't changed")
parser.add_argument(
"--skip_no_update_timeout",
action="store_true",
help="ignore skip logic if last processed >={} days ago".format(
C.DEFAULT_STALE_WINDOW))
parser.add_argument("--collect_only",
action="store_true",
help="stop after caching issues")
parser.add_argument("--skip_module_repos",
action="store_true",
help="ignore the module repos")
parser.add_argument("--module_repos_only",
action="store_true",
help="only process the module repos")
parser.add_argument("--force_rate_limit",
action="store_true",
help="debug: force the rate limit")
parser.add_argument("--sort",
default='desc',
choices=['asc', 'desc'],
help="Direction to sort issues [desc=9-0 asc=0-9]")
parser.add_argument("--logfile",
type=str,
default='/var/log/ansibullbot.log',
help="Send logging to this file")
parser.add_argument("--daemonize",
action="store_true",
help="run in a continuos loop")
parser.add_argument("--daemonize_interval",
type=int,
default=(30 * 60),
help="seconds to sleep between loop iterations")
parser.add_argument("--skiprepo",
action='append',
help="Github repo to skip triaging")
parser.add_argument("--repo",
"-r",
type=str,
help="Github repo to triage (defaults to all)")
parser.add_argument("--only_prs",
action="store_true",
help="Triage pullrequests only")
parser.add_argument("--only_issues",
action="store_true",
help="Triage issues only")
parser.add_argument("--only_open",
action="store_true",
help="Triage open issues|prs only")
parser.add_argument("--only_closed",
action="store_true",
help="Triage closed issues|prs only")
parser.add_argument("--verbose",
"-v",
action="store_true",
help="Verbose output")
parser.add_argument("--dry-run",
"-n",
action="store_true",
help="Don't make any changes")
parser.add_argument("--force",
"-f",
action="store_true",
help="Do not ask questions")
parser.add_argument("--safe_force",
action="store_true",
help="Prompt only on specific actions")
parser.add_argument("--safe_force_script",
type=str,
help="Script to check safe force")
parser.add_argument("--debug",
"-d",
action="store_true",
help="Debug output")
parser.add_argument("--pause",
"-p",
action="store_true",
help="Always pause between prs|issues")
parser.add_argument("--ignore_state",
action="store_true",
help="Do not skip processing closed issues")
# ALWAYS ON NOW
#parser.add_argument("--issue_component_matching", action="store_true",
# help="Try to enumerate the component labels for issues")
parser.add_argument(
"--pr",
"--id",
type=str,
help="Triage only the specified pr|issue (separated by commas)")
parser.add_argument("--start-at",
"--resume_id",
type=int,
help="Start triage at the specified pr|issue")
parser.add_argument("--resume",
action="store_true",
help="pickup right after where the bot last stopped")
parser.add_argument("--no_since",
action="store_true",
help="Do not use the since keyword to fetch issues")
parser.add_argument("--force_description_fixer",
action="store_true",
help="Always invoke the description fixer")
# useful for debugging
parser.add_argument("--dump_actions",
action="store_true",
help="serialize the actions to disk [/tmp/actions]")
parser.add_argument(
"--botmetafile",
type=str,
default=None,
help="Use this filepath for botmeta instead of from the repo")
args = parser.parse_args()
# Run the triager ...
AnsibleTriage(args).start()