def process_feed(feed):
channel = fetch_feed(feed)
import epdb; epdb.st()
if feed.etag == channel.etag or channel.status == 304:
print "Feed has not changed since we last checked."
return None
if channel.status >= 400:
print "There was an error parsing this feed."
return None
# the feed has changed (or it is the first time we parse it)
# saving the etag and last_modified fields
feed.etag = channel.get('etag', '')
# some times this is None (it never should) *sigh*
if feed.etag is None:
feed.etag = ''
try:
feed.last_modified = datetime.datetime.fromtimestamp(time.mktime(channel.modified))
except:
feed.last_modified = None
feed.feed_title = channel.feed.get('title', feed.feed_title)
#feed.tagline = channel.feed.get('tagline', feed.tagline)
feed.site_url = channel.feed.get('link', feed.feed_url)
feed.last_checked = datetime.datetime.now()
print "Feed updated"
feed.save()
return channel
def is_pr_merged(self, number, repo):
'''Check if a PR# has been merged or not'''
if number is None:
if C.DEFAULT_BREAKPOINTS:
logging.error('breakpoint!')
import epdb
epdb.st()
raise Exception('Can not check merge state on the number: None')
merged = False
pr = None
try:
pr = repo.get_pullrequest(number)
except Exception as e:
print(e)
if pr:
try:
merged = pr.merged
except Exception as e:
logging.debug(e)
if C.DEFAULT_BREAKPOINTS:
logging.error('breakpoint!')
import epdb
epdb.st()
return merged
def _find_match(self, pattern, exact=False):
logging.debug('exact:{} matching on {}'.format(exact, pattern))
matches = []
if isinstance(pattern, unicode):
pattern = pattern.encode('ascii', 'ignore')
for k, v in self.modules.iteritems():
if v['name'] == pattern:
logging.debug('match {} on name: {}'.format(k, v['name']))
matches = [v]
break
if not matches:
# search by key ... aka the filepath
for k, v in self.modules.iteritems():
if k == pattern:
logging.debug('match {} on key: {}'.format(k, k))
matches = [v]
break
if not matches and not exact:
# search by properties
for k, v in self.modules.iteritems():
for subkey in v.keys():
if v[subkey] == pattern:
logging.debug('match {} on subkey: {}'.format(
k, subkey))
matches.append(v)
if not matches and not exact:
# Levenshtein distance should workaround most typos
distance_map = {}
for k, v in self.modules.iteritems():
mname = v.get('name')
if not mname:
continue
if isinstance(mname, unicode):
mname = mname.encode('ascii', 'ignore')
try:
res = Levenshtein.distance(pattern, mname)
except TypeError as e:
logging.error(e)
import epdb
epdb.st()
distance_map[mname] = [res, k]
res = sorted(distance_map.items(),
key=lambda x: x[1],
reverse=True)
if len(pattern) > 3 and res[-1][1] < 3:
logging.debug('levenshtein ratio match: ({}) {} {}'.format(
res[-1][-1], res[-1][0], pattern))
matches = [self.modules[res[-1][-1]]]
#if matches:
# import epdb; epdb.st()
return matches
def _changes_requested_by(user_reviews, shipits, last_commit,
ready_for_review):
outstanding = set()
for actor, review in user_reviews.items():
if review[u'state'] == u'CHANGES_REQUESTED':
if actor in shipits:
review_time = strip_time_safely(review[u'submitted_at'])
review_time = pytz.utc.localize(review_time)
shipit_time = shipits[actor]
if review_time < shipit_time:
# ignore review older than shipit
# https://github.com/ansible/ansibullbot/issues/671
continue
if ready_for_review:
review_time = strip_time_safely(review[u'submitted_at'])
review_time = pytz.utc.localize(review_time)
if review[
u'commit_id'] != last_commit and review_time < ready_for_review:
# ignore review older than ready_for_review comment wrote by submitter
# but only if the pull request has been updated (meaning the
# last commit isn't the reviewed commit).
continue
outstanding.add(actor)
elif review[u'state'] not in [u'APPROVED', u'COMMENTED']:
logging.error(u'%s unhandled' % review[u'state'])
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb
epdb.st()
return list(outstanding)
def __init__(self, issue, usecache=True, cachedir=None, exclude_users=[]):
self.issue = issue
self.maincache = cachedir
self._waffled_labels = None
if issue.repo.repo_path not in cachedir and 'issues' not in cachedir:
self.cachefile = os.path.join(self.maincache, issue.repo.repo_path,
'issues', str(issue.instance.number),
'history.pickle')
elif issue.repo.repo_path not in cachedir:
self.cachefile = os.path.join(self.maincache, issue.repo.repo_path,
'issues', str(issue.instance.number),
'history.pickle')
elif 'issues' not in cachedir:
self.cachefile = os.path.join(self.maincache, 'issues',
str(issue.instance.number),
'history.pickle')
else:
self.cachefile = os.path.join(self.maincache,
str(issue.instance.number),
'history.pickle')
self.cachedir = os.path.dirname(self.cachefile)
if 'issues' not in self.cachedir:
logging.error(self.cachedir)
if C.DEFAULT_BREAKPOINTS:
logging.error('breakpoint!')
import epdb
epdb.st()
else:
raise Exception('')
if not usecache:
self.history = self.process()
else:
"""Building history is expensive and slow"""
cache = self._load_cache()
if not cache:
logging.info('empty history cache, rebuilding')
self.history = self.process()
logging.info('dumping newly created history cache')
self._dump_cache()
else:
if cache['updated_at'] >= self.issue.instance.updated_at:
logging.info('use cached history')
self.history = cache['history']
else:
logging.info('history out of date, updating')
self.history = self.process()
logging.info('dumping newly created history cache')
self._dump_cache()
if exclude_users:
tmp_history = [x for x in self.history]
for x in tmp_history:
if x['actor'] in exclude_users:
self.history.remove(x)
self.fix_history_tz()
self.history = sorted(self.history, key=itemgetter('created_at'))
def make_www(self):
apps_path = os.path.join(self.webroot, 'apps')
chrome_src = os.path.join(self.checkouts_root, 'insights-chrome')
if not os.path.exists(apps_path):
os.makedirs(apps_path)
'''
# apps/chrome should point at the chrome build path
if not os.path.exists(os.path.join(apps_path, 'chrome')):
cmd = f'ln -s ../../{chrome_src} chrome'
print(cmd)
subprocess.run(cmd, cwd=apps_path, shell=True)
'''
# get index.html and make it point at the right chrome css file ...
if not os.path.exists(os.path.join(self.webroot, 'index.html')):
cmd = 'curl -o index.html https://cloud.redhat.com'
res = subprocess.run(cmd, cwd=self.webroot, shell=True)
import epdb
epdb.st()
cmd = 'sed -i.bak "s/chrome\..*\.css/chrome\.css/" index.html && rm -f index.html.bak'
res = subprocess.run(cmd, cwd=self.webroot, shell=True)
import epdb
epdb.st()
# symlink the silent-check-sso.html
ssof = os.path.join(self.checkouts_root, 'landing-page-frontend',
'dist', 'silent-check-sso.html')
dst = os.path.join(self.webroot, 'silent-check-sso.html')
if not os.path.exists(dst):
os.link(ssof, dst)
def scrape_pullrequest_summaries(self):
prs = {}
base_url = 'https://github.com'
url = base_url
url += '/'
url += self.repo_path
url += '/pulls?'
url += urllib2.quote('q=is open')
page_count = 0
while url:
page_count += 1
rr = self._request_url(url)
if rr.status_code != 200:
break
soup = BeautifulSoup(rr.text, 'html.parser')
data = self._parse_pullrequests_summary_page(soup)
if data['next_page']:
url = base_url + data['next_page']
else:
url = None
if data['prs']:
prs.update(data['prs'])
else:
import epdb; epdb.st()
return prs
def scrape_pullrequest_summaries(self):
prs = {}
url = self.baseurl
url += '/'
url += self.repo_path
url += '/pulls?'
url += urllib2.quote('q=is open')
page_count = 0
while url:
page_count += 1
rr = self._request_url(url)
if rr.status_code != 200:
break
soup = BeautifulSoup(rr.text, 'html.parser')
data = self._parse_pullrequests_summary_page(soup)
if data['next_page']:
url = self.baseurl + data['next_page']
else:
url = None
if data['prs']:
prs.update(data['prs'])
else:
if C.DEFAULT_BREAKPOINTS:
logging.error('breakpoint!')
import epdb
epdb.st()
else:
raise Exception('no "prs" key in data')
return prs
def _fetch_api_url(self, url):
# fetch the url and parse to json
'''
jdata = None
try:
resp = self.instance._requester.requestJson(
'GET',
url
)
data = resp[2]
jdata = json.loads(data)
except Exception as e:
logging.error(e)
'''
jdata = None
while True:
resp = self.instance._requester.requestJson(
'GET',
url
)
data = resp[2]
jdata = json.loads(data)
if isinstance(jdata, dict) and jdata.get('documentation_url'):
if C.DEFAULT_BREAKPOINTS:
import epdb; epdb.st()
else:
raise RateLimitError("rate limited")
else:
break
return jdata
def migrated(self):
if self._migrated is None:
if self.body and 'Copied from original issue' in self.body:
self._migrated = True
migrated_issue = None
idx = self.body.find('Copied from original issue')
msg = self.body[idx:]
try:
migrated_issue = msg.split()[4]
except Exception as e:
logging.error(e)
if C.DEFAULT_BREAKPOINTS:
logging.error('breakpoint!')
import epdb; epdb.st()
else:
raise Exception('split failed')
if migrated_issue.endswith('_'):
migrated_issue = migrated_issue.rstrip('_')
self._migrated_from = migrated_issue
else:
for comment in self.comments:
if comment.body.lower().startswith('migrated from'):
self._migrated = True
bparts = comment.body.split()
self._migrated_from = bparts[2]
break
return self._migrated
def filter_csv(csv_data, columns, separator=';'):
""" Split apart a block of csv text and delete unwanted columns """
try:
rows = csv_data.split('\n')
except ValueError:
import epdb; epdb.st()
rows = [ x.split(separator) for x in rows if x != '' ]
#import epdb; epdb.st()
while True:
remove_col = None
headers = rows[0]
for idx, elem in enumerate(headers):
if elem not in columns:
remove_col = idx
if remove_col:
# delete the column
for idx, valx in enumerate(rows):
del rows[idx][remove_col]
else:
# exit the loop
break
#import epdb; epdb.st()
rows = [ ';'.join(x) for x in rows ]
rows = '\n'.join(rows)
return rows
def data_to_dict(self, url, key=None, usecache=False):
""" Paginate a REST url and convert all data to a dictionary """
datadict = {}
gpages = []
pages = []
# recursively get all pages for this resource
thisurl, pages = self.get_all_urls(url, gpages=None)
for gp in pages:
thisdata = None
try:
thisdata = json.loads(gp.text or gp.content)
except ValueError:
epdb.st()
sys.exit(1)
# add each element to the dict by key
for t in thisdata:
try:
datadict[t[key]] = t
except TypeError:
epdb.st()
return datadict
def run(self):
a_files = sorted(glob.glob('%s/*.png' % self._a))
b_files = sorted(glob.glob('%s/*.png' % self._b))
a = [os.path.basename(x) for x in a_files]
b = [os.path.basename(x) for x in b_files]
c = sorted(set(a + b))
if a != b:
for x in c:
if x not in a:
print('%s does not have %s' % (self._a, x))
if x not in b:
print('%s does not have %s' % (self._b, x))
for x in c:
Ia = Image.open(os.path.join(self._a, x))
Ia = np.array(Ia)
Ib = Image.open(os.path.join(self._b, x))
Ib = np.array(Ib)
this_mse = mse(Ia, Ib)
print(this_mse)
if this_mse > 0.0:
print('%s differs from %s' % (os.path.join(self._a, x), os.path.join(self._b, x)))
make_diff(os.path.join(self._a, x), os.path.join(self._b, x))
import epdb; epdb.st()
import epdb; epdb.st()
def apply_actions(self, iw, actions):
action_meta = {'REDO': False}
if actions.count() > 0:
if self.dump_actions:
self.dump_action_dict(iw, actions)
if self.dry_run:
print("Dry-run specified, skipping execution of actions")
else:
if self.force:
print("Running actions non-interactive as you forced.")
self.execute_actions(iw, actions)
return action_meta
cont = input("Take recommended actions (y/N/a/R/T/DEBUG)? ")
if cont in ('a', 'A'):
sys.exit(0)
if cont in ('Y', 'y'):
self.execute_actions(iw, actions)
if cont == 'T':
self.template_wizard(iw)
action_meta['REDO'] = True
if cont in ('r', 'R'):
action_meta['REDO'] = True
if cont == 'DEBUG':
# put the user into a breakpoint to do live debug
action_meta['REDO'] = True
import epdb; epdb.st()
elif self.always_pause:
print("Skipping, but pause.")
cont = input("Continue (Y/n/a/R/T/DEBUG)? ")
if cont in ('a', 'A', 'n', 'N'):
sys.exit(0)
if cont == 'T':
self.template_wizard(iw)
action_meta['REDO'] = True
elif cont in ('r', 'R'):
action_meta['REDO'] = True
elif cont == 'DEBUG':
# put the user into a breakpoint to do live debug
import epdb; epdb.st()
action_meta['REDO'] = True
elif self.force_description_fixer:
# FIXME: self.FIXED_ISSUES not defined since 1cf9674cd38edbd17aff906d72296c99043e5c13
# either define self.FIXED_ISSUES, either remove this method
# FIXME force_description_fixer is not known by DefaultTriager (only
# by AnsibleTriage): if not removed, move it to AnsibleTriage
if iw.html_url not in self.FIXED_ISSUES:
if self.meta['template_missing_sections']:
changed = self.template_wizard(iw)
if changed:
action_meta['REDO'] = True
self.FIXED_ISSUES.append(iw.html_url)
else:
print("Skipping.")
# let the upper level code redo this issue
return action_meta
def get_pull_request_patches(self):
for k in self.datadict.keys():
#epdb.st()
i = self.datadict[k]['number']
pr = self.datadict[k]['pull_request']
if pr['patch_url'] is not None:
patchfile = os.path.join(self.cachedir, "%s.patch" % k)
if not os.path.isfile(patchfile):
patch_page = self.get_one_page(pr['patch_url'])
self.datadict[k]['patch_text'] = patch_page.text
else:
patch_text = open(patchfile).read()
self.datadict[k]['patch_text'] = patch_text
# generate synthetic meta
patch_meta = self.parse_patch(self.datadict[k]['patch_text'])
for pk in patch_meta.keys():
self.datadict[k][pk] = patch_meta[pk]
try:
open(patchfile, "wb").write(self.datadict[k]['patch_text'])
except UnicodeEncodeError:
pass
except:
import epdb; epdb.st()
def check_response(self, response):
if response and response.status_code == 404:
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb; epdb.st()
else:
raise Exception(u'shippable 404')
def changes_requested_by(user_reviews, shipits, last_commit, ready_for_review):
outstanding = set()
for actor, review in user_reviews.items():
if review[u'state'] == u'CHANGES_REQUESTED':
if actor in shipits:
review_time = datetime.datetime.strptime(review[u'submitted_at'], u'%Y-%m-%dT%H:%M:%SZ')
review_time = pytz.utc.localize(review_time)
shipit_time = shipits[actor]
if review_time < shipit_time:
# ignore review older than shipit
# https://github.com/ansible/ansibullbot/issues/671
continue
if ready_for_review:
review_time = datetime.datetime.strptime(review[u'submitted_at'], u'%Y-%m-%dT%H:%M:%SZ')
review_time = pytz.utc.localize(review_time)
if review[u'commit_id'] != last_commit and review_time < ready_for_review:
# ignore review older than ready_for_review comment wrote by submitter
# but only if the pull request has been updated (meaning the
# last commit isn't the reviewed commit).
continue
outstanding.add(actor)
elif review[u'state'] not in [u'APPROVED', u'COMMENTED']:
logging.error(u'%s unhandled' % review[u'state'])
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb; epdb.st()
return list(outstanding)
def _fetch_api_url(self, url):
# fetch the url and parse to json
'''
jdata = None
try:
resp = self.instance._requester.requestJson(
'GET',
url
)
data = resp[2]
jdata = json.loads(data)
except Exception as e:
logging.error(e)
'''
jdata = None
while True:
resp = self.instance._requester.requestJson(
u'GET',
url
)
data = resp[2]
jdata = json.loads(data)
if isinstance(jdata, dict) and jdata.get(u'documentation_url'):
if C.DEFAULT_BREAKPOINTS:
import epdb; epdb.st()
else:
raise RateLimitError("rate limited")
else:
break
return jdata
def check_public(self, statement, node):
if isinstance(node, astng.Function):
if (node.is_method() and
(not node.name[0] == '_' or (
node.name[0:2] == '__' and node.name[-2:] == '__'))):
classNode = node.parent.parent
if (self.marked_public(classNode)
or self.marked_public(node.root())):
return
if not self.is_function_public(node):
objType, nodeName = self.get_node_name(node)
if nodeName == 'dict.itervalues':
import epdb
epdb.st()
self.add_message('C1000', node=statement.statement(),
args=(objType, nodeName))
elif isinstance(node, astng.Class):
if not self.is_class_public(node):
objType, nodeName = self.get_node_name(node)
self.add_message('C1000', node=statement.statement(),
args=(objType, nodeName))
elif isinstance(node, astng.Yes):
pass
else:
import epdb
epdb.st()
def eval_query(issue, query):
# need a custom filter for ~= and !~
if '~' in query:
import epdb; epdb.st()
res = None
try:
t = Template("{{ %s }}" % query)
res = t.render(**issue)
except Exception as e:
print(e)
pass
if not res:
return False
'''
if res.lower() == 'true':
print "query: %s" % query
print "res: %s" % res
'''
if res.lower() == 'true':
return True
else:
return False
def migrated(self):
if self._migrated is None:
if self.body and u'Copied from original issue' in self.body:
self._migrated = True
migrated_issue = None
idx = self.body.find(u'Copied from original issue')
msg = self.body[idx:]
try:
migrated_issue = msg.split()[4]
except Exception as e:
logging.error(e)
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb; epdb.st()
else:
raise Exception(u'split failed')
if migrated_issue.endswith(u'_'):
migrated_issue = migrated_issue.rstrip(u'_')
self._migrated_from = migrated_issue
else:
for comment in self.comments:
if comment.body.lower().startswith(u'migrated from'):
self._migrated = True
bparts = comment.body.split()
self._migrated_from = bparts[2]
break
return self._migrated
def addJob(self, job):
"""
Add a job to the worker pool.
"""
# Make sure job is hashable.
if isinstance(job, list):
job = tuple(job)
elif isinstance(job, set):
job = frozenset(job)
args = list(self._threadArgs)
args.append(job)
worker = self.workerClass(self._status, args)
if worker.workerId in self._workers:
log.critical("job already being monitored: %s" % (job,))
import epdb
epdb.st()
return
self._workers[worker.workerId] = worker
self._retries.addJob(worker.workerId)
worker.daemon = True
worker.start()
def bulkload_data_dicts(self, keymap):
# bulkload multiple issues at the same time
increment = 50
total = len(self.issue_files)
for i in xrange(increment,total,increment):
indexes = [x for x in xrange((i-increment), (i))]
jfiles = [self.issue_files[x] for x in indexes]
issues = {}
for jfile in jfiles:
jdata = self.load_json_file(jfile)
jdata = self.clean_datadict(jdata, keymap)
jdata['json_file'] = jfile
try:
key = str(jdata['github_namespace']) + \
'_' + str(jdata['github_repo']) + \
'_' + str(jdata['github_number'])
except Exception as e:
print(e)
import epdb; epdb.st()
issues[key] = jdata
if issues:
bulk_add_indexes('github', 'issue', issues)
def scrape_pullrequest_summaries(self):
prs = {}
url = self.baseurl
url += u'/'
url += self.repo_path
url += u'/pulls?'
url += urllib2.quote(u'q=is open')
page_count = 0
while url:
page_count += 1
rr = self._request_url(url)
if rr.status_code != 200:
break
soup = BeautifulSoup(rr.text, u'html.parser')
data = self._parse_pullrequests_summary_page(soup)
if data[u'next_page']:
url = self.baseurl + data[u'next_page']
else:
url = None
if data[u'prs']:
prs.update(data[u'prs'])
else:
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb; epdb.st()
else:
raise Exception(u'no "prs" key in data')
return prs
def inner(*args, **kwargs):
success = False
count = 0
while not success:
count += 1
logging.info('rl\'ed call #%s' % count)
sminutes = 5
try:
x = fn(*args, **kwargs)
success = True
except Exception as e:
# e.status == 403 == blocked from content creation
print(e)
if hasattr(e, 'data') and e.data.get('message'):
if 'blocked from content creation' in e.data['message']:
logging.warning('content creation rate limit exceeded')
sminutes = 2
elif 'rate limit exceeded' in e.data['message']:
logging.warning('general rate limit exceeded')
sminutes = 61
elif isinstance(e, socket.error):
logging.warning('socket error')
sminutes = 5
else:
import epdb; epdb.st()
else:
import epdb; epdb.st()
logging.warning('sleeping %s minutes' % sminutes)
time.sleep(sminutes*60)
return x
def main():
script = "#!/bin/bash\n"
script += "\n"
script += "URL='https://github.com/ansible/ansible/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aopen%20biomassives'\n"
script += "PYTHONPATH=$(pwd) scripts/scrape_github_issues_url $URL\n"
(rc, so, se) = runscript(script)
numbers = json.loads(so)
numbers = sorted(set(numbers))
for number in numbers:
print(number)
iurl = 'https://api.github.com/repos/ansible/ansible/issues/{}'.format(number)
irr = requests.get(iurl, headers=get_headers())
idata = irr.json()
curl = idata['comments_url']
crr = requests.get(curl, headers=get_headers())
comments = crr.json()
if crr.links:
import epdb; epdb.st()
for comment in comments:
if comment['user']['login'] == 'biomassives':
drr = requests.delete(comment['url'], headers=get_headers())
if drr.status_code != 204:
import epdb; epdb.st()
print('done with {}'.format(number))
print('DONE')
def _checkMissingPackages(self):
"""
Check previously committed buckets for any missing source
packages. These are usually caused by delayed releases of errata
into repositories, which play havoc with timestamp ordering.
"""
log.info('checking for any source packages missing from repository')
# Get current group; needed for latest commit timestamp.
group = self._groupmgr.getGroup()
log.info('querying buildLabel %s for all committed packages' %
self._updater._conaryhelper._ccfg.buildLabel)
# Get all versions of all on buildLabel committed to repo.
allPackageVersions = self._updater._conaryhelper._repos.getTroveVersionsByLabel({None: {self._updater._conaryhelper._ccfg.buildLabel: None}})
allSourceVersions = {}
# Build dict of all source versions found in repo.
for source, versions in allPackageVersions.iteritems():
if source.endswith(':source'):
allSourceVersions[source.replace(':source', '')] = [
x.trailingRevision().version for x in versions.keys() ]
missingPackages = {}
missingOrder = {}
# Check all previously ordered/committed buckets for packages
# missing from the repo.
for bucket, packages in sorted(self._errata._order.iteritems()):
if bucket <= group.errataState:
for package in packages:
ver = package.version + '_' + package.release
try:
if ver not in allSourceVersions[package.name]:
# Handle all missing-version cases as exception.
raise KeyError
except KeyError:
try:
if package.getNevra() in self._cfg.allowMissingPackage[bucket]:
log.warn('explicitly allowing missing repository package %s at %s' % (package, bucket))
continue
except KeyError:
pass
log.warn('%s missing from repository' % package)
log.info('? reorderSource %s otherId>%s %s' % (
bucket, group.errataState,
' '.join(package.getNevra())))
missingPackages[package] = bucket
missingOrder.setdefault(bucket, set()).add(package)
if len(missingPackages) > 0:
log.error('missing %s ordered source packages from repository; inspect missingPackages and missingOrder' % len(missingPackages))
import epdb ; epdb.st()
else:
log.info('all expected source packages found in repository')
return missingPackages, missingOrder
def makecheckout(self):
if not os.path.isdir(self.repo_path):
cmd = "git clone %s -b %s %s" % (self.repo_url, self.branch, self.repo_path)
print "# %s" % cmd
rc, out, err = run_command(cmd, cwd=self.tmp_path, shell=False)
if rc != 0:
import epdb; epdb.st()
def crossbreed(MINDIST,NUMLIST,MATCHLIST,APPENDS): NEWVALUES=[] OLDVALUES=[] for row in NUMLIST: if len(str(row)) > 10: OLDVALUES.append(row) epdb.st() elif len(str(row)) < 9: OLDVALUES.append(row) epdb.st() try: SDEX=random.randint(0,int(len(MATCHLIST))-1) except ValueError: SDEX=0 if NUMLIST[row] > (MINDIST+CROSSLIMIT) : NEWVALUES,APPENDS=choose_swap_method(NUMLIST,MATCHLIST,row,APPENDS,NEWVALUES) OLDVALUES.append(row) APPENDS+=1 else: if random.random()<MUTATIONRATE: if in_matchlist(MATCHLIST,row): pass else: OLDVALUES.append(row) NEWVAL=mutate(NUMLIST,MATCHLIST,row) NEWVALUES.append(NEWVAL) APPENDS+=1 NUMLIST=finalize_dchanges(NUMLIST,NEWVALUES,OLDVALUES) return NUMLIST,APPENDS,len(NEWVALUES),len(OLDVALUES)
def main():
args = parse_args()
lines = sys.stdin.readlines()
for line in lines:
numbers = json.loads(line)
numbers = sorted(set(numbers))
for number in numbers:
print(number)
iurl = ISSUE_URL_FMT.format(number)
ir = requests.get(iurl, headers=HEADERS)
idata = ir.json()
try:
labels = [l['name'] for l in idata['labels']]
except KeyError:
continue
if args.remove_label in labels:
url = LABEL_URL_FMT.format(number, '/'+args.remove_label)
r = requests.delete(url, headers=HEADERS)
if r.status_code != 200:
import epdb; epdb.st()
if args.add_label not in labels:
url = LABEL_URL_FMT.format(number, '')
r = requests.post(url, data=json.dumps([args.add_label]), headers=HEADERS)
if r.status_code != 200:
import epdb; epdb.st()
def testWms(base, path, branch=None):
from forester.scm import wms
_wms = wms.WmsRepository(base,path,branch=branch)
stuff = _wms.parseRevisionsFromUri()
print stuff
import epdb;epdb.st()
def get_all_urls(self, url, gpages=[], usecache=False):
""" Recursively fetch all pages for a url """
if not gpages:
gpages = []
i = None
if url not in [x.url for x in gpages]:
i = self.get_one_page(url, usecache=usecache)
if i.url == url:
if i.url in [x.url for x in gpages]:
print "WHAT!!!!"
epdb.st()
else:
gpages.append(i)
# TBD
"""
if hasattr(i, 'links'):
if 'next' in i.links:
if i.links['next']['url'] not in self.fetched:
import epdb; epdb.st()
next_url = i.links['next']['url']
j, gpages = self.get_all_urls(next_url, pages=gpages, usecache=usecache)
"""
else:
pass
return url, gpages
def get_all_pages(self, url, fetched=[], data=[]):
next_page = None
i = self.get_one_page(url)
fetched.append(url)
if not i.ok:
pprint(i)
sys.exit(1)
try:
thisdata = json.loads(i.text or i.content)
except ValueError:
epdb.st()
sys.exit(1)
for t in thisdata:
data.append(t)
if 'link' in i.headers:
#print "# %s" % i.headers['link']
next_page = i.headers['link'].split(';')[0]
next_page = next_page.replace('<', '')
next_page = next_page.replace('>', '')
if next_page is None or next_page in fetched:
return data
else:
data += self.get_all_pages(next_page, fetched=fetched, data=data)
return data
def inner(*args, **kwargs):
success = False
count = 0
while not success:
count += 1
rl = get_rate_limit()
logging.debug('ratelimited call #%s [%s] [%s] [%s]' %
(count,
str(type(args[0])),
fn.func_name,
rl['resources']['core']['remaining']))
if count > 10:
logging.error('HIT 10 loop iteration on call, giving up')
sys.exit(1)
# default to 5 minute sleep
stime = 5*60
try:
x = fn(*args, **kwargs)
success = True
except socket.error as e:
logging.warning('socket error: sleeping 2 minutes %s' % e)
time.sleep(2*60)
except Exception as e:
print(e)
if hasattr(e, 'data') and e.data.get('message'):
if 'blocked from content creation' in e.data['message']:
logging.warning('content creation rate limit exceeded')
stime = 2*60
elif 'Label does not exist' in e.data['message']:
return None
elif 'rate limit exceeded' in e.data['message']:
logging.warning('general rate limit exceeded')
stime = get_reset_time(fn, args)
elif isinstance(e, socket.error):
logging.warning('socket error')
stime = 5*60
elif 'Server Error' in e.data.get('message'):
logging.warning('server error')
stime = 2*60
elif 'Not Found' in e.data.get('message'):
logging.info('object not found')
#stime = 0
#success = True
return None
else:
logging.error('breakpoint!')
import epdb; epdb.st()
else:
logging.error('breakpoint!')
import epdb; epdb.st()
logging.warning('sleeping %s minutes' % (stime/60))
time.sleep(stime)
return x
def loadZMX(zmxfile, ndim=3):
surfaces = []
try:
# First try it as a unicode file.
import io
with io.open(zmxfile, 'r', encoding='utf-16-le') as fh:
lines = fh.readlines()
if not lines[0].startswith(u'\ufeffVERS '):
raise IOError('This is not a utf-16-le .zmx file.')
except IOError:
# Fall back to plain text.
with open(zmxfile, 'r') as fh:
lines = fh.readlines()
if not lines[0].startswith('VERS '):
raise IOError('This is not a text .zmx file.')
apertureStop = None
i = -1
while i < len(lines) - 1:
i += 1
line = lines[i]
if line.startswith('SURF'):
isStop = False
glass = Air()
while i < len(lines) - 1:
i += 1
line = lines[i]
if 'STOP' in line:
isStop = True
if 'TYPE STANDARD' in line or 'TYPE EVENASPH' in line:
ctor = StandardSurface
if 'CURV' in line:
curv = float(line.split()[1])
R = 1.0 / curv if curv != 0 else np.inf
if 'DIAM' in line: # Note, this seems to be the semidiameter even though it's called DIAM!
semidiam = float(line.split()[1])
if semidiam < 0.1: semidiam = 5.0
if 'DISZ' in line:
thickness = float(line.split()[1])
if 'GLAS' in line:
parts = line.split()
glass = SimpleGlass(float(parts[4]), name=parts[1])
if not line.startswith(' '):
i -= 1
try:
surfaces.append(
ctor(thickness=thickness,
R=R,
glass=glass,
semidiam=semidiam))
if isStop:
apertureStop = surfaces[-1]
except Exception as e:
print ctor, e
import epdb
epdb.st()
break
return System(surfaces, apertureStop=apertureStop, ndim=ndim)
def apply_actions(self):
action_meta = {'REDO': False}
if self.safe_force:
self.check_safe_match()
if self.action_count() > 0:
if self.dry_run:
print("Dry-run specified, skipping execution of actions")
else:
if self.force:
print("Running actions non-interactive as you forced.")
self.execute_actions()
return action_meta
cont = raw_input(
"Take recommended actions (y/N/a/R/T/DEBUG)? ")
if cont in ('a', 'A'):
sys.exit(0)
if cont in ('Y', 'y'):
self.execute_actions()
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
if cont == 'r' or cont == 'R':
action_meta['REDO'] = True
if cont == 'DEBUG':
# put the user into a breakpoint to do live debug
action_meta['REDO'] = True
import epdb
epdb.st()
elif self.always_pause:
print("Skipping, but pause.")
cont = raw_input("Continue (Y/n/a/R/T/DEBUG)? ")
if cont in ('a', 'A', 'n', 'N'):
sys.exit(0)
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
elif cont == 'REDO':
action_meta['REDO'] = True
elif cont == 'DEBUG':
# put the user into a breakpoint to do live debug
import epdb
epdb.st()
action_meta['REDO'] = True
elif self.args.force_description_fixer:
if self.issue.html_url not in self.FIXED_ISSUES:
if self.meta['template_missing_sections']:
#import epdb; epdb.st()
changed = self.template_wizard()
if changed:
action_meta['REDO'] = True
self.FIXED_ISSUES.append(self.issue.html_url)
else:
print("Skipping.")
# let the upper level code redo this issue
return action_meta
def check_response(self, response):
if response and response.status_code == 404:
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb
epdb.st()
else:
raise Exception(u'shippable 404')
def testWmsController(base, path, branch=None):
import controller
_c = controller.WmsController(base,path,branch=branch)
repos = _c.findrepos()
print repos
print _c.uri
import epdb;epdb.st()
def elem_to_string(elem):
if isinstance(elem, bs4.element.Tag):
return elem.prettify()
elif isinstance(elem, bs4.element.NavigableString):
return str(elem).replace('\n', '')
else:
import epdb
epdb.st()
def create_topic_json(request):
import epdb
epdb.st()
form = CreateTopicForm(request.POST)
if form.is_valid():
topic = form.save(creator=request.user)
return HttpResponseRedirect('/')
return HttpResponse(status=400)
def _load_events(self):
# parse the yaml events into mocked objects
for ev in self._events:
# make a mocked actor object
actor = ActorMock()
actor.login = ev['actor']['login']
# build an event -or- a comment
if ev['event'] == 'commented':
comment = CommentMock()
comment.actor = actor
comment.user = actor
comment.body = ev['body']
comment.created_at = ev['created_at']
self.comments.append(comment)
elif ev['event'] == 'committed':
commit = CommitTopMock()
#import epdb; epdb.st()
dts = ev['created_at']
commit.commit.committer.date = dts
self.commits.append(commit)
else:
event = EventMock()
event.raw_data = ev.copy()
event.actor = actor
event.id = ev['id']
event.event = ev['event']
event.created_at = ev['created_at']
if ev['event'] == 'labeled' or ev['event'] == 'unlabeled':
label = LabelMock()
label.name = ev['label']['name']
event.label = label
if ev['event'] == 'labeled':
current = [
x.name for x in self.labels if x.name == label.name
]
if not current:
self.labels.append(label)
elif ev['event'] == 'unlabeled':
current = [
x.name for x in self.labels if x.name == label.name
]
if current:
for idx, x in enumerate(self.labels):
if x.name == label.name:
del self.labels[idx]
break
#import epdb; epdb.st()
else:
import epdb
epdb.st()
self.events.append(event)
def create_body(self):
# cleanup remnant colons
for k, v in self.sections.iteritems():
if v.startswith(':\n'):
self.sections[k] = v[2:]
elif v.startswith(': \n'):
self.sections[k] = v[3:]
elif v.startswith(':'):
self.sections[k] = v[1:]
if self.retemplate:
# render to text
for section in self.section_order:
data = self.sections.get(section)
if data is None:
data = ''
self.new_description += '##### ' + section.upper() + '\n'
if section == 'issue type':
self.new_description += data.title()
self.new_description += '\n'
else:
self.new_description += data + '\n'
self.new_description += '\n'
else:
dlines = self.original.split('\n')
for msection in self.missing:
midx = self.section_order.index(msection)
post_section = self.section_order[midx + 1]
if post_section not in self.section_map:
if C.DEFAULT_BREAKPOINTS:
logging.error('breakpoint!')
import epdb
epdb.st()
else:
raise Exception('section not in map')
post_line = self.section_map[post_section]
new_section = ['##### %s' % msection.upper()]
if msection == 'component name':
if not self.meta['is_module']:
if self.issuewrapper.github_type == 'pullrequest':
new_section += self.issuewrapper.files
else:
new_section.append('core')
else:
new_section.append(self.meta['module_match']['name'] +
' module')
new_section.append('')
#import epdb; epdb.st()
for x in reversed(new_section):
dlines.insert(post_line, x)
#import epdb; epdb.st()
self.new_description = '\n'.join(dlines)
def merge(self):
# https://developer.github.com/v3/repos/merging/
# def merge(self, commit_message=github.GithubObject.NotSet)
# squash if 1 committer or just a few commits?
# rebase if >1 committer
# error otherwise
# DEBUG - skipping merges for now until shipit algo is fully vetted
return None
if len(self.commits) == 1 and not self.merge_commits:
url = url = os.path.join(self.pullrequest.url, 'merge')
headers = {}
headers['Accept'] = 'application/vnd.github.polaris-preview+json'
params = {}
params['merge_method'] = 'squash'
resp = self.pullrequest._requester.requestJson("PUT",
url,
headers=headers,
input=params)
if resp[0] != 200 or 'successfully merged' not in resp[2]:
logging.error('merge failed on %s' % self.number)
logging.error('breakpoint!')
import epdb
epdb.st()
sys.exit(1)
else:
logging.error('merge successful for %s' % self.number)
elif (len(self.commits) == len(self.committer_emails)) and \
len(self.commits) <= 10:
url = url = os.path.join(self.pullrequest.url, 'merge')
headers = {}
headers['Accept'] = 'application/vnd.github.polaris-preview+json'
params = {}
params['merge_method'] = 'rebase'
resp = self.pullrequest._requester.requestJson("PUT",
url,
headers=headers,
input=params)
if resp[0] != 200 or 'successfully merged' not in resp[2]:
logging.error('merge failed on %s' % self.number)
logging.error('breakpoint!')
import epdb
epdb.st()
sys.exit(1)
else:
logging.error('merge successful for %s' % self.number)
else:
logging.error('merge skipped for %s' % self.number)
pass
def clean_extra_lines(rawtext):
lines = rawtext.split('\n')
imports_start = None
imports_stop = None
for idx, x in enumerate(lines):
if imports_start is None:
if x.startswith('from ') and not 'absolute_import' in x:
imports_start = idx
continue
if not x:
continue
if x.startswith('from '):
continue
if imports_start and imports_stop is None:
if x[0].isalnum():
imports_stop = idx
break
empty_lines = [x for x in range(imports_start, imports_stop)]
empty_lines = [x for x in empty_lines if not lines[x].strip()]
if not empty_lines:
return rawtext
if len(empty_lines) == 1:
return rawtext
# keep 2 empty lines between imports and definitions
if len(empty_lines) == 2 and (empty_lines[-1] - empty_lines[-2] == 1):
return rawtext
print(lines[imports_start:imports_stop])
while empty_lines:
try:
print('DELETING: %s' % lines[empty_lines[0]])
except IndexError as e:
print(e)
import epdb
epdb.st()
del lines[empty_lines[0]]
del empty_lines[0]
empty_lines = [x - 1 for x in empty_lines]
if [x for x in empty_lines if x <= 0]:
break
if len(empty_lines) <= 2:
break
#import epdb; epdb.st()
rawtext = '\n'.join(lines)
return rawtext
def split_eq_on_highest_addition(eq):
plus_positions = [m.start() for m in re.finditer('\+', eq)]
for plus_position in plus_positions:
child1 = eq[:plus_position]
child2 = eq[plus_position + 1:]
if all([is_closed_form(child) for child in [child1, child2]]):
return [child1, child2]
import epdb
epdb.st()
def testController(controltype, base, path, branch=None):
import controller
_c = controller.Controller.create(controltype,
base,path,branch=branch)
repos = _c.findrepos()
print repos
print _c.uri
import epdb;epdb.st()
def smart_match_module(self):
'''Fuzzy matching for modules'''
if hasattr(self, 'meta'):
self.meta['smart_match_module_called'] = True
match = None
known_modules = []
for k, v in self.module_indexer.modules.iteritems():
known_modules.append(v['name'])
title = self.issue.instance.title.lower()
title = title.replace(':', '')
title_matches = [x for x in known_modules if x + ' module' in title]
if not title_matches:
title_matches = [
x for x in known_modules if title.startswith(x + ' ')
]
if not title_matches:
title_matches = [
x for x in known_modules if ' ' + x + ' ' in title
]
cmatches = None
if self.template_data.get('component name'):
component = self.template_data.get('component name')
cmatches = [x for x in known_modules if x in component]
cmatches = [x for x in cmatches if not '_' + x in component]
import epdb
epdb.st()
# use title ... ?
if title_matches:
cmatches = [x for x in cmatches if x in title_matches]
if cmatches:
if len(cmatches) >= 1:
match = cmatches[0]
if not match:
if 'docs.ansible.com' in component:
#import epdb; epdb.st()
pass
else:
#import epdb; epdb.st()
pass
#import epdb; epdb.st()
if not match:
if len(title_matches) == 1:
match = title_matches[0]
else:
print("module - title matches: %s" % title_matches)
print("module - component matches: %s" % cmatches)
return match
def read_gzip_json(cfile):
try:
with gzip.open(cfile, 'r') as f:
jdata = json.loads(f.read())
except json.decoder.JSONDecodeError as e:
logger.error(e)
import epdb
epdb.st()
return jdata
def module_stats():
global RESMAP
global MI
global MSM
update = True
if MI is None or update:
cfg = get_config()
MI = ModuleIndexer(
cachedir=cfg.get('github_cache_dir', '/tmp/mi.cache'),
checkoutdir=cfg.get('ansible_checkout', '/tmp/ansible.checkout')
)
MI.get_ansible_modules()
if MSM is None or update:
MSM = ModuleStatsMaker(MI)
module_df = MSM.get_grouped_data()
module_df.index.names = ['date']
added_rolmean = module_df['total added'].rolling(window=3)
module_df['total added (rolling mean)'] = added_rolmean.mean()
added_rolstd = pd.rolling_std(module_df['total added'], window=3)
added_rolstd = added_rolstd.to_frame()
module_df['total added (rolling std)'] = added_rolstd['total added']
# must be float for ARMA/ARIMA
cumsum = pd.to_numeric(module_df['cumulative sum'], downcast='float')
cumsum = cumsum.to_frame()
cumsum.columns = ['cumsum']
'''
# don't let the unfinished month screw up the prediction
cumsum.drop(cumsum.index[len(cumsum)-1])
# use the log to set the stationarity
ts_log_data = np.log(cumsum['cumsum'])
# create the model
model = sm.tsa.ARMA(ts_log_data, order=(1,1), freq='M').fit()
#start_date = ts_log_data.index[-1] + Day(1)
#end_date = ts_log_data.index[-1] + Day(60)
#y_forecast = model.predict('2017-04-30', '2018-01-31')
#y_vals = np.exp(y_forecast)
#print(module_df.to_csv())
print(np.exp(model.predict('2017-04-30', '2017-10-31')))
'''
print(module_df.to_csv())
import epdb; epdb.st()
def extract_pr_number_from_comment(rawtext, command='resolved_by_pr'):
# "resolved_by_pr 5136" --> 5136
# "resolved_by_pr #5136" --> 5136
# "resolved_by_pr https://github.com/ansible/ansible/issues/5136" --> 5136
# "resolved_by_pr https://github.com/ansible/ansible/issues/5136" --> 5136
# "resolved_by_pr #5319." --> 5319
index = rawtext.find(command)
index += len(command)
data = rawtext[index:]
data = data.strip()
words = data.split()
# remove non-digit chars
if words:
newword = u''
for char in words[0]:
if char.isdigit():
newword += to_text(char)
if newword:
words[0] = newword
if not words:
return None
number = words[0]
if number.isdigit():
number = int(number)
elif number.startswith(u'#'):
number = number[1:]
number = int(number)
elif number.startswith(u'http'):
urlparts = number.split(u'/')
number = urlparts[-1]
number = int(number)
elif rawtext.find(u'#'):
number = rawtext[rawtext.find(u'#'):]
number = number.replace(u'#', u'')
while number:
if not number[-1].isdigit():
number = number[:-1]
else:
break
try:
number = int(number)
except Exception:
number = None
else:
logging.error(u'NOT SURE HOW TO PARSE %s' % rawtext)
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb
epdb.st()
else:
raise Exception(u'parsing error')
return number
def create_body(self):
# cleanup remnant colons
for k, v in six.iteritems(self.sections):
if v.startswith(u':\n'):
self.sections[k] = v[2:]
elif v.startswith(u': \n'):
self.sections[k] = v[3:]
elif v.startswith(u':'):
self.sections[k] = v[1:]
if self.retemplate:
# render to text
for section in self.section_order:
data = self.sections.get(section)
if data is None:
data = u''
self.new_description += u'##### ' + section.upper() + u'\n'
if section == u'issue type':
self.new_description += data.title()
self.new_description += u'\n'
else:
self.new_description += data + u'\n'
self.new_description += u'\n'
else:
dlines = self.original.split(u'\n')
for msection in self.missing:
midx = self.section_order.index(msection)
post_section = self.section_order[midx + 1]
if post_section not in self.section_map:
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb; epdb.st()
else:
raise Exception(u'section not in map')
post_line = self.section_map[post_section]
new_section = [u'##### %s' % msection.upper()]
if msection == u'component name':
if not self.meta[u'is_module']:
if self.issuewrapper.github_type == u'pullrequest':
new_section += self.issuewrapper.files
else:
new_section.append(u'core')
else:
new_section.append(
self.meta[u'module_match'][u'name'] + u' module'
)
new_section.append(u'')
for x in reversed(new_section):
dlines.insert(post_line, x)
self.new_description = u'\n'.join(dlines)
def load_update_fetch(self, property_name):
'''Fetch a get() property for an object'''
edata = None
events = []
updated = None
update = False
write_cache = False
self.repo.update()
pfile = os.path.join(self.cachedir, u'%s.pickle' % property_name)
pdir = os.path.dirname(pfile)
if not os.path.isdir(pdir):
os.makedirs(pdir)
if os.path.isfile(pfile):
try:
with open(pfile, 'rb') as f:
edata = pickle_load(f)
except Exception as e:
update = True
write_cache = True
# check the timestamp on the cache
if edata:
updated = edata[0]
events = edata[1]
if updated < self.repo.updated_at:
update = True
write_cache = True
# pull all events if timestamp is behind or no events cached
if update or not events:
write_cache = True
updated = self.get_current_time()
try:
methodToCall = getattr(self.repo, u'get_' + property_name)
except Exception as e:
logging.error(e)
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb
epdb.st()
else:
raise Exception(u'unable to get %s' % property_name)
events = [x for x in methodToCall()]
if C.DEFAULT_PICKLE_ISSUES:
if write_cache or not os.path.isfile(pfile):
# need to dump the pickle back to disk
edata = [updated, events]
with open(pfile, 'wb') as f:
pickle_dump(edata, f)
return events
def _get_url(self, url, usecache=False, timeout=TIMEOUT):
cdir = os.path.join(self.cachedir, u'.raw')
if not os.path.isdir(cdir):
os.makedirs(cdir)
#cfile = url.replace(u'https://api.shippable.com/', u'')
cfile = url.replace(SHIPPABLE_URL + '/', u'')
cfile = cfile.replace(u'/', u'_')
cfile = os.path.join(cdir, cfile + u'.json')
gzfile = cfile + u'.gz'
# transparently compress old logs
if os.path.isfile(cfile) and not os.path.isfile(gzfile):
self._compress_cache_file(cfile, gzfile)
rc = None
jdata = None
if os.path.isfile(gzfile):
try:
fdata = self._load_cache_file(gzfile)
rc = fdata[0]
jdata = fdata[1]
except ValueError:
pass
if rc == 400:
return None
resp = None
if not os.path.isfile(gzfile) or not jdata or not usecache:
resp = self.fetch(url, timeout=timeout)
if not resp:
return None
if resp.status_code != 400:
jdata = resp.json()
self._write_cache_file(gzfile, [resp.status_code, jdata])
else:
self._write_cache_file(gzfile, [resp.status_code, {}])
return None
self.check_response(resp)
if not jdata:
#import epdb; epdb.st()
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb
epdb.st()
else:
import epdb
epdb.st()
#raise Exception(u'no json data')
raise ShippableNoData()
return jdata
def get_repo_for_collection(self, fqcn):
today = datetime.datetime.now()
if fqcn not in self._gitrepos:
# reduce the number of requests ...
try:
rurl = self._checkout_index.get(fqcn, {}).get('url')
except AttributeError as e:
print(e)
import epdb
epdb.st()
if rurl is None:
# https://galaxy.ansible.com/api/v2/collections/devoperate/base/
curl = self._baseurl + '/api/v2/collections/' + fqcn.replace(
'.', '/') + '/'
rr = requests.get(curl)
jdata = rr.json()
vurl = jdata['latest_version']['href']
rr2 = requests.get(vurl)
jdata2 = rr2.json()
rurl = jdata2.get('metadata', {}).get('repository')
# reduce the number of clones and rebases ...
needs_rebase = False
if fqcn not in self._checkout_index:
needs_rebase = True
elif not self._checkout_index.get(fqcn, {}).get('checkout'):
needs_rebase = True
elif not self._checkout_index.get(fqcn, {}).get('updated'):
needs_rebase = True
elif (today - self._checkout_index[fqcn]['updated']).days > 0:
needs_rebase = True
logging.info('checkout %s -> %s' % (fqcn, rurl))
grepo = GitRepoWrapper(cachedir=self.cachedir,
repo=rurl,
rebase=needs_rebase)
self._gitrepos[fqcn] = grepo
# keep the last updated time if not rebased ...
if needs_rebase:
updated = datetime.datetime.now()
else:
updated = self._checkout_index[fqcn]['updated']
self._checkout_index[fqcn] = {
'url': rurl,
'fqcn': fqcn,
'checkout': grepo.checkoutdir,
'updated': updated
}
self._save_checkout_index()
return self._gitrepos[fqcn]
def update(self):
self.instance.update()
self._history = \
HistoryWrapper(self, cachedir=self.cachedir, usecache=True)
if self.is_pullrequest():
self.pullrequest.update()
if self.instance.updated_at > self.pullrequest.updated_at:
logging.error('breakpoint!')
import epdb; epdb.st()
def parse_timestamp(self, timestamp):
try:
if 'Z' in timestamp:
timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
else:
timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S')
except Exception as e:
print(e)
import epdb; epdb.st()
return timestamp
def get_ansible_version_major_minor(self, version=None):
if not version:
# old workflow
if not hasattr(self, 'ansible_version'):
import epdb
epdb.st()
return self.version_indexer.get_major_minor(self.ansible_version)
else:
# v3 workflow
return self.version_indexer.get_major_minor(version)
def update(self):
self.instance.update()
self._history = \
HistoryWrapper(self, cachedir=self.cachedir, usecache=True)
if self.is_pullrequest():
self.pullrequest.update()
if self.instance.updated_at > self.pullrequest.updated_at:
import epdb
epdb.st()
def compare_task_sequence(self):
task_data = {}
for k, v in self.idata.items():
tasks = v['profiler_data'][:]
for idx, x in enumerate(tasks):
tn = '%s|%s' % (x['role'], x['task'])
if tn not in task_data:
task_data[tn] = {'observations': []}
# the 9 node results skew the data
if v['cluster_size'] == 9:
continue
task_data[tn]['observations'].append(
(v['cluster_size'], x['mem']))
# capture the mean mem per node for each task
mean_ratios = []
# divide total mem by the number of nodes
for k, v in task_data.items():
obs = v['observations'][:]
if not obs:
continue
obs_ratios = [x[1] / x[0] for x in obs]
obs_mean = np.mean(obs_ratios)
if np.isnan(obs_mean):
import epdb
epdb.st()
mean_ratios.append(obs_mean)
# make some baseline stats
mem_med = np.median(mean_ratios)
mem_mean = np.mean(mean_ratios)
mem_std = np.std(mean_ratios)
for k, v in task_data.items():
obs = v['observations'][:]
obs = sorted(obs, key=itemgetter(0))
obs_ratios = [x[1] / x[0] for x in obs]
if np.mean(obs_ratios) >= (mem_std * 3):
print('###################################')
print('task: ' + k)
print('nodecount+memused observations: ' + str(obs))
print('nodecount+memused ratio: ' + str(obs_ratios))
xvals = [0 for x in obs_ratios]
#import epdb; epdb.st()
import epdb
epdb.st()
def dump_summaries(self, repo_url, issues, filename="summaries"):
"""
[jtanner@fedmac ansibullbot]$ sudo ls -al /proc/10895/fd
total 0
dr-x------ 2 jtanner docker 0 Jan 13 08:51 .
dr-xr-xr-x 9 jtanner docker 0 Jan 13 08:42 ..
lrwx------ 1 jtanner docker 64 Jan 13 08:51 0 -> /dev/pts/2
lrwx------ 1 jtanner docker 64 Jan 13 08:51 1 -> /dev/pts/2
lr-x------ 1 jtanner docker 64 Jan 13 08:51 10 -> /dev/urandom
lrwx------ 1 jtanner d 64 Jan 13 08:51 11 -> /tmp/tmpag2rAb (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 12 -> /tmp/tmpD2plk9 (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 13 -> /tmp/tmpfkSSPA (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 14 -> /tmp/tmpIDY_wb (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 15 -> /tmp/tmpbQBvI2 (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 16 -> /tmp/tmpknP5os (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 17 -> /tmp/tmpDJgEnc (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 18 -> /tmp/tmprWLicP (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 19 -> /tmp/tmpm6d8Qx (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 2 -> /dev/pts/2
lrwx------ 1 jtanner d 64 Jan 13 08:51 20 -> /tmp/tmp_w9Sth (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 21 -> /tmp/tmpRGnb3p (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 22 -> /tmp/tmpiVYdTE (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 23 -> /tmp/tmpEyGXuP (deleted)
l-wx------ 1 jtanner d 64 Jan 13 08:51 3 -> /var/log/ansibullbot.log
lrwx------ 1 jtanner d 64 Jan 13 08:51 4 -> /tmp/tmpIlHOg_ (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 5 -> /tmp/tmp5P8Mya (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 6 -> /tmp/tmpDW4MRD (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 7 -> /tmp/tmpUyBIFB (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 8 -> /tmp/tmpYcWaLe (deleted)
lrwx------ 1 jtanner d 64 Jan 13 08:51 9 -> /tmp/tmp_Qcxrt (deleted)
"""
ns,repo = self.split_repo_url(repo_url)
cachefile = os.path.join(
self.cachedir,
ns,
repo,
'%s.json' % filename
)
if not issues:
if C.DEFAULT_BREAKPOINTS:
logging.error('breakpoint!')
import epdb; epdb.st()
else:
raise Exception('no issues')
tfh, tfn = tempfile.mkstemp()
os.close(tfh)
with open(tfn, 'wb') as f:
f.write(json.dumps(issues, sort_keys=True, indent=2))
if os.path.isfile(cachefile):
os.remove(cachefile)
shutil.move(tfn, cachefile)