def process_feed(feed):

    channel = fetch_feed(feed)

    import epdb; epdb.st()
    if feed.etag == channel.etag or channel.status == 304:
        print "Feed has not changed since we last checked."
        return None

    if channel.status >= 400:
        print "There was an error parsing this feed."
        return None

    # the feed has changed (or it is the first time we parse it)
    # saving the etag and last_modified fields
    feed.etag = channel.get('etag', '')
    # some times this is None (it never should) *sigh*
    if feed.etag is None:
        feed.etag = ''

    try:
        feed.last_modified = datetime.datetime.fromtimestamp(time.mktime(channel.modified))
    except:
        feed.last_modified = None

    feed.feed_title = channel.feed.get('title', feed.feed_title)
    #feed.tagline = channel.feed.get('tagline', feed.tagline)
    feed.site_url = channel.feed.get('link', feed.feed_url)
    feed.last_checked = datetime.datetime.now()

    print "Feed updated"
    feed.save()

    return channel
Exemplo n.º 2
0
    def is_pr_merged(self, number, repo):
        '''Check if a PR# has been merged or not'''

        if number is None:
            if C.DEFAULT_BREAKPOINTS:
                logging.error('breakpoint!')
                import epdb
                epdb.st()
            raise Exception('Can not check merge state on the number: None')

        merged = False
        pr = None
        try:
            pr = repo.get_pullrequest(number)
        except Exception as e:
            print(e)
        if pr:
            try:
                merged = pr.merged
            except Exception as e:
                logging.debug(e)
                if C.DEFAULT_BREAKPOINTS:
                    logging.error('breakpoint!')
                    import epdb
                    epdb.st()
        return merged
Exemplo n.º 3
0
    def _find_match(self, pattern, exact=False):

        logging.debug('exact:{} matching on {}'.format(exact, pattern))

        matches = []

        if isinstance(pattern, unicode):
            pattern = pattern.encode('ascii', 'ignore')

        for k, v in self.modules.iteritems():
            if v['name'] == pattern:
                logging.debug('match {} on name: {}'.format(k, v['name']))
                matches = [v]
                break

        if not matches:
            # search by key ... aka the filepath
            for k, v in self.modules.iteritems():
                if k == pattern:
                    logging.debug('match {} on key: {}'.format(k, k))
                    matches = [v]
                    break

        if not matches and not exact:
            # search by properties
            for k, v in self.modules.iteritems():
                for subkey in v.keys():
                    if v[subkey] == pattern:
                        logging.debug('match {} on subkey: {}'.format(
                            k, subkey))
                        matches.append(v)

        if not matches and not exact:
            # Levenshtein distance should workaround most typos
            distance_map = {}
            for k, v in self.modules.iteritems():
                mname = v.get('name')
                if not mname:
                    continue
                if isinstance(mname, unicode):
                    mname = mname.encode('ascii', 'ignore')
                try:
                    res = Levenshtein.distance(pattern, mname)
                except TypeError as e:
                    logging.error(e)
                    import epdb
                    epdb.st()
                distance_map[mname] = [res, k]
            res = sorted(distance_map.items(),
                         key=lambda x: x[1],
                         reverse=True)
            if len(pattern) > 3 and res[-1][1] < 3:
                logging.debug('levenshtein ratio match: ({}) {} {}'.format(
                    res[-1][-1], res[-1][0], pattern))
                matches = [self.modules[res[-1][-1]]]

            #if matches:
            #    import epdb; epdb.st()

        return matches
Exemplo n.º 4
0
def _changes_requested_by(user_reviews, shipits, last_commit,
                          ready_for_review):
    outstanding = set()
    for actor, review in user_reviews.items():
        if review[u'state'] == u'CHANGES_REQUESTED':
            if actor in shipits:
                review_time = strip_time_safely(review[u'submitted_at'])
                review_time = pytz.utc.localize(review_time)
                shipit_time = shipits[actor]
                if review_time < shipit_time:
                    # ignore review older than shipit
                    # https://github.com/ansible/ansibullbot/issues/671
                    continue

            if ready_for_review:
                review_time = strip_time_safely(review[u'submitted_at'])
                review_time = pytz.utc.localize(review_time)
                if review[
                        u'commit_id'] != last_commit and review_time < ready_for_review:
                    # ignore review older than ready_for_review comment wrote by submitter
                    # but only if the pull request has been updated (meaning the
                    # last commit isn't the reviewed commit).
                    continue

            outstanding.add(actor)
        elif review[u'state'] not in [u'APPROVED', u'COMMENTED']:
            logging.error(u'%s unhandled' % review[u'state'])
            if C.DEFAULT_BREAKPOINTS:
                logging.error(u'breakpoint!')
                import epdb
                epdb.st()
    return list(outstanding)
Exemplo n.º 5
0
    def __init__(self, issue, usecache=True, cachedir=None, exclude_users=[]):
        self.issue = issue
        self.maincache = cachedir
        self._waffled_labels = None

        if issue.repo.repo_path not in cachedir and 'issues' not in cachedir:
            self.cachefile = os.path.join(self.maincache, issue.repo.repo_path,
                                          'issues', str(issue.instance.number),
                                          'history.pickle')
        elif issue.repo.repo_path not in cachedir:
            self.cachefile = os.path.join(self.maincache, issue.repo.repo_path,
                                          'issues', str(issue.instance.number),
                                          'history.pickle')
        elif 'issues' not in cachedir:
            self.cachefile = os.path.join(self.maincache, 'issues',
                                          str(issue.instance.number),
                                          'history.pickle')
        else:
            self.cachefile = os.path.join(self.maincache,
                                          str(issue.instance.number),
                                          'history.pickle')

        self.cachedir = os.path.dirname(self.cachefile)
        if 'issues' not in self.cachedir:
            logging.error(self.cachedir)
            if C.DEFAULT_BREAKPOINTS:
                logging.error('breakpoint!')
                import epdb
                epdb.st()
            else:
                raise Exception('')

        if not usecache:
            self.history = self.process()
        else:
            """Building history is expensive and slow"""
            cache = self._load_cache()
            if not cache:
                logging.info('empty history cache, rebuilding')
                self.history = self.process()
                logging.info('dumping newly created history cache')
                self._dump_cache()
            else:
                if cache['updated_at'] >= self.issue.instance.updated_at:
                    logging.info('use cached history')
                    self.history = cache['history']
                else:
                    logging.info('history out of date, updating')
                    self.history = self.process()
                    logging.info('dumping newly created history cache')
                    self._dump_cache()

        if exclude_users:
            tmp_history = [x for x in self.history]
            for x in tmp_history:
                if x['actor'] in exclude_users:
                    self.history.remove(x)

        self.fix_history_tz()
        self.history = sorted(self.history, key=itemgetter('created_at'))
Exemplo n.º 6
0
    def make_www(self):
        apps_path = os.path.join(self.webroot, 'apps')
        chrome_src = os.path.join(self.checkouts_root, 'insights-chrome')

        if not os.path.exists(apps_path):
            os.makedirs(apps_path)
        '''
        # apps/chrome should point at the chrome build path
        if not os.path.exists(os.path.join(apps_path, 'chrome')):
            cmd = f'ln -s ../../{chrome_src} chrome'
            print(cmd)
            subprocess.run(cmd, cwd=apps_path, shell=True)
        '''

        # get index.html and make it point at the right chrome css file ...
        if not os.path.exists(os.path.join(self.webroot, 'index.html')):
            cmd = 'curl -o index.html https://cloud.redhat.com'
            res = subprocess.run(cmd, cwd=self.webroot, shell=True)
            import epdb
            epdb.st()
            cmd = 'sed -i.bak "s/chrome\..*\.css/chrome\.css/" index.html && rm -f index.html.bak'
            res = subprocess.run(cmd, cwd=self.webroot, shell=True)
            import epdb
            epdb.st()

        # symlink the silent-check-sso.html
        ssof = os.path.join(self.checkouts_root, 'landing-page-frontend',
                            'dist', 'silent-check-sso.html')
        dst = os.path.join(self.webroot, 'silent-check-sso.html')
        if not os.path.exists(dst):
            os.link(ssof, dst)
Exemplo n.º 7
0
    def scrape_pullrequest_summaries(self):

        prs = {}

        base_url = 'https://github.com'
        url = base_url
        url += '/'
        url += self.repo_path
        url += '/pulls?'
        url += urllib2.quote('q=is open')

        page_count = 0
        while url:
            page_count += 1
            rr = self._request_url(url)
            if rr.status_code != 200:
                break
            soup = BeautifulSoup(rr.text, 'html.parser')
            data = self._parse_pullrequests_summary_page(soup)
            if data['next_page']:
                url = base_url + data['next_page']
            else:
                url = None
            if data['prs']:
                prs.update(data['prs'])
            else:
                import epdb; epdb.st()

        return prs
Exemplo n.º 8
0
    def scrape_pullrequest_summaries(self):

        prs = {}

        url = self.baseurl
        url += '/'
        url += self.repo_path
        url += '/pulls?'
        url += urllib2.quote('q=is open')

        page_count = 0
        while url:
            page_count += 1
            rr = self._request_url(url)
            if rr.status_code != 200:
                break
            soup = BeautifulSoup(rr.text, 'html.parser')
            data = self._parse_pullrequests_summary_page(soup)
            if data['next_page']:
                url = self.baseurl + data['next_page']
            else:
                url = None
            if data['prs']:
                prs.update(data['prs'])
            else:
                if C.DEFAULT_BREAKPOINTS:
                    logging.error('breakpoint!')
                    import epdb
                    epdb.st()
                else:
                    raise Exception('no "prs" key in data')

        return prs
Exemplo n.º 9
0
    def _fetch_api_url(self, url):
        # fetch the url and parse to json
        '''
        jdata = None
        try:
            resp = self.instance._requester.requestJson(
                'GET',
                url
            )
            data = resp[2]
            jdata = json.loads(data)
        except Exception as e:
            logging.error(e)
        '''

        jdata = None
        while True:
            resp = self.instance._requester.requestJson(
                'GET',
                url
            )
            data = resp[2]
            jdata = json.loads(data)

            if isinstance(jdata, dict) and jdata.get('documentation_url'):
                if C.DEFAULT_BREAKPOINTS:
                    import epdb; epdb.st()
                else:
                    raise RateLimitError("rate limited")
            else:
                break

        return jdata
Exemplo n.º 10
0
 def migrated(self):
     if self._migrated is None:
         if self.body and 'Copied from original issue' in self.body:
             self._migrated = True
             migrated_issue = None
             idx = self.body.find('Copied from original issue')
             msg = self.body[idx:]
             try:
                 migrated_issue = msg.split()[4]
             except Exception as e:
                 logging.error(e)
                 if C.DEFAULT_BREAKPOINTS:
                     logging.error('breakpoint!')
                     import epdb; epdb.st()
                 else:
                     raise Exception('split failed')
             if migrated_issue.endswith('_'):
                 migrated_issue = migrated_issue.rstrip('_')
             self._migrated_from = migrated_issue
         else:
             for comment in self.comments:
                 if comment.body.lower().startswith('migrated from'):
                     self._migrated = True
                     bparts = comment.body.split()
                     self._migrated_from = bparts[2]
                     break
     return self._migrated
Exemplo n.º 11
0
def filter_csv(csv_data, columns, separator=';'):

    """ Split apart a block of csv text and delete unwanted columns """

    try:
        rows = csv_data.split('\n')    
    except ValueError:
        import epdb; epdb.st()
            
    rows = [ x.split(separator) for x in rows if x != '' ]
    #import epdb; epdb.st()

    while True:    
        remove_col = None

        headers = rows[0]          
        for idx, elem in enumerate(headers):
            if elem not in columns:
                remove_col = idx
        if remove_col:
            # delete the column
            for idx, valx in enumerate(rows):
                del rows[idx][remove_col]
        else:
            # exit the loop
            break        

    #import epdb; epdb.st()
    rows = [ ';'.join(x) for x in rows ]
    rows = '\n'.join(rows)
    return rows
Exemplo n.º 12
0
    def data_to_dict(self, url, key=None, usecache=False):

        """ Paginate a REST url and convert all data to a dictionary """

        datadict = {}
        gpages = []
        pages = []

        # recursively get all pages for this resource
        thisurl, pages = self.get_all_urls(url, gpages=None)
        for gp in pages:
            thisdata = None
            try:
                thisdata = json.loads(gp.text or gp.content)
            except ValueError:
                epdb.st()
                sys.exit(1)

            # add each element to the dict by key
            for t in thisdata:
                try:
                    datadict[t[key]] = t
                except TypeError:
                    epdb.st()

        return datadict
Exemplo n.º 13
0
    def run(self):
        a_files = sorted(glob.glob('%s/*.png' % self._a))
        b_files = sorted(glob.glob('%s/*.png' % self._b))

        a = [os.path.basename(x) for x in a_files]
        b = [os.path.basename(x) for x in b_files]
        c = sorted(set(a + b))

        if a != b:
            for x in c:
                if x not in a:
                    print('%s does not have %s' % (self._a, x))
                if x not in b:
                    print('%s does not have %s' % (self._b, x))

        for x in c:
            Ia = Image.open(os.path.join(self._a, x))
            Ia = np.array(Ia)
            Ib = Image.open(os.path.join(self._b, x))
            Ib = np.array(Ib)

            this_mse = mse(Ia, Ib)
            print(this_mse)
            if this_mse > 0.0:
                print('%s differs from %s' % (os.path.join(self._a, x), os.path.join(self._b, x)))
                make_diff(os.path.join(self._a, x), os.path.join(self._b, x))
                import epdb; epdb.st()

        import epdb; epdb.st()
Exemplo n.º 14
0
    def apply_actions(self, iw, actions):

        action_meta = {'REDO': False}

        if actions.count() > 0:

            if self.dump_actions:
                self.dump_action_dict(iw, actions)

            if self.dry_run:
                print("Dry-run specified, skipping execution of actions")
            else:
                if self.force:
                    print("Running actions non-interactive as you forced.")
                    self.execute_actions(iw, actions)
                    return action_meta
                cont = input("Take recommended actions (y/N/a/R/T/DEBUG)? ")
                if cont in ('a', 'A'):
                    sys.exit(0)
                if cont in ('Y', 'y'):
                    self.execute_actions(iw, actions)
                if cont == 'T':
                    self.template_wizard(iw)
                    action_meta['REDO'] = True
                if cont in ('r', 'R'):
                    action_meta['REDO'] = True
                if cont == 'DEBUG':
                    # put the user into a breakpoint to do live debug
                    action_meta['REDO'] = True
                    import epdb; epdb.st()
        elif self.always_pause:
            print("Skipping, but pause.")
            cont = input("Continue (Y/n/a/R/T/DEBUG)? ")
            if cont in ('a', 'A', 'n', 'N'):
                sys.exit(0)
            if cont == 'T':
                self.template_wizard(iw)
                action_meta['REDO'] = True
            elif cont in ('r', 'R'):
                action_meta['REDO'] = True
            elif cont == 'DEBUG':
                # put the user into a breakpoint to do live debug
                import epdb; epdb.st()
                action_meta['REDO'] = True
        elif self.force_description_fixer:
            # FIXME: self.FIXED_ISSUES not defined since 1cf9674cd38edbd17aff906d72296c99043e5c13
            #        either define self.FIXED_ISSUES, either remove this method
            # FIXME force_description_fixer is not known by DefaultTriager (only
            #       by AnsibleTriage): if not removed, move it to AnsibleTriage
            if iw.html_url not in self.FIXED_ISSUES:
                if self.meta['template_missing_sections']:
                    changed = self.template_wizard(iw)
                    if changed:
                        action_meta['REDO'] = True
                self.FIXED_ISSUES.append(iw.html_url)
        else:
            print("Skipping.")

        # let the upper level code redo this issue
        return action_meta
Exemplo n.º 15
0
    def get_pull_request_patches(self):
        for k in self.datadict.keys():
            #epdb.st()
            i = self.datadict[k]['number']
            pr = self.datadict[k]['pull_request']
            
            if pr['patch_url'] is not None:

                patchfile = os.path.join(self.cachedir, "%s.patch" % k)

                if not os.path.isfile(patchfile):
                    patch_page = self.get_one_page(pr['patch_url'])
                    self.datadict[k]['patch_text'] = patch_page.text
                else:
                    patch_text = open(patchfile).read()
                    self.datadict[k]['patch_text'] = patch_text

                # generate synthetic meta            
                patch_meta = self.parse_patch(self.datadict[k]['patch_text'])
                for pk in patch_meta.keys():
                    self.datadict[k][pk] = patch_meta[pk]
                    
                try:
                    open(patchfile, "wb").write(self.datadict[k]['patch_text'])
                except UnicodeEncodeError:
                    pass
                except:
                    import epdb; epdb.st()
Exemplo n.º 16
0
 def check_response(self, response):
     if response and response.status_code == 404:
         if C.DEFAULT_BREAKPOINTS:
             logging.error(u'breakpoint!')
             import epdb; epdb.st()
         else:
             raise Exception(u'shippable 404')
Exemplo n.º 17
0
def changes_requested_by(user_reviews, shipits, last_commit, ready_for_review):
    outstanding = set()
    for actor, review in user_reviews.items():
        if review[u'state'] == u'CHANGES_REQUESTED':
            if actor in shipits:
                review_time = datetime.datetime.strptime(review[u'submitted_at'], u'%Y-%m-%dT%H:%M:%SZ')
                review_time = pytz.utc.localize(review_time)
                shipit_time = shipits[actor]
                if review_time < shipit_time:
                    # ignore review older than shipit
                    # https://github.com/ansible/ansibullbot/issues/671
                    continue

            if ready_for_review:
                review_time = datetime.datetime.strptime(review[u'submitted_at'], u'%Y-%m-%dT%H:%M:%SZ')
                review_time = pytz.utc.localize(review_time)
                if review[u'commit_id'] != last_commit and review_time < ready_for_review:
                    # ignore review older than ready_for_review comment wrote by submitter
                    # but only if the pull request has been updated (meaning the
                    # last commit isn't the reviewed commit).
                    continue

            outstanding.add(actor)
        elif review[u'state'] not in [u'APPROVED', u'COMMENTED']:
            logging.error(u'%s unhandled' % review[u'state'])
            if C.DEFAULT_BREAKPOINTS:
                logging.error(u'breakpoint!')
                import epdb; epdb.st()
    return list(outstanding)
Exemplo n.º 18
0
    def _fetch_api_url(self, url):
        # fetch the url and parse to json
        '''
        jdata = None
        try:
            resp = self.instance._requester.requestJson(
                'GET',
                url
            )
            data = resp[2]
            jdata = json.loads(data)
        except Exception as e:
            logging.error(e)
        '''

        jdata = None
        while True:
            resp = self.instance._requester.requestJson(
                u'GET',
                url
            )
            data = resp[2]
            jdata = json.loads(data)

            if isinstance(jdata, dict) and jdata.get(u'documentation_url'):
                if C.DEFAULT_BREAKPOINTS:
                    import epdb; epdb.st()
                else:
                    raise RateLimitError("rate limited")
            else:
                break

        return jdata
Exemplo n.º 19
0
 def check_public(self, statement, node):
     if isinstance(node, astng.Function):
         if (node.is_method() and
             (not node.name[0] == '_' or (
                 node.name[0:2] == '__' and node.name[-2:] == '__'))):
             classNode = node.parent.parent
             if (self.marked_public(classNode) 
                 or self.marked_public(node.root())):
                 return
         if not self.is_function_public(node):
             objType, nodeName = self.get_node_name(node)
             if nodeName == 'dict.itervalues':
                 import epdb
                 epdb.st()
             self.add_message('C1000', node=statement.statement(),
                             args=(objType, nodeName))
     elif isinstance(node, astng.Class):
         if not self.is_class_public(node):
             objType, nodeName = self.get_node_name(node)
             self.add_message('C1000', node=statement.statement(),
                             args=(objType, nodeName))
     elif isinstance(node, astng.Yes):
         pass
     else:
         import epdb
         epdb.st()
Exemplo n.º 20
0
def eval_query(issue, query):

    # need a custom filter for ~= and !~
    if '~' in query:
        import epdb; epdb.st()

    res = None
    try:
        t = Template("{{ %s }}" % query)
        res = t.render(**issue)
    except Exception as e:
        print(e)
        pass

    if not res:
        return False

    '''
    if res.lower() == 'true':
        print "query: %s" % query
        print "res: %s" % res
    '''

    if res.lower() == 'true':
        return True
    else:
        return False
Exemplo n.º 21
0
 def migrated(self):
     if self._migrated is None:
         if self.body and u'Copied from original issue' in self.body:
             self._migrated = True
             migrated_issue = None
             idx = self.body.find(u'Copied from original issue')
             msg = self.body[idx:]
             try:
                 migrated_issue = msg.split()[4]
             except Exception as e:
                 logging.error(e)
                 if C.DEFAULT_BREAKPOINTS:
                     logging.error(u'breakpoint!')
                     import epdb; epdb.st()
                 else:
                     raise Exception(u'split failed')
             if migrated_issue.endswith(u'_'):
                 migrated_issue = migrated_issue.rstrip(u'_')
             self._migrated_from = migrated_issue
         else:
             for comment in self.comments:
                 if comment.body.lower().startswith(u'migrated from'):
                     self._migrated = True
                     bparts = comment.body.split()
                     self._migrated_from = bparts[2]
                     break
     return self._migrated
Exemplo n.º 22
0
    def addJob(self, job):
        """
        Add a job to the worker pool.
        """

        # Make sure job is hashable.
        if isinstance(job, list):
            job = tuple(job)
        elif isinstance(job, set):
            job = frozenset(job)

        args = list(self._threadArgs)
        args.append(job)

        worker = self.workerClass(self._status, args)

        if worker.workerId in self._workers:
            log.critical("job already being monitored: %s" % (job,))
            import epdb

            epdb.st()
            return

        self._workers[worker.workerId] = worker
        self._retries.addJob(worker.workerId)
        worker.daemon = True
        worker.start()
Exemplo n.º 23
0
    def bulkload_data_dicts(self, keymap):
        # bulkload multiple issues at the same time
        increment = 50
        total = len(self.issue_files)

        for i in xrange(increment,total,increment):
            indexes = [x for x in xrange((i-increment), (i))]
            jfiles = [self.issue_files[x] for x in indexes]

            issues = {}
            for jfile in jfiles:
                jdata = self.load_json_file(jfile)
                jdata = self.clean_datadict(jdata, keymap)
                jdata['json_file'] = jfile
                try:
                    key = str(jdata['github_namespace']) + \
                        '_' + str(jdata['github_repo']) + \
                        '_' + str(jdata['github_number'])
                except Exception as e:
                    print(e)
                    import epdb; epdb.st()
                issues[key] = jdata

            if issues:
                bulk_add_indexes('github', 'issue', issues)
Exemplo n.º 24
0
    def scrape_pullrequest_summaries(self):

        prs = {}

        url = self.baseurl
        url += u'/'
        url += self.repo_path
        url += u'/pulls?'
        url += urllib2.quote(u'q=is open')

        page_count = 0
        while url:
            page_count += 1
            rr = self._request_url(url)
            if rr.status_code != 200:
                break
            soup = BeautifulSoup(rr.text, u'html.parser')
            data = self._parse_pullrequests_summary_page(soup)
            if data[u'next_page']:
                url = self.baseurl + data[u'next_page']
            else:
                url = None
            if data[u'prs']:
                prs.update(data[u'prs'])
            else:
                if C.DEFAULT_BREAKPOINTS:
                    logging.error(u'breakpoint!')
                    import epdb; epdb.st()
                else:
                    raise Exception(u'no "prs" key in data')

        return prs
Exemplo n.º 25
0
    def inner(*args, **kwargs):

        success = False
        count = 0
        while not success:
            count += 1
            logging.info('rl\'ed call #%s' % count)
            sminutes = 5
            try:
                x = fn(*args, **kwargs)
                success = True
            except Exception as e:
                # e.status == 403 == blocked from content creation
                print(e)
                if hasattr(e, 'data') and e.data.get('message'):
                    if 'blocked from content creation' in e.data['message']:
                        logging.warning('content creation rate limit exceeded')
                        sminutes = 2
                    elif 'rate limit exceeded' in e.data['message']:
                        logging.warning('general rate limit exceeded')
                        sminutes = 61
                    elif isinstance(e, socket.error):
                        logging.warning('socket error')
                        sminutes = 5
                    else:
                        import epdb; epdb.st()
                else:
                    import epdb; epdb.st()

                logging.warning('sleeping %s minutes' % sminutes)
                time.sleep(sminutes*60)

        return x
def main():

    script = "#!/bin/bash\n"
    script += "\n"
    script += "URL='https://github.com/ansible/ansible/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aopen%20biomassives'\n"
    script += "PYTHONPATH=$(pwd) scripts/scrape_github_issues_url $URL\n"

    (rc, so, se) = runscript(script)
    numbers = json.loads(so)
    numbers = sorted(set(numbers))

    for number in numbers:
        print(number)
        iurl = 'https://api.github.com/repos/ansible/ansible/issues/{}'.format(number)
        irr = requests.get(iurl, headers=get_headers())
        idata = irr.json()
        curl = idata['comments_url']
        crr = requests.get(curl, headers=get_headers())
        comments = crr.json()
        if crr.links:
            import epdb; epdb.st()

        for comment in comments:
            if comment['user']['login'] == 'biomassives':
                drr = requests.delete(comment['url'], headers=get_headers())
                if drr.status_code != 204:
                    import epdb; epdb.st()

        print('done with {}'.format(number))

    print('DONE')
Exemplo n.º 27
0
    def _checkMissingPackages(self):
        """
        Check previously committed buckets for any missing source
        packages.  These are usually caused by delayed releases of errata
        into repositories, which play havoc with timestamp ordering.
        """

        log.info('checking for any source packages missing from repository')

        # Get current group; needed for latest commit timestamp.
        group = self._groupmgr.getGroup()

        log.info('querying buildLabel %s for all committed packages' %
                 self._updater._conaryhelper._ccfg.buildLabel)

        # Get all versions of all on buildLabel committed to repo.
        allPackageVersions = self._updater._conaryhelper._repos.getTroveVersionsByLabel({None: {self._updater._conaryhelper._ccfg.buildLabel: None}})

        allSourceVersions = {}

        # Build dict of all source versions found in repo.
        for source, versions in allPackageVersions.iteritems():
            if source.endswith(':source'):
                allSourceVersions[source.replace(':source', '')] = [
                        x.trailingRevision().version for x in versions.keys() ]

        missingPackages = {}
        missingOrder = {}

        # Check all previously ordered/committed buckets for packages
        # missing from the repo.
        for bucket, packages in sorted(self._errata._order.iteritems()):
            if bucket <= group.errataState:
                for package in packages:
                    ver = package.version + '_' + package.release
                    try:
                        if ver not in allSourceVersions[package.name]:
                            # Handle all missing-version cases as exception.
                            raise KeyError
                    except KeyError:
                        try:
                            if package.getNevra() in self._cfg.allowMissingPackage[bucket]:
                                log.warn('explicitly allowing missing repository package %s at %s' % (package, bucket))
                                continue
                        except KeyError:
                            pass
                        log.warn('%s missing from repository' % package)
                        log.info('? reorderSource %s otherId>%s %s' % (
                            bucket, group.errataState,
                            ' '.join(package.getNevra())))
                        missingPackages[package] = bucket
                        missingOrder.setdefault(bucket, set()).add(package)

        if len(missingPackages) > 0:
            log.error('missing %s ordered source packages from repository; inspect missingPackages and missingOrder' % len(missingPackages))
            import epdb ; epdb.st()
        else:
            log.info('all expected source packages found in repository')

        return missingPackages, missingOrder
Exemplo n.º 28
0
 def makecheckout(self):
     if not os.path.isdir(self.repo_path):
         cmd = "git clone %s -b %s %s" % (self.repo_url, self.branch, self.repo_path)
         print "# %s" % cmd
         rc, out, err = run_command(cmd, cwd=self.tmp_path, shell=False)
         if rc != 0:
             import epdb; epdb.st()
Exemplo n.º 29
0
def crossbreed(MINDIST,NUMLIST,MATCHLIST,APPENDS):
	NEWVALUES=[]
	OLDVALUES=[]
	for row in NUMLIST:
		if len(str(row)) > 10:
			OLDVALUES.append(row)
			epdb.st()
		elif len(str(row)) < 9:
			OLDVALUES.append(row)
			epdb.st()
		try:
			SDEX=random.randint(0,int(len(MATCHLIST))-1)
		except ValueError:
			SDEX=0
		if NUMLIST[row] > (MINDIST+CROSSLIMIT) :
			NEWVALUES,APPENDS=choose_swap_method(NUMLIST,MATCHLIST,row,APPENDS,NEWVALUES)
			OLDVALUES.append(row)
			APPENDS+=1 
		else:
			if random.random()<MUTATIONRATE:
				if in_matchlist(MATCHLIST,row):
					pass
				else:
					OLDVALUES.append(row)
					NEWVAL=mutate(NUMLIST,MATCHLIST,row)
					NEWVALUES.append(NEWVAL)
					APPENDS+=1
	NUMLIST=finalize_dchanges(NUMLIST,NEWVALUES,OLDVALUES)
	return NUMLIST,APPENDS,len(NEWVALUES),len(OLDVALUES)
Exemplo n.º 30
0
def main():
    args = parse_args()
    lines = sys.stdin.readlines()
    for line in lines:
        numbers = json.loads(line)
        numbers = sorted(set(numbers))

        for number in numbers:
            print(number)

            iurl = ISSUE_URL_FMT.format(number)
            ir = requests.get(iurl, headers=HEADERS)
            idata = ir.json()
            try:
                labels = [l['name'] for l in idata['labels']]
            except KeyError:
                continue

            if args.remove_label in labels:
                url = LABEL_URL_FMT.format(number, '/'+args.remove_label)
                r = requests.delete(url, headers=HEADERS)
                if r.status_code != 200:
                    import epdb; epdb.st()

                if args.add_label not in labels:
                    url = LABEL_URL_FMT.format(number, '')
                    r = requests.post(url, data=json.dumps([args.add_label]), headers=HEADERS)
                    if r.status_code != 200:
                        import epdb; epdb.st()
Exemplo n.º 31
0
def testWms(base, path, branch=None):
    from forester.scm import wms
    _wms = wms.WmsRepository(base,path,branch=branch)
    stuff = _wms.parseRevisionsFromUri()
    print stuff

    import epdb;epdb.st()
Exemplo n.º 32
0
    def get_all_urls(self, url, gpages=[], usecache=False):

        """ Recursively fetch all pages for a url """

        if not gpages:
            gpages = []

        i = None
        if url not in [x.url for x in gpages]:
            i = self.get_one_page(url, usecache=usecache)

            if i.url == url:

                if i.url in [x.url for x in gpages]:
                    print "WHAT!!!!"
                    epdb.st()
                else:
                    gpages.append(i)

                # TBD
                """
                if hasattr(i, 'links'):
                    if 'next' in i.links:
                        if i.links['next']['url'] not in self.fetched:
                            import epdb; epdb.st()
                            next_url = i.links['next']['url']
                            j, gpages = self.get_all_urls(next_url, pages=gpages, usecache=usecache)
                """

            else:
                pass

        return url, gpages
Exemplo n.º 33
0
    def get_all_pages(self, url, fetched=[], data=[]):
        next_page = None
        i = self.get_one_page(url)
        fetched.append(url)
        if not i.ok:
            pprint(i)
            sys.exit(1)

        try:
            thisdata = json.loads(i.text or i.content)
        except ValueError:
            epdb.st()
            sys.exit(1)
    
        for t in thisdata:
            data.append(t)

        if 'link' in i.headers:
            #print "# %s" % i.headers['link']
            next_page = i.headers['link'].split(';')[0]
            next_page = next_page.replace('<', '')        
            next_page = next_page.replace('>', '')        

        if next_page is None or next_page in fetched:
            return data
        else:
            data += self.get_all_pages(next_page, fetched=fetched, data=data)
            return data
Exemplo n.º 34
0
    def inner(*args, **kwargs):

        success = False
        count = 0
        while not success:
            count += 1
            rl = get_rate_limit()

            logging.debug('ratelimited call #%s [%s] [%s] [%s]' %
                          (count,
                           str(type(args[0])),
                           fn.func_name,
                           rl['resources']['core']['remaining']))

            if count > 10:
                logging.error('HIT 10 loop iteration on call, giving up')
                sys.exit(1)

            # default to 5 minute sleep
            stime = 5*60
            try:
                x = fn(*args, **kwargs)
                success = True
            except socket.error as e:
                logging.warning('socket error: sleeping 2 minutes %s' % e)
                time.sleep(2*60)
            except Exception as e:
                print(e)
                if hasattr(e, 'data') and e.data.get('message'):
                    if 'blocked from content creation' in e.data['message']:
                        logging.warning('content creation rate limit exceeded')
                        stime = 2*60
                    elif 'Label does not exist' in e.data['message']:
                        return None
                    elif 'rate limit exceeded' in e.data['message']:
                        logging.warning('general rate limit exceeded')
                        stime = get_reset_time(fn, args)
                    elif isinstance(e, socket.error):
                        logging.warning('socket error')
                        stime = 5*60
                    elif 'Server Error' in e.data.get('message'):
                        logging.warning('server error')
                        stime = 2*60
                    elif 'Not Found' in e.data.get('message'):
                        logging.info('object not found')
                        #stime = 0
                        #success = True
                        return None
                    else:
                        logging.error('breakpoint!')
                        import epdb; epdb.st()
                else:
                    logging.error('breakpoint!')
                    import epdb; epdb.st()

                logging.warning('sleeping %s minutes' % (stime/60))
                time.sleep(stime)

        return x
Exemplo n.º 35
0
    def loadZMX(zmxfile, ndim=3):
        surfaces = []
        try:
            # First try it as a unicode file.
            import io
            with io.open(zmxfile, 'r', encoding='utf-16-le') as fh:
                lines = fh.readlines()
            if not lines[0].startswith(u'\ufeffVERS '):
                raise IOError('This is not a utf-16-le .zmx file.')
        except IOError:
            # Fall back to plain text.
            with open(zmxfile, 'r') as fh:
                lines = fh.readlines()
            if not lines[0].startswith('VERS '):
                raise IOError('This is not a text .zmx file.')

        apertureStop = None
        i = -1
        while i < len(lines) - 1:
            i += 1
            line = lines[i]
            if line.startswith('SURF'):
                isStop = False
                glass = Air()
                while i < len(lines) - 1:
                    i += 1
                    line = lines[i]
                    if 'STOP' in line:
                        isStop = True
                    if 'TYPE STANDARD' in line or 'TYPE EVENASPH' in line:
                        ctor = StandardSurface
                    if 'CURV' in line:
                        curv = float(line.split()[1])
                        R = 1.0 / curv if curv != 0 else np.inf
                    if 'DIAM' in line:  # Note, this seems to be the semidiameter even though it's called DIAM!
                        semidiam = float(line.split()[1])
                        if semidiam < 0.1: semidiam = 5.0
                    if 'DISZ' in line:
                        thickness = float(line.split()[1])
                    if 'GLAS' in line:
                        parts = line.split()
                        glass = SimpleGlass(float(parts[4]), name=parts[1])
                    if not line.startswith(' '):
                        i -= 1
                        try:
                            surfaces.append(
                                ctor(thickness=thickness,
                                     R=R,
                                     glass=glass,
                                     semidiam=semidiam))
                            if isStop:
                                apertureStop = surfaces[-1]
                        except Exception as e:
                            print ctor, e
                            import epdb
                            epdb.st()
                        break

        return System(surfaces, apertureStop=apertureStop, ndim=ndim)
Exemplo n.º 36
0
    def apply_actions(self):

        action_meta = {'REDO': False}

        if self.safe_force:
            self.check_safe_match()

        if self.action_count() > 0:
            if self.dry_run:
                print("Dry-run specified, skipping execution of actions")
            else:
                if self.force:
                    print("Running actions non-interactive as you forced.")
                    self.execute_actions()
                    return action_meta
                cont = raw_input(
                    "Take recommended actions (y/N/a/R/T/DEBUG)? ")
                if cont in ('a', 'A'):
                    sys.exit(0)
                if cont in ('Y', 'y'):
                    self.execute_actions()
                if cont == 'T':
                    self.template_wizard()
                    action_meta['REDO'] = True
                if cont == 'r' or cont == 'R':
                    action_meta['REDO'] = True
                if cont == 'DEBUG':
                    # put the user into a breakpoint to do live debug
                    action_meta['REDO'] = True
                    import epdb
                    epdb.st()
        elif self.always_pause:
            print("Skipping, but pause.")
            cont = raw_input("Continue (Y/n/a/R/T/DEBUG)? ")
            if cont in ('a', 'A', 'n', 'N'):
                sys.exit(0)
            if cont == 'T':
                self.template_wizard()
                action_meta['REDO'] = True
            elif cont == 'REDO':
                action_meta['REDO'] = True
            elif cont == 'DEBUG':
                # put the user into a breakpoint to do live debug
                import epdb
                epdb.st()
                action_meta['REDO'] = True
        elif self.args.force_description_fixer:
            if self.issue.html_url not in self.FIXED_ISSUES:
                if self.meta['template_missing_sections']:
                    #import epdb; epdb.st()
                    changed = self.template_wizard()
                    if changed:
                        action_meta['REDO'] = True
                self.FIXED_ISSUES.append(self.issue.html_url)
        else:
            print("Skipping.")

        # let the upper level code redo this issue
        return action_meta
Exemplo n.º 37
0
 def check_response(self, response):
     if response and response.status_code == 404:
         if C.DEFAULT_BREAKPOINTS:
             logging.error(u'breakpoint!')
             import epdb
             epdb.st()
         else:
             raise Exception(u'shippable 404')
Exemplo n.º 38
0
def testWmsController(base, path, branch=None):
    import controller
    _c = controller.WmsController(base,path,branch=branch)

    repos = _c.findrepos()
    print repos
    print _c.uri
    import epdb;epdb.st()
Exemplo n.º 39
0
 def elem_to_string(elem):
     if isinstance(elem, bs4.element.Tag):
         return elem.prettify()
     elif isinstance(elem, bs4.element.NavigableString):
         return str(elem).replace('\n', '')
     else:
         import epdb
         epdb.st()
Exemplo n.º 40
0
def create_topic_json(request):
    import epdb
    epdb.st()
    form = CreateTopicForm(request.POST)
    if form.is_valid():
        topic = form.save(creator=request.user)
        return HttpResponseRedirect('/')
    return HttpResponse(status=400)
Exemplo n.º 41
0
    def _load_events(self):

        # parse the yaml events into mocked objects
        for ev in self._events:

            # make a mocked actor object
            actor = ActorMock()
            actor.login = ev['actor']['login']

            # build an event -or- a comment
            if ev['event'] == 'commented':
                comment = CommentMock()
                comment.actor = actor
                comment.user = actor
                comment.body = ev['body']
                comment.created_at = ev['created_at']
                self.comments.append(comment)

            elif ev['event'] == 'committed':
                commit = CommitTopMock()
                #import epdb; epdb.st()
                dts = ev['created_at']
                commit.commit.committer.date = dts
                self.commits.append(commit)

            else:
                event = EventMock()
                event.raw_data = ev.copy()
                event.actor = actor
                event.id = ev['id']
                event.event = ev['event']
                event.created_at = ev['created_at']

                if ev['event'] == 'labeled' or ev['event'] == 'unlabeled':
                    label = LabelMock()
                    label.name = ev['label']['name']
                    event.label = label
                    if ev['event'] == 'labeled':
                        current = [
                            x.name for x in self.labels if x.name == label.name
                        ]
                        if not current:
                            self.labels.append(label)
                    elif ev['event'] == 'unlabeled':
                        current = [
                            x.name for x in self.labels if x.name == label.name
                        ]
                        if current:
                            for idx, x in enumerate(self.labels):
                                if x.name == label.name:
                                    del self.labels[idx]
                                    break
                        #import epdb; epdb.st()
                else:
                    import epdb
                    epdb.st()

                self.events.append(event)
Exemplo n.º 42
0
    def create_body(self):

        # cleanup remnant colons
        for k, v in self.sections.iteritems():
            if v.startswith(':\n'):
                self.sections[k] = v[2:]
            elif v.startswith(': \n'):
                self.sections[k] = v[3:]
            elif v.startswith(':'):
                self.sections[k] = v[1:]

        if self.retemplate:
            # render to text
            for section in self.section_order:
                data = self.sections.get(section)
                if data is None:
                    data = ''
                self.new_description += '##### ' + section.upper() + '\n'
                if section == 'issue type':
                    self.new_description += data.title()
                    self.new_description += '\n'
                else:
                    self.new_description += data + '\n'
                self.new_description += '\n'
        else:
            dlines = self.original.split('\n')
            for msection in self.missing:
                midx = self.section_order.index(msection)
                post_section = self.section_order[midx + 1]

                if post_section not in self.section_map:
                    if C.DEFAULT_BREAKPOINTS:
                        logging.error('breakpoint!')
                        import epdb
                        epdb.st()
                    else:
                        raise Exception('section not in map')

                post_line = self.section_map[post_section]

                new_section = ['##### %s' % msection.upper()]
                if msection == 'component name':
                    if not self.meta['is_module']:
                        if self.issuewrapper.github_type == 'pullrequest':
                            new_section += self.issuewrapper.files
                        else:
                            new_section.append('core')
                    else:
                        new_section.append(self.meta['module_match']['name'] +
                                           ' module')
                new_section.append('')

                #import epdb; epdb.st()
                for x in reversed(new_section):
                    dlines.insert(post_line, x)

            #import epdb; epdb.st()
            self.new_description = '\n'.join(dlines)
Exemplo n.º 43
0
    def merge(self):

        # https://developer.github.com/v3/repos/merging/
        # def merge(self, commit_message=github.GithubObject.NotSet)

        # squash if 1 committer or just a few commits?
        # rebase if >1 committer
        # error otherwise

        # DEBUG - skipping merges for now until shipit algo is fully vetted
        return None

        if len(self.commits) == 1 and not self.merge_commits:

            url = url = os.path.join(self.pullrequest.url, 'merge')
            headers = {}
            headers['Accept'] = 'application/vnd.github.polaris-preview+json'
            params = {}
            params['merge_method'] = 'squash'
            resp = self.pullrequest._requester.requestJson("PUT",
                                                           url,
                                                           headers=headers,
                                                           input=params)

            if resp[0] != 200 or 'successfully merged' not in resp[2]:
                logging.error('merge failed on %s' % self.number)
                logging.error('breakpoint!')
                import epdb
                epdb.st()
                sys.exit(1)
            else:
                logging.error('merge successful for %s' % self.number)

        elif (len(self.commits) == len(self.committer_emails)) and \
                len(self.commits) <= 10:

            url = url = os.path.join(self.pullrequest.url, 'merge')
            headers = {}
            headers['Accept'] = 'application/vnd.github.polaris-preview+json'
            params = {}
            params['merge_method'] = 'rebase'
            resp = self.pullrequest._requester.requestJson("PUT",
                                                           url,
                                                           headers=headers,
                                                           input=params)

            if resp[0] != 200 or 'successfully merged' not in resp[2]:
                logging.error('merge failed on %s' % self.number)
                logging.error('breakpoint!')
                import epdb
                epdb.st()
                sys.exit(1)
            else:
                logging.error('merge successful for %s' % self.number)

        else:
            logging.error('merge skipped for %s' % self.number)
            pass
Exemplo n.º 44
0
def clean_extra_lines(rawtext):
    lines = rawtext.split('\n')

    imports_start = None
    imports_stop = None
    for idx, x in enumerate(lines):
        if imports_start is None:
            if x.startswith('from ') and not 'absolute_import' in x:
                imports_start = idx
                continue

        if not x:
            continue

        if x.startswith('from '):
            continue

        if imports_start and imports_stop is None:
            if x[0].isalnum():
                imports_stop = idx
                break

    empty_lines = [x for x in range(imports_start, imports_stop)]
    empty_lines = [x for x in empty_lines if not lines[x].strip()]

    if not empty_lines:
        return rawtext

    if len(empty_lines) == 1:
        return rawtext

    # keep 2 empty lines between imports and definitions
    if len(empty_lines) == 2 and (empty_lines[-1] - empty_lines[-2] == 1):
        return rawtext

    print(lines[imports_start:imports_stop])

    while empty_lines:
        try:
            print('DELETING: %s' % lines[empty_lines[0]])
        except IndexError as e:
            print(e)
            import epdb
            epdb.st()
        del lines[empty_lines[0]]
        del empty_lines[0]
        empty_lines = [x - 1 for x in empty_lines]
        if [x for x in empty_lines if x <= 0]:
            break

        if len(empty_lines) <= 2:
            break

        #import epdb; epdb.st()

    rawtext = '\n'.join(lines)
    return rawtext
Exemplo n.º 45
0
 def split_eq_on_highest_addition(eq):
     plus_positions = [m.start() for m in re.finditer('\+', eq)]
     for plus_position in plus_positions:
         child1 = eq[:plus_position]
         child2 = eq[plus_position + 1:]
         if all([is_closed_form(child) for child in [child1, child2]]):
             return [child1, child2]
     import epdb
     epdb.st()
Exemplo n.º 46
0
def testController(controltype, base, path, branch=None):
    import controller
    _c = controller.Controller.create(controltype, 
                                base,path,branch=branch)

    repos = _c.findrepos()
    print repos
    print _c.uri
    import epdb;epdb.st()
Exemplo n.º 47
0
    def smart_match_module(self):
        '''Fuzzy matching for modules'''

        if hasattr(self, 'meta'):
            self.meta['smart_match_module_called'] = True

        match = None
        known_modules = []

        for k, v in self.module_indexer.modules.iteritems():
            known_modules.append(v['name'])

        title = self.issue.instance.title.lower()
        title = title.replace(':', '')
        title_matches = [x for x in known_modules if x + ' module' in title]
        if not title_matches:
            title_matches = [
                x for x in known_modules if title.startswith(x + ' ')
            ]
            if not title_matches:
                title_matches = [
                    x for x in known_modules if ' ' + x + ' ' in title
                ]

        cmatches = None
        if self.template_data.get('component name'):
            component = self.template_data.get('component name')
            cmatches = [x for x in known_modules if x in component]
            cmatches = [x for x in cmatches if not '_' + x in component]

            import epdb
            epdb.st()

            # use title ... ?
            if title_matches:
                cmatches = [x for x in cmatches if x in title_matches]

            if cmatches:
                if len(cmatches) >= 1:
                    match = cmatches[0]
                if not match:
                    if 'docs.ansible.com' in component:
                        #import epdb; epdb.st()
                        pass
                    else:
                        #import epdb; epdb.st()
                        pass

        #import epdb; epdb.st()
        if not match:
            if len(title_matches) == 1:
                match = title_matches[0]
            else:
                print("module - title matches: %s" % title_matches)
                print("module - component matches: %s" % cmatches)

        return match
Exemplo n.º 48
0
def read_gzip_json(cfile):
    try:
        with gzip.open(cfile, 'r') as f:
            jdata = json.loads(f.read())
    except json.decoder.JSONDecodeError as e:
        logger.error(e)
        import epdb
        epdb.st()
    return jdata
Exemplo n.º 49
0
def module_stats():

    global RESMAP
    global MI
    global MSM

    update = True
    if MI is None or update:
        cfg = get_config()
        MI = ModuleIndexer(
            cachedir=cfg.get('github_cache_dir', '/tmp/mi.cache'),
            checkoutdir=cfg.get('ansible_checkout', '/tmp/ansible.checkout')
        )
        MI.get_ansible_modules()

    if MSM is None or update:
        MSM = ModuleStatsMaker(MI)

    module_df = MSM.get_grouped_data()
    module_df.index.names = ['date']

    added_rolmean = module_df['total added'].rolling(window=3)
    module_df['total added (rolling mean)'] = added_rolmean.mean()

    added_rolstd = pd.rolling_std(module_df['total added'], window=3)
    added_rolstd = added_rolstd.to_frame()
    module_df['total added (rolling std)'] = added_rolstd['total added']

    # must be float for ARMA/ARIMA
    cumsum = pd.to_numeric(module_df['cumulative sum'], downcast='float')
    cumsum = cumsum.to_frame()
    cumsum.columns = ['cumsum']

    '''
    # don't let the unfinished month screw up the prediction
    cumsum.drop(cumsum.index[len(cumsum)-1])

    # use the log to set the stationarity
    ts_log_data = np.log(cumsum['cumsum'])

    # create the model
    model = sm.tsa.ARMA(ts_log_data, order=(1,1), freq='M').fit()

    #start_date = ts_log_data.index[-1] + Day(1)
    #end_date = ts_log_data.index[-1] + Day(60)

    #y_forecast = model.predict('2017-04-30', '2018-01-31')
    #y_vals = np.exp(y_forecast)
    #print(module_df.to_csv())
    print(np.exp(model.predict('2017-04-30', '2017-10-31')))
    '''



    print(module_df.to_csv())
    import epdb; epdb.st()
Exemplo n.º 50
0
def extract_pr_number_from_comment(rawtext, command='resolved_by_pr'):
    # "resolved_by_pr 5136" --> 5136
    # "resolved_by_pr #5136" --> 5136
    # "resolved_by_pr https://github.com/ansible/ansible/issues/5136" --> 5136
    # "resolved_by_pr https://github.com/ansible/ansible/issues/5136" --> 5136
    # "resolved_by_pr #5319." --> 5319
    index = rawtext.find(command)
    index += len(command)
    data = rawtext[index:]
    data = data.strip()
    words = data.split()

    # remove non-digit chars
    if words:
        newword = u''
        for char in words[0]:
            if char.isdigit():
                newword += to_text(char)
        if newword:
            words[0] = newword

    if not words:
        return None
    number = words[0]

    if number.isdigit():
        number = int(number)
    elif number.startswith(u'#'):
        number = number[1:]
        number = int(number)
    elif number.startswith(u'http'):
        urlparts = number.split(u'/')
        number = urlparts[-1]
        number = int(number)
    elif rawtext.find(u'#'):
        number = rawtext[rawtext.find(u'#'):]
        number = number.replace(u'#', u'')
        while number:
            if not number[-1].isdigit():
                number = number[:-1]
            else:
                break
        try:
            number = int(number)
        except Exception:
            number = None
    else:
        logging.error(u'NOT SURE HOW TO PARSE %s' % rawtext)
        if C.DEFAULT_BREAKPOINTS:
            logging.error(u'breakpoint!')
            import epdb
            epdb.st()
        else:
            raise Exception(u'parsing error')

    return number
Exemplo n.º 51
0
    def create_body(self):

        # cleanup remnant colons
        for k, v in six.iteritems(self.sections):
            if v.startswith(u':\n'):
                self.sections[k] = v[2:]
            elif v.startswith(u': \n'):
                self.sections[k] = v[3:]
            elif v.startswith(u':'):
                self.sections[k] = v[1:]

        if self.retemplate:
            # render to text
            for section in self.section_order:
                data = self.sections.get(section)
                if data is None:
                    data = u''
                self.new_description += u'##### ' + section.upper() + u'\n'
                if section == u'issue type':
                    self.new_description += data.title()
                    self.new_description += u'\n'
                else:
                    self.new_description += data + u'\n'
                self.new_description += u'\n'
        else:
            dlines = self.original.split(u'\n')
            for msection in self.missing:
                midx = self.section_order.index(msection)
                post_section = self.section_order[midx + 1]

                if post_section not in self.section_map:
                    if C.DEFAULT_BREAKPOINTS:
                        logging.error(u'breakpoint!')
                        import epdb; epdb.st()
                    else:
                        raise Exception(u'section not in map')

                post_line = self.section_map[post_section]

                new_section = [u'##### %s' % msection.upper()]
                if msection == u'component name':
                    if not self.meta[u'is_module']:
                        if self.issuewrapper.github_type == u'pullrequest':
                            new_section += self.issuewrapper.files
                        else:
                            new_section.append(u'core')
                    else:
                        new_section.append(
                            self.meta[u'module_match'][u'name'] + u' module'
                        )
                new_section.append(u'')

                for x in reversed(new_section):
                    dlines.insert(post_line, x)

            self.new_description = u'\n'.join(dlines)
Exemplo n.º 52
0
    def load_update_fetch(self, property_name):
        '''Fetch a get() property for an object'''

        edata = None
        events = []
        updated = None
        update = False
        write_cache = False
        self.repo.update()

        pfile = os.path.join(self.cachedir, u'%s.pickle' % property_name)
        pdir = os.path.dirname(pfile)

        if not os.path.isdir(pdir):
            os.makedirs(pdir)

        if os.path.isfile(pfile):
            try:
                with open(pfile, 'rb') as f:
                    edata = pickle_load(f)
            except Exception as e:
                update = True
                write_cache = True

            # check the timestamp on the cache
            if edata:
                updated = edata[0]
                events = edata[1]
                if updated < self.repo.updated_at:
                    update = True
                    write_cache = True

        # pull all events if timestamp is behind or no events cached
        if update or not events:
            write_cache = True
            updated = self.get_current_time()
            try:
                methodToCall = getattr(self.repo, u'get_' + property_name)
            except Exception as e:
                logging.error(e)
                if C.DEFAULT_BREAKPOINTS:
                    logging.error(u'breakpoint!')
                    import epdb
                    epdb.st()
                else:
                    raise Exception(u'unable to get %s' % property_name)
            events = [x for x in methodToCall()]

        if C.DEFAULT_PICKLE_ISSUES:
            if write_cache or not os.path.isfile(pfile):
                # need to dump the pickle back to disk
                edata = [updated, events]
                with open(pfile, 'wb') as f:
                    pickle_dump(edata, f)

        return events
Exemplo n.º 53
0
    def _get_url(self, url, usecache=False, timeout=TIMEOUT):
        cdir = os.path.join(self.cachedir, u'.raw')
        if not os.path.isdir(cdir):
            os.makedirs(cdir)
        #cfile = url.replace(u'https://api.shippable.com/', u'')
        cfile = url.replace(SHIPPABLE_URL + '/', u'')
        cfile = cfile.replace(u'/', u'_')
        cfile = os.path.join(cdir, cfile + u'.json')
        gzfile = cfile + u'.gz'

        # transparently compress old logs
        if os.path.isfile(cfile) and not os.path.isfile(gzfile):
            self._compress_cache_file(cfile, gzfile)

        rc = None
        jdata = None
        if os.path.isfile(gzfile):
            try:
                fdata = self._load_cache_file(gzfile)
                rc = fdata[0]
                jdata = fdata[1]
            except ValueError:
                pass

            if rc == 400:
                return None

        resp = None
        if not os.path.isfile(gzfile) or not jdata or not usecache:

            resp = self.fetch(url, timeout=timeout)
            if not resp:
                return None

            if resp.status_code != 400:
                jdata = resp.json()
                self._write_cache_file(gzfile, [resp.status_code, jdata])
            else:
                self._write_cache_file(gzfile, [resp.status_code, {}])
                return None

        self.check_response(resp)

        if not jdata:
            #import epdb; epdb.st()
            if C.DEFAULT_BREAKPOINTS:
                logging.error(u'breakpoint!')
                import epdb
                epdb.st()
            else:
                import epdb
                epdb.st()
                #raise Exception(u'no json data')
                raise ShippableNoData()

        return jdata
Exemplo n.º 54
0
    def get_repo_for_collection(self, fqcn):
        today = datetime.datetime.now()

        if fqcn not in self._gitrepos:

            # reduce the number of requests ...
            try:
                rurl = self._checkout_index.get(fqcn, {}).get('url')
            except AttributeError as e:
                print(e)
                import epdb
                epdb.st()

            if rurl is None:
                # https://galaxy.ansible.com/api/v2/collections/devoperate/base/
                curl = self._baseurl + '/api/v2/collections/' + fqcn.replace(
                    '.', '/') + '/'
                rr = requests.get(curl)
                jdata = rr.json()
                vurl = jdata['latest_version']['href']
                rr2 = requests.get(vurl)
                jdata2 = rr2.json()
                rurl = jdata2.get('metadata', {}).get('repository')

            # reduce the number of clones and rebases ...
            needs_rebase = False
            if fqcn not in self._checkout_index:
                needs_rebase = True
            elif not self._checkout_index.get(fqcn, {}).get('checkout'):
                needs_rebase = True
            elif not self._checkout_index.get(fqcn, {}).get('updated'):
                needs_rebase = True
            elif (today - self._checkout_index[fqcn]['updated']).days > 0:
                needs_rebase = True

            logging.info('checkout %s -> %s' % (fqcn, rurl))
            grepo = GitRepoWrapper(cachedir=self.cachedir,
                                   repo=rurl,
                                   rebase=needs_rebase)
            self._gitrepos[fqcn] = grepo

            # keep the last updated time if not rebased ...
            if needs_rebase:
                updated = datetime.datetime.now()
            else:
                updated = self._checkout_index[fqcn]['updated']

            self._checkout_index[fqcn] = {
                'url': rurl,
                'fqcn': fqcn,
                'checkout': grepo.checkoutdir,
                'updated': updated
            }
            self._save_checkout_index()

        return self._gitrepos[fqcn]
Exemplo n.º 55
0
    def update(self):
        self.instance.update()
        self._history = \
            HistoryWrapper(self, cachedir=self.cachedir, usecache=True)
        if self.is_pullrequest():
            self.pullrequest.update()

            if self.instance.updated_at > self.pullrequest.updated_at:
                logging.error('breakpoint!')
                import epdb; epdb.st()
Exemplo n.º 56
0
 def parse_timestamp(self, timestamp):
     try:
         if 'Z' in timestamp:
             timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
         else:
             timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S')
     except Exception as e:
         print(e)
         import epdb; epdb.st()
     return timestamp
Exemplo n.º 57
0
 def get_ansible_version_major_minor(self, version=None):
     if not version:
         # old workflow
         if not hasattr(self, 'ansible_version'):
             import epdb
             epdb.st()
         return self.version_indexer.get_major_minor(self.ansible_version)
     else:
         # v3 workflow
         return self.version_indexer.get_major_minor(version)
Exemplo n.º 58
0
    def update(self):
        self.instance.update()
        self._history = \
            HistoryWrapper(self, cachedir=self.cachedir, usecache=True)
        if self.is_pullrequest():
            self.pullrequest.update()

            if self.instance.updated_at > self.pullrequest.updated_at:
                import epdb
                epdb.st()
Exemplo n.º 59
0
    def compare_task_sequence(self):

        task_data = {}

        for k, v in self.idata.items():
            tasks = v['profiler_data'][:]

            for idx, x in enumerate(tasks):
                tn = '%s|%s' % (x['role'], x['task'])
                if tn not in task_data:
                    task_data[tn] = {'observations': []}
                # the 9 node results skew the data
                if v['cluster_size'] == 9:
                    continue
                task_data[tn]['observations'].append(
                    (v['cluster_size'], x['mem']))

        # capture the mean mem per node for each task
        mean_ratios = []

        # divide total mem by the number of nodes
        for k, v in task_data.items():
            obs = v['observations'][:]
            if not obs:
                continue
            obs_ratios = [x[1] / x[0] for x in obs]
            obs_mean = np.mean(obs_ratios)
            if np.isnan(obs_mean):
                import epdb
                epdb.st()
            mean_ratios.append(obs_mean)

        # make some baseline stats
        mem_med = np.median(mean_ratios)
        mem_mean = np.mean(mean_ratios)
        mem_std = np.std(mean_ratios)

        for k, v in task_data.items():
            obs = v['observations'][:]
            obs = sorted(obs, key=itemgetter(0))
            obs_ratios = [x[1] / x[0] for x in obs]

            if np.mean(obs_ratios) >= (mem_std * 3):

                print('###################################')
                print('task: ' + k)
                print('nodecount+memused observations: ' + str(obs))
                print('nodecount+memused ratio: ' + str(obs_ratios))

                xvals = [0 for x in obs_ratios]

                #import epdb; epdb.st()

        import epdb
        epdb.st()
Exemplo n.º 60
0
    def dump_summaries(self, repo_url, issues, filename="summaries"):

        """
        [jtanner@fedmac ansibullbot]$ sudo ls -al /proc/10895/fd
        total 0
        dr-x------ 2 jtanner docker  0 Jan 13 08:51 .
        dr-xr-xr-x 9 jtanner docker  0 Jan 13 08:42 ..
        lrwx------ 1 jtanner docker 64 Jan 13 08:51 0 -> /dev/pts/2
        lrwx------ 1 jtanner docker 64 Jan 13 08:51 1 -> /dev/pts/2
        lr-x------ 1 jtanner docker 64 Jan 13 08:51 10 -> /dev/urandom
        lrwx------ 1 jtanner d 64 Jan 13 08:51 11 -> /tmp/tmpag2rAb (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 12 -> /tmp/tmpD2plk9 (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 13 -> /tmp/tmpfkSSPA (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 14 -> /tmp/tmpIDY_wb (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 15 -> /tmp/tmpbQBvI2 (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 16 -> /tmp/tmpknP5os (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 17 -> /tmp/tmpDJgEnc (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 18 -> /tmp/tmprWLicP (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 19 -> /tmp/tmpm6d8Qx (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 2 -> /dev/pts/2
        lrwx------ 1 jtanner d 64 Jan 13 08:51 20 -> /tmp/tmp_w9Sth (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 21 -> /tmp/tmpRGnb3p (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 22 -> /tmp/tmpiVYdTE (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 23 -> /tmp/tmpEyGXuP (deleted)
        l-wx------ 1 jtanner d 64 Jan 13 08:51 3 -> /var/log/ansibullbot.log
        lrwx------ 1 jtanner d 64 Jan 13 08:51 4 -> /tmp/tmpIlHOg_ (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 5 -> /tmp/tmp5P8Mya (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 6 -> /tmp/tmpDW4MRD (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 7 -> /tmp/tmpUyBIFB (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 8 -> /tmp/tmpYcWaLe (deleted)
        lrwx------ 1 jtanner d 64 Jan 13 08:51 9 -> /tmp/tmp_Qcxrt (deleted)
        """

        ns,repo = self.split_repo_url(repo_url)
        cachefile = os.path.join(
            self.cachedir,
            ns,
            repo,
            '%s.json' % filename
        )
        if not issues:
            if C.DEFAULT_BREAKPOINTS:
                logging.error('breakpoint!')
                import epdb; epdb.st()
            else:
                raise Exception('no issues')

        tfh, tfn = tempfile.mkstemp()
        os.close(tfh)
        with open(tfn, 'wb') as f:
            f.write(json.dumps(issues, sort_keys=True, indent=2))

        if os.path.isfile(cachefile):
            os.remove(cachefile)
        shutil.move(tfn, cachefile)