Example #1
0
def unknown_commit_ids(all_commit_ids):
    '''filter out all commit ids that have already been cached'''
    result = []
    for chunk in utils.chunked_iter(all_commit_ids, QSIZE):
        chunk = list(chunk)
        q = CommitDoc.m.find(dict(_id={'$in': chunk}))
        known_commit_ids = set(ci._id for ci in q)
        result += [oid for oid in chunk if oid not in known_commit_ids]
    return result
Example #2
0
def unknown_commit_ids(all_commit_ids):
    '''filter out all commit ids that have already been cached'''
    result = []
    for chunk in utils.chunked_iter(all_commit_ids, QSIZE):
        chunk = list(chunk)
        q = CommitDoc.m.find(dict(_id={'$in': chunk}))
        known_commit_ids = set(ci._id for ci in q)
        result += [oid for oid in chunk if oid not in known_commit_ids]
    return result
Example #3
0
def send_notifications(repo, commit_ids):
    '''Create appropriate notification and feed objects for a refresh'''
    from allura.model import Feed, Notification
    commit_msgs = []
    base_url = tg.config['base_url']
    for oids in utils.chunked_iter(commit_ids, QSIZE):
        chunk = list(oids)
        index = dict(
            (doc._id, doc)
            for doc in Commit.query.find(dict(_id={'$in': chunk})))
        for oid in chunk:
            ci = index[oid]
            href = repo.url_for_commit(oid)
            title = _title(ci.message)
            summary = _summarize(ci.message)
            Feed.post(
                repo, title=title,
                description='%s<br><a href="%s">View Changes</a>' % (
                    summary, href),
                author_link=ci.author_url,
                author_name=ci.authored.name,
                link=href,
                unique_id=href)
            branches = repo.symbolics_for_commit(ci)[0]
            commit_msgs.append('%s: %s by %s %s%s' % (
                ",".join(b for b in branches),
                summary, ci.authored.name, base_url, ci.url()))
    if commit_msgs:
        if len(commit_msgs) > 1:
            subject = '%d new commits to %s %s' % (
                len(commit_msgs), repo.app.project.name, repo.app.config.options.mount_label)
            text = '\n\n'.join(commit_msgs)
        else:
            subject = '{0} - {1}: {2}'.format(
                repo.shorthand_for_commit(ci._id),
                ci.authored.name,
                summary)
            branches = repo.symbolics_for_commit(ci)[0]
            text_branches = ('%s: ' % ",".join(b for b in branches)
                             if branches else '')
            text = "%s%s %s%s" % (text_branches,
                                  ci.message,
                                  base_url, ci.url())

        Notification.post(
            artifact=repo,
            topic='metadata',
            subject=subject,
            text=text)
Example #4
0
 def log_iter(self, skip, count):
     for oids in utils.chunked_iter(commitlog([self._id]), QSIZE):
         oids = list(oids)
         commits = dict((ci._id, ci)
                        for ci in self.query.find(dict(_id={'$in': oids})))
         for oid in oids:
             if skip:
                 skip -= 1
                 continue
             if count:
                 count -= 1
                 ci = commits[oid]
                 ci.set_context(self.repo)
                 yield ci
             else:
                 break
Example #5
0
 def log_iter(self, skip, count):
     for oids in utils.chunked_iter(commitlog([self._id]), QSIZE):
         oids = list(oids)
         commits = dict(
             (ci._id, ci) for ci in self.query.find(dict(
                     _id={'$in': oids})))
         for oid in oids:
             if skip:
                 skip -= 1
                 continue
             if count:
                 count -= 1
                 ci = commits[oid]
                 ci.set_context(self.repo)
                 yield ci
             else:
                 break
Example #6
0
def refresh_commit_repos(all_commit_ids, repo):
    """Refresh the list of repositories within which a set of commits are
    contained"""
    for oids in utils.chunked_iter(all_commit_ids, QSIZE):
        for ci in CommitDoc.m.find(dict(_id={"$in": list(oids)}, repo_ids={"$ne": repo._id})):
            oid = ci._id
            ci.repo_ids.append(repo._id)
            index_id = "allura.model.repo.Commit#" + oid
            ref = ArtifactReferenceDoc(
                dict(
                    _id=index_id,
                    artifact_reference=dict(
                        cls=bson.Binary(dumps(Commit)),
                        project_id=repo.app.config.project_id,
                        app_config_id=repo.app.config._id,
                        artifact_id=oid,
                    ),
                    references=[],
                )
            )
            link0 = ShortlinkDoc(
                dict(
                    _id=bson.ObjectId(),
                    ref_id=index_id,
                    project_id=repo.app.config.project_id,
                    app_config_id=repo.app.config._id,
                    link=repo.shorthand_for_commit(oid)[1:-1],
                    url=repo.url_for_commit(oid),
                )
            )
            # Always create a link for the full commit ID
            link1 = ShortlinkDoc(
                dict(
                    _id=bson.ObjectId(),
                    ref_id=index_id,
                    project_id=repo.app.config.project_id,
                    app_config_id=repo.app.config._id,
                    link=oid,
                    url=repo.url_for_commit(oid),
                )
            )
            ci.m.save(safe=False, validate=False)
            ref.m.save(safe=False, validate=False)
            link0.m.save(safe=False, validate=False)
            link1.m.save(safe=False, validate=False)
def send_notifications(repo, commit_ids):
    '''Create appropriate notification and feed objects for a refresh'''
    from allura.model import Feed, Notification
    commit_msgs = []
    base_url = tg.config.get('base_url', 'sourceforge.net')
    for oids in utils.chunked_iter(commit_ids, QSIZE):
        chunk = list(oids)
        index = dict(
            (doc._id, doc)
            for doc in Commit.query.find(dict(_id={'$in':chunk})))
        for oid in chunk:
            ci = index[oid]
            href = repo.url_for_commit(oid)
            summary = _summarize(ci.message)
            Feed.post(
                repo, title='New commit',
                description='%s<br><a href="%s">View Changes</a>' % (
                    summary, href),
                author_link=ci.author_url,
                author_name=ci.authored.name)
            branches = repo.symbolics_for_commit(ci)[0]
            commit_msgs.append('%s: %s by %s %s%s' % (
                    ",".join(b for b in branches),
                    summary, ci.authored.name, base_url, ci.url()))
    if commit_msgs:
        if len(commit_msgs) > 1:
            subject = '%d new commits to %s %s' % (
                len(commit_msgs), repo.app.project.name, repo.app.config.options.mount_label)
            text='\n\n'.join(commit_msgs)
        else:
            subject = '%s committed to %s %s: %s' % (
                ci.authored.name,
                repo.app.project.name,
                repo.app.config.options.mount_label,
                summary)
            branches = repo.symbolics_for_commit(ci)[0]
            text = "%s: %s %s%s" % (",".join(b for b in branches),
                               ci.message,
                               base_url, ci.url())

        Notification.post(
            artifact=repo,
            topic='metadata',
            subject=subject,
            text=text)
 def run(self):
     '''Build up the runs'''
     for oids in utils.chunked_iter(self.commit_ids, QSIZE):
         oids = list(oids)
         for ci in CommitDoc.m.find(dict(_id={'$in': oids})):
             if ci._id in self.run_index: continue
             self.run_index[ci._id] = ci._id
             self.runs[ci._id] = CommitRunDoc(
                 dict(_id=ci._id,
                      parent_commit_ids=ci.parent_ids,
                      commit_ids=[ci._id],
                      commit_times=[ci.authored['date']]))
         self.merge_runs()
     log.info('%d runs', len(self.runs))
     for rid, run in sorted(self.runs.items()):
         log.info('%32s: %r', self.reasons.get(rid, 'none'), run._id)
     for run in self.runs.itervalues():
         run.m.save()
     return self.runs
Example #9
0
 def _all_runs(self):
     """Find all runs containing this builder's commit IDs"""
     runs = {}
     for oids in utils.chunked_iter(self.commit_ids, QSIZE):
         oids = list(oids)
         for run in CommitRunDoc.m.find(dict(commit_ids={"$in": oids})):
             runs[run._id] = run
         for run in CommitRunDoc.m.find(dict(parent_commit_ids={"$in": oids})):
             runs[run._id] = run
     seen_run_ids = set()
     runs = runs.values()
     while runs:
         run = runs.pop()
         if run._id in seen_run_ids:
             continue
         seen_run_ids.add(run._id)
         yield run
         for run in CommitRunDoc.m.find(dict(commit_ids={"$in": run.parent_commit_ids})):
             runs.append(run)
 def _all_runs(self):
     '''Find all runs containing this builder's commit IDs'''
     runs = {}
     for oids in utils.chunked_iter(self.commit_ids, QSIZE):
         oids = list(oids)
         for run in CommitRunDoc.m.find(dict(commit_ids={'$in': oids})):
             runs[run._id] = run
         for run in CommitRunDoc.m.find(dict(parent_commit_ids={'$in': oids})):
             runs[run._id] = run
     seen_run_ids = set()
     runs = runs.values()
     while runs:
         run = runs.pop()
         if run._id in seen_run_ids: continue
         seen_run_ids.add(run._id)
         yield run
         for run in CommitRunDoc.m.find(
             dict(commit_ids={'$in':run.parent_commit_ids})):
             runs.append(run)
 def run(self):
     '''Build up the runs'''
     for oids in utils.chunked_iter(self.commit_ids, QSIZE):
         oids = list(oids)
         for ci in CommitDoc.m.find(dict(_id={'$in':oids})):
             if ci._id in self.run_index: continue
             self.run_index[ci._id] = ci._id
             self.runs[ci._id] = CommitRunDoc(dict(
                     _id=ci._id,
                     parent_commit_ids=ci.parent_ids,
                     commit_ids=[ci._id],
                     commit_times=[ci.authored['date']]))
         self.merge_runs()
     log.info('%d runs', len(self.runs))
     for rid, run in sorted(self.runs.items()):
         log.info('%32s: %r', self.reasons.get(rid, 'none'), run._id)
     for run in self.runs.itervalues():
         run.m.save()
     return self.runs
Example #12
0
 def run(self):
     """Build up the runs"""
     for oids in utils.chunked_iter(self.commit_ids, QSIZE):
         oids = list(oids)
         for ci in CommitDoc.m.find(dict(_id={"$in": oids})):
             if ci._id in self.run_index:
                 continue
             self.run_index[ci._id] = ci._id
             self.runs[ci._id] = CommitRunDoc(
                 dict(
                     _id=ci._id,
                     parent_commit_ids=ci.parent_ids,
                     commit_ids=[ci._id],
                     commit_times=[ci.authored["date"]],
                 )
             )
         self.merge_runs()
     log.info("%d runs", len(self.runs))
     for rid, run in sorted(self.runs.items()):
         log.info("%32s: %r", self.reasons.get(rid, "none"), run._id)
     for run in self.runs.itervalues():
         run.m.save()
     return self.runs
Example #13
0
def refresh_commit_repos(all_commit_ids, repo):
    '''Refresh the list of repositories within which a set of commits are
    contained'''
    for oids in utils.chunked_iter(all_commit_ids, QSIZE):
        for ci in CommitDoc.m.find(dict(
                _id={'$in': list(oids)},
                repo_ids={'$ne': repo._id})):
            oid = ci._id
            ci.repo_ids.append(repo._id)
            index_id = 'allura.model.repository.Commit#' + oid
            ref = ArtifactReferenceDoc(dict(
                _id=index_id,
                artifact_reference=dict(
                    cls=bson.Binary(dumps(Commit)),
                    project_id=repo.app.config.project_id,
                    app_config_id=repo.app.config._id,
                    artifact_id=oid),
                references=[]))
            link0 = ShortlinkDoc(dict(
                _id=bson.ObjectId(),
                ref_id=index_id,
                project_id=repo.app.config.project_id,
                app_config_id=repo.app.config._id,
                link=repo.shorthand_for_commit(oid)[1:-1],
                url=repo.url_for_commit(oid)))
            # Always create a link for the full commit ID
            link1 = ShortlinkDoc(dict(
                _id=bson.ObjectId(),
                ref_id=index_id,
                project_id=repo.app.config.project_id,
                app_config_id=repo.app.config._id,
                link=oid,
                url=repo.url_for_commit(oid)))
            ci.m.save(safe=False, validate=False)
            ref.m.save(safe=False, validate=False)
            link0.m.save(safe=False, validate=False)
            link1.m.save(safe=False, validate=False)
Example #14
0
def send_notifications(repo, commit_ids):
    """Create appropriate notification and feed objects for a refresh

    :param repo: A repository artifact instance.
    :type repo: Repository

    :param commit_ids: A list of commit hash strings, oldest to newest
    :type commit_ids: list
    """
    from allura.model import Feed, Notification
    commit_msgs = []
    base_url = tg.config['base_url']
    for oids in utils.chunked_iter(commit_ids, QSIZE):
        chunk = list(oids)
        index = dict((doc._id, doc)
                     for doc in Commit.query.find(dict(_id={'$in': chunk})))
        for oid in chunk:
            ci = index[oid]
            href = repo.url_for_commit(oid)
            title = _title(ci.message)
            summary = _summarize(ci.message)
            Feed.post(repo,
                      title=title,
                      description='%s<br><a href="%s">View Changes</a>' %
                      (summary, href),
                      author_link=ci.author_url,
                      author_name=ci.authored.name,
                      link=href,
                      unique_id=href)

            summary = g.markdown_commit.convert(
                ci.message) if ci.message else ""
            current_branch = repo.symbolics_for_commit(ci)[
                0]  # only the head of a branch will have this
            commit_msgs.append(
                dict(author=ci.authored.name,
                     date=ci.authored.date.strftime("%m/%d/%Y %H:%M"),
                     summary=summary,
                     branches=current_branch,
                     commit_url=base_url + href,
                     shorthand_id=ci.shorthand_id()))

    # fill out the branch info for all the other commits
    prev_branch = None
    for c_msg in reversed(commit_msgs):
        if not c_msg['branches']:
            c_msg['branches'] = prev_branch
        prev_branch = c_msg['branches']

    # mark which ones are first on a branch and need the branch name shown
    last_branch = None
    for c_msg in commit_msgs:
        if c_msg['branches'] != last_branch:
            c_msg['show_branch_name'] = True
        last_branch = c_msg['branches']

    if commit_msgs:
        if len(commit_msgs) > 1:
            subject = u"{} new commits to {}".format(
                len(commit_msgs), repo.app.config.options.mount_label)
        else:
            commit = commit_msgs[0]
            subject = u'New commit {} by {}'.format(commit['shorthand_id'],
                                                    commit['author'])
        text = g.jinja2_env.get_template(
            "allura:templates/mail/commits.md").render(
                commit_msgs=commit_msgs,
                max_num_commits=asint(
                    tg.config.get('scm.notify.max_commits', 100)),
            )

        Notification.post(artifact=repo,
                          topic='metadata',
                          subject=subject,
                          text=text)
Example #15
0
def send_notifications(repo, commit_ids):
    """Create appropriate notification and feed objects for a refresh

    :param repo: A repository artifact instance.
    :type repo: Repository

    :param commit_ids: A list of commit hash strings.
    :type commit_ids: list
    """
    from allura.model import Feed, Notification
    commit_msgs = []
    base_url = tg.config['base_url']
    last_branch = []
    for oids in utils.chunked_iter(commit_ids, QSIZE):
        chunk = list(oids)
        index = dict(
            (doc._id, doc)
            for doc in Commit.query.find(dict(_id={'$in': chunk})))
        for oid in chunk:
            ci = index[oid]
            href = repo.url_for_commit(oid)
            title = _title(ci.message)
            summary = _summarize(ci.message)
            Feed.post(
                repo, title=title,
                description='%s<br><a href="%s">View Changes</a>' % (
                    summary, href),
                author_link=ci.author_url,
                author_name=ci.authored.name,
                link=href,
                unique_id=href)

            summary = g.markdown_commit.convert(ci.message) if ci.message else ""

            current_branch = repo.symbolics_for_commit(ci)[0]
            if last_branch == current_branch:
                branches = []
            else:
                branches = current_branch
                last_branch = branches

            commit_msgs.append(dict(
                author=ci.authored.name,
                date=ci.authored.date.strftime("%m/%d/%Y %H:%M"),
                summary=summary,
                branches=branches,
                commit_url=base_url + href))

    if commit_msgs:
        if len(commit_msgs) > 1:
            subject = u"{} new commits to {}".format(len(commit_msgs), repo.app.config.options.mount_label)
        else:
            subject = u'New commit by {}'.format(commit_msgs[0]['author'])
        template = g.jinja2_env.get_template("allura:templates/mail/commits.md")
        text = u"\n\n-----".join([template.render(d) for d in commit_msgs])

        Notification.post(
            artifact=repo,
            topic='metadata',
            subject=subject,
            text=text)
Example #16
0
def send_notifications(repo, commit_ids):
    """Create appropriate notification and feed objects for a refresh

    :param repo: A repository artifact instance.
    :type repo: Repository

    :param commit_ids: A list of commit hash strings, oldest to newest
    :type commit_ids: list
    """
    from allura.model import Feed, Notification
    commit_msgs = []
    base_url = tg.config['base_url']
    for oids in utils.chunked_iter(commit_ids, QSIZE):
        chunk = list(oids)
        index = dict(
            (doc._id, doc)
            for doc in Commit.query.find(dict(_id={'$in': chunk})))
        for oid in chunk:
            ci = index[oid]
            href = repo.url_for_commit(oid)
            title = _title(ci.message)
            summary = _summarize(ci.message)
            Feed.post(
                repo, title=title,
                description='%s<br><a href="%s">View Changes</a>' % (
                    summary, href),
                author_link=ci.author_url,
                author_name=ci.authored.name,
                link=href,
                unique_id=href)

            summary = g.markdown_commit.convert(ci.message) if ci.message else ""
            current_branch = repo.symbolics_for_commit(ci)[0]  # only the head of a branch will have this
            commit_msgs.append(dict(
                author=ci.authored.name,
                date=ci.authored.date.strftime("%m/%d/%Y %H:%M"),
                summary=summary,
                branches=current_branch,
                commit_url=base_url + href,
                shorthand_id=ci.shorthand_id()))

    # fill out the branch info for all the other commits
    prev_branch = None
    for c_msg in reversed(commit_msgs):
        if not c_msg['branches']:
            c_msg['branches'] = prev_branch
        prev_branch = c_msg['branches']

    # mark which ones are first on a branch and need the branch name shown
    last_branch = None
    for c_msg in commit_msgs:
        if c_msg['branches'] != last_branch:
            c_msg['show_branch_name'] = True
        last_branch = c_msg['branches']

    if commit_msgs:
        if len(commit_msgs) > 1:
            subject = u"{} new commits to {}".format(len(commit_msgs), repo.app.config.options.mount_label)
        else:
            commit = commit_msgs[0]
            subject = u'New commit {} by {}'.format(commit['shorthand_id'], commit['author'])
        text = g.jinja2_env.get_template("allura:templates/mail/commits.md").render(
            commit_msgs=commit_msgs,
            max_num_commits=asint(tg.config.get('scm.notify.max_commits', 100)),
        )

        Notification.post(
            artifact=repo,
            topic='metadata',
            subject=subject,
            text=text)