def unknown_commit_ids(all_commit_ids): '''filter out all commit ids that have already been cached''' result = [] for chunk in utils.chunked_iter(all_commit_ids, QSIZE): chunk = list(chunk) q = CommitDoc.m.find(dict(_id={'$in': chunk})) known_commit_ids = set(ci._id for ci in q) result += [oid for oid in chunk if oid not in known_commit_ids] return result
def send_notifications(repo, commit_ids): '''Create appropriate notification and feed objects for a refresh''' from allura.model import Feed, Notification commit_msgs = [] base_url = tg.config['base_url'] for oids in utils.chunked_iter(commit_ids, QSIZE): chunk = list(oids) index = dict( (doc._id, doc) for doc in Commit.query.find(dict(_id={'$in': chunk}))) for oid in chunk: ci = index[oid] href = repo.url_for_commit(oid) title = _title(ci.message) summary = _summarize(ci.message) Feed.post( repo, title=title, description='%s<br><a href="%s">View Changes</a>' % ( summary, href), author_link=ci.author_url, author_name=ci.authored.name, link=href, unique_id=href) branches = repo.symbolics_for_commit(ci)[0] commit_msgs.append('%s: %s by %s %s%s' % ( ",".join(b for b in branches), summary, ci.authored.name, base_url, ci.url())) if commit_msgs: if len(commit_msgs) > 1: subject = '%d new commits to %s %s' % ( len(commit_msgs), repo.app.project.name, repo.app.config.options.mount_label) text = '\n\n'.join(commit_msgs) else: subject = '{0} - {1}: {2}'.format( repo.shorthand_for_commit(ci._id), ci.authored.name, summary) branches = repo.symbolics_for_commit(ci)[0] text_branches = ('%s: ' % ",".join(b for b in branches) if branches else '') text = "%s%s %s%s" % (text_branches, ci.message, base_url, ci.url()) Notification.post( artifact=repo, topic='metadata', subject=subject, text=text)
def log_iter(self, skip, count): for oids in utils.chunked_iter(commitlog([self._id]), QSIZE): oids = list(oids) commits = dict((ci._id, ci) for ci in self.query.find(dict(_id={'$in': oids}))) for oid in oids: if skip: skip -= 1 continue if count: count -= 1 ci = commits[oid] ci.set_context(self.repo) yield ci else: break
def log_iter(self, skip, count): for oids in utils.chunked_iter(commitlog([self._id]), QSIZE): oids = list(oids) commits = dict( (ci._id, ci) for ci in self.query.find(dict( _id={'$in': oids}))) for oid in oids: if skip: skip -= 1 continue if count: count -= 1 ci = commits[oid] ci.set_context(self.repo) yield ci else: break
def refresh_commit_repos(all_commit_ids, repo): """Refresh the list of repositories within which a set of commits are contained""" for oids in utils.chunked_iter(all_commit_ids, QSIZE): for ci in CommitDoc.m.find(dict(_id={"$in": list(oids)}, repo_ids={"$ne": repo._id})): oid = ci._id ci.repo_ids.append(repo._id) index_id = "allura.model.repo.Commit#" + oid ref = ArtifactReferenceDoc( dict( _id=index_id, artifact_reference=dict( cls=bson.Binary(dumps(Commit)), project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, artifact_id=oid, ), references=[], ) ) link0 = ShortlinkDoc( dict( _id=bson.ObjectId(), ref_id=index_id, project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, link=repo.shorthand_for_commit(oid)[1:-1], url=repo.url_for_commit(oid), ) ) # Always create a link for the full commit ID link1 = ShortlinkDoc( dict( _id=bson.ObjectId(), ref_id=index_id, project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, link=oid, url=repo.url_for_commit(oid), ) ) ci.m.save(safe=False, validate=False) ref.m.save(safe=False, validate=False) link0.m.save(safe=False, validate=False) link1.m.save(safe=False, validate=False)
def send_notifications(repo, commit_ids): '''Create appropriate notification and feed objects for a refresh''' from allura.model import Feed, Notification commit_msgs = [] base_url = tg.config.get('base_url', 'sourceforge.net') for oids in utils.chunked_iter(commit_ids, QSIZE): chunk = list(oids) index = dict( (doc._id, doc) for doc in Commit.query.find(dict(_id={'$in':chunk}))) for oid in chunk: ci = index[oid] href = repo.url_for_commit(oid) summary = _summarize(ci.message) Feed.post( repo, title='New commit', description='%s<br><a href="%s">View Changes</a>' % ( summary, href), author_link=ci.author_url, author_name=ci.authored.name) branches = repo.symbolics_for_commit(ci)[0] commit_msgs.append('%s: %s by %s %s%s' % ( ",".join(b for b in branches), summary, ci.authored.name, base_url, ci.url())) if commit_msgs: if len(commit_msgs) > 1: subject = '%d new commits to %s %s' % ( len(commit_msgs), repo.app.project.name, repo.app.config.options.mount_label) text='\n\n'.join(commit_msgs) else: subject = '%s committed to %s %s: %s' % ( ci.authored.name, repo.app.project.name, repo.app.config.options.mount_label, summary) branches = repo.symbolics_for_commit(ci)[0] text = "%s: %s %s%s" % (",".join(b for b in branches), ci.message, base_url, ci.url()) Notification.post( artifact=repo, topic='metadata', subject=subject, text=text)
def run(self): '''Build up the runs''' for oids in utils.chunked_iter(self.commit_ids, QSIZE): oids = list(oids) for ci in CommitDoc.m.find(dict(_id={'$in': oids})): if ci._id in self.run_index: continue self.run_index[ci._id] = ci._id self.runs[ci._id] = CommitRunDoc( dict(_id=ci._id, parent_commit_ids=ci.parent_ids, commit_ids=[ci._id], commit_times=[ci.authored['date']])) self.merge_runs() log.info('%d runs', len(self.runs)) for rid, run in sorted(self.runs.items()): log.info('%32s: %r', self.reasons.get(rid, 'none'), run._id) for run in self.runs.itervalues(): run.m.save() return self.runs
def _all_runs(self): """Find all runs containing this builder's commit IDs""" runs = {} for oids in utils.chunked_iter(self.commit_ids, QSIZE): oids = list(oids) for run in CommitRunDoc.m.find(dict(commit_ids={"$in": oids})): runs[run._id] = run for run in CommitRunDoc.m.find(dict(parent_commit_ids={"$in": oids})): runs[run._id] = run seen_run_ids = set() runs = runs.values() while runs: run = runs.pop() if run._id in seen_run_ids: continue seen_run_ids.add(run._id) yield run for run in CommitRunDoc.m.find(dict(commit_ids={"$in": run.parent_commit_ids})): runs.append(run)
def _all_runs(self): '''Find all runs containing this builder's commit IDs''' runs = {} for oids in utils.chunked_iter(self.commit_ids, QSIZE): oids = list(oids) for run in CommitRunDoc.m.find(dict(commit_ids={'$in': oids})): runs[run._id] = run for run in CommitRunDoc.m.find(dict(parent_commit_ids={'$in': oids})): runs[run._id] = run seen_run_ids = set() runs = runs.values() while runs: run = runs.pop() if run._id in seen_run_ids: continue seen_run_ids.add(run._id) yield run for run in CommitRunDoc.m.find( dict(commit_ids={'$in':run.parent_commit_ids})): runs.append(run)
def run(self): '''Build up the runs''' for oids in utils.chunked_iter(self.commit_ids, QSIZE): oids = list(oids) for ci in CommitDoc.m.find(dict(_id={'$in':oids})): if ci._id in self.run_index: continue self.run_index[ci._id] = ci._id self.runs[ci._id] = CommitRunDoc(dict( _id=ci._id, parent_commit_ids=ci.parent_ids, commit_ids=[ci._id], commit_times=[ci.authored['date']])) self.merge_runs() log.info('%d runs', len(self.runs)) for rid, run in sorted(self.runs.items()): log.info('%32s: %r', self.reasons.get(rid, 'none'), run._id) for run in self.runs.itervalues(): run.m.save() return self.runs
def run(self): """Build up the runs""" for oids in utils.chunked_iter(self.commit_ids, QSIZE): oids = list(oids) for ci in CommitDoc.m.find(dict(_id={"$in": oids})): if ci._id in self.run_index: continue self.run_index[ci._id] = ci._id self.runs[ci._id] = CommitRunDoc( dict( _id=ci._id, parent_commit_ids=ci.parent_ids, commit_ids=[ci._id], commit_times=[ci.authored["date"]], ) ) self.merge_runs() log.info("%d runs", len(self.runs)) for rid, run in sorted(self.runs.items()): log.info("%32s: %r", self.reasons.get(rid, "none"), run._id) for run in self.runs.itervalues(): run.m.save() return self.runs
def refresh_commit_repos(all_commit_ids, repo): '''Refresh the list of repositories within which a set of commits are contained''' for oids in utils.chunked_iter(all_commit_ids, QSIZE): for ci in CommitDoc.m.find(dict( _id={'$in': list(oids)}, repo_ids={'$ne': repo._id})): oid = ci._id ci.repo_ids.append(repo._id) index_id = 'allura.model.repository.Commit#' + oid ref = ArtifactReferenceDoc(dict( _id=index_id, artifact_reference=dict( cls=bson.Binary(dumps(Commit)), project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, artifact_id=oid), references=[])) link0 = ShortlinkDoc(dict( _id=bson.ObjectId(), ref_id=index_id, project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, link=repo.shorthand_for_commit(oid)[1:-1], url=repo.url_for_commit(oid))) # Always create a link for the full commit ID link1 = ShortlinkDoc(dict( _id=bson.ObjectId(), ref_id=index_id, project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, link=oid, url=repo.url_for_commit(oid))) ci.m.save(safe=False, validate=False) ref.m.save(safe=False, validate=False) link0.m.save(safe=False, validate=False) link1.m.save(safe=False, validate=False)
def send_notifications(repo, commit_ids): """Create appropriate notification and feed objects for a refresh :param repo: A repository artifact instance. :type repo: Repository :param commit_ids: A list of commit hash strings, oldest to newest :type commit_ids: list """ from allura.model import Feed, Notification commit_msgs = [] base_url = tg.config['base_url'] for oids in utils.chunked_iter(commit_ids, QSIZE): chunk = list(oids) index = dict((doc._id, doc) for doc in Commit.query.find(dict(_id={'$in': chunk}))) for oid in chunk: ci = index[oid] href = repo.url_for_commit(oid) title = _title(ci.message) summary = _summarize(ci.message) Feed.post(repo, title=title, description='%s<br><a href="%s">View Changes</a>' % (summary, href), author_link=ci.author_url, author_name=ci.authored.name, link=href, unique_id=href) summary = g.markdown_commit.convert( ci.message) if ci.message else "" current_branch = repo.symbolics_for_commit(ci)[ 0] # only the head of a branch will have this commit_msgs.append( dict(author=ci.authored.name, date=ci.authored.date.strftime("%m/%d/%Y %H:%M"), summary=summary, branches=current_branch, commit_url=base_url + href, shorthand_id=ci.shorthand_id())) # fill out the branch info for all the other commits prev_branch = None for c_msg in reversed(commit_msgs): if not c_msg['branches']: c_msg['branches'] = prev_branch prev_branch = c_msg['branches'] # mark which ones are first on a branch and need the branch name shown last_branch = None for c_msg in commit_msgs: if c_msg['branches'] != last_branch: c_msg['show_branch_name'] = True last_branch = c_msg['branches'] if commit_msgs: if len(commit_msgs) > 1: subject = u"{} new commits to {}".format( len(commit_msgs), repo.app.config.options.mount_label) else: commit = commit_msgs[0] subject = u'New commit {} by {}'.format(commit['shorthand_id'], commit['author']) text = g.jinja2_env.get_template( "allura:templates/mail/commits.md").render( commit_msgs=commit_msgs, max_num_commits=asint( tg.config.get('scm.notify.max_commits', 100)), ) Notification.post(artifact=repo, topic='metadata', subject=subject, text=text)
def send_notifications(repo, commit_ids): """Create appropriate notification and feed objects for a refresh :param repo: A repository artifact instance. :type repo: Repository :param commit_ids: A list of commit hash strings. :type commit_ids: list """ from allura.model import Feed, Notification commit_msgs = [] base_url = tg.config['base_url'] last_branch = [] for oids in utils.chunked_iter(commit_ids, QSIZE): chunk = list(oids) index = dict( (doc._id, doc) for doc in Commit.query.find(dict(_id={'$in': chunk}))) for oid in chunk: ci = index[oid] href = repo.url_for_commit(oid) title = _title(ci.message) summary = _summarize(ci.message) Feed.post( repo, title=title, description='%s<br><a href="%s">View Changes</a>' % ( summary, href), author_link=ci.author_url, author_name=ci.authored.name, link=href, unique_id=href) summary = g.markdown_commit.convert(ci.message) if ci.message else "" current_branch = repo.symbolics_for_commit(ci)[0] if last_branch == current_branch: branches = [] else: branches = current_branch last_branch = branches commit_msgs.append(dict( author=ci.authored.name, date=ci.authored.date.strftime("%m/%d/%Y %H:%M"), summary=summary, branches=branches, commit_url=base_url + href)) if commit_msgs: if len(commit_msgs) > 1: subject = u"{} new commits to {}".format(len(commit_msgs), repo.app.config.options.mount_label) else: subject = u'New commit by {}'.format(commit_msgs[0]['author']) template = g.jinja2_env.get_template("allura:templates/mail/commits.md") text = u"\n\n-----".join([template.render(d) for d in commit_msgs]) Notification.post( artifact=repo, topic='metadata', subject=subject, text=text)
def send_notifications(repo, commit_ids): """Create appropriate notification and feed objects for a refresh :param repo: A repository artifact instance. :type repo: Repository :param commit_ids: A list of commit hash strings, oldest to newest :type commit_ids: list """ from allura.model import Feed, Notification commit_msgs = [] base_url = tg.config['base_url'] for oids in utils.chunked_iter(commit_ids, QSIZE): chunk = list(oids) index = dict( (doc._id, doc) for doc in Commit.query.find(dict(_id={'$in': chunk}))) for oid in chunk: ci = index[oid] href = repo.url_for_commit(oid) title = _title(ci.message) summary = _summarize(ci.message) Feed.post( repo, title=title, description='%s<br><a href="%s">View Changes</a>' % ( summary, href), author_link=ci.author_url, author_name=ci.authored.name, link=href, unique_id=href) summary = g.markdown_commit.convert(ci.message) if ci.message else "" current_branch = repo.symbolics_for_commit(ci)[0] # only the head of a branch will have this commit_msgs.append(dict( author=ci.authored.name, date=ci.authored.date.strftime("%m/%d/%Y %H:%M"), summary=summary, branches=current_branch, commit_url=base_url + href, shorthand_id=ci.shorthand_id())) # fill out the branch info for all the other commits prev_branch = None for c_msg in reversed(commit_msgs): if not c_msg['branches']: c_msg['branches'] = prev_branch prev_branch = c_msg['branches'] # mark which ones are first on a branch and need the branch name shown last_branch = None for c_msg in commit_msgs: if c_msg['branches'] != last_branch: c_msg['show_branch_name'] = True last_branch = c_msg['branches'] if commit_msgs: if len(commit_msgs) > 1: subject = u"{} new commits to {}".format(len(commit_msgs), repo.app.config.options.mount_label) else: commit = commit_msgs[0] subject = u'New commit {} by {}'.format(commit['shorthand_id'], commit['author']) text = g.jinja2_env.get_template("allura:templates/mail/commits.md").render( commit_msgs=commit_msgs, max_num_commits=asint(tg.config.get('scm.notify.max_commits', 100)), ) Notification.post( artifact=repo, topic='metadata', subject=subject, text=text)