def __get_desc(self, cs):
desc_msg = [(_('%s committed on %s')
% (h.person(cs.author), h.fmt_date(cs.date))) + '<br/>']
#branches, tags, bookmarks
if cs.branch:
desc_msg.append('branch: %s<br/>' % cs.branch)
if h.is_hg(c.db_repo_scm_instance):
for book in cs.bookmarks:
desc_msg.append('bookmark: %s<br/>' % book)
for tag in cs.tags:
desc_msg.append('tag: %s<br/>' % tag)
diff_processor, changes = self.__changes(cs)
# rev link
_url = h.canonical_url('changeset_home', repo_name=c.db_repo.repo_name,
revision=cs.raw_id)
desc_msg.append('changeset: <a href="%s">%s</a>' % (_url, cs.raw_id[:8]))
desc_msg.append('<pre>')
desc_msg.append(h.urlify_text(cs.message))
desc_msg.append('\n')
desc_msg.extend(changes)
if self.include_diff:
desc_msg.append('\n\n')
desc_msg.append(diff_processor.as_raw())
desc_msg.append('</pre>')
return map(safe_unicode, desc_msg)
def __get_desc(self, cs):
desc_msg = [(_('%s committed on %s') %
(h.person(cs.author), h.fmt_date(cs.date))) + '<br/>']
#branches, tags, bookmarks
if cs.branch:
desc_msg.append('branch: %s<br/>' % cs.branch)
if h.is_hg(c.db_repo_scm_instance):
for book in cs.bookmarks:
desc_msg.append('bookmark: %s<br/>' % book)
for tag in cs.tags:
desc_msg.append('tag: %s<br/>' % tag)
diff_processor, changes = self.__changes(cs)
# rev link
_url = h.canonical_url('changeset_home',
repo_name=c.db_repo.repo_name,
revision=cs.raw_id)
desc_msg.append('changeset: <a href="%s">%s</a>' %
(_url, cs.raw_id[:8]))
desc_msg.append('<pre>')
desc_msg.append(h.urlify_text(cs.message))
desc_msg.append('\n')
desc_msg.extend(changes)
if self.include_diff:
desc_msg.append('\n\n')
desc_msg.append(diff_processor.as_raw())
desc_msg.append('</pre>')
return map(safe_unicode, desc_msg)
def index(self, repo_name, revision, f_path, annotate=False):
# redirect to given revision from form if given
post_revision = request.POST.get('at_rev', None)
if post_revision:
cs = self.__get_cs(post_revision) # FIXME - unused!
c.revision = revision
c.changeset = self.__get_cs(revision)
c.branch = request.GET.get('branch', None)
c.f_path = f_path
c.annotate = annotate
cur_rev = c.changeset.revision
# prev link
try:
prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(c.branch)
c.url_prev = url('files_home', repo_name=c.repo_name,
revision=prev_rev.raw_id, f_path=f_path)
if c.branch:
c.url_prev += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_prev = '#'
# next link
try:
next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(c.branch)
c.url_next = url('files_home', repo_name=c.repo_name,
revision=next_rev.raw_id, f_path=f_path)
if c.branch:
c.url_next += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_next = '#'
# files or dirs
try:
c.file = c.changeset.get_node(f_path)
if c.file.is_file():
c.load_full_history = False
file_last_cs = c.file.last_changeset
c.file_changeset = (c.changeset
if c.changeset.revision < file_last_cs.revision
else file_last_cs)
#determine if we're on branch head
_branches = c.db_repo_scm_instance.branches
c.on_branch_head = revision in _branches.keys() + _branches.values()
_hist = []
c.file_history = []
if c.load_full_history:
c.file_history, _hist = self._get_node_history(c.changeset, f_path)
c.authors = []
for a in set([x.author for x in _hist]):
c.authors.append((h.email(a), h.person(a)))
else:
c.authors = c.file_history = []
except RepositoryError, e:
h.flash(safe_str(e), category='error')
raise HTTPNotFound()
def authors(self, repo_name, revision, f_path):
changeset = self.__get_cs(revision)
_file = changeset.get_node(f_path)
if _file.is_file():
file_history, _hist = self._get_node_history(changeset, f_path)
c.authors = []
for a in set([x.author for x in _hist]):
c.authors.append((h.email(a), h.person(a)))
return render('files/files_history_box.html')
def authors(self, repo_name, revision, f_path):
changeset = self.__get_cs(revision)
_file = changeset.get_node(f_path)
if _file.is_file():
file_history, _hist = self._get_node_history(changeset, f_path)
c.authors = []
for a in set([x.author for x in _hist]):
c.authors.append((h.email(a), h.person(a)))
return render('files/files_history_box.html')
def index(self, format='html'):
"""GET /repo_groups: All items in the collection"""
# url('repos_groups')
_list = RepoGroup.query()\
.order_by(func.lower(RepoGroup.group_name))\
.all()
group_iter = RepoGroupList(_list, perm_set=['group.admin'])
repo_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = kallithea.CONFIG['pylons.app_globals'].mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
repo_group_name = lambda repo_group_name, children_groups: (
template.get_def("repo_group_name").render(
repo_group_name, children_groups, _=_, h=h, c=c))
repo_group_actions = lambda repo_group_id, repo_group_name, gr_count: (
template.get_def("repo_group_actions").render(repo_group_id,
repo_group_name,
gr_count,
_=_,
h=h,
c=c,
ungettext=ungettext))
for repo_gr in group_iter:
children_groups = map(
h.safe_unicode,
itertools.chain((g.name for g in repo_gr.parents),
(x.name for x in [repo_gr])))
repo_count = repo_gr.repositories.count()
repo_groups_data.append({
"raw_name":
repo_gr.group_name,
"group_name":
repo_group_name(repo_gr.group_name, children_groups),
"desc":
repo_gr.group_description,
"repos":
repo_count,
"owner":
h.person(repo_gr.user),
"action":
repo_group_actions(repo_gr.group_id, repo_gr.group_name,
repo_count)
})
c.data = json.dumps({
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": repo_groups_data
})
return render('admin/repo_groups/repo_groups.html')
def __add_reviewers(self, pr, reviewers, mention_recipients=None):
#members
for member in set(reviewers):
_usr = self._get_user(member)
reviewer = PullRequestReviewers(_usr, pr)
Session().add(reviewer)
revision_data = [(x.raw_id, x.message)
for x in map(pr.org_repo.get_changeset, pr.revisions)]
#notification to reviewers
pr_url = pr.url(canonical=True)
threading = [h.canonical_url('pullrequest_show', repo_name=pr.other_repo.repo_name,
pull_request_id=pr.pull_request_id)]
subject = safe_unicode(
h.link_to(
_('%(user)s wants you to review pull request #%(pr_id)s: %(pr_title)s') % \
{'user': pr.author.username,
'pr_title': pr.title,
'pr_id': pr.pull_request_id},
pr_url)
)
body = pr.description
_org_ref_type, org_ref_name, _org_rev = pr.org_ref.split(':')
email_kwargs = {
'pr_title': pr.title,
'pr_user_created': h.person(pr.author),
'pr_repo_url': h.canonical_url('summary_home', repo_name=pr.other_repo.repo_name),
'pr_url': pr_url,
'pr_revisions': revision_data,
'repo_name': pr.other_repo.repo_name,
'pr_id': pr.pull_request_id,
'ref': org_ref_name,
'pr_username': pr.author.username,
'threading': threading,
'is_mention': False,
}
if reviewers:
NotificationModel().create(created_by=pr.author, subject=subject, body=body,
recipients=reviewers,
type_=Notification.TYPE_PULL_REQUEST,
email_kwargs=email_kwargs)
if mention_recipients:
mention_recipients.discard(None)
mention_recipients.difference_update(reviewers)
if mention_recipients:
email_kwargs['is_mention'] = True
subject = _('[Mention]') + ' ' + subject
NotificationModel().create(created_by=pr.author, subject=subject, body=body,
recipients=mention_recipients,
type_=Notification.TYPE_PULL_REQUEST,
email_kwargs=email_kwargs)
def index(self, format='html'):
_list = RepoGroup.query(sorted=True).all()
group_iter = RepoGroupList(_list, perm_level='admin')
repo_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = app_globals.mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
repo_group_name = lambda repo_group_name, children_groups: (
template.get_def("repo_group_name").render(
repo_group_name, children_groups, _=_, h=h, c=c))
repo_group_actions = lambda repo_group_id, repo_group_name, gr_count: (
template.get_def("repo_group_actions").render(repo_group_id,
repo_group_name,
gr_count,
_=_,
h=h,
c=c,
ungettext=ungettext))
for repo_gr in group_iter:
children_groups = map(
h.safe_unicode,
itertools.chain((g.name for g in repo_gr.parents),
(x.name for x in [repo_gr])))
repo_count = repo_gr.repositories.count()
repo_groups_data.append({
"raw_name":
repo_gr.group_name,
"group_name":
repo_group_name(repo_gr.group_name, children_groups),
"desc":
h.escape(repo_gr.group_description),
"repos":
repo_count,
"owner":
h.person(repo_gr.owner),
"action":
repo_group_actions(repo_gr.group_id, repo_gr.group_name,
repo_count)
})
c.data = {
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": repo_groups_data
}
return render('admin/repo_groups/repo_groups.html')
def index(self, format='html'):
"""GET /users_groups: All items in the collection"""
# url('users_groups')
_list = UserGroup.query()\
.order_by(func.lower(UserGroup.users_group_name))\
.all()
group_iter = UserGroupList(_list, perm_set=['usergroup.admin'])
user_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = kallithea.CONFIG['pylons.app_globals'].mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
user_group_name = lambda user_group_id, user_group_name: (
template.get_def("user_group_name").render(
user_group_id, user_group_name, _=_, h=h, c=c))
user_group_actions = lambda user_group_id, user_group_name: (
template.get_def("user_group_actions").render(
user_group_id, user_group_name, _=_, h=h, c=c))
for user_gr in group_iter:
user_groups_data.append({
"raw_name":
user_gr.users_group_name,
"group_name":
user_group_name(user_gr.users_group_id,
user_gr.users_group_name),
"desc":
user_gr.user_group_description,
"members":
len(user_gr.members),
"active":
h.boolicon(user_gr.users_group_active),
"owner":
h.person(user_gr.user.username),
"action":
user_group_actions(user_gr.users_group_id,
user_gr.users_group_name)
})
c.data = json.dumps({
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": user_groups_data
})
return render('admin/user_groups/user_groups.html')
def index(self, format='html'):
"""GET /repo_groups: All items in the collection"""
# url('repos_groups')
_list = RepoGroup.query()\
.order_by(func.lower(RepoGroup.group_name))\
.all()
group_iter = RepoGroupList(_list, perm_set=['group.admin'])
repo_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = kallithea.CONFIG['pylons.app_globals'].mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
repo_group_name = lambda repo_group_name, children_groups: (
template.get_def("repo_group_name")
.render(repo_group_name, children_groups, _=_, h=h, c=c)
)
repo_group_actions = lambda repo_group_id, repo_group_name, gr_count: (
template.get_def("repo_group_actions")
.render(repo_group_id, repo_group_name, gr_count, _=_, h=h, c=c,
ungettext=ungettext)
)
for repo_gr in group_iter:
children_groups = map(h.safe_unicode,
itertools.chain((g.name for g in repo_gr.parents),
(x.name for x in [repo_gr])))
repo_count = repo_gr.repositories.count()
repo_groups_data.append({
"raw_name": repo_gr.group_name,
"group_name": repo_group_name(repo_gr.group_name, children_groups),
"desc": h.escape(repo_gr.group_description),
"repos": repo_count,
"owner": h.person(repo_gr.user),
"action": repo_group_actions(repo_gr.group_id, repo_gr.group_name,
repo_count)
})
c.data = json.dumps({
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": repo_groups_data
})
return render('admin/repo_groups/repo_groups.html')
def index(self, format='html'):
_list = UserGroup.query() \
.order_by(func.lower(UserGroup.users_group_name)) \
.all()
group_iter = UserGroupList(_list, perm_level='admin')
user_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = app_globals.mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
user_group_name = lambda user_group_id, user_group_name: (
template.get_def("user_group_name").render(
user_group_id, user_group_name, _=_, h=h, c=c))
user_group_actions = lambda user_group_id, user_group_name: (
template.get_def("user_group_actions").render(
user_group_id, user_group_name, _=_, h=h, c=c))
for user_gr in group_iter:
user_groups_data.append({
"raw_name":
user_gr.users_group_name,
"group_name":
user_group_name(user_gr.users_group_id,
user_gr.users_group_name),
"desc":
h.escape(user_gr.user_group_description),
"members":
len(user_gr.members),
"active":
h.boolicon(user_gr.users_group_active),
"owner":
h.person(user_gr.owner.username),
"action":
user_group_actions(user_gr.users_group_id,
user_gr.users_group_name)
})
c.data = {
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": user_groups_data
}
return render('admin/user_groups/user_groups.html')
def index(self, format='html'):
"""GET /users_groups: All items in the collection"""
# url('users_groups')
_list = UserGroup.query()\
.order_by(func.lower(UserGroup.users_group_name))\
.all()
group_iter = UserGroupList(_list, perm_set=['usergroup.admin'])
user_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = kallithea.CONFIG['pylons.app_globals'].mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
user_group_name = lambda user_group_id, user_group_name: (
template.get_def("user_group_name")
.render(user_group_id, user_group_name, _=_, h=h, c=c)
)
user_group_actions = lambda user_group_id, user_group_name: (
template.get_def("user_group_actions")
.render(user_group_id, user_group_name, _=_, h=h, c=c)
)
for user_gr in group_iter:
user_groups_data.append({
"raw_name": user_gr.users_group_name,
"group_name": user_group_name(user_gr.users_group_id,
user_gr.users_group_name),
"desc": user_gr.user_group_description,
"members": len(user_gr.members),
"active": h.boolicon(user_gr.users_group_active),
"owner": h.person(user_gr.user.username),
"action": user_group_actions(user_gr.users_group_id, user_gr.users_group_name)
})
c.data = json.dumps({
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": user_groups_data
})
return render('admin/user_groups/user_groups.html')
def index(self, format='html'):
_list = RepoGroup.query(sorted=True).all()
group_iter = RepoGroupList(_list, perm_level='admin')
repo_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = app_globals.mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
repo_group_name = lambda repo_group_name, children_groups: (
template.get_def("repo_group_name")
.render(repo_group_name, children_groups, _=_, h=h, c=c)
)
repo_group_actions = lambda repo_group_id, repo_group_name, gr_count: (
template.get_def("repo_group_actions")
.render(repo_group_id, repo_group_name, gr_count, _=_, h=h, c=c,
ungettext=ungettext)
)
for repo_gr in group_iter:
children_groups = map(h.safe_unicode,
itertools.chain((g.name for g in repo_gr.parents),
(x.name for x in [repo_gr])))
repo_count = repo_gr.repositories.count()
repo_groups_data.append({
"raw_name": repo_gr.group_name,
"group_name": repo_group_name(repo_gr.group_name, children_groups),
"desc": h.escape(repo_gr.group_description),
"repos": repo_count,
"owner": h.person(repo_gr.owner),
"action": repo_group_actions(repo_gr.group_id, repo_gr.group_name,
repo_count)
})
c.data = {
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": repo_groups_data
}
return render('admin/repo_groups/repo_groups.html')
def index(self, format='html'):
_list = UserGroup.query() \
.order_by(func.lower(UserGroup.users_group_name)) \
.all()
group_iter = UserGroupList(_list, perm_level='admin')
user_groups_data = []
total_records = len(group_iter)
_tmpl_lookup = app_globals.mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
user_group_name = lambda user_group_id, user_group_name: (
template.get_def("user_group_name")
.render(user_group_id, user_group_name, _=_, h=h, c=c)
)
user_group_actions = lambda user_group_id, user_group_name: (
template.get_def("user_group_actions")
.render(user_group_id, user_group_name, _=_, h=h, c=c)
)
for user_gr in group_iter:
user_groups_data.append({
"raw_name": user_gr.users_group_name,
"group_name": user_group_name(user_gr.users_group_id,
user_gr.users_group_name),
"desc": h.escape(user_gr.user_group_description),
"members": len(user_gr.members),
"active": h.boolicon(user_gr.users_group_active),
"owner": h.person(user_gr.owner.username),
"action": user_group_actions(user_gr.users_group_id, user_gr.users_group_name)
})
c.data = {
"totalRecords": total_records,
"startIndex": 0,
"sort": None,
"dir": "asc",
"records": user_groups_data
}
return render('admin/user_groups/user_groups.html')
def index(self, format='html'):
_list = RepoGroup.query(sorted=True).all()
group_iter = RepoGroupList(_list, perm_level='admin')
repo_groups_data = []
_tmpl_lookup = app_globals.mako_lookup
template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
def repo_group_name(repo_group_name, children_groups):
return template.get_def("repo_group_name") \
.render_unicode(repo_group_name, children_groups, _=_, h=h, c=c)
def repo_group_actions(repo_group_id, repo_group_name, gr_count):
return template.get_def("repo_group_actions") \
.render_unicode(repo_group_id, repo_group_name, gr_count, _=_, h=h, c=c,
ungettext=ungettext)
for repo_gr in group_iter:
children_groups = [g.name for g in repo_gr.parents] + [repo_gr.name]
repo_count = repo_gr.repositories.count()
repo_groups_data.append({
"raw_name": repo_gr.group_name,
"group_name": repo_group_name(repo_gr.group_name, children_groups),
"desc": h.escape(repo_gr.group_description),
"repos": repo_count,
"owner": h.person(repo_gr.owner),
"action": repo_group_actions(repo_gr.group_id, repo_gr.group_name,
repo_count)
})
c.data = {
"sort": None,
"dir": "asc",
"records": repo_groups_data
}
return render('admin/repo_groups/repo_groups.html')
def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
log = get_logger(get_commits_stats)
DBS = get_session()
lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
ts_max_y)
lockkey_path = config['app_conf']['cache_dir']
log.info('running task with lockkey %s', lockkey)
try:
lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))
# for js data compatibility cleans the key for person from '
akc = lambda k: person(k).replace('"', "")
co_day_auth_aggr = {}
commits_by_day_aggregate = {}
repo = Repository.get_by_repo_name(repo_name)
if repo is None:
return True
repo = repo.scm_instance
repo_size = repo.count()
# return if repo have no revisions
if repo_size < 1:
lock.release()
return True
skip_date_limit = True
parse_limit = int(config['app_conf'].get('commit_parse_limit'))
last_rev = None
last_cs = None
timegetter = itemgetter('time')
dbrepo = DBS.query(Repository) \
.filter(Repository.repo_name == repo_name).scalar()
cur_stats = DBS.query(Statistics) \
.filter(Statistics.repository == dbrepo).scalar()
if cur_stats is not None:
last_rev = cur_stats.stat_on_revision
if last_rev == repo.get_changeset().revision and repo_size > 1:
# pass silently without any work if we're not on first revision or
# current state of parsing revision(from db marker) is the
# last revision
lock.release()
return True
if cur_stats:
commits_by_day_aggregate = OrderedDict(json.loads(
cur_stats.commit_activity_combined))
co_day_auth_aggr = json.loads(cur_stats.commit_activity)
log.debug('starting parsing %s', parse_limit)
lmktime = mktime
last_rev = last_rev + 1 if last_rev >= 0 else 0
log.debug('Getting revisions from %s to %s',
last_rev, last_rev + parse_limit
)
for cs in repo[last_rev:last_rev + parse_limit]:
log.debug('parsing %s', cs)
last_cs = cs # remember last parsed changeset
k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
if akc(cs.author) in co_day_auth_aggr:
try:
l = [timegetter(x) for x in
co_day_auth_aggr[akc(cs.author)]['data']]
time_pos = l.index(k)
except ValueError:
time_pos = None
if time_pos >= 0 and time_pos is not None:
datadict = \
co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
datadict["commits"] += 1
datadict["added"] += len(cs.added)
datadict["changed"] += len(cs.changed)
datadict["removed"] += len(cs.removed)
else:
if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
datadict = {"time": k,
"commits": 1,
"added": len(cs.added),
"changed": len(cs.changed),
"removed": len(cs.removed),
}
co_day_auth_aggr[akc(cs.author)]['data'] \
.append(datadict)
else:
if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
co_day_auth_aggr[akc(cs.author)] = {
"label": akc(cs.author),
"data": [{"time":k,
"commits":1,
"added":len(cs.added),
"changed":len(cs.changed),
"removed":len(cs.removed),
}],
"schema": ["commits"],
}
#gather all data by day
if k in commits_by_day_aggregate:
commits_by_day_aggregate[k] += 1
else:
commits_by_day_aggregate[k] = 1
overview_data = sorted(commits_by_day_aggregate.items(),
key=itemgetter(0))
if not co_day_auth_aggr:
co_day_auth_aggr[akc(repo.contact)] = {
"label": akc(repo.contact),
"data": [0, 1],
"schema": ["commits"],
}
stats = cur_stats if cur_stats else Statistics()
stats.commit_activity = json.dumps(co_day_auth_aggr)
stats.commit_activity_combined = json.dumps(overview_data)
log.debug('last revision %s', last_rev)
leftovers = len(repo.revisions[last_rev:])
log.debug('revisions to parse %s', leftovers)
if last_rev == 0 or leftovers < parse_limit:
log.debug('getting code trending stats')
stats.languages = json.dumps(__get_codes_stats(repo_name))
try:
stats.repository = dbrepo
stats.stat_on_revision = last_cs.revision if last_cs else 0
DBS.add(stats)
DBS.commit()
except:
log.error(traceback.format_exc())
DBS.rollback()
lock.release()
return False
# final release
lock.release()
# execute another task if celery is enabled
if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:
recurse_limit -= 1
run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y,
recurse_limit)
if recurse_limit <= 0:
log.debug('Breaking recursive mode due to reach of recurse limit')
return True
except LockHeld:
log.info('LockHeld')
return 'Task with key %s already running' % lockkey
def get_repos_as_dict(self, repos_list=None, admin=False, perm_check=True,
super_user_actions=False):
_render = self._render_datatable
from pylons import tmpl_context as c
def quick_menu(repo_name):
return _render('quick_menu', repo_name)
def repo_lnk(name, rtype, rstate, private, fork_of):
return _render('repo_name', name, rtype, rstate, private, fork_of,
short_name=not admin, admin=False)
def last_change(last_change):
return _render("last_change", last_change)
def rss_lnk(repo_name):
return _render("rss", repo_name)
def atom_lnk(repo_name):
return _render("atom", repo_name)
def last_rev(repo_name, cs_cache):
return _render('revision', repo_name, cs_cache.get('revision'),
cs_cache.get('raw_id'), cs_cache.get('author'),
cs_cache.get('message'))
def desc(desc):
if c.visual.stylify_metatags:
return h.urlify_text(h.desc_stylize(h.truncate(desc, 60)))
else:
return h.urlify_text(h.truncate(desc, 60))
def state(repo_state):
return _render("repo_state", repo_state)
def repo_actions(repo_name):
return _render('repo_actions', repo_name, super_user_actions)
def owner_actions(user_id, username):
return _render('user_name', user_id, username)
repos_data = []
for repo in repos_list:
if perm_check:
# check permission at this level
if not HasRepoPermissionAny(
'repository.read', 'repository.write',
'repository.admin'
)(repo.repo_name, 'get_repos_as_dict check'):
continue
cs_cache = repo.changeset_cache
row = {
"menu": quick_menu(repo.repo_name),
"raw_name": repo.repo_name.lower(),
"name": repo_lnk(repo.repo_name, repo.repo_type,
repo.repo_state, repo.private, repo.fork),
"last_change": last_change(repo.last_db_change),
"last_changeset": last_rev(repo.repo_name, cs_cache),
"last_rev_raw": cs_cache.get('revision'),
"desc": desc(repo.description),
"owner": h.person(repo.user),
"state": state(repo.repo_state),
"rss": rss_lnk(repo.repo_name),
"atom": atom_lnk(repo.repo_name),
}
if admin:
row.update({
"action": repo_actions(repo.repo_name),
"owner": owner_actions(repo.user.user_id,
h.person(repo.user))
})
repos_data.append(row)
return {
"totalRecords": len(repos_list),
"startIndex": 0,
"sort": "name",
"dir": "asc",
"records": repos_data
}
def index(self, repo_name, revision, f_path, annotate=False):
# redirect to given revision from form if given
post_revision = request.POST.get('at_rev', None)
if post_revision:
cs = self.__get_cs(post_revision) # FIXME - unused!
c.revision = revision
c.changeset = self.__get_cs(revision)
c.branch = request.GET.get('branch', None)
c.f_path = f_path
c.annotate = annotate
cur_rev = c.changeset.revision
c.fulldiff = request.GET.get('fulldiff')
# prev link
try:
prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(c.branch)
c.url_prev = url('files_home', repo_name=c.repo_name,
revision=prev_rev.raw_id, f_path=f_path)
if c.branch:
c.url_prev += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_prev = '#'
# next link
try:
next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(c.branch)
c.url_next = url('files_home', repo_name=c.repo_name,
revision=next_rev.raw_id, f_path=f_path)
if c.branch:
c.url_next += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_next = '#'
# files or dirs
try:
c.file = c.changeset.get_node(f_path)
if c.file.is_file():
c.load_full_history = False
file_last_cs = c.file.last_changeset
c.file_changeset = (c.changeset
if c.changeset.revision < file_last_cs.revision
else file_last_cs)
#determine if we're on branch head
_branches = c.db_repo_scm_instance.branches
c.on_branch_head = revision in _branches.keys() + _branches.values()
_hist = []
c.file_history = []
if c.load_full_history:
c.file_history, _hist = self._get_node_history(c.changeset, f_path)
c.authors = []
for a in set([x.author for x in _hist]):
c.authors.append((h.email(a), h.person(a)))
else:
c.authors = c.file_history = []
except RepositoryError as e:
h.flash(safe_str(e), category='error')
raise HTTPNotFound()
if request.environ.get('HTTP_X_PARTIAL_XHR'):
return render('files/files_ypjax.html')
# TODO: tags and bookmarks?
c.revision_options = [(c.changeset.raw_id,
_('%s at %s') % (c.changeset.branch, h.short_id(c.changeset.raw_id)))] + \
[(n, b) for b, n in c.db_repo_scm_instance.branches.items()]
if c.db_repo_scm_instance.closed_branches:
prefix = _('(closed)') + ' '
c.revision_options += [('-', '-')] + \
[(n, prefix + b) for b, n in c.db_repo_scm_instance.closed_branches.items()]
return render('files/files.html')
def get_repos_as_dict(self,
repos_list,
repo_groups_list=None,
admin=False,
short_name=False):
"""Return repository list for use by DataTable.
repos_list: list of repositories - but will be filtered for read permission.
repo_groups_list: added at top of list without permission check.
admin: return data for action column.
"""
_render = self._render_datatable
from tg import tmpl_context as c, request
from kallithea.model.scm import ScmModel
def repo_lnk(name, rtype, rstate, private, fork_of):
return _render('repo_name',
name,
rtype,
rstate,
private,
fork_of,
short_name=short_name)
def following(repo_id, is_following):
return _render('following', repo_id, is_following)
def last_change(last_change):
return _render("last_change", last_change)
def rss_lnk(repo_name):
return _render("rss", repo_name)
def atom_lnk(repo_name):
return _render("atom", repo_name)
def last_rev(repo_name, cs_cache):
return _render('revision', repo_name, cs_cache.get('revision'),
cs_cache.get('raw_id'), cs_cache.get('author'),
cs_cache.get('message'))
def desc(desc):
return h.urlify_text(desc,
truncate=80,
stylize=c.visual.stylify_metalabels)
def state(repo_state):
return _render("repo_state", repo_state)
def repo_actions(repo_name):
return _render('repo_actions', repo_name)
def owner_actions(owner_id, username):
return _render('user_name', owner_id, username)
repos_data = []
for gr in repo_groups_list or []:
repos_data.append(
dict(
raw_name='\0' + gr.name, # sort before repositories
just_name=gr.name,
name=_render('group_name_html',
group_name=gr.group_name,
name=gr.name),
desc=gr.group_description))
for repo in repos_list:
if not HasRepoPermissionLevel('read')(repo.repo_name,
'get_repos_as_dict check'):
continue
cs_cache = repo.changeset_cache
row = {
"raw_name":
repo.repo_name,
"just_name":
repo.just_name,
"name":
repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
repo.private, repo.fork),
"following":
following(
repo.repo_id,
ScmModel().is_following_repo(repo.repo_name,
request.authuser.user_id),
),
"last_change_iso":
repo.last_db_change.isoformat(),
"last_change":
last_change(repo.last_db_change),
"last_changeset":
last_rev(repo.repo_name, cs_cache),
"last_rev_raw":
cs_cache.get('revision'),
"desc":
desc(repo.description),
"owner":
h.person(repo.owner),
"state":
state(repo.repo_state),
"rss":
rss_lnk(repo.repo_name),
"atom":
atom_lnk(repo.repo_name),
}
if admin:
row.update({
"action":
repo_actions(repo.repo_name),
"owner":
owner_actions(repo.owner_id, h.person(repo.owner))
})
repos_data.append(row)
return {"sort": "name", "dir": "asc", "records": repos_data}
def index(self, repo_name, revision, f_path, annotate=False):
# redirect to given revision from form if given
post_revision = request.POST.get('at_rev', None)
if post_revision:
cs = self.__get_cs(post_revision) # FIXME - unused!
c.revision = revision
c.changeset = self.__get_cs(revision)
c.branch = request.GET.get('branch', None)
c.f_path = f_path
c.annotate = annotate
cur_rev = c.changeset.revision
# used in files_source.html:
c.cut_off_limit = self.cut_off_limit
c.fulldiff = request.GET.get('fulldiff')
# prev link
try:
prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(
c.branch)
c.url_prev = url('files_home',
repo_name=c.repo_name,
revision=prev_rev.raw_id,
f_path=f_path)
if c.branch:
c.url_prev += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_prev = '#'
# next link
try:
next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(
c.branch)
c.url_next = url('files_home',
repo_name=c.repo_name,
revision=next_rev.raw_id,
f_path=f_path)
if c.branch:
c.url_next += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_next = '#'
# files or dirs
try:
c.file = c.changeset.get_node(f_path)
if c.file.is_submodule():
raise HTTPFound(location=c.file.url)
elif c.file.is_file():
c.load_full_history = False
# determine if we're on branch head
_branches = c.db_repo_scm_instance.branches
c.on_branch_head = revision in _branches or revision in _branches.values(
)
_hist = []
c.file_history = []
if c.load_full_history:
c.file_history, _hist = self._get_node_history(
c.changeset, f_path)
c.authors = []
for a in set([x.author for x in _hist]):
c.authors.append((h.email(a), h.person(a)))
else:
c.authors = c.file_history = []
except RepositoryError as e:
h.flash(e, category='error')
raise HTTPNotFound()
if request.environ.get('HTTP_X_PARTIAL_XHR'):
return render('files/files_ypjax.html')
# TODO: tags and bookmarks?
c.revision_options = [(c.changeset.raw_id,
_('%s at %s') % (b, h.short_id(c.changeset.raw_id))) for b in c.changeset.branches] + \
[(n, b) for b, n in c.db_repo_scm_instance.branches.items()]
if c.db_repo_scm_instance.closed_branches:
prefix = _('(closed)') + ' '
c.revision_options += [('-', '-')] + \
[(n, prefix + b) for b, n in c.db_repo_scm_instance.closed_branches.items()]
return render('files/files.html')
def _get_notification_data(self,
repo,
comment,
user,
comment_text,
line_no=None,
revision=None,
pull_request=None,
status_change=None,
closing_pr=False):
"""
Get notification data
:param comment_text:
:param line:
:returns: tuple (subj,body,recipients,notification_type,email_kwargs)
"""
# make notification
body = comment_text # text of the comment
line = ''
if line_no:
line = _('on line %s') % line_no
#changeset
if revision:
notification_type = Notification.TYPE_CHANGESET_COMMENT
cs = repo.scm_instance.get_changeset(revision)
desc = "%s" % (cs.short_id)
threading = [
'%s-rev-%s@%s' %
(repo.repo_name, revision, h.canonical_hostname())
]
if line_no: # TODO: url to file _and_ line number
threading.append('%s-rev-%s-line-%s@%s' %
(repo.repo_name, revision, line_no,
h.canonical_hostname()))
comment_url = h.canonical_url('changeset_home',
repo_name=repo.repo_name,
revision=revision,
anchor='comment-%s' %
comment.comment_id)
subj = safe_unicode(
h.link_to('Re changeset: %(desc)s %(line)s' % \
{'desc': desc, 'line': line},
comment_url)
)
# get the current participants of this changeset
recipients = ChangesetComment.get_users(revision=revision)
# add changeset author if it's in kallithea system
cs_author = User.get_from_cs_author(cs.author)
if not cs_author:
#use repo owner if we cannot extract the author correctly
cs_author = repo.user
recipients += [cs_author]
email_kwargs = {
'status_change':
status_change,
'cs_comment_user':
h.person(user, 'username_and_name'),
'cs_target_repo':
h.canonical_url('summary_home', repo_name=repo.repo_name),
'cs_comment_url':
comment_url,
'raw_id':
revision,
'message':
cs.message,
'repo_name':
repo.repo_name,
'short_id':
h.short_id(revision),
'branch':
cs.branch,
'comment_username':
user.username,
'threading':
threading,
}
#pull request
elif pull_request:
notification_type = Notification.TYPE_PULL_REQUEST_COMMENT
desc = comment.pull_request.title
_org_ref_type, org_ref_name, _org_rev = comment.pull_request.org_ref.split(
':')
threading = [
'%s-pr-%s@%s' %
(pull_request.other_repo.repo_name,
pull_request.pull_request_id, h.canonical_hostname())
]
if line_no: # TODO: url to file _and_ line number
threading.append('%s-pr-%s-line-%s@%s' %
(pull_request.other_repo.repo_name,
pull_request.pull_request_id, line_no,
h.canonical_hostname()))
comment_url = pull_request.url(canonical=True,
anchor='comment-%s' %
comment.comment_id)
subj = safe_unicode(
h.link_to('Re pull request #%(pr_id)s: %(desc)s %(line)s' % \
{'desc': desc,
'pr_id': comment.pull_request.pull_request_id,
'line': line},
comment_url)
)
# get the current participants of this pull request
recipients = ChangesetComment.get_users(
pull_request_id=pull_request.pull_request_id)
# add pull request author
recipients += [pull_request.author]
# add the reviewers to notification
recipients += [x.user for x in pull_request.reviewers]
#set some variables for email notification
email_kwargs = {
'pr_title':
pull_request.title,
'pr_id':
pull_request.pull_request_id,
'status_change':
status_change,
'closing_pr':
closing_pr,
'pr_comment_url':
comment_url,
'pr_comment_user':
h.person(user, 'username_and_name'),
'pr_target_repo':
h.canonical_url('summary_home',
repo_name=pull_request.other_repo.repo_name),
'repo_name':
pull_request.other_repo.repo_name,
'ref':
org_ref_name,
'comment_username':
user.username,
'threading':
threading,
}
return subj, body, recipients, notification_type, email_kwargs
def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
log = get_logger(get_commits_stats)
DBS = get_session()
lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, ts_max_y)
lockkey_path = config['app_conf']['cache_dir']
log.info('running task with lockkey %s' % lockkey)
try:
lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))
# for js data compatibility cleans the key for person from '
akc = lambda k: person(k).replace('"', "")
co_day_auth_aggr = {}
commits_by_day_aggregate = {}
repo = Repository.get_by_repo_name(repo_name)
if repo is None:
return True
repo = repo.scm_instance
repo_size = repo.count()
# return if repo have no revisions
if repo_size < 1:
lock.release()
return True
skip_date_limit = True
parse_limit = int(config['app_conf'].get('commit_parse_limit'))
last_rev = None
last_cs = None
timegetter = itemgetter('time')
dbrepo = DBS.query(Repository)\
.filter(Repository.repo_name == repo_name).scalar()
cur_stats = DBS.query(Statistics)\
.filter(Statistics.repository == dbrepo).scalar()
if cur_stats is not None:
last_rev = cur_stats.stat_on_revision
if last_rev == repo.get_changeset().revision and repo_size > 1:
# pass silently without any work if we're not on first revision or
# current state of parsing revision(from db marker) is the
# last revision
lock.release()
return True
if cur_stats:
commits_by_day_aggregate = OrderedDict(
json.loads(cur_stats.commit_activity_combined))
co_day_auth_aggr = json.loads(cur_stats.commit_activity)
log.debug('starting parsing %s' % parse_limit)
lmktime = mktime
last_rev = last_rev + 1 if last_rev >= 0 else 0
log.debug('Getting revisions from %s to %s' %
(last_rev, last_rev + parse_limit))
for cs in repo[last_rev:last_rev + parse_limit]:
log.debug('parsing %s' % cs)
last_cs = cs # remember last parsed changeset
k = lmktime([
cs.date.timetuple()[0],
cs.date.timetuple()[1],
cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0
])
if akc(cs.author) in co_day_auth_aggr:
try:
l = [
timegetter(x)
for x in co_day_auth_aggr[akc(cs.author)]['data']
]
time_pos = l.index(k)
except ValueError:
time_pos = None
if time_pos >= 0 and time_pos is not None:
datadict = \
co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
datadict["commits"] += 1
datadict["added"] += len(cs.added)
datadict["changed"] += len(cs.changed)
datadict["removed"] += len(cs.removed)
else:
if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
datadict = {
"time": k,
"commits": 1,
"added": len(cs.added),
"changed": len(cs.changed),
"removed": len(cs.removed),
}
co_day_auth_aggr[akc(cs.author)]['data']\
.append(datadict)
else:
if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
co_day_auth_aggr[akc(cs.author)] = {
"label":
akc(cs.author),
"data": [{
"time": k,
"commits": 1,
"added": len(cs.added),
"changed": len(cs.changed),
"removed": len(cs.removed),
}],
"schema": ["commits"],
}
#gather all data by day
if k in commits_by_day_aggregate:
commits_by_day_aggregate[k] += 1
else:
commits_by_day_aggregate[k] = 1
overview_data = sorted(commits_by_day_aggregate.items(),
key=itemgetter(0))
if not co_day_auth_aggr:
co_day_auth_aggr[akc(repo.contact)] = {
"label": akc(repo.contact),
"data": [0, 1],
"schema": ["commits"],
}
stats = cur_stats if cur_stats else Statistics()
stats.commit_activity = json.dumps(co_day_auth_aggr)
stats.commit_activity_combined = json.dumps(overview_data)
log.debug('last revision %s' % last_rev)
leftovers = len(repo.revisions[last_rev:])
log.debug('revisions to parse %s' % leftovers)
if last_rev == 0 or leftovers < parse_limit:
log.debug('getting code trending stats')
stats.languages = json.dumps(__get_codes_stats(repo_name))
try:
stats.repository = dbrepo
stats.stat_on_revision = last_cs.revision if last_cs else 0
DBS.add(stats)
DBS.commit()
except:
log.error(traceback.format_exc())
DBS.rollback()
lock.release()
return False
# final release
lock.release()
# execute another task if celery is enabled
if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:
recurse_limit -= 1
run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y,
recurse_limit)
if recurse_limit <= 0:
log.debug('Breaking recursive mode due to reach of recurse limit')
return True
except LockHeld:
log.info('LockHeld')
return 'Task with key %s already running' % lockkey
def _get_notification_data(self, repo, comment, user, comment_text,
line_no=None, revision=None, pull_request=None,
status_change=None, closing_pr=False):
"""
Get notification data
:param comment_text:
:param line:
:returns: tuple (subj,body,recipients,notification_type,email_kwargs)
"""
# make notification
body = comment_text # text of the comment
line = ''
if line_no:
line = _('on line %s') % line_no
#changeset
if revision:
notification_type = Notification.TYPE_CHANGESET_COMMENT
cs = repo.scm_instance.get_changeset(revision)
desc = "%s" % (cs.short_id)
threading = ['%s-rev-%s@%s' % (repo.repo_name, revision, h.canonical_hostname())]
if line_no: # TODO: url to file _and_ line number
threading.append('%s-rev-%s-line-%s@%s' % (repo.repo_name, revision, line_no,
h.canonical_hostname()))
comment_url = h.canonical_url('changeset_home',
repo_name=repo.repo_name,
revision=revision,
anchor='comment-%s' % comment.comment_id)
subj = safe_unicode(
h.link_to('Re changeset: %(desc)s %(line)s' % \
{'desc': desc, 'line': line},
comment_url)
)
# get the current participants of this changeset
recipients = ChangesetComment.get_users(revision=revision)
# add changeset author if it's in kallithea system
cs_author = User.get_from_cs_author(cs.author)
if not cs_author:
#use repo owner if we cannot extract the author correctly
cs_author = repo.user
recipients += [cs_author]
email_kwargs = {
'status_change': status_change,
'cs_comment_user': h.person(user, 'username_and_name'),
'cs_target_repo': h.canonical_url('summary_home', repo_name=repo.repo_name),
'cs_comment_url': comment_url,
'raw_id': revision,
'message': cs.message,
'repo_name': repo.repo_name,
'short_id': h.short_id(revision),
'branch': cs.branch,
'comment_username': user.username,
'threading': threading,
}
#pull request
elif pull_request:
notification_type = Notification.TYPE_PULL_REQUEST_COMMENT
desc = comment.pull_request.title
_org_ref_type, org_ref_name, _org_rev = comment.pull_request.org_ref.split(':')
threading = ['%s-pr-%s@%s' % (pull_request.other_repo.repo_name,
pull_request.pull_request_id,
h.canonical_hostname())]
if line_no: # TODO: url to file _and_ line number
threading.append('%s-pr-%s-line-%s@%s' % (pull_request.other_repo.repo_name,
pull_request.pull_request_id, line_no,
h.canonical_hostname()))
comment_url = pull_request.url(canonical=True,
anchor='comment-%s' % comment.comment_id)
subj = safe_unicode(
h.link_to('Re pull request #%(pr_id)s: %(desc)s %(line)s' % \
{'desc': desc,
'pr_id': comment.pull_request.pull_request_id,
'line': line},
comment_url)
)
# get the current participants of this pull request
recipients = ChangesetComment.get_users(pull_request_id=
pull_request.pull_request_id)
# add pull request author
recipients += [pull_request.author]
# add the reviewers to notification
recipients += [x.user for x in pull_request.reviewers]
#set some variables for email notification
email_kwargs = {
'pr_title': pull_request.title,
'pr_id': pull_request.pull_request_id,
'status_change': status_change,
'closing_pr': closing_pr,
'pr_comment_url': comment_url,
'pr_comment_user': h.person(user, 'username_and_name'),
'pr_target_repo': h.canonical_url('summary_home',
repo_name=pull_request.other_repo.repo_name),
'repo_name': pull_request.other_repo.repo_name,
'ref': org_ref_name,
'comment_username': user.username,
'threading': threading,
}
return subj, body, recipients, notification_type, email_kwargs
def get_repos_as_dict(self,
repos_list=None,
admin=False,
perm_check=True,
super_user_actions=False,
short_name=False):
_render = self._render_datatable
from tg import tmpl_context as c
def repo_lnk(name, rtype, rstate, private, fork_of):
return _render('repo_name',
name,
rtype,
rstate,
private,
fork_of,
short_name=short_name,
admin=False)
def last_change(last_change):
return _render("last_change", last_change)
def rss_lnk(repo_name):
return _render("rss", repo_name)
def atom_lnk(repo_name):
return _render("atom", repo_name)
def last_rev(repo_name, cs_cache):
return _render('revision', repo_name, cs_cache.get('revision'),
cs_cache.get('raw_id'), cs_cache.get('author'),
cs_cache.get('message'))
def desc(desc):
return h.urlify_text(desc,
truncate=80,
stylize=c.visual.stylify_metatags)
def state(repo_state):
return _render("repo_state", repo_state)
def repo_actions(repo_name):
return _render('repo_actions', repo_name, super_user_actions)
def owner_actions(owner_id, username):
return _render('user_name', owner_id, username)
repos_data = []
for repo in repos_list:
if perm_check:
# check permission at this level
if not HasRepoPermissionLevel('read')(
repo.repo_name, 'get_repos_as_dict check'):
continue
cs_cache = repo.changeset_cache
row = {
"raw_name":
repo.repo_name,
"just_name":
repo.just_name,
"name":
repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
repo.private, repo.fork),
"last_change_iso":
repo.last_db_change.isoformat(),
"last_change":
last_change(repo.last_db_change),
"last_changeset":
last_rev(repo.repo_name, cs_cache),
"last_rev_raw":
cs_cache.get('revision'),
"desc":
desc(repo.description),
"owner":
h.person(repo.owner),
"state":
state(repo.repo_state),
"rss":
rss_lnk(repo.repo_name),
"atom":
atom_lnk(repo.repo_name),
}
if admin:
row.update({
"action":
repo_actions(repo.repo_name),
"owner":
owner_actions(repo.owner_id, h.person(repo.owner))
})
repos_data.append(row)
return {
"totalRecords": len(repos_list),
"startIndex": 0,
"sort": "name",
"dir": "asc",
"records": repos_data
}
def akc(k):
return person(k).replace('"', '')
