def diff(self, pid=None, tid=None, cid=None, gid=None,
dfrom=None, dto=None, dfromref=None, dtoref=None,
inc_merge_commit=None, inc_repos=None, metadata=None,
exc_groups=None, limit=None, inc_groups=None):
if not dfrom or not dto:
abort(404,
detail="Must specify dfrom and dto dates for the new "
"contributors")
if not dfromref or not dtoref:
abort(404,
detail="Must specify dfromref and dtoref dates for the "
"reference period to compute new contributors")
# Get contributors for the new period
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit, metadata,
exc_groups, inc_groups)
authors_new = self.gbycommits(
c, idents, query_kwargs, top=-1,
resolv_name=False, clean_email=False)
# Now get contributors for the old reference period
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfromref, dtoref, inc_repos, inc_merge_commit, metadata,
exc_groups, inc_groups)
authors_old = self.gbycommits(
c, idents, query_kwargs, top=-1,
resolv_name=False, clean_email=False)
# And compute the difference
cids_new = set([auth['cid'] for auth in authors_new]) - \
set([auth['cid'] for auth in authors_old])
authors_diff = [author for author in authors_new
if author['cid'] in cids_new]
if limit is None:
limit = 10
else:
limit = int(limit)
# If limit set to a negative value all results will be returned
if limit >= 0:
authors_diff = authors_diff[:limit]
self.resolv_name(c, authors_diff)
return authors_diff
def diff(self, pid=None, tid=None, cid=None, gid=None,
dfrom=None, dto=None, dfromref=None, dtoref=None,
inc_merge_commit=None, inc_repos=None, metadata=None,
exc_groups=None, limit=None, inc_groups=None):
if not dfrom or not dto:
abort(404,
detail="Must specify dfrom and dto dates for the new "
"contributors")
if not dfromref or not dtoref:
abort(404,
detail="Must specify dfromref and dtoref dates for the "
"reference period to compute new contributors")
# Get contributors for the new period
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit, metadata,
exc_groups, inc_groups)
authors_new = self.gbycommits(
c, idents, query_kwargs, top=-1,
resolv_name=False, clean_email=False)
# Now get contributors for the old reference period
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfromref, dtoref, inc_repos, inc_merge_commit, metadata,
exc_groups, inc_groups)
authors_old = self.gbycommits(
c, idents, query_kwargs, top=-1,
resolv_name=False, clean_email=False)
# And compute the difference
cids_new = set([auth['cid'] for auth in authors_new]) - \
set([auth['cid'] for auth in authors_old])
authors_diff = [author for author in authors_new
if author['cid'] in cids_new]
if limit is None:
limit = 10
else:
limit = int(limit)
# If limit set to a negative value all results will be returned
if limit >= 0:
authors_diff = authors_diff[:limit]
self.resolv_name(c, authors_diff)
return authors_diff
def bycommits(self,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
metadata=None,
exc_groups=None,
inc_repos_detail=None,
inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, metadata,
exc_groups, inc_groups)
return self.gbycommits(c, projects_index, query_kwargs,
inc_repos_detail)
def tags(self, pid=None, tid=None, dfrom=None, dto=None, inc_repos=None):
t = Tags(index.Connector())
projects_index = Projects()
query_kwargs = utils.resolv_filters(projects_index, None, pid, tid,
None, None, dfrom, dto, inc_repos,
None, None, None, None)
p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']]
dfrom = query_kwargs['fromdate']
dto = query_kwargs['todate']
ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)]
# TODO: if tid is given we can include user defined releases
# for repo tagged with tid.
if not pid:
return ret
# now append user defined releases
ur = {}
project = projects_index.get_projects()[pid]
for repo in project['repos']:
if 'releases' in repo:
for release in repo['releases']:
ur[release['name']] = {
'name': release['name'],
'date': release['date'],
'repo': repo['name']
}
for rel in ur.values():
ret.append(rel)
return ret
def tags(self, pid=None, tid=None,
dfrom=None, dto=None, inc_repos=None):
t = Tags(index.Connector(index_suffix='tags'))
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, None, None,
dfrom, dto, inc_repos, None, None, None, None)
p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']]
dfrom = query_kwargs['fromdate']
dto = query_kwargs['todate']
ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)]
# TODO: if tid is given we can include user defined releases
# for repo tagged with tid.
if not pid:
return ret
# now append user defined releases
ur = {}
project = projects_index.get(pid, source=['refs', 'releases'])
for release in project.get('releases', []):
ur[release['name']] = release
for ref in project['refs']:
for release in ref.get('releases', []):
ur[release['name']] = release
for rel in ur.values():
ret.append(rel)
return ret
def authors(self, pid=None, tid=None, cid=None, gid=None,
dfrom=None, dto=None, inc_merge_commit=None,
inc_repos=None, metadata=None, exc_groups=None,
inc_groups=None):
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit,
metadata, exc_groups, inc_groups)
c = Commits(index.Connector())
if not c.get_commits_amount(**query_kwargs):
return []
ret = c.get_authors_histo(**query_kwargs)[1]
for bucket in ret:
_idents = idents.get_idents_by_emails(bucket['authors_email'])
bucket['value'] = len(_idents)
bucket['date'] = bucket['key_as_string']
del bucket['authors_email']
del bucket['doc_count']
del bucket['key_as_string']
del bucket['key']
return ret
def commits(self, pid=None, tid=None, cid=None, gid=None,
start=0, limit=10,
dfrom=None, dto=None, inc_merge_commit=None,
inc_repos=None, metadata=None, exc_groups=None,
inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit,
metadata, exc_groups, inc_groups)
query_kwargs.update(
{'start': start, 'limit': limit})
resp = c.get_commits(**query_kwargs)
for cmt in resp[2]:
# Get extra metadata keys
extra = set(cmt.keys()) - set(PROPERTIES.keys())
cmt['metadata'] = list(extra)
cmt['repos'] = [r for r in cmt['repos']
if not r.startswith('meta_ref: ')]
# Compute link to access commit diff based on the
# URL template provided in projects.yaml
cmt['gitwebs'] = [
projects_index.get_gitweb_link(r) %
{'sha': cmt['sha']} for r in cmt['repos']]
cmt['projects'] = utils.get_projects_from_references(
projects_index, cmt['repos'])
# Also remove the URI part
cmt['repos'] = [":".join(p.split(':')[-2:]) for
p in cmt['repos']]
# Request the ident index to fetch author/committer name/email
for elm in ('author', 'committer'):
ident = list(idents.get_idents_by_emails(
cmt['%s_email' % elm]).values())[0]
cmt['%s_email' % elm] = ident['default-email']
if ident['name']:
cmt['%s_name' % elm] = ident['name']
# Convert the TTL to something human readable
cmt['ttl'] = str((datetime.fromtimestamp(cmt['ttl']) -
datetime.fromtimestamp(0)))
cmt['author_gravatar'] = \
hashlib.md5(cmt['author_email'].encode(
errors='ignore')).hexdigest()
cmt['committer_gravatar'] = \
hashlib.md5(cmt['committer_email'].encode(
errors='ignore')).hexdigest()
if len(cmt['commit_msg']) > 80:
cmt['commit_msg'] = cmt['commit_msg'][0:76] + '...'
# Add cid and ccid
cmt['cid'] = utils.encrypt(xorkey, cmt['author_email'])
cmt['ccid'] = utils.encrypt(xorkey, cmt['committer_email'])
# Remove email details
del cmt['author_email']
del cmt['committer_email']
return resp
def metadata(self,
key=None,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
exc_groups=None,
inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, None, exc_groups,
inc_groups)
del query_kwargs['metadata']
if not key:
keys = c.get_metadata_keys(**query_kwargs)
return keys
else:
vals = c.get_metadata_key_values(key, **query_kwargs)
return vals
def commits(self,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
metadata=None,
exc_groups=None,
inc_groups=None):
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, metadata,
exc_groups, inc_groups)
c = Commits(index.Connector())
if not c.get_commits_amount(**query_kwargs):
return []
ret = c.get_commits_histo(**query_kwargs)
ret = [{
'date': d['key_as_string'],
'value': d['doc_count']
} for d in ret[1]]
return ret
def authors(self,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
metadata=None,
exc_groups=None,
inc_groups=None):
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, metadata,
exc_groups, inc_groups)
c = Commits(index.Connector())
if not c.get_commits_amount(**query_kwargs):
return []
ret = c.get_authors_histo(**query_kwargs)[1]
for bucket in ret:
_idents = idents.get_idents_by_emails(bucket['authors_email'])
bucket['value'] = len(_idents)
bucket['date'] = bucket['key_as_string']
del bucket['authors_email']
del bucket['doc_count']
del bucket['key_as_string']
del bucket['key']
return ret
def index(self, prefix=None, nameonly='false', withstats='false',
pid=None, dfrom=None, dto=None, inc_merge_commit=None):
ci = Commits(index.Connector())
contributors_index = Contributors()
groups = contributors_index.get_groups()
if withstats == 'true':
projects_index = Projects()
if nameonly == 'true':
ret = dict([(k, None) for k in groups.keys()])
if prefix:
ret = dict([(k, None) for k in ret.keys() if
k.lower().startswith(prefix)])
return ret
ret_groups = {}
for group, data in groups.items():
if prefix and not group.lower().startswith(prefix.lower()):
continue
rg = {'members': {},
'description': data.get('description', ''),
'domains': data.get('domains', [])}
emails = list(data['emails'].keys())
members = contributors_index.get_idents_by_emails(emails)
for id, member in members.items():
member['gravatar'] = hashlib.md5(
member['default-email'].encode(
errors='ignore')).hexdigest()
# TODO(fbo): bounces should be a list of bounce
# Let's deactivate that for now
# member['bounces'] = bounces
del member['emails']
if not member['name']:
# Try to find it among commits
suggested = ci.get_commits_author_name_by_emails(
[member['default-email']])
name = suggested.get(member['default-email'],
'Unnamed')
member['name'] = name
del member['default-email']
rg['members'][utils.encrypt(xorkey, id)] = member
if withstats == 'true':
# Fetch the number of projects and repos contributed to
query_kwargs = utils.resolv_filters(
projects_index, contributors_index, pid, None, None, group,
dfrom, dto, None, inc_merge_commit, None, None, None)
repos = [r for r in ci.get_repos(**query_kwargs)[1]
if not r.startswith('meta_ref: ')]
projects = utils.get_projects_from_references(
projects_index, repos)
rg['repos_amount'] = len(repos)
rg['projects_amount'] = len(projects)
ret_groups[group] = rg
return ret_groups
def bylchanged(self, pid=None, tid=None, cid=None, gid=None,
dfrom=None, dto=None, inc_merge_commit=None,
inc_repos=None, metadata=None, exc_groups=None,
limit=None, inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit, metadata,
exc_groups, inc_groups)
return self.gbylchanged(c, idents, query_kwargs, limit)
def metadata(self, key=None, pid=None, tid=None, cid=None, gid=None,
dfrom=None, dto=None, inc_merge_commit=None,
inc_repos=None, exc_groups=None, inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit, None, exc_groups,
inc_groups)
del query_kwargs['metadata']
if not key:
keys = c.get_metadata_keys(**query_kwargs)
return keys
else:
vals = c.get_metadata_key_values(key, **query_kwargs)
return vals
def commits(self, pid=None, tid=None, cid=None, gid=None,
dfrom=None, dto=None, inc_merge_commit=None,
inc_repos=None, metadata=None, exc_groups=None,
inc_groups=None):
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit,
metadata, exc_groups, inc_groups)
c = Commits(index.Connector())
if not c.get_commits_amount(**query_kwargs):
return []
ret = c.get_commits_histo(**query_kwargs)
ret = [{'date': d['key_as_string'],
'value': d['doc_count']} for d in ret[1]]
return ret
def index(self,
prefix=None,
nameonly='false',
withstats='false',
pid=None,
dfrom=None,
dto=None,
inc_merge_commit=None):
ci = Commits(index.Connector())
contributors_index = Contributors()
groups = contributors_index.get_groups()
if withstats == 'true':
projects_index = Projects()
if nameonly == 'true':
ret = dict([(k, None) for k in groups.keys()])
if prefix:
ret = dict([(k, None) for k in ret.keys()
if k.lower().startswith(prefix)])
return ret
ret_groups = {}
for group, data in groups.items():
if prefix and not group.lower().startswith(prefix.lower()):
continue
rg = {
'members': {},
'description': data.get('description', ''),
'domains': data.get('domains', [])
}
emails = list(data['emails'].keys())
members = contributors_index.get_idents_by_emails(emails)
for id, member in members.items():
member['gravatar'] = hashlib.md5(
member['default-email'].encode(
errors='ignore')).hexdigest()
# TODO(fbo): bounces should be a list of bounce
# Let's deactivate that for now
# member['bounces'] = bounces
del member['emails']
if not member['name']:
# Try to find it among commits
suggested = ci.get_commits_author_name_by_emails(
[member['default-email']])
name = suggested.get(member['default-email'], 'Unnamed')
member['name'] = name
del member['default-email']
rg['members'][utils.encrypt(xorkey, id)] = member
if withstats == 'true':
# Fetch the number of projects and repos contributed to
query_kwargs = utils.resolv_filters(
projects_index, contributors_index, pid, None, None, group,
dfrom, dto, None, inc_merge_commit, None, None, None)
repos = [
r for r in ci.get_repos(**query_kwargs)[1]
if not r.startswith('meta_ref: ')
]
projects = utils.get_projects_from_references(
projects_index, repos)
rg['repos_amount'] = len(repos)
rg['projects_amount'] = len(projects)
ret_groups[group] = rg
return ret_groups
