def diff(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, dfromref=None, dtoref=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, limit=None, inc_groups=None): if not dfrom or not dto: abort(404, detail="Must specify dfrom and dto dates for the new " "contributors") if not dfromref or not dtoref: abort(404, detail="Must specify dfromref and dtoref dates for the " "reference period to compute new contributors") # Get contributors for the new period c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) authors_new = self.gbycommits( c, idents, query_kwargs, top=-1, resolv_name=False, clean_email=False) # Now get contributors for the old reference period query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfromref, dtoref, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) authors_old = self.gbycommits( c, idents, query_kwargs, top=-1, resolv_name=False, clean_email=False) # And compute the difference cids_new = set([auth['cid'] for auth in authors_new]) - \ set([auth['cid'] for auth in authors_old]) authors_diff = [author for author in authors_new if author['cid'] in cids_new] if limit is None: limit = 10 else: limit = int(limit) # If limit set to a negative value all results will be returned if limit >= 0: authors_diff = authors_diff[:limit] self.resolv_name(c, authors_diff) return authors_diff
def diff(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, dfromref=None, dtoref=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, limit=None, inc_groups=None): if not dfrom or not dto: abort(404, detail="Must specify dfrom and dto dates for the new " "contributors") if not dfromref or not dtoref: abort(404, detail="Must specify dfromref and dtoref dates for the " "reference period to compute new contributors") # Get contributors for the new period c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) authors_new = self.gbycommits( c, idents, query_kwargs, top=-1, resolv_name=False, clean_email=False) # Now get contributors for the old reference period query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfromref, dtoref, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) authors_old = self.gbycommits( c, idents, query_kwargs, top=-1, resolv_name=False, clean_email=False) # And compute the difference cids_new = set([auth['cid'] for auth in authors_new]) - \ set([auth['cid'] for auth in authors_old]) authors_diff = [author for author in authors_new if author['cid'] in cids_new] if limit is None: limit = 10 else: limit = int(limit) # If limit set to a negative value all results will be returned if limit >= 0: authors_diff = authors_diff[:limit] self.resolv_name(c, authors_diff) return authors_diff
def bycommits(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_repos_detail=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) return self.gbycommits(c, projects_index, query_kwargs, inc_repos_detail)
def tags(self, pid=None, tid=None, dfrom=None, dto=None, inc_repos=None): t = Tags(index.Connector()) projects_index = Projects() query_kwargs = utils.resolv_filters(projects_index, None, pid, tid, None, None, dfrom, dto, inc_repos, None, None, None, None) p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']] dfrom = query_kwargs['fromdate'] dto = query_kwargs['todate'] ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)] # TODO: if tid is given we can include user defined releases # for repo tagged with tid. if not pid: return ret # now append user defined releases ur = {} project = projects_index.get_projects()[pid] for repo in project['repos']: if 'releases' in repo: for release in repo['releases']: ur[release['name']] = { 'name': release['name'], 'date': release['date'], 'repo': repo['name'] } for rel in ur.values(): ret.append(rel) return ret
def tags(self, pid=None, tid=None, dfrom=None, dto=None, inc_repos=None): t = Tags(index.Connector(index_suffix='tags')) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, None, None, dfrom, dto, inc_repos, None, None, None, None) p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']] dfrom = query_kwargs['fromdate'] dto = query_kwargs['todate'] ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)] # TODO: if tid is given we can include user defined releases # for repo tagged with tid. if not pid: return ret # now append user defined releases ur = {} project = projects_index.get(pid, source=['refs', 'releases']) for release in project.get('releases', []): ur[release['name']] = release for ref in project['refs']: for release in ref.get('releases', []): ur[release['name']] = release for rel in ur.values(): ret.append(rel) return ret
def authors(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) c = Commits(index.Connector()) if not c.get_commits_amount(**query_kwargs): return [] ret = c.get_authors_histo(**query_kwargs)[1] for bucket in ret: _idents = idents.get_idents_by_emails(bucket['authors_email']) bucket['value'] = len(_idents) bucket['date'] = bucket['key_as_string'] del bucket['authors_email'] del bucket['doc_count'] del bucket['key_as_string'] del bucket['key'] return ret
def commits(self, pid=None, tid=None, cid=None, gid=None, start=0, limit=10, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) query_kwargs.update( {'start': start, 'limit': limit}) resp = c.get_commits(**query_kwargs) for cmt in resp[2]: # Get extra metadata keys extra = set(cmt.keys()) - set(PROPERTIES.keys()) cmt['metadata'] = list(extra) cmt['repos'] = [r for r in cmt['repos'] if not r.startswith('meta_ref: ')] # Compute link to access commit diff based on the # URL template provided in projects.yaml cmt['gitwebs'] = [ projects_index.get_gitweb_link(r) % {'sha': cmt['sha']} for r in cmt['repos']] cmt['projects'] = utils.get_projects_from_references( projects_index, cmt['repos']) # Also remove the URI part cmt['repos'] = [":".join(p.split(':')[-2:]) for p in cmt['repos']] # Request the ident index to fetch author/committer name/email for elm in ('author', 'committer'): ident = list(idents.get_idents_by_emails( cmt['%s_email' % elm]).values())[0] cmt['%s_email' % elm] = ident['default-email'] if ident['name']: cmt['%s_name' % elm] = ident['name'] # Convert the TTL to something human readable cmt['ttl'] = str((datetime.fromtimestamp(cmt['ttl']) - datetime.fromtimestamp(0))) cmt['author_gravatar'] = \ hashlib.md5(cmt['author_email'].encode( errors='ignore')).hexdigest() cmt['committer_gravatar'] = \ hashlib.md5(cmt['committer_email'].encode( errors='ignore')).hexdigest() if len(cmt['commit_msg']) > 80: cmt['commit_msg'] = cmt['commit_msg'][0:76] + '...' # Add cid and ccid cmt['cid'] = utils.encrypt(xorkey, cmt['author_email']) cmt['ccid'] = utils.encrypt(xorkey, cmt['committer_email']) # Remove email details del cmt['author_email'] del cmt['committer_email'] return resp
def metadata(self, key=None, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, exc_groups=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, None, exc_groups, inc_groups) del query_kwargs['metadata'] if not key: keys = c.get_metadata_keys(**query_kwargs) return keys else: vals = c.get_metadata_key_values(key, **query_kwargs) return vals
def commits(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) c = Commits(index.Connector()) if not c.get_commits_amount(**query_kwargs): return [] ret = c.get_commits_histo(**query_kwargs) ret = [{ 'date': d['key_as_string'], 'value': d['doc_count'] } for d in ret[1]] return ret
def authors(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) c = Commits(index.Connector()) if not c.get_commits_amount(**query_kwargs): return [] ret = c.get_authors_histo(**query_kwargs)[1] for bucket in ret: _idents = idents.get_idents_by_emails(bucket['authors_email']) bucket['value'] = len(_idents) bucket['date'] = bucket['key_as_string'] del bucket['authors_email'] del bucket['doc_count'] del bucket['key_as_string'] del bucket['key'] return ret
def index(self, prefix=None, nameonly='false', withstats='false', pid=None, dfrom=None, dto=None, inc_merge_commit=None): ci = Commits(index.Connector()) contributors_index = Contributors() groups = contributors_index.get_groups() if withstats == 'true': projects_index = Projects() if nameonly == 'true': ret = dict([(k, None) for k in groups.keys()]) if prefix: ret = dict([(k, None) for k in ret.keys() if k.lower().startswith(prefix)]) return ret ret_groups = {} for group, data in groups.items(): if prefix and not group.lower().startswith(prefix.lower()): continue rg = {'members': {}, 'description': data.get('description', ''), 'domains': data.get('domains', [])} emails = list(data['emails'].keys()) members = contributors_index.get_idents_by_emails(emails) for id, member in members.items(): member['gravatar'] = hashlib.md5( member['default-email'].encode( errors='ignore')).hexdigest() # TODO(fbo): bounces should be a list of bounce # Let's deactivate that for now # member['bounces'] = bounces del member['emails'] if not member['name']: # Try to find it among commits suggested = ci.get_commits_author_name_by_emails( [member['default-email']]) name = suggested.get(member['default-email'], 'Unnamed') member['name'] = name del member['default-email'] rg['members'][utils.encrypt(xorkey, id)] = member if withstats == 'true': # Fetch the number of projects and repos contributed to query_kwargs = utils.resolv_filters( projects_index, contributors_index, pid, None, None, group, dfrom, dto, None, inc_merge_commit, None, None, None) repos = [r for r in ci.get_repos(**query_kwargs)[1] if not r.startswith('meta_ref: ')] projects = utils.get_projects_from_references( projects_index, repos) rg['repos_amount'] = len(repos) rg['projects_amount'] = len(projects) ret_groups[group] = rg return ret_groups
def bylchanged(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, limit=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) return self.gbylchanged(c, idents, query_kwargs, limit)
def metadata(self, key=None, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, exc_groups=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, None, exc_groups, inc_groups) del query_kwargs['metadata'] if not key: keys = c.get_metadata_keys(**query_kwargs) return keys else: vals = c.get_metadata_key_values(key, **query_kwargs) return vals
def commits(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) c = Commits(index.Connector()) if not c.get_commits_amount(**query_kwargs): return [] ret = c.get_commits_histo(**query_kwargs) ret = [{'date': d['key_as_string'], 'value': d['doc_count']} for d in ret[1]] return ret
def index(self, prefix=None, nameonly='false', withstats='false', pid=None, dfrom=None, dto=None, inc_merge_commit=None): ci = Commits(index.Connector()) contributors_index = Contributors() groups = contributors_index.get_groups() if withstats == 'true': projects_index = Projects() if nameonly == 'true': ret = dict([(k, None) for k in groups.keys()]) if prefix: ret = dict([(k, None) for k in ret.keys() if k.lower().startswith(prefix)]) return ret ret_groups = {} for group, data in groups.items(): if prefix and not group.lower().startswith(prefix.lower()): continue rg = { 'members': {}, 'description': data.get('description', ''), 'domains': data.get('domains', []) } emails = list(data['emails'].keys()) members = contributors_index.get_idents_by_emails(emails) for id, member in members.items(): member['gravatar'] = hashlib.md5( member['default-email'].encode( errors='ignore')).hexdigest() # TODO(fbo): bounces should be a list of bounce # Let's deactivate that for now # member['bounces'] = bounces del member['emails'] if not member['name']: # Try to find it among commits suggested = ci.get_commits_author_name_by_emails( [member['default-email']]) name = suggested.get(member['default-email'], 'Unnamed') member['name'] = name del member['default-email'] rg['members'][utils.encrypt(xorkey, id)] = member if withstats == 'true': # Fetch the number of projects and repos contributed to query_kwargs = utils.resolv_filters( projects_index, contributors_index, pid, None, None, group, dfrom, dto, None, inc_merge_commit, None, None, None) repos = [ r for r in ci.get_repos(**query_kwargs)[1] if not r.startswith('meta_ref: ') ] projects = utils.get_projects_from_references( projects_index, repos) rg['repos_amount'] = len(repos) rg['projects_amount'] = len(projects) ret_groups[group] = rg return ret_groups