def tags(self, pid=None, tid=None, dfrom=None, dto=None, inc_repos=None): t = Tags(index.Connector()) projects_index = Projects() query_kwargs = utils.resolv_filters(projects_index, None, pid, tid, None, None, dfrom, dto, inc_repos, None, None, None, None) p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']] dfrom = query_kwargs['fromdate'] dto = query_kwargs['todate'] ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)] # TODO: if tid is given we can include user defined releases # for repo tagged with tid. if not pid: return ret # now append user defined releases ur = {} project = projects_index.get_projects()[pid] for repo in project['repos']: if 'releases' in repo: for release in repo['releases']: ur[release['name']] = { 'name': release['name'], 'date': release['date'], 'repo': repo['name'] } for rel in ur.values(): ret.append(rel) return ret
def commits(self, pid=None, tid=None, cid=None, gid=None, start=0, limit=10, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) query_kwargs.update( {'start': start, 'limit': limit}) resp = c.get_commits(**query_kwargs) for cmt in resp[2]: # Get extra metadata keys extra = set(cmt.keys()) - set(PROPERTIES.keys()) cmt['metadata'] = list(extra) cmt['repos'] = [r for r in cmt['repos'] if not r.startswith('meta_ref: ')] # Compute link to access commit diff based on the # URL template provided in projects.yaml cmt['gitwebs'] = [ projects_index.get_gitweb_link(r) % {'sha': cmt['sha']} for r in cmt['repos']] cmt['projects'] = utils.get_projects_from_references( projects_index, cmt['repos']) # Also remove the URI part cmt['repos'] = [":".join(p.split(':')[-2:]) for p in cmt['repos']] # Request the ident index to fetch author/committer name/email for elm in ('author', 'committer'): ident = list(idents.get_idents_by_emails( cmt['%s_email' % elm]).values())[0] cmt['%s_email' % elm] = ident['default-email'] if ident['name']: cmt['%s_name' % elm] = ident['name'] # Convert the TTL to something human readable cmt['ttl'] = str((datetime.fromtimestamp(cmt['ttl']) - datetime.fromtimestamp(0))) cmt['author_gravatar'] = \ hashlib.md5(cmt['author_email'].encode( errors='ignore')).hexdigest() cmt['committer_gravatar'] = \ hashlib.md5(cmt['committer_email'].encode( errors='ignore')).hexdigest() if len(cmt['commit_msg']) > 80: cmt['commit_msg'] = cmt['commit_msg'][0:76] + '...' # Add cid and ccid cmt['cid'] = utils.encrypt(xorkey, cmt['author_email']) cmt['ccid'] = utils.encrypt(xorkey, cmt['committer_email']) # Remove email details del cmt['author_email'] del cmt['committer_email'] return resp
def tags(self, pid=None, tid=None, dfrom=None, dto=None, inc_repos=None): t = Tags(index.Connector(index_suffix='tags')) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, None, None, dfrom, dto, inc_repos, None, None, None, None) p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']] dfrom = query_kwargs['fromdate'] dto = query_kwargs['todate'] ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)] # TODO: if tid is given we can include user defined releases # for repo tagged with tid. if not pid: return ret # now append user defined releases ur = {} project = projects_index.get(pid, source=['refs', 'releases']) for release in project.get('releases', []): ur[release['name']] = release for ref in project['refs']: for release in ref.get('releases', []): ur[release['name']] = release for rel in ur.values(): ret.append(rel) return ret
def get_projects(self): projects_index = Projects() projects = projects_index.get_projects() projects = OrderedDict( sorted(projects.items(), key=lambda t: t[0])) tags = projects_index.get_tags() return {'projects': projects, 'tags': tags.keys()}
def get_status(self): projects_index = Projects() projects = projects_index.get_projects(source=['name', 'refs']) num_projects = len(projects) num_repos = len(set([ ref['name'] for ref in itertools.chain( *[p['refs'] for p in list(projects.values())])])) return {'customtext': index_custom_html, 'projects': num_projects, 'repos': num_repos, 'users_endpoint': conf.get('users_endpoint', False), 'version': rx_version}
def get_repos(self, pid=None, tid=None): projects_index = Projects() if pid: repos = projects_index.get_projects().get(pid) elif tid: repos = projects_index.get_tags().get(tid) else: abort(404, detail="A tag ID or project ID must be passed as parameter") if repos is None: abort(404, detail='Project ID or Tag ID has not been found') return repos
def commits(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) c = Commits(index.Connector()) if not c.get_commits_amount(**query_kwargs): return [] ret = c.get_commits_histo(**query_kwargs) ret = [{ 'date': d['key_as_string'], 'value': d['doc_count'] } for d in ret[1]] return ret
def repos(self, pid=None, tid=None): projects_index = Projects() if not pid and not tid: abort(404, detail="A tag ID or project ID must be passed as parameter") if pid: project = projects_index.get(pid) else: if tid in projects_index.get_tags(): refs = projects_index.get_references_from_tags(tid) project = {'refs': refs} else: project = None if not project: abort(404, detail='Project ID or Tag ID has not been found') return project['refs']
def metadata(self, key=None, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, exc_groups=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, None, exc_groups, inc_groups) del query_kwargs['metadata'] if not key: keys = c.get_metadata_keys(**query_kwargs) return keys else: vals = c.get_metadata_key_values(key, **query_kwargs) return vals
def bycommits(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_repos_detail=None, inc_groups=None): c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) return self.gbycommits(c, projects_index, query_kwargs, inc_repos_detail)
def authors(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, inc_groups=None): projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) c = Commits(index.Connector()) if not c.get_commits_amount(**query_kwargs): return [] ret = c.get_authors_histo(**query_kwargs)[1] for bucket in ret: _idents = idents.get_idents_by_emails(bucket['authors_email']) bucket['value'] = len(_idents) bucket['date'] = bucket['key_as_string'] del bucket['authors_email'] del bucket['doc_count'] del bucket['key_as_string'] del bucket['key'] return ret
def get_status(self): projects_index = Projects() projects = projects_index.get_projects(source=['name', 'refs']) num_projects = len(projects) num_repos = len( set([ ref['name'] for ref in itertools.chain( *[p['refs'] for p in projects.values()]) ])) return { 'customtext': index_custom_html, 'projects': num_projects, 'repos': num_repos, 'users_endpoint': conf.get('users_endpoint', False), 'version': rx_version }
def index(self, prefix=None, nameonly='false', withstats='false'): ci = Commits(index.Connector()) contributors_index = Contributors() groups = contributors_index.get_groups() if nameonly == 'true': ret = dict([(k, None) for k in groups.keys()]) if prefix: ret = dict([(k, None) for k in ret.keys() if k.lower().startswith(prefix)]) return ret ret_groups = {} for group, data in groups.items(): if prefix and not group.lower().startswith(prefix.lower()): continue rg = {'members': {}, 'description': data['description'], 'domains': data.get('domains', [])} emails = data['emails'].keys() members = contributors_index.get_idents_by_emails(emails) for id, member in members.items(): member['gravatar'] = hashlib.md5( member['default-email']).hexdigest() # TODO(fbo): bounces should be a list of bounce # Let's deactivate that for now # member['bounces'] = bounces del member['emails'] if not member['name']: # Try to find it among commits suggested = ci.get_commits_author_name_by_emails( [member['default-email']]) name = suggested.get(member['default-email'], 'Unknown name') member['name'] = name del member['default-email'] rg['members'][utils.encrypt(xorkey, id)] = member if withstats == 'true': # TODO(fbo): This endpoint needs to handle some filters like # dates bounces to return more accurate stats # Fetch the number of projects and repos contributed to p_filter = {} query_kwargs = { 'mails': data['emails'], 'merge_commit': False, 'repos': p_filter, } projects = Projects() tops_ctl = tops.TopProjectsController() top_projects = tops_ctl.gbycommits( ci, projects, query_kwargs, False) top_repos = tops_ctl.gbycommits( ci, projects, query_kwargs, True) rg['projects_amount'] = len(top_projects) rg['repos_amount'] = len(top_repos) ret_groups[group] = rg return ret_groups
def diff(self, pid=None, tid=None, cid=None, gid=None, dfrom=None, dto=None, dfromref=None, dtoref=None, inc_merge_commit=None, inc_repos=None, metadata=None, exc_groups=None, limit=None, inc_groups=None): if not dfrom or not dto: abort(404, detail="Must specify dfrom and dto dates for the new " "contributors") if not dfromref or not dtoref: abort(404, detail="Must specify dfromref and dtoref dates for the " "reference period to compute new contributors") # Get contributors for the new period c = Commits(index.Connector()) projects_index = Projects() idents = Contributors() query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfrom, dto, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) authors_new = self.gbycommits( c, idents, query_kwargs, top=-1, resolv_name=False, clean_email=False) # Now get contributors for the old reference period query_kwargs = utils.resolv_filters( projects_index, idents, pid, tid, cid, gid, dfromref, dtoref, inc_repos, inc_merge_commit, metadata, exc_groups, inc_groups) authors_old = self.gbycommits( c, idents, query_kwargs, top=-1, resolv_name=False, clean_email=False) # And compute the difference cids_new = set([auth['cid'] for auth in authors_new]) - \ set([auth['cid'] for auth in authors_old]) authors_diff = [author for author in authors_new if author['cid'] in cids_new] if limit is None: limit = 10 else: limit = int(limit) # If limit set to a negative value all results will be returned if limit >= 0: authors_diff = authors_diff[:limit] self.resolv_name(c, authors_diff) return authors_diff
def contributor(self, cid=None): if not cid: abort(404, detail="No contributor specified") try: cid = utils.decrypt(xorkey, cid) except Exception: abort(404, detail="The cid is incorrectly formated") c = Commits(index.Connector()) idents = Contributors() projects = Projects() _, ident = idents.get_ident_by_id(cid) if not ident: # No ident has been declared for that contributor ident = idents.get_idents_by_emails(cid).values()[0] mails = ident['emails'] name = ident['name'] if not name: raw_names = c.get_commits_author_name_by_emails([cid]) if cid not in raw_names: # TODO: get_commits_author_name_by_emails must # support look by committer email too name = 'Unnamed' else: name = raw_names[cid] p_filter = {} query_kwargs = { 'mails': mails, 'merge_commit': False, 'repos': p_filter, } tops_ctl = tops.TopProjectsController() top_projects = tops_ctl.gbycommits(c, projects, query_kwargs, False) top_repos = tops_ctl.gbycommits(c, projects, query_kwargs, True) infos = {} infos['name'] = name infos['mails_amount'] = len(mails) infos['projects_amount'] = len(top_projects) infos['repos_amount'] = len(top_repos) infos['gravatar'] = hashlib.md5(ident['default-email']).hexdigest() return infos
def projects(self, pid=None): projects_index = Projects() if pid: project = projects_index.get(pid) if not project: abort(404, detail="Project ID has not been found") return {pid: projects_index.get(pid)} else: projects = projects_index.get_projects( source=['name', 'description', 'logo', 'refs']) _projects = OrderedDict( sorted(projects.items(), key=lambda t: t[0])) return {'projects': _projects, 'tags': projects_index.get_tags()}
def set_projects_definition(con, projects_file=None): default_projects_file = """ project-templates: default: uri: https://github.com/nakata/%(name)s.git branches: - master projects: test: repos: monkey: template: default """ projects_file = projects_file or default_projects_file db = tempfile.mkdtemp() open(os.path.join(db, 'projects.yaml'), 'w+').write(projects_file) index.conf['db_path'] = db index.conf['db_default_file'] = None Projects(db_path=db, con=con, dump_yaml_in_index=True) return db
def index(self, prefix=None, nameonly='false', withstats='false', pid=None, dfrom=None, dto=None, inc_merge_commit=None): ci = Commits(index.Connector()) contributors_index = Contributors() groups = contributors_index.get_groups() if withstats == 'true': projects_index = Projects() if nameonly == 'true': ret = dict([(k, None) for k in groups.keys()]) if prefix: ret = dict([(k, None) for k in ret.keys() if k.lower().startswith(prefix)]) return ret ret_groups = {} for group, data in groups.items(): if prefix and not group.lower().startswith(prefix.lower()): continue rg = { 'members': {}, 'description': data.get('description', ''), 'domains': data.get('domains', []) } emails = list(data['emails'].keys()) members = contributors_index.get_idents_by_emails(emails) for id, member in members.items(): member['gravatar'] = hashlib.md5( member['default-email'].encode( errors='ignore')).hexdigest() # TODO(fbo): bounces should be a list of bounce # Let's deactivate that for now # member['bounces'] = bounces del member['emails'] if not member['name']: # Try to find it among commits suggested = ci.get_commits_author_name_by_emails( [member['default-email']]) name = suggested.get(member['default-email'], 'Unnamed') member['name'] = name del member['default-email'] rg['members'][utils.encrypt(xorkey, id)] = member if withstats == 'true': # Fetch the number of projects and repos contributed to query_kwargs = utils.resolv_filters( projects_index, contributors_index, pid, None, None, group, dfrom, dto, None, inc_merge_commit, None, None, None) repos = [ r for r in ci.get_repos(**query_kwargs)[1] if not r.startswith('meta_ref: ') ] projects = utils.get_projects_from_references( projects_index, repos) rg['repos_amount'] = len(repos) rg['projects_amount'] = len(projects) ret_groups[group] = rg return ret_groups