def setUpClass(cls):
cls.con = index.Connector(index='repoxplorertest')
cls.conp = index.Connector(
index='repoxplorertest', index_suffix='projects')
c = Commits(cls.con)
c.add_commits(COMMITS)
cls.db = set_projects_definition(cls.conp)
t = Tags(index.Connector(
index='repoxplorertest', index_suffix='tags'))
tags = [
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b184',
'date': 1410456005,
'repo':
'https://github.com/nakata/monkey.git:monkey',
'name': 'tag1',
},
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b1845',
'date': 1410456005,
'repo':
'https://github.com/nakata/monkey.git:monkey',
'name': 'tag2',
}]
t.add_tags(tags)
def setUpClass(cls):
cls.con = index.Connector(index='repoxplorertest')
cls.conp = index.Connector(index='repoxplorertest',
index_suffix='projects')
c = Commits(cls.con)
c.add_commits(COMMITS)
cls.db = set_projects_definition(cls.conp)
def setUpClass(cls):
cls.con = index.Connector(index='repoxplorertest')
cls.conp = index.Connector(index='repoxplorertest',
index_suffix='projects')
c = Commits(cls.con)
c.add_commits(COMMITS)
projects_file = """
project-templates:
default:
uri: https://github.com/nakata/%(name)s.git
branches:
- master
projects:
test:
repos:
monkey:
template: default
test2:
repos:
monkey:
template: default
tags:
- python
"""
cls.db = set_projects_definition(cls.conp, projects_file)
def setUp(self):
FunctionalTest.setUp(self)
self.con = index.Connector(index='repoxplorertest')
self.conp = index.Connector(index='repoxplorertest',
index_suffix='projects')
c = Commits(self.con)
c.add_commits(COMMITS)
self.db = set_projects_definition(self.conp)
def __init__(self, projects, con=None):
if not con:
self.con = index.Connector()
else:
self.con = con
self.projects = projects
self.c = Commits(self.con)
self.t = Tags(
index.Connector(index=self.con.index, index_suffix='tags'))
self.seen_refs_path = os.path.join(conf.db_path, SEEN_REFS_CACHED)
self.current_base_ids = set()
def __init__(self, name, uri, parsers=None, con=None, meta_ref=None):
if not con:
self.con = index.Connector()
else:
self.con = con
self.c = Commits(self.con)
self.t = Tags(
index.Connector(index=self.con.index, index_suffix='tags'))
if not os.path.isdir(conf.git_store):
os.makedirs(conf.git_store)
self.name = name
self.uri = uri
self.base_id = '%s:%s' % (self.uri, self.name)
self.seen_refs_path = os.path.join(conf.db_cache_path,
SEEN_REFS_CACHED)
if meta_ref:
self.meta_ref = 'meta_ref: %s' % meta_ref
else:
self.meta_ref = None
if not parsers:
self.parsers = []
else:
self.parsers = parsers
self.parsers_compiled = False
self.local = os.path.join(conf.git_store, self.name,
self.uri.replace('/', '_'))
if not os.path.isdir(self.local):
os.makedirs(self.local)
self.credentials_helper_path = getattr(conf,
'git_credential_helper_path',
None)
if not (self.credentials_helper_path
and self.credentials_helper_path.startswith('/')
and os.path.isfile(self.credentials_helper_path)):
if self.credentials_helper_path:
logger.warning(
'Configured git_credential_helper %s not found' %
(self.credentials_helper_path))
self.credentials_helper_path = None
# Look at the default installation pathes
if not self.credentials_helper_path:
self.credentials_helper_path = os.path.join(
sys.prefix, 'bin', 'repoxplorer-git-credentials-helper')
if not os.path.isfile(self.credentials_helper_path):
self.credentials_helper_path = shutil.which(
'repoxplorer-git-credentials-helper')
if not self.credentials_helper_path:
logger.warning(
'Default repoxplorer-git-credential-helper command '
'not found')
def authors(self,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
metadata=None,
exc_groups=None,
inc_groups=None):
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, metadata,
exc_groups, inc_groups)
c = Commits(index.Connector())
if not c.get_commits_amount(**query_kwargs):
return []
ret = c.get_authors_histo(**query_kwargs)[1]
for bucket in ret:
_idents = idents.get_idents_by_emails(bucket['authors_email'])
bucket['value'] = len(_idents)
bucket['date'] = bucket['key_as_string']
del bucket['authors_email']
del bucket['doc_count']
del bucket['key_as_string']
del bucket['key']
return ret
def __init__(self, db_path=None, db_default_file=None, vonly=False):
YAMLDefinition.__init__(self, db_path, db_default_file)
self.enriched_groups = False
self.enriched_idents = False
if not vonly:
self._users = users.Users(
index.Connector(index_suffix='users'))
def search_authors(self, query=""):
ret_limit = 100
c = Commits(index.Connector())
ret = c.es.search(index=c.index,
q=query,
df="author_name",
size=10000,
default_operator="AND",
_source_includes=["author_name", "author_email"])
ret = ret['hits']['hits']
if not len(ret):
return {}
idents = Contributors()
authors = dict([(d['_source']['author_email'],
d['_source']['author_name']) for d in ret])
result = {}
_idents = idents.get_idents_by_emails(list(authors.keys())[:ret_limit])
for iid, ident in _idents.items():
email = ident['default-email']
name = ident['name'] or authors[email]
result[utils.encrypt(xorkey, iid)] = {
'name': name,
'gravatar':
hashlib.md5(email.encode(errors='ignore')).hexdigest()
}
result = OrderedDict(
sorted(list(result.items()), key=lambda t: t[1]['name']))
return result
def commits(self, pid=None, tid=None, cid=None, gid=None,
start=0, limit=10,
dfrom=None, dto=None, inc_merge_commit=None,
inc_repos=None, metadata=None, exc_groups=None,
inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit,
metadata, exc_groups, inc_groups)
query_kwargs.update(
{'start': start, 'limit': limit})
resp = c.get_commits(**query_kwargs)
for cmt in resp[2]:
# Get extra metadata keys
extra = set(cmt.keys()) - set(PROPERTIES.keys())
cmt['metadata'] = list(extra)
cmt['repos'] = [r for r in cmt['repos']
if not r.startswith('meta_ref: ')]
# Compute link to access commit diff based on the
# URL template provided in projects.yaml
cmt['gitwebs'] = [
projects_index.get_gitweb_link(r) %
{'sha': cmt['sha']} for r in cmt['repos']]
cmt['projects'] = utils.get_projects_from_references(
projects_index, cmt['repos'])
# Also remove the URI part
cmt['repos'] = [":".join(p.split(':')[-2:]) for
p in cmt['repos']]
# Request the ident index to fetch author/committer name/email
for elm in ('author', 'committer'):
ident = list(idents.get_idents_by_emails(
cmt['%s_email' % elm]).values())[0]
cmt['%s_email' % elm] = ident['default-email']
if ident['name']:
cmt['%s_name' % elm] = ident['name']
# Convert the TTL to something human readable
cmt['ttl'] = str((datetime.fromtimestamp(cmt['ttl']) -
datetime.fromtimestamp(0)))
cmt['author_gravatar'] = \
hashlib.md5(cmt['author_email'].encode(
errors='ignore')).hexdigest()
cmt['committer_gravatar'] = \
hashlib.md5(cmt['committer_email'].encode(
errors='ignore')).hexdigest()
if len(cmt['commit_msg']) > 80:
cmt['commit_msg'] = cmt['commit_msg'][0:76] + '...'
# Add cid and ccid
cmt['cid'] = utils.encrypt(xorkey, cmt['author_email'])
cmt['ccid'] = utils.encrypt(xorkey, cmt['committer_email'])
# Remove email details
del cmt['author_email']
del cmt['committer_email']
return resp
def tags(self, pid=None, tid=None,
dfrom=None, dto=None, inc_repos=None):
t = Tags(index.Connector(index_suffix='tags'))
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, None, None,
dfrom, dto, inc_repos, None, None, None, None)
p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']]
dfrom = query_kwargs['fromdate']
dto = query_kwargs['todate']
ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)]
# TODO: if tid is given we can include user defined releases
# for repo tagged with tid.
if not pid:
return ret
# now append user defined releases
ur = {}
project = projects_index.get(pid, source=['refs', 'releases'])
for release in project.get('releases', []):
ur[release['name']] = release
for ref in project['refs']:
for release in ref.get('releases', []):
ur[release['name']] = release
for rel in ur.values():
ret.append(rel)
return ret
def contributor(self, cid=None):
if not cid:
abort(404, detail="No contributor specified")
c = Commits(index.Connector())
idents = Contributors()
try:
cid = utils.decrypt(xorkey, cid)
except Exception:
abort(404, detail="The cid is incorrectly formated")
_, ident = idents.get_ident_by_id(cid)
if not ident:
# No ident has been declared for that contributor
ident = list(idents.get_idents_by_emails(cid).values())[0]
mails = ident['emails']
name = ident['name']
if not name:
raw_names = c.get_commits_author_name_by_emails([cid])
if cid not in raw_names:
# TODO: get_commits_author_name_by_emails must
# support look by committer email too
name = 'Unnamed'
else:
name = raw_names[cid]
infos = {}
infos['name'] = name
infos['mails_amount'] = len(mails)
infos['gravatar'] = hashlib.md5(
ident['default-email'].encode(errors='ignore')).hexdigest()
return infos
def metadata(self,
key=None,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
exc_groups=None,
inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, None, exc_groups,
inc_groups)
del query_kwargs['metadata']
if not key:
keys = c.get_metadata_keys(**query_kwargs)
return keys
else:
vals = c.get_metadata_key_values(key, **query_kwargs)
return vals
def commits(self,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
metadata=None,
exc_groups=None,
inc_groups=None):
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, metadata,
exc_groups, inc_groups)
c = Commits(index.Connector())
if not c.get_commits_amount(**query_kwargs):
return []
ret = c.get_commits_histo(**query_kwargs)
ret = [{
'date': d['key_as_string'],
'value': d['doc_count']
} for d in ret[1]]
return ret
def tags(self, pid=None, tid=None, dfrom=None, dto=None, inc_repos=None):
t = Tags(index.Connector())
projects_index = Projects()
query_kwargs = utils.resolv_filters(projects_index, None, pid, tid,
None, None, dfrom, dto, inc_repos,
None, None, None, None)
p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']]
dfrom = query_kwargs['fromdate']
dto = query_kwargs['todate']
ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)]
# TODO: if tid is given we can include user defined releases
# for repo tagged with tid.
if not pid:
return ret
# now append user defined releases
ur = {}
project = projects_index.get_projects()[pid]
for repo in project['repos']:
if 'releases' in repo:
for release in repo['releases']:
ur[release['name']] = {
'name': release['name'],
'date': release['date'],
'repo': repo['name']
}
for rel in ur.values():
ret.append(rel)
return ret
def setUpClass(cls):
indexer.conf['git_store'] = tempfile.mkdtemp()
indexer.conf['db_path'] = tempfile.mkdtemp()
indexer.conf['elasticsearch_index'] = 'repoxplorertest'
indexer.get_commits_desc = lambda path, shas: []
cls.con = index.Connector()
cls.cmts = commits.Commits(cls.con)
def bycommits(self,
pid=None,
tid=None,
cid=None,
gid=None,
dfrom=None,
dto=None,
inc_merge_commit=None,
inc_repos=None,
metadata=None,
exc_groups=None,
inc_repos_detail=None,
inc_groups=None):
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(projects_index, idents, pid, tid,
cid, gid, dfrom, dto, inc_repos,
inc_merge_commit, metadata,
exc_groups, inc_groups)
return self.gbycommits(c, projects_index, query_kwargs,
inc_repos_detail)
def __init__(self,
db_path=None,
db_default_file=None,
db_cache_path=None,
con=None,
dump_yaml_in_index=None,
vonly=False):
self.db_path = db_path or conf.get('db_path')
self.db_default_file = db_default_file or conf.get('db_default_file')
self.db_cache_path = db_cache_path or conf.get('db_cache_path')
if vonly:
return
# Use a separate index for projects (same as for users) as mapping
# name/type collision will occured as commits have dynamic mapping
self.eprojects = EProjects(
connector=(con or index.Connector(index_suffix='projects')))
self.el_version = self.eprojects.es.info().get('version',
{}).get('number', '')
if dump_yaml_in_index:
YAMLDefinition.__init__(self, self.db_path, self.db_default_file,
self.db_cache_path)
issues = self.validate()
if issues:
raise RuntimeError(issues)
self._enrich_projects()
projects, rid2projects = self._flatten_projects()
self.eprojects.load(projects, rid2projects)
def setUpClass(cls):
cls.con = index.Connector(index='repoxplorertest',
index_suffix='users')
cls.c = Users(cls.con)
cls.user = {
'uid':
'123',
'name':
'saboten',
'default-email':
'saboten@domain1',
'emails': [{
'email': 'saboten@domain1'
}, {
'email':
'saboten@domain2',
'groups': [{
'group': 'ugroup1',
'begin-date': '2016-01-01',
'end-date': '2016-01-09'
}],
}],
'last_cnx':
1410456005
}
cls.user2 = {
'uid': '124',
'name': 'ampanman',
'default-email': 'ampanman@domain1',
'emails': [{
'email': 'ampanman@domain1'
}],
'last_cnx': 1410456006
}
def __init__(self, name, uri, parsers=None, con=None, config=None):
if config:
configuration.set_config(config)
if not con:
self.con = index.Connector()
else:
self.con = con
self.c = Commits(self.con)
self.t = Tags(self.con)
if not os.path.isdir(conf.git_store):
os.makedirs(conf.git_store)
self.name = name
self.uri = uri
self.base_id = '%s:%s' % (self.uri, self.name)
self.seen_refs_path = os.path.join(conf.db_path, SEEN_REFS_CACHED)
if not parsers:
self.parsers = []
else:
self.parsers = parsers
self.parsers_compiled = False
self.local = os.path.join(conf.git_store, self.name,
self.uri.replace('/', '_'))
if not os.path.isdir(self.local):
os.makedirs(self.local)
self.credentials_helper_path = os.path.join(
sys.prefix, 'bin', 'repoxplorer-git-credentials-helper')
def process_commits(options):
path, ref_id, shas = options
c = Commits(index.Connector())
logger.info("Worker %s started to extract and index %s commits" %
(mp.current_process(), len(shas)))
buf = get_commits_desc(path, shas)
c.add_commits(process_commits_desc_output(buf, ref_id))
def provision_user(self, request):
raw_token = self._get_raw_token(request)
# verified before so it's totally okay
claims = jwt.decode(raw_token, verify=False)
# TODO assuming the presence of claims, but a specific scope might be
# needed.
# These are expected to be standard though, see
# https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
email = claims['email']
uid = claims['preferred_username']
name = claims['name']
_users = users.Users(index.Connector(index_suffix='users'))
u = _users.get(uid)
infos = {
'uid': uid,
'name': name,
'default-email': email,
'emails': [{
'email': email
}]
}
if u:
_users.update(infos)
else:
_users.create(infos)
def delete(self, uid):
self._authorize(uid)
_users = users.Users(index.Connector(index_suffix='users'))
u = _users.get(uid)
if not u:
abort(404)
_users.delete(uid)
def index(self, prefix=None, nameonly='false', withstats='false'):
ci = Commits(index.Connector())
contributors_index = Contributors()
groups = contributors_index.get_groups()
if nameonly == 'true':
ret = dict([(k, None) for k in groups.keys()])
if prefix:
ret = dict([(k, None) for k in ret.keys() if
k.lower().startswith(prefix)])
return ret
ret_groups = {}
for group, data in groups.items():
if prefix and not group.lower().startswith(prefix.lower()):
continue
rg = {'members': {},
'description': data['description'],
'domains': data.get('domains', [])}
emails = data['emails'].keys()
members = contributors_index.get_idents_by_emails(emails)
for id, member in members.items():
member['gravatar'] = hashlib.md5(
member['default-email']).hexdigest()
# TODO(fbo): bounces should be a list of bounce
# Let's deactivate that for now
# member['bounces'] = bounces
del member['emails']
if not member['name']:
# Try to find it among commits
suggested = ci.get_commits_author_name_by_emails(
[member['default-email']])
name = suggested.get(member['default-email'],
'Unknown name')
member['name'] = name
del member['default-email']
rg['members'][utils.encrypt(xorkey, id)] = member
if withstats == 'true':
# TODO(fbo): This endpoint needs to handle some filters like
# dates bounces to return more accurate stats
# Fetch the number of projects and repos contributed to
p_filter = {}
query_kwargs = {
'mails': data['emails'],
'merge_commit': False,
'repos': p_filter,
}
projects = Projects()
tops_ctl = tops.TopProjectsController()
top_projects = tops_ctl.gbycommits(
ci, projects, query_kwargs, False)
top_repos = tops_ctl.gbycommits(
ci, projects, query_kwargs, True)
rg['projects_amount'] = len(top_projects)
rg['repos_amount'] = len(top_repos)
ret_groups[group] = rg
return ret_groups
def get(self, uid):
self._authorize(uid)
_users = users.Users(index.Connector(index_suffix='users'))
u = _users.get(uid)
if not u:
abort(404)
u['cid'] = utils.encrypt(xorkey, u['default-email'])
return u
def __init__(self, projects, con=None, config=None):
if config:
configuration.set_config(config)
if not con:
self.con = index.Connector()
else:
self.con = con
self.projects = projects
self.c = Commits(self.con)
self.seen_refs_path = os.path.join(conf.db_path, SEEN_REFS_CACHED)
def setUpClass(cls):
cls.con = index.Connector(index='repoxplorertest')
cls.c = Commits(cls.con)
cls.t = CommitsAmountTrend(cls.con)
cls.commits = [
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b184',
'author_date': 1410456005,
'committer_date': 1410456010,
'ttl': 5,
'author_name': 'Nakata Daisuke',
'committer_name': 'Nakata Daisuke',
'author_email': '*****@*****.**',
'committer_email': '*****@*****.**',
'repos': [
'https://github.com/nakata/monkey.git:monkey:master',
],
'line_modifieds': 10,
'merge_commit': False,
'commit_msg': 'Add init method',
},
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b185',
'author_date': 1410457005,
'committer_date': 1410457005,
'ttl': 0,
'author_name': 'Keiko Amura',
'committer_name': 'Keiko Amura',
'author_email': '*****@*****.**',
'committer_email': '*****@*****.**',
'repos': [
'https://github.com/nakata/monkey.git:monkey:master',
],
'line_modifieds': 100,
'merge_commit': False,
'commit_msg': 'Merge "Fix sanity unittest"',
},
{
'sha': '3597334f2cb10772950c97ddf2f6cc17b186',
'author_date': 1410458005,
'committer_date': 1410458005,
'ttl': 0,
'author_name': 'Jean Bon',
'committer_name': 'Jean Bon',
'author_email': '*****@*****.**',
'committer_email': '*****@*****.**',
'repos': [
'https://github.com/nakata/monkey.git:monkey:master',
],
'line_modifieds': 200,
'merge_commit': False,
'commit_msg': 'Add request customer feature 19',
},
]
cls.c.add_commits(cls.commits)
def diff(self, pid=None, tid=None, cid=None, gid=None,
dfrom=None, dto=None, dfromref=None, dtoref=None,
inc_merge_commit=None, inc_repos=None, metadata=None,
exc_groups=None, limit=None, inc_groups=None):
if not dfrom or not dto:
abort(404,
detail="Must specify dfrom and dto dates for the new "
"contributors")
if not dfromref or not dtoref:
abort(404,
detail="Must specify dfromref and dtoref dates for the "
"reference period to compute new contributors")
# Get contributors for the new period
c = Commits(index.Connector())
projects_index = Projects()
idents = Contributors()
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfrom, dto, inc_repos, inc_merge_commit, metadata,
exc_groups, inc_groups)
authors_new = self.gbycommits(
c, idents, query_kwargs, top=-1,
resolv_name=False, clean_email=False)
# Now get contributors for the old reference period
query_kwargs = utils.resolv_filters(
projects_index, idents, pid, tid, cid, gid,
dfromref, dtoref, inc_repos, inc_merge_commit, metadata,
exc_groups, inc_groups)
authors_old = self.gbycommits(
c, idents, query_kwargs, top=-1,
resolv_name=False, clean_email=False)
# And compute the difference
cids_new = set([auth['cid'] for auth in authors_new]) - \
set([auth['cid'] for auth in authors_old])
authors_diff = [author for author in authors_new
if author['cid'] in cids_new]
if limit is None:
limit = 10
else:
limit = int(limit)
# If limit set to a negative value all results will be returned
if limit >= 0:
authors_diff = authors_diff[:limit]
self.resolv_name(c, authors_diff)
return authors_diff
def __init__(self, db_path=None, db_default_file=None, vonly=False,
db_cache_path=None):
self.db_path = db_path or conf.get('db_path')
self.db_default_file = db_default_file or conf.get('db_default_file')
self.db_cache_path = db_cache_path or conf.get('db_cache_path')
YAMLDefinition.__init__(
self, self.db_path, self.db_default_file, self.db_cache_path)
self.enriched_groups = False
self.enriched_idents = False
if not vonly:
self._users = users.Users(
index.Connector(index_suffix='users'))
def setUpClass(cls):
cls.con = index.Connector(index='repoxplorertest')
cls.projects = {
'test': {
'repos': [{
'uri': 'https://github.com/nakata/monkey.git',
'name': 'monkey',
'branch': 'master'
}]
}
}
cls.c = Commits(cls.con)
cls.c.add_commits(COMMITS)