def get_tags(self, repos, fromdate=None, todate=None):
qfilter = {
"bool": {
"must": [],
"should": [],
}
}
for repo in repos:
should_repo_clause = {"bool": {"must": []}}
should_repo_clause["bool"]["must"].append({"term": {"repo": repo}})
qfilter["bool"]["should"].append(should_repo_clause)
qfilter["bool"]["must"].append(
{"range": {
"date": {
"gte": fromdate,
"lt": todate,
}
}})
body = {"query": {"bool": {"filter": qfilter}}}
body = clean_empty(body)
return [
t for t in scanner(
self.es, query=body, index=self.index, doc_type=self.dbname)
]
def _scan(es, index, repository_fullname, params):
body = {
# "_source": "repository_fullname_and_number",
"_source": params.get('field', []),
"query": generate_filter(repository_fullname, params),
}
scanner_params = {'index': index, 'doc_type': index, 'query': body}
data = scanner(es, **scanner_params)
return [d['_source'] for d in data]
def get_all(self, source=True, type=None):
query = {
'_source': source,
'query': {
'match_all': {}
}
}
return scanner(self.es, query=query, index=self.index,
doc_type=type or self.dbname)
def get_commits(self,
mails=[],
repos=[],
fromdate=None,
todate=None,
start=0,
limit=100,
sort='desc',
scan=False,
merge_commit=None,
metadata=[],
mails_neg=False,
domains=None,
blacklisted_mails=None):
""" Return the list of commits for authors and/or repos.
"""
params = {'index': self.index, 'doc_type': self.dbname}
body = {
"filter":
self.get_filter(mails, repos, metadata, mails_neg, domains,
blacklisted_mails),
}
# If None both are returned. If you expect to skip merge commits
# then set merge_commit to False
if merge_commit is not None:
body["filter"]["bool"]["must"].append(
{"term": {
"merge_commit": merge_commit
}})
body["filter"]["bool"]["must"].append(
{"range": {
"committer_date": {
"gte": fromdate,
"lt": todate,
}
}})
if scan:
return scanner(self.es,
query=body,
index=self.index,
doc_type=self.dbname)
params['body'] = body
params['size'] = limit
params['from_'] = start
params['sort'] = "committer_date:%s,author_date:%s" % (sort, sort)
res = self.es.search(**params)
took = res['took']
hits = res['hits']['total']
commits = [r['_source'] for r in res['hits']['hits']]
return took, hits, commits
def scan(self):
return scanner(
es,
query={
**self.query_sort(),
"_source": False,
"query": self.query(),
},
preserve_order=bool(self.sort_by),
)
def _get_all(es, index):
body = {
"query": {"match_all": {}},
}
scanner_params = {"index": index, "query": body}
data = scanner(es, **scanner_params)
ret = []
for d in data:
ret.append(d["_source"])
return ret
def get_commits(self, mails=[], repos=[],
fromdate=None, todate=None, start=0, limit=100,
sort='desc', scan=False, merge_commit=None,
metadata=[], mails_neg=False, domains=None,
blacklisted_mails=None):
""" Return the list of commits for authors and/or repos.
"""
params = {'index': self.index, 'doc_type': self.dbname}
qfilter = self.get_filter(
mails, repos, metadata,
mails_neg, domains, blacklisted_mails)
# If None both are returned. If you expect to skip merge commits
# then set merge_commit to False
if merge_commit is not None:
qfilter["bool"]["must"].append(
{"term": {"merge_commit": merge_commit}}
)
qfilter["bool"]["must"].append(
{
"range": {
"committer_date": {
"gte": fromdate,
"lt": todate,
}
}
}
)
body = {
"query": {
"bool": {
"filter": qfilter,
}
}
}
if scan:
return scanner(self.es, query=body,
index=self.index,
doc_type=self.dbname)
params['body'] = body
params['size'] = limit
params['from_'] = start
params['sort'] = "committer_date:%s,author_date:%s" % (sort, sort)
res = self.es.search(**params)
took = res['took']
hits = res['hits']['total']
commits = [r['_source'] for r in res['hits']['hits']]
return took, hits, commits
def _scan(es, index, repository_fullname, params):
body = {
# "_source": "change_id",
"_source": params.get("field", []),
"query": generate_filter(es, index, repository_fullname, params),
}
scanner_params = {"index": index, "query": body}
data = scanner(es, **scanner_params)
ret = []
size = params.get("size")
for d in data:
if size and len(ret) == size:
break
ret.append(d["_source"])
return ret
def _scan(es, index, repository_fullname, params):
body = {
# "_source": "change_id",
"_source": params.get('field', []),
"query": generate_filter(repository_fullname, params),
}
scanner_params = {'index': index, 'doc_type': index, 'query': body}
data = scanner(es, **scanner_params)
ret = []
size = params.get('size')
for d in data:
if size and len(ret) == size:
break
ret.append(d['_source'])
return ret
def get_tags(self, repos, fromdate=None, todate=None):
qfilter = {
"bool": {
"must": [],
"should": [],
}
}
for repo in repos:
should_repo_clause = {
"bool": {
"must": []
}
}
should_repo_clause["bool"]["must"].append(
{"term": {"repo": repo}}
)
qfilter["bool"]["should"].append(should_repo_clause)
qfilter["bool"]["must"].append(
{
"range": {
"date": {
"gte": fromdate,
"lt": todate,
}
}
}
)
body = {
"query": {
"bool": {
"filter": qfilter
}
}
}
return [t for t in scanner(self.es, query=body,
index=self.index, doc_type=self.dbname)]
def get_commits(self,
mails=[],
repos=[],
fromdate=None,
todate=None,
start=0,
limit=100,
sort='desc',
scan=False,
merge_commit=None,
metadata=[],
mails_neg=False,
domains=None,
blacklisted_mails=None):
""" Return the list of commits for authors and/or repos.
"""
params = {'index': self.index}
qfilter = self.get_filter(mails, repos, metadata, mails_neg, domains,
blacklisted_mails)
# If None both are returned. If you expect to skip merge commits
# then set merge_commit to False
if merge_commit is not None:
qfilter["bool"]["must"].append(
{"term": {
"merge_commit": merge_commit
}})
qfilter["bool"]["must"].append(
{"range": {
"committer_date": {
"gte": fromdate,
"lt": todate,
}
}})
body = {
"query": {
"bool": {
"filter": qfilter,
}
}
}
if scan:
return scanner(self.es, query=body, index=self.index)
params['body'] = body
params['size'] = limit
params['from_'] = start
params['sort'] = "committer_date:%s,author_date:%s" % (sort, sort)
params = clean_empty(params)
res = self.es.search(**params)
took = res['took']
hits = res['hits']['total']
if isinstance(hits, dict) and 'value' in hits:
hits = hits.get('value')
commits = [r['_source'] for r in res['hits']['hits']]
return took, hits, commits
