def get_tags(self, repos, fromdate=None, todate=None): qfilter = { "bool": { "must": [], "should": [], } } for repo in repos: should_repo_clause = {"bool": {"must": []}} should_repo_clause["bool"]["must"].append({"term": {"repo": repo}}) qfilter["bool"]["should"].append(should_repo_clause) qfilter["bool"]["must"].append( {"range": { "date": { "gte": fromdate, "lt": todate, } }}) body = {"query": {"bool": {"filter": qfilter}}} body = clean_empty(body) return [ t for t in scanner( self.es, query=body, index=self.index, doc_type=self.dbname) ]
def _scan(es, index, repository_fullname, params): body = { # "_source": "repository_fullname_and_number", "_source": params.get('field', []), "query": generate_filter(repository_fullname, params), } scanner_params = {'index': index, 'doc_type': index, 'query': body} data = scanner(es, **scanner_params) return [d['_source'] for d in data]
def get_all(self, source=True, type=None): query = { '_source': source, 'query': { 'match_all': {} } } return scanner(self.es, query=query, index=self.index, doc_type=type or self.dbname)
def get_commits(self, mails=[], repos=[], fromdate=None, todate=None, start=0, limit=100, sort='desc', scan=False, merge_commit=None, metadata=[], mails_neg=False, domains=None, blacklisted_mails=None): """ Return the list of commits for authors and/or repos. """ params = {'index': self.index, 'doc_type': self.dbname} body = { "filter": self.get_filter(mails, repos, metadata, mails_neg, domains, blacklisted_mails), } # If None both are returned. If you expect to skip merge commits # then set merge_commit to False if merge_commit is not None: body["filter"]["bool"]["must"].append( {"term": { "merge_commit": merge_commit }}) body["filter"]["bool"]["must"].append( {"range": { "committer_date": { "gte": fromdate, "lt": todate, } }}) if scan: return scanner(self.es, query=body, index=self.index, doc_type=self.dbname) params['body'] = body params['size'] = limit params['from_'] = start params['sort'] = "committer_date:%s,author_date:%s" % (sort, sort) res = self.es.search(**params) took = res['took'] hits = res['hits']['total'] commits = [r['_source'] for r in res['hits']['hits']] return took, hits, commits
def scan(self): return scanner( es, query={ **self.query_sort(), "_source": False, "query": self.query(), }, preserve_order=bool(self.sort_by), )
def _get_all(es, index): body = { "query": {"match_all": {}}, } scanner_params = {"index": index, "query": body} data = scanner(es, **scanner_params) ret = [] for d in data: ret.append(d["_source"]) return ret
def get_commits(self, mails=[], repos=[], fromdate=None, todate=None, start=0, limit=100, sort='desc', scan=False, merge_commit=None, metadata=[], mails_neg=False, domains=None, blacklisted_mails=None): """ Return the list of commits for authors and/or repos. """ params = {'index': self.index, 'doc_type': self.dbname} qfilter = self.get_filter( mails, repos, metadata, mails_neg, domains, blacklisted_mails) # If None both are returned. If you expect to skip merge commits # then set merge_commit to False if merge_commit is not None: qfilter["bool"]["must"].append( {"term": {"merge_commit": merge_commit}} ) qfilter["bool"]["must"].append( { "range": { "committer_date": { "gte": fromdate, "lt": todate, } } } ) body = { "query": { "bool": { "filter": qfilter, } } } if scan: return scanner(self.es, query=body, index=self.index, doc_type=self.dbname) params['body'] = body params['size'] = limit params['from_'] = start params['sort'] = "committer_date:%s,author_date:%s" % (sort, sort) res = self.es.search(**params) took = res['took'] hits = res['hits']['total'] commits = [r['_source'] for r in res['hits']['hits']] return took, hits, commits
def _scan(es, index, repository_fullname, params): body = { # "_source": "change_id", "_source": params.get("field", []), "query": generate_filter(es, index, repository_fullname, params), } scanner_params = {"index": index, "query": body} data = scanner(es, **scanner_params) ret = [] size = params.get("size") for d in data: if size and len(ret) == size: break ret.append(d["_source"]) return ret
def _scan(es, index, repository_fullname, params): body = { # "_source": "change_id", "_source": params.get('field', []), "query": generate_filter(repository_fullname, params), } scanner_params = {'index': index, 'doc_type': index, 'query': body} data = scanner(es, **scanner_params) ret = [] size = params.get('size') for d in data: if size and len(ret) == size: break ret.append(d['_source']) return ret
def get_tags(self, repos, fromdate=None, todate=None): qfilter = { "bool": { "must": [], "should": [], } } for repo in repos: should_repo_clause = { "bool": { "must": [] } } should_repo_clause["bool"]["must"].append( {"term": {"repo": repo}} ) qfilter["bool"]["should"].append(should_repo_clause) qfilter["bool"]["must"].append( { "range": { "date": { "gte": fromdate, "lt": todate, } } } ) body = { "query": { "bool": { "filter": qfilter } } } return [t for t in scanner(self.es, query=body, index=self.index, doc_type=self.dbname)]
def get_commits(self, mails=[], repos=[], fromdate=None, todate=None, start=0, limit=100, sort='desc', scan=False, merge_commit=None, metadata=[], mails_neg=False, domains=None, blacklisted_mails=None): """ Return the list of commits for authors and/or repos. """ params = {'index': self.index} qfilter = self.get_filter(mails, repos, metadata, mails_neg, domains, blacklisted_mails) # If None both are returned. If you expect to skip merge commits # then set merge_commit to False if merge_commit is not None: qfilter["bool"]["must"].append( {"term": { "merge_commit": merge_commit }}) qfilter["bool"]["must"].append( {"range": { "committer_date": { "gte": fromdate, "lt": todate, } }}) body = { "query": { "bool": { "filter": qfilter, } } } if scan: return scanner(self.es, query=body, index=self.index) params['body'] = body params['size'] = limit params['from_'] = start params['sort'] = "committer_date:%s,author_date:%s" % (sort, sort) params = clean_empty(params) res = self.es.search(**params) took = res['took'] hits = res['hits']['total'] if isinstance(hits, dict) and 'value' in hits: hits = hits.get('value') commits = [r['_source'] for r in res['hits']['hits']] return took, hits, commits