def __init__(self, host=None, port=None, index=None, index_suffix=None):
self.host = (host or getattr(conf, 'elasticsearch_host', None)
or 'localhost')
self.port = (port or getattr(conf, 'elasticsearch_port', None) or 9200)
self.index = (index or getattr(conf, 'elasticsearch_index', None)
or 'repoxplorer')
if index_suffix:
self.index += "-%s" % index_suffix
if (getattr(conf, 'elasticsearch_user', None)
and getattr(conf, 'elasticsearch_password', None)):
self.http_auth = "%s:%s" % (getattr(
conf, 'elasticsearch_user',
None), getattr(conf, 'elasticsearch_password', None))
# NOTE(dpawlik) Opendistro is using self signed certs,
# so verify_certs is set to False.
self.es = client.Elasticsearch([{
"host": self.host,
"port": self.port,
"http_auth": self.http_auth,
"use_ssl": True,
"verify_certs": False,
"ssl_show_warn": True
}],
timeout=60)
else:
self.es = client.Elasticsearch([{
"host": self.host,
"port": self.port
}],
timeout=60)
self.ic = client.IndicesClient(self.es)
if not self.ic.exists(index=self.index):
self.ic.create(index=self.index)
# Give some time to have the index fully created
time.sleep(1)
def __init__(self, model, doc_type):
assert model and issubclass(
model, SerializerElasticModel), 'Неверная модель: {}'.format(model)
self.model = model
self.doc_type = doc_type
self.connection = client.Elasticsearch(**settings.ELASTIC_CONNECTION)
self.index = settings.ELASTIC_INDEX
self._mapping = None
def __init__(self, host=None, port=None, index=None, index_suffix=None):
self.host = (host or getattr(conf, 'elasticsearch_host', None)
or 'localhost')
self.port = (port or getattr(conf, 'elasticsearch_port', None) or 9200)
self.index = (index or getattr(conf, 'elasticsearch_index', None)
or 'repoxplorer')
if index_suffix:
self.index += "-%s" % index_suffix
self.es = client.Elasticsearch([{
"host": self.host,
"port": self.port
}],
timeout=60)
self.ic = client.IndicesClient(self.es)
if not self.ic.exists(index=self.index):
self.ic.create(index=self.index)
# Give some time to have the index fully created
time.sleep(1)
def __init__(
self,
elastic_conn='localhost:9200',
index=None,
timeout=10,
prefix=CHANGE_PREFIX,
create=True,
):
host, port = elastic_conn.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ip = socket.gethostbyname(host)
self.log.info('ES IP is %s' % ip)
self.log.info('ES prefix is %s' % prefix)
while True:
try:
s.connect((ip, int(port)))
s.shutdown(2)
s.close()
break
except Exception as excpt:
self.log.info(
'Unable to connect to %s: %s. Sleeping for %ds.' %
(elastic_conn, excpt, timeout))
time.sleep(timeout)
self.log.info('Connecting to ES server at %s' % elastic_conn)
self.es = client.Elasticsearch(elastic_conn)
self.log.info(self.es.info())
self.prefix = prefix
if not index:
self.log.info('No index provided')
return
self.index = '{}{}'.format(self.prefix, index)
self.log.info('Using ES index %s' % self.index)
self.mapping = {
"properties": {
"id": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"number": {
"type": "keyword"
},
"change_id": {
"type": "keyword"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 8191
}
},
},
"text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 8191
}
},
},
"url": {
"type": "keyword"
},
"commit_count": {
"type": "integer"
},
"additions": {
"type": "integer"
},
"deletions": {
"type": "integer"
},
"changed_files_count": {
"type": "integer"
},
"changed_files": {
"properties": {
"additions": {
"type": "integer"
},
"deletions": {
"type": "integer"
},
"path": {
"type": "keyword"
},
}
},
"commits": {
"properties": {
"sha": {
"type": "keyword"
},
"author": {
"type": "keyword"
},
"committer": {
"type": "keyword"
},
"authored_at": {
"type": "date",
"format": "date_time_no_millis",
},
"committed_at": {
"type": "date",
"format": "date_time_no_millis",
},
"additions": {
"type": "integer"
},
"deletions": {
"type": "integer"
},
"title": {
"type": "text"
},
}
},
"repository_prefix": {
"type": "keyword"
},
"repository_fullname": {
"type": "keyword"
},
"repository_shortname": {
"type": "keyword"
},
"author": {
"type": "keyword"
},
"on_author": {
"type": "keyword"
},
"committer": {
"type": "keyword"
},
"merged_by": {
"type": "keyword"
},
"branch": {
"type": "keyword"
},
"target_branch": {
"type": "keyword"
},
"created_at": {
"type": "date",
"format": "date_time_no_millis"
},
"on_created_at": {
"type": "date",
"format": "date_time_no_millis"
},
"merged_at": {
"type": "date",
"format": "date_time_no_millis"
},
"updated_at": {
"type": "date",
"format": "date_time_no_millis"
},
"closed_at": {
"type": "date",
"format": "date_time_no_millis"
},
"state": {
"type": "keyword"
},
"duration": {
"type": "integer"
},
"mergeable": {
"type": "keyword"
},
"label": {
"type": "keyword"
},
"assignee": {
"type": "keyword"
},
"approval": {
"type": "keyword"
},
"draft": {
"type": "boolean"
},
"self_merged": {
"type": "boolean"
},
}
}
settings = {'mappings': self.mapping}
self.ic = client.IndicesClient(self.es)
if create:
self.ic.create(index=self.index, ignore=400, body=settings)
# The authors_histo is failing on some context with this error when the
# time slice is large: Must be less than or equal to: [10000] but was [10001]. ()This limit can be
# set by changing the [search.max_buckets] cluster level)
# This is an attempt to mitigate the issue
cluster_settings = {'transient': {'search.max_buckets': 100000}}
self.es.cluster.put_settings(body=cluster_settings)
def __init__(self, index='monocle'):
self.es = client.Elasticsearch('localhost:9200')
self.index = index
self.mapping = {
self.index: {
"properties": {
"id": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"number": {
"type": "keyword"
},
"repository_fullname_and_number": {
"type": "keyword"
},
"title": {
"type": "keyword"
},
"commits": {
"type": "keyword"
},
"additions": {
"type": "integer"
},
"deletions": {
"type": "integer"
},
"changed_files": {
"type": "integer"
},
"repository_prefix": {
"type": "keyword"
},
"repository_fullname": {
"type": "keyword"
},
"repository_shortname": {
"type": "keyword"
},
"author": {
"type": "keyword"
},
"on_author": {
"type": "keyword"
},
"committer": {
"type": "keyword"
},
"merged_by": {
"type": "keyword"
},
"created_at": {
"type": "date",
"format": "date_time_no_millis"
},
"on_created_at": {
"type": "date",
"format": "date_time_no_millis"
},
"merged_at": {
"type": "date",
"format": "date_time_no_millis"
},
"updated_at": {
"type": "date",
"format": "date_time_no_millis"
},
"closed_at": {
"type": "date",
"format": "date_time_no_millis"
},
"authored_at": {
"type": "date",
"format": "date_time_no_millis"
},
"committed_at": {
"type": "date",
"format": "date_time_no_millis"
},
"state": {
"type": "keyword"
},
"duration": {
"type": "integer"
},
"mergeable": {
"type": "keyword"
},
"label": {
"type": "keyword"
},
"assignee": {
"type": "keyword"
},
"approval": {
"type": "keyword"
},
}
}
}
settings = {'mappings': self.mapping}
self.ic = client.IndicesClient(self.es)
self.ic.create(index=self.index, ignore=400, body=settings)