def create_es_connection(self): ks = KeyStore() es_conf = ks.get_elasticsearch_credentials() self.es = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=False, http_auth=(es_conf["es_user"], es_conf["es_password"]))
def fix_es_mapping(index_name): ks = KeyStore() es_conf = ks.get_elasticsearch_credentials() mapping_url = "{es_url}/{index_name}/_mapping".format( index_name=index_name, **es_conf) res = requests.get(mapping_url, auth=(es_conf["es_user"], es_conf["es_password"])) output = res.json()[index_name] output['mappings']['test_stats']['dynamic'] = False output['mappings']['test_stats']['properties']['coredumps'] = dict( type='object') output['mappings']['test_stats']['properties']['setup_details'][ 'properties']['db_cluster_details'] = dict(type='object') output['mappings']['test_stats']['properties']['system_details'] = { "dynamic": False, "properties": {} } res = requests.put(mapping_url + "/test_stats", json=output['mappings'], auth=(es_conf["es_user"], es_conf["es_password"])) print(res.text) res.raise_for_status() click.secho("fixed {index_name}".format(index_name=index_name), fg='green')
def migrate(old_index_name, dry_run, new_index, days): # pylint: disable=too-many-locals logging.basicConfig(level=logging.DEBUG) logging.config.dictConfig({ 'version': 1, 'disable_existing_loggers': True, 'formatters': { 'standard': { 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' }, }, 'handlers': { 'default': { 'level': 'INFO', 'formatter': 'standard', 'class': 'logging.StreamHandler', 'stream': 'ext://sys.stdout', # Default is stderr }, }, 'loggers': { '': { # root logger 'handlers': ['default'], 'level': 'WARNING', 'propagate': False }, '__main__': { # if __name__ == '__main__' 'handlers': ['default'], 'level': 'DEBUG', 'propagate': False }, } }) ks = KeyStore() es_conf = ks.get_elasticsearch_credentials() elastic_search = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=True, http_auth=(es_conf["es_user"], es_conf["es_password"])) if not elastic_search.indices.exists(index=new_index): elastic_search.indices.create(index=new_index) def post_to_new(doc): if dry_run: return elastic_search.index(index=new_index, doc_type='nemesis', body=doc) res = scan(elastic_search, index=old_index_name, query={ "query": { "range": { "test_details.start_time": { "gte": (datetime.datetime.utcnow() - datetime.timedelta(days=days)).timestamp(), "lte": datetime.datetime.utcnow().timestamp(), "boost": 2.0 } } } }, size=300, scroll='3h') for num, hit in enumerate(res): nemesis_list = hit["_source"]["nemesis"] test_data = hit["_source"] LOGGER.info("%s: %s", num, test_data['test_details']['test_id']) if 'scylla-server' not in test_data['versions']: LOGGER.debug("%s: No version for %s - %s", num, test_data['test_details']['test_id'], test_data['test_details']['job_name']) if not test_data['test_details']['job_name']: LOGGER.debug(test_data) continue for nemesis_class, data in nemesis_list.items(): for failure in data['failures']: new_nemesis_data = dict( test_id=test_data['test_details']['test_id'], job_name=test_data['test_details']['job_name'], test_name=test_data['test_details']['test_name'], scylla_version=test_data['versions']['scylla-server'] ['version'], scylla_git_sha=test_data['versions']['scylla-server'] ['commit_id'], ) new_nemesis_data.update( dict(nemesis_name=nemesis_class, nemesis_duration=failure['duration'], start_time=datetime.datetime.utcfromtimestamp( failure['start']), end_time=datetime.datetime.utcfromtimestamp( failure['end']), target_node=failure['node'], outcome="failure", failure_message=failure['error'])) post_to_new(new_nemesis_data) for run in data['runs']: new_nemesis_data = dict( test_id=test_data['test_details']['test_id'], job_name=test_data['test_details']['job_name'], test_name=test_data['test_details']['test_name'], scylla_version=test_data['versions']['scylla-server'] ['version'], scylla_git_sha=test_data['versions']['scylla-server'] ['commit_id'], ) new_nemesis_data.update( dict(nemesis_name=nemesis_class, nemesis_duration=run['duration'], start_time=datetime.datetime.utcfromtimestamp( run['start']), end_time=datetime.datetime.utcfromtimestamp( run['end']), target_node=run['node'], outcome="passed")) if run.get('type', '') == 'skipped': new_nemesis_data['outcome'] = 'skipped' new_nemesis_data['skip_reason'] = run['skip_reason'] post_to_new(new_nemesis_data)
class ES(elasticsearch.Elasticsearch): """ Provides interface for Elasticsearch DB """ def __init__(self): self._conf = self.get_conf() super().__init__(hosts=[self._conf["es_url"]], verify_certs=False, http_auth=(self._conf["es_user"], self._conf["es_password"])) def get_conf(self): self.key_store = KeyStore() return self.key_store.get_elasticsearch_credentials() def _create_index(self, index): self.indices.create(index=index, ignore=400) # pylint: disable=unexpected-keyword-arg def create_doc(self, index, doc_type, doc_id, body): """ Add document in json format """ LOGGER.info('Create doc') LOGGER.info('INDEX: %s', index) LOGGER.info('DOC_TYPE: %s', doc_type) LOGGER.info('DOC_ID: %s', doc_id) LOGGER.info('BODY: %s', body) self._create_index(index) if self.exists(index=index, doc_type=doc_type, id=doc_id): self.update(index=index, doc_type=doc_type, id=doc_id, body={'doc': body}) else: self.create(index=index, doc_type=doc_type, id=doc_id, body=body) def update_doc(self, index, doc_type, doc_id, body): """ Update document with partial data """ LOGGER.info('Update doc %s with info %s', doc_id, body) self.update(index=index, doc_type=doc_type, id=doc_id, body=dict(doc=body)) def get_all(self, index, limit=1000): """ Search for documents for the certain index """ return self.search(index=index, size=limit) # pylint: disable=unexpected-keyword-arg def get_doc(self, index, doc_id, doc_type='_all'): """ Get document by id """ doc = self.get( index=index, doc_type=doc_type, id=doc_id, ignore=[ # pylint: disable=unexpected-keyword-arg 400, 404 ]) if not doc['found']: LOGGER.warning('Document not found: %s %s', doc_id, doc_type) return None return doc def delete_doc(self, index, doc_type, doc_id): """ Delete document """ if self.get_doc(index, doc_id, doc_type): self.delete(index=index, doc_type=doc_type, id=doc_id)