def retrieve(self, _id): self._db.indices.refresh(self._index) res = self._db.get(self._index, self._doc_type, id=_id) if not res['found']: raise es.ElasticsearchException('Object not found.') obj = self._cls.from_dict(res['_source']) obj.reg_id = res['_id'] return obj
def create(self, obj: dao.Model): res = self._db.index(index=self._index, doc_type=self._doc_type, body=obj.to_dict(), id=obj.reg_id, refresh=True) if not res['result'] in ('created', 'updated'): raise es.ElasticsearchException( 'Fail creating or updating the object in the database')
def write(self, results): """Write malice plugin results to Elasticsearch database.""" # sample already in malice DB (update sample with plugin results) if self.es.exists(index=self.index, doc_type=self.doc_type, id=results['id']): update_scan = { 'scan_date': datetime.now(), 'plugins': { results['category']: { results['name']: results['results'] } } } resp = self.es.update(index=self.index, doc_type=self.doc_type, id=results['id'], refresh="wait_for", retry_on_conflict=3, body=dict(doc=update_scan)) if not resp.get("result"): raise elasticsearch.ElasticsearchException( "[UPDATE] failed to update doc with id: {}".format( results['id'])) return scan = { 'scan_date': datetime.now(), 'plugins': { results['category']: { results['name']: results['results'] } } } resp = self.es.index(index=self.index, doc_type=self.doc_type, id=results['id'], body=scan) if not resp.get("result"): raise elasticsearch.ElasticsearchException( "[INDEX] failed to write doc with id: {}".format( results['id']))
def __init__(self, elastic_host, timeout=0): self.es = elasticsearch.Elasticsearch(elastic_host) # wait for elasticsearch to finish starting for _ in range(timeout): if self.es.ping(): break time.sleep(1) if not self.es.ping(): raise elasticsearch.ElasticsearchException("[PING] cannot connect to host: {}".format(elastic_host)) # create malice index self.es.indices.create(index="malice", ignore=400)
def __init__(self, elastic_host, timeout=0): self.index = os.environ.get('MALICE_ELASTICSEARCH_INDEX', 'malice') self.doc_type = os.environ.get('MALICE_ELASTICSEARCH_TYPE', 'samples') self.username = os.environ.get('MALICE_ELASTICSEARCH_USERNAME', '') self.password = os.environ.get('MALICE_ELASTICSEARCH_PASSWORD', '') self.es = elasticsearch.Elasticsearch(elastic_host, http_auth=(self.username, self.password)) # wait for elasticsearch to finish starting for _ in range(timeout): if self.es.ping(): break time.sleep(1) if not self.es.ping(): raise elasticsearch.ElasticsearchException( "[PING] cannot connect to host: {}".format(elastic_host)) # create malice index self.es.indices.create(index=self.index, ignore=400)
def get_client(**kwargs): """ Return an :class:`elasticsearch.Elasticsearch` client object using the provided parameters. Any of the keyword arguments the :class:`elasticsearch.Elasticsearch` client object can receive are valid, such as: :arg hosts: A list of one or more Elasticsearch client hostnames or IP addresses to connect to. Can send a single host. :type port: list :arg port: The Elasticsearch client port to connect to. :type port: int :arg url_prefix: `Optional` url prefix, if needed to reach the Elasticsearch API (i.e., it's not at the root level) :type url_prefix: str :arg use_ssl: Whether to connect to the client via SSL/TLS :type use_ssl: bool :arg certificate: Path to SSL/TLS certificate :arg client_cert: Path to SSL/TLS client certificate (public key) :arg client_key: Path to SSL/TLS private key :arg aws_key: AWS IAM Access Key (Only used if the :mod:`requests-aws4auth` python module is installed) :arg aws_secret_key: AWS IAM Secret Access Key (Only used if the :mod:`requests-aws4auth` python module is installed) :arg aws_region: AWS Region (Only used if the :mod:`requests-aws4auth` python module is installed) :arg ssl_no_validate: If `True`, do not validate the certificate chain. This is an insecure option and you will see warnings in the log output. :type ssl_no_validate: bool :arg http_auth: Authentication credentials in `user:pass` format. :type http_auth: str :arg timeout: Number of seconds before the client will timeout. :type timeout: int :arg master_only: If `True`, the client will `only` connect if the endpoint is the elected master node of the cluster. **This option does not work if `hosts` has more than one value.** It will raise an Exception in that case. :type master_only: bool :rtype: :class:`elasticsearch.Elasticsearch` """ if 'url_prefix' in kwargs: if (type(kwargs['url_prefix']) == type(None) or kwargs['url_prefix'] == "None"): kwargs['url_prefix'] = '' kwargs['hosts'] = '127.0.0.1' if not 'hosts' in kwargs else kwargs['hosts'] kwargs['master_only'] = False if not 'master_only' in kwargs \ else kwargs['master_only'] kwargs['use_ssl'] = False if not 'use_ssl' in kwargs else kwargs['use_ssl'] kwargs['ssl_no_validate'] = False if not 'ssl_no_validate' in kwargs \ else kwargs['ssl_no_validate'] kwargs['certificate'] = False if not 'certificate' in kwargs \ else kwargs['certificate'] kwargs['client_cert'] = False if not 'client_cert' in kwargs \ else kwargs['client_cert'] kwargs['client_key'] = False if not 'client_key' in kwargs \ else kwargs['client_key'] kwargs['hosts'] = ensure_list(kwargs['hosts']) logger.debug("kwargs = {0}".format(kwargs)) master_only = kwargs.pop('master_only') if kwargs['use_ssl']: if kwargs['ssl_no_validate']: kwargs[ 'verify_certs'] = False # Not needed, but explicitly defined else: logger.info('Attempting to verify SSL certificate.') # If user provides a certificate: if kwargs['certificate']: kwargs['verify_certs'] = True kwargs['ca_certs'] = kwargs['certificate'] else: # Try to use certifi certificates: try: import certifi kwargs['verify_certs'] = True kwargs['ca_certs'] = certifi.where() except ImportError: logger.warn('Unable to verify SSL certificate.') try: from requests_aws4auth import AWS4Auth kwargs['aws_key'] = False if not 'aws_key' in kwargs \ else kwargs['aws_key'] kwargs['aws_secret_key'] = False if not 'aws_secret_key' in kwargs \ else kwargs['aws_secret_key'] kwargs['region'] = False if not 'region' in kwargs \ else kwargs['region'] if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs['region']: if not kwargs['aws_key'] and kwargs['aws_secret_key'] \ and kwargs['region']: raise MissingArgument( 'Missing one or more of "aws_key", "aws_secret_key", ' 'or "region".') # Override these kwargs kwargs['use_ssl'] = True kwargs['verify_certs'] = True kwargs['connection_class'] = elasticsearch.RequestsHttpConnection kwargs['http_auth'] = (AWS4Auth(kwargs['aws_key'], kwargs['aws_secret_key'], kwargs['region'], 'es')) else: logger.debug('"requests_aws4auth" module present, but not used.') except ImportError: logger.debug('Not using "requests_aws4auth" python module to connect.') if master_only: if len(kwargs['hosts']) > 1: raise ConfigurationError( '"master_only" cannot be True if more than one host is ' 'specified. Hosts = {0}'.format(kwargs['hosts'])) try: client = elasticsearch.Elasticsearch(**kwargs) # Verify the version is acceptable. check_version(client) # Verify "master_only" status, if applicable check_master(client, master_only=master_only) return client except Exception as e: raise elasticsearch.ElasticsearchException( 'Unable to create client connection to Elasticsearch. ' 'Error: {0}'.format(e))
def test_elasticsearch_msearch_error(self, es): """Se debería devolver un error 500 cuando falla la query MultiSearch.""" es.return_value.msearch.side_effect = \ elasticsearch.ElasticsearchException() self.assert_500_error(random.choice(ENDPOINTS))
def test_elasticsearch_connection_error(self, es): """Se debería devolver un error 500 cuando falla la conexión a Elasticsearch.""" es.side_effect = elasticsearch.ElasticsearchException() self.assert_500_error(random.choice(ENDPOINTS))
def delete(self, obj: dao.Model): response = self._db.delete(index=self._index, doc_type=self._doc_type, id=obj.reg_id) if not response['result'] == 'deleted': raise es.ElasticsearchException('Object not found.')
def createIndex(self): ''' The first event :param event: :return: ''' print("Creating Elastic Search Index --> %s" % INDEX) try: self.es.indices.create(INDEX, ignore=[400, 404], timeout=30) except elasticsearch.ElasticsearchException, e: raise e def deleteIndex(self): try: self.es.indices.delete(INDEX, ignore=[400, 404]) except Exception, e: raise e def sendEventToElastic(self, event): # Send events to Elastic try: self.es.index(index=INDEX, doc_type=DOC_TYPE, body=event) except elasticsearch.TransportError, e: if e.status_code == 409: self.createIndex() except elasticsearch.ElasticsearchException, e: print(e) raise elasticsearch.ElasticsearchException() except Exception: raise Exception