def retrieve(self, _id):
self._db.indices.refresh(self._index)
res = self._db.get(self._index, self._doc_type, id=_id)
if not res['found']:
raise es.ElasticsearchException('Object not found.')
obj = self._cls.from_dict(res['_source'])
obj.reg_id = res['_id']
return obj
def create(self, obj: dao.Model):
res = self._db.index(index=self._index,
doc_type=self._doc_type,
body=obj.to_dict(),
id=obj.reg_id,
refresh=True)
if not res['result'] in ('created', 'updated'):
raise es.ElasticsearchException(
'Fail creating or updating the object in the database')
def write(self, results):
"""Write malice plugin results to Elasticsearch database."""
# sample already in malice DB (update sample with plugin results)
if self.es.exists(index=self.index,
doc_type=self.doc_type,
id=results['id']):
update_scan = {
'scan_date': datetime.now(),
'plugins': {
results['category']: {
results['name']: results['results']
}
}
}
resp = self.es.update(index=self.index,
doc_type=self.doc_type,
id=results['id'],
refresh="wait_for",
retry_on_conflict=3,
body=dict(doc=update_scan))
if not resp.get("result"):
raise elasticsearch.ElasticsearchException(
"[UPDATE] failed to update doc with id: {}".format(
results['id']))
return
scan = {
'scan_date': datetime.now(),
'plugins': {
results['category']: {
results['name']: results['results']
}
}
}
resp = self.es.index(index=self.index,
doc_type=self.doc_type,
id=results['id'],
body=scan)
if not resp.get("result"):
raise elasticsearch.ElasticsearchException(
"[INDEX] failed to write doc with id: {}".format(
results['id']))
def __init__(self, elastic_host, timeout=0):
self.es = elasticsearch.Elasticsearch(elastic_host)
# wait for elasticsearch to finish starting
for _ in range(timeout):
if self.es.ping():
break
time.sleep(1)
if not self.es.ping():
raise elasticsearch.ElasticsearchException("[PING] cannot connect to host: {}".format(elastic_host))
# create malice index
self.es.indices.create(index="malice", ignore=400)
def __init__(self, elastic_host, timeout=0):
self.index = os.environ.get('MALICE_ELASTICSEARCH_INDEX', 'malice')
self.doc_type = os.environ.get('MALICE_ELASTICSEARCH_TYPE', 'samples')
self.username = os.environ.get('MALICE_ELASTICSEARCH_USERNAME', '')
self.password = os.environ.get('MALICE_ELASTICSEARCH_PASSWORD', '')
self.es = elasticsearch.Elasticsearch(elastic_host,
http_auth=(self.username,
self.password))
# wait for elasticsearch to finish starting
for _ in range(timeout):
if self.es.ping():
break
time.sleep(1)
if not self.es.ping():
raise elasticsearch.ElasticsearchException(
"[PING] cannot connect to host: {}".format(elastic_host))
# create malice index
self.es.indices.create(index=self.index, ignore=400)
def get_client(**kwargs):
"""
Return an :class:`elasticsearch.Elasticsearch` client object using the
provided parameters. Any of the keyword arguments the
:class:`elasticsearch.Elasticsearch` client object can receive are valid,
such as:
:arg hosts: A list of one or more Elasticsearch client hostnames or IP
addresses to connect to. Can send a single host.
:type port: list
:arg port: The Elasticsearch client port to connect to.
:type port: int
:arg url_prefix: `Optional` url prefix, if needed to reach the Elasticsearch
API (i.e., it's not at the root level)
:type url_prefix: str
:arg use_ssl: Whether to connect to the client via SSL/TLS
:type use_ssl: bool
:arg certificate: Path to SSL/TLS certificate
:arg client_cert: Path to SSL/TLS client certificate (public key)
:arg client_key: Path to SSL/TLS private key
:arg aws_key: AWS IAM Access Key (Only used if the :mod:`requests-aws4auth`
python module is installed)
:arg aws_secret_key: AWS IAM Secret Access Key (Only used if the
:mod:`requests-aws4auth` python module is installed)
:arg aws_region: AWS Region (Only used if the :mod:`requests-aws4auth`
python module is installed)
:arg ssl_no_validate: If `True`, do not validate the certificate
chain. This is an insecure option and you will see warnings in the
log output.
:type ssl_no_validate: bool
:arg http_auth: Authentication credentials in `user:pass` format.
:type http_auth: str
:arg timeout: Number of seconds before the client will timeout.
:type timeout: int
:arg master_only: If `True`, the client will `only` connect if the
endpoint is the elected master node of the cluster. **This option does
not work if `hosts` has more than one value.** It will raise an
Exception in that case.
:type master_only: bool
:rtype: :class:`elasticsearch.Elasticsearch`
"""
if 'url_prefix' in kwargs:
if (type(kwargs['url_prefix']) == type(None)
or kwargs['url_prefix'] == "None"):
kwargs['url_prefix'] = ''
kwargs['hosts'] = '127.0.0.1' if not 'hosts' in kwargs else kwargs['hosts']
kwargs['master_only'] = False if not 'master_only' in kwargs \
else kwargs['master_only']
kwargs['use_ssl'] = False if not 'use_ssl' in kwargs else kwargs['use_ssl']
kwargs['ssl_no_validate'] = False if not 'ssl_no_validate' in kwargs \
else kwargs['ssl_no_validate']
kwargs['certificate'] = False if not 'certificate' in kwargs \
else kwargs['certificate']
kwargs['client_cert'] = False if not 'client_cert' in kwargs \
else kwargs['client_cert']
kwargs['client_key'] = False if not 'client_key' in kwargs \
else kwargs['client_key']
kwargs['hosts'] = ensure_list(kwargs['hosts'])
logger.debug("kwargs = {0}".format(kwargs))
master_only = kwargs.pop('master_only')
if kwargs['use_ssl']:
if kwargs['ssl_no_validate']:
kwargs[
'verify_certs'] = False # Not needed, but explicitly defined
else:
logger.info('Attempting to verify SSL certificate.')
# If user provides a certificate:
if kwargs['certificate']:
kwargs['verify_certs'] = True
kwargs['ca_certs'] = kwargs['certificate']
else: # Try to use certifi certificates:
try:
import certifi
kwargs['verify_certs'] = True
kwargs['ca_certs'] = certifi.where()
except ImportError:
logger.warn('Unable to verify SSL certificate.')
try:
from requests_aws4auth import AWS4Auth
kwargs['aws_key'] = False if not 'aws_key' in kwargs \
else kwargs['aws_key']
kwargs['aws_secret_key'] = False if not 'aws_secret_key' in kwargs \
else kwargs['aws_secret_key']
kwargs['region'] = False if not 'region' in kwargs \
else kwargs['region']
if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs['region']:
if not kwargs['aws_key'] and kwargs['aws_secret_key'] \
and kwargs['region']:
raise MissingArgument(
'Missing one or more of "aws_key", "aws_secret_key", '
'or "region".')
# Override these kwargs
kwargs['use_ssl'] = True
kwargs['verify_certs'] = True
kwargs['connection_class'] = elasticsearch.RequestsHttpConnection
kwargs['http_auth'] = (AWS4Auth(kwargs['aws_key'],
kwargs['aws_secret_key'],
kwargs['region'], 'es'))
else:
logger.debug('"requests_aws4auth" module present, but not used.')
except ImportError:
logger.debug('Not using "requests_aws4auth" python module to connect.')
if master_only:
if len(kwargs['hosts']) > 1:
raise ConfigurationError(
'"master_only" cannot be True if more than one host is '
'specified. Hosts = {0}'.format(kwargs['hosts']))
try:
client = elasticsearch.Elasticsearch(**kwargs)
# Verify the version is acceptable.
check_version(client)
# Verify "master_only" status, if applicable
check_master(client, master_only=master_only)
return client
except Exception as e:
raise elasticsearch.ElasticsearchException(
'Unable to create client connection to Elasticsearch. '
'Error: {0}'.format(e))
def test_elasticsearch_msearch_error(self, es):
"""Se debería devolver un error 500 cuando falla la query
MultiSearch."""
es.return_value.msearch.side_effect = \
elasticsearch.ElasticsearchException()
self.assert_500_error(random.choice(ENDPOINTS))
def test_elasticsearch_connection_error(self, es):
"""Se debería devolver un error 500 cuando falla la conexión a
Elasticsearch."""
es.side_effect = elasticsearch.ElasticsearchException()
self.assert_500_error(random.choice(ENDPOINTS))
def delete(self, obj: dao.Model):
response = self._db.delete(index=self._index,
doc_type=self._doc_type,
id=obj.reg_id)
if not response['result'] == 'deleted':
raise es.ElasticsearchException('Object not found.')
def createIndex(self):
'''
The first event
:param event:
:return:
'''
print("Creating Elastic Search Index --> %s" % INDEX)
try:
self.es.indices.create(INDEX, ignore=[400, 404], timeout=30)
except elasticsearch.ElasticsearchException, e:
raise e
def deleteIndex(self):
try:
self.es.indices.delete(INDEX, ignore=[400, 404])
except Exception, e:
raise e
def sendEventToElastic(self, event):
# Send events to Elastic
try:
self.es.index(index=INDEX, doc_type=DOC_TYPE, body=event)
except elasticsearch.TransportError, e:
if e.status_code == 409:
self.createIndex()
except elasticsearch.ElasticsearchException, e:
print(e)
raise elasticsearch.ElasticsearchException()
except Exception:
raise Exception
