async def test_fluentd_forward_source_events(elasticsearch, http_client,
docker_ip):
"""FluentD forwards "native" records to ElasticSearch."""
# Grab the current date (we'll need to use it several times and we don't
# want to get flakey results when we execute the tests around midnight).
today = datetime.date.today()
# Clear the index.
url = urljoin(elasticsearch, 'events-%s' % (today.isoformat(), ))
async with http_client.delete(url) as resp:
# May get 404 if we're the first test to run, but we'll get 200 if we
# successfully delete the index.
assert resp.status in (200, 404)
# Post an event with a tag that matches `events.**` rule in `fluent.conf`.
fluent = FluentSender('events.test', host=docker_ip, port=24224)
fluent.emit('an-event', {
'some-field': 'some-value',
})
# Wait until the record shows up in search results.
await asyncio.sleep(datetime.timedelta(seconds=3).total_seconds())
# Grab the record.
url = urljoin(
elasticsearch, 'events-%04d-%02d-%02d/events/_search' % (
today.year,
today.month,
today.day,
))
async with http_client.get(url) as resp:
assert resp.status == 200
body = await resp.json()
assert body['hits']['total'] == 1
assert body['hits']['hits'][0]['_source'] == {
'service': 'test',
'event': 'an-event',
'some-field': 'some-value',
'@timestamp': mock.ANY,
}
# Grab index stats, check that index exists and that we have our data.
url = urljoin(elasticsearch, '_cat/indices?v')
async with http_client.get(url) as resp:
assert resp.status == 200
body = await resp.text()
print('STATS:')
print(body)
print('------')
lines = body.split('\n')
lines = [line.split() for line in lines if line.strip()]
assert len(lines) >= 2
stats = [dict(zip(lines[0], line)) for line in lines[1:]]
stats = {index['index']: index for index in stats}
index = stats.pop('events-%04d-%02d-%02d' %
(today.year, today.month, today.day))
assert int(index['docs.count']) >= 1
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-i",
"--ident",
help="Unique identifier of the honeypot.")
parser.add_argument("-n", "--hostname", help="Hostname of the honeypot.")
parser.add_argument("-t", "--type", help="Honeypot type.")
parser.add_argument("-a", "--address", help="IP address of the honeypot.")
parser.add_argument("--fluent-host",
default="fluentbit",
help="Hostname of Fluent Bit server")
parser.add_argument("--fluent-port",
type=int,
default=24284,
help="Port of Fluent Bit server")
parser.add_argument("--fluent-app",
default="stingar",
help="Application name for Fluent Bit server")
parser.add_argument("--tags", help="Comma separated tags for honeypot.")
args = parser.parse_args()
dt = datetime.datetime.now()
data = {
"identifier": args.ident,
"hostname": args.hostname,
"honeypot": args.type,
"ip": args.address,
"created": dt.strftime("%Y/%m/%d %H:%M:%S"),
"updated": dt.strftime("%Y/%m/%d %H:%M:%S"),
"tags": args.tags
}
sender = FluentSender(args.fluent_app,
host=args.fluent_host,
port=args.fluent_port)
sender.emit(SENSOR_TOPIC, data)
return 0
class LogSenseSender:
def __init__(self,
logsense_token=None,
tag='python',
meta={},
logsense_host=None,
logsense_port=None,
verbose=False,
nanosecond_precision=False):
internal_logger = logging.getLogger('logsense.sender')
self._logsense_token = logsense_token
if logsense_host:
self._logsense_host = logsense_host
else:
self._logsense_host = getenv('LOGSENSE_HOST', 'logs.logsense.com')
if logsense_port:
self._logsense_port = logsense_port
else:
self._logsense_port = int(getenv('LOGSENSE_PORT', '32714'))
if self._logsense_token is None:
self._logger = None
print("LOGSENSE_TOKEN not set - skipping handler")
else:
self._verbose = verbose
self._logger = FluentSender(
tag,
host=self._logsense_host,
ssl_server_hostname=self._logsense_host,
port=self._logsense_port,
use_ssl=True,
verbose=self._verbose)
self._base_dict = self.update_meta(meta)
self.nanosecond_precision = nanosecond_precision
def update_meta(self, new_meta):
self._base_dict = {
**new_meta,
**{
'cs_customer_token': self._logsense_token,
'cs_hostname': socket.gethostname()
}
}
return self._base_dict
def _convert_value_to_known_type(self, value):
if isinstance(value, str):
return value
elif isinstance(value, int):
return value
elif isinstance(value, float):
return value
elif isinstance(value, dict):
return value
elif isinstance(value, list):
return value
else:
# This fixes issues with e.g. NumPy serialization
return str(value)
def emit(self, data={}):
if self._logger:
if isinstance(data, dict):
converted_data = {
key: self._convert_value_to_known_type(value)
for key, value in data.items()
}
else:
converted_data = {'message': str(data)}
self._logger.emit('tag', {**converted_data, **self._base_dict})
def emit_with_time(self, label, timestamp, data):
event_timestamp = EventTime(timestamp)
if self._logger:
if isinstance(data, dict):
converted_data = {
key: self._convert_value_to_known_type(value)
for key, value in data.items()
}
else:
converted_data = {'message': str(data)}
self._logger.emit_with_time(label, event_timestamp, {
**converted_data,
**self._base_dict
})
@property
def last_error(self):
if self._logger:
return self._logger.last_error()
else:
return None
@last_error.setter
def last_error(self, err):
if self._logger:
self._logger.last_error(err)
def clear_last_error(self, _thread_id=None):
if self._logger:
self._logger.clear_last_error(_thread_id)
def close(self):
if self._logger:
self._logger.close()
from fluent.sender import FluentSender
sender = FluentSender('tag')
sender.emit('label', dict(a='data', b=dict(c='d')))
from fluent.sender import FluentSender
logger = FluentSender("app", host="localhost", port=24224)
logger.emit("follow", {"from": "userA", "to": "userB"})
logger.emit("bof", dict(beuha="aussi", age=42))
logger.close()
