class FluentTraceLogger( object ):
def __init__( self, name, host='localhost', port=24224 ):
if FluentSender is None:
raise Exception("Attempted to use FluentTraceLogger with not Fluent dependency available.")
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender( self.name, host=host, port=port )
def context_set( self, key, value ):
self.lock.acquire()
if not hasattr( self.thread_local, 'context' ):
self.thread_local.context = {}
self.thread_local.context[key] = value
self.lock.release()
def context_remove( self, key ):
self.lock.acquire()
del self.thread_local.context[key]
self.lock.release()
def log( self, label, time=None, **kwargs ):
self.lock.acquire()
if hasattr( self.thread_local, 'context' ):
kwargs.update( self.thread_local.context )
self.lock.release()
if time is None:
time = int( time.time() )
self.sender.emit_with_time( label, time, kwargs )
class FluentTraceLogger(object):
def __init__(self, name, host='localhost', port=24224):
if FluentSender is None:
raise Exception(
"Attempted to use FluentTraceLogger with not Fluent dependency available."
)
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender(self.name, host=host, port=port)
def context_set(self, key, value):
self.lock.acquire()
if not hasattr(self.thread_local, 'context'):
self.thread_local.context = {}
self.thread_local.context[key] = value
self.lock.release()
def context_remove(self, key):
self.lock.acquire()
del self.thread_local.context[key]
self.lock.release()
def log(self, label, time=None, **kwargs):
self.lock.acquire()
if hasattr(self.thread_local, 'context'):
kwargs.update(self.thread_local.context)
self.lock.release()
if time is None:
time = int(time.time())
self.sender.emit_with_time(label, time, kwargs)
class FluentTraceLogger( object ):
def __init__( self, name, host='localhost', port=24224 ):
assert FluentSender is not None, FLUENT_IMPORT_MESSAGE
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender( self.name, host=host, port=port )
def context_set( self, key, value ):
self.lock.acquire()
if not hasattr( self.thread_local, 'context' ):
self.thread_local.context = {}
self.thread_local.context[key] = value
self.lock.release()
def context_remove( self, key ):
self.lock.acquire()
del self.thread_local.context[key]
self.lock.release()
def log( self, label, event_time=None, **kwargs ):
self.lock.acquire()
if hasattr( self.thread_local, 'context' ):
kwargs.update( self.thread_local.context )
self.lock.release()
event_time = event_time or time.time()
self.sender.emit_with_time( label, int(event_time), kwargs )
async def test_fluentd_forward_source_events(elasticsearch, http_client,
docker_ip):
"""FluentD forwards "native" records to ElasticSearch."""
# Grab the current date (we'll need to use it several times and we don't
# want to get flakey results when we execute the tests around midnight).
today = datetime.date.today()
# Clear the index.
url = urljoin(elasticsearch, 'events-%s' % (today.isoformat(), ))
async with http_client.delete(url) as resp:
# May get 404 if we're the first test to run, but we'll get 200 if we
# successfully delete the index.
assert resp.status in (200, 404)
# Post an event with a tag that matches `events.**` rule in `fluent.conf`.
fluent = FluentSender('events.test', host=docker_ip, port=24224)
fluent.emit('an-event', {
'some-field': 'some-value',
})
# Wait until the record shows up in search results.
await asyncio.sleep(datetime.timedelta(seconds=3).total_seconds())
# Grab the record.
url = urljoin(
elasticsearch, 'events-%04d-%02d-%02d/events/_search' % (
today.year,
today.month,
today.day,
))
async with http_client.get(url) as resp:
assert resp.status == 200
body = await resp.json()
assert body['hits']['total'] == 1
assert body['hits']['hits'][0]['_source'] == {
'service': 'test',
'event': 'an-event',
'some-field': 'some-value',
'@timestamp': mock.ANY,
}
# Grab index stats, check that index exists and that we have our data.
url = urljoin(elasticsearch, '_cat/indices?v')
async with http_client.get(url) as resp:
assert resp.status == 200
body = await resp.text()
print('STATS:')
print(body)
print('------')
lines = body.split('\n')
lines = [line.split() for line in lines if line.strip()]
assert len(lines) >= 2
stats = [dict(zip(lines[0], line)) for line in lines[1:]]
stats = {index['index']: index for index in stats}
index = stats.pop('events-%04d-%02d-%02d' %
(today.year, today.month, today.day))
assert int(index['docs.count']) >= 1
def __init__(self, tag, pdir, stream_cfg, send_term,
max_send_fail, echo, encoding, lines_on_start,
max_between_data):
"""
Trailer common class initialization
Args:
tag: Classification tag for Fluentd
pdir: Position file directory
stream_cfg: Log streaming service config (Fluentd / Kinesis)
strem_cfg: Log transmission time interval
max_send_fail: Maximum number of retries in case of transmission
failure
echo: Whether to save sent messages
encoding: Original message encoding
lines_on_start: How many lines of existing log will be resent
at startup (for debugging)
max_between_data: When the service is restarted, unsent logs
smaller than this amount are sent.
"""
super(BaseTailer, self).__init__()
self.fsender = self.kclient = None
self.ksent_seqn = self.ksent_shid = None
self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail))
self.last_get_hinfo = 0
self.sname, self.saddr = self.get_host_info()
tag = "{}.{}".format(self.sname.lower() if self.sname is not None else
None, tag)
self.linfo(1, "tag: '{}'".format(tag))
self.tag = tag
self.send_term = send_term
self.last_send_try = 0
self.last_update = 0
self.kpk_cnt = 0 # count for kinesis partition key
self.pdir = pdir
max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL
tstc = type(stream_cfg)
if tstc == FluentCfg:
host, port = stream_cfg
self.fsender = FluentSender(tag, host, port,
max_send_fail=max_send_fail)
elif tstc == KinesisCfg:
stream_name, region, access_key, secret_key = stream_cfg
self.kstream_name = stream_name
self.ldebug('query_aws_client kinesis {}'.format(region))
self.kclient = query_aws_client('kinesis', region, access_key,
secret_key)
self.kagg = aggregator.RecordAggregator()
self.send_retry = 0
self.echo_file = StringIO() if echo else None
self.cache_sent_pos = {}
self.encoding = encoding
self.lines_on_start = lines_on_start if lines_on_start else 0
self.max_between_data = max_between_data if max_between_data else\
MAX_BETWEEN_DATA
def __init__(self, name, host='localhost', port=24224):
if FluentSender is None:
raise Exception(
"Attempted to use FluentTraceLogger with not Fluent dependency available."
)
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender(self.name, host=host, port=port)
def loop(fs: sender.FluentSender, connection_id: str, interval: int,
verbose: bool):
while True:
results, timestamp = run_speedtest()
if results:
if not fs.emit_with_time(connection_id, timestamp,
clean_results(results, verbose)):
logger.error("fluent error: %s", fs.last_error)
fs.clear_last_error()
logger.info("Sleeping for %s", interval)
time.sleep(interval)
def connect(self, tag=None):
"""
Connect to a fluentd logger
"""
if not tag:
tag = self.tag
timeout = float(self.timeout)
port = int(self.port)
buffer_max = int(self.buffer_max)
self.sender = FluentSender(tag, self.host, port, buffer_max, timeout)
class FluentdConnector(StoqConnectorPlugin):
def __init__(self):
super().__init__()
def activate(self, stoq):
self.stoq = stoq
super().activate()
def save(self, payload, archive=False, **kwargs):
"""
Save results to fluentd logger
:param bytes payload: Content to be sent to fluentd
:param **kwargs index: Index name to save content to
"""
# Define the index name, if available. Will default to the plugin name
index = kwargs.get('index', self.parentname)
for save_attempt in range(3):
try:
self.sender.emit(index, payload)
except AttributeError:
self.connect()
super().save()
def connect(self, tag=None):
"""
Connect to a fluentd logger
"""
if not tag:
tag = self.tag
timeout = float(self.timeout)
port = int(self.port)
buffer_max = int(self.buffer_max)
self.sender = FluentSender(tag, self.host, port, buffer_max, timeout)
def disconnect(self):
"""
Disconnect to a fluentd logger
"""
self.sender._close()
def init_global_fsender(tag, host, port):
from fluent.sender import FluentSender
global fsender
if fsender is None:
fsender = FluentSender(tag, host, port)
linfo("init_global_fsender")
def __init__( self, name, host='localhost', port=24224 ):
if FluentSender is None:
raise Exception("Attempted to use FluentTraceLogger with not Fluent dependency available.")
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender( self.name, host=host, port=port )
def __init__(
self,
mongos,
mongor_ssl=False,
cursor_timeout=600, #10 Minutes
fluentd="localhost:24224",
logtag="rabidmongoose"):
self.connections = {}
self.cursor_timeout = cursor_timeout
self.mongor_uri = "".join(mongos)
self.mongor_host = self.mongor_uri[10:].split(":")[0]
self.mongor_port = int(self.mongor_uri[10:].split(":")[1])
self.mongor_ssl = mongor_ssl
LOGGER.debug('INIT - URI: %s, Host: %s, Port: %s' %
(self.mongor_uri, self.mongor_host, self.mongor_port))
if HAS_FLUENT:
self.sender = FluentSender(host=fluentd.split(":")[0],
port=int(fluentd.split(":")[1]),
tag=logtag)
LOGGER.addHandler(
FluentHandler("%s.debug" % (logtag),
host=fluentd.split(":")[0],
port=int(fluentd.split(":")[1])))
else:
self.sender = FakeFluentSender()
class FluentdConnector(StoqConnectorPlugin):
def __init__(self):
super().__init__()
def activate(self, stoq):
self.stoq = stoq
super().activate()
def save(self, payload, archive=False, **kwargs):
"""
Save results to fluentd logger
:param bytes payload: Content to be sent to fluentd
:param **kwargs kwargs: Additional attributes (unused)
"""
for save_attempt in range(3):
try:
self.sender.emit(self.parentname, payload)
except AttributeError:
self.connect()
super().save()
def connect(self, tag=None):
"""
Connect to a fluentd logger
"""
if not tag:
tag = self.tag
timeout = float(self.timeout)
port = int(self.port)
buffer_max = int(self.buffer_max)
self.sender = FluentSender(tag, self.host, port, buffer_max, timeout)
def disconnect(self):
"""
Disconnect to a fluentd logger
"""
self.sender._close()
def __init__(self, tag, pdir, stream_cfg, send_term,
max_send_fail, echo, encoding, lines_on_start,
max_between_data):
"""
Trailer common class initialization
Args:
tag: Classification tag for Fluentd
pdir: Position file directory
stream_cfg: Log streaming service config (Fluentd / Kinesis)
strem_cfg: Log transmission time interval
max_send_fail: Maximum number of retries in case of transmission
failure
echo: Whether to save sent messages
encoding: Original message encoding
lines_on_start: How many lines of existing log will be resent
at startup (for debugging)
max_between_data: When the service is restarted, unsent logs
smaller than this amount are sent.
"""
super(BaseTailer, self).__init__()
self.fsender = self.kclient = None
self.ksent_seqn = self.ksent_shid = None
self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail))
self.last_get_hinfo = 0
self.sname, self.saddr = self.get_host_info()
tag = "{}.{}".format(self.sname.lower() if self.sname is not None else
None, tag)
self.linfo(1, "tag: '{}'".format(tag))
self.tag = tag
self.send_term = send_term
self.last_send_try = 0
self.last_update = 0
self.kpk_cnt = 0 # count for kinesis partition key
self.pdir = pdir
max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL
tstc = type(stream_cfg)
if tstc == FluentCfg:
host, port = stream_cfg
self.fsender = FluentSender(tag, host, port,
max_send_fail=max_send_fail)
elif tstc == KinesisCfg:
stream_name, region, access_key, secret_key = stream_cfg
self.kstream_name = stream_name
self.ldebug('query_aws_client kinesis {}'.format(region))
self.kclient = query_aws_client('kinesis', region, access_key,
secret_key)
self.kagg = aggregator.RecordAggregator()
self.send_retry = 0
self.echo_file = StringIO() if echo else None
self.cache_sent_pos = {}
self.encoding = encoding
self.lines_on_start = lines_on_start if lines_on_start else 0
self.max_between_data = max_between_data if max_between_data else\
MAX_BETWEEN_DATA
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-i",
"--ident",
help="Unique identifier of the honeypot.")
parser.add_argument("-n", "--hostname", help="Hostname of the honeypot.")
parser.add_argument("-t", "--type", help="Honeypot type.")
parser.add_argument("-a", "--address", help="IP address of the honeypot.")
parser.add_argument("--fluent-host",
default="fluentbit",
help="Hostname of Fluent Bit server")
parser.add_argument("--fluent-port",
type=int,
default=24284,
help="Port of Fluent Bit server")
parser.add_argument("--fluent-app",
default="stingar",
help="Application name for Fluent Bit server")
parser.add_argument("--tags", help="Comma separated tags for honeypot.")
args = parser.parse_args()
dt = datetime.datetime.now()
data = {
"identifier": args.ident,
"hostname": args.hostname,
"honeypot": args.type,
"ip": args.address,
"created": dt.strftime("%Y/%m/%d %H:%M:%S"),
"updated": dt.strftime("%Y/%m/%d %H:%M:%S"),
"tags": args.tags
}
sender = FluentSender(args.fluent_app,
host=args.fluent_host,
port=args.fluent_port)
sender.emit(SENSOR_TOPIC, data)
return 0
def connect(self, tag=None):
"""
Connect to a fluentd logger
"""
if not tag:
tag = self.tag
timeout = float(self.timeout)
port = int(self.port)
buffer_max = int(self.buffer_max)
self.sender = FluentSender(tag, self.host, port, buffer_max, timeout)
def __init__(self,
logsense_token=None,
tag='python',
meta={},
logsense_host=None,
logsense_port=None,
verbose=False,
nanosecond_precision=False):
internal_logger = logging.getLogger('logsense.sender')
self._logsense_token = logsense_token
if logsense_host:
self._logsense_host = logsense_host
else:
self._logsense_host = getenv('LOGSENSE_HOST', 'logs.logsense.com')
if logsense_port:
self._logsense_port = logsense_port
else:
self._logsense_port = int(getenv('LOGSENSE_PORT', '32714'))
if self._logsense_token is None:
self._logger = None
print("LOGSENSE_TOKEN not set - skipping handler")
else:
self._verbose = verbose
self._logger = FluentSender(
tag,
host=self._logsense_host,
ssl_server_hostname=self._logsense_host,
port=self._logsense_port,
use_ssl=True,
verbose=self._verbose)
self._base_dict = self.update_meta(meta)
self.nanosecond_precision = nanosecond_precision
def _get_sender(self, tag, host, port, bufmax, timeout):
"""
Get the Fluentd sender for the given arguments.
Arguments:
tag -- The base tag of the log event, representing the
application.
host -- The host name or IP address of the fluentd server.
port -- The port number the fluentd server is listeneing on.
bufmax -- The size of the buffer.
timeout -- The timeout
Returns:
A FluentSender with the given configuration.
"""
sender = None
key = "%s%s%d%d%f" % (tag, host, port, bufmax, timeout)
if key in self._senders:
sender = self._senders[key]
else:
sender = FluentSender(tag, host, port, bufmax, timeout)
self._senders[key] = sender
return sender
def __init__( self, name, host='localhost', port=24224 ):
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender( self.name, host=host, port=port )
class LogSenseSender:
def __init__(self,
logsense_token=None,
tag='python',
meta={},
logsense_host=None,
logsense_port=None,
verbose=False,
nanosecond_precision=False):
internal_logger = logging.getLogger('logsense.sender')
self._logsense_token = logsense_token
if logsense_host:
self._logsense_host = logsense_host
else:
self._logsense_host = getenv('LOGSENSE_HOST', 'logs.logsense.com')
if logsense_port:
self._logsense_port = logsense_port
else:
self._logsense_port = int(getenv('LOGSENSE_PORT', '32714'))
if self._logsense_token is None:
self._logger = None
print("LOGSENSE_TOKEN not set - skipping handler")
else:
self._verbose = verbose
self._logger = FluentSender(
tag,
host=self._logsense_host,
ssl_server_hostname=self._logsense_host,
port=self._logsense_port,
use_ssl=True,
verbose=self._verbose)
self._base_dict = self.update_meta(meta)
self.nanosecond_precision = nanosecond_precision
def update_meta(self, new_meta):
self._base_dict = {
**new_meta,
**{
'cs_customer_token': self._logsense_token,
'cs_hostname': socket.gethostname()
}
}
return self._base_dict
def _convert_value_to_known_type(self, value):
if isinstance(value, str):
return value
elif isinstance(value, int):
return value
elif isinstance(value, float):
return value
elif isinstance(value, dict):
return value
elif isinstance(value, list):
return value
else:
# This fixes issues with e.g. NumPy serialization
return str(value)
def emit(self, data={}):
if self._logger:
if isinstance(data, dict):
converted_data = {
key: self._convert_value_to_known_type(value)
for key, value in data.items()
}
else:
converted_data = {'message': str(data)}
self._logger.emit('tag', {**converted_data, **self._base_dict})
def emit_with_time(self, label, timestamp, data):
event_timestamp = EventTime(timestamp)
if self._logger:
if isinstance(data, dict):
converted_data = {
key: self._convert_value_to_known_type(value)
for key, value in data.items()
}
else:
converted_data = {'message': str(data)}
self._logger.emit_with_time(label, event_timestamp, {
**converted_data,
**self._base_dict
})
@property
def last_error(self):
if self._logger:
return self._logger.last_error()
else:
return None
@last_error.setter
def last_error(self, err):
if self._logger:
self._logger.last_error(err)
def clear_last_error(self, _thread_id=None):
if self._logger:
self._logger.clear_last_error(_thread_id)
def close(self):
if self._logger:
self._logger.close()
from fluent.sender import FluentSender
logger = FluentSender("app", host="localhost", port=24224)
logger.emit("follow", {"from": "userA", "to": "userB"})
logger.emit("bof", dict(beuha="aussi", age=42))
logger.close()
class BaseTailer(object):
def __init__(self, tag, pdir, stream_cfg, send_term,
max_send_fail, echo, encoding, lines_on_start,
max_between_data):
"""
Trailer common class initialization
Args:
tag: Classification tag for Fluentd
pdir: Position file directory
stream_cfg: Log streaming service config (Fluentd / Kinesis)
strem_cfg: Log transmission time interval
max_send_fail: Maximum number of retries in case of transmission
failure
echo: Whether to save sent messages
encoding: Original message encoding
lines_on_start: How many lines of existing log will be resent
at startup (for debugging)
max_between_data: When the service is restarted, unsent logs
smaller than this amount are sent.
"""
super(BaseTailer, self).__init__()
self.fsender = self.kclient = None
self.ksent_seqn = self.ksent_shid = None
self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail))
self.last_get_hinfo = 0
self.sname, self.saddr = self.get_host_info()
tag = "{}.{}".format(self.sname.lower() if self.sname is not None else
None, tag)
self.linfo(1, "tag: '{}'".format(tag))
self.tag = tag
self.send_term = send_term
self.last_send_try = 0
self.last_update = 0
self.kpk_cnt = 0 # count for kinesis partition key
self.pdir = pdir
max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL
tstc = type(stream_cfg)
if tstc == FluentCfg:
host, port = stream_cfg
self.fsender = FluentSender(tag, host, port,
max_send_fail=max_send_fail)
elif tstc == KinesisCfg:
stream_name, region, access_key, secret_key = stream_cfg
self.kstream_name = stream_name
self.ldebug('query_aws_client kinesis {}'.format(region))
self.kclient = query_aws_client('kinesis', region, access_key,
secret_key)
self.kagg = aggregator.RecordAggregator()
self.send_retry = 0
self.echo_file = StringIO() if echo else None
self.cache_sent_pos = {}
self.encoding = encoding
self.lines_on_start = lines_on_start if lines_on_start else 0
self.max_between_data = max_between_data if max_between_data else\
MAX_BETWEEN_DATA
def get_host_info(self):
sname = saddr = None
try:
sname = socket.gethostname()
self.linfo(" host name: {}".format(sname))
saddr = socket.gethostbyname(sname)
self.linfo(" host addr: {}".format(saddr))
except Exception as e:
self.lerror("Fail to get host info: {}".format(e))
return sname, saddr
def query_host_info(self):
invalid = (self.sname is None) or (self.saddr is None)
if invalid:
if time.time() - self.last_get_hinfo > GET_HINFO_TERM:
self.sname, self.saddr = self.get_host_info()
self.last_get_hinfo = time.time()
self.linfo(" self.sname {}, self.saddr {}".format(self.sname, self.saddr))
return self.sname, self.saddr
def ldebug(self, tabfunc, msg=""):
_log(self, 'debug', tabfunc, msg)
def linfo(self, tabfunc, msg=""):
_log(self, 'info', tabfunc, msg)
def lwarning(self, tabfunc, msg=""):
_log(self, 'warning', tabfunc, msg)
def lerror(self, tabfunc, msg=""):
_log(self, 'error', tabfunc, msg)
def tmain(self):
cur = time.time()
self.ldebug("tmain {}".format(cur))
return cur
def read_sent_pos(self, target, con):
"""Update the sent position of the target so far.
Arguments:
target: file path for FileTailer, table name for DBTailer
con(DBConnector): DB Connection
Returns:
(position type): Parsed position type
`int` for FileTailer.
`datetime` for TableTailer.
"""
self.linfo("read_sent_pos", "updating for '{}'..".format(target))
tname = escape_path(target)
ppath = os.path.join(self.pdir, tname + '.pos')
pos = None
if os.path.isfile(ppath):
with open(ppath, 'r') as f:
pos = f.readline()
self.linfo(1, "found pos file - {}: {}".format(ppath, pos))
parsed_pos = self.parse_sent_pos(pos)
if parsed_pos is None:
self.lerror("Invalid pos file: '{}'".format(pos))
pos = None
else:
pos = parsed_pos
if pos is None:
pos = self.get_initial_pos(con)
self.linfo(1, "can't find valid pos for {}, save as "
"initial value {}".format(target, pos))
self._save_sent_pos(target, pos)
pos = self.parse_sent_pos(pos)
return pos
def _save_sent_pos(self, target, pos):
"""Save sent position for a target flie.
Args:
target: A target file for which position will be saved.
pos: Sent position to save.
"""
self.linfo(1, "_save_sent_pos for {} - {}".format(target, pos))
tname = escape_path(target)
path = os.path.join(self.pdir, tname + '.pos')
try:
with open(path, 'w') as f:
f.write("{}\n".format(pos))
except Exception as e:
self.lerror("Fail to write pos file: {} {}".format(e, path))
self.cache_sent_pos[target] = pos
def _send_newline(self, msg, msgs):
"""Send new lines
This does not send right away, but waits for a certain number of
messages to send for efficiency.
Args:
msg: A message to send
msgs: Bulk message buffer
"""
# self.ldebug("_send_newline {}".format(msg))
ts = int(time.time())
self.may_echo(msg)
msgs.append((ts, msg))
if len(msgs) >= BULK_SEND_SIZE:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
msgs[:] = []
def _send_remain_msgs(self, msgs):
"""Send bulk remain messages."""
if len(msgs) > 0:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
def _handle_send_fail(self, e, rbytes):
"""Handle send exception.
Args:
e: Exception instance
rbytes: Size of send message in bytes.
Raises:
Re-raise send exception
"""
self.lwarning(1, "send fail '{}'".format(e))
self.send_retry += 1
if self.send_retry < MAX_SEND_RETRY:
self.lerror(1, "Not exceed max retry({} < {}), will try "
"again".format(self.send_retry,
MAX_SEND_RETRY))
raise
else:
self.lerror(1, "Exceed max retry, Giving up this change({}"
" Bytes)!!".format(rbytes))
def _make_fluent_bulk(self, msgs):
"""Make bulk payload for fluentd"""
tag = '.'.join((self.tag, "data"))
bulk = [msgpack.packb((tag, ts, data)) for ts, data in msgs]
return ''.join(bulk)
def may_echo(self, line):
"""Echo sent message for debugging
Args:
line: Sent message
"""
if self.echo_file:
self.echo_file.write('{}\n'.format(line))
self.echo_file.flush()
def _kinesis_put(self, msgs):
"""Send to AWS Kinesis
Make aggregated message and send it.
Args:
msgs: Messages to send
"""
self.linfo('_kinesis_put {} messages'.format(len(msgs)))
self.kpk_cnt += 1 # round robin shards
for aggd in self._iter_kinesis_aggrec(msgs):
pk, ehk, data = aggd.get_contents()
self.linfo(" kinesis aggregated put_record: {} "
"bytes".format(len(data)))
st = time.time()
ret = self.kclient.put_record(
StreamName=self.kstream_name,
Data=data,
PartitionKey=pk,
ExplicitHashKey=ehk
)
stat = ret['ResponseMetadata']['HTTPStatusCode']
shid = ret['ShardId']
seqn = ret['SequenceNumber']
self.ksent_seqn = seqn
self.ksent_shid = shid
elp = time.time() - st
if stat == 200:
self.linfo("Kinesis put success in {}: ShardId: {}, "
"SequenceNumber: {}".format(elp, shid, seqn))
else:
self.error("Kineis put failed in {}!: "
"{}".format(elp, ret['ResponseMetadata']))
def _iter_kinesis_aggrec(self, msgs):
for msg in msgs:
data = {'tag_': self.tag + '.data', 'ts_': msg[0]}
if isinstance(msg[1], dict):
data.update(msg[1])
else:
data['value_'] = msg[1]
pk = str(uuid.uuid4())
res = self.kagg.add_user_record(pk, str(data))
# if payload fits max send size, send it
if res:
yield self.kagg.clear_and_get()
# send remain payload
yield self.kagg.clear_and_get()
from fluent.sender import FluentSender
sender = FluentSender('tag')
sender.emit('label', dict(a='data', b=dict(c='d')))
class BaseTailer(object):
def __init__(self, tag, pdir, stream_cfg, send_term,
max_send_fail, echo, encoding, lines_on_start,
max_between_data):
"""
Trailer common class initialization
Args:
tag: Classification tag for Fluentd
pdir: Position file directory
stream_cfg: Log streaming service config (Fluentd / Kinesis)
strem_cfg: Log transmission time interval
max_send_fail: Maximum number of retries in case of transmission
failure
echo: Whether to save sent messages
encoding: Original message encoding
lines_on_start: How many lines of existing log will be resent
at startup (for debugging)
max_between_data: When the service is restarted, unsent logs
smaller than this amount are sent.
"""
super(BaseTailer, self).__init__()
self.fsender = self.kclient = None
self.ksent_seqn = self.ksent_shid = None
self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail))
self.last_get_hinfo = 0
self.sname, self.saddr = self.get_host_info()
tag = "{}.{}".format(self.sname.lower() if self.sname is not None else
None, tag)
self.linfo(1, "tag: '{}'".format(tag))
self.tag = tag
self.send_term = send_term
self.last_send_try = 0
self.last_update = 0
self.kpk_cnt = 0 # count for kinesis partition key
self.pdir = pdir
max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL
tstc = type(stream_cfg)
if tstc == FluentCfg:
host, port = stream_cfg
self.fsender = FluentSender(tag, host, port,
max_send_fail=max_send_fail)
elif tstc == KinesisCfg:
stream_name, region, access_key, secret_key = stream_cfg
self.kstream_name = stream_name
self.ldebug('query_aws_client kinesis {}'.format(region))
self.kclient = query_aws_client('kinesis', region, access_key,
secret_key)
self.kagg = aggregator.RecordAggregator()
self.send_retry = 0
self.echo_file = StringIO() if echo else None
self.cache_sent_pos = {}
self.encoding = encoding
self.lines_on_start = lines_on_start if lines_on_start else 0
self.max_between_data = max_between_data if max_between_data else\
MAX_BETWEEN_DATA
def get_host_info(self):
sname = saddr = None
try:
sname = socket.gethostname()
self.linfo(" host name: {}".format(sname))
saddr = socket.gethostbyname(sname)
self.linfo(" host addr: {}".format(saddr))
except Exception as e:
self.lerror("Fail to get host info: {}".format(e))
return sname, saddr
def query_host_info(self):
invalid = (self.sname is None) or (self.saddr is None)
if invalid:
if time.time() - self.last_get_hinfo > GET_HINFO_TERM:
self.sname, self.saddr = self.get_host_info()
self.last_get_hinfo = time.time()
self.linfo(" self.sname {}, self.saddr {}".format(self.sname,
self.saddr))
return self.sname, self.saddr
def ldebug(self, tabfunc, msg=""):
_log(self, 'debug', tabfunc, msg)
def linfo(self, tabfunc, msg=""):
_log(self, 'info', tabfunc, msg)
def lwarning(self, tabfunc, msg=""):
_log(self, 'warning', tabfunc, msg)
def lerror(self, tabfunc, msg=""):
_log(self, 'error', tabfunc, msg)
def tmain(self):
cur = time.time()
self.ldebug("tmain {}".format(cur))
return cur
def read_sent_pos(self, target, con):
"""Update the sent position of the target so far.
Arguments:
target: file path for FileTailer, table name for DBTailer
con(DBConnector): DB Connection
Returns:
(position type): Parsed position type
`int` for FileTailer.
`datetime` for TableTailer.
"""
self.linfo("read_sent_pos", "updating for '{}'..".format(target))
tname = escape_path(target)
ppath = os.path.join(self.pdir, tname + '.pos')
pos = None
if os.path.isfile(ppath):
with open(ppath, 'r') as f:
pos = f.readline()
self.linfo(1, "found pos file - {}: {}".format(ppath, pos))
parsed_pos = self.parse_sent_pos(pos)
if parsed_pos is None:
self.lerror("Invalid pos file: '{}'".format(pos))
pos = None
else:
pos = parsed_pos
if pos is None:
pos = self.get_initial_pos(con)
self.linfo(1, "can't find valid pos for {}, save as "
"initial value {}".format(target, pos))
self._save_sent_pos(target, pos)
pos = self.parse_sent_pos(pos)
return pos
def _save_sent_pos(self, target, pos):
"""Save sent position for a target flie.
Args:
target: A target file for which position will be saved.
pos: Sent position to save.
"""
self.linfo(1, "_save_sent_pos for {} - {}".format(target, pos))
tname = escape_path(target)
path = os.path.join(self.pdir, tname + '.pos')
try:
with open(path, 'w') as f:
f.write("{}\n".format(pos))
except Exception as e:
self.lerror("Fail to write pos file: {} {}".format(e, path))
self.cache_sent_pos[target] = pos
def _send_newline(self, msg, msgs):
"""Send new lines
This does not send right away, but waits for a certain number of
messages to send for efficiency.
Args:
msg: A message to send
msgs: Bulk message buffer
"""
# self.ldebug("_send_newline {}".format(msg))
ts = int(time.time())
self.may_echo(msg)
msgs.append((ts, msg))
if len(msgs) >= BULK_SEND_SIZE:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
msgs[:] = []
def _send_remain_msgs(self, msgs):
"""Send bulk remain messages."""
if len(msgs) > 0:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
def _handle_send_fail(self, e, rbytes):
"""Handle send exception.
Args:
e: Exception instance
rbytes: Size of send message in bytes.
Raises:
Re-raise send exception
"""
self.lwarning(1, "send fail '{}'".format(e))
self.send_retry += 1
if self.send_retry < MAX_SEND_RETRY:
self.lerror(1, "Not exceed max retry({} < {}), will try "
"again".format(self.send_retry,
MAX_SEND_RETRY))
raise
else:
self.lerror(1, "Exceed max retry, Giving up this change({}"
" Bytes)!!".format(rbytes))
def _make_fluent_bulk(self, msgs):
"""Make bulk payload for fluentd"""
tag = '.'.join((self.tag, "data"))
bulk = [msgpack.packb((tag, ts, data)) for ts, data in msgs]
return ''.join(bulk)
def may_echo(self, line):
"""Echo sent message for debugging
Args:
line: Sent message
"""
if self.echo_file:
self.echo_file.write('{}\n'.format(line))
self.echo_file.flush()
def _kinesis_put(self, msgs):
"""Send to AWS Kinesis
Make aggregated message and send it.
Args:
msgs: Messages to send
"""
self.linfo('_kinesis_put {} messages'.format(len(msgs)))
self.kpk_cnt += 1 # round robin shards
for aggd in self._iter_kinesis_aggrec(msgs):
pk, ehk, data = aggd.get_contents()
self.linfo(" kinesis aggregated put_record: {} "
"bytes".format(len(data)))
st = time.time()
ret = self.kclient.put_record(
StreamName=self.kstream_name,
Data=data,
PartitionKey=pk,
ExplicitHashKey=ehk
)
stat = ret['ResponseMetadata']['HTTPStatusCode']
shid = ret['ShardId']
seqn = ret['SequenceNumber']
self.ksent_seqn = seqn
self.ksent_shid = shid
elp = time.time() - st
if stat == 200:
self.linfo("Kinesis put success in {}: ShardId: {}, "
"SequenceNumber: {}".format(elp, shid, seqn))
else:
self.error("Kineis put failed in {}!: "
"{}".format(elp, ret['ResponseMetadata']))
def _iter_kinesis_aggrec(self, msgs):
for msg in msgs:
data = {'tag_': self.tag + '.data', 'ts_': msg[0]}
if isinstance(msg[1], dict):
data.update(msg[1])
else:
data['value_'] = msg[1]
pk = str(uuid.uuid4())
res = self.kagg.add_user_record(pk, str(data))
# if payload fits max send size, send it
if res:
yield self.kagg.clear_and_get()
# send remain payload
yield self.kagg.clear_and_get()
def __init__( self, name, host='localhost', port=24224 ):
assert FluentSender is not None, FLUENT_IMPORT_MESSAGE
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender( self.name, host=host, port=port )
def main():
parser = argparse.ArgumentParser(
add_help=False,
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument(
'--help',
action='help',
default=argparse.SUPPRESS,
help=argparse._('show this help message and exit')
)
parser.add_argument(
'-v', '--verbose', action='store_true', help='Display debug messages'
)
parser.add_argument('-h', '--host', type=str, default='127.0.0.1',
help='Listen host')
parser.add_argument('-p', '--port', type=int, default=8080,
help='Listen port')
parser.add_argument('-w', '--workers', type=int,
default=max(1, multiprocessing.cpu_count() - 1),
help='Number of workers')
parser.add_argument('--sentry-dsn', type=str, help='Sentry DSN')
parser.add_argument('--redis-host', type=str, default='127.0.0.1',
help='Redis host')
parser.add_argument('--redis-port', type=str, default=6379,
help='Redis port')
parser.add_argument('--redis-password', type=str, default=None,
help='Redis password')
parser.add_argument('--disable-fluent', action='store_true', default=False,
help='If set, do not send logs to fluent')
parser.add_argument('--fluent-host', type=str, default='127.0.0.1',
help='Fluentd host')
parser.add_argument('--fluent-port', type=int, default=24224,
help='Fluentd port')
parser.add_argument('--auth-enabled', action='store_true', default=False,
help='Enable authentication')
parser.add_argument('--api-url', type=str, default='http://127.0.0.1:5000',
help='APITaxi URL, used when authentication is enabled to retrieve users')
args = parser.parse_args()
if args.sentry_dsn:
sentry_sdk.init(args.sentry_dsn, traces_sample_rate=1.0)
loglevel = logging.DEBUG if args.verbose else logging.INFO
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'default': {
'()': FormatWithPID,
'format': '%(asctime)s (pid %(pid)s) %(message)s'
}
},
'handlers': {
'console': {
'level': loglevel,
'class': 'logging.StreamHandler',
'formatter': 'default',
}
},
'loggers': {
'': {
'handlers': ['console'],
'level': loglevel,
}
}
})
if not args.auth_enabled:
logger.warning('Authentication is not enabled')
api_key = os.getenv('API_KEY')
if args.auth_enabled and not api_key:
parser.error('--enable-auth is set but API_KEY environment variable is not set')
if args.disable_fluent:
fluent = None
else:
fluent = FluentSender('geotaxi', host=args.fluent_host, port=args.fluent_port)
redis = Redis(
host=args.redis_host,
port=args.redis_port,
password=args.redis_password,
socket_keepalive=True,
)
worker = Worker(
redis,
fluent=fluent,
auth_enabled=args.auth_enabled, api_url=args.api_url, api_key=api_key
)
run_server(args.workers, args.host, args.port, worker)
def __init__(self, name, host='localhost', port=24224):
self.lock = threading.Lock()
self.thread_local = threading.local()
self.name = name
self.sender = FluentSender(self.name, host=host, port=port)
