class BaseTailer(object):
def __init__(self, tag, pdir, stream_cfg, send_term,
max_send_fail, echo, encoding, lines_on_start,
max_between_data):
"""
Trailer common class initialization
Args:
tag: Classification tag for Fluentd
pdir: Position file directory
stream_cfg: Log streaming service config (Fluentd / Kinesis)
strem_cfg: Log transmission time interval
max_send_fail: Maximum number of retries in case of transmission
failure
echo: Whether to save sent messages
encoding: Original message encoding
lines_on_start: How many lines of existing log will be resent
at startup (for debugging)
max_between_data: When the service is restarted, unsent logs
smaller than this amount are sent.
"""
super(BaseTailer, self).__init__()
self.fsender = self.kclient = None
self.ksent_seqn = self.ksent_shid = None
self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail))
self.last_get_hinfo = 0
self.sname, self.saddr = self.get_host_info()
tag = "{}.{}".format(self.sname.lower() if self.sname is not None else
None, tag)
self.linfo(1, "tag: '{}'".format(tag))
self.tag = tag
self.send_term = send_term
self.last_send_try = 0
self.last_update = 0
self.kpk_cnt = 0 # count for kinesis partition key
self.pdir = pdir
max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL
tstc = type(stream_cfg)
if tstc == FluentCfg:
host, port = stream_cfg
self.fsender = FluentSender(tag, host, port,
max_send_fail=max_send_fail)
elif tstc == KinesisCfg:
stream_name, region, access_key, secret_key = stream_cfg
self.kstream_name = stream_name
self.ldebug('query_aws_client kinesis {}'.format(region))
self.kclient = query_aws_client('kinesis', region, access_key,
secret_key)
self.kagg = aggregator.RecordAggregator()
self.send_retry = 0
self.echo_file = StringIO() if echo else None
self.cache_sent_pos = {}
self.encoding = encoding
self.lines_on_start = lines_on_start if lines_on_start else 0
self.max_between_data = max_between_data if max_between_data else\
MAX_BETWEEN_DATA
def get_host_info(self):
sname = saddr = None
try:
sname = socket.gethostname()
self.linfo(" host name: {}".format(sname))
saddr = socket.gethostbyname(sname)
self.linfo(" host addr: {}".format(saddr))
except Exception as e:
self.lerror("Fail to get host info: {}".format(e))
return sname, saddr
def query_host_info(self):
invalid = (self.sname is None) or (self.saddr is None)
if invalid:
if time.time() - self.last_get_hinfo > GET_HINFO_TERM:
self.sname, self.saddr = self.get_host_info()
self.last_get_hinfo = time.time()
self.linfo(" self.sname {}, self.saddr {}".format(self.sname,
self.saddr))
return self.sname, self.saddr
def ldebug(self, tabfunc, msg=""):
_log(self, 'debug', tabfunc, msg)
def linfo(self, tabfunc, msg=""):
_log(self, 'info', tabfunc, msg)
def lwarning(self, tabfunc, msg=""):
_log(self, 'warning', tabfunc, msg)
def lerror(self, tabfunc, msg=""):
_log(self, 'error', tabfunc, msg)
def tmain(self):
cur = time.time()
self.ldebug("tmain {}".format(cur))
return cur
def read_sent_pos(self, target, con):
"""Update the sent position of the target so far.
Arguments:
target: file path for FileTailer, table name for DBTailer
con(DBConnector): DB Connection
Returns:
(position type): Parsed position type
`int` for FileTailer.
`datetime` for TableTailer.
"""
self.linfo("read_sent_pos", "updating for '{}'..".format(target))
tname = escape_path(target)
ppath = os.path.join(self.pdir, tname + '.pos')
pos = None
if os.path.isfile(ppath):
with open(ppath, 'r') as f:
pos = f.readline()
self.linfo(1, "found pos file - {}: {}".format(ppath, pos))
parsed_pos = self.parse_sent_pos(pos)
if parsed_pos is None:
self.lerror("Invalid pos file: '{}'".format(pos))
pos = None
else:
pos = parsed_pos
if pos is None:
pos = self.get_initial_pos(con)
self.linfo(1, "can't find valid pos for {}, save as "
"initial value {}".format(target, pos))
self._save_sent_pos(target, pos)
pos = self.parse_sent_pos(pos)
return pos
def _save_sent_pos(self, target, pos):
"""Save sent position for a target flie.
Args:
target: A target file for which position will be saved.
pos: Sent position to save.
"""
self.linfo(1, "_save_sent_pos for {} - {}".format(target, pos))
tname = escape_path(target)
path = os.path.join(self.pdir, tname + '.pos')
try:
with open(path, 'w') as f:
f.write("{}\n".format(pos))
except Exception as e:
self.lerror("Fail to write pos file: {} {}".format(e, path))
self.cache_sent_pos[target] = pos
def _send_newline(self, msg, msgs):
"""Send new lines
This does not send right away, but waits for a certain number of
messages to send for efficiency.
Args:
msg: A message to send
msgs: Bulk message buffer
"""
# self.ldebug("_send_newline {}".format(msg))
ts = int(time.time())
self.may_echo(msg)
msgs.append((ts, msg))
if len(msgs) >= BULK_SEND_SIZE:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
msgs[:] = []
def _send_remain_msgs(self, msgs):
"""Send bulk remain messages."""
if len(msgs) > 0:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
def _handle_send_fail(self, e, rbytes):
"""Handle send exception.
Args:
e: Exception instance
rbytes: Size of send message in bytes.
Raises:
Re-raise send exception
"""
self.lwarning(1, "send fail '{}'".format(e))
self.send_retry += 1
if self.send_retry < MAX_SEND_RETRY:
self.lerror(1, "Not exceed max retry({} < {}), will try "
"again".format(self.send_retry,
MAX_SEND_RETRY))
raise
else:
self.lerror(1, "Exceed max retry, Giving up this change({}"
" Bytes)!!".format(rbytes))
def _make_fluent_bulk(self, msgs):
"""Make bulk payload for fluentd"""
tag = '.'.join((self.tag, "data"))
bulk = [msgpack.packb((tag, ts, data)) for ts, data in msgs]
return ''.join(bulk)
def may_echo(self, line):
"""Echo sent message for debugging
Args:
line: Sent message
"""
if self.echo_file:
self.echo_file.write('{}\n'.format(line))
self.echo_file.flush()
def _kinesis_put(self, msgs):
"""Send to AWS Kinesis
Make aggregated message and send it.
Args:
msgs: Messages to send
"""
self.linfo('_kinesis_put {} messages'.format(len(msgs)))
self.kpk_cnt += 1 # round robin shards
for aggd in self._iter_kinesis_aggrec(msgs):
pk, ehk, data = aggd.get_contents()
self.linfo(" kinesis aggregated put_record: {} "
"bytes".format(len(data)))
st = time.time()
ret = self.kclient.put_record(
StreamName=self.kstream_name,
Data=data,
PartitionKey=pk,
ExplicitHashKey=ehk
)
stat = ret['ResponseMetadata']['HTTPStatusCode']
shid = ret['ShardId']
seqn = ret['SequenceNumber']
self.ksent_seqn = seqn
self.ksent_shid = shid
elp = time.time() - st
if stat == 200:
self.linfo("Kinesis put success in {}: ShardId: {}, "
"SequenceNumber: {}".format(elp, shid, seqn))
else:
self.error("Kineis put failed in {}!: "
"{}".format(elp, ret['ResponseMetadata']))
def _iter_kinesis_aggrec(self, msgs):
for msg in msgs:
data = {'tag_': self.tag + '.data', 'ts_': msg[0]}
if isinstance(msg[1], dict):
data.update(msg[1])
else:
data['value_'] = msg[1]
pk = str(uuid.uuid4())
res = self.kagg.add_user_record(pk, str(data))
# if payload fits max send size, send it
if res:
yield self.kagg.clear_and_get()
# send remain payload
yield self.kagg.clear_and_get()
class BaseTailer(object):
def __init__(self, tag, pdir, stream_cfg, send_term,
max_send_fail, echo, encoding, lines_on_start,
max_between_data):
"""
Trailer common class initialization
Args:
tag: Classification tag for Fluentd
pdir: Position file directory
stream_cfg: Log streaming service config (Fluentd / Kinesis)
strem_cfg: Log transmission time interval
max_send_fail: Maximum number of retries in case of transmission
failure
echo: Whether to save sent messages
encoding: Original message encoding
lines_on_start: How many lines of existing log will be resent
at startup (for debugging)
max_between_data: When the service is restarted, unsent logs
smaller than this amount are sent.
"""
super(BaseTailer, self).__init__()
self.fsender = self.kclient = None
self.ksent_seqn = self.ksent_shid = None
self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail))
self.last_get_hinfo = 0
self.sname, self.saddr = self.get_host_info()
tag = "{}.{}".format(self.sname.lower() if self.sname is not None else
None, tag)
self.linfo(1, "tag: '{}'".format(tag))
self.tag = tag
self.send_term = send_term
self.last_send_try = 0
self.last_update = 0
self.kpk_cnt = 0 # count for kinesis partition key
self.pdir = pdir
max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL
tstc = type(stream_cfg)
if tstc == FluentCfg:
host, port = stream_cfg
self.fsender = FluentSender(tag, host, port,
max_send_fail=max_send_fail)
elif tstc == KinesisCfg:
stream_name, region, access_key, secret_key = stream_cfg
self.kstream_name = stream_name
self.ldebug('query_aws_client kinesis {}'.format(region))
self.kclient = query_aws_client('kinesis', region, access_key,
secret_key)
self.kagg = aggregator.RecordAggregator()
self.send_retry = 0
self.echo_file = StringIO() if echo else None
self.cache_sent_pos = {}
self.encoding = encoding
self.lines_on_start = lines_on_start if lines_on_start else 0
self.max_between_data = max_between_data if max_between_data else\
MAX_BETWEEN_DATA
def get_host_info(self):
sname = saddr = None
try:
sname = socket.gethostname()
self.linfo(" host name: {}".format(sname))
saddr = socket.gethostbyname(sname)
self.linfo(" host addr: {}".format(saddr))
except Exception as e:
self.lerror("Fail to get host info: {}".format(e))
return sname, saddr
def query_host_info(self):
invalid = (self.sname is None) or (self.saddr is None)
if invalid:
if time.time() - self.last_get_hinfo > GET_HINFO_TERM:
self.sname, self.saddr = self.get_host_info()
self.last_get_hinfo = time.time()
self.linfo(" self.sname {}, self.saddr {}".format(self.sname, self.saddr))
return self.sname, self.saddr
def ldebug(self, tabfunc, msg=""):
_log(self, 'debug', tabfunc, msg)
def linfo(self, tabfunc, msg=""):
_log(self, 'info', tabfunc, msg)
def lwarning(self, tabfunc, msg=""):
_log(self, 'warning', tabfunc, msg)
def lerror(self, tabfunc, msg=""):
_log(self, 'error', tabfunc, msg)
def tmain(self):
cur = time.time()
self.ldebug("tmain {}".format(cur))
return cur
def read_sent_pos(self, target, con):
"""Update the sent position of the target so far.
Arguments:
target: file path for FileTailer, table name for DBTailer
con(DBConnector): DB Connection
Returns:
(position type): Parsed position type
`int` for FileTailer.
`datetime` for TableTailer.
"""
self.linfo("read_sent_pos", "updating for '{}'..".format(target))
tname = escape_path(target)
ppath = os.path.join(self.pdir, tname + '.pos')
pos = None
if os.path.isfile(ppath):
with open(ppath, 'r') as f:
pos = f.readline()
self.linfo(1, "found pos file - {}: {}".format(ppath, pos))
parsed_pos = self.parse_sent_pos(pos)
if parsed_pos is None:
self.lerror("Invalid pos file: '{}'".format(pos))
pos = None
else:
pos = parsed_pos
if pos is None:
pos = self.get_initial_pos(con)
self.linfo(1, "can't find valid pos for {}, save as "
"initial value {}".format(target, pos))
self._save_sent_pos(target, pos)
pos = self.parse_sent_pos(pos)
return pos
def _save_sent_pos(self, target, pos):
"""Save sent position for a target flie.
Args:
target: A target file for which position will be saved.
pos: Sent position to save.
"""
self.linfo(1, "_save_sent_pos for {} - {}".format(target, pos))
tname = escape_path(target)
path = os.path.join(self.pdir, tname + '.pos')
try:
with open(path, 'w') as f:
f.write("{}\n".format(pos))
except Exception as e:
self.lerror("Fail to write pos file: {} {}".format(e, path))
self.cache_sent_pos[target] = pos
def _send_newline(self, msg, msgs):
"""Send new lines
This does not send right away, but waits for a certain number of
messages to send for efficiency.
Args:
msg: A message to send
msgs: Bulk message buffer
"""
# self.ldebug("_send_newline {}".format(msg))
ts = int(time.time())
self.may_echo(msg)
msgs.append((ts, msg))
if len(msgs) >= BULK_SEND_SIZE:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
msgs[:] = []
def _send_remain_msgs(self, msgs):
"""Send bulk remain messages."""
if len(msgs) > 0:
if self.fsender:
bytes_ = self._make_fluent_bulk(msgs)
self.fsender._send(bytes_)
elif self.kclient:
self._kinesis_put(msgs)
def _handle_send_fail(self, e, rbytes):
"""Handle send exception.
Args:
e: Exception instance
rbytes: Size of send message in bytes.
Raises:
Re-raise send exception
"""
self.lwarning(1, "send fail '{}'".format(e))
self.send_retry += 1
if self.send_retry < MAX_SEND_RETRY:
self.lerror(1, "Not exceed max retry({} < {}), will try "
"again".format(self.send_retry,
MAX_SEND_RETRY))
raise
else:
self.lerror(1, "Exceed max retry, Giving up this change({}"
" Bytes)!!".format(rbytes))
def _make_fluent_bulk(self, msgs):
"""Make bulk payload for fluentd"""
tag = '.'.join((self.tag, "data"))
bulk = [msgpack.packb((tag, ts, data)) for ts, data in msgs]
return ''.join(bulk)
def may_echo(self, line):
"""Echo sent message for debugging
Args:
line: Sent message
"""
if self.echo_file:
self.echo_file.write('{}\n'.format(line))
self.echo_file.flush()
def _kinesis_put(self, msgs):
"""Send to AWS Kinesis
Make aggregated message and send it.
Args:
msgs: Messages to send
"""
self.linfo('_kinesis_put {} messages'.format(len(msgs)))
self.kpk_cnt += 1 # round robin shards
for aggd in self._iter_kinesis_aggrec(msgs):
pk, ehk, data = aggd.get_contents()
self.linfo(" kinesis aggregated put_record: {} "
"bytes".format(len(data)))
st = time.time()
ret = self.kclient.put_record(
StreamName=self.kstream_name,
Data=data,
PartitionKey=pk,
ExplicitHashKey=ehk
)
stat = ret['ResponseMetadata']['HTTPStatusCode']
shid = ret['ShardId']
seqn = ret['SequenceNumber']
self.ksent_seqn = seqn
self.ksent_shid = shid
elp = time.time() - st
if stat == 200:
self.linfo("Kinesis put success in {}: ShardId: {}, "
"SequenceNumber: {}".format(elp, shid, seqn))
else:
self.error("Kineis put failed in {}!: "
"{}".format(elp, ret['ResponseMetadata']))
def _iter_kinesis_aggrec(self, msgs):
for msg in msgs:
data = {'tag_': self.tag + '.data', 'ts_': msg[0]}
if isinstance(msg[1], dict):
data.update(msg[1])
else:
data['value_'] = msg[1]
pk = str(uuid.uuid4())
res = self.kagg.add_user_record(pk, str(data))
# if payload fits max send size, send it
if res:
yield self.kagg.clear_and_get()
# send remain payload
yield self.kagg.clear_and_get()
