def ftail2():
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
ptrn = "tailtest2_*-*-*.log"
ftail2 = FileTailer(bdir, ptrn, 'wdfwd.tail2', pos_dir, fcfg, send_term=0,
update_term=0, echo=True, max_between_data=100*100)
return ftail2
def _ftail():
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
ptrn = finfo['pattern']
tag = finfo['tag']
tail = FileTailer(bdir, ptrn, tag, pos_dir, fcfg,
send_term=0, update_term=0, echo=True,
max_between_data=100 * 100)
return tail
def _ktail():
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
ptrn = finfo['pattern']
tag = finfo['tag']
kcfg = KinesisCfg(KN_TEST_STREAM, 'ap-northeast-1', None, None)
tail = FileTailer(bdir, ptrn, tag, pos_dir, kcfg,
send_term=0, update_term=0, echo=True,
max_between_data=100 * 100)
return tail
def ftail_fmt():
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
ptrn = finfo['pattern']
tag = finfo['tag']
fmt = '(?P<dt_>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})'
tail = FileTailer(bdir, ptrn, tag, pos_dir, fcfg,
send_term=0, update_term=0, echo=True,
max_between_data=100 * 100, format=fmt)
return tail
def test_tail_file_startline(rmlogs):
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
ptrn = finfo['pattern']
tag = finfo['tag']
path = os.path.join(bdir, 'tailtest_2016-03-30.log')
with open(path, 'w') as f:
for i in range(100):
f.write('{}\n'.format(i))
tail = FileTailer(bdir, ptrn, tag, pos_dir, fcfg, send_term=0,
update_term=0, echo=True, lines_on_start=0)
tail.update_target(True)
assert tail.may_send_newlines() == 100
tail = FileTailer(bdir, ptrn, tag, pos_dir, fcfg, send_term=0,
update_term=0, echo=True, lines_on_start=10)
tail.update_target(True)
assert tail.may_send_newlines() == 10
def start_tailing():
linfo("start_tailing")
supress_boto3_log()
if not tailc:
ldebug("no tailing config. return")
return
for i, ti in enumerate(iter_tail_info(tailc)):
if isinstance(ti, TableTailInfo):
linfo("start table tail - {}".format(ti))
tailer = TableTailer(tailc,
ti.table,
ti.tag,
ti.pos_dir,
ti.scfg,
ti.datefmt,
ti.col_names,
send_term=ti.send_term,
encoding=ti.encoding,
lines_on_start=ti.lines_on_start,
max_between_data=ti.max_between_data,
millisec_ndigit=ti.millisec_ndigit,
key_idx=ti.key_idx,
start_key_sp=ti.start_key_sp,
latest_rows_sp=ti.latest_rows_sp)
elif isinstance(ti, FileTailInfo):
linfo("start file tail - {}".format(ti))
tailer = FileTailer(ti.bdir,
ti.ptrn,
ti.tag,
ti.pos_dir,
ti.scfg,
send_term=ti.send_term,
update_term=ti.update_term,
elatest=ti.latest,
encoding=ti.file_enc,
lines_on_start=ti.lines_on_start,
max_between_data=ti.max_between_data,
format=ti.format,
parser=ti.parser,
order_ptrn=ti.order_ptrn,
reverse_order=ti.reverse_order)
name = ti.tag
linfo("create & start {} thread".format(name))
trd = TailThread(name, tailer)
tail_threads.append(trd)
trd.start()
def test_tail_file_format(rmlogs):
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
fmt = r'\s(?P<lvl>\S+):\s(?P<_json_>.+)'
with pytest.raises(InvalidLogFormat):
FileTailer(bdir, "tailtest4_*-*-*.log", "wdfwd.tail4", pos_dir, fcfg,
echo=True, format=fmt)
fmt = r'(?P<dt_>\d+-\d+-\d+ \d+:\d+:\S+)\s(?P<lvl>\S+):\s(?P<_json_>.+)'
tail = FileTailer(bdir, "tailtest4_*-*-*.log", "wdfwd.tail4", pos_dir,
fcfg, echo=True, format=fmt)
tail.saddr = tail.sname = None
data="""2016-03-30 16:10:50.5503 INFO: {"dt_": "to-be-overwritten", "obj":[{"Delegate":{},"target0":{"returnObj":{"FirstItems":[{"ProductDisplaySeq":388}],"ProductDisplaySeq":389,"SecondItems":[{"ParentSeq":388,"ProductDisplaySeq":389},{"ParentSeq":388,"ProductDisplaySeq":461}],"ThirdItems":[],"Message":null,"Return":true,"ReturnCode":0,"TraceId":"6c8b7c6c-6c6a-4fcc-b879-72a64a4e57e5"},"parentSeq":0,"salesZone":421,"userSeq":0,"accountID":"","clientIp":"10.1.18.22"}}]}"""
path = os.path.join(bdir, 'tailtest4_2016-03-30.log')
with open(path, 'w') as f:
f.write(data)
tail.update_target(True)
assert tail.may_send_newlines() == 1
assert tail.sname is not None
assert tail.saddr is not None
echo = tail.echo_file.getvalue()
assert 'dt_' in echo
assert 'sname_' in echo
assert 'saddr_' in echo
assert 'lvl' in echo
fmt = r'(?P<dt_>\d+-\d+-\d+ \d+:\d+:\S+)\s(?P<lvl>\S+):\s(?P<_text_>.+)'
tail = FileTailer(bdir, "tailtest5_*-*-*.log", "wdfwd.tail4", pos_dir,
fcfg, echo=True, format=fmt)
data="""2016-03-30 16:10:50.5503 INFO: Plain Text Message"""
path = os.path.join(bdir, 'tailtest5_2016-03-30.log')
with open(path, 'w') as f:
f.write(data)
tail.update_target(True)
assert tail.may_send_newlines() == 1
echo = tail.echo_file.getvalue()
assert 'message' in echo
assert 'dt' in echo
assert 'lvl' in echo
def test_tail_file_elatest3(rmlogs):
"""
CASE 3: rotate elatest, reversed pre-elatest order
"""
import shutil
EXP_LATEST = 'z1_action.log'
PRE_LATEST = 'z1_action.log.1'
PRE_LATEST2 = 'z1_action.log.2'
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
epath = os.path.join(bdir, EXP_LATEST)
with open(epath, 'w') as f:
f.write('A\n')
ptrn = "z*_action.*"
ftail = FileTailer(bdir, ptrn, 'wdfwd.tail', pos_dir, fcfg, 0,
elatest=EXP_LATEST, reverse_order=True)
ftail.update_target()
rotated, psent_pos, sent_line = ftail.tmain()
assert ftail.target_path.endswith(EXP_LATEST)
assert ftail.elatest_fid is not None
assert sent_line == 1
elatest_sent_pos = ftail.get_sent_pos()
with open(epath, 'a') as f:
f.write('B\n')
f.write('C\n')
# rotate latest
pre_sent_pos = ftail.get_sent_pos()
pre_path = os.path.join(bdir, PRE_LATEST)
shutil.move(epath, pre_path)
assert os.path.isfile(pre_path)
assert not os.path.isfile(epath)
# new elatest
with open(epath, 'w') as f:
f.write('a\n')
f.write('b\n')
f.write('c\n')
time.sleep(1)
rotated, psent_pos, sent_line = ftail.tmain()
assert rotated
# pre-elatest file is target
assert ftail.target_path == pre_path
# pre-elatest sent_pos is equal to elatest's
assert psent_pos == elatest_sent_pos
# send remain 2 lines
assert sent_line == 2
# now new elatest is target
ftail.update_target()
assert ftail.target_path == epath
assert ftail.may_send_newlines() == 3
# rotate again
time.sleep(2)
pre_path2 = os.path.join(bdir, PRE_LATEST2)
shutil.move(pre_path, pre_path2)
shutil.move(epath, pre_path)
rotated = ftail.handle_elatest_rotation(cur=0)
assert rotated
ftail.update_target()
assert ftail.target_path == pre_path
def test_tail_file_elatest2(rmlogs):
"""
CASE 2: rotate elatest, new elatest, update and process
"""
import shutil
EXP_LATEST = 'exp_latest.log'
PRE_LATEST = 'tailtest3_2016-03-30.log'
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
epath = os.path.join(bdir, EXP_LATEST)
with open(epath, 'w') as f:
f.write('A\n')
ptrn = "tailtest3_*-*-*.log"
ftail = FileTailer(bdir, ptrn, 'wdfwd.tail', pos_dir, fcfg, 0,
elatest=EXP_LATEST)
ftail.update_target()
rotated, psent_pos, sent_line = ftail.tmain()
assert ftail.target_path.endswith(EXP_LATEST)
assert ftail.elatest_fid is not None
assert sent_line == 1
elatest_sent_pos = ftail.get_sent_pos()
with open(epath, 'a') as f:
f.write('B\n')
f.write('C\n')
# rotate latest
pre_sent_pos = ftail.get_sent_pos()
pre_path = os.path.join(bdir, PRE_LATEST)
shutil.move(epath, pre_path)
assert os.path.isfile(pre_path)
assert not os.path.isfile(epath)
# new elatest
with open(epath, 'w') as f:
f.write('a\n')
f.write('b\n')
f.write('c\n')
time.sleep(1)
rotated, psent_pos, sent_line = ftail.tmain()
assert rotated
# pre-elatest file is target
assert ftail.target_path == pre_path
# pre-elatest sent_pos is equal to elatest's
assert psent_pos == elatest_sent_pos
# send remain 2 lines
assert sent_line == 2
# now new elatest is target
ftail.update_target()
assert ftail.target_path == epath
assert ftail.may_send_newlines() == 3
def test_tail_file_elatest1(rmlogs):
"""
CASE 1: rotate elatest, process and new elatest
"""
EXP_LATEST = 'exp_latest.log'
PRE_LATEST = 'tailtest3_2016-03-30.9.log'
PRE_LATEST2 = 'tailtest3_2016-03-30.10.log'
finfo = tcfg['from'][0]['file']
bdir = finfo['dir']
epath = os.path.join(bdir, EXP_LATEST)
with open(epath, 'w') as f:
f.write('A\n')
ptrn = "tailtest3_*-*-*.log"
ftail = FileTailer(bdir, ptrn, 'wdfwd.tail', pos_dir, fcfg, 0,
elatest=EXP_LATEST,
order_ptrn=r'(?P<date>[^\.]+)\.(?P<order>\d+)\.log')
ftail.update_target()
rotated = ftail.handle_elatest_rotation()
assert not rotated
epath, efid = ftail.get_elatest_info()
assert epath and efid
assert ftail.target_path.endswith(EXP_LATEST)
assert ftail.elatest_fid is not None
# update target again
ftail.update_target()
assert ftail.target_path.endswith(EXP_LATEST)
assert ftail.may_send_newlines() == 1
elatest_sent_pos = ftail.get_sent_pos()
with open(epath, 'a') as f:
f.write('B\n')
f.write('C\n')
### CASE 1: rotate elatest, process and new elatest
## rotate latest
pre_sent_pos = ftail.get_sent_pos()
pre_path = os.path.join(bdir, PRE_LATEST)
shutil.move(epath, pre_path)
assert os.path.isfile(pre_path)
assert not os.path.isfile(epath)
## check elatest rotation
rotated = ftail.handle_elatest_rotation(cur=0)
assert rotated
# now elatest fid is None
assert ftail.elatest_fid is None
ftail.update_target()
# pre-elatest file is target
assert ftail.target_path == pre_path
# pre-elatest sent_pos is equal to elatest's
assert ftail.get_sent_pos() == elatest_sent_pos
# elatest sent_pos is cleared
assert ftail.get_sent_pos(epath) == 0
# send remain 2 lines
assert ftail.may_send_newlines() == 2
# new elatest
with open(epath, 'w') as f:
f.write('a\n')
f.write('b\n')
f.write('c\n')
rotated = ftail.handle_elatest_rotation(cur=0)
assert rotated == False
ftail.update_target()
assert ftail.target_path.endswith(EXP_LATEST)
epath, efid = ftail.get_elatest_info()
assert epath and efid
assert ftail.get_sent_pos() == 0
assert ftail.may_send_newlines() == 3
# rotate again
time.sleep(2)
pre_path2 = os.path.join(bdir, PRE_LATEST2)
shutil.move(epath, pre_path2)
rotated = ftail.handle_elatest_rotation(cur=0)
assert rotated
ftail.update_target()
assert ftail.target_path == pre_path2