def bootstrap_run():
""" temporary bootstrap """
try:
#log in stdout
log_utils.LoggerFactory.setup_simple_stdout_handler()
tailer = GEMSTailer()
tailer.tail()
sys.exit(1)
except KeyboardInterrupt:
#CTRL^C pressed so silently quit
sys.exit(0)
except Exception, e:
# report error and proceed
error_str = utils.get_exception_traceback()
log = log_utils.LoggerFactory.get_logger('bootstrap_run')
log.error("Received error %s. traceback = %s\nPlease contact your administrator." %(e, error_str))
sys.exit(1)
def bootstrap_run():
""" temporary bootstrap """
try:
#log in stdout
log_utils.LoggerFactory.setup_simple_stdout_handler()
tailer = GEMSTailer()
tailer.tail()
sys.exit(1)
except KeyboardInterrupt:
#CTRL^C pressed so silently quit
sys.exit(0)
except Exception, e:
# report error and proceed
error_str = utils.get_exception_traceback()
log = log_utils.LoggerFactory.get_logger('bootstrap_run')
log.error(
"Received error %s. traceback = %s\nPlease contact your administrator."
% (e, error_str))
sys.exit(1)
def analyze_from_tailed_file(self, a_file_paths, a_go_to_the_end = True): #pylint: disable-msg=R0912
"""
Analyze from an aggregated file containing xferlog, dirmon.log and send.log
"""
files = []
for f_path in a_file_paths:
files.append(open(f_path, 'r'))
f_iter = multitail.MultiFileTailer.tail(files, delay=0.4, go_to_the_end = a_go_to_the_end)
last_time_display = None
on_error = False
try:
#init print
self.print_on_display(self.display, None)
for (f_line, _) in f_iter:
#process only tuples (other lines should be ignored)
matched = Analyzer.expr_re.match(f_line)
if matched:
line = matched.group('line')
filename = matched.group('filename')
# sometimes the tail can eat (bug) part of the line
# ignore this exception
try:
self.analyze(self.mem_db, line, filename)
except tellicastlog_parser.InvalidTellicastlogFormatError, err:
error_str = utils.get_exception_traceback()
Analyzer.LOG.error("Parser Exception %s, traceback %s" %(err, error_str))
last_time_display = self.print_on_display(self.display, last_time_display)
kb_input = self.display.check_for_input()
if kb_input and kb_input == 'QUIT':
break # quit loop
elif kb_input and kb_input == 'STOPACCEPTING':
Analyzer.LOG.error("*********** Stop Accepting")
self._accepting_new = False
elif kb_input and kb_input == 'RESTARTACCEPTING':
Analyzer.LOG.error("*********** Restart Accepting")
self._accepting_new = True
elif kb_input and kb_input == 'OLDEST':
Analyzer.LOG.error("*********** OLDEST records first")
self._sort_criteria = 'OLDEST'
elif kb_input and kb_input == 'NEWEST':
Analyzer.LOG.error("*********** NEWEST records first")
self._sort_criteria = 'NEWEST'
else:
def test_player_with_cp():
"""
:return:
"""
#xferlog_dir = 'e:/IPPS-Data/One-Day-Replay/xferlog-lftp/xferlog'
#files = ['xferlog.dwd.txt', 'xferlog.ears.txt', 'xferlog.eps-prime.txt', 'xferlog.hrit-0.txt', 'xferlog.hrit-rss.txt', 'xferlog.other.txt',
# 'xferlog.saf.txt', 'xferlog.saflsa.txt', 'xferlog.safo3m.txt', 'xferlog.wmora1.txt', 'xferlog.wmora6.txt'
#files = ['xferlog.ears.txt']
#index_file = 'H:/index.cache'
#destination = { 'dest-dir' : 'e:/IPPS-Data/One-Day-Replay/test-dir' }
try:
conf = read_configuration_file(
"C:/Dev/ecli-workspace/rodd/etc/diss-player/cp_test.json")
j_type = None
if conf["job_type"] == "cp_job":
j_type = cp_job
elif conf["job_type"] == "scp_job":
j_type = scp_job
else:
raise Exception("Error. Unknown job type %s" % (conf["job_type"]))
player = DisseminationPlayer(conf['xferlog_dir'], conf['files'],
conf['index_file'], j_type,
conf['destination'])
player.add_jobs()
player.start()
except Exception, e:
tracebk = utils.get_exception_traceback()
print(e)
print("Traceback %s" % (tracebk))
sys.exit(1)
def test_player_with_cp():
"""
:return:
"""
#xferlog_dir = 'e:/IPPS-Data/One-Day-Replay/xferlog-lftp/xferlog'
#files = ['xferlog.dwd.txt', 'xferlog.ears.txt', 'xferlog.eps-prime.txt', 'xferlog.hrit-0.txt', 'xferlog.hrit-rss.txt', 'xferlog.other.txt',
# 'xferlog.saf.txt', 'xferlog.saflsa.txt', 'xferlog.safo3m.txt', 'xferlog.wmora1.txt', 'xferlog.wmora6.txt'
#files = ['xferlog.ears.txt']
#index_file = 'H:/index.cache'
#destination = { 'dest-dir' : 'e:/IPPS-Data/One-Day-Replay/test-dir' }
try:
conf = read_configuration_file("C:/Dev/ecli-workspace/rodd/etc/diss-player/cp_test.json")
j_type = None
if conf["job_type"] == "cp_job":
j_type = cp_job
elif conf["job_type"] == "scp_job":
j_type = scp_job
else:
raise Exception("Error. Unknown job type %s" % (conf["job_type"]))
player = DisseminationPlayer(conf['xferlog_dir'], conf['files'] , conf['index_file'], j_type, conf['destination'])
player.add_jobs()
player.start()
except Exception, e:
tracebk = utils.get_exception_traceback()
print(e)
print("Traceback %s" % (tracebk))
sys.exit(1)
Analyzer.LOG.error("*********** NEWEST records first")
self._sort_criteria = 'NEWEST'
else:
#force update
self.print_on_display(self.mem_db, self.display, None)
except KeyboardInterrupt:
#CTRL^C pressed so silently quit
sys.exit(0)
except Exception, err:
Analyzer.LOG.error("In Error")
error_str = utils.get_exception_traceback()
Analyzer.LOG.error("received error %s. Traceback = %s" %(err, error_str))
on_error = True
finally:
#whatever the case always reset the screen
self.display.reset_screen()
if on_error:
print("Exiting on error")
return 1 #pylint: disable-msg=W0150
else:
print("Exiting gracefully")
return 0 #pylint: disable-msg=W0150
if __name__ == '__main__':
j_type = cp_job
elif conf["job_type"] == "scp_job":
j_type = scp_job
else:
raise SimpleException("Error. Unknown job type %s" % (conf["job_type"]))
player = DisseminationPlayer(conf['top_data_dir'], conf['index_file'], conf['xferlog_dir'], conf['files'] , j_type, conf['destination'])
player.add_jobs()
player.start()
except SimpleException, se:
print(se)
parsed_args['parser'].die_with_usage()
except Exception, e:
tracebk = utils.get_exception_traceback()
print(e)
print("Traceback %s" % (tracebk))
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
#test_parser()
#test_index()
run_cmd()
elif conf["job_type"] == "scp_job":
j_type = scp_job
else:
raise SimpleException("Error. Unknown job type %s" %
(conf["job_type"]))
player = DisseminationPlayer(conf['top_data_dir'], conf['index_file'],
conf['xferlog_dir'], conf['files'],
j_type, conf['destination'])
player.add_jobs()
player.start()
except SimpleException, se:
print(se)
parsed_args['parser'].die_with_usage()
except Exception, e:
tracebk = utils.get_exception_traceback()
print(e)
print("Traceback %s" % (tracebk))
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
#test_parser()
#test_index()
run_cmd()
def analyze_from_tailed_file(
self,
a_file_paths,
a_go_to_the_end=True): #pylint: disable-msg=R0912
"""
Analyze from an aggregated file containing xferlog, dirmon.log and send.log
"""
files = []
for f_path in a_file_paths:
files.append(open(f_path, 'r'))
f_iter = multitail.MultiFileTailer.tail(files,
delay=0.4,
go_to_the_end=a_go_to_the_end)
last_time_display = None
on_error = False
try:
#init print
self.print_on_display(self.display, None)
for (f_line, _) in f_iter:
#process only tuples (other lines should be ignored)
matched = Analyzer.expr_re.match(f_line)
if matched:
line = matched.group('line')
filename = matched.group('filename')
# sometimes the tail can eat (bug) part of the line
# ignore this exception
try:
self.analyze(self.mem_db, line, filename)
except tellicastlog_parser.InvalidTellicastlogFormatError, err:
error_str = utils.get_exception_traceback()
Analyzer.LOG.error(
"Parser Exception %s, traceback %s" %
(err, error_str))
last_time_display = self.print_on_display(
self.display, last_time_display)
kb_input = self.display.check_for_input()
if kb_input and kb_input == 'QUIT':
break # quit loop
elif kb_input and kb_input == 'STOPACCEPTING':
Analyzer.LOG.error("*********** Stop Accepting")
self._accepting_new = False
elif kb_input and kb_input == 'RESTARTACCEPTING':
Analyzer.LOG.error("*********** Restart Accepting")
self._accepting_new = True
elif kb_input and kb_input == 'OLDEST':
Analyzer.LOG.error("*********** OLDEST records first")
self._sort_criteria = 'OLDEST'
elif kb_input and kb_input == 'NEWEST':
Analyzer.LOG.error("*********** NEWEST records first")
self._sort_criteria = 'NEWEST'
else:
Analyzer.LOG.error("*********** NEWEST records first")
self._sort_criteria = 'NEWEST'
else:
#force update
self.print_on_display(self.mem_db, self.display, None)
except KeyboardInterrupt:
#CTRL^C pressed so silently quit
sys.exit(0)
except Exception, err:
Analyzer.LOG.error("In Error")
error_str = utils.get_exception_traceback()
Analyzer.LOG.error("received error %s. Traceback = %s" %
(err, error_str))
on_error = True
finally:
#whatever the case always reset the screen
self.display.reset_screen()
if on_error:
print("Exiting on error")
return 1 #pylint: disable-msg=W0150
else:
print("Exiting gracefully")
return 0 #pylint: disable-msg=W0150
