def get_stack_dump():
id2name = dict([(th.ident, th.name) for th in threading_enumerate()])
code = ["Stack dump:"]
for threadId, stack in sys._current_frames().items():
code.append("")
code.append("# Thread: %s(%d)" % (id2name.get(threadId,""), threadId))
for filename, lineno, name, line in traceback.extract_stack(stack):
code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
if line:
code.append(" %s" % (line.strip()))
return code
def get_stack_dump():
id2name = dict([(th.ident, th.name) for th in threading_enumerate()])
code = ["Stack dump:"]
for threadId, stack in sys._current_frames().items():
code.append("")
code.append("# Thread: %s(%d)" % (id2name.get(threadId,""), threadId))
for filename, lineno, name, line in traceback.extract_stack(stack):
code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
if line:
code.append(" %s" % (line.strip()))
return code
def init_main():
"""Main initializator for catching exceptions"""
try:
#Call the main()
main_log = get_logger("main")
error_flag = Event()
result = controlled_exception(main, main_log, error_flag, True)
main_log.debug("main() return: '%s'" % result)
main_log.debug("Active Threads: %s" % threading_enumerate())
unreachable = gc.collect()
main_log.debug("Garbage Collector stats: unreachable %s, total %s, list %s" % (unreachable, len(gc.garbage), gc.garbage))
if len(gc.garbage):
for target in gc.garbage:
main_log.warning("target %s, referrers %s, referents %s" % (target, gc.get_referrers(target), gc.get_referents(target)))
logging.shutdown()
except: # pylint: disable=W0702
print_exc()
print("Error ocurred at init_main")
def _i_am_orphan(self): current = currentThread() for thread in threading_enumerate(): if not thread.isDaemon() and (thread is not current): return False return True
def main():
# Initialize logging
logger = logging.getLogger(LOGGER_NAME)
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s %(name)-4s %(levelname)-4s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.info('starting up')
# Load global configs
load_config()
logger.debug('config.json loaded')
try:
if sys.argv[1] == 'client':
logger.debug('entering client mode')
cli()
exit()
except IndexError:
pass
# Check if requirements have changed and if so, update
req_updater = RequirementsUpdater()
if not req_updater.check():
logger.info('requirements have change since last time, updating')
if not req_updater.update():
logger.error('requirements update FAILED!')
else:
logger.info('restarting in order to apply new updates')
runtime_restart()
# Load all profiles
logger.info('Loading profiles')
load_profiles()
logger.info('Profiles loaded')
# Initialize database
Database.init(os.path.join(CONFIG_DIR, 'db.db'))
logger.info('Blocking all IPs saved in database')
IPBlocker.init()
IPBlocker.block_all_banned()
# Check for updates and perform automatic update if enabled and available
Updater.init()
update_available = Updater.update_available()
logger.info('You are up to date' if not update_available else
'There is another version on the server: %s (you have %s)' %
(Updater.get_latest_name(), VERSION_TAG))
if update_available and CONFIG.get('updater', {}).get('autoupdate', False):
Updater.update()
# Load online config
if os.path.isfile(os.path.join(CONFIG_DIR, 'server.json')):
logger.debug('loading online config')
with open(os.path.join(CONFIG_DIR, 'server.json'), 'r') as f:
ONLINE_DATA.update(json.load(f))
if ONLINE_DATA['loggedIn']:
init_online()
# Start scanning of the logs
ThreadScanner().start()
# Terminate the program when CTRL+C is pressed
while AppRunning.is_running():
try:
AppRunning.sleep_while_running(10)
except KeyboardInterrupt:
AppRunning.set_running(False)
while True:
print('waiting on all threads to terminate...')
if len(threading_enumerate()) == 1:
print('ok, bye')
break
print('%d left' % len(threading_enumerate()), threading_enumerate())
time.sleep(1)
exit(AppRunning.exit_code[0])
def _main_thread_alive():
return any((i.name == "MainThread") and i.is_alive()
for i in threading_enumerate())
def _main_thread_alive():
return any(
(i.name == "MainThread") and i.is_alive() for i in
threading_enumerate())
def upload_task(values):
"""
Execute either a new upload task or resume last unsuccessful upload task
from the DB.
This method returns true when all file upload tasks are executed and
postprocess and error queues are emptied.
Return: False is the upload is not executed due to an existing ongoing task.
"""
global ongoing_upload_task
global input_csv_error
if (ongoing_upload_task and
ongoing_upload_task.get_status() != BatchUploadTask.STATUS_FINISHED):
# Ongoing task exists
return False
try:
ongoing_upload_task = BatchUploadTask()
ongoing_upload_task.set_status(BatchUploadTask.STATUS_RUNNING)
postprocess_queue = ongoing_upload_task.postprocess_queue
def _postprocess(func=None, *args):
func and func(*args)
postprocess_thread = QueueFunctionThread(postprocess_queue, _postprocess)
postprocess_thread.start()
def _error(item):
logger.error(item)
# error_thread is a new thread logging the errors.
error_queue = ongoing_upload_task.error_queue
error_thread = QueueFunctionThread(error_queue, _error)
error_thread.start()
# Multi-threaded from here.
try:
_upload_images(ongoing_upload_task, values)
except (ClientException, IOError):
error_queue.put(str(IOError))
while not postprocess_queue.empty():
sleep(0.2)
postprocess_thread.abort = True
while postprocess_thread.isAlive():
postprocess_thread.join(0.01)
try:
_upload_csv(_get_conn())
if (ongoing_upload_task.get_fails() == 0
and ongoing_upload_task.csv_uploaded()): # All done.
ongoing_upload_task.batch.finish_time = datetime.now()
model.commit()
except (ClientException, IOError):
error_queue.put(str(IOError))
while not error_queue.empty():
sleep(0.01)
error_thread.abort = True
while error_thread.isAlive():
error_thread.join(0.01)
logger.info("Upload task execution completed.")
except InputCSVException as e:
logger.debug("Input CSV File error ")
input_csv_error = True
except (SystemExit, Exception) as ex:
logger.error("Error happens in _upload: %s" %ex)
logger.error("Aborting all threads...")
for thread in threading_enumerate():
thread.abort = True
raise
finally:
# Reset of singleton task in the module.
ongoing_upload_task.set_status(BatchUploadTask.STATUS_FINISHED)
def check_thread_tasks_and_restart(
need_to_be_monitored_thread_tasks_info_list: list,
sleep_time=30,
logger=None,
):
"""
监控线程tasks状态, 挂掉的进行重启
simple use:
def print_num(num):
sleep(5.)
print(num)
class TestThread(Thread):
def __init__(self, args: (list, tuple)=()):
super(TestThread, self).__init__()
self.args = args
def run(self):
sleep(5.)
print(self.args[0])
# 任务信息列表 eg: [{'thread_name': 'thread_task:print_ip:is_class_False:21113cf4-cc69-11e9-bdef-68fef70d1e6e', 'func_args': [xxx,]}, 'is_class': False]
# 存储所有需要监控并重启的初始化线程对象list
need_to_be_monitored_thread_tasks_info_list = []
tasks = []
# 函数类型的
for num in range(0, 3):
func_args = [
num,
]
t = Thread(
target=print_num,
args=func_args)
thread_task_name = 'thread_task:{}:{}'.format(
'print_ip',
get_uuid1())
t.setName(thread_task_name)
tasks.append(t)
need_to_be_monitored_thread_tasks_info_list.append({
'func_name': print_num,
'thread_name': thread_task_name,
'func_args': func_args,
'is_class': False, # 是否是类, 函数为False
})
# 继承自Thread的类
for num in range(3, 4):
func_args = [
num,
]
task = TestThread(args=func_args)
thread_task_name = 'thread_task:{}:{}'.format(
'TestThread',
get_uuid1(),)
task.setName(thread_task_name)
tasks.append(task)
need_to_be_monitored_thread_tasks_info_list.append({
'func_name': TestThread,
'thread_name': thread_task_name,
'func_args': func_args,
'is_class': True,
})
for t in tasks:
t.start()
# pprint(need_to_be_monitored_thread_tasks_info_list)
# 用来检测是否有线程down并重启down线程
check_thread_task = Thread(
target=check_thread_tasks_and_restart,
args=(
need_to_be_monitored_thread_tasks_info_list,
6,
))
check_thread_task.setName('thread_task:check_thread_task_and_restart')
check_thread_task.start()
:param need_to_be_monitored_thread_tasks_info_list: 需要监控并重启的线程对象list, eg: [{'func_name': do_something, 'thread_name': 'thread_task:print_ip:21113cf4-cc69-11e9-bdef-68fef70d1e6e', 'func_args': [xxx,], 'is_class': False},], is_class为是否是类(继承自Thread)
:param sleep_time:
:param logger:
:return:
"""
while True:
# 获取当前线程名list
try:
now_threads_name_list = [
thread_obj.getName() for thread_obj in threading_enumerate()
]
# pprint(now_threads_name_list)
except Exception as e:
_print(msg='遇到错误:', logger=logger, log_level=2, exception=e)
continue
for thread_obj_info in need_to_be_monitored_thread_tasks_info_list:
thread_name = thread_obj_info['thread_name']
func_args = thread_obj_info['func_args']
func_name = thread_obj_info['func_name']
is_class = thread_obj_info['is_class']
# _print(msg='thread_name: {}'.format(thread_name), logger=logger)
if thread_name in now_threads_name_list:
# 当前某线程名包含在初始化线程组中，可以认为线程仍在运行
continue
else:
_print(
msg='thread_name: {} stopped, now restart!'.format(
thread_name),
logger=logger,
)
# 报错: RuntimeError: threads can only be started once
# 每个线程对象只能运行一次, 所以每次得重新thread_obj = Thread()线程对象, 然后thread_obj.start()
# 因此必须把相关运行参数传进来
if not is_class:
thread_task = Thread(
target=func_name,
name=thread_name,
args=func_args,
)
else:
# print(func_name)
# 此处继承自类的格式必须为
"""
eg:
class TestThread(Thread):
def __init__(self, args: (list, tuple) = ()):
super(TestThread, self).__init__()
self.args = args
def run(self):
pass
"""
thread_task = func_name(args=func_args)
thread_task.setName(name=thread_name)
thread_task.start()
sleep(sleep_time)
