def start_task(self, task):
self.log_reader = InstallLogReader(path=self.core.settings.tmp_logs_path, task_id=str(task.id))
common_log = self.log_reader.common_working_log()
self.logger.addHandler(TaskDbLogHandler(task))
f_out = open(common_log, 'w')
self.process = subprocess.Popen(
(
sys.executable,
'-u',
sys.argv[0],
'--task-work={0}'.format(task.id),
),
stderr=f_out,
stdout=f_out
)
print("start child task with pid %i" % self.process.pid)
task.parent_pid = os.getpid()
task.pid = self.process.pid
task.save()
self.configs["task"] = task
self.timer_check_logdir = PeriodicCallback(lambda: TornadoWorker.periodic_check_logdir(self), 700)
self.timer_read_log = PeriodicCallback(lambda: TornadoWorker.periodic_read_logs(self), 300)
self.timer_read_log.start()
self.timer_check_logdir.start()
class TornadoWorker(object):
def __init__(self, *args):
self.web_frontend_url = "http://127.0.0.1:7799/api/1/core/sync/"
self.process = None
self.logger = None
self.timer_read_log = None
self.timer_check_logdir = None
self.check_new_tasks = None
self.decoder = OutputDecoder()
# TODO remove port from code
self.port = 7798
self.logger = logging.getLogger()
self.log_reader = None
# remove all handlers
self.logger.handlers = []
# add only db handler
self.core = Core.get_instance()
self.configs = {
'max_messages_per': 100,
"queue": Queue(),
'buffer': 1,
'sessions': {},
"state": "reading",
'status_done': False,
'task': None
}
self.application = tornado.web.Application([
(r"/socket/log", WebSocketHandler, {"configs": self.configs}),
(r"/test", MainHandler, {"configs": self.configs})
])
# check weather the child process is working now
def wait_worker_process(self):
logging.debug('thread for waiting of worker process started')
if self.process:
if self.process.poll() is not None:
logging.debug("process is finished")
self.process = None
# stop interaval timers
self.timer_read_log.stop()
self.timer_check_logdir.stop()
self.logger.info('install worker process is finished check last lines from logs')
self.sync_core_frontend()
TornadoWorker.periodic_read_logs(self)
def sync_core_frontend(self):
print(self.web_frontend_url)
urllib.request.urlopen(self.web_frontend_url)
# start eventloop, webserver and periodic reading
def start(self):
self.application.listen(self.port)
main_loop = tornado.ioloop.IOLoop.instance()
self.check_new_tasks = PeriodicCallback(lambda: TornadoWorker.periodic_check_new_tasks(self), 2000)
self.check_new_tasks.start()
main_loop.start()
# delete locks after work
@staticmethod
def clear_lock(key):
try:
obj = Locks.get(Locks.title == key)
obj.delete_instance()
return True
except Locks.DoesNotExist:
return False
# try to start locking of a task
@staticmethod
def lock_obj(key, val):
try:
lk = Locks()
lk.title = key
lk.value = val
lk.save()
return val
except BaseException:
obj = Locks.get(Locks.title == key)
return obj.value
# just check lock on given key
@staticmethod
def get_lock(key):
try:
obj = Locks.get(Locks.title == key)
return obj.value
except Locks.DoesNotExist:
return False
# make task fail
@staticmethod
def task2fail(task_id):
try:
obj = Task.get(Task.id == task_id)
obj.status = Task.STATUS_FAILED
obj.save()
except Task.DoesNotExist:
return True
# check if task is alive
@staticmethod
def is_alive_task(task_id, core):
try:
obj = Task.get(Task.id == task_id)
# may be wrong
if obj.status == Task.STATUS_EXCEPTION:
return False
if obj.status == Task.STATUS_FAILED:
return False
if obj.status == Task.STATUS_DONE:
return False
print("find possible alive task %i pid is %i" % (task_id, obj.pid) )
pid = obj.pid
return core.api.os.shell.is_alive(pid)
except Task.DoesNotExist:
return False
# check weather the running task is zombie
# may be there are some sense to do like decorator
def check_zombie(self, task_id):
try:
self.process_check_new_tasks()
except BaseException:
print(sys.exc_info())
# checking for new tasks in database
# may be there are some sense to do like decorator
def periodic_check_new_tasks(self):
# try:
self.process_check_new_tasks()
# except BaseException:
# print(sys.exc_info())
# check the lock and failed tasks
def check_free4run(self):
ret = TornadoWorker.get_lock(Task.LOCK)
if not ret:
return True
else:
task_id = int(ret)
if TornadoWorker.is_alive_task(task_id, self.core):
print("task is really working we will wait")
return False
else:
print("task to fail %i" % task_id)
TornadoWorker.task2fail(task_id)
TornadoWorker.clear_lock(Task.LOCK)
# TODO add clearing all task with the same session
return True
# process tasks to work
def process_check_new_tasks(self):
# User.select().where(User.active == True).order_by(User.username)
print("i'm checking a new tasks")
query = Task.select().where(Task.status == Task.STATUS_PENDING).order_by(Task.id)
if not self.check_free4run():
print("lock is busy")
return False
# race condition ?????
for task in query:
print("i have found a new task %s" % str(task))
if TornadoWorker.lock_obj(Task.LOCK, str(task.id)):
# if it failed ?
self.start_task(task)
return True
else:
return False
# start working on task
def task(self, task):
self.log_reader = InstallLogReader(path=self.core.settings.tmp_logs_path, task_id=str(task.id))
common_log = self.log_reader.common_working_log()
self.logger.addHandler(TaskDbLogHandler(task))
f_out = open(common_log, 'w')
self.process = subprocess.Popen(
(
sys.executable,
'-u',
sys.argv[0],
'--task-work={0}'.format(task.id),
),
stderr=f_out,
stdout=f_out
)
print("start child task with pid %i" % self.process.pid)
task.parent_pid = os.getpid()
task.pid = self.process.pid
task.save()
self.configs["task"] = task
self.timer_check_logdir = PeriodicCallback(lambda: TornadoWorker.periodic_check_logdir(self), 700)
self.timer_read_log = PeriodicCallback(lambda: TornadoWorker.periodic_read_logs(self), 300)
self.timer_read_log.start()
self.timer_check_logdir.start()
# start working on task
def start_task(self, task):
self.log_reader = InstallLogReader(path=self.core.settings.tmp_logs_path, task_id=str(task.id))
common_log = self.log_reader.common_working_log()
self.logger.addHandler(TaskDbLogHandler(task))
f_out = open(common_log, 'w')
self.process = subprocess.Popen(
(
sys.executable,
'-u',
sys.argv[0],
'--task-work={0}'.format(task.id),
),
stderr=f_out,
stdout=f_out
)
print("start child task with pid %i" % self.process.pid)
task.parent_pid = os.getpid()
task.pid = self.process.pid
task.save()
self.configs["task"] = task
self.timer_check_logdir = PeriodicCallback(lambda: TornadoWorker.periodic_check_logdir(self), 700)
self.timer_read_log = PeriodicCallback(lambda: TornadoWorker.periodic_read_logs(self), 300)
self.timer_read_log.start()
self.timer_check_logdir.start()
# pseudo static method of reading new lines from directory
@staticmethod
def periodic_read_logs(self):
lines = self.log_reader.read_new()
line = "".join(lines)
# logging.debug(line.decode('ascii', errors='ignore').rstrip())
if line != "":
self.configs["queue"].put(line, False)
# pseudo static method of checking is the directory has changed
def periodic_check_logdir(self):
if self.log_reader.is_changed():
self.log_reader.update_list_checking_files()
self.wait_worker_process()
@staticmethod
def get_object_or_404(id):
try:
obj = Task.get(Task.id == id)
return obj
except Task.DoesNotExist:
return []
# stopping tornado
def stop(self):
# schedule stopping
self.configs["queue"].put("#last_record#")