def __init__(self, stream):
"""
Creates thread which wait data from stream and put they in inter-thread queue
:param stream: stream to read
"""
# input stream to read
self.stream = stream
# inter-thread queue to pass data from stream
self.queue = Queue()
# stream-read thread
self.thread = Thread(target=self.wait_output)
self.thread.daemon = True
self.thread.start()
self.decoder = OutputDecoder()
def __init__(self, *args):
self.decoder = OutputDecoder()
# TODO remove port from code
self.main_loop = None
self.buffer = {"buffer": {}}
self.application_handlers = [(r"/console2", WebSocketHandler, {
"configs": self.buffer
})]
def __init__(self):
self.pid = None
self.process = None
self.task = None
self.wait_worker_process_thread = None
self.stdout_read_thread = None
self.stderr_read_thread = None
self.logger = None
self.decoder = OutputDecoder()
def __init__(self, *args):
self.logger = None
# deprecated
self.decoder = OutputDecoder()
# TODO remove port from code
self.main_loop = tornado.ioloop.IOLoop.instance()
self.pool = {"buffer": {}, "task": None}
self.application_handlers = [(r"/socket/log", WebSocketHandler, {
"configs": self.pool
}), (r"/test", MainHandler, {
"configs": self.pool
})]
def __init__(self, stream):
"""
Creates thread which wait data from stream and put they in inter-thread queue
:param stream: stream to read
"""
# input stream to read
self.stream = stream
# inter-thread queue to pass data from stream
self.queue = Queue()
# stream-read thread
self.thread = Thread(target=self.wait_output)
self.thread.daemon = True
self.thread.start()
self.decoder = OutputDecoder()
class OutputReader(object):
"""
Non-blocking stream reader.
Uses in WebConsole.
"""
def __init__(self, stream):
"""
Creates thread which wait data from stream and put they in inter-thread queue
:param stream: stream to read
"""
# input stream to read
self.stream = stream
# inter-thread queue to pass data from stream
self.queue = Queue()
# stream-read thread
self.thread = Thread(target=self.wait_output)
self.thread.daemon = True
self.thread.start()
self.decoder = OutputDecoder()
def wait_output(self):
"""
Reading thread entry point
"""
for line in iter(self.stream.readline, b''):
self.queue.put(line)
self.stream.close()
def read_nowait(self):
"""
Non-blocking read from stream.
:return: string from stream
"""
result = []
try:
for i in range(0, 100):
# get bytes from inter-thread queue
b = self.queue.get_nowait()
if b:
# decode bytes to string
s = self.decoder.try_decode(b)
s = s.rstrip()
result.append(s)
except Empty:
pass
# add empty line to add \n at the end of result
result.append('')
# join result list of string to string
return '\n'.join(result)
class OutputReader(object):
"""
Non-blocking stream reader.
Uses in WebConsole.
"""
def __init__(self, stream):
"""
Creates thread which wait data from stream and put they in inter-thread queue
:param stream: stream to read
"""
# input stream to read
self.stream = stream
# inter-thread queue to pass data from stream
self.queue = Queue()
# stream-read thread
self.thread = Thread(target=self.wait_output)
self.thread.daemon = True
self.thread.start()
self.decoder = OutputDecoder()
def wait_output(self):
"""
Reading thread entry point
"""
for line in iter(self.stream.readline, b''):
self.queue.put(line)
self.stream.close()
def read_nowait(self):
"""
Non-blocking read from stream.
:return: string from stream
"""
result = []
try:
for i in range(0, 100):
# get bytes from inter-thread queue
b = self.queue.get_nowait()
if b:
# decode bytes to string
s = self.decoder.try_decode(b)
s = s.rstrip()
result.append(s)
except Empty:
pass
# add empty line to add \n at the end of result
result.append('')
# join result list of string to string
return '\n'.join(result)
class TaskManager:
"""
управление воркером
создает дочерний процесс
ждет его завершения
следит чтобы не создавалось больше одного воркера
читает его лог, std_out и сохраняет это в базе данных
"""
_instance = None
@classmethod
def get_instance(cls):
"""
:rtype : TaskManager
"""
if not cls._instance:
cls._instance = TaskManager()
# cls._instance.create_job_object()
return cls._instance
def __init__(self):
self.pid = None
self.process = None
self.task = None
self.wait_worker_process_thread = None
self.stdout_read_thread = None
self.stderr_read_thread = None
self.logger = None
self.decoder = OutputDecoder()
def has_active_install(self):
"""
Есть ли уже активный воркер?
:return:
"""
is_running = (self.process and self.process.poll() is None)
return is_running
def run_worker(self, task_id):
"""
Запустить воркер, с заданием task_id
:param task_id:
:raise Exception:
"""
if self.has_active_install():
raise Exception("Can't run new worker, another one still exists")
self.create_worker(task_id)
def cancel(self):
"""
отменить, убить процесс
"""
if self.process:
self.process.terminate()
self.task = None
def create_worker(self, task_id):
"""
создать воркер, передать task_id дочернему процессу
zoocmd --start-install-worker= task_id
новый процесс сам все сделает дальше
:param task_id:
"""
self.logger = logging.getLogger('zoo-worker-task-' + str(task_id))
self.logger.propagate = False
self.logger.handlers = []
self.logger.addHandler(TaskDbLogHandler(self.task))
self.process = subprocess.Popen(
(
sys.executable,
sys.argv[0],
'--start-install-worker={0}'.format(task_id),
#'--log-level=debug'
),
# stdout = subprocess.PIPE,
# stderr = subprocess.PIPE
)
# self.pid = self.process.pid
#
self.wait_worker_process_thread = Thread(target=self.wait_worker_process)
self.wait_worker_process_thread.start()
#
# self.stdout_read_thread = Thread(target=self.read_stdout)
# self.stderr_read_thread = Thread(target=self.read_stderr)
#
# self.stdout_read_thread.start()
# self.stderr_read_thread.start()
def wait_worker_process(self):
logging.debug('thread for waiting of worker process started')
while self.process:
if self.process.poll() is not None:
logging.debug("process is finished")
self.process = None
break
else:
logging.debug('sleep')
time.sleep(1)
self.logger.info('install worker process is finished')
self.sync_core()
def read_stdout(self):
"""
читать лог воркера, это должно происходить в отдельном потоке чтобы не блокировать работу воркера
"""
while self.process:
if self.process.poll() is not None:
break
if self.process.stdout.closed:
logging.debug("stdout is closed")
break
line = self.process.stdout.readline()
if line != b'':
s = self.decoder.try_decode(line)
s = s.rstrip()
self.logger.info(s)
logging.debug(line.decode('ascii', errors='ignore').rstrip())
else:
self.process.stdout.close()
logging.debug("read worker stdout thread is finished")
def read_stderr(self):
"""
читать лог воркера, это должно происходить в отдельном потоке чтобы не блокировать работу воркера
"""
while self.process:
if self.process.poll() is not None:
break
if self.process.stderr.closed:
logging.debug("stderr is closed")
break
line = self.process.stderr.readline()
if line != b'':
s = self.decoder.try_decode(line)
self.logger.warning(s)
logging.debug(line.decode('ascii', errors='ignore').rstrip())
else:
self.process.stderr.close()
logging.debug("read worker stderr thread is finished")
@staticmethod
def sync_core():
"""
После каждой установки выполнить sync чтобы current.yaml был в актуальном сосотоянии
"""
logging.debug('start core updating after install process finished')
Core.get_instance().set_expired(True)
Core.get_instance().update()
def create_install_task(self, requested_products: list, products: ProductCollection, parameter_manager: ParametersManager):
"""
фабричный метод: создать такс с переданными параметрами, с типом COMMAND_INSTALL
сохранить в базе данных
зхапустить воркер
:param requested_products: те продукты которые выбрал пользователь
:param products: продукты для установки (продукты которые выбрал пользователь + зависимости)
:param parameter_manager:
:return: номер таска
"""
title = 'Installing products: {0}'.format(', '.join([product_name for product_name in requested_products]))
state = {
'requested_products': requested_products,
'products': products.to_json_list(),
'parameters': parameter_manager.get_state()
}
settings = Core.get_instance().settings.get_state()
logging.debug('state: {0}'.format(state))
state2save = json.dumps(state, sort_keys=True, indent=1)
task = Task(
command=Task.COMMAND_INSTALL,
title=title,
params=state2save,
settings=json.dumps(settings, sort_keys=True, indent=1)
)
task.save()
self.task = task
# production
self.run_worker(task.id)
# For debug worker uncomment this
#self.task.execute()
return task.id
def create_upgrade_task(self, requested_products: list, products: ProductCollection):
"""
фабричный метод: создать такс с переданными параметрами, с типом COMMAND_UPGRADE
сохранить в базе данных
зхапустить воркер
:param requested_products:
:param products:
:param parameter_manager:
:return:
"""
title = 'Upgrading products: {0}'.format(', '.join([product_name for product_name in requested_products]))
state = {
'requested_products': requested_products,
'products': products.get_names_list(),
'parameters': {}
}
settings = Core.get_instance().settings.get_state()
logging.debug('state: {0}'.format(state))
task = Task(
command=Task.COMMAND_UPGRADE,
title=title,
params=json.dumps(state, sort_keys=True, indent=1),
settings=json.dumps(settings, sort_keys=True, indent=1)
)
task.save()
self.task = task
# production
self.run_worker(task.id)
# For debug worker uncomment this
#self.task.execute()
return task.id
def create_uninstall_task(self, products: ProductCollection):
"""
фабричный метод: создать такс с переданными параметрами, с типом COMMAND_UNINSTALL
сохранить в базе данных
зхапустить воркер
:param requested_products:
:param products:
:param parameter_manager:
:return:
"""
title = 'Uninstalling products: {0}'.format(', '.join([product.title for product in products]))
state = {
'products': products.get_names_list(),
}
settings = Core.get_instance().settings.get_state()
task = Task(
command=Task.COMMAND_UNINSTALL,
title=title,
params=json.dumps(state, sort_keys=True, indent=1),
settings=json.dumps(settings, sort_keys=True, indent=1))
task.save()
self.task = task
self.run_worker(task.id)
return task.id
def rerun_task(self, task):
"""
повтонрно выполнить задание N
удобно для оталадки,
удобно если что-то упало, повторить снова
:param task:
:return:
"""
task.status = Task.STATUS_PENDING
task.logmessage_set.all().delete()
task.error_message = ''
task.save()
self.task = task
self.run_worker(task.id)
return task.id
