def __init__(self, app_logger, queue_ip="127.0.0.1", queue_port=6379):
StoppableThread.__init__(self)
self.log_queue = QueueManager(host=queue_ip,
port=queue_port,
qname="logging")
self.logger = app_logger
class LogWriter(StoppableThread):
"""Pops element from the logging queue and write them in the proper directory
"""
def __init__(self, app_logger):
StoppableThread.__init__(self)
self.log_queue = QueueManager(qname="logging")
self.logger = app_logger
def write_logs(self):
"""Write logs to a local file
"""
while not self.log_queue.is_empty():
log_json = self.log_queue.pop()
log_data = json.loads(log_json)
self.logger.log(log_data["lvl"], "["+log_data["uid"]+"] "+log_data["msg"])
sleep(0.2)
def run(self):
self.logger.debug("Logger initiatied")
while not self.stop_event.isSet():
self.write_logs()
sleep(0.5)
self.write_logs()
self.logger.info("Logger stopped")
def __init__(self, app_config):
StoppableThread.__init__(self)
# ip = app_config["machines"]["master"][0].split('@')
# master_ip = ip[-1:][0]
redis_ip = app_config["redis"]["host"]
redis_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), redis_ip,
redis_port)
self.log_writer = LogWriter(logging.getLogger("app"), redis_ip,
redis_port)
self.command_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="commands")
self.finished_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="finished")
# self.fman = FileManager(master_ip, master_queue_port)
self.fman = FileManager(app_config)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
class LogWriter(StoppableThread):
"""Pops element from the logging queue and write them in the proper directory
"""
def __init__(self, app_logger, queue_ip="127.0.0.1", queue_port=6379):
StoppableThread.__init__(self)
self.log_queue = QueueManager(host=queue_ip,
port=queue_port,
qname="logging")
self.logger = app_logger
def write_logs(self):
"""Write logs to a local file
"""
while not self.log_queue.is_empty():
log_json = self.log_queue.pop()
log_data = json.loads(log_json)
self.logger.log(log_data["lvl"],
"[" + log_data["uid"] + "] " + log_data["msg"])
sleep(0.2)
def run(self):
self.logger.debug("Logger initiatied")
while not self.stop_event.isSet():
self.write_logs()
sleep(0.5)
self.write_logs()
self.logger.info("Logger stopped")
def __init__(self, app_config):
StoppableThread.__init__(self)
self.config = app_config
# ip = app_config["machines"]["master"][0].split('@')
# master_ip = ip[-1:][0]
redis_ip = app_config["redis"]["host"]
redis_port = app_config["redis"]["port"]
self.command_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="commands")
self.finished_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="finished")
# self.fman = FileManager(master_ip, master_queue_port)
slave_ip = gethostbyname(gethostname())
slave_pid = getpid()
uid = slave_ip + "::" + str(slave_pid)
self.logger = AppLogger(uid, logging.getLogger("local"), redis_ip,
redis_port)
self.max_tries = app_config["commands"]["tries"]
self.logger.info("Slave initiated [redis on " + redis_ip + "]")
def __init__(self,
uid,
local_logger,
queue_ip="127.0.0.1",
queue_port=6379):
self.log_queue = QueueManager(host=queue_ip,
port=queue_port,
qname="logging")
self.logger = local_logger
# Identify the logging process
self.uid = uid
def __init__(self, app_config):
StoppableThread.__init__(self)
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port)
self.log_writer = LogWriter(logging.getLogger("app"))
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def __init__(self, app_config):
StoppableThread.__init__(self)
self.config = app_config
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
slave_ip = gethostbyname(gethostname())
slave_pid = getpid()
uid = slave_ip + "::" + str(slave_pid)
self.logger = AppLogger(uid, logging.getLogger("local"), master_ip, master_queue_port)
self.max_tries = app_config["commands"]["tries"]
self.logger.info("Slave initiated [redis on "+master_ip+"]")
class Slave(StoppableThread):
"""Slave worker
"""
def __init__(self, app_config):
StoppableThread.__init__(self)
self.config = app_config
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
slave_ip = gethostbyname(gethostname())
slave_pid = getpid()
uid = slave_ip + "::" + str(slave_pid)
self.logger = AppLogger(uid, logging.getLogger("local"), master_ip, master_queue_port)
self.max_tries = app_config["commands"]["tries"]
self.logger.info("Slave initiated [redis on "+master_ip+"]")
def run(self):
self.logger.debug("Running slave...")
while not self.is_stopped():
if not self.command_queue.is_empty():
cmd_json = self.command_queue.pop()
cmd = CommandQueueItem(jsondata=cmd_json, logger=self.logger, config=self.config)
status = cmd.execute()
# Job returned an error and has reached the limit of tries
if status == 1 and cmd.tries >= self.max_tries:
self.logger.error("Error when processing command")
continue
if cmd.current_step == -1:
self.logger.info("Pushing to finished queue")
self.finished_queue.push(cmd.filename)
self.logger.info("Job done")
continue
self.command_queue.push(cmd)
sleep(1) # Avoid CPU consumption while waiting
def stop(self):
self.logger.info("Slave stopped")
StoppableThread.stop(self)
def __init__(self, app_logger):
StoppableThread.__init__(self)
self.log_queue = QueueManager(qname="logging")
self.logger = app_logger
def __init__(self, uid, local_logger, queue_ip="127.0.0.1", queue_port=6379):
self.log_queue = QueueManager(host=queue_ip, port=queue_port, qname="logging")
self.logger = local_logger
# Identify the logging process
self.uid = uid
class AppLogger(object):
"""Logger publishing to a redis queue
"""
def __init__(self, uid, local_logger, queue_ip="127.0.0.1", queue_port=6379):
self.log_queue = QueueManager(host=queue_ip, port=queue_port, qname="logging")
self.logger = local_logger
# Identify the logging process
self.uid = uid
def log(self, level, message):
"""Log a message with a given level
Parameters
level (int): Log level
message (str): Message to store
"""
log_mess = {
"uid": self.uid,
"lvl": level,
"msg": message
}
self.log_queue.push(json.dumps(log_mess))
self.logger.log(level, "["+self.uid+"] "+message)
def debug(self, message):
"""Log a debug message
Parameters
message (str): Message to store
"""
self.log(logging.DEBUG, message)
def info(self, message):
"""Log an info message
Parameters
message (str): Message to store
"""
self.log(logging.INFO, message)
def warning(self, message):
"""Log a warning message
Parameters
message (str): Message to store
"""
self.log(logging.WARNING, message)
def error(self, message):
"""Log an error message
Parameters
message (str): Message to store
"""
self.log(logging.ERROR, message)
def fatal(self, message):
"""Log a fatal message
Parameters
message (str): Message to store
"""
self.log(logging.FATAL, message)
class Master(StoppableThread):
"""Master worker
"""
def __init__(self, app_config):
StoppableThread.__init__(self)
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port)
self.log_writer = LogWriter(logging.getLogger("app"))
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def run(self):
self.log_writer.start()
self.logger.debug("Running master...")
processed_filenames = []
while not self.is_stopped():
self.logger.debug("Reading directory...")
filenames = [f for f in listdir(self.input) if f not in processed_filenames]
if len(filenames) > 0:
self.logger.info(str(len(filenames)) + " file(s) to put in the queue")
for filename in filenames:
full_filename = join(self.input, filename)
dirname = create_data_directory(full_filename)
if dirname is not None:
archive = zip_directory(dirname)
self.fman.store_file(archive)
self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config))
processed_filenames.append(filename)
if len(self.finished_queue) > 0:
self.logger.info("Finished queue not empty")
while not self.finished_queue.is_empty():
filename = self.finished_queue.pop()
self.fman.retrieve_file(filename)
output_file_path = join(self.config["dirs"]["output"], split(filename)[1])
if exists(output_file_path):
remove(output_file_path)
move(filename, self.config["dirs"]["output"])
self.fman.delete_file(filename)
self.logger.info("No more finished job to process")
sleep(60) # Avoid CPU consuption while waiting
def stop(self):
self.logger.info("Master stopped")
self.log_writer.stop()
StoppableThread.stop(self)
class AppLogger(object):
"""Logger publishing to a redis queue
"""
def __init__(self,
uid,
local_logger,
queue_ip="127.0.0.1",
queue_port=6379):
self.log_queue = QueueManager(host=queue_ip,
port=queue_port,
qname="logging")
self.logger = local_logger
# Identify the logging process
self.uid = uid
def log(self, level, message):
"""Log a message with a given level
Parameters
level (int): Log level
message (str): Message to store
"""
log_mess = {"uid": self.uid, "lvl": level, "msg": message}
self.log_queue.push(json.dumps(log_mess))
self.logger.log(level, "[" + self.uid + "] " + message)
def debug(self, message):
"""Log a debug message
Parameters
message (str): Message to store
"""
self.log(logging.DEBUG, message)
def info(self, message):
"""Log an info message
Parameters
message (str): Message to store
"""
self.log(logging.INFO, message)
def warning(self, message):
"""Log a warning message
Parameters
message (str): Message to store
"""
self.log(logging.WARNING, message)
def error(self, message):
"""Log an error message
Parameters
message (str): Message to store
"""
self.log(logging.ERROR, message)
def fatal(self, message):
"""Log a fatal message
Parameters
message (str): Message to store
"""
self.log(logging.FATAL, message)
class Master(StoppableThread):
""" Master worker
"""
def __init__(self, app_config):
StoppableThread.__init__(self)
# ip = app_config["machines"]["master"][0].split('@')
# master_ip = ip[-1:][0]
redis_ip = app_config["redis"]["host"]
redis_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), redis_ip,
redis_port)
self.log_writer = LogWriter(logging.getLogger("app"), redis_ip,
redis_port)
self.command_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="commands")
self.finished_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="finished")
# self.fman = FileManager(master_ip, master_queue_port)
self.fman = FileManager(app_config)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def run(self):
self.log_writer.start()
self.logger.info("Starting master...")
# processed_filenames = []
while not self.is_stopped():
self.logger.info("Reading input directory...")
# filenames = [f for f in listdir(self.input) if f not in processed_filenames]
filenames = listdir(self.input)
if len(filenames) > 0:
self.logger.info(
str(len(filenames)) + " file(s) to put in the queue")
for filename in filenames:
self.logger.debug("Processing %s..." % filename)
full_filename = join(self.input, filename)
dirname = create_data_directory(
full_filename, self.config["dirs"]["temp"])
self.logger.debug("%s has been created." % dirname)
if dirname is not None:
# archive = zip_directory(dirname)
# self.fman.store_file(archive)
self.command_queue.push(
CommandQueueItem(filename=dirname,
logger=self.logger,
config=self.config))
# processed_filenames.append(filename)
self.logger.info(
"Incoming files have been put in the queue")
if len(self.finished_queue) > 0:
self.logger.info("Finished queue not empty")
while not self.finished_queue.is_empty():
filename = self.finished_queue.pop()
# self.fman.retrieve_file(filename)
output_file_path = join(self.config["dirs"]["output"],
split(filename)[1])
if exists(output_file_path):
remove(output_file_path)
move(filename, self.config["dirs"]["output"])
# self.fman.delete_file(filename)
self.logger.info("No more finished job to process")
sleep(self.config["sleep"]
["master"]) # Avoid CPU consuption while waiting
def stop(self):
self.logger.info("Master stopped")
self.log_writer.stop()
StoppableThread.stop(self)
class Slave(StoppableThread):
""" Slave worker
"""
def __init__(self, app_config):
StoppableThread.__init__(self)
self.config = app_config
# ip = app_config["machines"]["master"][0].split('@')
# master_ip = ip[-1:][0]
redis_ip = app_config["redis"]["host"]
redis_port = app_config["redis"]["port"]
self.command_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="commands")
self.finished_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="finished")
# self.fman = FileManager(master_ip, master_queue_port)
slave_ip = gethostbyname(gethostname())
slave_pid = getpid()
uid = slave_ip + "::" + str(slave_pid)
self.logger = AppLogger(uid, logging.getLogger("local"), redis_ip,
redis_port)
self.max_tries = app_config["commands"]["tries"]
self.logger.info("Slave initiated [redis on " + redis_ip + "]")
def run(self):
self.logger.info("Starting slave...")
while not self.is_stopped():
if not self.command_queue.is_empty():
cmd_json = self.command_queue.pop()
self.logger.debug("CommandQueueItem(jsondata=%s, ...)" %
str(cmd_json))
cmd = CommandQueueItem(jsondata=cmd_json,
logger=self.logger,
config=self.config)
# Start the job after waiting sync between master and worker
sleep(self.config["sleep"]["job"])
status = cmd.execute()
# Job returned an error and has reached the limit of tries
if status == 1 and cmd.tries >= self.max_tries:
self.logger.error("Error when processing command")
continue
if cmd.current_step == -1:
self.logger.info("Pushing to finished queue")
self.finished_queue.push(cmd.filename)
self.logger.info("Job done")
continue
self.command_queue.push(cmd)
sleep(self.config["sleep"]
["worker"]) # Avoid CPU consumption while waiting
def stop(self):
self.logger.info("Slave stopped")
StoppableThread.stop(self)
