def __init__(self, app_config):
StoppableThread.__init__(self)
# ip = app_config["machines"]["master"][0].split('@')
# master_ip = ip[-1:][0]
redis_ip = app_config["redis"]["host"]
redis_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), redis_ip,
redis_port)
self.log_writer = LogWriter(logging.getLogger("app"), redis_ip,
redis_port)
self.command_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="commands")
self.finished_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="finished")
# self.fman = FileManager(master_ip, master_queue_port)
self.fman = FileManager(app_config)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def __init__(self, app_config):
StoppableThread.__init__(self)
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port)
self.log_writer = LogWriter(logging.getLogger("app"))
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
class Master(StoppableThread):
"""Master worker
"""
def __init__(self, app_config):
StoppableThread.__init__(self)
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port)
self.log_writer = LogWriter(logging.getLogger("app"))
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def run(self):
self.log_writer.start()
self.logger.debug("Running master...")
processed_filenames = []
while not self.is_stopped():
self.logger.debug("Reading directory...")
filenames = [f for f in listdir(self.input) if f not in processed_filenames]
if len(filenames) > 0:
self.logger.info(str(len(filenames)) + " file(s) to put in the queue")
for filename in filenames:
full_filename = join(self.input, filename)
dirname = create_data_directory(full_filename)
if dirname is not None:
archive = zip_directory(dirname)
self.fman.store_file(archive)
self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config))
processed_filenames.append(filename)
if len(self.finished_queue) > 0:
self.logger.info("Finished queue not empty")
while not self.finished_queue.is_empty():
filename = self.finished_queue.pop()
self.fman.retrieve_file(filename)
output_file_path = join(self.config["dirs"]["output"], split(filename)[1])
if exists(output_file_path):
remove(output_file_path)
move(filename, self.config["dirs"]["output"])
self.fman.delete_file(filename)
self.logger.info("No more finished job to process")
sleep(60) # Avoid CPU consuption while waiting
def stop(self):
self.logger.info("Master stopped")
self.log_writer.stop()
StoppableThread.stop(self)
class Master(StoppableThread):
""" Master worker
"""
def __init__(self, app_config):
StoppableThread.__init__(self)
# ip = app_config["machines"]["master"][0].split('@')
# master_ip = ip[-1:][0]
redis_ip = app_config["redis"]["host"]
redis_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), redis_ip,
redis_port)
self.log_writer = LogWriter(logging.getLogger("app"), redis_ip,
redis_port)
self.command_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="commands")
self.finished_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="finished")
# self.fman = FileManager(master_ip, master_queue_port)
self.fman = FileManager(app_config)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def run(self):
self.log_writer.start()
self.logger.info("Starting master...")
# processed_filenames = []
while not self.is_stopped():
self.logger.info("Reading input directory...")
# filenames = [f for f in listdir(self.input) if f not in processed_filenames]
filenames = listdir(self.input)
if len(filenames) > 0:
self.logger.info(
str(len(filenames)) + " file(s) to put in the queue")
for filename in filenames:
self.logger.debug("Processing %s..." % filename)
full_filename = join(self.input, filename)
dirname = create_data_directory(
full_filename, self.config["dirs"]["temp"])
self.logger.debug("%s has been created." % dirname)
if dirname is not None:
# archive = zip_directory(dirname)
# self.fman.store_file(archive)
self.command_queue.push(
CommandQueueItem(filename=dirname,
logger=self.logger,
config=self.config))
# processed_filenames.append(filename)
self.logger.info(
"Incoming files have been put in the queue")
if len(self.finished_queue) > 0:
self.logger.info("Finished queue not empty")
while not self.finished_queue.is_empty():
filename = self.finished_queue.pop()
# self.fman.retrieve_file(filename)
output_file_path = join(self.config["dirs"]["output"],
split(filename)[1])
if exists(output_file_path):
remove(output_file_path)
move(filename, self.config["dirs"]["output"])
# self.fman.delete_file(filename)
self.logger.info("No more finished job to process")
sleep(self.config["sleep"]
["master"]) # Avoid CPU consuption while waiting
def stop(self):
self.logger.info("Master stopped")
self.log_writer.stop()
StoppableThread.stop(self)
