class Command(object):
"""Main command object
"""
def __init__(self, filename, logger, app_config):
self.logger = logger
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.fman = FileManager(master_ip, master_queue_port)
self.filename = filename
self.unzipped = None
self.config = app_config
def get_file(self):
"""Retrieve file from redis and unzip it to the local filesystem
"""
# Get hash from redis
self.logger.debug("Retrieving "+self.filename+"...")
self.fman.retrieve_file(self.filename)
# Write it in the tmp folder
self._unzip_file()
def store_file(self):
"""Zip file on the local filesystem and store it to redis
"""
self._zip_file()
# Store it in redis
self.fman.store_file(self.filename)
def _zip_file(self):
"""Check if the file can be zipped and zip it
"""
if self.unzipped is None:
self.logger.error("Zipped directory has not been unzipped")
return
zip_directory(self.unzipped)
self.unzipped = None
def _unzip_file(self):
"""Check if the file can be unzipped and unzip it
"""
if self.unzipped is not None:
self.logger.error("Archive already unzipped")
return
self.unzipped = unzip_directory(self.filename)
def __init__(self, app_config):
StoppableThread.__init__(self)
# ip = app_config["machines"]["master"][0].split('@')
# master_ip = ip[-1:][0]
redis_ip = app_config["redis"]["host"]
redis_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), redis_ip,
redis_port)
self.log_writer = LogWriter(logging.getLogger("app"), redis_ip,
redis_port)
self.command_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="commands")
self.finished_queue = QueueManager(host=redis_ip,
port=redis_port,
qname="finished")
# self.fman = FileManager(master_ip, master_queue_port)
self.fman = FileManager(app_config)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def __init__(self, filename, logger, app_config):
self.logger = logger
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.fman = FileManager(master_ip, master_queue_port)
self.filename = filename
self.unzipped = None
self.config = app_config
def __init__(self, app_config):
StoppableThread.__init__(self)
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port)
self.log_writer = LogWriter(logging.getLogger("app"))
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
class Master(StoppableThread):
"""Master worker
"""
def __init__(self, app_config):
StoppableThread.__init__(self)
ip = app_config["machines"]["master"][0].split('@')
master_ip = ip[-1:][0]
master_queue_port = app_config["redis"]["port"]
self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port)
self.log_writer = LogWriter(logging.getLogger("app"))
self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
self.fman = FileManager(master_ip, master_queue_port)
self.config = app_config
self.input = app_config["dirs"]["input"]
self.output = app_config["dirs"]["output"]
def run(self):
self.log_writer.start()
self.logger.debug("Running master...")
processed_filenames = []
while not self.is_stopped():
self.logger.debug("Reading directory...")
filenames = [f for f in listdir(self.input) if f not in processed_filenames]
if len(filenames) > 0:
self.logger.info(str(len(filenames)) + " file(s) to put in the queue")
for filename in filenames:
full_filename = join(self.input, filename)
dirname = create_data_directory(full_filename)
if dirname is not None:
archive = zip_directory(dirname)
self.fman.store_file(archive)
self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config))
processed_filenames.append(filename)
if len(self.finished_queue) > 0:
self.logger.info("Finished queue not empty")
while not self.finished_queue.is_empty():
filename = self.finished_queue.pop()
self.fman.retrieve_file(filename)
output_file_path = join(self.config["dirs"]["output"], split(filename)[1])
if exists(output_file_path):
remove(output_file_path)
move(filename, self.config["dirs"]["output"])
self.fman.delete_file(filename)
self.logger.info("No more finished job to process")
sleep(60) # Avoid CPU consuption while waiting
def stop(self):
self.logger.info("Master stopped")
self.log_writer.stop()
StoppableThread.stop(self)
