class Command(object): """Main command object """ def __init__(self, filename, logger, app_config): self.logger = logger ip = app_config["machines"]["master"][0].split('@') master_ip = ip[-1:][0] master_queue_port = app_config["redis"]["port"] self.fman = FileManager(master_ip, master_queue_port) self.filename = filename self.unzipped = None self.config = app_config def get_file(self): """Retrieve file from redis and unzip it to the local filesystem """ # Get hash from redis self.logger.debug("Retrieving "+self.filename+"...") self.fman.retrieve_file(self.filename) # Write it in the tmp folder self._unzip_file() def store_file(self): """Zip file on the local filesystem and store it to redis """ self._zip_file() # Store it in redis self.fman.store_file(self.filename) def _zip_file(self): """Check if the file can be zipped and zip it """ if self.unzipped is None: self.logger.error("Zipped directory has not been unzipped") return zip_directory(self.unzipped) self.unzipped = None def _unzip_file(self): """Check if the file can be unzipped and unzip it """ if self.unzipped is not None: self.logger.error("Archive already unzipped") return self.unzipped = unzip_directory(self.filename)
def __init__(self, app_config): StoppableThread.__init__(self) # ip = app_config["machines"]["master"][0].split('@') # master_ip = ip[-1:][0] redis_ip = app_config["redis"]["host"] redis_port = app_config["redis"]["port"] self.logger = AppLogger("master", logging.getLogger("local"), redis_ip, redis_port) self.log_writer = LogWriter(logging.getLogger("app"), redis_ip, redis_port) self.command_queue = QueueManager(host=redis_ip, port=redis_port, qname="commands") self.finished_queue = QueueManager(host=redis_ip, port=redis_port, qname="finished") # self.fman = FileManager(master_ip, master_queue_port) self.fman = FileManager(app_config) self.config = app_config self.input = app_config["dirs"]["input"] self.output = app_config["dirs"]["output"]
def __init__(self, filename, logger, app_config): self.logger = logger ip = app_config["machines"]["master"][0].split('@') master_ip = ip[-1:][0] master_queue_port = app_config["redis"]["port"] self.fman = FileManager(master_ip, master_queue_port) self.filename = filename self.unzipped = None self.config = app_config
def __init__(self, app_config): StoppableThread.__init__(self) ip = app_config["machines"]["master"][0].split('@') master_ip = ip[-1:][0] master_queue_port = app_config["redis"]["port"] self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port) self.log_writer = LogWriter(logging.getLogger("app")) self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands") self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished") self.fman = FileManager(master_ip, master_queue_port) self.config = app_config self.input = app_config["dirs"]["input"] self.output = app_config["dirs"]["output"]
class Master(StoppableThread): """Master worker """ def __init__(self, app_config): StoppableThread.__init__(self) ip = app_config["machines"]["master"][0].split('@') master_ip = ip[-1:][0] master_queue_port = app_config["redis"]["port"] self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port) self.log_writer = LogWriter(logging.getLogger("app")) self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands") self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished") self.fman = FileManager(master_ip, master_queue_port) self.config = app_config self.input = app_config["dirs"]["input"] self.output = app_config["dirs"]["output"] def run(self): self.log_writer.start() self.logger.debug("Running master...") processed_filenames = [] while not self.is_stopped(): self.logger.debug("Reading directory...") filenames = [f for f in listdir(self.input) if f not in processed_filenames] if len(filenames) > 0: self.logger.info(str(len(filenames)) + " file(s) to put in the queue") for filename in filenames: full_filename = join(self.input, filename) dirname = create_data_directory(full_filename) if dirname is not None: archive = zip_directory(dirname) self.fman.store_file(archive) self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config)) processed_filenames.append(filename) if len(self.finished_queue) > 0: self.logger.info("Finished queue not empty") while not self.finished_queue.is_empty(): filename = self.finished_queue.pop() self.fman.retrieve_file(filename) output_file_path = join(self.config["dirs"]["output"], split(filename)[1]) if exists(output_file_path): remove(output_file_path) move(filename, self.config["dirs"]["output"]) self.fman.delete_file(filename) self.logger.info("No more finished job to process") sleep(60) # Avoid CPU consuption while waiting def stop(self): self.logger.info("Master stopped") self.log_writer.stop() StoppableThread.stop(self)