def __init__(self, app_config): StoppableThread.__init__(self) # ip = app_config["machines"]["master"][0].split('@') # master_ip = ip[-1:][0] redis_ip = app_config["redis"]["host"] redis_port = app_config["redis"]["port"] self.logger = AppLogger("master", logging.getLogger("local"), redis_ip, redis_port) self.log_writer = LogWriter(logging.getLogger("app"), redis_ip, redis_port) self.command_queue = QueueManager(host=redis_ip, port=redis_port, qname="commands") self.finished_queue = QueueManager(host=redis_ip, port=redis_port, qname="finished") # self.fman = FileManager(master_ip, master_queue_port) self.fman = FileManager(app_config) self.config = app_config self.input = app_config["dirs"]["input"] self.output = app_config["dirs"]["output"]
def __init__(self, app_config): StoppableThread.__init__(self) ip = app_config["machines"]["master"][0].split('@') master_ip = ip[-1:][0] master_queue_port = app_config["redis"]["port"] self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port) self.log_writer = LogWriter(logging.getLogger("app")) self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands") self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished") self.fman = FileManager(master_ip, master_queue_port) self.config = app_config self.input = app_config["dirs"]["input"] self.output = app_config["dirs"]["output"]
class Master(StoppableThread): """Master worker """ def __init__(self, app_config): StoppableThread.__init__(self) ip = app_config["machines"]["master"][0].split('@') master_ip = ip[-1:][0] master_queue_port = app_config["redis"]["port"] self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port) self.log_writer = LogWriter(logging.getLogger("app")) self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands") self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished") self.fman = FileManager(master_ip, master_queue_port) self.config = app_config self.input = app_config["dirs"]["input"] self.output = app_config["dirs"]["output"] def run(self): self.log_writer.start() self.logger.debug("Running master...") processed_filenames = [] while not self.is_stopped(): self.logger.debug("Reading directory...") filenames = [f for f in listdir(self.input) if f not in processed_filenames] if len(filenames) > 0: self.logger.info(str(len(filenames)) + " file(s) to put in the queue") for filename in filenames: full_filename = join(self.input, filename) dirname = create_data_directory(full_filename) if dirname is not None: archive = zip_directory(dirname) self.fman.store_file(archive) self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config)) processed_filenames.append(filename) if len(self.finished_queue) > 0: self.logger.info("Finished queue not empty") while not self.finished_queue.is_empty(): filename = self.finished_queue.pop() self.fman.retrieve_file(filename) output_file_path = join(self.config["dirs"]["output"], split(filename)[1]) if exists(output_file_path): remove(output_file_path) move(filename, self.config["dirs"]["output"]) self.fman.delete_file(filename) self.logger.info("No more finished job to process") sleep(60) # Avoid CPU consuption while waiting def stop(self): self.logger.info("Master stopped") self.log_writer.stop() StoppableThread.stop(self)
class Master(StoppableThread): """ Master worker """ def __init__(self, app_config): StoppableThread.__init__(self) # ip = app_config["machines"]["master"][0].split('@') # master_ip = ip[-1:][0] redis_ip = app_config["redis"]["host"] redis_port = app_config["redis"]["port"] self.logger = AppLogger("master", logging.getLogger("local"), redis_ip, redis_port) self.log_writer = LogWriter(logging.getLogger("app"), redis_ip, redis_port) self.command_queue = QueueManager(host=redis_ip, port=redis_port, qname="commands") self.finished_queue = QueueManager(host=redis_ip, port=redis_port, qname="finished") # self.fman = FileManager(master_ip, master_queue_port) self.fman = FileManager(app_config) self.config = app_config self.input = app_config["dirs"]["input"] self.output = app_config["dirs"]["output"] def run(self): self.log_writer.start() self.logger.info("Starting master...") # processed_filenames = [] while not self.is_stopped(): self.logger.info("Reading input directory...") # filenames = [f for f in listdir(self.input) if f not in processed_filenames] filenames = listdir(self.input) if len(filenames) > 0: self.logger.info( str(len(filenames)) + " file(s) to put in the queue") for filename in filenames: self.logger.debug("Processing %s..." % filename) full_filename = join(self.input, filename) dirname = create_data_directory( full_filename, self.config["dirs"]["temp"]) self.logger.debug("%s has been created." % dirname) if dirname is not None: # archive = zip_directory(dirname) # self.fman.store_file(archive) self.command_queue.push( CommandQueueItem(filename=dirname, logger=self.logger, config=self.config)) # processed_filenames.append(filename) self.logger.info( "Incoming files have been put in the queue") if len(self.finished_queue) > 0: self.logger.info("Finished queue not empty") while not self.finished_queue.is_empty(): filename = self.finished_queue.pop() # self.fman.retrieve_file(filename) output_file_path = join(self.config["dirs"]["output"], split(filename)[1]) if exists(output_file_path): remove(output_file_path) move(filename, self.config["dirs"]["output"]) # self.fman.delete_file(filename) self.logger.info("No more finished job to process") sleep(self.config["sleep"] ["master"]) # Avoid CPU consuption while waiting def stop(self): self.logger.info("Master stopped") self.log_writer.stop() StoppableThread.stop(self)