コード例 #1
0
ファイル: logger.py プロジェクト: pdessauw/ocr-pipeline
class LogWriter(StoppableThread):
    """Pops element from the logging queue and write them in the proper directory
    """

    def __init__(self, app_logger):
        StoppableThread.__init__(self)

        self.log_queue = QueueManager(qname="logging")
        self.logger = app_logger

    def write_logs(self):
        """Write logs to a local file
        """
        while not self.log_queue.is_empty():
            log_json = self.log_queue.pop()
            log_data = json.loads(log_json)

            self.logger.log(log_data["lvl"], "["+log_data["uid"]+"] "+log_data["msg"])
            sleep(0.2)

    def run(self):
        self.logger.debug("Logger initiatied")

        while not self.stop_event.isSet():
            self.write_logs()
            sleep(0.5)

        self.write_logs()
        self.logger.info("Logger stopped")
コード例 #2
0
class LogWriter(StoppableThread):
    """Pops element from the logging queue and write them in the proper directory
    """
    def __init__(self, app_logger, queue_ip="127.0.0.1", queue_port=6379):
        StoppableThread.__init__(self)

        self.log_queue = QueueManager(host=queue_ip,
                                      port=queue_port,
                                      qname="logging")
        self.logger = app_logger

    def write_logs(self):
        """Write logs to a local file
        """
        while not self.log_queue.is_empty():
            log_json = self.log_queue.pop()
            log_data = json.loads(log_json)

            self.logger.log(log_data["lvl"],
                            "[" + log_data["uid"] + "] " + log_data["msg"])
            sleep(0.2)

    def run(self):
        self.logger.debug("Logger initiatied")

        while not self.stop_event.isSet():
            self.write_logs()
            sleep(0.5)

        self.write_logs()
        self.logger.info("Logger stopped")
コード例 #3
0
ファイル: actors.py プロジェクト: pdessauw/ocr-pipeline
class Slave(StoppableThread):
    """Slave worker
    """

    def __init__(self, app_config):
        StoppableThread.__init__(self)

        self.config = app_config

        ip = app_config["machines"]["master"][0].split('@')
        master_ip = ip[-1:][0]
        master_queue_port = app_config["redis"]["port"]

        self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
        self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
        self.fman = FileManager(master_ip, master_queue_port)

        slave_ip = gethostbyname(gethostname())
        slave_pid = getpid()
        uid = slave_ip + "::" + str(slave_pid)

        self.logger = AppLogger(uid, logging.getLogger("local"), master_ip, master_queue_port)
        self.max_tries = app_config["commands"]["tries"]

        self.logger.info("Slave initiated [redis on "+master_ip+"]")

    def run(self):
        self.logger.debug("Running slave...")

        while not self.is_stopped():
            if not self.command_queue.is_empty():
                cmd_json = self.command_queue.pop()
                cmd = CommandQueueItem(jsondata=cmd_json, logger=self.logger, config=self.config)

                status = cmd.execute()

                # Job returned an error and has reached the limit of tries
                if status == 1 and cmd.tries >= self.max_tries:
                    self.logger.error("Error when processing command")
                    continue

                if cmd.current_step == -1:
                    self.logger.info("Pushing to finished queue")
                    self.finished_queue.push(cmd.filename)
                    self.logger.info("Job done")
                    continue

                self.command_queue.push(cmd)

            sleep(1)  # Avoid CPU consumption while waiting

    def stop(self):
        self.logger.info("Slave stopped")
        StoppableThread.stop(self)
コード例 #4
0
ファイル: actors.py プロジェクト: pdessauw/ocr-pipeline
class Master(StoppableThread):
    """Master worker
    """

    def __init__(self, app_config):
        StoppableThread.__init__(self)

        ip = app_config["machines"]["master"][0].split('@')
        master_ip = ip[-1:][0]
        master_queue_port = app_config["redis"]["port"]

        self.logger = AppLogger("master", logging.getLogger("local"), master_ip, master_queue_port)
        self.log_writer = LogWriter(logging.getLogger("app"))

        self.command_queue = QueueManager(host=master_ip, port=master_queue_port, qname="commands")
        self.finished_queue = QueueManager(host=master_ip, port=master_queue_port, qname="finished")
        self.fman = FileManager(master_ip, master_queue_port)

        self.config = app_config
        self.input = app_config["dirs"]["input"]
        self.output = app_config["dirs"]["output"]

    def run(self):
        self.log_writer.start()
        self.logger.debug("Running master...")

        processed_filenames = []

        while not self.is_stopped():
            self.logger.debug("Reading directory...")
            filenames = [f for f in listdir(self.input) if f not in processed_filenames]

            if len(filenames) > 0:
                self.logger.info(str(len(filenames)) + " file(s) to put in the queue")

            for filename in filenames:
                full_filename = join(self.input, filename)
                dirname = create_data_directory(full_filename)

                if dirname is not None:
                    archive = zip_directory(dirname)

                    self.fman.store_file(archive)
                    self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config))

                processed_filenames.append(filename)

            if len(self.finished_queue) > 0:
                self.logger.info("Finished queue not empty")

                while not self.finished_queue.is_empty():
                    filename = self.finished_queue.pop()
                    self.fman.retrieve_file(filename)

                    output_file_path = join(self.config["dirs"]["output"], split(filename)[1])
                    if exists(output_file_path):
                        remove(output_file_path)

                    move(filename, self.config["dirs"]["output"])
                    self.fman.delete_file(filename)

                self.logger.info("No more finished job to process")

            sleep(60)  # Avoid CPU consuption while waiting

    def stop(self):
        self.logger.info("Master stopped")

        self.log_writer.stop()
        StoppableThread.stop(self)
コード例 #5
0
ファイル: actors.py プロジェクト: williammo2016/ocr-pipeline
class Master(StoppableThread):
    """ Master worker
    """
    def __init__(self, app_config):
        StoppableThread.__init__(self)

        # ip = app_config["machines"]["master"][0].split('@')
        # master_ip = ip[-1:][0]
        redis_ip = app_config["redis"]["host"]
        redis_port = app_config["redis"]["port"]

        self.logger = AppLogger("master", logging.getLogger("local"), redis_ip,
                                redis_port)
        self.log_writer = LogWriter(logging.getLogger("app"), redis_ip,
                                    redis_port)

        self.command_queue = QueueManager(host=redis_ip,
                                          port=redis_port,
                                          qname="commands")
        self.finished_queue = QueueManager(host=redis_ip,
                                           port=redis_port,
                                           qname="finished")
        # self.fman = FileManager(master_ip, master_queue_port)
        self.fman = FileManager(app_config)

        self.config = app_config
        self.input = app_config["dirs"]["input"]
        self.output = app_config["dirs"]["output"]

    def run(self):
        self.log_writer.start()
        self.logger.info("Starting master...")

        # processed_filenames = []

        while not self.is_stopped():
            self.logger.info("Reading input directory...")
            # filenames = [f for f in listdir(self.input) if f not in processed_filenames]
            filenames = listdir(self.input)

            if len(filenames) > 0:
                self.logger.info(
                    str(len(filenames)) + " file(s) to put in the queue")

                for filename in filenames:
                    self.logger.debug("Processing %s..." % filename)
                    full_filename = join(self.input, filename)
                    dirname = create_data_directory(
                        full_filename, self.config["dirs"]["temp"])
                    self.logger.debug("%s has been created." % dirname)

                    if dirname is not None:
                        # archive = zip_directory(dirname)

                        # self.fman.store_file(archive)
                        self.command_queue.push(
                            CommandQueueItem(filename=dirname,
                                             logger=self.logger,
                                             config=self.config))

                    # processed_filenames.append(filename)
                    self.logger.info(
                        "Incoming files have been put in the queue")

            if len(self.finished_queue) > 0:
                self.logger.info("Finished queue not empty")

                while not self.finished_queue.is_empty():
                    filename = self.finished_queue.pop()
                    # self.fman.retrieve_file(filename)

                    output_file_path = join(self.config["dirs"]["output"],
                                            split(filename)[1])
                    if exists(output_file_path):
                        remove(output_file_path)

                    move(filename, self.config["dirs"]["output"])
                    # self.fman.delete_file(filename)

                self.logger.info("No more finished job to process")

            sleep(self.config["sleep"]
                  ["master"])  # Avoid CPU consuption while waiting

    def stop(self):
        self.logger.info("Master stopped")

        self.log_writer.stop()
        StoppableThread.stop(self)
コード例 #6
0
ファイル: actors.py プロジェクト: williammo2016/ocr-pipeline
class Slave(StoppableThread):
    """ Slave worker
    """
    def __init__(self, app_config):
        StoppableThread.__init__(self)

        self.config = app_config

        # ip = app_config["machines"]["master"][0].split('@')
        # master_ip = ip[-1:][0]
        redis_ip = app_config["redis"]["host"]
        redis_port = app_config["redis"]["port"]

        self.command_queue = QueueManager(host=redis_ip,
                                          port=redis_port,
                                          qname="commands")
        self.finished_queue = QueueManager(host=redis_ip,
                                           port=redis_port,
                                           qname="finished")
        # self.fman = FileManager(master_ip, master_queue_port)

        slave_ip = gethostbyname(gethostname())
        slave_pid = getpid()
        uid = slave_ip + "::" + str(slave_pid)

        self.logger = AppLogger(uid, logging.getLogger("local"), redis_ip,
                                redis_port)
        self.max_tries = app_config["commands"]["tries"]

        self.logger.info("Slave initiated [redis on " + redis_ip + "]")

    def run(self):
        self.logger.info("Starting slave...")

        while not self.is_stopped():
            if not self.command_queue.is_empty():
                cmd_json = self.command_queue.pop()
                self.logger.debug("CommandQueueItem(jsondata=%s, ...)" %
                                  str(cmd_json))
                cmd = CommandQueueItem(jsondata=cmd_json,
                                       logger=self.logger,
                                       config=self.config)

                # Start the job after waiting sync between master and worker
                sleep(self.config["sleep"]["job"])
                status = cmd.execute()

                # Job returned an error and has reached the limit of tries
                if status == 1 and cmd.tries >= self.max_tries:
                    self.logger.error("Error when processing command")
                    continue

                if cmd.current_step == -1:
                    self.logger.info("Pushing to finished queue")
                    self.finished_queue.push(cmd.filename)
                    self.logger.info("Job done")
                    continue

                self.command_queue.push(cmd)

            sleep(self.config["sleep"]
                  ["worker"])  # Avoid CPU consumption while waiting

    def stop(self):
        self.logger.info("Slave stopped")
        StoppableThread.stop(self)