예제 #1
0
    def _zip_file(self):
        """Check if the file can be zipped and zip it
        """
        if self.unzipped is None:
            self.logger.error("Zipped directory has not been unzipped")
            return

        zip_directory(self.unzipped)
        self.unzipped = None
예제 #2
0
    def run(self):
        self.log_writer.start()
        self.logger.debug("Running master...")

        processed_filenames = []

        while not self.is_stopped():
            self.logger.debug("Reading directory...")
            filenames = [f for f in listdir(self.input) if f not in processed_filenames]

            if len(filenames) > 0:
                self.logger.info(str(len(filenames)) + " file(s) to put in the queue")

            for filename in filenames:
                full_filename = join(self.input, filename)
                dirname = create_data_directory(full_filename)

                if dirname is not None:
                    archive = zip_directory(dirname)

                    self.fman.store_file(archive)
                    self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config))

                processed_filenames.append(filename)

            if len(self.finished_queue) > 0:
                self.logger.info("Finished queue not empty")

                while not self.finished_queue.is_empty():
                    filename = self.finished_queue.pop()
                    self.fman.retrieve_file(filename)

                    output_file_path = join(self.config["dirs"]["output"], split(filename)[1])
                    if exists(output_file_path):
                        remove(output_file_path)

                    move(filename, self.config["dirs"]["output"])
                    self.fman.delete_file(filename)

                self.logger.info("No more finished job to process")

            sleep(60)  # Avoid CPU consuption while waiting