def _zip_file(self):
"""Check if the file can be zipped and zip it
"""
if self.unzipped is None:
self.logger.error("Zipped directory has not been unzipped")
return
zip_directory(self.unzipped)
self.unzipped = None
def run(self):
self.log_writer.start()
self.logger.debug("Running master...")
processed_filenames = []
while not self.is_stopped():
self.logger.debug("Reading directory...")
filenames = [f for f in listdir(self.input) if f not in processed_filenames]
if len(filenames) > 0:
self.logger.info(str(len(filenames)) + " file(s) to put in the queue")
for filename in filenames:
full_filename = join(self.input, filename)
dirname = create_data_directory(full_filename)
if dirname is not None:
archive = zip_directory(dirname)
self.fman.store_file(archive)
self.command_queue.push(CommandQueueItem(filename=archive, logger=self.logger, config=self.config))
processed_filenames.append(filename)
if len(self.finished_queue) > 0:
self.logger.info("Finished queue not empty")
while not self.finished_queue.is_empty():
filename = self.finished_queue.pop()
self.fman.retrieve_file(filename)
output_file_path = join(self.config["dirs"]["output"], split(filename)[1])
if exists(output_file_path):
remove(output_file_path)
move(filename, self.config["dirs"]["output"])
self.fman.delete_file(filename)
self.logger.info("No more finished job to process")
sleep(60) # Avoid CPU consuption while waiting