def main(): logfile = join(settings.sanitised_directory, "publish.log") fh = logging.FileHandler(logfile) log.addHandler(fh) manager = Manager() semaphore = manager.Semaphore(cpu_count()) pool = Pool(processes=cpu_count()) report_counter = 0 # iterate over report files report_files = list_report_files(settings.sanitised_directory) while True: try: semaphore.acquire() report_file = report_files.next() log.info("Importing %s" % report_file) pool.apply_async(ReportInserter, (report_file, semaphore)) report_counter += 1 except StopIteration: break log.info("Waiting for all the tasks to finish") pool.close() pool.join() log.info("Imported %d reports" % report_counter)
def main(): if not os.path.isdir(settings.reports_directory): log.error(settings.reports_directory + " does not exist") sys.exit(1) logfile = os.path.join(settings.reports_directory, "sanitise.log") fh = logging.FileHandler(logfile) log.addHandler(fh) if not os.path.isdir(settings.archive_directory): log.error(settings.archive_directory + " does not exist") sys.exit(1) if not os.path.isfile(settings.bridge_db_mapping_file): log.error(settings.bridge_db_mapping_file + " does not exist") sys.exit(1) if not os.path.isdir(settings.sanitised_directory): log.error(settings.sanitised_directory + " does not exist") sys.exit(1) report_counter = 0 manager = Manager() semaphore = manager.Semaphore(cpu_count()) pool = Pool(processes=cpu_count()) # iterate over report files report_files = list_report_files(settings.reports_directory) while True: try: semaphore.acquire() report_file = report_files.next() pool.apply_async(sanitise_report, (report_file, semaphore)) report_counter += 1 except StopIteration: break log.info("Waiting for all the tasks to finish") pool.close() pool.join() if report_counter > 0: log.info(str(report_counter) + " reports archived") else: log.info("No reports were found in the reports directory: " + settings.reports_directory)