def main():
logfile = join(settings.sanitised_directory, "publish.log")
fh = logging.FileHandler(logfile)
log.addHandler(fh)
manager = Manager()
semaphore = manager.Semaphore(cpu_count())
pool = Pool(processes=cpu_count())
report_counter = 0
# iterate over report files
report_files = list_report_files(settings.sanitised_directory)
while True:
try:
semaphore.acquire()
report_file = report_files.next()
log.info("Importing %s" % report_file)
pool.apply_async(ReportInserter, (report_file, semaphore))
report_counter += 1
except StopIteration:
break
log.info("Waiting for all the tasks to finish")
pool.close()
pool.join()
log.info("Imported %d reports" % report_counter)
def main():
if not os.path.isdir(settings.reports_directory):
log.error(settings.reports_directory + " does not exist")
sys.exit(1)
logfile = os.path.join(settings.reports_directory, "sanitise.log")
fh = logging.FileHandler(logfile)
log.addHandler(fh)
if not os.path.isdir(settings.archive_directory):
log.error(settings.archive_directory + " does not exist")
sys.exit(1)
if not os.path.isfile(settings.bridge_db_mapping_file):
log.error(settings.bridge_db_mapping_file + " does not exist")
sys.exit(1)
if not os.path.isdir(settings.sanitised_directory):
log.error(settings.sanitised_directory + " does not exist")
sys.exit(1)
report_counter = 0
manager = Manager()
semaphore = manager.Semaphore(cpu_count())
pool = Pool(processes=cpu_count())
# iterate over report files
report_files = list_report_files(settings.reports_directory)
while True:
try:
semaphore.acquire()
report_file = report_files.next()
pool.apply_async(sanitise_report, (report_file, semaphore))
report_counter += 1
except StopIteration:
break
log.info("Waiting for all the tasks to finish")
pool.close()
pool.join()
if report_counter > 0:
log.info(str(report_counter) + " reports archived")
else:
log.info("No reports were found in the reports directory: " +
settings.reports_directory)
def main():
if not os.path.isdir(settings.reports_directory):
log.error(settings.reports_directory + " does not exist")
sys.exit(1)
logfile = os.path.join(settings.reports_directory, "sanitise.log")
fh = logging.FileHandler(logfile)
log.addHandler(fh)
if not os.path.isdir(settings.archive_directory):
log.error(settings.archive_directory + " does not exist")
sys.exit(1)
if not os.path.isfile(settings.bridge_db_mapping_file):
log.error(settings.bridge_db_mapping_file + " does not exist")
sys.exit(1)
if not os.path.isdir(settings.sanitised_directory):
log.error(settings.sanitised_directory + " does not exist")
sys.exit(1)
report_counter = 0
manager = Manager()
semaphore = manager.Semaphore(cpu_count())
pool = Pool(processes=cpu_count())
# iterate over report files
report_files = list_report_files(settings.reports_directory)
while True:
try:
semaphore.acquire()
report_file = report_files.next()
pool.apply_async(sanitise_report, (report_file, semaphore))
report_counter += 1
except StopIteration:
break
log.info("Waiting for all the tasks to finish")
pool.close()
pool.join()
if report_counter > 0:
log.info(str(report_counter) + " reports archived")
else:
log.info("No reports were found in the reports directory: " + settings.reports_directory)
