# Test Code Only #db.purge_table() #time.sleep(5) # Test Code Only #db.create_items() #time.sleep() if os.path.exists(OUTPUT_DIR): shutil.rmtree(OUTPUT_DIR) os.makedirs(OUTPUT_DIR) while True: db.display_items() rows = db.get_unclassified_items() # Process Items if rows != None: jobs = [] cnt = 0 for row in rows: cnt += 1 p = multiprocessing.Process(target=process_item, args=(row, )) jobs.append(p) # Start processes if cnt % PARALLEL_PROCS == 0: for job in jobs: job.start() for job in jobs: job.join()
#db.delete_by_id('Entrance') #db.add_classified() #db.reset_classified() print "\n\n\n" cnt = 0 for row in db.get_classified_items(): cnt += 1 print "# Classified Item Count", cnt cnt = 0 d = {} for row in db.get_unclassified_items(): d[row['RASP_NAME']] = d.get(row['RASP_NAME'],0) + 1 cnt += 1 print "# Unclassified Item Count", cnt print "# Unclassified Items", d cnt = 0 for row in db.get_processed_items(): cnt += 1 print "\n# Processed Item Count", cnt cnt = 0 d = {} for row in db.get_unprocessed_items(): d[row['RASP_NAME']] = d.get(row['RASP_NAME'],0) + 1 cnt += 1
# Test Code Only # db.purge_table() # time.sleep(5) # Test Code Only # db.create_items() # time.sleep() if os.path.exists(OUTPUT_DIR): shutil.rmtree(OUTPUT_DIR) os.makedirs(OUTPUT_DIR) while True: db.display_items() rows = db.get_unclassified_items() # Process Items if rows != None: jobs = [] cnt = 0 for row in rows: cnt += 1 p = multiprocessing.Process(target=process_item, args=(row,)) jobs.append(p) # Start processes if cnt % PARALLEL_PROCS == 0: for job in jobs: job.start() for job in jobs: job.join()