def main(): # Getting the models. db_connector = dbutils.connect_to_main_database() # get the number of rgb artifacts select_sql_statement = "SELECT path FROM artifact WHERE type='pcrgb';" pcd_paths = db_connector.execute(select_sql_statement, fetch_all=True) #[0][0] print ('Available fused data: ' + str(len(pcd_paths))) # # todo: remove the (1) or (2) backup ? # unique_qr_codes = [x[0] for x in unique_qr_codes] # initialze log file logging.basicConfig(filename='/tmp/command_update_depth.log',level=logging.DEBUG, format='%(asctime)s %(message)s') # Run this in multiprocess mode. utils.multiprocess(pcd_paths, process_method = update_depth, process_individial_entries = False, number_of_workers = 8, pass_process_index = True, progressbar = False, disable_gpu = True) print("*** Done ***.")
def update_artifactsquality_with_pointcloud_data(): # Get all pointclouds. sql_script = "SELECT id, path FROM artifact WHERE type='pcd'" db_connector = dbutils.connect_to_main_database() pointcloud_entries = db_connector.execute(sql_script, fetch_all=True) print("Found {} pointclouds.".format(len(pointcloud_entries))) db_type = "pcd" def process_pointcloud_entries(pointcloud_entries): db_connector = dbutils.connect_to_main_database() # Go through all entries. bar = progressbar.ProgressBar(max_value=len(pointcloud_entries)) for index, (artifact_id, path) in enumerate(pointcloud_entries): bar.update(index) pointcloud_values = get_pointcloud_values(path) for db_key, db_value in pointcloud_values.items(): # Check if there is already an entry. select_sql_statement = "" select_sql_statement += "SELECT COUNT(*) FROM artifact_quality" select_sql_statement += " WHERE artifact_id='{}'".format( artifact_id) select_sql_statement += " AND type='{}'".format(db_type) select_sql_statement += " AND key='{}'".format(db_key) results = db_connector.execute(select_sql_statement, fetch_one=True)[0] # There is an entry. Skip if results != 0: continue # Create an SQL statement for insertion. sql_statement = "" sql_statement += "INSERT INTO artifact_quality (type, key, value, artifact_id, misc)" sql_statement += " VALUES(\'{}\', \'{}\', \'{}\', \'{}\', \'{}\');".format( db_type, db_key, db_value, artifact_id, "") # Call database. try: result = db_connector.execute(sql_statement) except: print(sql_statement, pointcloud_values) exit(0) bar.finish() # Run this in multiprocess mode. utils.multiprocess(pointcloud_entries, process_method=process_pointcloud_entries, process_individial_entries=False, progressbar=False) print("Done.")
def update_artifactsquality_with_posenet(): # Get all images. sql_script = "SELECT id, path FROM artifact WHERE type='jpg'" db_connector = dbutils.connect_to_main_database() image_entries = db_connector.execute(sql_script, fetch_all=True) print("Found {} images.".format(len(image_entries))) db_type = "rgb" db_key = "No of People" def process_image_entries(image_entries): db_connector = dbutils.connect_to_main_database() # Go through all entries. bar = progressbar.ProgressBar(max_value=len(image_entries)) for index, (artifact_id, path) in enumerate(image_entries): bar.update(index) # Check if there is already an entry. select_sql_statement = "" select_sql_statement += "SELECT COUNT(*) FROM artifact_quality" select_sql_statement += " WHERE artifact_id='{}'".format( artifact_id) select_sql_statement += " AND type='{}'".format(db_type) select_sql_statement += " AND key='{}'".format(db_key) results = db_connector.execute(select_sql_statement, fetch_one=True)[0] # There is an entry. Skip if results != 0: continue Pose = get_pose(path) # Create an SQL statement for insertion. sql_statement = "" sql_statement += "INSERT INTO artifact_quality (type, key, value, artifact_id, misc)" sql_statement += " VALUES(\'{}\', \'{}\', \'{}\', \'{}\', \'{}\');".format( db_type, db_key, Pose, artifact_id, "") # Call database. result = db_connector.execute(sql_statement) bar.finish() # Run this in multiprocess mode. utils.multiprocess(image_entries, process_method=process_image_entries, process_individial_entries=False, progressbar=False, number_of_workers=1) print("Done.")
def get_artifact_paths(file_extensions): # Get all persons. person_search_path = os.path.join(whhdata_path, media_subpath, "*") person_paths = [ path for path in glob.glob(person_search_path) if os.path.isdir(path) ] # Method for multiprocessing. def process_person_paths(person_paths): artifact_paths_per_process = [] for person_path in person_paths: for file_extension in file_extensions: glob_search_path = os.path.join( person_path, "**/*.{}".format(file_extension)) artifact_paths_per_process.extend(glob.glob(glob_search_path)) return artifact_paths_per_process # Use multiprocessing. artifact_paths = utils.multiprocess(person_paths, process_method=process_person_paths, process_individial_entries=False, progressbar=False, number_of_workers=None) return artifact_paths
def process_all(all_path, output_path): # Getting the paths of the qr-codes. logger.info("Gathering all qr codes...") qrcode_paths = glob.glob(os.path.join(all_path, "*")) qrcode_paths = [path for path in qrcode_paths if os.path.isdir(path) == True] qrcode_paths = [path for path in qrcode_paths if os.path.exists(os.path.join(path, "measurements"))] qrcode_paths = sorted(qrcode_paths) # Do a quality check on the qr code paths. if len(qrcode_paths) == 0: logger.info("No measurements found at \"{}\"!".format(all_path)) # This method is called in multiple processes. def process_qrcode_path(qrcode_path): results_all = [] scan_paths = glob.glob(os.path.join(qrcode_path, "measurements", "*")) scan_paths = [path for path in scan_paths if os.path.isdir(path) == True] for scan_path in scan_paths: process_scan(scan_path, output_path) # Run this in multiprocess mode. results = utils.multiprocess( qrcode_paths, process_method=process_qrcode_path, process_individial_entries=True, progressbar=False, number_of_workers=10, disable_gpu=True ) return []
def get_artifact_paths(file_extensions): config.art # Get all persons. person_search_path = os.path.join(config.artifacts_path, "*") person_paths = [ path for path in glob.glob(person_search_path) if os.path.isdir(path) ] print("Found {} persons.".format(len(person_paths))) # Method for multiprocessing. def process_person_paths(person_paths, process_index): artifact_paths_per_process = [] for person_path in tqdm(person_paths, position=process_index): for file_extension in file_extensions: glob_search_path = os.path.join( person_path, "**/*.{}".format(file_extension)) artifact_paths_per_process.extend(glob.glob(glob_search_path)) return artifact_paths_per_process # Use multiprocessing. artifact_paths = utils.multiprocess(person_paths, process_method=process_person_paths, process_individial_entries=False, progressbar=False, pass_process_index=True, number_of_workers=None) return artifact_paths
def main(): # get a list of all unique qr_codes sql_statement = "SELECT DISTINCT artifact.qr_code FROM artifact ORDER BY qr_code ASC;" unique_qr_codes = main_connector.execute(sql_statement, fetch_all=True) # todo: remove the (1) or (2) backup ? unique_qr_codes = [x[0] for x in unique_qr_codes] # initialze log file logging.basicConfig(filename='/tmp/command_update_fusion.log', level=logging.DEBUG, format='%(asctime)s %(message)s') # Run this in multiprocess mode. utils.multiprocess(unique_qr_codes, process_method=update_qrs, process_individial_entries=False, number_of_workers=4, pass_process_index=True, progressbar=False, disable_gpu=True) print("*** Done ***.")
def main(): # Check the arguments. if len(sys.argv) != 2: raise Exception("ERROR! Must provide model filename.") model_path = sys.argv[1] if not os.path.exists(model_path): raise Exception("ERROR! \"{}\" does not exist.".format(model_path)) # Get the training QR-codes. #search_path = os.path.join(os.path.dirname(model_path), "*.p") #paths = glob.glob(search_path) #details_path = [path for path in paths if "details" in path][0] #details = pickle.load(open(details_path, "rb")) #qrcodes_train = details["qrcodes_train"] #qrcodes_validate = details["qrcodes_validate"] #print("QR codes train:", len(qrcodes_train), "QR codes validate:", len(qrcodes_validate)) # Query the database for artifacts. print("Getting all artifacts...") db_connector = dbutils.connect_to_main_database() sql_statement = "" sql_statement += "SELECT id, path FROM artifact" sql_statement += " WHERE type='pcd'" sql_statement += ";" artifacts = db_connector.execute(sql_statement, fetch_all=True) print("Found {} artifacts.".format(len(artifacts))) # Method for processing a set of artifacts. # Note: This method will run in its own process. def process_artifacts(artifacts, process_index): # Create database connection. db_connector = dbutils.connect_to_main_database() # Load the model first. model_weights_path = [ x for x in glob.glob((os.path.join(model_path, "*"))) if x.endswith("-model-weights.h5") ][0] model_details_path = [ x for x in glob.glob((os.path.join(model_path, "*"))) if x.endswith("-details.p") ][0] model_name = model_path.split("/")[-1] model_details = pickle.load(open(model_details_path, "rb")) pointcloud_target_size = model_details["dataset_parameters"][ "pointcloud_target_size"] pointcloud_subsampling_method = model_details["dataset_parameters"][ "pointcloud_subsampling_method"] target_key = model_details["dataset_parameters"]["output_targets"][0] model = load_model(model_weights_path, pointcloud_target_size) # Evaluate and create SQL-statements. for artifact_index, artifact in enumerate( tqdm(artifacts, position=process_index)): # Unpack fields. artifact_id, pcd_path = artifact # Check if there is already an entry. select_sql_statement = "" select_sql_statement += "SELECT COUNT(*) FROM artifact_result" select_sql_statement += " WHERE artifact_id='{}'".format( artifact_id) select_sql_statement += " AND model_name='{}'".format(model_name) select_sql_statement += " AND target_key='{}'".format(target_key) results = db_connector.execute(select_sql_statement, fetch_one=True)[0] # There is an entry. Skip if results != 0: continue # Execute SQL statement. try: # Load the artifact and evaluate. pcd_path = pcd_path.replace("/whhdata/qrcode", "/localssd/qrcode") pcd_array = utils.load_pcd_as_ndarray(pcd_path) pcd_array = utils.subsample_pointcloud( pcd_array, pointcloud_target_size, subsampling_method=pointcloud_subsampling_method) value = model.predict(np.expand_dims(pcd_array, axis=0), verbose=0)[0][0] # Create an SQL statement. sql_statement = "" sql_statement += "INSERT INTO artifact_result (model_name, target_key, value, artifact_id)" sql_statement += " VALUES(\'{}\', \'{}\', \'{}\', \'{}\');".format( model_name, target_key, value, artifact_id) # Call database. result = db_connector.execute(sql_statement) except psycopg2.IntegrityError as e: #print("Already in DB. Skipped.", pcd_path) pass except ValueError as e: #print("Skipped.", pcd_path) pass # Run this in multiprocess mode. utils.multiprocess(artifacts, process_method=process_artifacts, process_individial_entries=False, progressbar=False, pass_process_index=True, disable_gpu=True) print("Done.")
def execute_command_preprocess(preprocess_pcds=True, preprocess_jpgs=False): print("Preprocessing data-set...") # Create the base-folder. #datetime_path = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") #base_path = os.path.join(preprocessed_root_path, datetime_path) #os.mkdir(base_path) #if preprocess_pcds == True: # os.mkdir(os.path.join(base_path, "pcd")) #if preprocess_jpgs == True: # os.mkdir(os.path.join(base_path, "jpg")) #print("Writing preprocessed data to {}...".format(base_path)) # Process the filtered PCDs. if preprocess_pcds == True: # Filter parameters. number_of_points_threshold=10000 confidence_avg_threshold=0.75 remove_unreasonable=True remove_errors=True remove_rejects=True # Save filter parameters. filter_parameters_path = os.path.join(base_path, "filter_parameters.txt") with open(filter_parameters_path, "w") as filter_parameters_file: filter_parameters_file.write("number_of_points_threshold" + "," + str(number_of_points_threshold) + "\n") filter_parameters_file.write("confidence_avg_threshold" + "," + str(confidence_avg_threshold) + "\n") filter_parameters_file.write("remove_unreasonable" + "," + str(remove_unreasonable) + "\n") filter_parameters_file.write("remove_errors" + "," + str(remove_errors) + "\n") filter_parameters_file.write("remove_rejects" + "," + str(remove_rejects) + "\n") # Get filtered entries. entries = filterpcds( number_of_points_threshold=number_of_points_threshold, confidence_avg_threshold=confidence_avg_threshold, remove_unreasonable=remove_unreasonable, remove_errors=remove_errors, remove_rejects=remove_rejects )["results"] print("Found {} PCDs. Processing...".format(len(entries))) # Method for processing a single entry. def process_pcd_entry(entry): path = entry["artifact_path"] if os.path.exists(path) == False: print("\n", "File {} does not exist!".format(path), "\n") return pointcloud = utils.load_pcd_as_ndarray(path) targets = np.array([entry["height"], entry["weight"]]) qrcode = entry["qr_code"] pickle_filename = os.path.basename(entry["artifact_path"]).replace(".pcd", ".p") qrcode_path = os.path.join(base_path, "pcd", qrcode) if os.path.exists(qrcode_path) == False: os.mkdir(qrcode_path) pickle_output_path = os.path.join(qrcode_path, pickle_filename) pickle.dump((pointcloud, targets), open(pickle_output_path, "wb")) # Start multiprocessing. utils.multiprocess(entries, process_pcd_entry) # Process the filtered JPGs. if preprocess_jpgs == True: assert False entries = filterjpgs()["results"] print("Found {} JPGs. Processing...".format(len(entries))) bar = progressbar.ProgressBar(max_value=len(entries)) # Method for processing a single entry. def process_jpg_entry(entry): path = entry["path"] if os.path.exists(path) == False: print("\n", "File {} does not exist!".format(path), "\n") return image = cv2.imread(path) targets = np.array([entry["height_cms"], entry["weight_kgs"]]) qrcode = entry["qrcode"] pickle_filename = os.path.basename(entry["path"]).replace(".jpg", ".p") qrcode_path = os.path.join(base_path, "jpg", qrcode) if os.path.exists(qrcode_path) == False: os.mkdir(qrcode_path) pickle_output_path = os.path.join(qrcode_path, pickle_filename) pickle.dump((image, targets), open(pickle_output_path, "wb")) # Start multiprocessing. utils.multiprocess(entries, process_pcd_entry)
def execute_command_preprocess(preprocess_pcds=True, preprocess_ply=False, preprocess_jpgs=False, preprocess_npy=False, preprocess_png=False, path_suffix=""): print("Preprocessing data-set...") print("Using '{}'".format(config.preprocessed_root_path)) if os.path.exists(config.preprocessed_root_path) == False: print("Folder does not exists. Creating...") os.mkdir(config.preprocessed_root_path) # Create the base-folder. if path_suffix != "": path_suffix = "-" + path_suffix datetime_path = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") base_path = os.path.join(config.preprocessed_root_path, datetime_path + path_suffix) os.mkdir(base_path) if preprocess_pcds == True: os.mkdir(os.path.join(base_path, "pcd")) if preprocess_ply == True: os.mkdir(os.path.join(base_path, "ply")) if preprocess_jpgs == True: os.mkdir(os.path.join(base_path, "jpg")) if preprocess_npy == True: os.mkdir(os.path.join(base_path, "npy")) print("Writing preprocessed data to {}...".format(base_path)) # Process the filtered PCDs. if preprocess_pcds == True: # Get entries. sql_statement = """ SELECT artifact_path, qr_code, height, weight FROM artifacts_with_targets WHERE type='pcd' AND status='standing' ; """ main_connector = dbutils.connect_to_main_database() entries = main_connector.execute(sql_statement, fetch_all=True) print("Found {} PCDs. Processing...".format(len(entries))) # Method for processing a single entry. def process_pcd_entry(entry): path, qr_code, height, weight = entry if os.path.exists(path) == False: print("\n", "File {} does not exist!".format(path), "\n") return try: pointcloud = utils.load_pcd_as_ndarray(path) targets = np.array([height, weight]) pickle_filename = os.path.basename(path).replace(".pcd", ".p") qrcode_path = os.path.join(base_path, "pcd", qr_code) if os.path.exists(qrcode_path) == False: os.mkdir(qrcode_path) pickle_output_path = os.path.join(qrcode_path, pickle_filename) pickle.dump((pointcloud, targets), open(pickle_output_path, "wb")) except: pass # Start multiprocessing. utils.multiprocess(entries, process_pcd_entry) if preprocess_ply == True: # Get entries. sql_statement = """ SELECT artifact_path, qr_code, height, weight FROM artifacts_with_targets WHERE type='pcrgb' AND status='standing' ; """ main_connector = dbutils.connect_to_main_database() entries = main_connector.execute(sql_statement, fetch_all=True) print("Found {} Plys. Processing...".format(len(entries))) # Method for processing a single entry. def process_ply_entry(entry): path, qr_code, height, weight = entry print(path) if os.path.exists(path) == False: print("\n", "File {} does not exist!".format(path), "\n") return try: pointcloud = utils.load_pcd_as_ndarray(path) targets = np.array([height, weight]) pickle_filename = os.path.basename(path).replace(".ply", ".p") qrcode_path = os.path.join(base_path, "ply", qr_code) # print('qr code path: ' + qrcode_path) # if not os.path.exists(qrcode_path): os.mkdir(qrcode_path) print('creating path') pickle_output_path = os.path.join(qrcode_path, pickle_filename) pickle.dump((pointcloud, targets), open(pickle_output_path, "wb")) except BaseException as e: print(e) pass # Start multiprocessing. utils.multiprocess(entries, process_ply_entry) # Process the filtered JPGs. if preprocess_jpgs == True: assert False entries = filterjpgs()["results"] print("Found {} JPGs. Processing...".format(len(entries))) bar = progressbar.ProgressBar(max_value=len(entries)) # Method for processing a single entry. def process_jpg_entry(entry): path = entry["path"] if os.path.exists(path) == False: print("\n", "File {} does not exist!".format(path), "\n") return image = cv2.imread(path) targets = np.array([entry["height_cms"], entry["weight_kgs"]]) qrcode = entry["qrcode"] pickle_filename = os.path.basename(entry["path"]).replace( ".jpg", ".p") qrcode_path = os.path.join(base_path, "jpg", qrcode) if os.path.exists(qrcode_path) == False: os.mkdir(qrcode_path) pickle_output_path = os.path.join(qrcode_path, pickle_filename) pickle.dump((image, targets), open(pickle_output_path, "wb")) # Start multiprocessing. utils.multiprocess(entries, process_pcd_entry) # Process depth images numpy array if preprocess_npy == True: # Get entries. sql_statement = """ SELECT artifact_path, qr_code, height, weight FROM artifacts_with_targets WHERE type='depth_npy' AND status='standing' ; """ main_connector = dbutils.connect_to_main_database() entries = main_connector.execute(sql_statement, fetch_all=True) print("Found {} depth_npy. Processing...".format(len(entries))) # Method for processing a single entry. def process_pcd_entry(entry): path, qr_code, height, weight = entry if os.path.exists(path) == False: print("\n", "File {} does not exist!".format(path), "\n") return try: print(path) np_array = np.load(path) print("opened file") targets = np.array([height, weight]) pickle_filename = os.path.basename(path).replace(".npy", ".p") qrcode_path = os.path.join(base_path, "npy", qr_code) if os.path.exists(qrcode_path) == False: os.mkdir(qrcode_path) pickle_output_path = os.path.join(qrcode_path, pickle_filename) pickle.dump((np_array, targets), open(pickle_output_path, "wb")) except: print("exception fails") pass # Start multiprocessing. utils.multiprocess(entries, process_pcd_entry) # print(preprocess_png) # preprocess_png = True # Process the filtered depth pngs. if preprocess_png == True: # Get entries. sql_statement = """ SELECT artifact_path, qr_code, height, weight FROM artifacts_with_targets WHERE type='depth_png' AND status='standing' ; """ main_connector = dbutils.connect_to_main_database() entries = main_connector.execute(sql_statement, fetch_all=True) print("Found {} depth_npy. Processing...".format(len(entries))) # Method for processing a single entry. def process_pcd_entry(entry): path, qr_code, height, weight = entry if os.path.exists(path) == False: print("\n", "File {} does not exist!".format(path), "\n") return try: print(path) np_array = cv2.imread(path, cv2.CV_8U) print(type(np_array)) print("opened file") print(np_array.shape) targets = np.array([height, weight]) pickle_filename = os.path.basename(path).replace(".png", ".p") qrcode_path = os.path.join(base_path, "png", qr_code) print(qrcode_path) # if os.path.exists(qrcode_path) == False: try: os.makedirs(qrcode_path, exist_ok=True) except: print("can not create folder") pass # print("path created") # os.mkdir(qrcode_path) print("after path created") print("in advance to pickle") pickle_output_path = os.path.join(qrcode_path, pickle_filename) print("created pickle") pickle.dump((np_array, targets), open(pickle_output_path, "wb")) except: print("Failed to import image") pass # Start multiprocessing. utils.multiprocess(entries, process_pcd_entry)
def main(): # Check the arguments. if len(sys.argv) != 2: raise Exception("ERROR! Must provide model filename.") model_path = sys.argv[1] if not os.path.exists(model_path): raise Exception("ERROR! \"{}\" does not exist.".format(model_path)) if not os.path.isfile(model_path): raise Exception("ERROR! \"{}\" is not a file.".format(model_path)) # Get the training QR-codes. search_path = os.path.join(os.path.dirname(model_path), "*.p") paths = glob.glob(search_path) details_path = [path for path in paths if "details" in path][0] details = pickle.load(open(details_path, "rb")) qrcodes_train = details["qrcodes_train"] # Create database connection. db_connector = dbutils.connect_to_main_database() # Query the database for artifacts. print("Getting all artifacts...") sql_statement = "" # Select all artifacts. sql_statement += "SELECT pointcloud_data.id, pointcloud_data.path, measurements.height_cms, pointcloud_data.qrcode FROM pointcloud_data" # Join them with measurements. sql_statement += " INNER JOIN measurements ON pointcloud_data.measurement_id=measurements.id" # Only take into account manual measurements. sql_statement += " WHERE measurements.type=\'manual\'" artifacts = db_connector.execute(sql_statement, fetch_all=True) print("Found {} artifacts.".format(len(artifacts))) # Method for processing a set of artifacts. # Note: This method will run in its own process. def process_artifacts(artifacts): # Create database connection. db_connector = dbutils.connect_to_main_database() # Load the model first. model = load_model(model_path) model_name = model_path.split("/")[-2] # Evaluate and create SQL-statements. bar = progressbar.ProgressBar(max_value=len(artifacts)) for artifact_index, artifact in enumerate(artifacts): bar.update(artifact_index) # Execute SQL statement. try: # Load the artifact and evaluate. artifact_id, pcd_path, target_height, qrcode = artifact pcd_array = utils.load_pcd_as_ndarray(pcd_path) pcd_array = utils.subsample_pointcloud(pcd_array, 10000) mse, mae = model.evaluate(np.expand_dims(pcd_array, axis=0), np.array([target_height]), verbose=0) if qrcode in qrcodes_train: misc = "training" else: misc = "nottraining" # Create an SQL statement. sql_statement = "" sql_statement += "INSERT INTO artifact_quality (type, key, value, artifact_id, misc)" sql_statement += " VALUES(\'{}\', \'{}\', \'{}\', \'{}\', \'{}\');".format(model_name, "mae", mae, artifact_id, misc) # Call database. result = db_connector.execute(sql_statement) except psycopg2.IntegrityError: print("Already in DB. Skipped.", pcd_path) except ValueError: print("Skipped.", pcd_path) bar.finish() # Run this in multiprocess mode. utils.multiprocess(artifacts, process_method=process_artifacts, process_individial_entries=False, progressbar=False) print("Done.")
def execute_command_updatemedia(update_jpgs=False, update_pcds=True): print("Updating media...") # Getting all files. print("Searching artifacts...") file_extensions = [] if update_jpgs == True: file_extensions.append("jpg") if update_pcds == True: file_extensions.append("pcd") artifact_paths = get_artifact_paths(file_extensions) print("Found {} artifacts.".format(len(artifact_paths))) # Method for multiprocessing. def process_artifact_paths(artifact_paths): main_connector = dbutils.connect_to_main_database() table = "artifact" batch_size = 100 insert_count = 0 no_measurements_count = 0 skip_count = 0 bar = progressbar.ProgressBar(max_value=len(artifact_paths)) sql_statement = "" last_index = len(artifact_paths) - 1 for index, artifact_path in enumerate(artifact_paths): bar.update(index) # Check if there is already an entry. basename = os.path.basename(artifact_path) sql_statement_select = dbutils.create_select_statement( "artifact", ["id"], [basename]) # TODO is this the proper id? results = main_connector.execute(sql_statement_select, fetch_all=True) # No results found. Insert. if len(results) == 0: insert_data = {} insert_data["id"] = basename # TODO proper? # Process the artifact. default_values = get_default_values(artifact_path, table, main_connector) if default_values != None: insert_data.update(default_values) sql_statement += dbutils.create_insert_statement( table, insert_data.keys(), insert_data.values()) insert_count += 1 else: no_measurements_count += 1 # Found a result. Update. elif len(results) != 0: skip_count += 1 # Update database. if index != 0 and ( (index % batch_size) == 0) or index == last_index: if sql_statement != "": result = main_connector.execute(sql_statement) sql_statement = "" bar.finish() print("Inserted {} new entries.".format(insert_count)) print("No measurements for {} entries.".format(no_measurements_count)) print("Skipped {} entries.".format(skip_count)) # Start multiprocessing. utils.multiprocess(artifact_paths, process_method=process_artifact_paths, process_individial_entries=False, progressbar=False, number_of_workers=1)
def execute_command_update_artifacts(update_jpgs=False, update_pcds=False, update_pcrgb=True, update_depth_npy=False, update_depth_png=False): # Get all persons. print("Finding all persons at '{}'...".format(config.artifacts_path)) person_search_path = os.path.join(config.artifacts_path, "*") person_paths = [ path for path in glob.glob(person_search_path) if os.path.isdir(path) ] # TODO speedup... only for development! #pickle.dump(person_paths, open("temp.p", "wb")) #person_paths = pickle.load(open("temp.p", "rb")) #person_paths = person_paths[0:20] # Deleting the table. Be careful! #print("DELETING TABLE!") #dbutils.connect_to_main_database().execute("DELETE FROM artifact;") # Statistics. print("Found {} persons.".format(len(person_paths))) # Decide on the file-extensions. file_extensions = [] if update_jpgs == True: file_extensions.append("jpg") if update_pcds == True: file_extensions.append("pcd") if update_pcrgb == True: file_extensions.append("ply") if update_depth_npy == True: file_extensions.append("npy") if update_depth_png == True: file_extensions.append("png") # This method is executed in multi-processing mode. def process_person_paths(person_paths, process_index): #person_paths = person_paths[0:4] # TODO remove this! # Go through each person (qr-code). for person_path in tqdm(person_paths, position=process_index): person_path = person_path.replace('localssd/', 'localssd2/') print(person_path) # Find all artifacts for that person. artifact_paths = [] for file_extension in file_extensions: print(file_extension) glob_search_path = os.path.join( person_path, "**/*.{}".format(file_extension)) #print (glob_search_path) artifact_paths.extend(glob.glob(glob_search_path)) # print(artifact_paths) print("Found {} artifacts in {}".format(len(artifact_paths), person_path)) # Process those artifacts. main_connector = dbutils.connect_to_main_database() table = "artifact" batch_size = 100 insert_count = 0 no_measurements_count = 0 skip_count = 0 sql_statement = "" last_index = len(artifact_paths) - 1 for artifact_index, artifact_path in enumerate(artifact_paths): # Check if there is already an entry in the database. basename = os.path.basename(artifact_path) sql_statement_select = dbutils.create_select_statement( "artifact", ["id"], [basename]) results = main_connector.execute(sql_statement_select, fetch_all=True) # No results found. Insert. if len(results) == 0: insert_data = {} insert_data["id"] = basename # TODO proper? # Get the default values for the artifact. default_values = get_default_values( artifact_path, table, main_connector) # Check if there is a measure_id. if "measure_id" in default_values.keys(): insert_count += 1 else: no_measurements_count += 1 # Create SQL statement. insert_data.update(default_values) sql_statement_for_artifact = dbutils.create_insert_statement( table, insert_data.keys(), insert_data.values()) sql_statement += sql_statement_for_artifact # Found a result. Update. elif len(results) != 0: skip_count += 1 # Update database. if artifact_index != 0 and ( (artifact_index % batch_size) == 0) or artifact_index == last_index: if sql_statement != "": result = main_connector.execute(sql_statement) sql_statement = "" # Return statistics. return (insert_count, no_measurements_count, skip_count) # Start multiprocessing. results = utils.multiprocess(person_paths, process_method=process_person_paths, process_individial_entries=False, pass_process_index=True, progressbar=False, number_of_workers=6) if results == None: print("\n") print("No results.") return # Aggregate results total_insert_count = 0 total_no_measurements_count = 0 total_skip_count = 0 for (insert_count, no_measurements_count, skip_count) in results: total_insert_count += insert_count total_no_measurements_count += no_measurements_count total_skip_count += skip_count print("\n") print("Inserted {} new entries.".format(total_insert_count)) print( "No measurements for {} entries.".format(total_no_measurements_count)) print("Skipped {} entries.".format(total_skip_count))