def user_properties(user_name, import_path, process_file_list, organization_username=None, organization_key=None, private=False, verbose=False): # basic try: user_properties = uploader.authenticate_user(user_name) except: print("Error, user authentication failed for user " + user_name) return None if not user_properties: print("Error, user authentication failed for user " + user_name) return None # organization validation if organization_username or organization_key: organization_key = process_organization(user_properties, organization_username, organization_key, private) user_properties.update({ 'MAPOrganizationKey': organization_key, 'MAPPrivate': private }) # remove uneeded credentials if "user_upload_token" in user_properties: del user_properties["user_upload_token"] if "user_permission_hash" in user_properties: del user_properties["user_permission_hash"] if "user_signature_hash" in user_properties: del user_properties["user_signature_hash"] return user_properties
def download(import_path, user_name, output_folder, number_threads=10, verbose=False): total_files = uploader.get_total_file_list(import_path) rows = [] local_mapping = save_local_mapping(import_path) signal.signal(signal.SIGTERM, service_shutdown) signal.signal(signal.SIGINT, service_shutdown) try: user_properties = uploader.authenticate_user(user_name) except: print("Error, user authentication failed for user " + user_name) print( "Make sure your user credentials are correct, user authentication is required for images to be downloaded from Mapillary." ) return None if "user_upload_token" in user_properties: token = user_properties["user_upload_token"] else: print("Error, failed to obtain user token, please try again.") return None do_sleep = False while not check_files_downloaded(local_mapping, output_folder, do_sleep): do_sleep = True lock = threading.Lock() downloaded_images = { "failed": 0, "nbr": 0, "success": 0, } threads = [] try: for i in range(number_threads): t = BlurDownloader(lock, downloaded_images, local_mapping, output_folder, token) threads.append(t) t.start() while True: any_alive = False for t in threads: any_alive = (any_alive or t.is_alive()) if not any_alive: break time.sleep(0.5) except ServiceExit: for t in threads: t.shutdown_flag.set() for t in threads: t.join() break
def user_properties(user_name, import_path, process_file_list, organization_username=None, organization_key=None, private=False, verbose=False): # basic try: user_properties = uploader.authenticate_user(user_name) except: print("Error, user authentication failed for user " + user_name) return None # organization validation if organization_username or organization_key: organization_key = process_organization(user_properties, organization_username, organization_key, private) user_properties.update( {'MAPOrganizationKey': organization_key, 'MAPPrivate': private}) # remove uneeded credentials if "user_upload_token" in user_properties: del user_properties["user_upload_token"] if "user_permission_hash" in user_properties: del user_properties["user_permission_hash"] if "user_signature_hash" in user_properties: del user_properties["user_signature_hash"] return user_properties
def download(import_path, user_name, output_folder, number_threads=10, verbose=False): total_files = uploader.get_total_file_list(import_path) rows = [] local_mapping = save_local_mapping(import_path) signal.signal(signal.SIGTERM, service_shutdown) signal.signal(signal.SIGINT, service_shutdown) try: user_properties = uploader.authenticate_user(user_name) except: print("Error, user authentication failed for user " + user_name) print("Make sure your user credentials are correct, user authentication is required for images to be downloaded from Mapillary.") return None if "user_upload_token" in user_properties: token = user_properties["user_upload_token"] else: print("Error, failed to obtain user token, please try again.") return None do_sleep = False while not check_files_downloaded(local_mapping, output_folder, do_sleep): do_sleep = True lock = threading.Lock() downloaded_images = { "failed": 0, "nbr": 0, "success": 0, } threads = [] try: for i in range(number_threads): t = BlurDownloader(lock, downloaded_images, local_mapping, output_folder, token) threads.append(t) t.start() while True: any_alive = False for t in threads: any_alive = (any_alive or t.is_alive()) if not any_alive: break time.sleep(0.5) except ServiceExit: for t in threads: t.shutdown_flag.set() for t in threads: t.join() break
def get_token(user_name): """ :param user_name: user name, string :type user_name: string :return authentication token for requests :rtype string or Exception/None """ try: user_properties = uploader.authenticate_user(user_name) except Exception: print("Error, user authentication failed for user " + user_name) print("Make sure your user credentials are correct, user authentication is required for images to be downloaded from Mapillary.") return None if "user_upload_token" in user_properties: return user_properties["user_upload_token"]
def user_properties(user_name, import_path, process_file_list, organization_username=None, organization_key=None, private=False, verbose=False): # basic try: user_properties = uploader.authenticate_user(user_name) except: print_error("Error, user authentication failed for user " + user_name) print( "Make sure your user credentials are correct, user authentication is required for images to be uploaded to Mapillary." ) return None if not user_properties: print_error("Error, user authentication failed for user " + user_name) print( "Make sure your user credentials are correct, user authentication is required for images to be uploaded to Mapillary." ) return None # organization validation if organization_username or organization_key: organization_key = process_organization(user_properties, organization_username, organization_key, private) user_properties.update({ 'MAPOrganizationKey': organization_key, 'MAPPrivate': private }) # remove uneeded credentials if "user_upload_token" in user_properties: del user_properties["user_upload_token"] if "user_permission_hash" in user_properties: del user_properties["user_permission_hash"] if "user_signature_hash" in user_properties: del user_properties["user_signature_hash"] if "aws_access_key_id" in user_properties: del user_properties["aws_access_key_id"] return user_properties
def process_upload_params(import_path, user_name, master_upload=False, verbose=False, rerun=False, skip_subfolders=False): # basic check for all import_path = os.path.abspath(import_path) if not os.path.isdir(import_path): print("Error, import directory " + import_path + " doesnt not exist, exiting...") sys.exit() # get list of file to process process_file_list = processing.get_process_file_list( import_path, "upload_params_process", rerun, verbose, skip_subfolders) if not len(process_file_list): print("No images to run upload params process") print( "If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun" ) # sanity checks if not user_name: print("Error, must provide a valid user name, exiting...") processing.create_and_log_process_in_list( process_file_list, "upload_params_process" "failed", verbose) return if not master_upload: try: credentials = uploader.authenticate_user(user_name) except: print("Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list( process_file_list, "upload_params_process" "failed", verbose) return if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials: print("Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list( process_file_list, "upload_params_process" "failed", verbose) return user_upload_token = credentials["user_upload_token"] user_permission_hash = credentials["user_permission_hash"] user_signature_hash = credentials["user_signature_hash"] user_key = credentials["MAPSettingsUserKey"] for image in process_file_list: # check the status of the sequence processing log_root = uploader.log_rootpath(image) duplicate_flag_path = os.path.join(log_root, "duplicate") upload_params_path = os.path.join(log_root, "upload_params_process.json") if os.path.isfile(upload_params_path): os.remove(upload_params_path) if os.path.isfile(duplicate_flag_path) or master_upload: continue upload_params_properties = processing.get_upload_param_properties( log_root, image, user_name, user_upload_token, user_permission_hash, user_signature_hash, user_key, verbose) processing.create_and_log_process(image, "upload_params_process", "success", upload_params_properties, verbose=verbose) # flag manual upload log_manual_upload = os.path.join(log_root, "manual_upload") open(log_manual_upload, 'a').close()
def process_upload_params(import_path, user_name, master_upload=False, verbose=False, rerun=False, skip_subfolders=False, video_import_path=None): # sanity check if video file is passed if video_import_path and not os.path.isdir( video_import_path) and not os.path.isfile(video_import_path): print("Error, video path " + video_import_path + " does not exist, exiting...") sys.exit(1) # in case of video processing, adjust the import path if video_import_path: # set sampling path video_sampling_path = "mapillary_sampled_video_frames" video_dirname = video_import_path if os.path.isdir( video_import_path) else os.path.dirname(video_import_path) import_path = os.path.join( os.path.abspath(import_path), video_sampling_path) if import_path else os.path.join( os.path.abspath(video_dirname), video_sampling_path) # basic check for all if not import_path or not os.path.isdir(import_path): print_error("Error, import directory " + import_path + " does not exist, exiting...") sys.exit(1) # get list of file to process process_file_list = processing.get_process_file_list( import_path, "upload_params_process", rerun, verbose, skip_subfolders) if not len(process_file_list): print("No images to run upload params process") print( "If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun" ) # sanity checks if not user_name: print_error("Error, must provide a valid user name, exiting...") processing.create_and_log_process_in_list( process_file_list, "upload_params_process" "failed", verbose) sys.exit(1) if not master_upload: try: credentials = uploader.authenticate_user(user_name) except: print_error("Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list( process_file_list, "upload_params_process" "failed", verbose) sys.exit(1) if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials: print_error("Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list( process_file_list, "upload_params_process" "failed", verbose) sys.exit(1) user_upload_token = credentials["user_upload_token"] user_permission_hash = credentials["user_permission_hash"] user_signature_hash = credentials["user_signature_hash"] user_key = credentials["MAPSettingsUserKey"] for image in tqdm(process_file_list, desc="Processing image upload parameters"): # check the status of the sequence processing log_root = uploader.log_rootpath(image) duplicate_flag_path = os.path.join(log_root, "duplicate") upload_params_path = os.path.join(log_root, "upload_params_process.json") if os.path.isfile(upload_params_path): os.remove(upload_params_path) if os.path.isfile(duplicate_flag_path) or master_upload: continue upload_params_properties = processing.get_upload_param_properties( log_root, image, user_name, user_upload_token, user_permission_hash, user_signature_hash, user_key, verbose) processing.create_and_log_process(image, "upload_params_process", "success", upload_params_properties, verbose=verbose) # flag manual upload log_manual_upload = os.path.join(log_root, "manual_upload") open(log_manual_upload, 'a').close() print("Sub process ended")
def process_upload_params(import_path, user_name, master_upload=False, verbose=False, rerun=False, skip_subfolders=False, video_import_path=None): # sanity check if video file is passed if video_import_path and not os.path.isdir(video_import_path) and not os.path.isfile(video_import_path): print("Error, video path " + video_import_path + " does not exist, exiting...") sys.exit(1) # in case of video processing, adjust the import path if video_import_path: # set sampling path video_sampling_path = "mapillary_sampled_video_frames" video_dirname = video_import_path if os.path.isdir( video_import_path) else os.path.dirname(video_import_path) import_path = os.path.join(os.path.abspath(import_path), video_sampling_path) if import_path else os.path.join( os.path.abspath(video_dirname), video_sampling_path) # basic check for all if not import_path or not os.path.isdir(import_path): print_error("Error, import directory " + import_path + " does not exist, exiting...") sys.exit(1) # get list of file to process process_file_list = processing.get_process_file_list(import_path, "upload_params_process", rerun, verbose, skip_subfolders) if not len(process_file_list): print("No images to run upload params process") print("If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun") # sanity checks if not user_name: print_error("Error, must provide a valid user name, exiting...") processing.create_and_log_process_in_list(process_file_list, "upload_params_process" "failed", verbose) sys.exit(1) if not master_upload: try: credentials = uploader.authenticate_user(user_name) except: print_error( "Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list(process_file_list, "upload_params_process" "failed", verbose) sys.exit(1) if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials: print_error( "Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list(process_file_list, "upload_params_process" "failed", verbose) sys.exit(1) user_upload_token = credentials["user_upload_token"] user_permission_hash = credentials["user_permission_hash"] user_signature_hash = credentials["user_signature_hash"] user_key = credentials["MAPSettingsUserKey"] for image in tqdm(process_file_list, desc="Processing image upload parameters"): # check the status of the sequence processing log_root = uploader.log_rootpath(image) duplicate_flag_path = os.path.join(log_root, "duplicate") upload_params_path = os.path.join( log_root, "upload_params_process.json") if os.path.isfile(upload_params_path): os.remove(upload_params_path) if os.path.isfile(duplicate_flag_path) or master_upload: continue upload_params_properties = processing.get_upload_param_properties(log_root, image, user_name, user_upload_token, user_permission_hash, user_signature_hash, user_key, verbose) processing.create_and_log_process(image, "upload_params_process", "success", upload_params_properties, verbose=verbose) # flag manual upload log_manual_upload = os.path.join( log_root, "manual_upload") open(log_manual_upload, 'a').close() print("Sub process ended")
def process_upload_params(import_path, user_name, master_upload=False, verbose=False, rerun=False, skip_subfolders=False): # basic check for all import_path = os.path.abspath(import_path) if not os.path.isdir(import_path): print("Error, import directory " + import_path + " doesnt not exist, exiting...") sys.exit() # get list of file to process process_file_list = processing.get_process_file_list(import_path, "upload_params_process", rerun, verbose, skip_subfolders) if not len(process_file_list): print("No images to run upload params process") print("If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun") # sanity checks if not user_name: print("Error, must provide a valid user name, exiting...") processing.create_and_log_process_in_list(process_file_list, import_path, "upload_params_process" "failed", verbose) return if not master_upload: try: credentials = uploader.authenticate_user(user_name) except: print("Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list(process_file_list, import_path, "upload_params_process" "failed", verbose) return if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials: print("Error, user authentication failed for user " + user_name) processing.create_and_log_process_in_list(process_file_list, import_path, "upload_params_process" "failed", verbose) return user_upload_token = credentials["user_upload_token"] user_permission_hash = credentials["user_permission_hash"] user_signature_hash = credentials["user_signature_hash"] user_key = credentials["MAPSettingsUserKey"] for image in process_file_list: # check the status of the sequence processing log_root = uploader.log_rootpath(import_path, image) duplicate_flag_path = os.path.join(log_root, "duplicate") upload_params_path = os.path.join( log_root, "upload_params_process.json") if os.path.isfile(upload_params_path): os.remove(upload_params_path) if os.path.isfile(duplicate_flag_path) or master_upload: continue upload_params_properties = processing.get_upload_param_properties(log_root, image, user_name, user_upload_token, user_permission_hash, user_signature_hash, user_key, verbose) processing.create_and_log_process(image, import_path, "upload_params_process", "success", upload_params_properties, verbose=verbose) # flag manual upload log_manual_upload = os.path.join( log_root, "manual_upload") open(log_manual_upload, 'a').close()