def user_properties(user_name,
import_path,
process_file_list,
organization_username=None,
organization_key=None,
private=False,
verbose=False):
# basic
try:
user_properties = uploader.authenticate_user(user_name)
except:
print("Error, user authentication failed for user " + user_name)
return None
if not user_properties:
print("Error, user authentication failed for user " + user_name)
return None
# organization validation
if organization_username or organization_key:
organization_key = process_organization(user_properties,
organization_username,
organization_key, private)
user_properties.update({
'MAPOrganizationKey': organization_key,
'MAPPrivate': private
})
# remove uneeded credentials
if "user_upload_token" in user_properties:
del user_properties["user_upload_token"]
if "user_permission_hash" in user_properties:
del user_properties["user_permission_hash"]
if "user_signature_hash" in user_properties:
del user_properties["user_signature_hash"]
return user_properties
def download(import_path,
user_name,
output_folder,
number_threads=10,
verbose=False):
total_files = uploader.get_total_file_list(import_path)
rows = []
local_mapping = save_local_mapping(import_path)
signal.signal(signal.SIGTERM, service_shutdown)
signal.signal(signal.SIGINT, service_shutdown)
try:
user_properties = uploader.authenticate_user(user_name)
except:
print("Error, user authentication failed for user " + user_name)
print(
"Make sure your user credentials are correct, user authentication is required for images to be downloaded from Mapillary."
)
return None
if "user_upload_token" in user_properties:
token = user_properties["user_upload_token"]
else:
print("Error, failed to obtain user token, please try again.")
return None
do_sleep = False
while not check_files_downloaded(local_mapping, output_folder, do_sleep):
do_sleep = True
lock = threading.Lock()
downloaded_images = {
"failed": 0,
"nbr": 0,
"success": 0,
}
threads = []
try:
for i in range(number_threads):
t = BlurDownloader(lock, downloaded_images, local_mapping,
output_folder, token)
threads.append(t)
t.start()
while True:
any_alive = False
for t in threads:
any_alive = (any_alive or t.is_alive())
if not any_alive:
break
time.sleep(0.5)
except ServiceExit:
for t in threads:
t.shutdown_flag.set()
for t in threads:
t.join()
break
def user_properties(user_name,
import_path,
process_file_list,
organization_username=None,
organization_key=None,
private=False,
verbose=False):
# basic
try:
user_properties = uploader.authenticate_user(user_name)
except:
print("Error, user authentication failed for user " + user_name)
return None
# organization validation
if organization_username or organization_key:
organization_key = process_organization(user_properties,
organization_username,
organization_key,
private)
user_properties.update(
{'MAPOrganizationKey': organization_key, 'MAPPrivate': private})
# remove uneeded credentials
if "user_upload_token" in user_properties:
del user_properties["user_upload_token"]
if "user_permission_hash" in user_properties:
del user_properties["user_permission_hash"]
if "user_signature_hash" in user_properties:
del user_properties["user_signature_hash"]
return user_properties
def download(import_path, user_name, output_folder, number_threads=10, verbose=False):
total_files = uploader.get_total_file_list(import_path)
rows = []
local_mapping = save_local_mapping(import_path)
signal.signal(signal.SIGTERM, service_shutdown)
signal.signal(signal.SIGINT, service_shutdown)
try:
user_properties = uploader.authenticate_user(user_name)
except:
print("Error, user authentication failed for user " + user_name)
print("Make sure your user credentials are correct, user authentication is required for images to be downloaded from Mapillary.")
return None
if "user_upload_token" in user_properties:
token = user_properties["user_upload_token"]
else:
print("Error, failed to obtain user token, please try again.")
return None
do_sleep = False
while not check_files_downloaded(local_mapping, output_folder, do_sleep):
do_sleep = True
lock = threading.Lock()
downloaded_images = {
"failed": 0,
"nbr": 0,
"success": 0,
}
threads = []
try:
for i in range(number_threads):
t = BlurDownloader(lock, downloaded_images,
local_mapping, output_folder, token)
threads.append(t)
t.start()
while True:
any_alive = False
for t in threads:
any_alive = (any_alive or t.is_alive())
if not any_alive:
break
time.sleep(0.5)
except ServiceExit:
for t in threads:
t.shutdown_flag.set()
for t in threads:
t.join()
break
def get_token(user_name):
"""
:param user_name: user name, string
:type user_name: string
:return authentication token for requests
:rtype string or Exception/None
"""
try:
user_properties = uploader.authenticate_user(user_name)
except Exception:
print("Error, user authentication failed for user " + user_name)
print("Make sure your user credentials are correct, user authentication is required for images to be downloaded from Mapillary.")
return None
if "user_upload_token" in user_properties:
return user_properties["user_upload_token"]
def get_token(user_name):
"""
:param user_name: user name, string
:type user_name: string
:return authentication token for requests
:rtype string or Exception/None
"""
try:
user_properties = uploader.authenticate_user(user_name)
except Exception:
print("Error, user authentication failed for user " + user_name)
print("Make sure your user credentials are correct, user authentication is required for images to be downloaded from Mapillary.")
return None
if "user_upload_token" in user_properties:
return user_properties["user_upload_token"]
def user_properties(user_name,
import_path,
process_file_list,
organization_username=None,
organization_key=None,
private=False,
verbose=False):
# basic
try:
user_properties = uploader.authenticate_user(user_name)
except:
print_error("Error, user authentication failed for user " + user_name)
print(
"Make sure your user credentials are correct, user authentication is required for images to be uploaded to Mapillary."
)
return None
if not user_properties:
print_error("Error, user authentication failed for user " + user_name)
print(
"Make sure your user credentials are correct, user authentication is required for images to be uploaded to Mapillary."
)
return None
# organization validation
if organization_username or organization_key:
organization_key = process_organization(user_properties,
organization_username,
organization_key, private)
user_properties.update({
'MAPOrganizationKey': organization_key,
'MAPPrivate': private
})
# remove uneeded credentials
if "user_upload_token" in user_properties:
del user_properties["user_upload_token"]
if "user_permission_hash" in user_properties:
del user_properties["user_permission_hash"]
if "user_signature_hash" in user_properties:
del user_properties["user_signature_hash"]
if "aws_access_key_id" in user_properties:
del user_properties["aws_access_key_id"]
return user_properties
def process_upload_params(import_path,
user_name,
master_upload=False,
verbose=False,
rerun=False,
skip_subfolders=False):
# basic check for all
import_path = os.path.abspath(import_path)
if not os.path.isdir(import_path):
print("Error, import directory " + import_path +
" doesnt not exist, exiting...")
sys.exit()
# get list of file to process
process_file_list = processing.get_process_file_list(
import_path, "upload_params_process", rerun, verbose, skip_subfolders)
if not len(process_file_list):
print("No images to run upload params process")
print(
"If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun"
)
# sanity checks
if not user_name:
print("Error, must provide a valid user name, exiting...")
processing.create_and_log_process_in_list(
process_file_list, "upload_params_process"
"failed", verbose)
return
if not master_upload:
try:
credentials = uploader.authenticate_user(user_name)
except:
print("Error, user authentication failed for user " + user_name)
processing.create_and_log_process_in_list(
process_file_list, "upload_params_process"
"failed", verbose)
return
if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials:
print("Error, user authentication failed for user " + user_name)
processing.create_and_log_process_in_list(
process_file_list, "upload_params_process"
"failed", verbose)
return
user_upload_token = credentials["user_upload_token"]
user_permission_hash = credentials["user_permission_hash"]
user_signature_hash = credentials["user_signature_hash"]
user_key = credentials["MAPSettingsUserKey"]
for image in process_file_list:
# check the status of the sequence processing
log_root = uploader.log_rootpath(image)
duplicate_flag_path = os.path.join(log_root, "duplicate")
upload_params_path = os.path.join(log_root,
"upload_params_process.json")
if os.path.isfile(upload_params_path):
os.remove(upload_params_path)
if os.path.isfile(duplicate_flag_path) or master_upload:
continue
upload_params_properties = processing.get_upload_param_properties(
log_root, image, user_name, user_upload_token,
user_permission_hash, user_signature_hash, user_key, verbose)
processing.create_and_log_process(image,
"upload_params_process",
"success",
upload_params_properties,
verbose=verbose)
# flag manual upload
log_manual_upload = os.path.join(log_root, "manual_upload")
open(log_manual_upload, 'a').close()
def process_upload_params(import_path,
user_name,
master_upload=False,
verbose=False,
rerun=False,
skip_subfolders=False,
video_import_path=None):
# sanity check if video file is passed
if video_import_path and not os.path.isdir(
video_import_path) and not os.path.isfile(video_import_path):
print("Error, video path " + video_import_path +
" does not exist, exiting...")
sys.exit(1)
# in case of video processing, adjust the import path
if video_import_path:
# set sampling path
video_sampling_path = "mapillary_sampled_video_frames"
video_dirname = video_import_path if os.path.isdir(
video_import_path) else os.path.dirname(video_import_path)
import_path = os.path.join(
os.path.abspath(import_path),
video_sampling_path) if import_path else os.path.join(
os.path.abspath(video_dirname), video_sampling_path)
# basic check for all
if not import_path or not os.path.isdir(import_path):
print_error("Error, import directory " + import_path +
" does not exist, exiting...")
sys.exit(1)
# get list of file to process
process_file_list = processing.get_process_file_list(
import_path, "upload_params_process", rerun, verbose, skip_subfolders)
if not len(process_file_list):
print("No images to run upload params process")
print(
"If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun"
)
# sanity checks
if not user_name:
print_error("Error, must provide a valid user name, exiting...")
processing.create_and_log_process_in_list(
process_file_list, "upload_params_process"
"failed", verbose)
sys.exit(1)
if not master_upload:
try:
credentials = uploader.authenticate_user(user_name)
except:
print_error("Error, user authentication failed for user " +
user_name)
processing.create_and_log_process_in_list(
process_file_list, "upload_params_process"
"failed", verbose)
sys.exit(1)
if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials:
print_error("Error, user authentication failed for user " +
user_name)
processing.create_and_log_process_in_list(
process_file_list, "upload_params_process"
"failed", verbose)
sys.exit(1)
user_upload_token = credentials["user_upload_token"]
user_permission_hash = credentials["user_permission_hash"]
user_signature_hash = credentials["user_signature_hash"]
user_key = credentials["MAPSettingsUserKey"]
for image in tqdm(process_file_list,
desc="Processing image upload parameters"):
# check the status of the sequence processing
log_root = uploader.log_rootpath(image)
duplicate_flag_path = os.path.join(log_root, "duplicate")
upload_params_path = os.path.join(log_root,
"upload_params_process.json")
if os.path.isfile(upload_params_path):
os.remove(upload_params_path)
if os.path.isfile(duplicate_flag_path) or master_upload:
continue
upload_params_properties = processing.get_upload_param_properties(
log_root, image, user_name, user_upload_token,
user_permission_hash, user_signature_hash, user_key, verbose)
processing.create_and_log_process(image,
"upload_params_process",
"success",
upload_params_properties,
verbose=verbose)
# flag manual upload
log_manual_upload = os.path.join(log_root, "manual_upload")
open(log_manual_upload, 'a').close()
print("Sub process ended")
def process_upload_params(import_path,
user_name,
master_upload=False,
verbose=False,
rerun=False,
skip_subfolders=False,
video_import_path=None):
# sanity check if video file is passed
if video_import_path and not os.path.isdir(video_import_path) and not os.path.isfile(video_import_path):
print("Error, video path " + video_import_path +
" does not exist, exiting...")
sys.exit(1)
# in case of video processing, adjust the import path
if video_import_path:
# set sampling path
video_sampling_path = "mapillary_sampled_video_frames"
video_dirname = video_import_path if os.path.isdir(
video_import_path) else os.path.dirname(video_import_path)
import_path = os.path.join(os.path.abspath(import_path), video_sampling_path) if import_path else os.path.join(
os.path.abspath(video_dirname), video_sampling_path)
# basic check for all
if not import_path or not os.path.isdir(import_path):
print_error("Error, import directory " + import_path +
" does not exist, exiting...")
sys.exit(1)
# get list of file to process
process_file_list = processing.get_process_file_list(import_path,
"upload_params_process",
rerun,
verbose,
skip_subfolders)
if not len(process_file_list):
print("No images to run upload params process")
print("If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun")
# sanity checks
if not user_name:
print_error("Error, must provide a valid user name, exiting...")
processing.create_and_log_process_in_list(process_file_list,
"upload_params_process"
"failed",
verbose)
sys.exit(1)
if not master_upload:
try:
credentials = uploader.authenticate_user(user_name)
except:
print_error(
"Error, user authentication failed for user " + user_name)
processing.create_and_log_process_in_list(process_file_list,
"upload_params_process"
"failed",
verbose)
sys.exit(1)
if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials:
print_error(
"Error, user authentication failed for user " + user_name)
processing.create_and_log_process_in_list(process_file_list,
"upload_params_process"
"failed",
verbose)
sys.exit(1)
user_upload_token = credentials["user_upload_token"]
user_permission_hash = credentials["user_permission_hash"]
user_signature_hash = credentials["user_signature_hash"]
user_key = credentials["MAPSettingsUserKey"]
for image in tqdm(process_file_list, desc="Processing image upload parameters"):
# check the status of the sequence processing
log_root = uploader.log_rootpath(image)
duplicate_flag_path = os.path.join(log_root,
"duplicate")
upload_params_path = os.path.join(
log_root, "upload_params_process.json")
if os.path.isfile(upload_params_path):
os.remove(upload_params_path)
if os.path.isfile(duplicate_flag_path) or master_upload:
continue
upload_params_properties = processing.get_upload_param_properties(log_root,
image,
user_name,
user_upload_token,
user_permission_hash,
user_signature_hash,
user_key,
verbose)
processing.create_and_log_process(image,
"upload_params_process",
"success",
upload_params_properties,
verbose=verbose)
# flag manual upload
log_manual_upload = os.path.join(
log_root, "manual_upload")
open(log_manual_upload, 'a').close()
print("Sub process ended")
def process_upload_params(import_path,
user_name,
master_upload=False,
verbose=False,
rerun=False,
skip_subfolders=False):
# basic check for all
import_path = os.path.abspath(import_path)
if not os.path.isdir(import_path):
print("Error, import directory " + import_path +
" doesnt not exist, exiting...")
sys.exit()
# get list of file to process
process_file_list = processing.get_process_file_list(import_path,
"upload_params_process",
rerun,
verbose,
skip_subfolders)
if not len(process_file_list):
print("No images to run upload params process")
print("If the images have already been processed and not yet uploaded, they can be processed again, by passing the argument --rerun")
# sanity checks
if not user_name:
print("Error, must provide a valid user name, exiting...")
processing.create_and_log_process_in_list(process_file_list,
import_path,
"upload_params_process"
"failed",
verbose)
return
if not master_upload:
try:
credentials = uploader.authenticate_user(user_name)
except:
print("Error, user authentication failed for user " + user_name)
processing.create_and_log_process_in_list(process_file_list,
import_path,
"upload_params_process"
"failed",
verbose)
return
if credentials == None or "user_upload_token" not in credentials or "user_permission_hash" not in credentials or "user_signature_hash" not in credentials:
print("Error, user authentication failed for user " + user_name)
processing.create_and_log_process_in_list(process_file_list,
import_path,
"upload_params_process"
"failed",
verbose)
return
user_upload_token = credentials["user_upload_token"]
user_permission_hash = credentials["user_permission_hash"]
user_signature_hash = credentials["user_signature_hash"]
user_key = credentials["MAPSettingsUserKey"]
for image in process_file_list:
# check the status of the sequence processing
log_root = uploader.log_rootpath(import_path,
image)
duplicate_flag_path = os.path.join(log_root,
"duplicate")
upload_params_path = os.path.join(
log_root, "upload_params_process.json")
if os.path.isfile(upload_params_path):
os.remove(upload_params_path)
if os.path.isfile(duplicate_flag_path) or master_upload:
continue
upload_params_properties = processing.get_upload_param_properties(log_root,
image,
user_name,
user_upload_token,
user_permission_hash,
user_signature_hash,
user_key,
verbose)
processing.create_and_log_process(image,
import_path,
"upload_params_process",
"success",
upload_params_properties,
verbose=verbose)
# flag manual upload
log_manual_upload = os.path.join(
log_root, "manual_upload")
open(log_manual_upload, 'a').close()
