def list_images(directory):
'''
Create a list of image tuples sorted by capture timestamp.
@param directory: directory with JPEG files
@return: a list of image tuples with time, directory, lat,long...
'''
file_list = []
for root, sub_folders, files in os.walk(directory):
file_list += [
os.path.join(root, filename) for filename in files
if filename.lower().endswith(".jpg")
]
files = []
# get GPS data from the images and sort the list by timestamp
for filepath in file_list:
metadata = EXIF(filepath)
try:
t = metadata.extract_capture_time()
#lat = metadata["Exif.GPSInfo.GPSLatitude"].value
#latRef = metadata["Exif.GPSInfo.GPSLatitudeRef"].value
#lon = metadata["Exif.GPSInfo.GPSLongitude"].value
#lonRef = metadata["Exif.GPSInfo.GPSLongitudeRef"].value
geo = metadata.extract_geo()
direction = metadata.extract_direction()
files.append(
(filepath, geo["latitude"], geo["longitude"], direction))
except KeyError, e:
# if any of the required tags are not set the image is not added to the list
print("Skipping {0}: {1}".format(filename, e))
def _read_lat_lon(self, filename):
'''
Use EXIF class to parse latitude and longitude from EXIF.
'''
exif = EXIF(filename)
lon, lat = exif.extract_lon_lat()
return lat, lon
def _read_direction(self, filename):
'''
Use EXIF class to parse compass direction from EXIF.
'''
exif = EXIF(filename)
direction = exif.extract_direction()
return direction
def _read_lat_lon(self, filename):
'''
Use EXIF class to parse latitude and longitude from EXIF.
'''
exif = EXIF(filename)
lon, lat = exif.extract_lon_lat()
return lat, lon
def list_images(directory):
"""
Create a list of image tuples sorted by capture timestamp.
@param directory: directory with JPEG files
@return: a list of image tuples with time, directory, lat,long...
"""
file_list = []
for root, sub_folders, files in os.walk(directory):
file_list += [os.path.join(root, filename) for filename in files if filename.lower().endswith(".jpg")]
files = []
# get DateTimeOriginal data from the images and sort the list by timestamp
for filepath in file_list:
metadata = EXIF(filepath)
try:
t = metadata.extract_capture_time()
s = int(t.microsecond / 1000000)
files.append(Picture_infos(filepath, t, s))
# print t
# print type(t)
except KeyError as e:
# if any of the required tags are not set the image is not added to the list
print("Skipping {0}: {1}".format(filepath, e))
files.sort(key=lambda file: file.DateTimeOriginal)
# print_list(files)
return files
def _read_direction(self, filename):
'''
Use EXIF class to parse compass direction from EXIF.
'''
exif = EXIF(filename)
direction = exif.extract_direction()
return direction
def create_mapillary_description(filename,
username,
email,
upload_hash,
sequence_uuid,
interpolated_heading=None,
orientation=1,
verbose=False):
'''
Check that image file has the required EXIF fields.
Incompatible files will be ignored server side.
'''
# read exif
exif = EXIF(filename)
if not verify_exif(filename):
return False
# write the mapillary tag
mapillary_description = {}
mapillary_description["MAPLongitude"], mapillary_description[
"MAPLatitude"] = exif.extract_lon_lat()
#required date format: 2015_01_14_09_37_01_000
mapillary_description["MAPCaptureTime"] = datetime.datetime.strftime(
exif.extract_capture_time(), "%Y_%m_%d_%H_%M_%S_%f")[:-3]
mapillary_description["MAPOrientation"] = exif.extract_orientation()
heading = exif.extract_direction()
if heading is None:
heading = 0.0
heading = normalize_bearing(
interpolated_heading
) if interpolated_heading is not None else normalize_bearing(heading)
mapillary_description["MAPCompassHeading"] = {
"TrueHeading": heading,
"MagneticHeading": heading
}
mapillary_description["MAPSettingsUploadHash"] = upload_hash
mapillary_description["MAPSettingsEmail"] = email
mapillary_description["MAPSettingsUsername"] = username
settings_upload_hash = hashlib.sha256(
"%s%s%s" %
(upload_hash, email, base64.b64encode(filename))).hexdigest()
mapillary_description['MAPSettingsUploadHash'] = settings_upload_hash
mapillary_description['MAPPhotoUUID'] = str(uuid.uuid4())
mapillary_description['MAPSequenceUUID'] = str(sequence_uuid)
mapillary_description['MAPDeviceModel'] = exif.extract_model()
mapillary_description['MAPDeviceMake'] = exif.extract_make()
# write to file
json_desc = json.dumps(mapillary_description)
if verbose:
print "tag: {0}".format(json_desc)
metadata = ExifEdit(filename)
metadata.add_image_description(json_desc)
metadata.add_orientation(orientation)
metadata.add_direction(heading)
metadata.write()
def create_mapillary_description(filename, username, email,
upload_hash, sequence_uuid,
interpolated_heading=None,
orientation=1,
verbose=False):
'''
Check that image file has the required EXIF fields.
Incompatible files will be ignored server side.
'''
# read exif
exif = EXIF(filename)
if not verify_exif(filename):
return False
# write the mapillary tag
mapillary_description = {}
mapillary_description["MAPLongitude"], mapillary_description["MAPLatitude"] = exif.extract_lon_lat()
#required date format: 2015_01_14_09_37_01_000
mapillary_description["MAPCaptureTime"] = datetime.datetime.strftime(exif.extract_capture_time(), "%Y_%m_%d_%H_%M_%S_%f")[:-3]
mapillary_description["MAPOrientation"] = exif.extract_orientation()
heading = exif.extract_direction()
if heading is None:
heading = 0.0
heading = normalize_bearing(interpolated_heading) if interpolated_heading is not None else normalize_bearing(heading)
mapillary_description["MAPCompassHeading"] = {"TrueHeading": heading, "MagneticHeading": heading}
mapillary_description["MAPSettingsUploadHash"] = upload_hash
mapillary_description["MAPSettingsEmail"] = email
mapillary_description["MAPSettingsUsername"] = username
settings_upload_hash = hashlib.sha256("%s%s%s" % (upload_hash, email, base64.b64encode(filename))).hexdigest()
mapillary_description['MAPSettingsUploadHash'] = settings_upload_hash
mapillary_description['MAPPhotoUUID'] = str(uuid.uuid4())
mapillary_description['MAPSequenceUUID'] = str(sequence_uuid)
mapillary_description['MAPDeviceModel'] = exif.extract_model()
mapillary_description['MAPDeviceMake'] = exif.extract_make()
# write to file
json_desc = json.dumps(mapillary_description)
if verbose:
print "tag: {0}".format(json_desc)
metadata = ExifEdit(filename)
metadata.add_image_description(json_desc)
metadata.add_orientation(orientation)
metadata.add_direction(heading)
metadata.write()
def _read_capture_time(self, filename):
'''
Use EXIF class to parse capture time from EXIF.
'''
exif = EXIF(filename)
return exif.extract_capture_time()
# Handle EXIF errors
num_missing_date = 0
num_exif_error = 0
failed_folder = os.path.join(data_path, "failed")
io.mkdir_p(failed_folder)
valid_files = []
print("\nAdding GPS ...")
# Add GPS positions
for i, p in enumerate(meta["photos"]):
photo_file = p["photo_file_path"]
if os.path.isfile(photo_file):
exif = EXIF(photo_file)
try:
exif = EXIF(photo_file)
except Exception as e:
print "EXIF read error '{}' for image {}. Skipping".format(
e, photo_file)
shutil.move(
photo_file,
os.path.join(failed_folder, os.path.basename(photo_file)))
num_exif_error += 1
continue
try:
capture_time = exif.extract_capture_time()
except Exception as e:
print "Datetime error '{}' for image {}. Skipping".format(
action='store_true',
dest='backup',
help='store backup of overwritten values in Mapillary JSON')
args = parser.parse_args()
if args.orientation is not None:
exifOrientation = format_orientation(args.orientation)
s = Sequence(args.path)
if args.interpolate:
bearings = s.interpolate_direction()
for filename in s.get_file_list(args.path):
stat = os.stat(filename)
exifRead = EXIF(filename)
mapillaryTag = json.loads(exifRead.extract_image_description())
if args.interpolate:
bearing = bearings[filename]
else:
bearing = exifRead.extract_direction()
if args.offset:
bearing = offset_bearing(bearing, args.offset)
exifEdit = ExifEdit(filename)
if args.interpolate or args.offset:
exifEdit.add_direction(bearing, precision=10)
if (args.backup):
def exif_time(filename):
'''
Get image capture time from exif
'''
metadata = EXIF(filename)
return metadata.extract_capture_time()
# Handle EXIF errors
num_missing_date = 0
num_exif_error = 0
failed_folder = os.path.join(data_path, "failed")
io.mkdir_p(failed_folder)
valid_files = []
print("\nAdding GPS ...")
# Add GPS positions
for i, p in enumerate(meta["photos"]):
photo_file = p["photo_file_path"]
if os.path.isfile(photo_file):
exif = EXIF(photo_file)
try:
exif = EXIF(photo_file)
except Exception as e:
print "EXIF read error '{}' for image {}. Skipping".format(e, photo_file)
shutil.move(photo_file, os.path.join(failed_folder, os.path.basename(photo_file)))
num_exif_error += 1
continue
try:
capture_time = exif.extract_capture_time()
except Exception as e:
print "Datetime error '{}' for image {}. Skipping".format(e, photo_file)
shutil.move(photo_file, os.path.join(failed_folder, os.path.basename(photo_file)))
num_exif_error += 1
continue
# get the list of images in the folder
# Caution: all nested folders will be merged into one sequence!
s = Sequence(path, skip_folders=['success', 'duplicates'], skip_subfolders=skip_subfolders)
if len(s.file_list) == 0:
print('No images in the folder or all images have all ready been uploaded to Mapillary')
print('Note: If upload fails mid-sequence due to connection failure or similar, you should manually push the images to the server at http://www.mapillary.com/map/upload/im/ and pressing "push to Mapillary".')
sys.exit()
print("Uploading sequence {0}.".format(sequence_id))
# check mapillary tag and required exif
file_list = []
for filepath in s.file_list:
mapillary_tag_exists = EXIF(filepath).mapillary_tag_exists()
if mapillary_tag_exists:
print("File {} contains Mapillary EXIF tags, use upload.py instead.".format(filepath))
required_exif_exist = verify_exif(filepath)
if not required_exif_exist:
print("File {} missing required exif".format(filepath))
if required_exif_exist and (not mapillary_tag_exists):
file_list.append(filepath)
#upload valid files
upload_file_list(file_list, params)
# ask user if finalize upload to check that everything went fine
print("===\nFinalizing upload will submit all successful uploads and ignore all failed.\nIf all files were marked as successful, everything is fine, just press 'y'.")
sys.exit()
if path.lower().endswith(".jpg"):
# single file
file_list = [path]
else:
# folder(s)
file_list = []
for root, sub_folders, files in os.walk(path):
file_list += [
os.path.join(root, filename) for filename in files
if filename.lower().endswith(".jpg")
]
num_file = len(file_list)
for i, filepath in enumerate(file_list):
exif = EXIF(filepath)
description_ = exif.extract_image_description()
exif_edit = ExifEdit(filepath)
try:
description_ = json.loads(description_)
except:
description_ = {}
if 'MAPSettingsProject' not in description_ or overwrite:
description_['MAPSettingsProject'] = project_key
description_ = {}
exif_edit.add_image_description(description_)
exif_edit.write()
print("Done, processed %s files" % len(file_list))
def create_mapillary_description(filename,
username,
email,
userkey,
upload_hash,
sequence_uuid,
interpolated_heading=None,
offset_angle=0.0,
timestamp=None,
orientation=None,
project="",
secret_hash=None,
external_properties=None,
verbose=False):
'''
Check that image file has the required EXIF fields.
Incompatible files will be ignored server side.
'''
# read exif
exif = EXIF(filename)
if not verify_exif(filename):
return False
if orientation is None:
orientation = exif.extract_orientation()
# write the mapillary tag
mapillary_description = {}
mapillary_description["MAPLongitude"], mapillary_description[
"MAPLatitude"] = exif.extract_lon_lat()
mapillary_description["MAPAltitude"] = exif.extract_altitude()
# capture time: required date format: 2015_01_14_09_37_01_000
if timestamp is None:
timestamp = exif.extract_capture_time()
mapillary_description["MAPCaptureTime"] = datetime.datetime.strftime(
timestamp, "%Y_%m_%d_%H_%M_%S_%f")[:-3]
mapillary_description["MAPOrientation"] = orientation
heading = exif.extract_direction()
if heading is None:
heading = 0.0
heading = normalize_bearing(
interpolated_heading + offset_angle
) if interpolated_heading is not None else normalize_bearing(heading +
offset_angle)
mapillary_description["MAPCompassHeading"] = {
"TrueHeading": heading,
"MagneticHeading": heading
}
# authentication
assert (email is not None or userkey is not None)
if email is not None:
mapillary_description["MAPSettingsEmail"] = email
if username is not None:
mapillary_description["MAPSettingsUsername"] = username
if userkey is not None:
mapillary_description["MAPSettingsUserKey"] = userkey
if upload_hash is not None:
settings_upload_hash = hashlib.sha256(
"%s%s%s" %
(upload_hash, email, base64.b64encode(filename))).hexdigest()
mapillary_description['MAPSettingsUploadHash'] = settings_upload_hash
mapillary_description['MAPPhotoUUID'] = str(uuid.uuid4())
mapillary_description['MAPSequenceUUID'] = str(sequence_uuid)
mapillary_description['MAPDeviceModel'] = exif.extract_model()
mapillary_description['MAPDeviceMake'] = exif.extract_make()
if upload_hash is None and secret_hash is not None:
mapillary_description['MAPVideoSecure'] = secret_hash
mapillary_description["MAPSettingsProject"] = project
# external properties
if external_properties is not None:
mapillary_description['MAPExternalProperties'] = external_properties
# write to file
if verbose:
print("tag: {0}".format(mapillary_description))
metadata = ExifEdit(filename)
metadata.add_image_description(mapillary_description)
metadata.add_orientation(orientation)
metadata.add_direction(heading)
metadata.write()
def create_mapillary_description(filename,
username,
email,
userkey,
upload_hash,
sequence_uuid,
interpolated_heading=None,
offset_angle=0.0,
timestamp=None,
orientation=None,
project="",
secret_hash=None,
external_properties=None,
verbose=False,
make="",
model=""):
'''
Check that image file has the required EXIF fields.
Incompatible files will be ignored server side.
'''
# read exif
exif = EXIF(filename)
if not verify_exif(filename):
return False
if orientation is None:
orientation = exif.extract_orientation()
# write the mapillary tag
mapillary_description = {}
# lat, lon of the image, takes precedence over EXIF GPS values
mapillary_description["MAPLongitude"], mapillary_description[
"MAPLatitude"] = exif.extract_lon_lat()
# altitude of the image, takes precedence over EXIF GPS values, assumed 0
# if missing
mapillary_description["MAPAltitude"] = exif.extract_altitude()
# capture time: required date format: 2015_01_14_09_37_01_000, TZ MUST be
# UTC
if timestamp is None:
timestamp = exif.extract_capture_time()
# The capture time of the image in UTC. Will take precedence over any
# other time tags in the EXIF
mapillary_description["MAPCaptureTime"] = datetime.datetime.strftime(
timestamp, "%Y_%m_%d_%H_%M_%S_%f")[:-3]
# EXIF orientation of the image
mapillary_description["MAPOrientation"] = orientation
heading = exif.extract_direction()
if heading is None:
heading = 0.0
heading = normalize_bearing(
interpolated_heading + offset_angle
) if interpolated_heading is not None else normalize_bearing(heading +
offset_angle)
# bearing of the image
mapillary_description["MAPCompassHeading"] = {
"TrueHeading": heading,
"MagneticHeading": heading
}
# authentication
assert (email is not None or userkey is not None)
if email is not None:
mapillary_description["MAPSettingsEmail"] = email
if username is not None:
mapillary_description["MAPSettingsUsername"] = username
# use this if available, and omit MAPSettingsUsername and MAPSettingsEmail
# for privacy reasons
if userkey is not None:
mapillary_description["MAPSettingsUserKey"] = userkey
if upload_hash is not None:
settings_upload_hash = hashlib.sha256(
"%s%s%s" %
(upload_hash, email, base64.b64encode(filename))).hexdigest()
# this is not checked in the backend right now, will likely be changed to have user_key instead of email as part
# of the hash
mapillary_description['MAPSettingsUploadHash'] = settings_upload_hash
# a unique photo ID to check for duplicates in the backend in case the
# image gets uploaded more than once
mapillary_description['MAPPhotoUUID'] = str(uuid.uuid4())
# a sequene ID to make the images go together (order by MAPCaptureTime)
mapillary_description['MAPSequenceUUID'] = str(sequence_uuid)
# The device manufacturer
if make:
mapillary_description['MAPDeviceMake'] = make
else:
mapillary_description['MAPDeviceMake'] = exif.extract_make()
# The device model
if model:
mapillary_description['MAPDeviceModel'] = model
else:
mapillary_description['MAPDeviceModel'] = exif.extract_model()
if upload_hash is None and secret_hash is not None:
mapillary_description['MAPVideoSecure'] = secret_hash
if project:
mapillary_description["MAPSettingsProject"] = project
# external properties (optional)
if external_properties is not None:
# externl proerties can be saved and searched in Mapillary later on
mapillary_description['MAPExternalProperties'] = external_properties
if make:
mapillary_description['MAPDeviceMake'] = make
if model:
mapillary_description['MAPDeviceModel'] = model
# write to file
if verbose:
print("tag: {0}".format(mapillary_description))
metadata = ExifEdit(filename)
metadata.add_image_description(mapillary_description)
metadata.add_orientation(orientation)
metadata.add_direction(heading)
metadata.write()
def _read_capture_time(self, filename):
'''
Use EXIF class to parse capture time from EXIF.
'''
exif = EXIF(filename)
return exif.extract_capture_time()
def upload_file(filepath,
url,
permission,
signature,
key=None,
move_files=True,
keep_file_names=True):
'''
Upload file at filepath.
Move to subfolders 'success'/'failed' on completion if move_files is True.
'''
filename = os.path.basename(filepath)
if keep_file_names:
s3_filename = filename
else:
try:
s3_filename = EXIF(filepath).exif_name()
except:
s3_filename = filename
# add S3 'path' if given
if key is None:
s3_key = s3_filename
else:
s3_key = key + s3_filename
parameters = {
"key": s3_key,
"AWSAccessKeyId": "AKIAI2X3BJAT2W75HILA",
"acl": "private",
"policy": permission,
"signature": signature,
"Content-Type": "image/jpeg"
}
with open(filepath, "rb") as f:
encoded_string = f.read()
data, headers = encode_multipart(
parameters,
{'file': {
'filename': filename,
'content': encoded_string
}})
root_path = os.path.dirname(filepath)
success_path = os.path.join(root_path, 'success')
failed_path = os.path.join(root_path, 'failed')
lib.io.mkdir_p(success_path)
lib.io.mkdir_p(failed_path)
for attempt in range(MAX_ATTEMPTS):
try:
request = urllib2.Request(url, data=data, headers=headers)
response = urllib2.urlopen(request)
if response.getcode() == 204:
if move_files:
os.rename(filepath, os.path.join(success_path, filename))
# print("Success: {0}".format(filename))
else:
if move_files:
os.rename(filepath, os.path.join(failed_path, filename))
print("Failed: {0}".format(filename))
break # attempts
except urllib2.HTTPError as e:
print("HTTP error: {0} on {1}".format(e, filename))
time.sleep(5)
except urllib2.URLError as e:
print("URL error: {0} on {1}".format(e, filename))
time.sleep(5)
except OSError as e:
print("OS error: {0} on {1}".format(e, filename))
time.sleep(5)
except socket.timeout as e:
# Specific timeout handling for Python 2.7
print("Timeout error: {0} (retrying)".format(filename))
0: 1,
90: 6,
180: 3,
270: 8
} # see http://sylvana.net/jpegcrop/exif_orientation.html
if args.orientation is not None:
exifOrientation = orientations[args.orientation]
s = Sequence(args.path)
if args.interpolate:
bearings = s.interpolate_direction()
for filename in s.get_file_list(args.path):
stat = os.stat(filename)
exifRead = EXIF(filename)
mapillaryTag = json.loads(exifRead.tags['Image ImageDescription'].values)
# print filename, exifRead.extract_orientation(), mapillaryTag['MAPCameraRotation']
if args.interpolate:
bearing = bearings[filename];
else:
bearing = exifRead.extract_direction();
if args.offset:
bearing = offset_bearing(bearing, args.offset)
exifEdit = ExifEdit(filename)
if args.interpolate or args.offset:
exifEdit.add_direction(bearing, precision=10)