def send_2sa_notification(
smtp_email, smtp_password, smtp_host, smtp_port, smtp_no_tls, to_addr
):
"""Send an email notification when 2SA is expired"""
to_addr = to_addr if to_addr else smtp_email
from_addr = smtp_email if smtp_email else to_addr
logger = setup_logger()
logger.info("Sending 'two-step expired' notification via email...")
smtp = smtplib.SMTP()
smtp.set_debuglevel(0)
smtp.connect(smtp_host, smtp_port)
if not smtp_no_tls:
smtp.starttls()
if smtp_email is not None or smtp_password is not None:
smtp.login(smtp_email, smtp_password)
subj = "icloud_photos_downloader: Two step authentication has expired"
date = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
message_text = """Hello,
Two-step authentication has expired for the icloud_photos_downloader script.
Please log in to your server and run the script manually to update two-step authentication."""
msg = "From: %s\nTo: %s\nSubject: %s\nDate: %s\n\n%s" % (
"iCloud Photos Downloader <" + from_addr + ">",
to_addr,
subj,
date,
message_text,
)
smtp.sendmail(from_addr, to_addr, msg)
smtp.quit()
def send_2sa_notification(
smtp_email, smtp_password, smtp_host, smtp_port, smtp_no_tls, to_addr
):
"""Send an email notification when 2SA is expired"""
to_addr = to_addr if to_addr else smtp_email
logger = setup_logger()
logger.info("Sending 'two-step expired' notification via email...")
smtp = smtplib.SMTP()
smtp.set_debuglevel(0)
smtp.connect(smtp_host, smtp_port)
if not smtp_no_tls:
smtp.starttls()
smtp.login(smtp_email, smtp_password)
subj = "icloud_photos_downloader: Two step authentication has expired"
date = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
message_text = """Hello,
Two-step authentication has expired for the icloud_photos_downloader script.
Please log in to your server and run the script manually to update two-step authentication."""
from_addr = "iCloud Photos Downloader <" + smtp_email + ">"
msg = "From: %s\nTo: %s\nSubject: %s\nDate: %s\n\n%s" % (
from_addr,
to_addr,
subj,
date,
message_text,
)
smtp.sendmail(smtp_email, to_addr, msg)
smtp.quit()
def authenticate(
username,
password,
cookie_directory=None,
raise_error_on_2sa=False,
client_id=None
):
"""Authenticate with iCloud username and password"""
logger = setup_logger()
logger.debug("Authenticating...")
try:
# If password not provided on command line variable will be set to None
# and PyiCloud will attempt to retrieve from it's keyring
icloud = pyicloud_ipd.PyiCloudService(
username, password,
cookie_directory=cookie_directory,
client_id=client_id)
except pyicloud_ipd.exceptions.NoStoredPasswordAvailable:
# Prompt for password if not stored in PyiCloud's keyring
password = click.prompt("iCloud Password", hide_input=True)
icloud = pyicloud_ipd.PyiCloudService(
username, password,
cookie_directory=cookie_directory,
client_id=client_id)
if icloud.requires_2sa:
if raise_error_on_2sa:
raise TwoStepAuthRequiredError(
"Two-step/two-factor authentication is required!"
)
logger.info("Two-step/two-factor authentication is required!")
request_2sa(icloud, logger)
return icloud
def retaindelete_photos(icloud, folder_structure, directory):
"""
Scans the "Recently Deleted" folder and deletes any matching files
from the download directory.
(I.e. If you delete a photo on your phone, it's also deleted on your computer.)
"""
logger = setup_logger()
logger.info("Moving any files found in 'Recently Deleted'...")
recently_deleted = icloud.photos.albums["Recently Deleted"]
for media in recently_deleted:
created_date = media.created
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
for size in [None, "original", "medium", "thumb"]:
path = os.path.normpath(
local_download_path(media, size, download_dir))
if os.path.exists(path):
logger.info("Deleting %s!", path)
# os.remove(path)
retain_path = os.path.join(directory, "retain_delete")
if not os.path.exists(retain_path):
os.mkdir(retain_path)
os.rename(path, os.path.join(retain_path, path.split('/')[-1]))
def authenticate(username,
password,
cookie_directory=None,
raise_error_on_2sa=False,
client_id=None):
"""Authenticate with iCloud username and password"""
logger = setup_logger()
logger.debug("Authenticating...")
try:
# If password not provided on command line variable will be set to None
# and PyiCloud will attempt to retrieve from it's keyring
icloud = pyicloud_ipd.PyiCloudService(
username,
password,
cookie_directory=cookie_directory,
client_id=client_id)
except pyicloud_ipd.exceptions.NoStoredPasswordAvailable:
# Prompt for password if not stored in PyiCloud's keyring
password = click.prompt("iCloud Password", hide_input=True)
icloud = pyicloud_ipd.PyiCloudService(
username,
password,
cookie_directory=cookie_directory,
client_id=client_id)
if icloud.requires_2sa:
if raise_error_on_2sa:
raise TwoStepAuthRequiredError(
"Two-step/two-factor authentication is required!")
logger.info("Two-step/two-factor authentication is required!")
request_2sa(icloud, logger)
return icloud
def download_media(icloud, photo, download_path, size):
"""Download the photo to path, with retries and error handling"""
logger = setup_logger()
for retries in range(constants.MAX_RETRIES):
try:
photo_response = photo.download(size)
if photo_response:
temp_download_path = download_path + ".part"
with open(temp_download_path, "wb") as file_obj:
for chunk in photo_response.iter_content(chunk_size=1024):
if chunk:
file_obj.write(chunk)
os.rename(temp_download_path, download_path)
update_mtime(photo, download_path)
return True
logger.tqdm_write(
"Could not find URL to download %s for size %s!" %
(photo.filename, size),
logging.ERROR,
)
break
except (ConnectionError, socket.timeout,
PyiCloudAPIResponseError) as ex:
if "Invalid global session" in str(ex):
logger.tqdm_write("Session error, re-authenticating...",
logging.ERROR)
if retries > 0:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
else:
# you end up here when p.e. throttleing by Apple happens
wait_time = (retries + 1) * constants.WAIT_SECONDS
logger.tqdm_write(
"Error downloading %s, retrying after %d seconds..." %
(photo.filename, wait_time),
logging.ERROR,
)
time.sleep(wait_time)
except IOError:
logger.error(
"IOError while writing file to %s! "
"You might have run out of disk space, or the file "
"might be too large for your OS. "
"Skipping this file...", download_path)
break
else:
logger.tqdm_write("Could not download %s! Please try again later." %
photo.filename)
return False
def get_photo_exif(path):
"""Get EXIF date for a photo, return nothing if there is an error"""
try:
exif_dict = piexif.load(path)
return exif_dict.get("Exif").get(36867)
except (ValueError, InvalidImageDataError):
logger = setup_logger()
logger.debug("Error fetching EXIF data for %s", path)
return None
def download_media(icloud, photo, download_path, size):
"""Download the photo to path, with retries and error handling"""
logger = setup_logger()
for retries in range(constants.MAX_RETRIES):
try:
photo_response = photo.download(size)
if photo_response:
with open(download_path, "wb") as file_obj:
for chunk in photo_response.iter_content(chunk_size=1024):
if chunk:
file_obj.write(chunk)
update_mtime(photo, download_path)
return True
logger.tqdm_write(
"Could not find URL to download %s for size %s!"
% (photo.filename, size),
logging.ERROR,
)
break
except (ConnectionError, socket.timeout, PyiCloudAPIResponseError) as ex:
if "Invalid global session" in str(ex):
logger.tqdm_write(
"Session error, re-authenticating...",
logging.ERROR)
if retries > 0:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
else:
logger.tqdm_write(
"Error downloading %s, retrying after %d seconds..."
% (photo.filename, constants.WAIT_SECONDS),
logging.ERROR,
)
time.sleep(constants.WAIT_SECONDS)
except IOError:
logger.error(
"IOError while writing file to %s! "
"You might have run out of disk space, or the file "
"might be too large for your OS. "
"Skipping this file...", download_path
)
break
else:
logger.tqdm_write(
"Could not download %s! Please try again later." % photo.filename
)
return False
def download_photo(icloud, photo, download_path, size):
"""Download the photo to path, with retries and error handling"""
logger = setup_logger()
for _ in range(constants.MAX_RETRIES):
try:
photo_response = photo.download(size)
if photo_response:
with open(download_path, "wb") as file_obj:
for chunk in photo_response.iter_content(chunk_size=1024):
if chunk:
file_obj.write(chunk)
return True
logger.tqdm_write(
"Could not find URL to download %s for size %s!"
% (photo.filename, size),
logging.ERROR,
)
break
except (ConnectionError, socket.timeout, PyiCloudAPIResponseError) as ex:
if "Invalid global session" in str(ex):
logger.tqdm_write(
"Session error, re-authenticating...",
logging.ERROR)
icloud.authenticate()
# Wait a few seconds in case there are issues with Apple's
# servers
time.sleep(constants.WAIT_SECONDS)
else:
logger.tqdm_write(
"Error downloading %s, retrying after %d seconds..."
% (photo.filename, constants.WAIT_SECONDS),
logging.ERROR,
)
time.sleep(constants.WAIT_SECONDS)
except IOError:
logger.error(
"IOError while writing file to %s! "
"You might have run out of disk space, or the file "
"might be too large for your OS. "
"Skipping this file...", download_path
)
break
else:
logger.tqdm_write(
"Could not download %s! Please try again later." % photo.filename
)
return False
def set_photo_exif(path, date):
"""Set EXIF date on a photo, do nothing if there is an error"""
try:
exif_dict = piexif.load(path)
exif_dict.get("1st")[306] = date
exif_dict.get("Exif")[36867] = date
exif_dict.get("Exif")[36868] = date
exif_bytes = piexif.dump(exif_dict)
piexif.insert(exif_bytes, path)
except (ValueError, InvalidImageDataError):
logger = setup_logger()
logger.debug("Error setting EXIF data for %s", path)
return
def test_logger_output(self):
logger = setup_logger()
test_logger = logging.getLogger("icloudpd-test")
string_io = StringIO()
string_handler = logging.StreamHandler(stream=string_io)
string_handler.setFormatter(logger.handlers[0].formatter)
test_logger.addHandler(string_handler)
test_logger.setLevel(logging.DEBUG)
test_logger.info(u"Test info output")
test_logger.debug(u"Test debug output")
test_logger.error(u"Test error output")
output = string_io.getvalue().strip()
self.assertIn("2018-01-01", output)
self.assertIn("INFO Test info output", output)
self.assertIn("DEBUG Test debug output", output)
self.assertIn("ERROR Test error output", output)
def test_logger_output(self):
logger = setup_logger()
test_logger = logging.getLogger("icloudpd-test")
string_io = StringIO()
string_handler = logging.StreamHandler(stream=string_io)
string_handler.setFormatter(logger.handlers[0].formatter)
test_logger.addHandler(string_handler)
test_logger.setLevel(logging.DEBUG)
test_logger.info(u"Test info output")
test_logger.debug(u"Test debug output")
test_logger.error(u"Test error output")
output = string_io.getvalue().strip()
# 2018:01:01 00:00:00 utc
expectedDatetime = datetime.datetime(2018,1,1,0,0,0,tzinfo=datetime.timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S")
self.assertIn(expectedDatetime, output)
self.assertIn("INFO Test info output", output)
self.assertIn("DEBUG Test debug output", output)
self.assertIn("ERROR Test error output", output)
def autodelete_photos(icloud, folder_structure, directory):
"""
Scans the "Recently Deleted" folder and deletes any matching files
from the download directory.
(I.e. If you delete a photo on your phone, it's also deleted on your computer.)
"""
logger = setup_logger()
logger.info("Deleting any files found in 'Recently Deleted'...")
recently_deleted = icloud.photos.albums["Recently Deleted"]
for media in recently_deleted:
created_date = media.created
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
for size in [None, "original", "medium", "thumb"]:
path = local_download_path(media, size, download_dir)
if os.path.exists(path):
logger.info("Deleting %s!", path)
os.remove(path)
def send_2sa_notification(smtp_username, smtp_password, smtp_host, smtp_port,
smtp_no_tls, to_addr, from_addr):
"""Send an email notification when 2SA is expired"""
to_addr = to_addr if to_addr else smtp_username
from_addr = from_addr if from_addr else smtp_username
logger = setup_logger()
logger.info("Sending 'two-step expired' notification via email...")
smtp = smtplib.SMTP(smtp_host, smtp_port)
smtp.set_debuglevel(0)
# leaving explicit call of connect to not break unit tests, even though it is
# called implicitly via cunstructor parameters
smtp.connect(smtp_host, smtp_port)
if not smtp_no_tls:
smtp.starttls()
if smtp_username is not None or smtp_password is not None:
smtp.login(smtp_username, smtp_password)
subj = "icloud_photos_downloader: Two step authentication has expired"
date = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
message_text = """Hello,
Two-step authentication has expired for the icloud_photos_downloader script.
Please log in to your server and run the script manually to update two-step authentication."""
msg = "From: %s\nTo: %s\nSubject: %s\nDate: %s\n\n%s" % (
"iCloud Photos Downloader <" + from_addr + ">",
to_addr,
subj,
date,
message_text,
)
smtp.sendmail(from_addr, to_addr, msg)
smtp.quit()
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
album,
list_albums,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
threads_num,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (smtp_username is not None
or notification_email is not None
or notification_script is not None)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
sys.exit(1)
# Default album is "All Photos", so this is the same as
# calling `icloud.photos.all`.
photos = icloud.photos.albums[album]
if list_albums:
albums_dict = icloud.photos.albums
# Python2: itervalues, Python3: values()
if sys.version_info[0] >= 3:
albums = albums_dict.values() # pragma: no cover
else:
albums = albums_dict.itervalues() # pragma: no cover
album_titles = [str(a) for a in albums]
print(*album_titles, sep="\n")
sys.exit(0)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug("Looking up all photos%s from album %s...",
"" if skip_videos else " and videos", album)
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later.")
raise ex
logger.tqdm_write("Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (only_print_filenames
or no_progress_bar
or not sys.stdout.isatty()):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
def download_photo(counter, photo):
"""internal function for actually downloading the photos"""
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename)
return
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type))
return
try:
created_date = photo.created.astimezone(get_localzone())
except (ValueError, OSError):
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)" %
photo.created, logging.ERROR)
created_date = photo.created
try:
date_path = folder_structure.format(created_date)
except ValueError: # pragma: no cover
# This error only seems to happen in Python 2
logger.set_tqdm_description(
"Photo created date was not valid (%s)" % photo.created,
logging.ERROR)
# e.g. ValueError: year=5 is before 1900
# (https://github.com/ndbroadbent/icloud_photos_downloader/issues/122)
# Just use the Unix epoch
created_date = datetime.datetime.fromtimestamp(0)
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
try:
os.makedirs(download_dir)
except OSError: # pragma: no cover
pass # pragma: no cover
download_size = size
try:
versions = photo.versions
except KeyError as ex:
print("KeyError: %s attribute was not found in the photo fields!" %
ex)
with open('icloudpd-photo-error.json', 'w') as outfile:
# pylint: disable=protected-access
json.dump(
{
"master_record": photo._master_record,
"asset_record": photo._asset_record
}, outfile)
# pylint: enable=protected-access
print("icloudpd has saved the photo record to: "
"./icloudpd-photo-error.json")
print("Please create a Gist with the contents of this file: "
"https://gist.github.com")
print(
"Then create an issue on GitHub: "
"https://github.com/ndbroadbent/icloud_photos_downloader/issues"
)
print("Include a link to the Gist in your issue, so that we can "
"see what went wrong.\n")
return
if size not in versions and size != "original":
if force_size:
filename = photo.filename.encode("utf-8").decode(
"ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename),
logging.ERROR,
)
return
download_size = "original"
download_path = local_download_path(photo, download_size, download_dir)
file_exists = os.path.isfile(download_path)
if not file_exists and download_size == "original":
# Deprecation - We used to download files like IMG_1234-original.jpg,
# so we need to check for these.
# Now we match the behavior of iCloud for Windows: IMG_1234.jpg
original_download_path = ("-%s." % size).join(
download_path.rsplit(".", 1))
file_exists = os.path.isfile(original_download_path)
if file_exists:
counter.increment()
logger.set_tqdm_description("%s already exists." %
truncate_middle(download_path, 96))
else:
counter.reset()
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" % truncated_path)
download_result = download.download_media(
icloud, photo, download_path, download_size)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# Also download the live photo if present
if not skip_live_photos:
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
version = photo.versions[lp_size]
filename = version["filename"]
if live_photo_size != "original":
# Add size to filename if not original
filename = filename.replace(".MOV",
"-%s.MOV" % live_photo_size)
lp_download_path = os.path.join(download_dir, filename)
if only_print_filenames:
print(lp_download_path)
else:
if os.path.isfile(lp_download_path):
logger.set_tqdm_description(
"%s already exists." %
truncate_middle(lp_download_path, 96))
return
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download.download_media(icloud, photo, lp_download_path,
lp_size)
def get_threads_count():
"""Disable threads if we have until_found or recent arguments"""
if until_found is None and recent is None:
return threads_num # pragma: no cover
return 1
download_queue = queue.Queue(get_threads_count())
consecutive_files_found = Counter(0)
def should_break(counter):
"""Exit if until_found condition is reached"""
return until_found is not None and counter.value() >= until_found
def worker(counter):
"""Threaded worker"""
while True:
item = download_queue.get()
if item is None:
break
download_photo(counter, item)
download_queue.task_done()
threads = []
for _ in range(get_threads_count()):
thread = threading.Thread(target=worker,
args=(consecutive_files_found, ))
thread.daemon = True
thread.start()
threads.append(thread)
photos_iterator = iter(photos_enumerator)
while True:
try:
if should_break(consecutive_files_found):
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting"
% until_found)
break
download_queue.put(next(photos_iterator))
except StopIteration:
break
if not should_break(consecutive_files_found):
download_queue.join()
for _ in threads:
download_queue.put(None)
for thread in threads:
thread.join()
if only_print_filenames:
sys.exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (smtp_username is not None
or notification_email is not None
or notification_script is not None)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug("Looking up all photos%s...",
"" if skip_videos else " and videos")
photos = icloud.photos.all
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later.")
raise ex
logger.tqdm_write("Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (only_print_filenames
or no_progress_bar
or not sys.stdout.isatty()):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(constants.MAX_RETRIES):
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename)
break
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type))
break
try:
created_date = photo.created.astimezone(get_localzone())
except ValueError:
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)"
% photo.created,
logging.ERROR,
)
created_date = photo.created
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
download_size = size
# Fall back to original if requested size is not available
if size not in photo.versions and size != "original":
if force_size:
filename = photo.filename.encode("utf-8").decode(
"ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename),
logging.ERROR,
)
break
download_size = "original"
download_path = local_download_path(photo, download_size,
download_dir)
file_exists = os.path.isfile(download_path)
if not file_exists and download_size == "original":
# Deprecation - We used to download files like IMG_1234-original.jpg,
# so we need to check for these.
# Now we match the behavior of iCloud for Windows: IMG_1234.jpg
original_download_path = ("-%s." % size).join(
download_path.rsplit(".", 1))
file_exists = os.path.isfile(original_download_path)
if file_exists:
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description("%s already exists." %
truncate_middle(download_path, 96))
else:
if until_found is not None:
consecutive_files_found = 0
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download_result = download.download_media(
icloud, photo, download_path, download_size)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# Also download the live photo if present
if not skip_live_photos:
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
version = photo.versions[lp_size]
filename = version["filename"]
if live_photo_size != "original":
# Add size to filename if not original
filename = filename.replace(
".MOV", "-%s.MOV" % live_photo_size)
lp_download_path = os.path.join(download_dir, filename)
if only_print_filenames:
print(lp_download_path)
else:
if os.path.isfile(lp_download_path):
logger.set_tqdm_description(
"%s already exists." %
truncate_middle(lp_download_path, 96))
break
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download.download_media(icloud, photo,
lp_download_path, lp_size)
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting" %
until_found)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
size,
recent,
until_found,
skip_videos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
should_send_2sa_notification = smtp_username is not None
try:
icloud = authenticate(
username,
password,
should_send_2sa_notification,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug(
"Looking up all photos%s...",
"" if skip_videos else " and videos")
photos = icloud.photos.all
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str, size, plural_suffix, video_suffix, directory
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (
only_print_filenames or no_progress_bar or not sys.stdout.isatty()
):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(MAX_RETRIES):
if skip_videos and not photo.item_type == "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename
)
break
try:
created_date = photo.created.astimezone(get_localzone())
except ValueError:
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)" %
photo.created, logging.ERROR)
created_date = photo.created
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
download_size = size
# Fall back to original if requested size is not available
if size not in photo.versions and size != "original":
if force_size:
filename = photo.filename.encode(
"utf-8").decode("ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename), logging.ERROR)
break
download_size = "original"
download_path = local_download_path(
photo, download_size, download_dir)
download_path_without_size = local_download_path(
photo, None, download_dir)
# add a check if the "simple" name of the file is found if the size
# is original
if os.path.isfile(download_path) or (
download_size == "original" and
os.path.isfile(download_path_without_size)
):
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description(
"%s already exists." % truncate_middle(download_path, 96)
)
break
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" % truncated_path)
download_result = download.download_photo(
icloud, photo, download_path, download_size
)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path, date_str
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
if until_found is not None:
consecutive_files_found = 0
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting"
% until_found
)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (
smtp_username is not None
or notification_email is not None
or notification_script is not None
)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug(
"Looking up all photos%s...",
"" if skip_videos else " and videos")
photos = icloud.photos.all
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later."
)
raise ex
logger.tqdm_write(
"Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (
only_print_filenames or no_progress_bar or not sys.stdout.isatty()
):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(constants.MAX_RETRIES):
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename
)
break
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type)
)
break
try:
created_date = photo.created.astimezone(get_localzone())
except (ValueError, OSError):
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)" %
photo.created, logging.ERROR)
created_date = photo.created
try:
date_path = folder_structure.format(created_date)
except ValueError: # pragma: no cover
# This error only seems to happen in Python 2
logger.set_tqdm_description(
"Photo created date was not valid (%s)" %
photo.created, logging.ERROR)
# e.g. ValueError: year=5 is before 1900
# (https://github.com/ndbroadbent/icloud_photos_downloader/issues/122)
# Just use the Unix epoch
created_date = datetime.datetime.fromtimestamp(0)
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
download_size = size
try:
versions = photo.versions
except KeyError as ex:
print(
"KeyError: %s attribute was not found in the photo fields!" %
ex)
with open('icloudpd-photo-error.json', 'w') as outfile:
# pylint: disable=protected-access
json.dump({
"master_record": photo._master_record,
"asset_record": photo._asset_record
}, outfile)
# pylint: enable=protected-access
print("icloudpd has saved the photo record to: "
"./icloudpd-photo-error.json")
print("Please create a Gist with the contents of this file: "
"https://gist.github.com")
print(
"Then create an issue on GitHub: "
"https://github.com/ndbroadbent/icloud_photos_downloader/issues")
print(
"Include a link to the Gist in your issue, so that we can "
"see what went wrong.\n")
break
if size not in versions and size != "original":
if force_size:
filename = photo.filename.encode(
"utf-8").decode("ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename), logging.ERROR, )
break
download_size = "original"
download_path = local_download_path(
photo, download_size, download_dir)
file_exists = os.path.isfile(download_path)
if not file_exists and download_size == "original":
# Deprecation - We used to download files like IMG_1234-original.jpg,
# so we need to check for these.
# Now we match the behavior of iCloud for Windows: IMG_1234.jpg
original_download_path = ("-%s." % size).join(
download_path.rsplit(".", 1)
)
file_exists = os.path.isfile(original_download_path)
if file_exists:
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description(
"%s already exists." % truncate_middle(download_path, 96)
)
else:
if until_found is not None:
consecutive_files_found = 0
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description(
"Downloading %s" %
truncated_path)
download_result = download.download_media(
icloud, photo, download_path, download_size
)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# Also download the live photo if present
if not skip_live_photos:
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
version = photo.versions[lp_size]
filename = version["filename"]
if live_photo_size != "original":
# Add size to filename if not original
filename = filename.replace(
".MOV", "-%s.MOV" %
live_photo_size)
lp_download_path = os.path.join(download_dir, filename)
if only_print_filenames:
print(lp_download_path)
else:
if os.path.isfile(lp_download_path):
logger.set_tqdm_description(
"%s already exists."
% truncate_middle(lp_download_path, 96)
)
break
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description(
"Downloading %s" % truncated_path)
download.download_media(
icloud, photo, lp_download_path, lp_size
)
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting"
% until_found
)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
album,
list_albums,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
state_store,
state_path,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (smtp_username is not None
or notification_email is not None
or notification_script is not None)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# Default album is "All Photos", so this is the same as
# calling `icloud.photos.all`.
photos = icloud.photos.albums[album]
if list_albums:
albums_dict = icloud.photos.albums
# Python2: itervalues, Python3: values()
if sys.version_info[0] >= 3:
albums = albums_dict.values() # pragma: no cover
else:
albums = albums_dict.itervalues() # pragma: no cover
album_titles = [str(a) for a in albums]
print(*album_titles, sep="\n")
exit(0)
if state_store == "sqlite":
if state_path is None:
logger.tqdm_write("No state path supplied for sqlite")
exit(1)
statemgr = state_lib.SQLiteMediaManager(filepath=state_path)
elif state_store == "json":
if state_path is None:
logger.tqdm_write("No state path supplied for json state")
exit(1)
statemgr = state_lib.FileMediaManager(filepath=state_path)
else:
statemgr = state_lib.FilesMediaManager()
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug("Looking up all photos%s from album %s...",
"" if skip_videos else " and videos", album)
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later.")
raise ex
logger.tqdm_write("Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (only_print_filenames
or no_progress_bar
or not sys.stdout.isatty()):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(constants.MAX_RETRIES):
### Process media skipping rules
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename)
break
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type))
break
### Process photo ctime
try:
created_date = photo.created.astimezone(get_localzone())
except (ValueError, OSError):
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)"
% photo.created, logging.ERROR)
created_date = photo.created
# Process photo dir path schema
try:
date_path = folder_structure.format(created_date)
except ValueError: # pragma: no cover
# This error only seems to happen in Python 2
logger.set_tqdm_description(
"Photo created date was not valid (%s)" % photo.created,
logging.ERROR)
# e.g. ValueError: year=5 is before 1900
# (https://github.com/ndbroadbent/icloud_photos_downloader/issues/122)
# Just use the Unix epoch
created_date = datetime.datetime.fromtimestamp(0)
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
### process download size (user-defined vs available)
download_size = size
try:
versions = photo.versions
except KeyError as ex:
print(
"KeyError: %s attribute was not found in the photo fields!"
% ex)
with open('icloudpd-photo-error.json', 'w') as outfile:
# pylint: disable=protected-access
json.dump(
{
"master_record": photo._master_record,
"asset_record": photo._asset_record
}, outfile)
# pylint: enable=protected-access
print("icloudpd has saved the photo record to: "
"./icloudpd-photo-error.json")
print("Please create a Gist with the contents of this file: "
"https://gist.github.com")
print(
"Then create an issue on GitHub: "
"https://github.com/ndbroadbent/icloud_photos_downloader/issues"
)
print(
"Include a link to the Gist in your issue, so that we can "
"see what went wrong.\n")
break
# resort to "original" size by default
if size not in versions and size != "original":
if force_size:
filename = photo.filename.encode("utf-8").decode(
"ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename),
logging.ERROR,
)
break
download_size = "original"
download_path = local_download_path(photo, download_size,
download_dir)
### process downloading our photo
already_processed = statemgr.processed(
download_dir=download_dir,
download_size=download_size,
photo=photo,
)
if already_processed:
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description("%s already processed." %
truncate_middle(download_path, 96))
else:
if until_found is not None:
consecutive_files_found = 0
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
# update photo state to STARTED
statemgr.update(download_size=download_size,
photo=photo,
state=state_lib.STATE_ENUM["STARTED"])
download_result = download.download_media(
icloud, photo, download_path, download_size)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# update photo state to FINISHED
if download_result:
statemgr.update(download_size=download_size,
hash=download_result,
photo=photo,
state=state_lib.STATE_ENUM["FINISHED"])
### process downloading videos for any live photos
if not skip_live_photos:
# live photo video size
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
lp_download_path = local_download_path_lp(
media=photo,
lp_size=lp_size,
download_dir=download_dir)
if only_print_filenames:
print(lp_download_path)
else:
# skip if we've already processed this lp's video
if statemgr.processed(download_size=lp_size,
download_dir=download_dir,
photo=photo):
logger.set_tqdm_description(
"%s already processed." %
truncate_middle(lp_download_path, 96))
break
# update photo state to STARTED
statemgr.update(download_size=lp_size,
photo=photo,
state=state_lib.STATE_ENUM["STARTED"])
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download_result = download.download_media(
icloud, photo, lp_download_path, lp_size)
# update photo state to FINISHED
if download_result:
statemgr.update(
download_size=lp_size,
hash=download_result,
photo=photo,
state=state_lib.STATE_ENUM["FINISHED"])
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting" %
until_found)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
