def test_successful_auth(self):
with vcr.use_cassette("tests/vcr_cassettes/successful_auth.yml"):
authenticate(
"*****@*****.**",
"password1",
client_id="EC5646DE-9423-11E8-BF21-14109FE0B321",
)
def test_successful_auth(self):
with vcr.use_cassette("tests/vcr_cassettes/successful_auth.yml"):
authenticate(
"*****@*****.**",
"password1",
client_id="EC5646DE-9423-11E8-BF21-14109FE0B321",
)
def test_failed_auth(self):
with vcr.use_cassette("tests/vcr_cassettes/failed_auth.yml"):
with self.assertRaises(pyicloud_ipd.exceptions.
PyiCloudFailedLoginException) as context:
authenticate(
"bad_username",
"bad_password",
client_id="EC5646DE-9423-11E8-BF21-14109FE0B321",
)
self.assertTrue(
"Invalid email/password combination." in str(context.exception))
def test_failed_auth(self):
with vcr.use_cassette("tests/vcr_cassettes/failed_auth.yml"):
with self.assertRaises(
pyicloud_ipd.exceptions.PyiCloudFailedLoginException
) as context:
authenticate(
"bad_username",
"bad_password",
client_id="EC5646DE-9423-11E8-BF21-14109FE0B321",
)
self.assertTrue("Invalid email/password combination." in str(context.exception))
def test_2sa_required(self):
with vcr.use_cassette("tests/vcr_cassettes/auth_requires_2sa.yml"):
with self.assertRaises(TwoStepAuthRequiredError) as context:
# To re-record this HTTP request,
# delete ./tests/vcr_cassettes/auth_requires_2sa.yml,
# put your actual credentials in here, run the test,
# and then replace with dummy credentials.
authenticate(
"*****@*****.**",
"password1",
raise_error_on_2sa=True,
client_id="EC5646DE-9423-11E8-BF21-14109FE0B321",
)
self.assertTrue("Two-step/two-factor authentication is required!"
in str(context.exception))
def test_2sa_required(self):
with vcr.use_cassette("tests/vcr_cassettes/auth_requires_2sa.yml"):
with self.assertRaises(TwoStepAuthRequiredError) as context:
# To re-record this HTTP request,
# delete ./tests/vcr_cassettes/auth_requires_2sa.yml,
# put your actual credentials in here, run the test,
# and then replace with dummy credentials.
authenticate(
"*****@*****.**",
"password1",
raise_error_on_2sa=True,
client_id="EC5646DE-9423-11E8-BF21-14109FE0B321",
)
self.assertTrue(
"Two-step/two-factor authentication is required!"
in str(context.exception)
)
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
album,
list_albums,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
threads_num,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (smtp_username is not None
or notification_email is not None
or notification_script is not None)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
sys.exit(1)
# Default album is "All Photos", so this is the same as
# calling `icloud.photos.all`.
photos = icloud.photos.albums[album]
if list_albums:
albums_dict = icloud.photos.albums
# Python2: itervalues, Python3: values()
if sys.version_info[0] >= 3:
albums = albums_dict.values() # pragma: no cover
else:
albums = albums_dict.itervalues() # pragma: no cover
album_titles = [str(a) for a in albums]
print(*album_titles, sep="\n")
sys.exit(0)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug("Looking up all photos%s from album %s...",
"" if skip_videos else " and videos", album)
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later.")
raise ex
logger.tqdm_write("Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (only_print_filenames
or no_progress_bar
or not sys.stdout.isatty()):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
def download_photo(counter, photo):
"""internal function for actually downloading the photos"""
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename)
return
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type))
return
try:
created_date = photo.created.astimezone(get_localzone())
except (ValueError, OSError):
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)" %
photo.created, logging.ERROR)
created_date = photo.created
try:
date_path = folder_structure.format(created_date)
except ValueError: # pragma: no cover
# This error only seems to happen in Python 2
logger.set_tqdm_description(
"Photo created date was not valid (%s)" % photo.created,
logging.ERROR)
# e.g. ValueError: year=5 is before 1900
# (https://github.com/ndbroadbent/icloud_photos_downloader/issues/122)
# Just use the Unix epoch
created_date = datetime.datetime.fromtimestamp(0)
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
try:
os.makedirs(download_dir)
except OSError: # pragma: no cover
pass # pragma: no cover
download_size = size
try:
versions = photo.versions
except KeyError as ex:
print("KeyError: %s attribute was not found in the photo fields!" %
ex)
with open('icloudpd-photo-error.json', 'w') as outfile:
# pylint: disable=protected-access
json.dump(
{
"master_record": photo._master_record,
"asset_record": photo._asset_record
}, outfile)
# pylint: enable=protected-access
print("icloudpd has saved the photo record to: "
"./icloudpd-photo-error.json")
print("Please create a Gist with the contents of this file: "
"https://gist.github.com")
print(
"Then create an issue on GitHub: "
"https://github.com/ndbroadbent/icloud_photos_downloader/issues"
)
print("Include a link to the Gist in your issue, so that we can "
"see what went wrong.\n")
return
if size not in versions and size != "original":
if force_size:
filename = photo.filename.encode("utf-8").decode(
"ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename),
logging.ERROR,
)
return
download_size = "original"
download_path = local_download_path(photo, download_size, download_dir)
file_exists = os.path.isfile(download_path)
if not file_exists and download_size == "original":
# Deprecation - We used to download files like IMG_1234-original.jpg,
# so we need to check for these.
# Now we match the behavior of iCloud for Windows: IMG_1234.jpg
original_download_path = ("-%s." % size).join(
download_path.rsplit(".", 1))
file_exists = os.path.isfile(original_download_path)
if file_exists:
counter.increment()
logger.set_tqdm_description("%s already exists." %
truncate_middle(download_path, 96))
else:
counter.reset()
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" % truncated_path)
download_result = download.download_media(
icloud, photo, download_path, download_size)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# Also download the live photo if present
if not skip_live_photos:
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
version = photo.versions[lp_size]
filename = version["filename"]
if live_photo_size != "original":
# Add size to filename if not original
filename = filename.replace(".MOV",
"-%s.MOV" % live_photo_size)
lp_download_path = os.path.join(download_dir, filename)
if only_print_filenames:
print(lp_download_path)
else:
if os.path.isfile(lp_download_path):
logger.set_tqdm_description(
"%s already exists." %
truncate_middle(lp_download_path, 96))
return
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download.download_media(icloud, photo, lp_download_path,
lp_size)
def get_threads_count():
"""Disable threads if we have until_found or recent arguments"""
if until_found is None and recent is None:
return threads_num # pragma: no cover
return 1
download_queue = queue.Queue(get_threads_count())
consecutive_files_found = Counter(0)
def should_break(counter):
"""Exit if until_found condition is reached"""
return until_found is not None and counter.value() >= until_found
def worker(counter):
"""Threaded worker"""
while True:
item = download_queue.get()
if item is None:
break
download_photo(counter, item)
download_queue.task_done()
threads = []
for _ in range(get_threads_count()):
thread = threading.Thread(target=worker,
args=(consecutive_files_found, ))
thread.daemon = True
thread.start()
threads.append(thread)
photos_iterator = iter(photos_enumerator)
while True:
try:
if should_break(consecutive_files_found):
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting"
% until_found)
break
download_queue.put(next(photos_iterator))
except StopIteration:
break
if not should_break(consecutive_files_found):
download_queue.join()
for _ in threads:
download_queue.put(None)
for thread in threads:
thread.join()
if only_print_filenames:
sys.exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
size,
recent,
until_found,
skip_videos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
should_send_2sa_notification = smtp_username is not None
try:
icloud = authenticate(
username,
password,
should_send_2sa_notification,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug(
"Looking up all photos%s...",
"" if skip_videos else " and videos")
photos = icloud.photos.all
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str, size, plural_suffix, video_suffix, directory
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (
only_print_filenames or no_progress_bar or not sys.stdout.isatty()
):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(MAX_RETRIES):
if skip_videos and not photo.item_type == "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename
)
break
try:
created_date = photo.created.astimezone(get_localzone())
except ValueError:
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)" %
photo.created, logging.ERROR)
created_date = photo.created
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
download_size = size
# Fall back to original if requested size is not available
if size not in photo.versions and size != "original":
if force_size:
filename = photo.filename.encode(
"utf-8").decode("ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename), logging.ERROR)
break
download_size = "original"
download_path = local_download_path(
photo, download_size, download_dir)
download_path_without_size = local_download_path(
photo, None, download_dir)
# add a check if the "simple" name of the file is found if the size
# is original
if os.path.isfile(download_path) or (
download_size == "original" and
os.path.isfile(download_path_without_size)
):
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description(
"%s already exists." % truncate_middle(download_path, 96)
)
break
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" % truncated_path)
download_result = download.download_photo(
icloud, photo, download_path, download_size
)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path, date_str
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
if until_found is not None:
consecutive_files_found = 0
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting"
% until_found
)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
album,
list_albums,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
state_store,
state_path,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (smtp_username is not None
or notification_email is not None
or notification_script is not None)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# Default album is "All Photos", so this is the same as
# calling `icloud.photos.all`.
photos = icloud.photos.albums[album]
if list_albums:
albums_dict = icloud.photos.albums
# Python2: itervalues, Python3: values()
if sys.version_info[0] >= 3:
albums = albums_dict.values() # pragma: no cover
else:
albums = albums_dict.itervalues() # pragma: no cover
album_titles = [str(a) for a in albums]
print(*album_titles, sep="\n")
exit(0)
if state_store == "sqlite":
if state_path is None:
logger.tqdm_write("No state path supplied for sqlite")
exit(1)
statemgr = state_lib.SQLiteMediaManager(filepath=state_path)
elif state_store == "json":
if state_path is None:
logger.tqdm_write("No state path supplied for json state")
exit(1)
statemgr = state_lib.FileMediaManager(filepath=state_path)
else:
statemgr = state_lib.FilesMediaManager()
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug("Looking up all photos%s from album %s...",
"" if skip_videos else " and videos", album)
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later.")
raise ex
logger.tqdm_write("Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (only_print_filenames
or no_progress_bar
or not sys.stdout.isatty()):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(constants.MAX_RETRIES):
### Process media skipping rules
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename)
break
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type))
break
### Process photo ctime
try:
created_date = photo.created.astimezone(get_localzone())
except (ValueError, OSError):
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)"
% photo.created, logging.ERROR)
created_date = photo.created
# Process photo dir path schema
try:
date_path = folder_structure.format(created_date)
except ValueError: # pragma: no cover
# This error only seems to happen in Python 2
logger.set_tqdm_description(
"Photo created date was not valid (%s)" % photo.created,
logging.ERROR)
# e.g. ValueError: year=5 is before 1900
# (https://github.com/ndbroadbent/icloud_photos_downloader/issues/122)
# Just use the Unix epoch
created_date = datetime.datetime.fromtimestamp(0)
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
### process download size (user-defined vs available)
download_size = size
try:
versions = photo.versions
except KeyError as ex:
print(
"KeyError: %s attribute was not found in the photo fields!"
% ex)
with open('icloudpd-photo-error.json', 'w') as outfile:
# pylint: disable=protected-access
json.dump(
{
"master_record": photo._master_record,
"asset_record": photo._asset_record
}, outfile)
# pylint: enable=protected-access
print("icloudpd has saved the photo record to: "
"./icloudpd-photo-error.json")
print("Please create a Gist with the contents of this file: "
"https://gist.github.com")
print(
"Then create an issue on GitHub: "
"https://github.com/ndbroadbent/icloud_photos_downloader/issues"
)
print(
"Include a link to the Gist in your issue, so that we can "
"see what went wrong.\n")
break
# resort to "original" size by default
if size not in versions and size != "original":
if force_size:
filename = photo.filename.encode("utf-8").decode(
"ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename),
logging.ERROR,
)
break
download_size = "original"
download_path = local_download_path(photo, download_size,
download_dir)
### process downloading our photo
already_processed = statemgr.processed(
download_dir=download_dir,
download_size=download_size,
photo=photo,
)
if already_processed:
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description("%s already processed." %
truncate_middle(download_path, 96))
else:
if until_found is not None:
consecutive_files_found = 0
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
# update photo state to STARTED
statemgr.update(download_size=download_size,
photo=photo,
state=state_lib.STATE_ENUM["STARTED"])
download_result = download.download_media(
icloud, photo, download_path, download_size)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# update photo state to FINISHED
if download_result:
statemgr.update(download_size=download_size,
hash=download_result,
photo=photo,
state=state_lib.STATE_ENUM["FINISHED"])
### process downloading videos for any live photos
if not skip_live_photos:
# live photo video size
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
lp_download_path = local_download_path_lp(
media=photo,
lp_size=lp_size,
download_dir=download_dir)
if only_print_filenames:
print(lp_download_path)
else:
# skip if we've already processed this lp's video
if statemgr.processed(download_size=lp_size,
download_dir=download_dir,
photo=photo):
logger.set_tqdm_description(
"%s already processed." %
truncate_middle(lp_download_path, 96))
break
# update photo state to STARTED
statemgr.update(download_size=lp_size,
photo=photo,
state=state_lib.STATE_ENUM["STARTED"])
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download_result = download.download_media(
icloud, photo, lp_download_path, lp_size)
# update photo state to FINISHED
if download_result:
statemgr.update(
download_size=lp_size,
hash=download_result,
photo=photo,
state=state_lib.STATE_ENUM["FINISHED"])
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting" %
until_found)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (
smtp_username is not None
or notification_email is not None
or notification_script is not None
)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug(
"Looking up all photos%s...",
"" if skip_videos else " and videos")
photos = icloud.photos.all
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later."
)
raise ex
logger.tqdm_write(
"Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (
only_print_filenames or no_progress_bar or not sys.stdout.isatty()
):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(constants.MAX_RETRIES):
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename
)
break
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type)
)
break
try:
created_date = photo.created.astimezone(get_localzone())
except (ValueError, OSError):
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)" %
photo.created, logging.ERROR)
created_date = photo.created
try:
date_path = folder_structure.format(created_date)
except ValueError: # pragma: no cover
# This error only seems to happen in Python 2
logger.set_tqdm_description(
"Photo created date was not valid (%s)" %
photo.created, logging.ERROR)
# e.g. ValueError: year=5 is before 1900
# (https://github.com/ndbroadbent/icloud_photos_downloader/issues/122)
# Just use the Unix epoch
created_date = datetime.datetime.fromtimestamp(0)
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
download_size = size
try:
versions = photo.versions
except KeyError as ex:
print(
"KeyError: %s attribute was not found in the photo fields!" %
ex)
with open('icloudpd-photo-error.json', 'w') as outfile:
# pylint: disable=protected-access
json.dump({
"master_record": photo._master_record,
"asset_record": photo._asset_record
}, outfile)
# pylint: enable=protected-access
print("icloudpd has saved the photo record to: "
"./icloudpd-photo-error.json")
print("Please create a Gist with the contents of this file: "
"https://gist.github.com")
print(
"Then create an issue on GitHub: "
"https://github.com/ndbroadbent/icloud_photos_downloader/issues")
print(
"Include a link to the Gist in your issue, so that we can "
"see what went wrong.\n")
break
if size not in versions and size != "original":
if force_size:
filename = photo.filename.encode(
"utf-8").decode("ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename), logging.ERROR, )
break
download_size = "original"
download_path = local_download_path(
photo, download_size, download_dir)
file_exists = os.path.isfile(download_path)
if not file_exists and download_size == "original":
# Deprecation - We used to download files like IMG_1234-original.jpg,
# so we need to check for these.
# Now we match the behavior of iCloud for Windows: IMG_1234.jpg
original_download_path = ("-%s." % size).join(
download_path.rsplit(".", 1)
)
file_exists = os.path.isfile(original_download_path)
if file_exists:
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description(
"%s already exists." % truncate_middle(download_path, 96)
)
else:
if until_found is not None:
consecutive_files_found = 0
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description(
"Downloading %s" %
truncated_path)
download_result = download.download_media(
icloud, photo, download_path, download_size
)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# Also download the live photo if present
if not skip_live_photos:
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
version = photo.versions[lp_size]
filename = version["filename"]
if live_photo_size != "original":
# Add size to filename if not original
filename = filename.replace(
".MOV", "-%s.MOV" %
live_photo_size)
lp_download_path = os.path.join(download_dir, filename)
if only_print_filenames:
print(lp_download_path)
else:
if os.path.isfile(lp_download_path):
logger.set_tqdm_description(
"%s already exists."
% truncate_middle(lp_download_path, 96)
)
break
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description(
"Downloading %s" % truncated_path)
download.download_media(
icloud, photo, lp_download_path, lp_size
)
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting"
% until_found
)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
def main(
directory,
username,
password,
cookie_directory,
size,
live_photo_size,
recent,
until_found,
skip_videos,
skip_live_photos,
force_size,
auto_delete,
only_print_filenames,
folder_structure,
set_exif_datetime,
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
log_level,
no_progress_bar,
notification_script,
):
"""Download all iCloud photos to a local directory"""
logger = setup_logger()
if only_print_filenames:
logger.disabled = True
else:
# Need to make sure disabled is reset to the correct value,
# because the logger instance is shared between tests.
logger.disabled = False
if log_level == "debug":
logger.setLevel(logging.DEBUG)
elif log_level == "info":
logger.setLevel(logging.INFO)
elif log_level == "error":
logger.setLevel(logging.ERROR)
raise_error_on_2sa = (smtp_username is not None
or notification_email is not None
or notification_script is not None)
try:
icloud = authenticate(
username,
password,
cookie_directory,
raise_error_on_2sa,
client_id=os.environ.get("CLIENT_ID"),
)
except TwoStepAuthRequiredError:
if notification_script is not None:
subprocess.call([notification_script])
if smtp_username is not None or notification_email is not None:
send_2sa_notification(
smtp_username,
smtp_password,
smtp_host,
smtp_port,
smtp_no_tls,
notification_email,
)
exit(1)
# For Python 2.7
if hasattr(directory, "decode"):
directory = directory.decode("utf-8") # pragma: no cover
directory = os.path.normpath(directory)
logger.debug("Looking up all photos%s...",
"" if skip_videos else " and videos")
photos = icloud.photos.all
def photos_exception_handler(ex, retries):
"""Handles session errors in the PhotoAlbum photos iterator"""
if "Invalid global session" in str(ex):
if retries > constants.MAX_RETRIES:
logger.tqdm_write(
"iCloud re-authentication failed! Please try again later.")
raise ex
logger.tqdm_write("Session error, re-authenticating...",
logging.ERROR)
if retries > 1:
# If the first reauthentication attempt failed,
# start waiting a few seconds before retrying in case
# there are some issues with the Apple servers
time.sleep(constants.WAIT_SECONDS)
icloud.authenticate()
photos.exception_handler = photos_exception_handler
photos_count = len(photos)
# Optional: Only download the x most recent photos.
if recent is not None:
photos_count = recent
photos = itertools.islice(photos, recent)
tqdm_kwargs = {"total": photos_count}
if until_found is not None:
del tqdm_kwargs["total"]
photos_count = "???"
# ensure photos iterator doesn't have a known length
photos = (p for p in photos)
plural_suffix = "" if photos_count == 1 else "s"
video_suffix = ""
photos_count_str = "the first" if photos_count == 1 else photos_count
if not skip_videos:
video_suffix = " or video" if photos_count == 1 else " and videos"
logger.info(
"Downloading %s %s photo%s%s to %s/ ...",
photos_count_str,
size,
plural_suffix,
video_suffix,
directory,
)
consecutive_files_found = 0
# Use only ASCII characters in progress bar
tqdm_kwargs["ascii"] = True
# Skip the one-line progress bar if we're only printing the filenames,
# or if the progress bar is explicity disabled,
# or if this is not a terminal (e.g. cron or piping output to file)
if not os.environ.get("FORCE_TQDM") and (only_print_filenames
or no_progress_bar
or not sys.stdout.isatty()):
photos_enumerator = photos
logger.set_tqdm(None)
else:
photos_enumerator = tqdm(photos, **tqdm_kwargs)
logger.set_tqdm(photos_enumerator)
# pylint: disable-msg=too-many-nested-blocks
for photo in photos_enumerator:
for _ in range(constants.MAX_RETRIES):
if skip_videos and photo.item_type != "image":
logger.set_tqdm_description(
"Skipping %s, only downloading photos." % photo.filename)
break
if photo.item_type != "image" and photo.item_type != "movie":
logger.set_tqdm_description(
"Skipping %s, only downloading photos and videos. "
"(Item type was: %s)" % (photo.filename, photo.item_type))
break
try:
created_date = photo.created.astimezone(get_localzone())
except ValueError:
logger.set_tqdm_description(
"Could not convert photo created date to local timezone (%s)"
% photo.created,
logging.ERROR,
)
created_date = photo.created
date_path = folder_structure.format(created_date)
download_dir = os.path.join(directory, date_path)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
download_size = size
# Fall back to original if requested size is not available
if size not in photo.versions and size != "original":
if force_size:
filename = photo.filename.encode("utf-8").decode(
"ascii", "ignore")
logger.set_tqdm_description(
"%s size does not exist for %s. Skipping..." %
(size, filename),
logging.ERROR,
)
break
download_size = "original"
download_path = local_download_path(photo, download_size,
download_dir)
file_exists = os.path.isfile(download_path)
if not file_exists and download_size == "original":
# Deprecation - We used to download files like IMG_1234-original.jpg,
# so we need to check for these.
# Now we match the behavior of iCloud for Windows: IMG_1234.jpg
original_download_path = ("-%s." % size).join(
download_path.rsplit(".", 1))
file_exists = os.path.isfile(original_download_path)
if file_exists:
if until_found is not None:
consecutive_files_found += 1
logger.set_tqdm_description("%s already exists." %
truncate_middle(download_path, 96))
else:
if until_found is not None:
consecutive_files_found = 0
if only_print_filenames:
print(download_path)
else:
truncated_path = truncate_middle(download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download_result = download.download_media(
icloud, photo, download_path, download_size)
if download_result and set_exif_datetime:
if photo.filename.lower().endswith((".jpg", ".jpeg")):
if not exif_datetime.get_photo_exif(download_path):
# %Y:%m:%d looks wrong but it's the correct format
date_str = created_date.strftime(
"%Y:%m:%d %H:%M:%S")
logger.debug(
"Setting EXIF timestamp for %s: %s",
download_path,
date_str,
)
exif_datetime.set_photo_exif(
download_path,
created_date.strftime("%Y:%m:%d %H:%M:%S"),
)
else:
timestamp = time.mktime(created_date.timetuple())
os.utime(download_path, (timestamp, timestamp))
# Also download the live photo if present
if not skip_live_photos:
lp_size = live_photo_size + "Video"
if lp_size in photo.versions:
version = photo.versions[lp_size]
filename = version["filename"]
if live_photo_size != "original":
# Add size to filename if not original
filename = filename.replace(
".MOV", "-%s.MOV" % live_photo_size)
lp_download_path = os.path.join(download_dir, filename)
if only_print_filenames:
print(lp_download_path)
else:
if os.path.isfile(lp_download_path):
logger.set_tqdm_description(
"%s already exists." %
truncate_middle(lp_download_path, 96))
break
truncated_path = truncate_middle(lp_download_path, 96)
logger.set_tqdm_description("Downloading %s" %
truncated_path)
download.download_media(icloud, photo,
lp_download_path, lp_size)
break
if until_found is not None and consecutive_files_found >= until_found:
logger.tqdm_write(
"Found %d consecutive previously downloaded photos. Exiting" %
until_found)
if hasattr(photos_enumerator, "close"):
photos_enumerator.close()
break
if only_print_filenames:
exit(0)
logger.info("All photos have been downloaded!")
if auto_delete:
autodelete_photos(icloud, folder_structure, directory)
