def test_separators(self):
self.assertEqual(get_full_path('moo/boo', 'foo.jpg'),
''.join(['moo_boo', os.path.sep, 'foo.jpg']))
self.assertEqual(get_full_path('moo', 'foo/faa.jpg'),
''.join(['moo', os.path.sep, 'foo_faa.jpg']))
self.assertEqual(get_full_path('moo/boo', 'foo/faa.jpg'),
''.join(['moo_boo', os.path.sep, 'foo_faa.jpg']))
def download_set(set_id, get_filename, size_label=None):
"""
Download the set with 'set_id' to the current directory.
@param set_id: str, id of the photo set
@param get_filename: Function, function that creates a filename for the photo
@param size_label: str|None, size to download (or None for largest available)
"""
suffix = " ({})".format(size_label) if size_label else ""
pset = Flickr.Photoset(id=set_id)
photos = pset.getPhotos()
pagenum = 2
while True:
try:
page = pset.getPhotos(page=pagenum)
photos.extend(page)
pagenum += 1
except FlickrAPIError as ex:
if ex.code == 1:
break
raise
# we need to convert pathname separator to something else to create a valid directory
dirname = pset.title.replace(os.sep, "_")
if not os.path.exists(dirname):
os.mkdir(dirname)
for photo in photos:
fname = get_full_path(dirname, get_filename(pset, photo, suffix))
if os.path.exists(fname):
# TODO: Ideally we should check for file size / md5 here
# to handle failed downloads.
print('Skipping {0}, as it exists already'.format(fname))
continue
print('Saving: {0}'.format(fname))
photo.save(fname, size_label)
# Set file times to when the photo was taken
info = photo.getInfo()
taken = parser.parse(info['taken'])
taken_unix = time.mktime(taken.timetuple())
os.utime(fname, (taken_unix, taken_unix))
def do_download_photo(dirname, pset, photo, size_label, suffix, get_filename):
"""
Handle the downloading of a single photo
@param dirname: str, directory to download to
@param pset: FlickrList, photo list to download
@param photo: FlickrPhoto, photo to download
@param size_label: str|None, size to download (or None for largest available)
@param suffix: str|None, optional suffix to add to file name
@param get_filename: Function, function that creates a filename for the photo
"""
fname = get_full_path(dirname, get_filename(pset, photo, suffix))
if 'video' in photo.getInfo():
if 'HD MP4' in photo.getSizes():
photo_size_label = 'HD MP4'
else:
# Fall back for old 'short videos'
photo_size_label = 'Site MP4'
fname = fname + '.mp4'
else:
photo_size_label = size_label
fname = fname + '.jpg'
if os.path.exists(fname):
# TODO: Ideally we should check for file size / md5 here
# to handle failed downloads.
print('Skipping {0}, as it exists already'.format(fname))
return
print('Saving: {} ({})'.format(fname, photo.getPageUrl()))
try:
photo.save(fname, photo_size_label)
except IOError, ex:
logging.warning('IO error saving photo: {}'.format(ex.strerror))
return
def do_download_photo(dirname, pset, photo, size_label, suffix, get_filename):
"""
Handle the downloading of a single photo
@param dirname: str, directory to download to
@param pset: FlickrList, photo list to download
@param photo: FlickrPhoto, photo to download
@param size_label: str|None, size to download (or None for largest available)
@param suffix: str|None, optional suffix to add to file name
@param get_filename: Function, function that creates a filename for the photo
"""
fname = get_full_path(dirname, get_filename(pset, photo, suffix))
if 'video' in photo.getInfo():
if 'HD MP4' in photo.getSizes():
photo_size_label = 'HD MP4'
else:
# Fall back for old 'short videos'
photo_size_label = 'Site MP4'
fname = fname + '.mp4'
else:
photo_size_label = size_label
fname = fname + '.jpg'
if os.path.exists(fname):
# TODO: Ideally we should check for file size / md5 here
# to handle failed downloads.
print('Skipping {0}, as it exists already'.format(fname))
return
print('Saving: {} ({})'.format(fname, photo.getPageUrl()))
try:
photo.save(fname, photo_size_label)
except IOError, ex:
logging.warning('IO error saving photo: {}'.format(ex.strerror))
return
def do_download_photo(dirname, pset, photo, size_label, suffix, get_filename, skip_download=False,
save_json=False):
"""
Handle the downloading of a single photo
@param dirname: str, directory to download to
@param pset: FlickrList, photo list to download
@param photo: FlickrPhoto, photo to download
@param size_label: str|None, size to download (or None for largest available)
@param suffix: str|None, optional suffix to add to file name
@param get_filename: Function, function that creates a filename for the photo
@param skip_download: bool, do not actually download the photo
@param save_json: bool, save photo info as .json file
"""
fname = get_full_path(dirname, get_filename(pset, photo, suffix))
jsonFname = fname + '.json'
pInfo = {}
try:
with Timer('getInfo()'):
pInfo = photo.getInfo()
except FlickrError:
print('Skipping {0}, because cannot get info from Flickr'.format(fname))
return
if save_json:
try:
print('Saving photo info: {}'.format(jsonFname))
jsonFile = open(jsonFname, "w")
jsonFile.write(json.dumps(pInfo, default=serialize_json, indent=2, sort_keys=True))
jsonFile.close()
except Exception:
print("Trouble saving photo info:", sys.exc_info()[0])
if 'video' in pInfo:
with Timer('getSizes()'):
pSizes = get_photo_sizes(photo)
if 'HD MP4' in pSizes:
photo_size_label = 'HD MP4'
else:
# Fall back for old 'short videos'
photo_size_label = 'Site MP4'
fname = fname + '.mp4'
else:
photo_size_label = size_label
suffix = '.jpg'
# Flickr returns JPEG, except for when downloading originals. The only way to find the
# original type it seems is through the source filename. This is not pretty...
if (photo_size_label == 'Original' or not photo_size_label):
with Timer('getSizes()'):
pSizes = get_photo_sizes(photo)
meta = pSizes.get('Original')
if (meta and meta['source']):
ext = os.path.splitext(meta['source'])[1]
if (ext):
suffix = ext
fname = fname + suffix
if os.path.exists(fname):
# TODO: Ideally we should check for file size / md5 here
# to handle failed downloads.
print('Skipping {0}, as it exists already'.format(fname))
return
print('Saving: {} ({})'.format(fname, get_photo_page(pInfo)))
if skip_download:
return
try:
with Timer('save()'):
photo.save(fname, photo_size_label)
except IOError, ex:
logging.warning('IO error saving photo: {}'.format(ex.strerror))
return
def test_simple(self):
self.assertEqual(get_full_path('moo', 'foo.jpg'),
''.join(['moo', os.path.sep, 'foo.jpg']))
def do_download_photo(dirname,
pset,
photo,
size_label,
suffix,
get_filename,
skip_download=False):
"""
Handle the downloading of a single photo
@param dirname: str, directory to download to
@param pset: FlickrList, photo list to download
@param photo: FlickrPhoto, photo to download
@param size_label: str|None, size to download (or None for largest available)
@param suffix: str|None, optional suffix to add to file name
@param get_filename: Function, function that creates a filename for the photo
@param skip_download: bool, do not actually download the photo
"""
fname = get_full_path(dirname, get_filename(pset, photo, suffix))
with Timer('getInfo()'):
pInfo = photo.getInfo()
if 'video' in pInfo:
with Timer('getSizes()'):
pSizes = photo.getSizes()
if 'HD MP4' in pSizes:
photo_size_label = 'HD MP4'
else:
# Fall back for old 'short videos'
photo_size_label = 'Site MP4'
fname = fname + '.mp4'
else:
photo_size_label = size_label
suffix = '.jpg'
# Flickr returns JPEG, except for when downloading originals. The only way to find the
# original type it seems is through the source filename. This is not pretty...
if (photo_size_label == 'Original' or not photo_size_label):
with Timer('getSizes()'):
pSizes = photo.getSizes()
meta = pSizes.get('Original')
if (meta and meta['source']):
ext = os.path.splitext(meta['source'])[1]
if (ext):
suffix = ext
fname = fname + suffix
if os.path.exists(fname):
# TODO: Ideally we should check for file size / md5 here
# to handle failed downloads.
print('Skipping {0}, as it exists already'.format(fname))
return
print('Saving: {} ({})'.format(fname, get_photo_page(pInfo)))
if skip_download:
return
try:
with Timer('save()'):
photo.save(fname, photo_size_label)
except IOError as ex:
logging.warning('IO error saving photo: {}'.format(ex.strerror))
return
except FlickrError as ex:
logging.warning('Flickr error saving photo: {}'.format(str(ex)))
return
# Set file times to when the photo was taken
taken = parser.parse(pInfo['taken'])
taken_unix = time.mktime(taken.timetuple())
os.utime(fname, (taken_unix, taken_unix))
