def fill_queue(self):
page = random.randint(1, 250)
url = 'https://api.unsplash.com/photos/?page=%d&per_page=30&client_id=%s' % (page, UnsplashDownloader.CLIENT_ID)
logger.info(lambda: "Filling Unsplash queue from " + url)
r = Util.request(url)
if int(r.headers.get('X-Ratelimit-Remaining', 1000000)) < 100:
UnsplashDownloader.rate_limiting_started_time = time.time()
for item in r.json():
try:
width = item['width']
height = item['height']
if self.parent and not self.parent.size_ok(width, height):
continue
image_url = item['links']['download']
origin_url = item['links']['html']
filename = os.path.join(self.target_folder, Util.sanitize_filename(image_url.split('/')[-2] + '.jpg'))
extra_metadata = {
'sourceType': 'unsplash',
'sfwRating': 100,
'author': item['user']['name'],
'authorURL': item['user']['links']['html'],
'keywords': [cat['title'].lower().strip() for cat in item['categories']]
}
self.queue.append((origin_url, image_url, extra_metadata, filename))
except:
logger.exception(lambda: "Could not process an item from Unsplash")
raise
random.shuffle(self.queue)
logger.info(lambda: "Unsplash populated with %d URLs" % len(self.queue))
def fill_queue(self):
page = random.randint(1, 250)
url = 'https://api.unsplash.com/photos/?page=%d&per_page=30&client_id=%s' % (
page, UnsplashDownloader.CLIENT_ID)
logger.info(lambda: "Filling Unsplash queue from " + url)
r = Util.request(url)
if int(r.headers.get('X-Ratelimit-Remaining', 1000000)) < 100:
UnsplashDownloader.rate_limiting_started_time = time.time()
for item in r.json():
try:
width = item['width']
height = item['height']
if self.parent and not self.parent.size_ok(width, height):
continue
image_url = item['links']['download']
origin_url = item['links']['html']
filename = os.path.join(
self.target_folder,
Util.sanitize_filename(image_url.split('/')[-2] + '.jpg'))
extra_metadata = {
'sourceType':
'unsplash',
'sfwRating':
100,
'author':
item['user']['name'],
'authorURL':
item['user']['links']['html'],
'keywords': [
cat['title'].lower().strip()
for cat in item['categories']
]
}
self.queue.append(
(origin_url, image_url, extra_metadata, filename))
except:
logger.exception(
lambda: "Could not process an item from Unsplash")
raise
random.shuffle(self.queue)
logger.info(
lambda: "Unsplash populated with %d URLs" % len(self.queue))
def test_sanitize_filename(self):
self.assertEqual("i_m____g_.jpg",
Util.sanitize_filename("i?m?*%^g_.jpg"))
def test_sanitize_filename(self):
self.assertEqual("i_m____g_.jpg", Util.sanitize_filename("i?m?*%^g_.jpg"))
