def main():
# Set up commandline arguments
parser = argparse.ArgumentParser()
parser.add_argument('query', help='item to download images for')
parser.add_argument('count',
help='number of images to download',
default=20)
parser.add_argument('type', help='extension to save images as')
# Set up browser
browser = mechanize.Browser()
user_agent = 'Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36'
browser.addheaders = [('User-agent', user_agent)]
browser.set_handle_robots(False)
# Extract commandline arguments
args = parser.parse_args()
query = args.query
count = int(args.count)
extension = args.type
# Run downloads
downloader = ImageDownloader(browser, query)
try:
print 'Bringing your imaginations to reality...'
downloader.download_images(count, extension)
except KeyboardInterrupt:
print '\nBye. See you soon :)'
sys.exit(0)
def test_resolution_valid_value(self):
'''Ensure if we can create ImageDownloader object with valid input'''
test_cases = ['640x480', '1280x1024', '1920x1080']
for value in test_cases:
with self.subTest(x=value):
image_downloader = ImageDownloader(value)
self.assertEqual(
image_downloader.validate_input(value), value)
self.assertEqual(image_downloader.resolution, value)
def test_is_downloaded_file_valid_with_valid_scenario():
response = Mock()
response.headers = {'content-type': 'image/jpg'}
is_valid = ImageDownloader().is_downloaded_file_valid(response)
assert is_valid, 'Not an Image file'
def test_getting_image_links(self):
'''Ensure if ImageDownloader object can return links from html content'''
response_content = '''
<a href=http://files.com/wallpapers/cal/may-19-hello-spring-cal-800x480.png \
title="Hello Spring! - 800x480">800x480</a>,<a href=http://files.com/wallpapers/nocal\
/may-19-hello-spring-nocal-800x480.png title="Hello Spring! - 800x480">800x480</a>,\
<a href=http://files.com/wallpapers/nocal/may-19-hello-spring-nocal-1024x768.png \
title="Hello Spring! - 1024x768">1024x768</a>'''
test_cases = [
(
'800x480',
[
'http://files.com/wallpapers/cal/may-19-hello-spring-cal-800x480.png',
'http://files.com/wallpapers/nocal/may-19-hello-spring-nocal-800x480.png'
]
),
(
'1024x768',
['http://files.com/wallpapers/nocal/may-19-hello-spring-nocal-1024x768.png']
)
]
for resolution, exp_output in test_cases:
with self.subTest(x=resolution):
self.assertEqual(
ImageDownloader(resolution).get_image_links(response_content),
exp_output
)
def test_fetch_valid_input(self, mock_fetch):
'''Ensure if we can get valid url with valid input data'''
url = 'http://someurl.com/test'
base_resolution = '640x480'
test_cases = [
'-r 1280x1024 -m 1 -y 2015',
'-r 1280x1024 -m 12 -y 2019',
'-r 1920x1080 -m May -y 2015',
'-r 1920x1080 -m December -y 2019',
]
for x in test_cases:
with self.subTest(x=x):
self.runner.invoke(main, x.split(), input='3')
_, res, _, month, _, year = x.split()
month = Month(month)
# image_downloader = ImageDownloader(
# url, month.number, month.name, int(year))
new_url = ImageDownloader(base_resolution).get_url(
url, month.number, month.name, int(year))
self.assertTrue(
# ImageDownloader.get_url.called_with(
mock_fetch.called_with(
new_url,
timeout=5
)
)
def test_is_downloaded_file_valid_with_wrong_file_type():
response = Mock()
response.headers = {
'content-disposition':
'attachment; filename="test.pdf"; filename*=UTF-8\\test.pdf'
}
with pytest.raises(InvalidFormatException):
ImageDownloader().is_downloaded_file_valid(response)
def test_is_downloaded_file_valid_with_content_disposition_header():
response = Mock()
response.headers = {
'content-disposition':
'attachment; filename="test.png"; filename*=UTF-8\\test.png'
}
is_valid = ImageDownloader().is_downloaded_file_valid(response)
assert is_valid, 'Not an Image file'
async def main(start, stop, workers, loop):
downloader = ImageDownloader(folder_path='./data')
datalogger = DataLogger(folder_path='./logs')
fetcher = DataFetcher(data_parser=DataParser(),
downloader=downloader,
datalogger=datalogger)
ids = range(start, stop, workers)
for release_id in ids:
tasks = [fetcher.process(release_id + i) for i in range(workers)]
await asyncio.wait(tasks, loop=loop)
def test_getting_url(self):
'''Ensure if ImageDownloader object can return url in expected format'''
test_cases = [
(
('abc.com', 5, 'May', 2015),
'https://abc.com/2015/04/desktop-wallpaper-calendars-may-2015/'
),
(
('abc.com', 11, 'November', 2013),
'https://abc.com/2013/10/desktop-wallpaper-calendars-november-2013/'
),
(
('abc.com', 1, 'January', 2019),
'https://abc.com/2018/12/desktop-wallpaper-calendars-january-2019/'
)
]
image_downloader = ImageDownloader(self.base_resolution)
for x, exp_output in test_cases:
with self.subTest(x=x):
self.assertEqual(image_downloader.get_url(*x), exp_output)
def test_fetching_content(self, mock_get):
'''Ensure if ImageDownloader object can make valid request'''
image_downloader = ImageDownloader(self.base_resolution)
resp = image_downloader.fetch_content('http://someurl.com/test')
self.assertEqual(resp, 'some_data')
resp = image_downloader.fetch_content('http://someotherurl.com/test')
self.assertEqual(resp, 'other_data')
with self.assertRaises(Exception) as err:
image_downloader.fetch_content('http://nonexistenturl.com/test')
self.assertIn('Error', err.exception.args[0])
with self.assertRaises(Exception) as err:
image_downloader.fetch_content('http://unabletoconnecturl.com/test')
self.assertIn('Connection timed out', err.exception.args)
with self.assertRaises(Exception) as err:
image_downloader.fetch_content('http://otherunknownurl.com/test')
self.assertIn('Unable to establish connection', err.exception.args)
def test_creating_directory(self, mock_os):
'''Ensure if ImageDownloader object can create directory'''
base_directory, month, year = '/', 'may', 2015
directory_name = 'Smashing_wallpaper_{0}_{1}'.format(month, str(year))
ImageDownloader(self.base_resolution).create_directory(
base_directory, month, year)
self.assertTrue(
mock_os.makedirs.called_with(
mock_os.path.join(base_directory, directory_name),
exist_ok=True
)
)
def crawl(self):
thumbnails, originals = ImageDownloader.create_playlist_directory(
self.playlist.id)
videos = []
next_page_token = None
while True:
response = self.request_playlist_page(next_page_token)
for item in response['items']:
video = Video()
self.set_video_details(video, item, originals, thumbnails)
videos.append(video)
next_page_token = response.get('nextPageToken')
if not next_page_token:
break
self.set_video_stats(videos)
return videos
def test_is_downloaded_file_valid_with_no_content_disposition_header():
response = Mock()
response.headers = {'content-type': 'application/json'}
with pytest.raises(InvalidFormatException):
ImageDownloader().is_downloaded_file_valid(response)