def main(): # Set up commandline arguments parser = argparse.ArgumentParser() parser.add_argument('query', help='item to download images for') parser.add_argument('count', help='number of images to download', default=20) parser.add_argument('type', help='extension to save images as') # Set up browser browser = mechanize.Browser() user_agent = 'Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36' browser.addheaders = [('User-agent', user_agent)] browser.set_handle_robots(False) # Extract commandline arguments args = parser.parse_args() query = args.query count = int(args.count) extension = args.type # Run downloads downloader = ImageDownloader(browser, query) try: print 'Bringing your imaginations to reality...' downloader.download_images(count, extension) except KeyboardInterrupt: print '\nBye. See you soon :)' sys.exit(0)
def test_resolution_valid_value(self): '''Ensure if we can create ImageDownloader object with valid input''' test_cases = ['640x480', '1280x1024', '1920x1080'] for value in test_cases: with self.subTest(x=value): image_downloader = ImageDownloader(value) self.assertEqual( image_downloader.validate_input(value), value) self.assertEqual(image_downloader.resolution, value)
def test_is_downloaded_file_valid_with_valid_scenario(): response = Mock() response.headers = {'content-type': 'image/jpg'} is_valid = ImageDownloader().is_downloaded_file_valid(response) assert is_valid, 'Not an Image file'
def test_getting_image_links(self): '''Ensure if ImageDownloader object can return links from html content''' response_content = ''' <a href=http://files.com/wallpapers/cal/may-19-hello-spring-cal-800x480.png \ title="Hello Spring! - 800x480">800x480</a>,<a href=http://files.com/wallpapers/nocal\ /may-19-hello-spring-nocal-800x480.png title="Hello Spring! - 800x480">800x480</a>,\ <a href=http://files.com/wallpapers/nocal/may-19-hello-spring-nocal-1024x768.png \ title="Hello Spring! - 1024x768">1024x768</a>''' test_cases = [ ( '800x480', [ 'http://files.com/wallpapers/cal/may-19-hello-spring-cal-800x480.png', 'http://files.com/wallpapers/nocal/may-19-hello-spring-nocal-800x480.png' ] ), ( '1024x768', ['http://files.com/wallpapers/nocal/may-19-hello-spring-nocal-1024x768.png'] ) ] for resolution, exp_output in test_cases: with self.subTest(x=resolution): self.assertEqual( ImageDownloader(resolution).get_image_links(response_content), exp_output )
def test_fetch_valid_input(self, mock_fetch): '''Ensure if we can get valid url with valid input data''' url = 'http://someurl.com/test' base_resolution = '640x480' test_cases = [ '-r 1280x1024 -m 1 -y 2015', '-r 1280x1024 -m 12 -y 2019', '-r 1920x1080 -m May -y 2015', '-r 1920x1080 -m December -y 2019', ] for x in test_cases: with self.subTest(x=x): self.runner.invoke(main, x.split(), input='3') _, res, _, month, _, year = x.split() month = Month(month) # image_downloader = ImageDownloader( # url, month.number, month.name, int(year)) new_url = ImageDownloader(base_resolution).get_url( url, month.number, month.name, int(year)) self.assertTrue( # ImageDownloader.get_url.called_with( mock_fetch.called_with( new_url, timeout=5 ) )
def test_is_downloaded_file_valid_with_wrong_file_type(): response = Mock() response.headers = { 'content-disposition': 'attachment; filename="test.pdf"; filename*=UTF-8\\test.pdf' } with pytest.raises(InvalidFormatException): ImageDownloader().is_downloaded_file_valid(response)
def test_is_downloaded_file_valid_with_content_disposition_header(): response = Mock() response.headers = { 'content-disposition': 'attachment; filename="test.png"; filename*=UTF-8\\test.png' } is_valid = ImageDownloader().is_downloaded_file_valid(response) assert is_valid, 'Not an Image file'
async def main(start, stop, workers, loop): downloader = ImageDownloader(folder_path='./data') datalogger = DataLogger(folder_path='./logs') fetcher = DataFetcher(data_parser=DataParser(), downloader=downloader, datalogger=datalogger) ids = range(start, stop, workers) for release_id in ids: tasks = [fetcher.process(release_id + i) for i in range(workers)] await asyncio.wait(tasks, loop=loop)
def test_getting_url(self): '''Ensure if ImageDownloader object can return url in expected format''' test_cases = [ ( ('abc.com', 5, 'May', 2015), 'https://abc.com/2015/04/desktop-wallpaper-calendars-may-2015/' ), ( ('abc.com', 11, 'November', 2013), 'https://abc.com/2013/10/desktop-wallpaper-calendars-november-2013/' ), ( ('abc.com', 1, 'January', 2019), 'https://abc.com/2018/12/desktop-wallpaper-calendars-january-2019/' ) ] image_downloader = ImageDownloader(self.base_resolution) for x, exp_output in test_cases: with self.subTest(x=x): self.assertEqual(image_downloader.get_url(*x), exp_output)
def test_fetching_content(self, mock_get): '''Ensure if ImageDownloader object can make valid request''' image_downloader = ImageDownloader(self.base_resolution) resp = image_downloader.fetch_content('http://someurl.com/test') self.assertEqual(resp, 'some_data') resp = image_downloader.fetch_content('http://someotherurl.com/test') self.assertEqual(resp, 'other_data') with self.assertRaises(Exception) as err: image_downloader.fetch_content('http://nonexistenturl.com/test') self.assertIn('Error', err.exception.args[0]) with self.assertRaises(Exception) as err: image_downloader.fetch_content('http://unabletoconnecturl.com/test') self.assertIn('Connection timed out', err.exception.args) with self.assertRaises(Exception) as err: image_downloader.fetch_content('http://otherunknownurl.com/test') self.assertIn('Unable to establish connection', err.exception.args)
def test_creating_directory(self, mock_os): '''Ensure if ImageDownloader object can create directory''' base_directory, month, year = '/', 'may', 2015 directory_name = 'Smashing_wallpaper_{0}_{1}'.format(month, str(year)) ImageDownloader(self.base_resolution).create_directory( base_directory, month, year) self.assertTrue( mock_os.makedirs.called_with( mock_os.path.join(base_directory, directory_name), exist_ok=True ) )
def crawl(self): thumbnails, originals = ImageDownloader.create_playlist_directory( self.playlist.id) videos = [] next_page_token = None while True: response = self.request_playlist_page(next_page_token) for item in response['items']: video = Video() self.set_video_details(video, item, originals, thumbnails) videos.append(video) next_page_token = response.get('nextPageToken') if not next_page_token: break self.set_video_stats(videos) return videos
def test_is_downloaded_file_valid_with_no_content_disposition_header(): response = Mock() response.headers = {'content-type': 'application/json'} with pytest.raises(InvalidFormatException): ImageDownloader().is_downloaded_file_valid(response)