def test_handle_received_image__no_result_return(self, mocker):
dummy_result_original = {
'image': Image.Image(),
'requested_at': datetime.now(),
'url': 'http://example.com',
'error': None
}
dummy_result_after_1 = dummy_result_original.copy()
dummy_result_after_1['handler_1'] = 'done'
dummy_result_after_3 = dummy_result_after_1.copy()
dummy_result_after_3['handler_3'] = 'done'
mock_handler_1 = mocker.Mock(return_value=dummy_result_after_1)
mock_handler_2 = mocker.Mock(return_value=None)
mock_handler_3 = mocker.Mock(return_value=dummy_result_after_3)
grabber = Grabber('http://example.com')
mocker.patch.object(grabber,
'get_result_handlers',
return_value=(mock_handler_1, mock_handler_2,
mock_handler_3))
result = grabber.handle_received_image(dummy_result_original)
grabber.get_result_handlers.assert_called_once_with()
mock_handler_1.assert_called_once_with(dummy_result_original, grabber)
mock_handler_2.assert_called_once_with(dummy_result_after_1, grabber)
mock_handler_3.assert_called_once_with(dummy_result_after_1, grabber)
assert result == dummy_result_after_3
def test_download_image__exception_ignored(self, mocker):
correct_url = 'http://example.com'
now = datetime.now()
e = Exception()
dummy_request = {'requested_at': now, 'url': correct_url}
dummy_downloader = mocker.Mock(side_effect=e)
grabber = Grabber('http://bad-url.com')
grabber.get_download_callable = mocker.Mock(
grabber.get_download_callable,
autospec=True,
return_value=dummy_downloader,
)
grabber.ignore_download_exception = mocker.Mock(
grabber.ignore_download_exception,
autospec=True,
return_value=True)
expected_result = {
'requested_at': now,
'url': correct_url,
'image': None,
'error': e
}
result = grabber.download_image(dummy_request)
assert result == expected_result
grabber.get_download_callable.assert_called_once_with()
dummy_downloader.assert_called_once_with(correct_url, grabber)
grabber.ignore_download_exception.assert_called_once_with(e)
def test_ignore_download_exception(self, mocker):
url = 'http://example.com'
socket_errors = ((socket.timeout(), 'ignore_timeout'), )
urllib_errors = (
(URLError(reason=socket.timeout(),
filename=None), 'ignore_timeout'),
(URLError(reason=socket.gaierror(),
filename=None), 'ignore_network'),
(URLError(reason=OSError(), filename=None), 'ignore_network'),
)
http_status_codes = (403, 404, 500)
http_errors = []
for code in http_status_codes:
http_errors.append((
HTTPError(url=url, code=code, msg='msg', hdrs={}, fp=None),
'ignore_{}'.format(code),
))
errors_and_flags = (tuple(socket_errors) + tuple(urllib_errors) +
tuple(http_errors))
grabber = Grabber('http://example.com')
for e, flag in errors_and_flags:
# Ignore error
setattr(grabber, flag, True)
assert grabber.ignore_download_exception(e) is True
# Don't ignore error
setattr(grabber, flag, False)
assert grabber.ignore_download_exception(e) is False
def test_get_download_callable(self, mocker):
grabber = Grabber('http://example.com')
mock_downloader = mocker.Mock()
grabber.download_callable = mock_downloader
assert grabber.get_download_callable() is mock_downloader
def test_download_image(self, mocker):
correct_url = 'http://example.com'
dummy_image = Image.Image()
now = datetime.now()
dummy_request = {'requested_at': now, 'url': correct_url}
dummy_downloader = mocker.Mock(return_value=dummy_image)
grabber = Grabber('http://bad-url.com', save_dir='some/dir')
grabber.get_download_callable = mocker.Mock(
grabber.get_download_callable,
autospec=True,
return_value=dummy_downloader)
expected_result = {
'requested_at': now,
'url': correct_url,
'image': dummy_image,
'error': None,
}
result = grabber.download_image(dummy_request)
assert result == expected_result
dummy_downloader.assert_called_once_with(correct_url, grabber)
grabber.get_download_callable.assert_called_once_with()
def test_slugify(self):
grabber = Grabber('http://example.com')
tests = (
('abcDEFghi', 'abcdefghi'),
('a-a b_b c.c d:d', 'a-a-b_b-c-c-d-d'),
)
for value, expected in tests:
assert grabber.slugify(value) == expected
def test_create_save_path_dirs(self, mocker):
mocked_makedirs = mocker.patch('camgrab.camgrab.makedirs')
path = 'a_dir/somewhere/another_dir/final_file.jpg'
grabber = Grabber('http://example.com')
grabber.make_save_path_dirs(path)
mocked_makedirs.assert_called_once_with('a_dir/somewhere/another_dir',
exist_ok=True)
def test_tick__exception(self, mocker):
grabber = Grabber('http://example.com')
an_exception = IOError('some exception')
mocker.patch.object(grabber,
'create_request',
side_effect=an_exception)
with pytest.raises(IOError):
grabber.tick()
assert grabber.failed_exception is an_exception
def test_format_path(self, mocker):
request = {
'requested_at': datetime(2017, 1, 2, 12, 13, 14, 987654),
'url': 'http://a-url.co.uk:1234/some_path/0123456789/some-file.jpg'
}
path_format = 'a_dir/{url}/{Y}{m}{d}/{H}{M}{S}-{f}.jpg'
expected = ('a_dir/'
'http-a-url-co-uk-1234-some_path-0123456789-some-file-jpg/'
'20170102/121314-987654.jpg')
grabber = Grabber('http://example.com')
result = grabber.format_path(path_format, request)
assert result == expected
def test_create_request(self, mocker):
mocked_datetime = mocker.patch('camgrab.camgrab.datetime',
autospec=True)
dummy_datetime = datetime(2017, 1, 2, 12, 13, 14, 987654)
mocked_datetime.now = mocker.Mock(return_value=dummy_datetime)
url = 'http://example.com'
grabber = Grabber(url)
expected_request = {
'url': url,
'requested_at': dummy_datetime,
}
assert grabber.create_request() == expected_request
mocked_datetime.now.assert_called_once_with()
def test_do_save_image__no_image(self, mocker):
grabber = Grabber('http://example.com')
dummy_save_path = 'some/{Y}/path.jpg'
dummy_save_path_full = 'some/2017/path.jpg'
mocker.patch.object(grabber,
'get_save_path',
return_value=dummy_save_path)
mocker.patch.object(grabber, 'make_save_path_dirs')
mocker.patch.object(grabber, 'should_save_image', return_value=True)
mocker.patch.object(grabber,
'format_path',
return_value=dummy_save_path_full)
dummy_result = {
'image': None,
'url': 'http://example.com',
'error': None,
'requested_at': datetime.now(),
}
expected_result = dummy_result.copy()
expected_result['is_saved'] = False
expected_result['save_dir'] = grabber.save_dir
expected_result['save_path'] = dummy_save_path
expected_result['save_path_full'] = dummy_save_path_full
result = do_save_image(dummy_result, grabber)
grabber.get_save_path.assert_called_once_with()
grabber.should_save_image.assert_not_called()
grabber.make_save_path_dirs.assert_not_called()
grabber.format_path.assert_called_once_with(dummy_save_path, result)
assert result == expected_result
def test_tick(self, mocker):
im = Image.Image()
dummy_request = {
'url': 'http://example.com',
'requested_at': datetime.now()
}
grabber = Grabber('http://example.com')
grabber.create_request = mocker.Mock(grabber.create_request,
autospec=True,
return_value=dummy_request)
dummy_result = {'image': im, 'requested_at': datetime.now()}
grabber.download_image = mocker.Mock(grabber.download_image,
autospec=True,
return_value=dummy_result)
grabber.handle_received_image = mocker.Mock(
grabber.handle_received_image, autospec=True)
grabber.tick()
grabber.create_request.assert_called_once_with()
grabber.download_image.assert_called_once_with(dummy_request)
grabber.handle_received_image.assert_called_once_with(dummy_result)
def test_get_image_from_url(self):
dummy_url = 'http://a-url.com/'
dummy_timeout = 123
dummy_image_path = os.path.join(os.path.dirname(__file__), 'assets',
'kitty.jpg')
dummy_body = open(dummy_image_path, 'rb').read()
expected_result = Image.open(dummy_image_path)
httpretty.register_uri(httpretty.GET,
dummy_url,
body=dummy_body,
content_type='image/jpeg')
grabber = Grabber('http://example.com')
grabber.timeout = dummy_timeout
result = get_image_from_url(dummy_url, grabber)
assert result == expected_result
def test_handle_received_image(self, mocker):
grabber = Grabber('http://example.com')
dummy_result = {
'image': Image.Image(),
'requested_at': datetime.now(),
'url': 'http://example.com',
'error': None
}
# If no handlers, then should just return a dict of data
grabber.get_result_handlers = mocker.Mock(
'grabber.get_result_handlers', return_value=())
assert grabber.handle_received_image(dummy_result) == dummy_result
grabber.get_result_handlers.assert_called_once_with()
mock_handler_1_return = dummy_result.copy()
mock_handler_1_return['some_key'] = 'value'
mock_handler_1 = mocker.Mock(return_value=mock_handler_1_return)
mock_handler_2_return = dummy_result.copy()
mock_handler_2_return['another_key'] = 'value'
mock_handler_2 = mocker.Mock(return_value=mock_handler_2_return)
grabber.get_result_handlers = mocker.Mock(
'grabber.get_result_handlers',
return_value=(mock_handler_1, mock_handler_2),
)
result = grabber.handle_received_image(dummy_result)
assert result == mock_handler_2_return
mock_handler_1.assert_called_once_with(dummy_result, grabber)
mock_handler_2.assert_called_once_with(mock_handler_1_return, grabber)
def test_download_image__exception_raised(self, mocker):
correct_url = 'http://example.com'
now = datetime.now()
e = Exception()
dummy_request = {'requested_at': now, 'url': correct_url}
dummy_downloader = mocker.Mock(side_effect=e)
grabber = Grabber('http://bad-url.com')
grabber.get_download_callable = mocker.Mock(
grabber.get_download_callable,
autospec=True,
return_value=dummy_downloader,
)
grabber.ignore_download_exception = mocker.Mock(
grabber.ignore_download_exception,
autospec=True,
return_value=False)
with pytest.raises(Exception):
grabber.download_image(dummy_request)
grabber.get_download_callable.assert_called_once_with()
dummy_downloader.assert_called_once_with(correct_url, grabber)
grabber.ignore_download_exception.assert_called_once_with(e)
def test_init(self):
url = 'http://example.com'
grabber = Grabber(url)
# Default attribute values
assert grabber.url == url
assert grabber.every == 2
assert grabber.save_dir == 'grabbed_images'
assert grabber.download_callable == get_image_from_url
assert grabber.default_result_handlers == (do_save_image, )
assert grabber.extra_result_handlers == []
assert grabber.result_handlers is None
assert grabber.timeout == 30
assert grabber.save_filename == (
'{Y}{m}{d}/{H}/{Y}{m}{d}-{H}{M}{S}-{f}.jpg')
assert grabber.save is True
assert grabber.ignore_timeout is True
assert grabber.ignore_network is True
assert grabber.failed_exception is None
ignore_status_codes = (307, 400, 408, 409, 429, 444, 451, 499, 500,
502, 503, 504, 507, 599)
for code in ignore_status_codes:
assert getattr(grabber, 'ignore_{}'.format(code)) is True
assert grabber._test_max_ticks is None
kwargs = {
'url': url,
'every': 0.5,
'save_dir': 'somewhere_else',
'download_callable': lambda x: x,
'extra_result_handlers': [lambda x: x],
}
grabber = Grabber(**kwargs)
for key, value in kwargs.items():
assert getattr(grabber, key) == value
def test_ignore_http_code(self):
grabber = Grabber('http://example.com')
status_code = 999 # Definitely won't exist
assert grabber.ignore_http_code(status_code) is False
grabber.ignore_999 = True
assert grabber.ignore_http_code(status_code) is True
grabber.ignore_999 = False
assert grabber.ignore_http_code(status_code) is False
def test_begin(self, mocker):
grabber = Grabber('http://example.com')
grabber._test_max_ticks = 5
grabber.every = 13
grabber.tick = mocker.Mock(grabber.tick, autospec=True)
mock_sleep = mocker.patch('camgrab.camgrab.sleep', autospec=True)
grabber.begin()
assert grabber.tick.call_count == 5
assert mock_sleep.call_count == 5
mock_sleep.assert_called_with(13)
def test_get_save_path(self, mocker):
mocked_datetime = mocker.patch('camgrab.camgrab.datetime',
autospec=True)
fake_datetime = datetime(2017, 1, 2, 12, 13, 14, 987654)
mocked_datetime.now = mocker.Mock(return_value=fake_datetime)
grabber = Grabber('http://example.com')
grabber.save_dir = 'a_dir'
grabber.save_filename = '{Y}{m}{d}/blah/{H}{M}{S}{f}.jpg'
expected = 'a_dir/{Y}{m}{d}/blah/{H}{M}{S}{f}.jpg'
grabber.format_path = mocker.Mock(grabber.format_path,
autospec=True,
return_value=expected)
full_save_path = grabber.get_save_path()
assert full_save_path == expected
def test_save_simple_image(self, mocker):
dummy_datetime = datetime(2017, 1, 2, 12, 13, 14, 987654)
mocked_datetime = mocker.patch('camgrab.camgrab.datetime',
autospec=True,
return_value=dummy_datetime)
mocked_datetime.now = mocker.Mock(return_value=dummy_datetime)
mock_sleep = mocker.patch('camgrab.camgrab.sleep', autospec=True)
mock_image_save = mocker.patch('camgrab.camgrab.Image.Image.save')
mock_makedirs = mocker.patch('camgrab.camgrab.makedirs')
dummy_url = 'http://some-url.com/image.jpg'
dummy_image_path = os.path.join(os.path.dirname(__file__), 'assets',
'kitty.jpg')
dummy_body = open(dummy_image_path, 'rb').read()
httpretty.register_uri(httpretty.GET,
dummy_url,
body=dummy_body,
content_type='image/jpeg')
num_ticks = 5
every = 5
save_dir = 'some_dir'
grabber = Grabber(dummy_url, every=every, save_dir=save_dir)
grabber._test_max_ticks = num_ticks
grabber.save_filename = '{Y}{m}{d}/{H}/{Y}{m}{d}-{H}{M}{S}-{f}.jpg'
expected_dir = 'some_dir/20170102/12'
expected_filename = 'some_dir/20170102/12/20170102-121314-987654.jpg'
grabber.begin()
assert mock_sleep.call_count == num_ticks
mock_sleep.assert_called_with(every)
assert mock_image_save.call_count == num_ticks
mock_image_save.assert_called_with(expected_filename)
assert mock_makedirs.call_count == num_ticks
mock_makedirs.assert_called_with(expected_dir, exist_ok=True)
def test_ignore_download_exception__ignores_general_exceptions(self):
grabber = Grabber('http://example.com')
e = Exception()
assert grabber.ignore_download_exception(e) is False
def test_get_result_handlers(self, mocker):
grabber = Grabber('http://example.com')
# When nothing is set, should still return an iterable
grabber.default_result_handlers = None
grabber.extra_result_handlers = None
grabber.result_handlers = None
assert grabber.get_result_handlers() == ()
grabber.default_result_handlers = ()
grabber.extra_result_handlers = ()
grabber.result_handlers = None
assert grabber.get_result_handlers() == ()
grabber.default_result_handlers = ()
grabber.extra_result_handlers = ()
grabber.result_handlers = ()
assert grabber.get_result_handlers() == ()
mock_handler_1 = mocker.Mock()
mock_handler_2 = mocker.Mock()
mock_handler_3 = mocker.Mock()
mock_handler_4 = mocker.Mock()
grabber.default_result_handlers = (mock_handler_1, )
grabber.extra_result_handlers = ()
grabber.result_handlers = None
assert grabber.get_result_handlers() == (mock_handler_1, )
grabber.default_result_handlers = ()
grabber.extra_result_handlers = (mock_handler_1, )
grabber.result_handlers = None
assert grabber.get_result_handlers() == (mock_handler_1, )
grabber.default_result_handlers = (mock_handler_1, )
grabber.extra_result_handlers = (mock_handler_1, )
grabber.result_handlers = None
assert grabber.get_result_handlers() == (mock_handler_1,
mock_handler_1)
grabber.default_result_handlers = (mock_handler_1, )
grabber.extra_result_handlers = (mock_handler_1, )
grabber.result_handlers = (mock_handler_3, mock_handler_4)
assert grabber.get_result_handlers() == (mock_handler_3,
mock_handler_4)
grabber.default_result_handlers = (mock_handler_1, mock_handler_2)
grabber.extra_result_handlers = (mock_handler_1, mock_handler_3)
grabber.result_handlers = None
assert grabber.get_result_handlers() == (mock_handler_1,
mock_handler_2,
mock_handler_1,
mock_handler_3)
def test_should_save_image(self):
good_grabber = Grabber('http://example.com')
good_grabber.save = True
good_grabber.save_filename = 'some_filename'
good_grabber.save_dir = 'some_dir'
assert good_grabber.should_save_image() is True
no_save_grabber = Grabber('http://example.com')
no_save_filename_grabber = Grabber('http://example.com')
no_save_dir_grabber = Grabber('http://example.com')
no_save_grabber.save = False
no_save_grabber.save_filename = 'some_filename'
no_save_grabber.save_dir = 'some_dir'
no_save_filename_grabber.save = True
no_save_filename_grabber.save_filename = None
no_save_filename_grabber.save_dir = 'some_dir'
no_save_dir_grabber.save = True
no_save_dir_grabber.save_filename = 'some_filename'
no_save_dir_grabber.save_dir = None
bad_grabbers = (no_save_grabber, no_save_dir_grabber,
no_save_filename_grabber)
for grabber in bad_grabbers:
assert grabber.should_save_image() is False
