def test_get_url_ok_urlsplit(self):
with patch('lib.make_pycurl_request', Mock(return_value=('content', 'market://url'))):
with patch('lib.prepare_url', Mock(return_value='prepare_url')):
prepare_url, redirect_type, content = get_url('url', 5, 'user_agent')
self.assertEqual(prepare_url, 'prepare_url')
self.assertEqual(redirect_type, 'http_status')
self.assertEqual(content, 'content')
def test_get_url_with_meta_redirect(self):
with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content_with_meta, None))):
with mock.patch('lib.prepare_url', mock.Mock(return_value=self.meta_url)) as prepare:
self.assertEqual(get_url(self.url, self.timeout, None),
(self.meta_url, REDIRECT_META, self.content_with_meta),
'meta redirect')
prepare.assert_called_once_with(self.meta_url)
def test_get_url_ok_redirect(self):
with patch('lib.make_pycurl_request', Mock(return_value=('content',
'http://www.odnoklassniki.ru/sdfst.redirect'))):
prepare_url, redirect_type, content = get_url('url', 5, 'user_agent')
self.assertEqual(prepare_url, None)
self.assertEqual(redirect_type, None)
self.assertEqual(content, 'content')
def test_get_url_request_exception(self):
url = 'url'
with patch('lib.make_pycurl_request', Mock(side_effect=ValueError)):
prepared_redirect_url, redirect_type, content = get_url(url, 11, 'user_agent')
assert prepared_redirect_url == url
assert redirect_type == 'ERROR'
assert content is None
def Load(recipe, version, **kwargs):
url = get_url(recipe, version)
print(url)
try:
return Flow(load(url, name=recipe, format='csv'))
except:
print(recipe)
def test_get_url_check_for_meta_bad(self):
with patch('lib.make_pycurl_request', Mock(return_value=('content', None))):
with patch('lib.check_for_meta', Mock(return_value=None)):
with patch('lib.prepare_url', Mock(return_value='prepare_url')):
prepare_url, redirect_type, content = get_url('url', 5, 'user_agent')
self.assertEqual(prepare_url, 'prepare_url')
self.assertEqual(redirect_type, None)
self.assertEqual(content, 'content')
def Load(recipe, version, **kwargs):
url = get_url(recipe, version)
print(url)
return Flow(
load(url, name=recipe, format='csv', force_strings=True)
)
def test_get_url_ok_urlsplit(self):
with patch('lib.make_pycurl_request',
Mock(return_value=('content', 'market://url'))):
with patch('lib.prepare_url', Mock(return_value='prepare_url')):
prepare_url, redirect_type, content = get_url(
'url', 5, 'user_agent')
self.assertEqual(prepare_url, 'prepare_url')
self.assertEqual(redirect_type, 'http_status')
self.assertEqual(content, 'content')
def test_get_url_ok_redirect(self):
with patch(
'lib.make_pycurl_request',
Mock(return_value=(
'content', 'http://www.odnoklassniki.ru/sdfst.redirect'))):
prepare_url, redirect_type, content = get_url(
'url', 5, 'user_agent')
self.assertEqual(prepare_url, None)
self.assertEqual(redirect_type, None)
self.assertEqual(content, 'content')
def test_get_url_ok_login_redirect(self):
my_content = 'content'
redirect_url = 'http://www.odnoklassniki.ru/st.redirect'
with patch('lib.make_pycurl_request', Mock(return_value=(my_content, redirect_url))):
prepared_redirect_url, redirect_type, content = get_url('url', 11, 'user_agent')
assert prepared_redirect_url is None
assert redirect_type is None
assert content == my_content
def test_get_url_redirect_url_none_meta_none(self):
my_content = 'content'
with patch('lib.make_pycurl_request', Mock(return_value=(my_content, None))):
with patch('lib.prepare_url', Mock(side_effect=self.mocked_lib_prepare_url)):
with patch('lib.check_for_meta', Mock(return_value=None)):
prepared_redirect_url, redirect_type, content = get_url('url', 11)
assert prepared_redirect_url is None
assert redirect_type is None
assert content == my_content
def test_get_url(self):
my_content = 'content'
my_prepared_redirect_url = 'redirect_url'
with patch('lib.make_pycurl_request', Mock(return_value=(my_content, 'redirect_url'))):
with patch('lib.prepare_url', Mock(side_effect=self.mocked_lib_prepare_url)):
prepared_redirect_url, redirect_type, content = get_url('url', 11, 'user_agent')
assert prepared_redirect_url == my_prepared_redirect_url
assert redirect_type == REDIRECT_HTTP
assert content == my_content
def test_get_url_check_for_meta_bad(self):
with patch('lib.make_pycurl_request',
Mock(return_value=('content', None))):
with patch('lib.check_for_meta', Mock(return_value=None)):
with patch('lib.prepare_url',
Mock(return_value='prepare_url')):
prepare_url, redirect_type, content = get_url(
'url', 5, 'user_agent')
self.assertEqual(prepare_url, 'prepare_url')
self.assertEqual(redirect_type, None)
self.assertEqual(content, 'content')
def test_get_url_redirect_url_none_meta_ok(self):
my_content = 'content'
my_prepared_redirect_url = 'redirect_url_form_meta'
with patch('lib.make_pycurl_request', Mock(return_value=(my_content, None))):
with patch('lib.prepare_url', Mock(side_effect=self.mocked_lib_prepare_url)):
with patch('lib.check_for_meta', Mock(return_value='redirect_url_form_meta')):
prepared_redirect_url, redirect_type, content = get_url('url', 11)
assert prepared_redirect_url == my_prepared_redirect_url
assert redirect_type == REDIRECT_META
assert content == my_content
def test_get_url_error(self):
url_test = 'url_test'
timeout_test = 400
mock_make_pycurl = mock.Mock(side_effect=pycurl.error)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(url_test, url)
self.assertEqual('ERROR', red)
self.assertEqual(None, con)
def test_get_url_error(self):
url_test = 'url_test'
timeout_test = 400
mock_make_pycurl = mock.Mock(side_effect=pycurl.error)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(url_test, url)
self.assertEqual('ERROR', red)
self.assertEqual(None, con)
def test_get_url_none(self):
content = 'test'
new_redirect = 'http://www.odnoklassniki.ru/st.redirect/'
url_test = 'url_test'
timeout_test = 400
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.prepare_url', mock.Mock(return_value=None)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(None, url)
self.assertEqual(None, red)
self.assertEqual(content, con)
def test_get_url_none(self):
content = 'test'
new_redirect = 'http://www.odnoklassniki.ru/st.redirect/'
url_test = 'url_test'
timeout_test = 400
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.prepare_url', mock.Mock(return_value=None)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(None, url)
self.assertEqual(None, red)
self.assertEqual(content, con)
def test_get_url_http(self):
content = 'content'
new_redirect = 'http://test.com'
url_test = 'url_test'
timeout_test = 10
REDIRECT_HTTP = 'http_status'
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
mock_check_for_meta = mock.Mock(return_value=new_redirect)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.check_for_meta', mock_check_for_meta):
with mock.patch('lib.prepare_url', mock.Mock(return_value=new_redirect)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(new_redirect, url)
self.assertEqual(REDIRECT_HTTP, red)
self.assertEqual(content, con)
def test_get_url_http(self):
content = 'content'
new_redirect = 'http://test.com'
url_test = 'url_test'
timeout_test = 10
REDIRECT_HTTP = 'http_status'
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
mock_check_for_meta = mock.Mock(return_value=new_redirect)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.check_for_meta', mock_check_for_meta):
with mock.patch('lib.prepare_url',
mock.Mock(return_value=new_redirect)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(new_redirect, url)
self.assertEqual(REDIRECT_HTTP, red)
self.assertEqual(content, con)
def test_get_url_meta(self):
content = 'content'
new_redirect = None
new_redirect_meta = 'new_red_meta'
url_test = 'url_test'
timeout_test = 10
REDIRECT_META = 'meta_tag'
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
mock_check_for_meta = mock.Mock(return_value=new_redirect_meta)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.check_for_meta', mock_check_for_meta):
with mock.patch('lib.prepare_url', mock.Mock(return_value=new_redirect_meta)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(new_redirect_meta, url)
self.assertEqual(REDIRECT_META, red)
self.assertEqual(content, con)
def test_get_url_meta(self):
content = 'content'
new_redirect = None
new_redirect_meta = 'new_red_meta'
url_test = 'url_test'
timeout_test = 10
REDIRECT_META = 'meta_tag'
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
mock_check_for_meta = mock.Mock(return_value=new_redirect_meta)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.check_for_meta', mock_check_for_meta):
with mock.patch('lib.prepare_url',
mock.Mock(return_value=new_redirect_meta)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
self.assertEqual(new_redirect_meta, url)
self.assertEqual(REDIRECT_META, red)
self.assertEqual(content, con)
def test_get_url_market(self):
content = 'test'
new_redirect = 'market://www.test/'
return_fix_market = 'http://play.google.com/store/apps/www.test/'
url_test = 'url_test'
timeout_test = 400
REDIRECT_HTTP = 'http_status'
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
mock_fix_market_url = mock.Mock(return_value=return_fix_market)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.fix_market_url', mock_fix_market_url):
with mock.patch('lib.prepare_url', mock.Mock(return_value=return_fix_market)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
mock_fix_market_url.assert_called_once_with(new_redirect)
self.assertEqual(return_fix_market, url)
self.assertEqual(REDIRECT_HTTP, red)
self.assertEqual(content, con)
def test_get_url_market(self):
content = 'test'
new_redirect = 'market://www.test/'
return_fix_market = 'http://play.google.com/store/apps/www.test/'
url_test = 'url_test'
timeout_test = 400
REDIRECT_HTTP = 'http_status'
mock_make_pycurl = mock.Mock(return_value=(content, new_redirect))
mock_fix_market_url = mock.Mock(return_value=return_fix_market)
with mock.patch('lib.make_pycurl_request', mock_make_pycurl):
with mock.patch('lib.fix_market_url', mock_fix_market_url):
with mock.patch('lib.prepare_url',
mock.Mock(return_value=return_fix_market)):
url, red, con = get_url(url_test, timeout_test)
mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None)
mock_fix_market_url.assert_called_once_with(new_redirect)
self.assertEqual(return_fix_market, url)
self.assertEqual(REDIRECT_HTTP, red)
self.assertEqual(content, con)
def test_get_url_with_error(self):
self.assertEqual(get_url(self.url, self.timeout, None), (self.url, ERROR_GET_URL, None), 'error should occur')
def test_get_url_http_redirect(self):
with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content, self.http_redir))):
with mock.patch('lib.prepare_url', mock.Mock(return_value=self.http_redir)) as prepare:
self.assertEqual(get_url(self.url, self.timeout, None), (self.http_redir, REDIRECT_HTTP, self.content),
'simple http redir')
prepare.assert_called_once_with(self.http_redir)
def test_get_url_exc(self):
with patch('lib.make_pycurl_request',
Mock(side_effect=ValueError('ValueError'))):
self.assertRaises(Exception, get_url('url', 5, 'user_agent'))
def test_get_url_with_market_redirect(self):
with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content, self.market_redir))):
self.assertEqual(get_url(self.url, self.timeout, None),
(GOOGLE_MARKET_URL + self.market_url, REDIRECT_HTTP, self.content),
' market redirect')
def test_get_url_exc(self):
with patch('lib.make_pycurl_request', Mock(side_effect=ValueError('ValueError'))):
self.assertRaises(Exception, get_url('url', 5, 'user_agent'))
def test_get_url_odnoklassniki_redirect(self):
with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content, self.ok_redir_url))):
self.assertEqual(get_url(self.url, self.timeout, None), (None, None, self.content),
'ignore ok login redirects')
def Load(recipe, version, **kwargs):
url = get_url(recipe, version)
print(url)
return Flow(load(url, name=recipe))