def test_get_url_ok_urlsplit(self): with patch('lib.make_pycurl_request', Mock(return_value=('content', 'market://url'))): with patch('lib.prepare_url', Mock(return_value='prepare_url')): prepare_url, redirect_type, content = get_url('url', 5, 'user_agent') self.assertEqual(prepare_url, 'prepare_url') self.assertEqual(redirect_type, 'http_status') self.assertEqual(content, 'content')
def test_get_url_with_meta_redirect(self): with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content_with_meta, None))): with mock.patch('lib.prepare_url', mock.Mock(return_value=self.meta_url)) as prepare: self.assertEqual(get_url(self.url, self.timeout, None), (self.meta_url, REDIRECT_META, self.content_with_meta), 'meta redirect') prepare.assert_called_once_with(self.meta_url)
def test_get_url_ok_redirect(self): with patch('lib.make_pycurl_request', Mock(return_value=('content', 'http://www.odnoklassniki.ru/sdfst.redirect'))): prepare_url, redirect_type, content = get_url('url', 5, 'user_agent') self.assertEqual(prepare_url, None) self.assertEqual(redirect_type, None) self.assertEqual(content, 'content')
def test_get_url_request_exception(self): url = 'url' with patch('lib.make_pycurl_request', Mock(side_effect=ValueError)): prepared_redirect_url, redirect_type, content = get_url(url, 11, 'user_agent') assert prepared_redirect_url == url assert redirect_type == 'ERROR' assert content is None
def Load(recipe, version, **kwargs): url = get_url(recipe, version) print(url) try: return Flow(load(url, name=recipe, format='csv')) except: print(recipe)
def test_get_url_check_for_meta_bad(self): with patch('lib.make_pycurl_request', Mock(return_value=('content', None))): with patch('lib.check_for_meta', Mock(return_value=None)): with patch('lib.prepare_url', Mock(return_value='prepare_url')): prepare_url, redirect_type, content = get_url('url', 5, 'user_agent') self.assertEqual(prepare_url, 'prepare_url') self.assertEqual(redirect_type, None) self.assertEqual(content, 'content')
def Load(recipe, version, **kwargs): url = get_url(recipe, version) print(url) return Flow( load(url, name=recipe, format='csv', force_strings=True) )
def test_get_url_ok_urlsplit(self): with patch('lib.make_pycurl_request', Mock(return_value=('content', 'market://url'))): with patch('lib.prepare_url', Mock(return_value='prepare_url')): prepare_url, redirect_type, content = get_url( 'url', 5, 'user_agent') self.assertEqual(prepare_url, 'prepare_url') self.assertEqual(redirect_type, 'http_status') self.assertEqual(content, 'content')
def test_get_url_ok_redirect(self): with patch( 'lib.make_pycurl_request', Mock(return_value=( 'content', 'http://www.odnoklassniki.ru/sdfst.redirect'))): prepare_url, redirect_type, content = get_url( 'url', 5, 'user_agent') self.assertEqual(prepare_url, None) self.assertEqual(redirect_type, None) self.assertEqual(content, 'content')
def test_get_url_ok_login_redirect(self): my_content = 'content' redirect_url = 'http://www.odnoklassniki.ru/st.redirect' with patch('lib.make_pycurl_request', Mock(return_value=(my_content, redirect_url))): prepared_redirect_url, redirect_type, content = get_url('url', 11, 'user_agent') assert prepared_redirect_url is None assert redirect_type is None assert content == my_content
def test_get_url_redirect_url_none_meta_none(self): my_content = 'content' with patch('lib.make_pycurl_request', Mock(return_value=(my_content, None))): with patch('lib.prepare_url', Mock(side_effect=self.mocked_lib_prepare_url)): with patch('lib.check_for_meta', Mock(return_value=None)): prepared_redirect_url, redirect_type, content = get_url('url', 11) assert prepared_redirect_url is None assert redirect_type is None assert content == my_content
def test_get_url(self): my_content = 'content' my_prepared_redirect_url = 'redirect_url' with patch('lib.make_pycurl_request', Mock(return_value=(my_content, 'redirect_url'))): with patch('lib.prepare_url', Mock(side_effect=self.mocked_lib_prepare_url)): prepared_redirect_url, redirect_type, content = get_url('url', 11, 'user_agent') assert prepared_redirect_url == my_prepared_redirect_url assert redirect_type == REDIRECT_HTTP assert content == my_content
def test_get_url_check_for_meta_bad(self): with patch('lib.make_pycurl_request', Mock(return_value=('content', None))): with patch('lib.check_for_meta', Mock(return_value=None)): with patch('lib.prepare_url', Mock(return_value='prepare_url')): prepare_url, redirect_type, content = get_url( 'url', 5, 'user_agent') self.assertEqual(prepare_url, 'prepare_url') self.assertEqual(redirect_type, None) self.assertEqual(content, 'content')
def test_get_url_redirect_url_none_meta_ok(self): my_content = 'content' my_prepared_redirect_url = 'redirect_url_form_meta' with patch('lib.make_pycurl_request', Mock(return_value=(my_content, None))): with patch('lib.prepare_url', Mock(side_effect=self.mocked_lib_prepare_url)): with patch('lib.check_for_meta', Mock(return_value='redirect_url_form_meta')): prepared_redirect_url, redirect_type, content = get_url('url', 11) assert prepared_redirect_url == my_prepared_redirect_url assert redirect_type == REDIRECT_META assert content == my_content
def test_get_url_error(self): url_test = 'url_test' timeout_test = 400 mock_make_pycurl = mock.Mock(side_effect=pycurl.error) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(url_test, url) self.assertEqual('ERROR', red) self.assertEqual(None, con)
def test_get_url_error(self): url_test = 'url_test' timeout_test = 400 mock_make_pycurl = mock.Mock(side_effect=pycurl.error) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(url_test, url) self.assertEqual('ERROR', red) self.assertEqual(None, con)
def test_get_url_none(self): content = 'test' new_redirect = 'http://www.odnoklassniki.ru/st.redirect/' url_test = 'url_test' timeout_test = 400 mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.prepare_url', mock.Mock(return_value=None)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(None, url) self.assertEqual(None, red) self.assertEqual(content, con)
def test_get_url_none(self): content = 'test' new_redirect = 'http://www.odnoklassniki.ru/st.redirect/' url_test = 'url_test' timeout_test = 400 mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.prepare_url', mock.Mock(return_value=None)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(None, url) self.assertEqual(None, red) self.assertEqual(content, con)
def test_get_url_http(self): content = 'content' new_redirect = 'http://test.com' url_test = 'url_test' timeout_test = 10 REDIRECT_HTTP = 'http_status' mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) mock_check_for_meta = mock.Mock(return_value=new_redirect) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.check_for_meta', mock_check_for_meta): with mock.patch('lib.prepare_url', mock.Mock(return_value=new_redirect)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(new_redirect, url) self.assertEqual(REDIRECT_HTTP, red) self.assertEqual(content, con)
def test_get_url_http(self): content = 'content' new_redirect = 'http://test.com' url_test = 'url_test' timeout_test = 10 REDIRECT_HTTP = 'http_status' mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) mock_check_for_meta = mock.Mock(return_value=new_redirect) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.check_for_meta', mock_check_for_meta): with mock.patch('lib.prepare_url', mock.Mock(return_value=new_redirect)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(new_redirect, url) self.assertEqual(REDIRECT_HTTP, red) self.assertEqual(content, con)
def test_get_url_meta(self): content = 'content' new_redirect = None new_redirect_meta = 'new_red_meta' url_test = 'url_test' timeout_test = 10 REDIRECT_META = 'meta_tag' mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) mock_check_for_meta = mock.Mock(return_value=new_redirect_meta) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.check_for_meta', mock_check_for_meta): with mock.patch('lib.prepare_url', mock.Mock(return_value=new_redirect_meta)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(new_redirect_meta, url) self.assertEqual(REDIRECT_META, red) self.assertEqual(content, con)
def test_get_url_meta(self): content = 'content' new_redirect = None new_redirect_meta = 'new_red_meta' url_test = 'url_test' timeout_test = 10 REDIRECT_META = 'meta_tag' mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) mock_check_for_meta = mock.Mock(return_value=new_redirect_meta) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.check_for_meta', mock_check_for_meta): with mock.patch('lib.prepare_url', mock.Mock(return_value=new_redirect_meta)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) self.assertEqual(new_redirect_meta, url) self.assertEqual(REDIRECT_META, red) self.assertEqual(content, con)
def test_get_url_market(self): content = 'test' new_redirect = 'market://www.test/' return_fix_market = 'http://play.google.com/store/apps/www.test/' url_test = 'url_test' timeout_test = 400 REDIRECT_HTTP = 'http_status' mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) mock_fix_market_url = mock.Mock(return_value=return_fix_market) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.fix_market_url', mock_fix_market_url): with mock.patch('lib.prepare_url', mock.Mock(return_value=return_fix_market)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) mock_fix_market_url.assert_called_once_with(new_redirect) self.assertEqual(return_fix_market, url) self.assertEqual(REDIRECT_HTTP, red) self.assertEqual(content, con)
def test_get_url_market(self): content = 'test' new_redirect = 'market://www.test/' return_fix_market = 'http://play.google.com/store/apps/www.test/' url_test = 'url_test' timeout_test = 400 REDIRECT_HTTP = 'http_status' mock_make_pycurl = mock.Mock(return_value=(content, new_redirect)) mock_fix_market_url = mock.Mock(return_value=return_fix_market) with mock.patch('lib.make_pycurl_request', mock_make_pycurl): with mock.patch('lib.fix_market_url', mock_fix_market_url): with mock.patch('lib.prepare_url', mock.Mock(return_value=return_fix_market)): url, red, con = get_url(url_test, timeout_test) mock_make_pycurl.assert_called_once_with(url_test, timeout_test, None) mock_fix_market_url.assert_called_once_with(new_redirect) self.assertEqual(return_fix_market, url) self.assertEqual(REDIRECT_HTTP, red) self.assertEqual(content, con)
def test_get_url_with_error(self): self.assertEqual(get_url(self.url, self.timeout, None), (self.url, ERROR_GET_URL, None), 'error should occur')
def test_get_url_http_redirect(self): with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content, self.http_redir))): with mock.patch('lib.prepare_url', mock.Mock(return_value=self.http_redir)) as prepare: self.assertEqual(get_url(self.url, self.timeout, None), (self.http_redir, REDIRECT_HTTP, self.content), 'simple http redir') prepare.assert_called_once_with(self.http_redir)
def test_get_url_exc(self): with patch('lib.make_pycurl_request', Mock(side_effect=ValueError('ValueError'))): self.assertRaises(Exception, get_url('url', 5, 'user_agent'))
def test_get_url_with_market_redirect(self): with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content, self.market_redir))): self.assertEqual(get_url(self.url, self.timeout, None), (GOOGLE_MARKET_URL + self.market_url, REDIRECT_HTTP, self.content), ' market redirect')
def test_get_url_exc(self): with patch('lib.make_pycurl_request', Mock(side_effect=ValueError('ValueError'))): self.assertRaises(Exception, get_url('url', 5, 'user_agent'))
def test_get_url_odnoklassniki_redirect(self): with mock.patch('lib.make_pycurl_request', mock.Mock(return_value=(self.content, self.ok_redir_url))): self.assertEqual(get_url(self.url, self.timeout, None), (None, None, self.content), 'ignore ok login redirects')
def Load(recipe, version, **kwargs): url = get_url(recipe, version) print(url) return Flow(load(url, name=recipe))