def _on_request(
self,
adapter: "HTTPAdapter",
request: "PreparedRequest",
*,
retries: Optional["_Retry"] = None,
**kwargs: Any,
) -> "models.Response":
# add attributes params and req_kwargs to 'request' object for further match comparison
# original request object does not have these attributes
request.params = self._parse_request_params(request.path_url) # type: ignore[attr-defined]
request.req_kwargs = kwargs # type: ignore[attr-defined]
request_url = str(request.url)
match, match_failed_reasons = self._find_match(request)
resp_callback = self.response_callback
if match is None:
if any(
[
p.match(request_url)
if isinstance(p, Pattern)
else request_url.startswith(p)
for p in self.passthru_prefixes
]
):
logger.info("request.allowed-passthru", extra={"url": request_url})
return _real_send(adapter, request, **kwargs)
error_msg = (
"Connection refused by Responses - the call doesn't "
"match any registered mock.\n\n"
"Request: \n"
f"- {request.method} {request_url}\n\n"
"Available matches:\n"
)
for i, m in enumerate(self.registered()):
error_msg += "- {} {} {}\n".format(
m.method, m.url, match_failed_reasons[i]
)
if self.passthru_prefixes:
error_msg += "Passthru prefixes:\n"
for p in self.passthru_prefixes:
error_msg += "- {}\n".format(p)
response = ConnectionError(error_msg)
response.request = request
self._calls.add(request, response)
raise response
if match.passthrough:
logger.info("request.passthrough-response", extra={"url": request_url})
response = _real_send(adapter, request, **kwargs) # type: ignore[assignment]
else:
try:
response = adapter.build_response( # type: ignore[no-untyped-call]
request, match.get_response(request)
)
except BaseException as response:
match.call_count += 1
self._calls.add(request, response)
raise
if resp_callback:
response = resp_callback(response) # type: ignore[misc]
match.call_count += 1
self._calls.add(request, response) # type: ignore[misc]
retries = retries or adapter.max_retries
# first validate that current request is eligible to be retried.
# See ``requests.packages.urllib3.util.retry.Retry`` documentation.
if retries.is_retry(
method=response.request.method, status_code=response.status_code # type: ignore[misc]
):
try:
retries = retries.increment(
method=response.request.method, # type: ignore[misc]
url=response.url, # type: ignore[misc]
response=response, # type: ignore[misc]
)
return self._on_request(adapter, request, retries=retries, **kwargs)
except MaxRetryError:
if retries.raise_on_status:
raise
return response
return response
def callback(uploader):
"""Mimic a connection issue after chunk 1 is sent."""
if len(uploader.blob.uploadedChunkIds) > 1:
raise ConnectionError("Mocked error")
def test_devices_authenticate_view_returns_false_if_hue_bridge_not_reachable(
request_mock, browser, auth_url):
request_mock.side_effect = [ConnectionError('Not reachable')]
assert browser.post_json(auth_url, {}).json == {"id": "ph1", "authenticated": False}
def generate_proxied_request(self,
url,
method="GET",
params={},
data={},
headers={},
req_timeout=30):
try:
random.shuffle(self.proxy_list)
# req_headers = dict(params.items() + self.generate_random_request_headers().items())
req_headers = dict(params.items())
req_headers_random = dict(
self.generate_random_request_headers().items())
req_headers.update(req_headers_random)
if not self.sustain:
self.randomize_proxy()
headers.update(req_headers)
self.logger.debug("Using proxy: {0}".format(str(
self.current_proxy)))
request = requests.request(method,
url,
proxies={"http": self.current_proxy},
headers=headers,
data=data,
params=params,
timeout=req_timeout)
# Avoid HTTP request errors
if request.status_code == 409:
raise ConnectionError(
"HTTP Response [409] - Possible Cloudflare DNS resolution error"
)
elif request.status_code == 403:
raise ConnectionError(
"HTTP Response [403] - Permission denied error")
elif request.status_code == 503:
raise ConnectionError(
"HTTP Response [503] - Service unavailable error")
print('RR Status {}'.format(request.status_code))
return request
except ConnectionError:
try:
self.proxy_list.remove(self.current_proxy)
except ValueError:
pass
self.logger.debug(
"Proxy unreachable - Removed Straggling proxy: {0} PL Size = {1}"
.format(self.current_proxy, len(self.proxy_list)))
self.randomize_proxy()
except ReadTimeout:
try:
self.proxy_list.remove(self.current_proxy)
except ValueError:
pass
self.logger.debug(
"Read timed out - Removed Straggling proxy: {0} PL Size = {1}".
format(self.current_proxy, len(self.proxy_list)))
self.randomize_proxy()
except ChunkedEncodingError:
try:
self.proxy_list.remove(self.current_proxy)
except ValueError:
pass
self.logger.debug(
"Wrong server chunked encoding - Removed Straggling proxy: {0} PL Size = {1}"
.format(self.current_proxy, len(self.proxy_list)))
self.randomize_proxy()
def send(self,
request,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None):
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a (`connect timeout, read
timeout <user/advanced.html#timeouts>`_) tuple.
:type timeout: float or tuple
:param verify: (optional) Whether to verify SSL certificates.
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
"""
conn = self.get_connection(request.url, proxies, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(request)
chunked = not (request.body is None
or 'Content-Length' in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
timeout = TimeoutSauce(connect=connect, read=read)
except ValueError as e:
# this may raise a string formatting error.
err = ("Invalid timeout {0}. Pass a (connect, read) "
"timeout tuple, or a single float to set "
"both timeouts to the same value".format(timeout))
raise ValueError(err)
else:
timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
if not chunked:
resp = conn.urlopen(method=request.method,
url=url,
body=request.body,
headers=request.headers,
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=timeout)
# Send the request.
else:
if hasattr(conn, 'proxy_pool'):
conn = conn.proxy_pool
low_conn = conn._get_conn(timeout=timeout)
try:
low_conn.putrequest(request.method,
url,
skip_accept_encoding=True)
for header, value in request.headers.items():
low_conn.putheader(header, value)
low_conn.endheaders()
for i in request.body:
low_conn.send(hex(len(i))[2:].encode('utf-8'))
low_conn.send(b'\r\n')
low_conn.send(i)
low_conn.send(b'\r\n')
low_conn.send(b'0\r\n\r\n')
r = low_conn.getresponse()
resp = HTTPResponse.from_httplib(r,
pool=conn,
connection=low_conn,
preload_content=False,
decode_content=False)
except:
# If we hit any problems here, clean up the connection.
# Then, reraise so that we can handle the actual exception.
low_conn.close()
raise
else:
# All is well, return the connection to the pool.
conn._put_conn(low_conn)
except (ProtocolError, socket.error) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
raise ConnectionError(e, request=request)
except _ProxyError as e:
raise ProxyError(e)
except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
raise ReadTimeout(e, request=request)
else:
raise
return self.build_response(request, resp)
def test_connection_error(self):
with patch('requests.post', side_effect=ConnectionError()), \
patch('logging.warning'):
self._test('"test" en',
'Could not connect to Google Translate. .',
'test_languages')
def mock_raise_response_error(arg):
raise ConnectionError("Connection Error")
def raise_connectionerror(*args, **kwargs):
raise ConnectionError("qwe")
def _mock_connection_error(url):
mocked_responses.add(
responses.GET, url, body=ConnectionError("Connection error. ")
)
def raise_connection_error(self, reason):
return MagicMock(side_effect=ConnectionError(reason))
def send(self,
request,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None):
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
conn = self.get_connection(request.url, proxies)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(request)
chunked = not (request.body is None
or 'Content-Length' in request.headers)
#非chunked还存在request.body的情况,自动转换为trunked
if not chunked and request.body is not None:
if request.headers.get("Transfer-Encoding", None) != "chunked":
request.headers["Transfer-Encoding"] = "chunked"
if 'Content-Length' in request.headers:
del request.headers['Content-Length']
request.body = self.gen_data(request.body)
chunked = True
if isinstance(timeout, tuple):
try:
connect, read = timeout
timeout = TimeoutSauce(connect=connect, read=read)
except ValueError as e:
# this may raise a string formatting error.
err = ("Invalid timeout {0}. Pass a (connect, read) "
"timeout tuple, or a single float to set "
"both timeouts to the same value".format(timeout))
raise ValueError(err)
elif isinstance(timeout, TimeoutSauce):
pass
else:
timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
if not chunked:
resp = conn.urlopen(method=request.method,
url=url,
body=request.body,
headers=request.headers,
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=timeout)
# Send the request.
else:
if hasattr(conn, 'proxy_pool'):
conn = conn.proxy_pool
low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
try:
low_conn.putrequest(request.method,
url,
skip_accept_encoding=True)
for header, value in request.headers.items():
low_conn.putheader(header, value)
low_conn.endheaders()
is_first = True
for i in request.body:
low_conn.send((hex(len(i))[2:] + ";" +
self.get_random()).encode('utf-8'))
low_conn.send(b'\r\n')
#Python3适配，不确定会不会有问题┑(￣Д ￣)┍
if self.py_version == 2 or isinstance(i, bytes):
low_conn.send(i)
else:
low_conn.send(i.encode())
low_conn.send(b'\r\n')
low_conn.send(b'0\r\n\r\n')
# Receive the response from the server
try:
# For Python 2.7+ versions, use buffering of HTTP
# responses
r = low_conn.getresponse(buffering=True)
except TypeError:
# For compatibility with Python 2.6 versions and back
r = low_conn.getresponse()
resp = HTTPResponse.from_httplib(r,
pool=conn,
connection=low_conn,
preload_content=False,
decode_content=False)
except:
# If we hit any problems here, clean up the connection.
# Then, reraise so that we can handle the actual exception.
low_conn.close()
raise
except (ProtocolError, socket.error) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
raise SSLError(e, request=request)
raise ConnectionError(e, request=request)
except ClosedPoolError as e:
raise ConnectionError(e, request=request)
except _ProxyError as e:
raise ProxyError(e)
except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
raise ReadTimeout(e, request=request)
else:
raise
return self.build_response(request, resp)
def test_connection_error(self) -> None:
with self.mock_config_info(self.normal_config), \
patch('requests.get', side_effect=ConnectionError()), \
patch('logging.exception'):
self.verify_reply('Wow !', 'Uh-Oh, couldn\'t process the request '
'right now.\nPlease again later')
def request_callback(request):
raise ConnectionError('test exception')
import requests
from requests.exceptions import ConnectionError
import requests_mock
import bugzilla
class TestBugzilla(unittest.TestCase):
def test_bug_id(self):
zilla = MyBugzilla('*****@*****.**', server = 'http://yeah')
link = zilla.bug_link(23)
self.assertEqual(link, 'http://yeah/show_bug.cgi?id=23')
@requests_mock.mock()
def test_get_new_bugs(self, mocker):
# mocking the requests call and send back two bugs
bugs = [{'id': 1184528}, {'id': 1184524}]
mocker.get(requests_mock.ANY, json={'bugs': bugs})
zilla = MyBugzilla('*****@*****.**', server = 'http://yeah')
bugs = list(zilla.get_new_bugs())
self.assertEqual(bugs[0]['link'], 'http://yeah/show_bug.cgi?id=1184528')
@mock.patch.object(requests, 'get', side_effect=ConnectionError('No network'))
def test_network_error(self, mocked):
# faking a connection error in request if the web is down
zilla = MyBugzilla('*****@*****.**', server='http://yeah')
bugs = list(zilla.get_new_bugs())
self.assertEqual(len(bugs), 0)
if __name__ == '__main__':
unittest.main()
def connection_error():
x = Response()
x.status_code = 500
x.reason = err
raise ConnectionError(err, response=x)
def mock_call_connection_error(mock_call_response):
return partial(mock_call_response, body=ConnectionError("Connection error."))
def send_data(*args, **kwargs):
"""Simulate an error."""
raise ConnectionError("Mocked error")
def test_query_transport_error():
graph = GraphAPI('<access token>')
mock_request.side_effect = ConnectionError('Max retries exceeded with url: /')
assert_raises(GraphAPI.HTTPError, graph.get, 'me')
def download(self, url, args={}):
"""
Downloader Download By tools
:return: response object
"""
if isinstance(args, basestring):
args = json.loads(args)
tools = args.get('tools', 'requests')
method = args.get('method', 'GET')
if tools == "requests":
self.reqst = requests.Session()
self.headers = {
'Accept': 'text/html, application/xhtml+xml, */*',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language':
'en-US, en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
'User-Agent': self.get_user_agent()
}
kwargs = {
'headers': args.get('headers', self.headers),
'cookies': args.get('cookies', None),
'proxies': args.get('proxies', None),
'timeout': args.get('timeout', 30)
}
if str(method).upper() == 'GET':
kwargs['params'] = args.get('params', {})
elif str(method).upper() == 'POST':
kwargs['data'] = args.get('data', {})
# try:
resp = self.reqst.request(method=method, url=url, **kwargs)
if resp.status_code != 200:
raise ConnectionError("ConnectionError, {0}".format(
resp.status_code))
resp.doc = PyQuery(resp.content)
return resp
# except Exception:
# print traceback.format_exc()
# raise Exception
elif tools == 'phantomjs':
"""
Download by Phantomjs
"""
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap['phantomjs.page.settings.userAgent'] = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
)
# driver = webdriver.PhantomJS(desired_capabilities=dcap) # 指定使用的浏览器
# driver = webdriver.PhantomJS()
driver = webdriver.Chrome()
try:
print 'new get url: %s' % (url)
driver.get(url)
time.sleep(4)
js = args.get('js_code', "var q=document.body.scrollTop=10000")
driver.execute_script(js) # 可执行js，模仿用户操作。此处为将页面拉至最底端。
time.sleep(5)
body = driver.page_source.encode('utf-8')
print(u"访问" + url)
except Exception as e:
body = u'<html>有异常出现了</html>'.encode('utf-8')
print str(e)
traceback.print_exc()
finally:
current_url = driver.current_url
driver.close()
resp = Response()
# resp.status_code = 200
resp._content = body
resp.url = current_url
resp.doc = PyQuery(body)
return resp
def request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
json=None,
**kwargs):
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
:param data: (optional) Dictionary or bytes to send in the body of the
:class:`Request`.
:param json: (optional) json to send in the body of the
:class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the
:class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the
:class:`Request`.
:param files: (optional) Dictionary of ``'filename': file-like-objects``
for multipart encoding upload.
:param auth: (optional) Auth tuple or callable to enable
Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a (`connect timeout, read
timeout <user/advanced.html#timeouts>`_) tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Set to True by default.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol to the URL of
the proxy.
:param stream: (optional) whether to immediately download the response
content. Defaults to ``False``.
:param verify: (optional) if ``True``, the SSL cert will be verified.
A CA_BUNDLE path can also be provided.
:param cert: (optional) if String, path to ssl client cert file (.pem).
If Tuple, ('cert', 'key') pair.
"""
#===============================================================================================================
# add by mz
error_type = kwargs.get("error_type")
if error_type:
from requests.exceptions import InvalidURL, URLRequired, ConnectTimeout, ConnectionError, SSLError, ReadTimeout
from requests.exceptions import InvalidSchema, MissingSchema, ChunkedEncodingError, ContentDecodingError
from requests.exceptions import RequestException, HTTPError, ProxyError, Timeout, RetryError, StreamConsumedError
get_error = {
"InvalidURL": InvalidURL(),
"URLRequired": URLRequired(),
"ConnectTimeout": ConnectTimeout(),
"ConnectionError": ConnectionError(),
"SSLError": SSLError(),
"ReadTimeout": ReadTimeout(),
"InvalidSchema": InvalidSchema(),
"MissingSchema": MissingSchema(),
"ChunkedEncodingError": ChunkedEncodingError(),
"ContentDecodingError": ContentDecodingError(),
"StreamConsumedError": StreamConsumedError(),
"TooManyRedirects": TooManyRedirects(),
"RequestException": RequestException(),
"HTTPError": HTTPError(),
"ProxyError": ProxyError(),
"Timeout": Timeout(),
"RetryError": RetryError
}
error_ = get_error[error_type]
raise error_
#===============================================================================================================
method = to_native_string(method)
# Create the Request.
req = Request(
method=method.upper(),
url=url,
headers=headers,
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
return resp
def test_error_traceback(get_response):
exc = ConnectionError('Connection aborted')
exc.request = Request(method='GET', url='http://www.google.com')
get_response.side_effect = exc
with raises(ConnectionError):
ret = main(['--ignore-stdin', '--traceback', 'www.google.com'])
class TestForumTopicRest(BaseTestRest):
def _add_test_data(self):
self.waypoint = Waypoint(
waypoint_type='summit', elevation=2203)
self.locale_en = WaypointLocale(
lang='en', title='Mont Granier', description='...',
access='yep')
self.waypoint.locales.append(self.locale_en)
self.waypoint.geometry = DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)')
self.session.add(self.waypoint)
self.waypoint_with_topic = Waypoint(
waypoint_type='summit', elevation=2203)
document_topic = DocumentTopic(topic_id=1)
self.locale_en_with_topic = WaypointLocale(
lang='en', title='Mont Granier', description='...',
access='yep', document_topic=document_topic)
self.waypoint_with_topic.locales.append(self.locale_en_with_topic)
self.waypoint_with_topic.geometry = DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)')
self.session.add(self.waypoint_with_topic)
self.image = Image(
filename='image.jpg',
activities=['paragliding'], height=1500,
image_type='collaborative')
self.image_locale_en = DocumentLocale(
lang='en', title='', description='')
self.image.locales.append(self.image_locale_en)
self.image.geometry = DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)')
self.session.add(self.image)
self.outing = Outing(
activities=['skitouring'],
date_start=datetime.date(2016, 1, 1),
date_end=datetime.date(2016, 1, 1),
locales=[OutingLocale(lang='en',
title='Mont Granier / skitouring')]
)
self.session.add(self.outing)
self.session.flush()
for user_id in (
self.global_userids['contributor'],
self.global_userids['contributor2']
):
self.session.add(Association(
parent_document_id=user_id,
parent_document_type=USERPROFILE_TYPE,
child_document_id=self.outing.document_id,
child_document_type=OUTING_TYPE))
self.session.flush()
def setUp(self): # noqa
BaseTestRest.setUp(self)
self._add_test_data()
def test_post_document_not_exists(self):
json = self.post_json_with_contributor(
'/forum/topics',
{
'document_id': self.waypoint.document_id,
'lang': 'fr'
},
status=400)
errors = json.get('errors')
self.assertEqual('Document not found', errors[0].get('description'))
def test_post_topic_exists(self):
json = self.post_json_with_contributor(
'/forum/topics',
{
'document_id': self.waypoint_with_topic.document_id,
'lang': 'en'
},
status=400)
errors = json.get('errors')
self.assertEqual('Topic already exists', errors[0].get('description'))
self.assertEqual(1, errors[0].get('topic_id'))
@patch('pydiscourse.client.DiscourseClient.create_post',
side_effect=ConnectionError())
def test_post_discourse_down(self, create_post_mock):
self.post_json_with_contributor(
'/forum/topics',
{
'document_id': self.waypoint.document_id,
'lang': 'en'
},
status=500)
@patch('pydiscourse.client.DiscourseClient.create_post',
return_value={"topic_id": 10})
def test_post_success(self, create_post_mock):
version = self.locale_en.version
json = self.post_json_with_contributor(
'/forum/topics',
{
'document_id': self.waypoint.document_id,
'lang': 'en'
},
status=200)
self.session.expire(self.locale_en)
self.assertEqual(10, self.locale_en.topic_id)
self.assertEqual(version, self.locale_en.version)
self.assertEqual(10, json.get('topic_id'))
cache_version = self.session.query(CacheVersion).get(
self.waypoint.document_id)
self.assertEqual(cache_version.version, 2)
@patch('pydiscourse.client.DiscourseClient.create_post',
return_value={"topic_id": 10})
def test_post_without_title(self, create_post_mock):
"""Test topic link content for documents without title"""
locale = self.image_locale_en
referer = ('https://www.camptocamp.org/images/{}/{}'
.format(locale.document_id,
locale.lang))
self.post_json_with_contributor(
'/forum/topics',
{
'document_id': self.image.document_id,
'lang': 'en'
},
headers={
'Referer': referer
},
status=200)
create_post_mock.assert_called_with(
'<a href="{}">{}</a>'.format(
referer,
"/images/{}/{}".format(locale.document_id, locale.lang)),
title='{}_{}'.format(locale.document_id, locale.lang),
category='Commentaires')
@patch('pydiscourse.client.DiscourseClient._post',
side_effect=[{"topic_id": 10},
{},
{}])
def test_post_invite_participants(self, _post_mock):
"""Test outing participants are invited in the topic"""
self.post_json_with_contributor(
'/forum/topics',
{
'document_id': self.outing.document_id,
'lang': 'en'
},
status=200)
_post_mock.assert_has_calls([
call('/t/10/invite.json', user='******'),
call('/t/10/invite.json', user='******')])
class ImportContentTestCase(TestCase):
"""
Test case for the importcontent management command.
"""
fixtures = ["content_test.json"]
the_channel_id = "6199dde695db4ee4ab392222d5af1e5c"
def setUp(self):
LocalFile.objects.update(available=False)
@patch(
"kolibri.core.content.management.commands.importcontent.transfer.FileDownload"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
return_value=True,
)
def test_remote_cancel_immediately(
self,
is_cancelled_mock,
cancel_mock,
FileDownloadMock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
# Check behaviour if cancellation is called before any file download starts
FileDownloadMock.return_value.__iter__.return_value = [
"one", "two", "three"
]
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
call_command("importcontent", "network", self.the_channel_id)
is_cancelled_mock.assert_has_calls([call(), call()])
FileDownloadMock.assert_not_called()
cancel_mock.assert_called_with()
annotation_mock.mark_local_files_as_available.assert_not_called()
annotation_mock.set_leaf_node_availability_from_local_file_availability.assert_not_called(
)
annotation_mock.recurse_annotation_up_tree.assert_not_called()
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_remote_url"
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.transfer.FileDownload"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, True],
)
def test_remote_cancel_during_transfer(
self,
is_cancelled_mock,
cancel_mock,
FileDownloadMock,
local_path_mock,
remote_path_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
# If transfer is cancelled during transfer of first file
local_path = tempfile.mkstemp()[1]
local_path_mock.return_value = local_path
remote_path_mock.return_value = "notest"
# Mock this __iter__ so that the filetransfer can be looped over
FileDownloadMock.return_value.__iter__.side_effect = TransferCanceled()
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
call_command("importcontent", "network", self.the_channel_id)
# is_cancelled should be called thrice.
is_cancelled_mock.assert_has_calls([call(), call()])
# Should be set to the local path we mocked
FileDownloadMock.assert_called_with("notest",
local_path,
session=Any(Session),
cancel_check=is_cancelled_mock)
# Check that the command itself was also cancelled.
cancel_mock.assert_called_with()
annotation_mock.mark_local_files_as_available.assert_not_called()
annotation_mock.set_leaf_node_availability_from_local_file_availability.assert_not_called(
)
annotation_mock.recurse_annotation_up_tree.assert_not_called()
@patch(
"kolibri.core.content.management.commands.importcontent.compare_checksums",
return_value=True,
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_remote_url"
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.transfer.FileDownload"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, True, True],
)
def test_remote_cancel_after_file_copy_file_not_deleted(
self,
is_cancelled_mock,
cancel_mock,
FileDownloadMock,
local_path_mock,
remote_path_mock,
checksum_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
# If transfer is cancelled after transfer of first file
local_path_1 = tempfile.mkstemp()[1]
local_path_2 = tempfile.mkstemp()[1]
with open(local_path_1, "w") as f:
f.write("a")
local_path_mock.side_effect = [local_path_1, local_path_2]
remote_path_mock.return_value = "notest"
# Mock this __iter__ so that the filetransfer can be looped over
FileDownloadMock.return_value.__iter__.return_value = [
"one", "two", "three"
]
FileDownloadMock.return_value.total_size = 1
FileDownloadMock.return_value.dest = local_path_1
LocalFile.objects.update(file_size=1)
files_to_transfer_mock.return_value = (LocalFile.objects.all()[:3], 10)
call_command("importcontent", "network", self.the_channel_id)
# Check that the command itself was also cancelled.
cancel_mock.assert_called_with()
# Check that the temp file we created where the first file was being downloaded to has not been deleted
self.assertTrue(os.path.exists(local_path_1))
annotation_mock.set_content_visibility.assert_called()
@patch(
"kolibri.core.content.management.commands.importcontent.transfer.FileCopy"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
return_value=True,
)
def test_local_cancel_immediately(
self,
is_cancelled_mock,
cancel_mock,
FileCopyMock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
# Local version of test above
FileCopyMock.return_value.__iter__.return_value = [
"one", "two", "three"
]
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
call_command("importcontent", "disk", self.the_channel_id,
tempfile.mkdtemp())
is_cancelled_mock.assert_has_calls([call(), call()])
FileCopyMock.assert_not_called()
cancel_mock.assert_called_with()
annotation_mock.mark_local_files_as_available.assert_not_called()
annotation_mock.set_leaf_node_availability_from_local_file_availability.assert_not_called(
)
annotation_mock.recurse_annotation_up_tree.assert_not_called()
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.transfer.FileCopy"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, True],
)
def test_local_cancel_during_transfer(
self,
is_cancelled_mock,
cancel_mock,
FileCopyMock,
local_path_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
# Local version of test above
local_dest_path = tempfile.mkstemp()[1]
local_src_path = tempfile.mkstemp()[1]
local_path_mock.side_effect = [local_dest_path, local_src_path]
FileCopyMock.return_value.__iter__.side_effect = TransferCanceled()
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
call_command("importcontent", "disk", self.the_channel_id,
tempfile.mkdtemp())
is_cancelled_mock.assert_has_calls([call(), call()])
FileCopyMock.assert_called_with(local_src_path,
local_dest_path,
cancel_check=is_cancelled_mock)
cancel_mock.assert_called_with()
annotation_mock.set_content_visibility.assert_called()
@patch("kolibri.core.content.management.commands.importcontent.len")
@patch(
"kolibri.core.content.utils.transfer.Transfer.next",
side_effect=ConnectionError("connection error"),
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, True, True],
)
def test_remote_cancel_during_connect_error(
self,
is_cancelled_mock,
cancel_mock,
next_mock,
len_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
LocalFile.objects.filter(pk="6bdfea4a01830fdd4a585181c0b8068c").update(
file_size=2201062)
LocalFile.objects.filter(pk="211523265f53825b82f70ba19218a02e").update(
file_size=336974)
files_to_transfer_mock.return_value = (
LocalFile.objects.filter(pk__in=[
"6bdfea4a01830fdd4a585181c0b8068c",
"211523265f53825b82f70ba19218a02e",
]),
10,
)
call_command(
"importcontent",
"network",
self.the_channel_id,
node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"],
)
cancel_mock.assert_called_with()
len_mock.assert_not_called()
annotation_mock.set_content_visibility.assert_called()
@patch(
"kolibri.core.content.management.commands.importcontent.logger.warning"
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
def test_remote_import_httperror_404(
self,
path_mock,
logger_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
local_dest_path_1 = tempfile.mkstemp()[1]
local_dest_path_2 = tempfile.mkstemp()[1]
local_dest_path_3 = tempfile.mkstemp()[1]
path_mock.side_effect = [
local_dest_path_1,
local_dest_path_2,
local_dest_path_3,
]
ContentNode.objects.filter(
pk="2b6926ed22025518a8b9da91745b51d3").update(available=False)
LocalFile.objects.filter(
files__contentnode__pk="2b6926ed22025518a8b9da91745b51d3").update(
file_size=1, available=False)
files_to_transfer_mock.return_value = (
LocalFile.objects.filter(
files__contentnode__pk="2b6926ed22025518a8b9da91745b51d3"),
10,
)
node_id = ["2b6926ed22025518a8b9da91745b51d3"]
call_command(
"importcontent",
"network",
self.the_channel_id,
node_ids=node_id,
renderable_only=False,
)
logger_mock.assert_called_once()
self.assertTrue(
"3 files are skipped" in logger_mock.call_args_list[0][0][0])
annotation_mock.set_content_visibility.assert_called_with(
self.the_channel_id,
[],
node_ids=node_id,
exclude_node_ids=None,
public=False,
)
@patch("kolibri.core.content.utils.transfer.Transfer.next")
@patch("kolibri.core.content.utils.transfer.sleep")
@patch("kolibri.core.content.utils.transfer.requests.Session.get")
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, False, True, True, True],
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path",
return_value="test/test",
)
def test_remote_import_httperror_502(
self,
content_storage_file_path_mock,
is_cancelled_mock,
cancel_mock,
requests_get_mock,
sleep_mock,
transfer_next_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
response_mock = MagicMock()
response_mock.status_code = 502
exception_502 = HTTPError("Bad Gateway", response=response_mock)
requests_get_mock.return_value.raise_for_status.side_effect = exception_502
LocalFile.objects.filter(
files__contentnode__channel_id=self.the_channel_id).update(
file_size=1)
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
call_command("importcontent", "network", self.the_channel_id)
sleep_mock.assert_called_once()
transfer_next_mock.assert_not_called()
cancel_mock.assert_called_with()
annotation_mock.set_content_visibility.assert_called()
@patch("kolibri.core.content.utils.transfer.requests.Session.get")
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path",
return_value="test/test",
)
def test_remote_import_httperror_500(
self,
content_storage_file_path_mock,
requests_get_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
response_mock = MagicMock()
response_mock.status_code = 500
exception_500 = HTTPError("Internal Server Error",
response=response_mock)
requests_get_mock.return_value.raise_for_status.side_effect = exception_500
LocalFile.objects.filter(
files__contentnode__channel_id=self.the_channel_id).update(
file_size=1)
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
with self.assertRaises(HTTPError):
call_command("importcontent", "network", self.the_channel_id)
annotation_mock.set_content_visibility.assert_called_with(
self.the_channel_id, [],
node_ids=None,
exclude_node_ids=None,
public=False)
@patch("kolibri.core.content.management.commands.importcontent.len")
@patch("kolibri.core.content.utils.transfer.sleep")
@patch(
"kolibri.core.content.utils.transfer.Transfer.next",
side_effect=ChunkedEncodingError("Chunked Encoding Error"),
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, False, False, True, True],
)
def test_remote_import_chunkedencodingerror(
self,
is_cancelled_mock,
cancel_mock,
error_mock,
sleep_mock,
len_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
LocalFile.objects.filter(pk="6bdfea4a01830fdd4a585181c0b8068c").update(
file_size=2201062)
LocalFile.objects.filter(pk="211523265f53825b82f70ba19218a02e").update(
file_size=336974)
files_to_transfer_mock.return_value = (
LocalFile.objects.filter(pk__in=[
"6bdfea4a01830fdd4a585181c0b8068c",
"211523265f53825b82f70ba19218a02e",
]),
10,
)
call_command(
"importcontent",
"network",
self.the_channel_id,
node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"],
)
sleep_mock.assert_called_once()
cancel_mock.assert_called_with()
len_mock.assert_not_called()
annotation_mock.set_content_visibility.assert_called()
@patch(
"kolibri.core.content.management.commands.importcontent.logger.warning"
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, True, True],
)
def test_local_import_oserror_dne(
self,
is_cancelled_mock,
cancel_mock,
path_mock,
logger_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
dest_path = tempfile.mkstemp()[1]
path_mock.side_effect = [dest_path, "/test/dne"]
LocalFile.objects.filter(
files__contentnode__channel_id=self.the_channel_id).update(
file_size=1)
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
call_command("importcontent", "disk", self.the_channel_id,
"destination")
self.assertTrue(
"1 files are skipped" in logger_mock.call_args_list[0][0][0])
annotation_mock.set_content_visibility.assert_called()
@patch(
"kolibri.core.content.management.commands.importcontent.logger.error")
@patch("kolibri.core.content.utils.transfer.os.path.getsize")
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
def test_local_import_oserror_permission_denied(
self,
path_mock,
getsize_mock,
logger_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
dest_path = tempfile.mkstemp()[1]
path_mock.side_effect = [dest_path, "/test/dne"]
getsize_mock.side_effect = ["1", OSError("Permission denied")]
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
with self.assertRaises(OSError):
call_command("importcontent", "disk", self.the_channel_id,
"destination")
self.assertTrue(
"Permission denied" in logger_mock.call_args_list[0][0][0])
annotation_mock.set_content_visibility.assert_called()
@patch("kolibri.core.content.management.commands.importcontent.os.remove")
@patch(
"kolibri.core.content.management.commands.importcontent.os.path.isfile",
return_value=False,
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, False, True, True],
)
def test_local_import_source_corrupted(
self,
is_cancelled_mock,
cancel_mock,
path_mock,
isfile_mock,
remove_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
local_src_path = tempfile.mkstemp()[1]
local_dest_path = tempfile.mkstemp()[1]
LocalFile.objects.filter(
files__contentnode="32a941fb77c2576e8f6b294cde4c3b0c").update(
file_size=1)
path_mock.side_effect = [local_dest_path, local_src_path]
files_to_transfer_mock.return_value = (
LocalFile.objects.filter(
files__contentnode="32a941fb77c2576e8f6b294cde4c3b0c"),
10,
)
call_command(
"importcontent",
"disk",
self.the_channel_id,
"destination",
node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"],
)
cancel_mock.assert_called_with()
remove_mock.assert_called_with(local_dest_path)
@patch(
"kolibri.core.content.management.commands.importcontent.os.path.isfile",
return_value=False,
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
return_value=False,
)
def test_local_import_source_corrupted_full_progress(
self,
is_cancelled_mock,
cancel_mock,
path_mock,
isfile_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
"""
Ensure that when a file is imported that does not match the file size in the database
that the overall progress tracking for the content import process is properly updated
to reflect the size of the file in the database, not the file on disk.
This is important, as the total progress for the overall process is measured against
the total file size recorded in the database for all files, not for the the
transferred file size.
"""
local_src_path = tempfile.mkstemp()[1]
with open(local_src_path, "w") as f:
f.write("This is just a test")
src_file_size = os.path.getsize(local_src_path)
expected_file_size = 10000
local_dest_path = tempfile.mkstemp()[1]
os.remove(local_dest_path)
# Delete all but one file associated with ContentNode to reduce need for mocking
files = ContentNode.objects.get(
id="32a941fb77c2576e8f6b294cde4c3b0c").files.all()
first_file = files.first()
files.exclude(id=first_file.id).delete()
LocalFile.objects.filter(
files__contentnode="32a941fb77c2576e8f6b294cde4c3b0c").update(
file_size=expected_file_size)
files_to_transfer_mock.return_value = (
LocalFile.objects.filter(
files__contentnode="32a941fb77c2576e8f6b294cde4c3b0c"),
10,
)
path_mock.side_effect = [local_dest_path, local_src_path]
mock_overall_progress = MagicMock()
mock_file_progress = MagicMock()
with patch(
"kolibri.core.tasks.management.commands.base.ProgressTracker"
) as progress_mock:
progress_mock.return_value.__enter__.side_effect = [
mock_overall_progress,
mock_file_progress,
]
call_command(
"importcontent",
"disk",
self.the_channel_id,
"destination",
node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"],
)
mock_overall_progress.assert_called_with(expected_file_size -
src_file_size)
@patch(
"kolibri.core.content.management.commands.importcontent.transfer.FileDownload.finalize"
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
return_value=False,
)
def test_remote_import_source_corrupted(
self,
is_cancelled_mock,
path_mock,
finalize_dest_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
dest_path_1 = tempfile.mkstemp()[1]
dest_path_2 = tempfile.mkstemp()[1]
path_mock.side_effect = [dest_path_1, dest_path_2]
LocalFile.objects.filter(pk="6bdfea4a01830fdd4a585181c0b8068c").update(
file_size=2201062)
LocalFile.objects.filter(pk="211523265f53825b82f70ba19218a02e").update(
file_size=336974)
files_to_transfer_mock.return_value = (
LocalFile.objects.filter(pk__in=[
"6bdfea4a01830fdd4a585181c0b8068c",
"211523265f53825b82f70ba19218a02e",
]),
10,
)
call_command(
"importcontent",
"network",
self.the_channel_id,
node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"],
)
annotation_mock.set_content_visibility.assert_called_with(
self.the_channel_id,
[],
exclude_node_ids=None,
node_ids=["32a941fb77c2576e8f6b294cde4c3b0c"],
public=False,
)
@patch(
"kolibri.core.content.management.commands.importcontent.transfer.FileDownload.finalize"
)
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
return_value=False,
)
def test_remote_import_full_import(
self,
is_cancelled_mock,
path_mock,
finalize_dest_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
dest_path_1 = tempfile.mkstemp()[1]
dest_path_2 = tempfile.mkstemp()[1]
path_mock.side_effect = [dest_path_1, dest_path_2]
LocalFile.objects.filter(pk="6bdfea4a01830fdd4a585181c0b8068c").update(
file_size=2201062)
LocalFile.objects.filter(pk="211523265f53825b82f70ba19218a02e").update(
file_size=336974)
files_to_transfer_mock.return_value = (
LocalFile.objects.filter(pk__in=[
"6bdfea4a01830fdd4a585181c0b8068c",
"211523265f53825b82f70ba19218a02e",
]),
10,
)
call_command("importcontent", "network", self.the_channel_id)
annotation_mock.set_content_visibility.assert_called_with(
self.the_channel_id, [],
exclude_node_ids=None,
node_ids=None,
public=False)
@patch("kolibri.core.content.utils.transfer.sleep")
@patch("kolibri.core.content.utils.transfer.Transfer.next")
@patch("kolibri.core.content.utils.transfer.requests.Session.get")
@patch(
"kolibri.core.content.management.commands.importcontent.paths.get_content_storage_file_path",
return_value="test/test",
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.cancel"
)
@patch(
"kolibri.core.content.management.commands.importcontent.AsyncCommand.is_cancelled",
side_effect=[False, False, False, True, True, True],
)
def test_remote_import_file_compressed_on_gcs(
self,
is_cancelled_mock,
cancel_mock,
content_storage_file_path_mock,
requests_get_mock,
transfer_next_mock,
sleep_mock,
annotation_mock,
files_to_transfer_mock,
channel_list_status_mock,
):
response_mock = MagicMock()
response_mock.status_code = 503
exception_503 = HTTPError("Service Unavailable",
response=response_mock)
transfer_next_mock.side_effect = exception_503
requests_get_mock.return_value.headers = {
"X-Goog-Stored-Content-Length": "1"
}
LocalFile.objects.filter(
files__contentnode__channel_id=self.the_channel_id).update(
file_size=1)
files_to_transfer_mock.return_value = (LocalFile.objects.all(), 10)
m = mock_open()
with patch("kolibri.core.content.utils.transfer.open", m) as open_mock:
call_command("importcontent", "network", self.the_channel_id)
# Check if truncate() is called since byte-range file resuming is not supported
open_mock.assert_called_with("test/test.transfer", "wb")
open_mock.return_value.truncate.assert_called_once()
sleep_mock.assert_called_once()
annotation_mock.set_content_visibility.assert_called_with(
self.the_channel_id,
[],
node_ids=None,
exclude_node_ids=None,
public=False,
)
def find_solc_versions(
contracts: Dict[str, str],
install_needed: bool = False,
install_latest: bool = False,
silent: bool = True,
) -> Dict:
"""Analyzes contract pragmas and determines which solc version(s) to use.
Args:
contracts: a dictionary in the form of {'path': "source code"}
install_needed: if True, will install when no installed version matches
the contract pragma
install_latest: if True, will install when a newer version is available
than the installed one
silent: set to False to enable verbose reporting
Returns: dictionary of {'version': ['path', 'path', ..]}
"""
installed_versions = [
Version(i[1:]) for i in solcx.get_installed_solc_versions()
]
try:
available_versions = [
Version(i[1:]) for i in solcx.get_available_solc_versions()
]
except ConnectionError:
if not installed_versions:
raise ConnectionError(
"Solc not installed and cannot connect to GitHub")
available_versions = installed_versions
pragma_regex = re.compile(r"pragma +solidity([^;]*);")
pragma_specs: Dict = {}
to_install = set()
new_versions = set()
for path, source in contracts.items():
pragma_string = next(pragma_regex.finditer(source), None)
if pragma_string is None:
raise PragmaError(f"No version pragma in '{path}'")
pragma_specs[path] = NpmSpec(pragma_string.groups()[0])
version = pragma_specs[path].select(installed_versions)
if not version and not (install_needed or install_latest):
raise IncompatibleSolcVersion(
f"No installed solc version matching '{pragma_string[0]}' in '{path}'"
)
# if no installed version of solc matches the pragma, find the latest available version
latest = pragma_specs[path].select(available_versions)
if not version and not latest:
raise PragmaError(
f"No installable solc version matching '{pragma_string[0]}' in '{path}'"
)
if not version or (install_latest and latest > version):
to_install.add(latest)
elif latest > version:
new_versions.add(str(version))
# install new versions if needed
if to_install:
install_solc(*to_install)
installed_versions = [
Version(i[1:]) for i in solcx.get_installed_solc_versions()
]
elif new_versions and not silent:
print(
f"New compatible solc version{'s' if len(new_versions) > 1 else ''}"
f" available: {', '.join(new_versions)}")
# organize source paths by latest available solc version
compiler_versions: Dict = {}
for path, spec in pragma_specs.items():
version = spec.select(installed_versions)
compiler_versions.setdefault(str(version), []).append(path)
return compiler_versions
def link_blob_to_doc(self, *args, **kwargs):
"""Simulate a server error."""
raise ConnectionError("Mocked exception")
class RbacServiceTest(TestCase):
"""Test RbacService object."""
@patch("koku.rbac.requests.get", side_effect=mocked_requests_get_404_json)
def test_non_200_error_json(self, mock_get):
"""Test handling of request with non-200 response and json error."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
access = rbac._request_user_access(url, headers={})
self.assertEqual(access, [])
mock_get.assert_called()
@patch("koku.rbac.requests.get", side_effect=mocked_requests_get_500_text)
def test_500_error_json(self, mock_get):
"""Test handling of request with 500 response and json error."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
with self.assertRaises(RbacConnectionError):
rbac._request_user_access(url, headers={})
@patch("koku.rbac.requests.get", side_effect=mocked_requests_get_404_text)
def test_non_200_error_text(self, mock_get):
"""Test handling of request with non-200 response and non-json error."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
access = rbac._request_user_access(url, headers={})
self.assertEqual(access, [])
mock_get.assert_called()
@patch("koku.rbac.requests.get",
side_effect=mocked_requests_get_404_except)
def test_non_200_error_except(self, mock_get):
"""Test handling of request with non-200 response and non-json error."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
logging.disable(logging.NOTSET)
with self.assertLogs(logger="koku.rbac", level=logging.WARNING):
access = rbac._request_user_access(url, headers={})
self.assertEqual(access, [])
mock_get.assert_called()
@patch("koku.rbac.requests.get", side_effect=mocked_requests_get_200_text)
def test_200_text(self, mock_get):
"""Test handling of request with 200 response and non-json error."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
access = rbac._request_user_access(url, headers={})
self.assertEqual(access, [])
mock_get.assert_called()
@patch("koku.rbac.requests.get",
side_effect=mocked_requests_get_200_except)
def test_200_exception(self, mock_get):
"""Test handling of request with 200 response and raises a json error."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
access = rbac._request_user_access(url, headers={})
self.assertEqual(access, [])
mock_get.assert_called()
@patch("koku.rbac.requests.get",
side_effect=mocked_requests_get_200_no_next)
def test_200_all_results(self, mock_get):
"""Test handling of request with 200 response with no next link."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
access = rbac._request_user_access(url, headers={})
self.assertEqual(access, [LIMITED_AWS_ACCESS])
mock_get.assert_called()
@patch("koku.rbac.requests.get", side_effect=mocked_requests_get_200_next)
def test_200_results_next(self, mock_get):
"""Test handling of request with 200 response with next link."""
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
access = rbac._request_user_access(url, headers={})
self.assertEqual(access, [LIMITED_AWS_ACCESS, LIMITED_AWS_ACCESS])
mock_get.assert_called()
@patch("koku.rbac.requests.get",
side_effect=ConnectionError("test exception"))
def test_get_except(self, mock_get):
"""Test handling of request with ConnectionError."""
before = REGISTRY.get_sample_value("rbac_connection_errors_total")
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
with self.assertRaises(RbacConnectionError):
rbac._request_user_access(url, headers={})
after = REGISTRY.get_sample_value("rbac_connection_errors_total")
self.assertEqual(1, after - before)
mock_get.assert_called()
def test_process_acls_bad_permission(self):
"""Test function of _process_acls with a bad permission format."""
acls = [{"permission": "bad_permission"}]
access = _process_acls(acls)
self.assertIsNone(access)
def test_process_acls_multiple(self):
"""Test function of _process_acls with a bad permission format."""
acls = [
{
"permission": "cost-management:provider:read",
"resourceDefinitions": []
},
{
"permission":
"cost-management:provider:write",
"resourceDefinitions": [{
"attributeFilter": {
"key": "cost-management.provider",
"operation": "in",
"value": "1,3,5"
}
}],
},
{
"permission":
"cost-management:provider:write",
"resourceDefinitions": [{
"attributeFilter": {
"key": "cost-management.provider",
"operation": "equal",
"value": "8"
}
}],
},
]
access = _process_acls(acls)
expected = {
"provider": [
{
"operation": "read",
"resources": ["*"]
},
{
"operation": "write",
"resources": ["1", "3", "5"]
},
{
"operation": "write",
"resources": ["8"]
},
]
}
print()
print(access)
self.assertEqual(access, expected)
def test_get_operation_invalid_res_type(self):
"""Test invalid resource type."""
with self.assertRaises(ValueError):
_get_operation({"operation": "*"}, "invalid")
@patch("koku.rbac._get_operation", side_effect=mocked_get_operation)
def test_apply_access_except(self, mock_get_operation):
"""Test handling exception _get_operation used in apply access method."""
processed_acls = {"*": [{"operation": "*", "resources": ["1", "3"]}]}
res_access = _apply_access(processed_acls)
rw_access = {"write": [], "read": []}
read_access = {"read": []}
expected = {
"provider": rw_access,
"rate": rw_access,
"aws.account": read_access,
"aws.organizational_unit": read_access,
"azure.subscription_guid": read_access,
"openshift.cluster": read_access,
"openshift.node": read_access,
"openshift.project": read_access,
}
self.assertEqual(res_access, expected)
mock_get_operation.assert_called()
def test_apply_access_none(self):
"""Test handling none input for apply access method."""
res_access = _apply_access(None)
rw_access = {"write": [], "read": []}
read_access = {"read": []}
expected = {
"provider": rw_access,
"rate": rw_access,
"aws.account": read_access,
"aws.organizational_unit": read_access,
"azure.subscription_guid": read_access,
"openshift.cluster": read_access,
"openshift.node": read_access,
"openshift.project": read_access,
}
self.assertEqual(res_access, expected)
def test_apply_access_all_wildcard(self):
"""Test handling of wildcard data for apply access method."""
processed_acls = {"*": [{"operation": "*", "resources": ["1", "3"]}]}
res_access = _apply_access(processed_acls)
rw_access = {"write": ["1", "3"], "read": ["1", "3"]}
read_access = {"read": ["1", "3"]}
expected = {
"provider": rw_access,
"rate": rw_access,
"aws.account": read_access,
"aws.organizational_unit": read_access,
"azure.subscription_guid": read_access,
"openshift.cluster": read_access,
"openshift.node": read_access,
"openshift.project": read_access,
}
self.assertEqual(res_access, expected)
def test_apply_access_wildcard(self):
"""Test handling of wildcard data for apply access method."""
processed_acls = {
"*": [{
"operation": "write",
"resources": ["1", "3"]
}, {
"operation": "read",
"resources": ["2"]
}]
}
res_access = _apply_access(processed_acls)
rw_access = {"write": ["1", "3"], "read": ["1", "3", "2"]}
read_access = {"read": ["2"]}
expected = {
"provider": rw_access,
"rate": rw_access,
"aws.account": read_access,
"aws.organizational_unit": read_access,
"azure.subscription_guid": read_access,
"openshift.cluster": read_access,
"openshift.node": read_access,
"openshift.project": read_access,
}
self.assertEqual(res_access, expected)
def test_apply_access_limited(self):
"""Test handling of limited resource access data for apply access method."""
processed_acls = {
"provider": [{
"operation": "write",
"resources": ["1", "3"]
}, {
"operation": "read",
"resources": ["2"]
}]
}
res_access = _apply_access(processed_acls)
op_access = {"write": ["1", "3"], "read": ["1", "3", "2"]}
no_rw_access = {"write": [], "read": []}
no_access = {"read": []}
expected = {
"provider": op_access,
"rate": no_rw_access,
"aws.account": no_access,
"aws.organizational_unit": no_access,
"azure.subscription_guid": no_access,
"openshift.cluster": no_access,
"openshift.node": no_access,
"openshift.project": no_access,
}
self.assertEqual(res_access, expected)
def test_apply_case(self):
"""Test apply with mixed condition."""
processed_acls = {
"provider": [{
"operation": "*",
"resources": ["*"]
}],
"rate": [{
"operation": "*",
"resources": ["*"]
}],
"aws.account": [{
"operation": "read",
"resources": ["myaccount"]
}],
}
res_access = _apply_access(processed_acls)
op_access = {"read": ["myaccount"]}
rw_access = {"write": ["*"], "read": ["*"]}
no_access = {"read": []}
expected = {
"provider": rw_access,
"rate": rw_access,
"aws.account": op_access,
"aws.organizational_unit": no_access,
"azure.subscription_guid": no_access,
"openshift.cluster": no_access,
"openshift.node": no_access,
"openshift.project": no_access,
}
self.assertEqual(res_access, expected)
@patch("koku.rbac.requests.get",
side_effect=mocked_requests_get_200_except)
def test_get_access_for_user_none(self, mock_get):
"""Test handling of user request where no access returns None."""
rbac = RbacService()
mock_user = Mock()
mock_user.identity_header = {"encoded": "dGVzdCBoZWFkZXIgZGF0YQ=="}
access = rbac.get_access_for_user(mock_user)
self.assertIsNone(access)
mock_get.assert_called()
@patch("koku.rbac.requests.get",
side_effect=mocked_requests_get_200_no_next)
def test_get_access_for_user_data_limited(self, mock_get):
"""Test handling of user request where access returns data."""
rbac = RbacService()
mock_user = Mock()
mock_user.identity_header = {"encoded": "dGVzdCBoZWFkZXIgZGF0YQ=="}
access = rbac.get_access_for_user(mock_user)
expected = {
"provider": {
"write": [],
"read": []
},
"rate": {
"write": [],
"read": []
},
"aws.account": {
"read": ["123456"]
},
"aws.organizational_unit": {
"read": []
},
"azure.subscription_guid": {
"read": []
},
"openshift.cluster": {
"read": []
},
"openshift.node": {
"read": []
},
"openshift.project": {
"read": []
},
}
self.assertEqual(access, expected)
mock_get.assert_called()
@patch.dict(os.environ, {"RBAC_CACHE_TTL": "5"})
def test_get_cache_ttl(self):
"""Test to get the cache ttl value."""
rbac = RbacService()
self.assertEqual(rbac.get_cache_ttl(), 5)
}, {
'token': 'E80C89308D7A8B142861ADC4875D3D88'
}]
}
responses.replace(responses.GET, url, status=200, json=data)
data = api.export_push_tokens('token')
assert len(data) == 2
assert data[0]['token'] == 'D7BB4C4F8B3CF81488DEAAC8ABC1B955'
@pytest.mark.parametrize('params', [
{
'status': 500
},
{
'body': ConnectionError()
},
])
@responses.activate
def test_get_push_tokens_error(params, api):
url = urljoin(ExportAPI.base_url, 'push_tokens.json')
responses.add(responses.GET, url, **params)
with pytest.raises(AppMetricaExportPushTokenError):
api.export_push_tokens('token')
@responses.activate
def test_get_installations_success(api):
url = urljoin(ExportAPI.base_url, 'installations.json')
responses.add(responses.GET, url, status=202, body='')
def test_connection_error_when_validate_config(self) -> None:
error = ConnectionError()
with patch('requests.get', side_effect=ConnectionError()):
self.validate_invalid_config({'key': '12345678'}, str(error))
def test_udpipe_offline(self):
"""Test if UDPipe works offline"""
self.assertTrue(UDPipeModels().online)
with mock.patch('serverfiles.ServerFiles.listfiles',
**{'side_effect': ConnectionError()}):
self.assertFalse(UDPipeModels().online)
def get(address, headers=None, timeout=None):
match = re.match(regex, address)
if not match or match.groups()[0] == '4380':
return mock.DEFAULT
else:
raise ConnectionError()
