def test_download_invalid_spdx(
fake_repository, stringio, mock_put_license_in_file
):
"""An invalid SPDX identifier was provided."""
# pylint: disable=unused-argument
mock_put_license_in_file.side_effect = requests.RequestException()
result = main(["download", "does-not-exist"], out=stringio)
assert result == 1
assert "not a valid SPDX License Identifier" in stringio.getvalue()
def test_download_request_exception(
fake_repository, stringio, mock_put_license_in_file
):
"""There was an error while downloading the license file."""
# pylint: disable=unused-argument
mock_put_license_in_file.side_effect = requests.RequestException()
result = main(["download", "0BSD"], out=stringio)
assert result == 1
assert "internet" in stringio.getvalue()
def download_ext(extid='', filepath=''):
url = ext_download_url_base.format(id=extid)
res = requests.get(url, verify=False, stream=True,\
allow_redirects=True, timeout=10, headers=conf['HTTP_HEADERS'])
if res.status_code != 200:
raise requests.RequestException(u"Status code error: {}".format(res.status_code))
with io.open(filepath, 'wb') as f:
for chunk in res.iter_content(chunk_size=512 * 1024):
if chunk:
f.write(chunk)
return True
def request(self, method_name, *args, **kwargs):
# start_time = time.time()
self.websocket.send(str(Request(method_name, *args, **kwargs)))
response = self.websocket.recv()
# total_time = "%.3f" % (time.time() - start_time)
# print("[{}] http request - {} kb / {} sec - {} {}".format(datetime.datetime.now(), sys.getsizeof(str(response)) / 1000, total_time, method_name, list(args)))
if 'error' in response:
print("response error")
print(response)
raise requests.RequestException()
return json.loads(response)['result']
def test_get_receivers_list_request_exception(self):
class WF(object):
source = GitSource('git', 'foo', provider_params={'git_commit': 'foo'})
p = SendMailPlugin(None, WF())
flexmock(p).should_receive('_get_component_label').and_return('foo')
flexmock(p).should_receive('_get_pdc_token').and_return('foo')
flexmock(requests).should_receive('get').and_raise(requests.RequestException('foo'))
with pytest.raises(RuntimeError) as e:
p._get_receivers_list()
assert str(e.value) == 'foo'
def test_request_error(self, data, config, collectd, ClientV3, perf_req):
"""Test error raised by underlying requests module"""
# tell POST request to raise an exception
perf_req.side_effect = requests.RequestException('Test POST exception')
# ieit instance
instance = plugin.Plugin(collectd=collectd, config=config)
# write the value
self.assertRaises(requests.RequestException, instance.write, data)
def get_places(**kwargs):
url = "https://api.foursquare.com/v2/venues/search"
kwargs['client_id'] = '3BDM4BGF14QYWNQ5FOHULYUEIWQBTCYI5JTK53EL12R1ZYRW'
kwargs[
'client_secret'] = 'FSMFOMCF40UHZVHZLUP42AUP4ECZORNOBEFLZKR0YVUJ0D1W'
kwargs['categoryId'] = '4d4b7105d754a06374d81259'
kwargs['limit'] = 50
r = requests.get(url, kwargs)
if not r.ok:
raise requests.RequestException(r.text)
return r.json()
def get_arquivo(self, url, remove_CR=True):
# Pega o arquivo da CBLC
request = requests.get(url)
if (request.status_code != 200):
raise requests.RequestException('Não encontrou a página "{}". Status da requisição: "{}"'.format(url, str(request.status_code)))
arquivo = request.text.split('\n')
if remove_CR:
arquivo.pop(len(arquivo) - 1) # Remove CR
return arquivo
def test_request_critical_css_raises(self, requests_get_mock, settings):
settings.PENTHOUSE_URL = 'http://foobar:3000/'
settings.SESSION_COOKIE_SECURE = False
site = Site.objects.create(domain='test.com')
settings.SITE_ID = site.id
requests_get_mock.side_effect = requests.RequestException()
api = PenthouseApi()
with pytest.raises(PenthouseException):
api.request_critical_css('/foo', 'css/test.css')
def take_snapshot(
filename,
omega_url = f"http://{CAM_URL}/?action=snapshot"
):
r = requests.get(omega_url, stream=True)
if r.status_code == 200:
with open(filename, 'wb') as f:
for chunk in r.iter_content():
f.write(chunk)
else:
raise requests.RequestException()
def test_load_tasks_error_request(self, mc_get, init_jg):
mc_get.side_effect = requests.RequestException()
self.jg.route_build_task = MagicMock()
self.jg.event = MagicMock()
self.jg.process_action = MagicMock()
assert self.jg.load_tasks() is None
assert not self.jg.route_build_task.called
assert not self.jg.process_action.called
def ctfpad_set_description(self, challenge_id, description):
try:
self.etherpad_session.get(f'{self.etherpad_url}/api/1/setText',
params={
'padID': challenge_id,
'text': description
})
except:
raise requests.RequestException(
f'Could not connect to Etherpad-Lite on {self.etherpad_url}. Please verify URL and token.'
)
def health(self, sandbox_mode):
response = self.session.get(url=self.health_url,
headers={'sandbox-mode': sandbox_mode},
timeout=self.timeout)
if response.status_code == 200:
return response
raise requests.RequestException('{} {} {}'.format(
response.url, response.status_code, response.reason),
response=response)
def test_auth_connection_error(self, requests_post):
requests_post.side_effect = requests.RequestException()
api = TaigaAPI(host="host")
self.assertRaises(
taiga.exceptions.TaigaRestException,
api.auth_app,
"valid-app-id",
"valid-app-pass",
"valid-auth-code",
"valid-state",
)
def config(self):
if self.token is None:
raise ValueError("config property requires token authorization")
response = requests.get(self.base_url + "/1/config",
headers=self.headers)
if response.status_code != 200:
raise requests.RequestException("Error. Status Code {}".format(
response.status_code))
else:
self._config = response.json()
return self._config
def _call_search_students_recursively(self, sap_search_student_url, all_inactive_learners, page_size, start_at):
"""
Make recursive GET calls to traverse the paginated API response for search students.
"""
search_student_paginated_url = '{sap_search_student_url}&{pagination_criterion}'.format(
sap_search_student_url=sap_search_student_url,
pagination_criterion='$count=true&$top={page_size}&$skip={start_at}'.format(
page_size=page_size,
start_at=start_at,
),
)
try:
response = self.session.get(search_student_paginated_url)
sap_inactive_learners = response.json()
except ValueError:
raise requests.RequestException(response=response)
except (ConnectionError, Timeout):
LOGGER.warning(
'Unable to fetch inactive learners from SAP searchStudent API with url '
'"{%s}".', search_student_paginated_url,
)
return None
if 'error' in sap_inactive_learners:
LOGGER.warning(
'SAP searchStudent API for customer %s and base url %s returned response with '
'error message "%s" and with error code "%s".',
self.enterprise_configuration.enterprise_customer.name,
self.enterprise_configuration.sapsf_base_url,
sap_inactive_learners['error'].get('message'),
sap_inactive_learners['error'].get('code'),
)
return None
new_page_start_at = page_size + start_at
total_inactive_learners = sap_inactive_learners['@odata.count']
inactive_learners_on_page = sap_inactive_learners['value']
LOGGER.info(
'SAP SF searchStudent API returned [%d] inactive learners of total [%d] starting from [%d] for '
'enterprise customer [%s]',
len(inactive_learners_on_page), total_inactive_learners, start_at,
self.enterprise_configuration.enterprise_customer.name
)
all_inactive_learners += inactive_learners_on_page
if total_inactive_learners > new_page_start_at:
return self._call_search_students_recursively(
sap_search_student_url,
all_inactive_learners,
page_size=page_size,
start_at=new_page_start_at,
)
return all_inactive_learners
def send(self, request, **kwargs):
# start_time = time.time()
response = super(Client, self).send(request, **kwargs)
# total_time = "%.3f" % (time.time() - start_time)
# print("[{}] http request - {} kb / {} sec - {} {}".format(datetime.datetime.now(), sys.getsizeof(str(response)) / 1000, total_time, request, list(kwargs)))
# print(response)
if 'error' in response:
print("response error")
print(response)
raise requests.RequestException()
return response
def check_health(self, channel):
headers = self.get_api_headers(channel)
try:
response = requests.get(channel.config[Channel.CONFIG_BASE_URL] + "/v1/health", headers=headers)
except Exception as ex:
raise Exception(f"Could not establish a connection with the WhatsApp server: {ex}")
if response.status_code >= 400:
raise requests.RequestException(f"Error checking API health: {response.content}", response=response)
return response
def post(self, url, payload=None or {}):
if payload is None:
raise ValueError('Payload cannot be empty for post')
response = requests.post(url=url, data=payload)
if not response.ok:
print('Response from server: ', response.text)
raise requests.RequestException(
f'Something went wrong! {response.status_code}')
print(response.url)
data = response.json()
return data['data']
def definitions_from_clearlydefined(dependency: Dependency) -> Dict:
"""
:raises requests.RequestException: if the request could not be made.
"""
url = clearlydefined_url(dependency)
response = requests.get(url)
if response.status_code == 200:
return json.loads(response.text)
raise requests.RequestException(
f"Status code of '{url}' was {response.status_code}"
)
def load_data_from_api(api, headers = headers):
try:
resp = requests.get('https://restapi.tu.ac.th/tu_covid_api/v1/master/{}/getdata'.format(api), headers=headers)
if resp.status_code != 200:
# This means something went wrong.
raise requests.RequestException('GET /{}/ {}'.format(api, resp.status_code))
print('GET /{}/ {} OK'.format(api, resp.status_code))
return resp.json()
except Exception as e:
print(e)
return None
def test_unknown_error(self, request):
e = requests.RequestException('error')
e.response = mock.MagicMock()
e.response.status_code = 404
e.response.content = 'Error Code'
request().raise_for_status.side_effect = e
self.assertRaises(
SoftLayer.TransportError,
transports.make_rest_api_call,
'GET',
'http://something.com/path/to/resource.txt')
def get_oauth_access_token(url, client_id, client_secret, token_type='jwt', grant_type='client_credentials',
refresh_token=None,
timeout=(REQUEST_CONNECT_TIMEOUT, REQUEST_READ_TIMEOUT)):
""" Retrieves OAuth 2.0 access token using the given grant type.
Args:
url (str): Oauth2 access token endpoint, optionally including part of the path.
client_id (str): client ID
client_secret (str): client secret
Kwargs:
token_type (str): Type of token to return. Options include bearer and jwt.
grant_type (str): One of 'client_credentials' or 'refresh_token'
refresh_token (str): The previous access token (for grant_type=refresh_token)
Raises:
requests.RequestException if there is a problem retrieving the access token.
Returns:
tuple: Tuple containing (access token string, expiration datetime).
"""
now = datetime.datetime.utcnow()
data = {
'grant_type': grant_type,
'client_id': client_id,
'client_secret': client_secret,
'token_type': token_type,
}
if refresh_token:
data['refresh_token'] = refresh_token
else:
assert grant_type != 'refresh_token', "refresh_token parameter required"
response = requests.post(
_get_oauth_url(url),
data=data,
headers={
'User-Agent': USER_AGENT,
},
timeout=timeout
)
response.raise_for_status() # Raise an exception for bad status codes.
try:
data = response.json()
access_token = data['access_token']
expires_in = data['expires_in']
except (KeyError, json.decoder.JSONDecodeError):
raise requests.RequestException(response=response)
expires_at = now + datetime.timedelta(seconds=expires_in)
return access_token, expires_at
def fetch_data(data_l):
"""
Expects a iterable from make_data. Yields requests.
"""
if isinstance(data_l, str):
data_l = [data_l]
for data in data_l:
r = requests.get(URL, data=data, headers=HEADERS)
if r.status_code == 200:
yield r
else:
raise requests.RequestException(r)
def test_unknown_error(self, request):
e = requests.RequestException('error')
e.response = mock.MagicMock()
e.response.status_code = 404
e.response.content = 'Error Code'
request().raise_for_status.side_effect = e
req = transports.Request()
req.service = 'SoftLayer_Service'
req.method = 'getObject'
self.assertRaises(SoftLayer.TransportError, self.transport, req)
def requests_get_mock(url):
if url == 'https://github.com/my-project/uploads/image.png':
response_spec = create_autospec(requests.Response)
response = response_spec()
response.ok = True
response.headers = {
'Content-Disposition': 'filename="original-image.png"',
'Content-Type': 'image/png',
}
response.content = bytes('fake png', encoding='utf-8')
return response
raise requests.RequestException(f'Unhandled request to {url}')
def test_collector_should_fail_with_bad_repository_error(
self, requests_get_mock):
import requests
requests_get_mock.side_effect = requests.RequestException()
self.allowed_error_count = 1 # allow only single and final Error to be raised
self.run_bot(
parameters=SHOULD_FAIL_BECAUSE_REPOSITORY_IS_NOT_VALID_CONFIG[
'CONFIG'],
prepare=True)
self.assertRegexpMatchesLog(pattern=".*Unknown repository.*"
) # assert the expected ValueError msg
def test_get_pfs_info_error():
# test JSONDecodeError with correct data but formatted as a string
incorrect_json_info_data = {
"price_info": 5,
"network_info": {
"chain_id": 42,
"registry_address": pfs_test_default_registry_address
},
"version": "0.0.3",
"operator": "John Doe",
"message": "This is your favorite pathfinding service",
"payment_address": pfs_test_default_payment_address,
}
response = Mock()
response.configure_mock(status_code=200,
content=str(incorrect_json_info_data))
with patch.object(requests, "get", return_value=response):
with pytest.raises(ServiceRequestFailed) as error:
get_pfs_info("url")
assert "Expecting property name enclosed in double quotes:" in str(
error.value)
# test RequestException
with patch.object(requests, "get",
side_effect=requests.RequestException()):
with pytest.raises(ServiceRequestFailed) as error:
get_pfs_info("url")
# test KeyError with missing key 'price_info' and formatted as json
incorrect_info_data = {
"network_info": {
"chain_id": 42,
"registry_address": pfs_test_default_registry_address
},
"version": "0.0.3",
"operator": "John Doe",
"message": "This is your favorite pathfinding service",
"payment_address": pfs_test_default_payment_address,
}
response.configure_mock(status_code=200,
content=json.dumps(incorrect_info_data))
with patch.object(requests, "get", return_value=response):
with pytest.raises(ServiceRequestFailed) as error:
get_pfs_info("url")
assert "'price_info'" in str(error.value)
def export_terms(self,
guids,
csv_path,
glossary_name="Glossary",
glossary_guid=None):
"""
:param list(str) guids: List of guids that should be exported as csv.
:param str csv_path: Path to CSV that will be imported.
:param str glossary_name:
Name of the glossary. Defaults to 'Glossary'. Not used if
glossary_guid is provided.
:param str glossary_guid:
Guid of the glossary, optional if glossary_name is provided.
Otherwise, this parameter takes priority over glossary_name.
Providing glossary_guid is also faster as you avoid a lookup based
on glossary_name.
:return: A csv file is written to the csv_path.
:rtype: None
"""
if glossary_guid:
# Glossary guid is defined so we don't need to look up the guid
pass
elif glossary_name:
glossary = self.get_glossary(glossary_name)
glossary_guid = glossary["guid"]
else:
raise ValueError(
"Either glossary_name or glossary_guid must be defined.")
results = None
atlas_endpoint = self.endpoint_url + \
f"/glossary/{glossary_guid}/terms/export"
postResp = requests.post(
atlas_endpoint,
json=guids,
headers=self.authentication.get_authentication_headers())
# Can't use handle response since it expects json
try:
postResp.raise_for_status()
except requests.RequestException as e:
if "errorCode" in postResp:
raise AtlasException(postResp.text)
else:
raise requests.RequestException(postResp.text)
with open(csv_path, 'wb') as fp:
fp.write(postResp.content)
return None
def download_ka_dubbed_video_csv(download_url=None,
is_khan_csv=False,
cache_filepath=None):
"""
Function to do the heavy lifting in getting the dubbed videos map.
Could be moved into utils
"""
# Get the redirect url
if not download_url:
csv_url = "http://learningequality.org/r/translationmapping"
if is_khan_csv:
csv_url = "http://www.khanacademy.org/r/translationmapping"
logging.info("Getting spreadsheet location from (%s)" % csv_url)
try:
download_url = urllib.request.urlopen(csv_url).geturl()
if "docs.google.com" not in download_url:
logging.warn(
"Redirect location no longer in Google docs (%s)" %
download_url)
else:
download_url = download_url.replace("/edit",
"/export?format=csv")
except:
# TODO: have django email admins when we hit this exception
raise Exception("Expected redirect response from (%s)" % csv_url)
logging.info("Downloading dubbed video data from %s" % download_url)
data = requests.get(download_url)
attempts = 1
while data.status_code != 200 and attempts <= 100:
time.sleep(30)
data = requests.get(download_url)
attempts += 1
if data.status_code != 200:
raise requests.RequestException(
"Failed to download dubbed video CSV data: %s" % data.content)
csv_data = data.content
# Dump the data to a local cache file
csv_data = csv_data.decode("utf-8")
try:
ensure_dir(os.path.dirname(cache_filepath))
with open(cache_filepath, "w") as fp:
fp.write(csv_data)
except Exception as e:
logging.error(
"Failed to make a local cache of the CSV data: %s; parsing local data"
% e)
return csv_data