def fetch_bgp(
use_metadata: bool = True,
headers: Dict[str, str] = {},
instance: Optional[str] = None,
) -> Any:
url = "https://metadata.packet.net/metadata"
ip_addresses = []
if not use_metadata:
if not instance:
raise ValueError(
"Instance ID must be specified when not using metadata")
url = "https://api.packet.net/devices/{}/bgp/neighbors".format(
instance)
ip_addresses = fetch_ip_addresses(headers=headers, instance=instance)
response = requests.get(url, headers=headers)
try:
response_payload = response.json()
if not use_metadata:
response_payload["network"] = {"addresses": ip_addresses}
return response_payload
except JSONDecodeError as e:
raise JSONDecodeError(
"Unable to decode API/metadata response for {}. {}".format(
url, e.msg),
e.doc,
e.pos,
)
def test_parse_negative(self, reader, requests, mocker):
configs = {
'cookies': {},
'headers': {},
'parameters': {},
'value': "avascan",
'excludes': ["/admin"],
'domain': "example.com"
}
har = {'log': {'entries': [{'request': requests[0]}]}}
# missing har request
mocker.patch("builtins.open")
mocker.patch("json.load", return_value=har)
del har['log']['entries'][0]['request']
with pytest.raises(InvalidFormatException):
reader.parse(configs)
# invalid har log
mocker.patch("json.load", return_value={'log': {'creator': {}}})
with pytest.raises(InvalidFormatException):
reader.parse(configs)
# invalid json
mocker.patch("json.load", side_effect=JSONDecodeError("", "", 0))
with pytest.raises(InvalidFormatException):
reader.parse(configs)
def decode(self, string):
# replacing the mongodb objects in the extended json to the strict json version
# try:
string = re.sub(r'ObjectId\(\"(\S+)\"\)', r'"\1"', string)
string = re.sub(r'NumberInt\((\S+)\)', r'\1', string)
string = re.sub(r'NumberLong\((\S+)\)', r'\1', string)
string = re.sub(r'NumberDecimal\((\S+)\)', r'\1', string)
string = re.sub(r'Date\((\S+)\)', r'{"date":"\1"}', string)
string = re.sub(r'Timestamp\((\S+)[,][ ]*(\S+)\)',
r'{"timestamp":{"t":"\1","i":"\2"}}', string)
string = re.sub(r'BinData\((\S+)[,][ ]*[\"](\S+)[\"]\)',
r'{"binary":"\1","type":"\2"}', string)
string = re.sub(r'DBRef\([\"](\S+)[\"][,][ ]*[\"](\S+)[\"]\)',
r'{"ref":"\1","id":"\2"}', string)
# string = re.sub(r':[ ]*undefined(\}|,)]', r'{"undefined":true}', string)
# string = re.sub(r':[ ]*MaxKey(\}|,)]', r'{"maxkey":1}', string)
# string = re.sub(r':[ ]*MinKey(\}|,)]', r'{"minkey":1}', string)
# string = re.sub(r':[ ]*//\S+//\S+(\}|,)]', r'{"regex":"\1", "options":"\2"}', string)
# string = re.sub(r':[ ]*//\S+//\S+(\}|,)]', r'{"regex":"\1", "options":"\2"}', string)
# except Exception as err:
# msg = "Could not parse the Extended Json."
# raise JSONDecodeError(msg, err)
# loading the json using the traditional json library
try:
data = json.loads(string, object_hook=self.object_hook)
except JSONDecodeError as err:
raise JSONDecodeError(err.msg)
return data
def loadCreds(self):
try:
with open(self.__credPath) as loadedCredFile:
credDict = json.load(loadedCredFile)
if "username" in credDict and "password" in credDict:
return credDict
else:
raise JSONDecodeError(
"JSON file does not contain username and password")
except Exception as e:
exceptionType = type(e).__name__
print(exceptionType)
if exceptionType == "FileNotFoundError":
print(
"\n####################\n CRED FILE NOT FOUND. \n YOU MUST IMPORT A VALID CRED FILE BY IMPORTING SOFTSERVE AND USING \n THE COMMAND softserve.importCreds()\n\n"
)
raise
if exceptionType == "JSONDecodeError":
print(
"\n####################\n CRED FILE EXIST BUT IS CORRUPTED. \n YOU MUST IMPORT A NEW CRED FILE BY IMPORTING SOFTSERVE AND USING \n THE COMMAND softserve.importCreds()\n\n"
)
raise
def test_bad_json(self):
with patch("openlibrary.core.fulltext.requests.get") as mock_get:
config.plugin_inside = {"search_endpoint": "mock"}
mock_response = Mock(json=Mock(side_effect=JSONDecodeError('Not JSON', 'Not JSON', 0)))
mock_get.return_value = mock_response
response = fulltext.fulltext_search_api({"q": "hello"})
assert response == {"error": "Error converting search engine data to JSON"}
def jsonloads_list(data: str) -> List:
"""Just like jsonloads but forces the result to be a List"""
value = json.loads(data)
if not isinstance(value, list):
raise JSONDecodeError(msg='Returned json is not a list',
doc='{}',
pos=0)
return value
def jsonloads_dict(data: str) -> Dict[str, Any]:
"""Just like jsonloads but forces the result to be a Dict"""
value = json.loads(data)
if not isinstance(value, dict):
raise JSONDecodeError(msg='Returned json is not a dict',
doc='{}',
pos=0)
return value
def test_get_validated_fields_with_malformed_json(self, view, mschema):
"""
.get_validated_fields should raise HTTPNotAcceptable, when provided json
is malformed
"""
mschema.load.side_effect = JSONDecodeError(None, "", 0)
with raises(HTTPNotAcceptable):
view.get_validated_fields(mschema)
def test_wls_handle_error_500_html():
response = MagicMock()
response.status_code = 500
response.text = "text here"
response.json = MagicMock()
response.json.side_effect = JSONDecodeError("No json here", '"', 1)
with pytest.raises(wls_rest_python.ServerErrorException,
match="text here"):
wls_rest_python.WLS._handle_error(response)
def get_json(self):
if len(self.request.body) == 0:
return {}
try:
return tornado.escape.json_decode(self.request.body)
except JSONDecodeError:
raise JSONDecodeError(
f'JSON decode of request body failed on {self.request.uri}.'
' Please ensure all requests are of type application/json.')
def test_raises_brickftperror_when_response_is_not_valid_json(client, mocker):
mock_requests = mocker.patch('brickftp.client.requests')
mock_requests.get().json.side_effect = JSONDecodeError(
msg='', doc=mocker.MagicMock(), pos=1)
with pytest.raises(BrickFTPError) as exc:
client.dir('/')
exc.match(r'Non-valid JSON response:.+')
def test_non_json_error(self, mocked_api):
mocked_response = MagicMock()
mocked_response.json.side_effect = JSONDecodeError("json error", "", 0)
mocked_response.status_code = 500
mocked_response.raise_for_status.side_effect = HTTPError()
mocked_api.session.request.return_value = mocked_response
with pytest.raises(HTTPError):
mocked_api.version_by_id("version_id")
mocked_response.raise_for_status.assert_called_once()
def validator_mock_raises_decode_error():
doc = MagicMock()
doc.return_value = doc
doc.count.return_value = 5
doc.rfind.return_value = 2
from json.decoder import JSONDecodeError
with patch(
"pytest_splunk_addon.standard_lib.cim_tests.json_schema.Draft7Validator",
MagicMock(side_effect=JSONDecodeError("error", doc, 9)),
):
yield
def test_json_decode_error(self):
"""
case when we get unparsable JSON response from ISPyB
"""
with patch("requests.get") as get_mock:
resp = Mock()
resp.status_code = 200
resp.json.side_effect = JSONDecodeError(None, "", 0)
get_mock.return_value = resp
self.assertRaisesRegex(
Exception, ".*invalid json reply",
lambda: ispyb._ispyb_get_proposals("host", "token"))
def fn(user_exists=True, verified=False, no_auth0=False):
mock_storage = mocker.patch('sfa_api.utils.storage.get_storage')
mock_storage.user_exists = mocker.Mock(return_value=user_exists)
mock_storage.create_new_user = mocker.Mock()
mock_storage.return_value = mock_storage
user_info = mocker.patch('sfa_api.utils.auth.request_user_info')
if no_auth0:
user_info.side_effect = JSONDecodeError('error', '{}', 1)
else:
user_info.return_value = {'email_verified': verified}
return (mock_storage.user_exists, user_info,
mock_storage.create_new_user)
def test_handles_responses_with_invalid_json(self, get_email_hash):
dm_mailchimp_client = DMMailChimpClient('username', DUMMY_MAILCHIMP_API_KEY, logging.getLogger('mailchimp'))
with mock.patch.object(
dm_mailchimp_client._client.lists.members, 'create_or_update', autospec=True) as create_or_update:
response = mock.Mock()
response.json.side_effect = JSONDecodeError('msg', 'doc', 0)
create_or_update.side_effect = RequestException("error sending", response=response)
with assert_external_service_log_entry(successful_call=False, extra_modules=['mailchimp']) as log_catcher:
res = dm_mailchimp_client.subscribe_new_email_to_list('list_id', '*****@*****.**')
assert res == {'error_type': 'unexpected_error', 'status': 'error', 'status_code': 500}
assert log_catcher.records[1].msg == 'Mailchimp failed to add user (foo) to list (list_id)'
assert log_catcher.records[1].error == "error sending"
assert log_catcher.records[1].levelname == 'ERROR'
def test_invalid_creds(self):
response = Mock()
response.text = ISPYB_ERR_MSG
response.json.side_effect = JSONDecodeError("", "", 0)
with patch("requests.post", Mock(return_value=response)) as post:
token = ispyb._ispyb_authenticate(self.HOST, self.SITE, USER, PASS)
self.assertIsNone(token)
post.assert_called_once_with(
"https://example.com/ispyb/ispyb-ws/rest/authenticate?site=TstSite",
data={
"login": "******",
"password": "******"
},
headers={"content-type": "application/x-www-form-urlencoded"})
def get_profile_ic(self, profile: List) -> Dict:
"""
Given a list of individuals, return their information content
"""
sim_response = get_attribute_information_profile(
self.url, frozenset(profile))
profile_ic = {}
try:
for cls in sim_response['input']:
profile_ic[cls['id']] = cls['IC']
except JSONDecodeError as json_exc:
raise JSONDecodeError(
"Cannot parse owlsim2 response: {}".format(json_exc.msg),
json_exc.doc, json_exc.pos)
return profile_ic
def test_allDataMocked_JSONDecodeError(self, mock_getAll, mock_get):
# arrange
mock_get.side_effect = JSONDecodeError('could not parse json', 'doc',
1)
mock_getAll.return_value = [{
'url': 'http://mock-server.com/rest/api',
'name': 'mock'
}]
# act
response = self.app.get('/allData')
# assert
self.assertEqual(response.status_code, 200)
self.assertIn(
f"could not json parse response from http://mock-server.com/rest/api",
response.get_data(as_text=True))
def test_get_hmac_for_secret():
with mock.patch(
'paasta_tools.secret_tools.open',
autospec=False,
) as mock_open, mock.patch(
'json.load',
autospec=True,
) as mock_json_load, mock.patch(
'paasta_tools.secret_tools.get_secret_name_from_ref',
autospec=True,
) as mock_get_secret_name_from_ref:
mock_json_load.return_value = {
'environments': {
'dev': {
'signature': 'notArealHMAC'
},
},
}
mock_get_secret_name_from_ref.return_value = 'secretsquirrel'
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", 'dev')
mock_get_secret_name_from_ref.assert_called_with(
"SECRET(secretsquirrel)")
mock_open.assert_called_with(
"/nail/blah/service-name/secrets/secretsquirrel.json", "r")
assert ret == 'notArealHMAC'
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", 'dev-what')
assert ret is None
mock_open.side_effect = IOError
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", 'dev')
assert ret is None
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", 'dev')
assert ret is None
mock_open.side_effect = None
mock_json_load.side_effect = JSONDecodeError('', '', 1)
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", 'dev')
assert ret is None
def fetch_ip_addresses(headers: Dict[str, str] = {},
instance: Optional[str] = None) -> Any:
url = "https://api.packet.net/devices/{}".format(instance)
response = requests.get(url, headers=headers)
try:
response_payload = response.json()
if "ip_addresses" not in response_payload:
return []
else:
return response_payload["ip_addresses"]
except JSONDecodeError as e:
raise JSONDecodeError(
"Unable to decode API/metadata response for {}. {}".format(
url, e.msg),
e.doc,
e.pos,
)
def convert_to_csv(json_file):
"""Read/load the json_file (local file downloaded to /tmp) and
convert/write it to defined csv_file.
The data is in mounts > collected
Catch bad JSON (JSONDecodeError) file content, in that case print the defined
EXCEPTION string ('exception caught') to stdout reraising the exception.
This is to make sure you actually caught this exception.
Example csv output:
creatureId,icon,isAquatic,isFlying,isGround,isJumping,itemId,name,qualityId,spellId
32158,ability_mount_drake_blue,False,True,True,False,44178,Albino Drake,4,60025
63502,ability_mount_hordescorpionamber,True,...
...
""" # noqa E501
csv_file = TMP / json_file.name.replace(".json", ".csv")
# you code
# load the file, catch the bad JSON and raise a new JSONDecodeError
with open(json_file) as f:
try:
data = json.load(f)
except JSONDecodeError as e:
# print EXCEPTION as requested by the test
print(EXCEPTION)
# and then the original exception
raise JSONDecodeError(e.msg, e.doc, e.pos)
# data is a dict, but we are looking more specifically
# for data["mounts"]["collected"]
# >>> type(data["mounts"]["collected"])
# <class 'list'>
collected = data["mounts"]["collected"]
fieldnames = collected[0].keys()
# write the portion of data that we are looking for
# into the csv file
with open(csv_file, "w", newline="") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for row in collected:
writer.writerow(row)
def test_store_tests_should_return_empty_results_if_json_decoder_exception_thrown(
mock_get_json_job_details, mock_ssl_create_default_context,
mock_socket):
"""Tests that `store_tests` translates the test data object properly to a BuildResults document.
"""
# Given
mock_socket = mock_socket.return_value
mock_context = MagicMock()
mock_ssl_create_default_context.return_value = mock_context
## Mocked arguments
mock_args = MagicMock()
mock_args.buildurl = "abc"
mock_args.buildid = "123"
mock_args.platform = "platform"
mock_args.productversion = "1234abc"
## Mock the JSON response from Jenkins REST APIs
mock_get_json_job_details.side_effect = [
{
"fullName": "a_job_name"
},
{
"url": "http://abc",
"timestamp": "1550567699000",
"result": "FAILURE"
},
JSONDecodeError("dummy message", "doc", 1),
]
# When
build_results = store(mock_args)
# Then
assert not build_results.br_tests_object.br_tests_failed_object
assert not build_results.br_tests_object.br_tests_passed_object
assert not build_results.br_tests_object.br_tests_skipped_object
assert not build_results.br_tests_object.br_summary_object.br_total_failed_count
assert not build_results.br_tests_object.br_summary_object.br_total_passed_count
assert not build_results.br_tests_object.br_summary_object.br_total_skipped_count
assert not build_results.br_tests_object.br_summary_object.br_total_count
assert not build_results.br_tests_object.br_suites_object
def test_get_hmac_for_secret():
with mock.patch(
"paasta_tools.secret_tools.open",
autospec=False) as mock_open, mock.patch(
"json.load", autospec=True) as mock_json_load, mock.patch(
"paasta_tools.secret_tools.get_secret_name_from_ref",
autospec=True) as mock_get_secret_name_from_ref:
mock_json_load.return_value = {
"environments": {
"dev": {
"signature": "notArealHMAC"
}
}
}
mock_get_secret_name_from_ref.return_value = "secretsquirrel"
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", "dev")
mock_get_secret_name_from_ref.assert_called_with(
"SECRET(secretsquirrel)")
mock_open.assert_called_with(
"/nail/blah/service-name/secrets/secretsquirrel.json", "r")
assert ret == "notArealHMAC"
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", "dev-what")
assert ret is None
mock_open.side_effect = IOError
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", "dev")
assert ret is None
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", "dev")
assert ret is None
mock_open.side_effect = None
mock_json_load.side_effect = JSONDecodeError("", "", 1)
ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name",
"/nail/blah", "dev")
assert ret is None
def get_monitoring_locations(self, registry_ml_endpoint):
"""
Get the monitoring location data.
"""
# initialize state of errors, URL, and results
json_fail_count = 0
fetches = 0
url = self.construct_url(registry_ml_endpoint)
results = []
with self.session() as session:
while url:
if fetches % self.FETCHES_PER_LOG == 0:
logging.info(f'Retrieving monitoring locations: {url}')
try:
fetches += 1
payload = self.fetch_record_block(url, session)
results.extend(payload.get('results'))
if payload.get('next') is None or payload.get(
'next') == '':
break
except JSONDecodeError as json_err:
json_fail_count += 1
if json_fail_count >= self.FETCH_JSON_ERROR_TOLERANCE:
logging.error(
'Abort: JSON errors exceeded. Set FETCH_JSON_ERROR_TOLERANCE to fine tune.'
)
raise JSONDecodeError('JSON errors exceeded.',
json_err.doc, json_err.pos)
else:
logging.warning(
f'JSON error occurred, {self.FETCH_JSON_ERROR_TOLERANCE-json_fail_count} before abort.'
)
except RequestException:
logging.error(
f'Unrecoverable error fetching data from {url}')
return []
url = self.construct_url(url)
logging.info(
f'Finished retrieving {len(results)} monitoring locations.')
return results
def _handle_response(self, resp):
"""
Safely handle an Atlas Response and return the results if valid.
:param Response resp: The response from the request method.
:return: A dict containing the results.
:rtype: dict
"""
try:
results = json.loads(resp.text)
resp.raise_for_status()
except JSONDecodeError:
raise JSONDecodeError("Error in parsing: {}".format(resp.text))
except requests.RequestException as e:
if "errorCode" in results:
raise AtlasException(resp.text)
else:
raise e(resp.text)
return results
def get_owlsim_stats(url) -> Tuple[IcStatistic, Dict[str, IcStatistic]]:
"""
:return Tuple[IcStatistic, Dict[str, IcStatistic]]
:raises JSONDecodeError: If the response body does not contain valid json
"""
scigraph = OntologyFactory().create('scigraph:ontology')
category_stats = {}
categories = [enum.value for enum in HpoUpperLevel]
sim_response = get_attribute_information_profile(url, categories=tuple(categories))
try:
global_stats = IcStatistic(
mean_mean_ic=float(sim_response['system_stats']['meanMeanIC']),
mean_sum_ic=float(sim_response['system_stats']['meanSumIC']),
mean_cls=float(sim_response['system_stats']['meanN']),
max_max_ic=float(sim_response['system_stats']['maxMaxIC']),
max_sum_ic=float(sim_response['system_stats']['maxSumIC']),
individual_count=int(sim_response['system_stats']['individuals']),
mean_max_ic=float(sim_response['system_stats']['meanMaxIC'])
)
for cat_stat in sim_response['categorical_scores']:
category_stats[cat_stat['id']] = IcStatistic(
mean_mean_ic=float(cat_stat['system_stats']['meanMeanIC']),
mean_sum_ic=float(cat_stat['system_stats']['meanSumIC']),
mean_cls=float(cat_stat['system_stats']['meanN']),
max_max_ic=float(cat_stat['system_stats']['maxMaxIC']),
max_sum_ic=float(cat_stat['system_stats']['maxSumIC']),
individual_count=int(cat_stat['system_stats']['individuals']),
mean_max_ic=float(cat_stat['system_stats']['meanMaxIC']),
descendants=scigraph.descendants(cat_stat['id'], relations=["subClassOf"])
)
except JSONDecodeError as json_exc:
raise JSONDecodeError(
"Cannot parse owlsim2 response: {}".format(json_exc.msg),
json_exc.doc,
json_exc.pos
)
return global_stats, category_stats
def test_import_load_file_failure(tmp_trestle_dir: pathlib.Path) -> None:
"""Test model load failures."""
# Create a file with bad json
sample_data = '"star": {'
rand_str = ''.join(random.choice(string.ascii_letters) for x in range(16))
bad_file = pathlib.Path(f'{tmp_trestle_dir.parent}/{rand_str}.json').open(
'w+', encoding='utf8')
bad_file.write(sample_data)
bad_file.close()
with patch('trestle.utils.fs.load_file') as load_file_mock:
load_file_mock.side_effect = err.TrestleError('stuff')
args = argparse.Namespace(file=bad_file.name,
output='imported',
verbose=True)
i = importcmd.ImportCmd()
rc = i._run(args)
assert rc == 1
# Force PermissionError:
with patch('trestle.utils.fs.load_file') as load_file_mock:
load_file_mock.side_effect = PermissionError
args = argparse.Namespace(file=bad_file.name,
output='imported',
verbose=True)
i = importcmd.ImportCmd()
rc = i._run(args)
assert rc == 1
# Force JSONDecodeError:
with patch('trestle.utils.fs.load_file') as load_file_mock:
load_file_mock.side_effect = JSONDecodeError(msg='Extra data:',
doc=bad_file.name,
pos=0)
args = argparse.Namespace(file=bad_file.name,
output='imported',
verbose=True)
i = importcmd.ImportCmd()
rc = i._run(args)
assert rc == 1
# This is in case the same tmp_trestle_dir.parent is used, as across succeeding scopes of one pytest
os.chmod(bad_file.name, 0o600)
os.remove(bad_file.name)
def json_decode_error_raiser(*args, **kwargs):
raise JSONDecodeError('fake json decode error', 'credentials.json', 0)
def get_wallclock_history(self):
""" returns the wallclock time history for all the valid tests as a dictionary
of NumPy arrays. Set filter_times to False to return 0.0 as a placeholder
when there was no available execution time. """
def extract_time(file):
""" Helper function for getting runtimes """
for line in file:
if "Execution time" in line:
# this is of the form: <li>Execution time: 412.930 s
return float(line.split(":")[1].strip().split(" ")[0])
elif "(seconds)" in line:
# this is the older form -- split on "="
# form: <p><b>Execution Time</b> (seconds) = 399.414828
return float(line.split("=")[1])
raise RuntimeError()
json_file = self.get_wallclock_file()
if os.path.isfile(json_file):
try:
timings = json.load(open(json_file, 'r'))
# Check for proper format
item = next(iter(timings.values()))
if not isinstance(item, dict): raise JSONDecodeError()
return timings
except (IOError, OSError, JSONDecodeError, StopIteration):
pass
valid_dirs, all_tests = self.get_run_history(check_activity=False)
# store the timings in a dictionary
timings = dict()
for dir in valid_dirs:
# Get status files
dir_path = os.path.join(self.webTopDir, dir)
sfiles = glob.glob("{}/*.status".format(dir_path))
sfiles = list(filter(os.path.isfile, sfiles))
# Tests that should be counted
passed = set()
for i, file in enumerate(map(open, sfiles)):
contents = file.read()
if "PASSED" in contents:
filename = os.path.basename(sfiles[i])
passed.add(filename.split(".")[0])
file.close()
for test in filter(lambda x: x in passed, all_tests):
file = "{}/{}.html".format(dir_path, test)
try:
file = open(file)
except:
continue
try:
time = extract_time(file)
except RuntimeError:
continue
test_dict = timings.setdefault(test, self.timing_default)
test_dict["runtimes"].append(time)
test_dict["dates"].append(dir.rstrip("/"))
file.close()
return timings
