def fetch_bgp( use_metadata: bool = True, headers: Dict[str, str] = {}, instance: Optional[str] = None, ) -> Any: url = "https://metadata.packet.net/metadata" ip_addresses = [] if not use_metadata: if not instance: raise ValueError( "Instance ID must be specified when not using metadata") url = "https://api.packet.net/devices/{}/bgp/neighbors".format( instance) ip_addresses = fetch_ip_addresses(headers=headers, instance=instance) response = requests.get(url, headers=headers) try: response_payload = response.json() if not use_metadata: response_payload["network"] = {"addresses": ip_addresses} return response_payload except JSONDecodeError as e: raise JSONDecodeError( "Unable to decode API/metadata response for {}. {}".format( url, e.msg), e.doc, e.pos, )
def test_parse_negative(self, reader, requests, mocker): configs = { 'cookies': {}, 'headers': {}, 'parameters': {}, 'value': "avascan", 'excludes': ["/admin"], 'domain': "example.com" } har = {'log': {'entries': [{'request': requests[0]}]}} # missing har request mocker.patch("builtins.open") mocker.patch("json.load", return_value=har) del har['log']['entries'][0]['request'] with pytest.raises(InvalidFormatException): reader.parse(configs) # invalid har log mocker.patch("json.load", return_value={'log': {'creator': {}}}) with pytest.raises(InvalidFormatException): reader.parse(configs) # invalid json mocker.patch("json.load", side_effect=JSONDecodeError("", "", 0)) with pytest.raises(InvalidFormatException): reader.parse(configs)
def decode(self, string): # replacing the mongodb objects in the extended json to the strict json version # try: string = re.sub(r'ObjectId\(\"(\S+)\"\)', r'"\1"', string) string = re.sub(r'NumberInt\((\S+)\)', r'\1', string) string = re.sub(r'NumberLong\((\S+)\)', r'\1', string) string = re.sub(r'NumberDecimal\((\S+)\)', r'\1', string) string = re.sub(r'Date\((\S+)\)', r'{"date":"\1"}', string) string = re.sub(r'Timestamp\((\S+)[,][ ]*(\S+)\)', r'{"timestamp":{"t":"\1","i":"\2"}}', string) string = re.sub(r'BinData\((\S+)[,][ ]*[\"](\S+)[\"]\)', r'{"binary":"\1","type":"\2"}', string) string = re.sub(r'DBRef\([\"](\S+)[\"][,][ ]*[\"](\S+)[\"]\)', r'{"ref":"\1","id":"\2"}', string) # string = re.sub(r':[ ]*undefined(\}|,)]', r'{"undefined":true}', string) # string = re.sub(r':[ ]*MaxKey(\}|,)]', r'{"maxkey":1}', string) # string = re.sub(r':[ ]*MinKey(\}|,)]', r'{"minkey":1}', string) # string = re.sub(r':[ ]*//\S+//\S+(\}|,)]', r'{"regex":"\1", "options":"\2"}', string) # string = re.sub(r':[ ]*//\S+//\S+(\}|,)]', r'{"regex":"\1", "options":"\2"}', string) # except Exception as err: # msg = "Could not parse the Extended Json." # raise JSONDecodeError(msg, err) # loading the json using the traditional json library try: data = json.loads(string, object_hook=self.object_hook) except JSONDecodeError as err: raise JSONDecodeError(err.msg) return data
def loadCreds(self): try: with open(self.__credPath) as loadedCredFile: credDict = json.load(loadedCredFile) if "username" in credDict and "password" in credDict: return credDict else: raise JSONDecodeError( "JSON file does not contain username and password") except Exception as e: exceptionType = type(e).__name__ print(exceptionType) if exceptionType == "FileNotFoundError": print( "\n####################\n CRED FILE NOT FOUND. \n YOU MUST IMPORT A VALID CRED FILE BY IMPORTING SOFTSERVE AND USING \n THE COMMAND softserve.importCreds()\n\n" ) raise if exceptionType == "JSONDecodeError": print( "\n####################\n CRED FILE EXIST BUT IS CORRUPTED. \n YOU MUST IMPORT A NEW CRED FILE BY IMPORTING SOFTSERVE AND USING \n THE COMMAND softserve.importCreds()\n\n" ) raise
def test_bad_json(self): with patch("openlibrary.core.fulltext.requests.get") as mock_get: config.plugin_inside = {"search_endpoint": "mock"} mock_response = Mock(json=Mock(side_effect=JSONDecodeError('Not JSON', 'Not JSON', 0))) mock_get.return_value = mock_response response = fulltext.fulltext_search_api({"q": "hello"}) assert response == {"error": "Error converting search engine data to JSON"}
def jsonloads_list(data: str) -> List: """Just like jsonloads but forces the result to be a List""" value = json.loads(data) if not isinstance(value, list): raise JSONDecodeError(msg='Returned json is not a list', doc='{}', pos=0) return value
def jsonloads_dict(data: str) -> Dict[str, Any]: """Just like jsonloads but forces the result to be a Dict""" value = json.loads(data) if not isinstance(value, dict): raise JSONDecodeError(msg='Returned json is not a dict', doc='{}', pos=0) return value
def test_get_validated_fields_with_malformed_json(self, view, mschema): """ .get_validated_fields should raise HTTPNotAcceptable, when provided json is malformed """ mschema.load.side_effect = JSONDecodeError(None, "", 0) with raises(HTTPNotAcceptable): view.get_validated_fields(mschema)
def test_wls_handle_error_500_html(): response = MagicMock() response.status_code = 500 response.text = "text here" response.json = MagicMock() response.json.side_effect = JSONDecodeError("No json here", '"', 1) with pytest.raises(wls_rest_python.ServerErrorException, match="text here"): wls_rest_python.WLS._handle_error(response)
def get_json(self): if len(self.request.body) == 0: return {} try: return tornado.escape.json_decode(self.request.body) except JSONDecodeError: raise JSONDecodeError( f'JSON decode of request body failed on {self.request.uri}.' ' Please ensure all requests are of type application/json.')
def test_raises_brickftperror_when_response_is_not_valid_json(client, mocker): mock_requests = mocker.patch('brickftp.client.requests') mock_requests.get().json.side_effect = JSONDecodeError( msg='', doc=mocker.MagicMock(), pos=1) with pytest.raises(BrickFTPError) as exc: client.dir('/') exc.match(r'Non-valid JSON response:.+')
def test_non_json_error(self, mocked_api): mocked_response = MagicMock() mocked_response.json.side_effect = JSONDecodeError("json error", "", 0) mocked_response.status_code = 500 mocked_response.raise_for_status.side_effect = HTTPError() mocked_api.session.request.return_value = mocked_response with pytest.raises(HTTPError): mocked_api.version_by_id("version_id") mocked_response.raise_for_status.assert_called_once()
def validator_mock_raises_decode_error(): doc = MagicMock() doc.return_value = doc doc.count.return_value = 5 doc.rfind.return_value = 2 from json.decoder import JSONDecodeError with patch( "pytest_splunk_addon.standard_lib.cim_tests.json_schema.Draft7Validator", MagicMock(side_effect=JSONDecodeError("error", doc, 9)), ): yield
def test_json_decode_error(self): """ case when we get unparsable JSON response from ISPyB """ with patch("requests.get") as get_mock: resp = Mock() resp.status_code = 200 resp.json.side_effect = JSONDecodeError(None, "", 0) get_mock.return_value = resp self.assertRaisesRegex( Exception, ".*invalid json reply", lambda: ispyb._ispyb_get_proposals("host", "token"))
def fn(user_exists=True, verified=False, no_auth0=False): mock_storage = mocker.patch('sfa_api.utils.storage.get_storage') mock_storage.user_exists = mocker.Mock(return_value=user_exists) mock_storage.create_new_user = mocker.Mock() mock_storage.return_value = mock_storage user_info = mocker.patch('sfa_api.utils.auth.request_user_info') if no_auth0: user_info.side_effect = JSONDecodeError('error', '{}', 1) else: user_info.return_value = {'email_verified': verified} return (mock_storage.user_exists, user_info, mock_storage.create_new_user)
def test_handles_responses_with_invalid_json(self, get_email_hash): dm_mailchimp_client = DMMailChimpClient('username', DUMMY_MAILCHIMP_API_KEY, logging.getLogger('mailchimp')) with mock.patch.object( dm_mailchimp_client._client.lists.members, 'create_or_update', autospec=True) as create_or_update: response = mock.Mock() response.json.side_effect = JSONDecodeError('msg', 'doc', 0) create_or_update.side_effect = RequestException("error sending", response=response) with assert_external_service_log_entry(successful_call=False, extra_modules=['mailchimp']) as log_catcher: res = dm_mailchimp_client.subscribe_new_email_to_list('list_id', '*****@*****.**') assert res == {'error_type': 'unexpected_error', 'status': 'error', 'status_code': 500} assert log_catcher.records[1].msg == 'Mailchimp failed to add user (foo) to list (list_id)' assert log_catcher.records[1].error == "error sending" assert log_catcher.records[1].levelname == 'ERROR'
def test_invalid_creds(self): response = Mock() response.text = ISPYB_ERR_MSG response.json.side_effect = JSONDecodeError("", "", 0) with patch("requests.post", Mock(return_value=response)) as post: token = ispyb._ispyb_authenticate(self.HOST, self.SITE, USER, PASS) self.assertIsNone(token) post.assert_called_once_with( "https://example.com/ispyb/ispyb-ws/rest/authenticate?site=TstSite", data={ "login": "******", "password": "******" }, headers={"content-type": "application/x-www-form-urlencoded"})
def get_profile_ic(self, profile: List) -> Dict: """ Given a list of individuals, return their information content """ sim_response = get_attribute_information_profile( self.url, frozenset(profile)) profile_ic = {} try: for cls in sim_response['input']: profile_ic[cls['id']] = cls['IC'] except JSONDecodeError as json_exc: raise JSONDecodeError( "Cannot parse owlsim2 response: {}".format(json_exc.msg), json_exc.doc, json_exc.pos) return profile_ic
def test_allDataMocked_JSONDecodeError(self, mock_getAll, mock_get): # arrange mock_get.side_effect = JSONDecodeError('could not parse json', 'doc', 1) mock_getAll.return_value = [{ 'url': 'http://mock-server.com/rest/api', 'name': 'mock' }] # act response = self.app.get('/allData') # assert self.assertEqual(response.status_code, 200) self.assertIn( f"could not json parse response from http://mock-server.com/rest/api", response.get_data(as_text=True))
def test_get_hmac_for_secret(): with mock.patch( 'paasta_tools.secret_tools.open', autospec=False, ) as mock_open, mock.patch( 'json.load', autospec=True, ) as mock_json_load, mock.patch( 'paasta_tools.secret_tools.get_secret_name_from_ref', autospec=True, ) as mock_get_secret_name_from_ref: mock_json_load.return_value = { 'environments': { 'dev': { 'signature': 'notArealHMAC' }, }, } mock_get_secret_name_from_ref.return_value = 'secretsquirrel' ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", 'dev') mock_get_secret_name_from_ref.assert_called_with( "SECRET(secretsquirrel)") mock_open.assert_called_with( "/nail/blah/service-name/secrets/secretsquirrel.json", "r") assert ret == 'notArealHMAC' ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", 'dev-what') assert ret is None mock_open.side_effect = IOError ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", 'dev') assert ret is None ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", 'dev') assert ret is None mock_open.side_effect = None mock_json_load.side_effect = JSONDecodeError('', '', 1) ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", 'dev') assert ret is None
def fetch_ip_addresses(headers: Dict[str, str] = {}, instance: Optional[str] = None) -> Any: url = "https://api.packet.net/devices/{}".format(instance) response = requests.get(url, headers=headers) try: response_payload = response.json() if "ip_addresses" not in response_payload: return [] else: return response_payload["ip_addresses"] except JSONDecodeError as e: raise JSONDecodeError( "Unable to decode API/metadata response for {}. {}".format( url, e.msg), e.doc, e.pos, )
def convert_to_csv(json_file): """Read/load the json_file (local file downloaded to /tmp) and convert/write it to defined csv_file. The data is in mounts > collected Catch bad JSON (JSONDecodeError) file content, in that case print the defined EXCEPTION string ('exception caught') to stdout reraising the exception. This is to make sure you actually caught this exception. Example csv output: creatureId,icon,isAquatic,isFlying,isGround,isJumping,itemId,name,qualityId,spellId 32158,ability_mount_drake_blue,False,True,True,False,44178,Albino Drake,4,60025 63502,ability_mount_hordescorpionamber,True,... ... """ # noqa E501 csv_file = TMP / json_file.name.replace(".json", ".csv") # you code # load the file, catch the bad JSON and raise a new JSONDecodeError with open(json_file) as f: try: data = json.load(f) except JSONDecodeError as e: # print EXCEPTION as requested by the test print(EXCEPTION) # and then the original exception raise JSONDecodeError(e.msg, e.doc, e.pos) # data is a dict, but we are looking more specifically # for data["mounts"]["collected"] # >>> type(data["mounts"]["collected"]) # <class 'list'> collected = data["mounts"]["collected"] fieldnames = collected[0].keys() # write the portion of data that we are looking for # into the csv file with open(csv_file, "w", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() for row in collected: writer.writerow(row)
def test_store_tests_should_return_empty_results_if_json_decoder_exception_thrown( mock_get_json_job_details, mock_ssl_create_default_context, mock_socket): """Tests that `store_tests` translates the test data object properly to a BuildResults document. """ # Given mock_socket = mock_socket.return_value mock_context = MagicMock() mock_ssl_create_default_context.return_value = mock_context ## Mocked arguments mock_args = MagicMock() mock_args.buildurl = "abc" mock_args.buildid = "123" mock_args.platform = "platform" mock_args.productversion = "1234abc" ## Mock the JSON response from Jenkins REST APIs mock_get_json_job_details.side_effect = [ { "fullName": "a_job_name" }, { "url": "http://abc", "timestamp": "1550567699000", "result": "FAILURE" }, JSONDecodeError("dummy message", "doc", 1), ] # When build_results = store(mock_args) # Then assert not build_results.br_tests_object.br_tests_failed_object assert not build_results.br_tests_object.br_tests_passed_object assert not build_results.br_tests_object.br_tests_skipped_object assert not build_results.br_tests_object.br_summary_object.br_total_failed_count assert not build_results.br_tests_object.br_summary_object.br_total_passed_count assert not build_results.br_tests_object.br_summary_object.br_total_skipped_count assert not build_results.br_tests_object.br_summary_object.br_total_count assert not build_results.br_tests_object.br_suites_object
def test_get_hmac_for_secret(): with mock.patch( "paasta_tools.secret_tools.open", autospec=False) as mock_open, mock.patch( "json.load", autospec=True) as mock_json_load, mock.patch( "paasta_tools.secret_tools.get_secret_name_from_ref", autospec=True) as mock_get_secret_name_from_ref: mock_json_load.return_value = { "environments": { "dev": { "signature": "notArealHMAC" } } } mock_get_secret_name_from_ref.return_value = "secretsquirrel" ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev") mock_get_secret_name_from_ref.assert_called_with( "SECRET(secretsquirrel)") mock_open.assert_called_with( "/nail/blah/service-name/secrets/secretsquirrel.json", "r") assert ret == "notArealHMAC" ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev-what") assert ret is None mock_open.side_effect = IOError ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev") assert ret is None ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev") assert ret is None mock_open.side_effect = None mock_json_load.side_effect = JSONDecodeError("", "", 1) ret = get_hmac_for_secret("SECRET(secretsquirrel)", "service-name", "/nail/blah", "dev") assert ret is None
def get_monitoring_locations(self, registry_ml_endpoint): """ Get the monitoring location data. """ # initialize state of errors, URL, and results json_fail_count = 0 fetches = 0 url = self.construct_url(registry_ml_endpoint) results = [] with self.session() as session: while url: if fetches % self.FETCHES_PER_LOG == 0: logging.info(f'Retrieving monitoring locations: {url}') try: fetches += 1 payload = self.fetch_record_block(url, session) results.extend(payload.get('results')) if payload.get('next') is None or payload.get( 'next') == '': break except JSONDecodeError as json_err: json_fail_count += 1 if json_fail_count >= self.FETCH_JSON_ERROR_TOLERANCE: logging.error( 'Abort: JSON errors exceeded. Set FETCH_JSON_ERROR_TOLERANCE to fine tune.' ) raise JSONDecodeError('JSON errors exceeded.', json_err.doc, json_err.pos) else: logging.warning( f'JSON error occurred, {self.FETCH_JSON_ERROR_TOLERANCE-json_fail_count} before abort.' ) except RequestException: logging.error( f'Unrecoverable error fetching data from {url}') return [] url = self.construct_url(url) logging.info( f'Finished retrieving {len(results)} monitoring locations.') return results
def _handle_response(self, resp): """ Safely handle an Atlas Response and return the results if valid. :param Response resp: The response from the request method. :return: A dict containing the results. :rtype: dict """ try: results = json.loads(resp.text) resp.raise_for_status() except JSONDecodeError: raise JSONDecodeError("Error in parsing: {}".format(resp.text)) except requests.RequestException as e: if "errorCode" in results: raise AtlasException(resp.text) else: raise e(resp.text) return results
def get_owlsim_stats(url) -> Tuple[IcStatistic, Dict[str, IcStatistic]]: """ :return Tuple[IcStatistic, Dict[str, IcStatistic]] :raises JSONDecodeError: If the response body does not contain valid json """ scigraph = OntologyFactory().create('scigraph:ontology') category_stats = {} categories = [enum.value for enum in HpoUpperLevel] sim_response = get_attribute_information_profile(url, categories=tuple(categories)) try: global_stats = IcStatistic( mean_mean_ic=float(sim_response['system_stats']['meanMeanIC']), mean_sum_ic=float(sim_response['system_stats']['meanSumIC']), mean_cls=float(sim_response['system_stats']['meanN']), max_max_ic=float(sim_response['system_stats']['maxMaxIC']), max_sum_ic=float(sim_response['system_stats']['maxSumIC']), individual_count=int(sim_response['system_stats']['individuals']), mean_max_ic=float(sim_response['system_stats']['meanMaxIC']) ) for cat_stat in sim_response['categorical_scores']: category_stats[cat_stat['id']] = IcStatistic( mean_mean_ic=float(cat_stat['system_stats']['meanMeanIC']), mean_sum_ic=float(cat_stat['system_stats']['meanSumIC']), mean_cls=float(cat_stat['system_stats']['meanN']), max_max_ic=float(cat_stat['system_stats']['maxMaxIC']), max_sum_ic=float(cat_stat['system_stats']['maxSumIC']), individual_count=int(cat_stat['system_stats']['individuals']), mean_max_ic=float(cat_stat['system_stats']['meanMaxIC']), descendants=scigraph.descendants(cat_stat['id'], relations=["subClassOf"]) ) except JSONDecodeError as json_exc: raise JSONDecodeError( "Cannot parse owlsim2 response: {}".format(json_exc.msg), json_exc.doc, json_exc.pos ) return global_stats, category_stats
def test_import_load_file_failure(tmp_trestle_dir: pathlib.Path) -> None: """Test model load failures.""" # Create a file with bad json sample_data = '"star": {' rand_str = ''.join(random.choice(string.ascii_letters) for x in range(16)) bad_file = pathlib.Path(f'{tmp_trestle_dir.parent}/{rand_str}.json').open( 'w+', encoding='utf8') bad_file.write(sample_data) bad_file.close() with patch('trestle.utils.fs.load_file') as load_file_mock: load_file_mock.side_effect = err.TrestleError('stuff') args = argparse.Namespace(file=bad_file.name, output='imported', verbose=True) i = importcmd.ImportCmd() rc = i._run(args) assert rc == 1 # Force PermissionError: with patch('trestle.utils.fs.load_file') as load_file_mock: load_file_mock.side_effect = PermissionError args = argparse.Namespace(file=bad_file.name, output='imported', verbose=True) i = importcmd.ImportCmd() rc = i._run(args) assert rc == 1 # Force JSONDecodeError: with patch('trestle.utils.fs.load_file') as load_file_mock: load_file_mock.side_effect = JSONDecodeError(msg='Extra data:', doc=bad_file.name, pos=0) args = argparse.Namespace(file=bad_file.name, output='imported', verbose=True) i = importcmd.ImportCmd() rc = i._run(args) assert rc == 1 # This is in case the same tmp_trestle_dir.parent is used, as across succeeding scopes of one pytest os.chmod(bad_file.name, 0o600) os.remove(bad_file.name)
def json_decode_error_raiser(*args, **kwargs): raise JSONDecodeError('fake json decode error', 'credentials.json', 0)
def get_wallclock_history(self): """ returns the wallclock time history for all the valid tests as a dictionary of NumPy arrays. Set filter_times to False to return 0.0 as a placeholder when there was no available execution time. """ def extract_time(file): """ Helper function for getting runtimes """ for line in file: if "Execution time" in line: # this is of the form: <li>Execution time: 412.930 s return float(line.split(":")[1].strip().split(" ")[0]) elif "(seconds)" in line: # this is the older form -- split on "=" # form: <p><b>Execution Time</b> (seconds) = 399.414828 return float(line.split("=")[1]) raise RuntimeError() json_file = self.get_wallclock_file() if os.path.isfile(json_file): try: timings = json.load(open(json_file, 'r')) # Check for proper format item = next(iter(timings.values())) if not isinstance(item, dict): raise JSONDecodeError() return timings except (IOError, OSError, JSONDecodeError, StopIteration): pass valid_dirs, all_tests = self.get_run_history(check_activity=False) # store the timings in a dictionary timings = dict() for dir in valid_dirs: # Get status files dir_path = os.path.join(self.webTopDir, dir) sfiles = glob.glob("{}/*.status".format(dir_path)) sfiles = list(filter(os.path.isfile, sfiles)) # Tests that should be counted passed = set() for i, file in enumerate(map(open, sfiles)): contents = file.read() if "PASSED" in contents: filename = os.path.basename(sfiles[i]) passed.add(filename.split(".")[0]) file.close() for test in filter(lambda x: x in passed, all_tests): file = "{}/{}.html".format(dir_path, test) try: file = open(file) except: continue try: time = extract_time(file) except RuntimeError: continue test_dict = timings.setdefault(test, self.timing_default) test_dict["runtimes"].append(time) test_dict["dates"].append(dir.rstrip("/")) file.close() return timings