def test_empty_objects(self): s = '{}' self.assertEqual(dirtyjson.loads(s), eval(s)) s = '[]' self.assertEqual(dirtyjson.loads(s), eval(s)) s = '""' self.assertEqual(dirtyjson.loads(s), eval(s))
def test_failures(self): for idx, doc in enumerate(JSONDOCS): idx += 1 try: dirtyjson.loads(doc) except dirtyjson.Error: pass else: self.fail("Expected failure for fail%d.json: %r" % (idx, doc))
def test_object_keys(self): result = {"key": "value", "k": "v"} rval = dirtyjson.loads("""{"key": "value", "k": "v"}""") self.assertEqual(rval, result) rval = dirtyjson.loads("""{'key': 'value', 'k': 'v'}""") self.assertEqual(rval, result) rval = dirtyjson.loads("""{key: 'value', k: 'v'}""") self.assertEqual(rval, result) rval = dirtyjson.loads("""{key: 'value', k: 'v',}""") self.assertEqual(rval, result)
def test_scan_error(self): for t in (u, b): try: dirtyjson.loads(t('{"asdf": "')) except dirtyjson.Error: err = sys.exc_info()[1] else: self.fail('Expected JSONDecodeError') self.assertEqual(err.lineno, 1) self.assertEqual(err.colno, 10)
def test_array_decoder_issue46(self): # http://code.google.com/p/dirtyjson/issues/detail?id=46 for doc in [u'[,]', '[,]']: try: dirtyjson.loads(doc) except dirtyjson.Error: e = sys.exc_info()[1] self.assertEqual(e.pos, 1) self.assertEqual(e.lineno, 1) self.assertEqual(e.colno, 2) except Exception: e = sys.exc_info()[1] self.fail("Unexpected exception raised %r %s" % (e, e)) else: self.fail("Unexpected success parsing '[,]'")
async def login(self): async with self._session.get('http://{}/html/login/index.html'.format( self._host)) as resp: assert resp.status == 200, "Response status code for fetching index is {}".format( file, resp.status) re_res = re.search(r'[0-9a-zA-Z]{64}', await resp.text()) if re_res: challenge = re_res.group(0) encrypted_password = hashlib.sha256('{}:{}'.format( challenge, self._password).encode()).hexdigest() # noinspection SpellCheckingInspection request_data = { 'password': encrypted_password, 'csrf_token': 'nulltoken', 'showpw': '0', 'challengev': challenge, } headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'X-Requested-With': 'XMLHttpRequest', 'Referer': 'http://{}/html/login/index.html'.format(self._host), } async with self._session.post('http://{}/data/Login.json'.format( self._host), data=request_data, headers=headers) as resp: content = await resp.read() parsed = json.loads(content) data = self.parse_typed_dict(parsed) assert data[ 'login'] == 'success', "Login wasn't successful: {}".format( data) derived_key = hashlib.pbkdf2_hmac( 'sha1', hashlib.sha256(self._password.encode()).hexdigest().encode(), challenge[0:16].encode(), 1000, 16).hex() cookies = SimpleCookie() cookies['challengev'] = challenge cookies['challengev']['domain'] = self._host cookies['derivedk'] = derived_key cookies['derivedk']['domain'] = self._host self._session.cookie_jar.update_cookies(cookies) for cookie in self._session.cookie_jar: self.logger.info(cookie) if self._cookie_persistent_path: try: self._session.cookie_jar.save(self._cookie_persistent_path) except Exception as e: self.logger.error("Storing cookies failed", exc_info=True)
def load_json(fname): with open(fname, 'r') as read_file: input_json = read_file.read() # Keep the javascript preamble to reattach later preamble = input_json[0:15] data = dirtyjson.loads(input_json[15:]) return (preamble, data)
def test_ignore_single_line_comments(self): s = """ // here are some comments { // comments inside too test: 1, // and at the end of lines 'aack': 0x80, "bar": [ // even inside arrays 1, 2, 3, // and after trailing commas ], more: { // and inside objects once: true, twice: false, three_times3: null // and at the end } } console.log(b); """ rval = dirtyjson.loads(s) self.assertEqual( rval, { 'test': 1, 'aack': 0x80, 'bar': [1, 2, 3], 'more': { 'once': True, 'twice': False, 'three_times3': None } })
def _invoke_serverless(self, stage='', local=False): name = _get_serverless_name(self.config, self.target) command = 'serverless invoke' if local: command += ' local' if stage: command += ' --stage %s' % stage logger.debug("Invoking via 'serverless'") command += ' --log --verbose --function %s' % name output, retcode = next(run(command, out='oneshot')) logger.debug("Invocation logs from 'serverless':\n%s", output) if retcode != 0: error = RuntimeError('Invocation failed on Serverless: %s' % command) try: details = dirtyjson.loads(output) except ValueError: # No JSON in the output details = {} if isinstance(details, dict): details = dict(details) elif isinstance(details, list): details = list(details) error.logs = output error.details = details raise error return output, None
def dirty_json_or_none(text): # type: (Text) -> dict try: return dirtyjson.loads(text) except dirtyjson.error.Error: # logger.warning('^^^^^^^^^^ malformed json text: %s', text) malformed_json_text.append(text) return None
def test_not_at_beginning(self): s = """ // here are some comments var a = 1; // here is a line of regular JS var b = {test: 1, 'aack': 0x80, "bar": [1, 2, 3]}; console.log(b); """ first_object_index = s.index('{') rval = dirtyjson.loads(s, start_index=first_object_index) self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]}) rval = dirtyjson.loads(s, start_index=first_object_index + 1, search_for_first_object=True) self.assertEqual(rval, [1, 2, 3]) rval = dirtyjson.loads(s, search_for_first_object=True) self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]})
def parse(self, lint_data): messages = set() for msgdata in json.loads(lint_data).get("Issues", []): try: path = msgdata["Pos"]["Filename"] line = msgdata["Pos"]["Line"] msgbody = msgdata["FromLinter"] + ": " + msgdata["Text"] messages.add((path, line, msgbody)) except (ValueError, KeyError): print("Invalid message: {0}".format(msgdata)) return messages
def search(self, search_strings, age=0, ep_obj=None, **kwargs): """ Search a provider and parse the results. :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) """ results = [] if self.custom_url: if not validators.url(self.custom_url): log.warning('Invalid custom url: {0}', self.custom_url) return results self.url = self.custom_url self.urls = { 'login': urljoin(self.url, '/torrents/'), 'search': urljoin(self.url, '/t.json'), 'download': urljoin(self.url, '/download.php/') } if not self.login(): return results for mode in search_strings: log.debug('Search mode: {0}', mode) for search_string in search_strings[mode]: if mode != 'RSS': log.debug('Search string: {search}', {'search': search_string}) search_string = '+'.join(search_string.split()) params = dict({'q': search_string}, **self.categories[mode]) response = self.session.get(self.urls['search'], params=params) if not response or not response.text: log.debug('No data returned from provider') continue try: jdata = djson.loads(response.text) except ValueError as error: log.error( "Couldn't deserialize JSON document. Error: {0!r}", error) continue results += self.parse(jdata, mode) return results
def parse(self, lint_data): messages = set() for error_stanza in json.loads(lint_data): messages.add(( error_stanza.get("file", None), error_stanza.get("line", None), error_stanza.get("message", None), )) return messages
def parse(self, lint_data): messages = set() for msgdata in json.loads(lint_data): try: path = msgdata['path'] line = msgdata['line'] msgbody = msgdata['linter'] + ': ' + msgdata['message'] messages.add((path, line, msgbody)) except (ValueError, KeyError): print('Invalid message: {0}'.format(msgdata)) return messages
async def heartbeat(self) -> bool: # We shouldn't have caching issues, but maybe the speedport interprets the params params = {'_time': int(time.time()), '_rand': random.randint(1, 1000)} async with self._session.get('http://{}/data/heartbeat.json'.format( self._host), params=params) as resp: assert resp.status == 200, "Response status code for heartbeat is {}".format( resp.status) raw_data = json.loads(await resp.text()) data = self.parse_typed_dict(raw_data) return data['loginstate'] == '1'
def parse(self, lint_data): messages = set() for msgdata in json.loads(lint_data): try: path = msgdata["path"] line = msgdata["line"] msgbody = msgdata["linter"] + ": " + msgdata["message"] messages.add((path, line, msgbody)) except (ValueError, KeyError): print("Invalid message: {0}".format(msgdata)) return messages
def fetch_public_key(repo): """Download RSA public key Travis will use for this repo. Travis API docs: http://docs.travis-ci.com/api/#repository-keys """ keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo) data = json.loads(urlopen(keyurl).read().decode()) if 'key' not in data: errmsg = "Could not find public key for repo: {}.\n".format(repo) errmsg += "Have you already added your GitHub repo to Travis?" raise ValueError(errmsg) return data['key']
def parse(self, lint_data): messages = set() for file_path, output in lint_data: for msgdata in json.loads(output): try: path = file_path line = msgdata['line'] msgbody = msgdata['message'] messages.add((path, line, msgbody)) except (ValueError, KeyError): print('Invalid message: {0}'.format(msgdata)) return messages
def parse(self, lint_data): messages = set() for filedata in json.loads(lint_data): if filedata.get("file") and filedata.get("messages"): path = filedata["file"] for msgdata in filedata["messages"]: try: line = msgdata["line"] msgbody = msgdata["message"] messages.add((path, line, msgbody)) except (ValueError, KeyError): print("Invalid message: {0}".format(msgdata)) return messages
def parse(self, lint_data): messages = set() for filedata in json.loads(lint_data): if filedata.get('messages'): for msgdata in filedata['messages']: try: path = filedata['filePath'] line = msgdata['line'] msgbody = msgdata['message'] messages.add((path, line, msgbody)) except (ValueError, KeyError): print('Invalid message: {0}'.format(msgdata)) return messages
def test_truncated_input(self): test_cases = [ ('', 'Expecting value', 0), ('[', "Expecting value or ']'", 1), ('[42', "Expecting ',' delimiter", 3), ('[42,', 'Expecting value', 4), ('["', 'Unterminated string starting at', 1), ('["spam', 'Unterminated string starting at', 1), ('["spam"', "Expecting ',' delimiter", 7), ('["spam",', 'Expecting value', 8), ('{', 'Expecting property name', 1), ('{"', 'Unterminated string starting at', 1), ('{"spam', 'Unterminated string starting at', 1), ('{"spam"', "Expecting ':' delimiter", 7), ('{"spam":', 'Expecting value', 8), ('{"spam":42', "Expecting ',' delimiter", 10), ('{"spam":42,', 'Expecting property name', 11), ('"', 'Unterminated string starting at', 0), ('"spam', 'Unterminated string starting at', 0), ('[,', "Expecting value", 1), ] for data, msg, idx in test_cases: try: dirtyjson.loads(data) except dirtyjson.Error: e = sys.exc_info()[1] self.assertEqual( e.msg[:len(msg)], msg, "%r doesn't start with %r for %r" % (e.msg, msg, data)) self.assertEqual( e.pos, idx, "pos %r != %r for %r" % (e.pos, idx, data)) except Exception: e = sys.exc_info()[1] self.fail("Unexpected exception raised %r %s" % (e, e)) else: self.fail("Unexpected success parsing '%r'" % (data,))
def _extract_dirtyjson_definition(match_html, match_data, r, key, flags=0): m = re.search(r, match_html, flags) if m is None: return dirtyjson_string = m.group(1) fixed_dirtyjson_string = fix_dirtyjson(dirtyjson_string) value = dirtyjson.loads(fixed_dirtyjson_string) if value is None: return match_data[key] = value
def parse(self, lint_data): messages = set() msgdata = '' for line in lint_data.split('\n'): try: msgdata = json.loads(line) path = msgdata['location']['file'] line = msgdata['location']['line'] msgbody = msgdata['message'] messages.add((path, line, msgbody)) except (ValueError, KeyError): print('Invalid message: {0}'.format(msgdata)) return messages
def parse(self, lint_data): messages = set() for filename, msgs in json.loads(lint_data).items(): if msgs: for msgdata in msgs: try: path = filename line = msgdata['line'] msgbody = msgdata['message'] messages.add((path, line, msgbody)) except (ValueError, KeyError): print('Invalid message: {0}'.format(msgdata)) return messages
def parse(self, lint_data): messages = set() msgdata = "" for line in lint_data.split("\n"): try: msgdata = json.loads(line) path = msgdata["location"]["file"] line = msgdata["location"]["line"] msgbody = msgdata["message"] messages.add((path, line, msgbody)) except (ValueError, KeyError): print("({0}) Invalid message: {1}".format(type(self).__name__, msgdata)) return messages
def parse(self, lint_data): messages = set() for msgdata in json.loads(lint_data).get('messages'): try: path = msgdata['location']['path'] line = msgdata['location']['line'] msgbody = '{0}: {1} ({2})'.format(msgdata['source'], msgdata['message'], msgdata['code']) messages.add((path, line, msgbody)) except (ValueError, KeyError): print('Invalid message: {0}'.format(msgdata)) return messages
def parse_bad_contents(contents): """Recursively attempts to fix JSON until it succeeds. Args: contents - String contents to try parsing. Returns: Parsed JSON result. """ try: return dirtyjson.loads(contents) except Exception as e: print(contents) return parse_bad_contents(repair(e, contents))
async def fetch_data(self, file: str): with self.FETCH_EXCEPTIONS.labels(file).count_exceptions(): with self.FETCH_TIME.labels(file).time(): async with self._session.get('http://{}/data/{}.json'.format( self._host, file)) as resp: assert resp.status == 200, "Response status code for {} is {}".format( file, resp.status) raw = await resp.text() try: return json.loads(raw) except json.error.Error as e: self.logger.error("Error unmarshalling received data", exc_info=True) raise
def parse(self, lint_data): messages = set() for file_path, output in lint_data: try: for msgdata in json.loads(output): try: path = file_path line = msgdata["line"] msgbody = msgdata["message"] messages.add((path, line, msgbody)) except (ValueError, KeyError): print("Invalid message: {0}".format(msgdata)) except json.error.Error: print("Invalid message: {0}".format(output)) return messages
def parse(self, lint_data): messages = set() for file_path, output in lint_data: try: for error_stanza in json.loads(output): messages.add(( file_path, error_stanza.get("line", None), error_stanza.get("message", None), )) except Exception: traceback.print_exc() return messages
def search(self, search_strings, age=0, ep_obj=None, **kwargs): """ Search a provider and parse the results. :param search_strings: A dict with mode (key) and the search value (value) :param age: Not used :param ep_obj: Not used :returns: A list of search results (structure) """ results = [] if not self.login(): return results for mode in search_strings: log.debug('Search mode: {0}', mode) for search_string in search_strings[mode]: if mode != 'RSS': log.debug('Search string: {search}', {'search': search_string}) search_string = '+'.join(search_string.split()) params = dict({'q': search_string}, **self.categories[mode]) response = self.session.get(self.urls['search'], params=params) if not response or not response.text: log.debug('No data returned from provider') continue try: jdata = djson.loads(response.text) except ValueError as error: log.error("Couldn't deserialize JSON document. Error: {0!r}", error) continue results += self.parse(jdata, mode) return results
def test_ignore_single_line_comments(self): s = """ // here are some comments { // comments inside too test: 1, // and at the end of lines 'aack': 0x80, "bar": [ // even inside arrays 1, 2, 3, // and after trailing commas ], more: { // and inside objects once: true, twice: false, three_times3: null // and at the end } } console.log(b); """ rval = dirtyjson.loads(s) self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3], 'more': {'once': True, 'twice': False, 'three_times3': None}})
def test_ignore_inline_comments(self): s = """ /* here are some comments * that should all be skipped * right up until the terminator */ { /* comments inside too */ test: 1, /* and at the end of lines */ 'aack': 0x80, "bar": [ // even inside arrays 1, 2, 3, // and after trailing commas ], /* comment this block out more: { // and inside objects once: true, twice: false, three_times3: null // and at the end } */ } console.log(b); """ rval = dirtyjson.loads(s) self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]})
def test_degenerates_allow(self): for inf in (PosInf, NegInf): self.assertEqual(dirtyjson.loads(json.dumps(inf)), inf) # Python 2.5 doesn't have math.isnan nan = dirtyjson.loads(json.dumps(NaN)) self.assertTrue((0 + nan) != nan)
def test_empty_strings(self): self.assertEqual(dirtyjson.loads('""'), "") self.assertEqual(dirtyjson.loads(u'""'), u"") self.assertEqual(dirtyjson.loads('[""]'), [""]) self.assertEqual(dirtyjson.loads(u'[""]'), [u""])
def test_parse(self): # test in/out equivalence and parsing res = dirtyjson.loads(JSON) out = json.dumps(res) self.assertEqual(res, dirtyjson.loads(out))
def loads(self, s, **kw): sio = StringIO(s) res = dirtyjson.loads(s, **kw) self.assertEqual(res, dirtyjson.load(sio, **kw)) return res
def test_float(self): rval = dirtyjson.loads('1', parse_int=float) self.assertTrue(isinstance(rval, float)) self.assertEqual(rval, 1.0)
def test_decoder_optimizations(self): rval = dirtyjson.loads('{ "key" : "value" , "k":"v" }') self.assertEqual(rval, {"key": "value", "k": "v"})
def test_floats(self): for num in [1617161771.7650001, math.pi, math.pi ** 100, math.pi ** -100, 3.1]: self.assertEqual(dirtyjson.loads(json.dumps(num)), num) self.assertEqual(dirtyjson.loads(text_type(json.dumps(num))), num)
def test_ints(self): for num in [1, long_type(1), 1 << 32, 1 << 64]: self.assertEqual(dirtyjson.loads(json.dumps(num)), num) self.assertEqual(dirtyjson.loads(text_type(json.dumps(num))), num)
def test_decimal(self): rval = dirtyjson.loads('1.1', parse_float=decimal.Decimal) self.assertTrue(isinstance(rval, decimal.Decimal)) self.assertEqual(rval, decimal.Decimal('1.1'))
def test_empty_strings_with_single_quotes(self): self.assertEqual(dirtyjson.loads("''"), "") self.assertEqual(dirtyjson.loads(u"''"), u"") self.assertEqual(dirtyjson.loads("['']"), [""]) self.assertEqual(dirtyjson.loads(u"['']"), [u""])