def test_empty_objects(self):
s = '{}'
self.assertEqual(dirtyjson.loads(s), eval(s))
s = '[]'
self.assertEqual(dirtyjson.loads(s), eval(s))
s = '""'
self.assertEqual(dirtyjson.loads(s), eval(s))
def test_empty_objects(self):
s = '{}'
self.assertEqual(dirtyjson.loads(s), eval(s))
s = '[]'
self.assertEqual(dirtyjson.loads(s), eval(s))
s = '""'
self.assertEqual(dirtyjson.loads(s), eval(s))
def test_failures(self):
for idx, doc in enumerate(JSONDOCS):
idx += 1
try:
dirtyjson.loads(doc)
except dirtyjson.Error:
pass
else:
self.fail("Expected failure for fail%d.json: %r" % (idx, doc))
def test_failures(self):
for idx, doc in enumerate(JSONDOCS):
idx += 1
try:
dirtyjson.loads(doc)
except dirtyjson.Error:
pass
else:
self.fail("Expected failure for fail%d.json: %r" % (idx, doc))
def test_object_keys(self):
result = {"key": "value", "k": "v"}
rval = dirtyjson.loads("""{"key": "value", "k": "v"}""")
self.assertEqual(rval, result)
rval = dirtyjson.loads("""{'key': 'value', 'k': 'v'}""")
self.assertEqual(rval, result)
rval = dirtyjson.loads("""{key: 'value', k: 'v'}""")
self.assertEqual(rval, result)
rval = dirtyjson.loads("""{key: 'value', k: 'v',}""")
self.assertEqual(rval, result)
def test_scan_error(self):
for t in (u, b):
try:
dirtyjson.loads(t('{"asdf": "'))
except dirtyjson.Error:
err = sys.exc_info()[1]
else:
self.fail('Expected JSONDecodeError')
self.assertEqual(err.lineno, 1)
self.assertEqual(err.colno, 10)
def test_object_keys(self):
result = {"key": "value", "k": "v"}
rval = dirtyjson.loads("""{"key": "value", "k": "v"}""")
self.assertEqual(rval, result)
rval = dirtyjson.loads("""{'key': 'value', 'k': 'v'}""")
self.assertEqual(rval, result)
rval = dirtyjson.loads("""{key: 'value', k: 'v'}""")
self.assertEqual(rval, result)
rval = dirtyjson.loads("""{key: 'value', k: 'v',}""")
self.assertEqual(rval, result)
def test_array_decoder_issue46(self):
# http://code.google.com/p/dirtyjson/issues/detail?id=46
for doc in [u'[,]', '[,]']:
try:
dirtyjson.loads(doc)
except dirtyjson.Error:
e = sys.exc_info()[1]
self.assertEqual(e.pos, 1)
self.assertEqual(e.lineno, 1)
self.assertEqual(e.colno, 2)
except Exception:
e = sys.exc_info()[1]
self.fail("Unexpected exception raised %r %s" % (e, e))
else:
self.fail("Unexpected success parsing '[,]'")
def test_array_decoder_issue46(self):
# http://code.google.com/p/dirtyjson/issues/detail?id=46
for doc in [u'[,]', '[,]']:
try:
dirtyjson.loads(doc)
except dirtyjson.Error:
e = sys.exc_info()[1]
self.assertEqual(e.pos, 1)
self.assertEqual(e.lineno, 1)
self.assertEqual(e.colno, 2)
except Exception:
e = sys.exc_info()[1]
self.fail("Unexpected exception raised %r %s" % (e, e))
else:
self.fail("Unexpected success parsing '[,]'")
async def login(self):
async with self._session.get('http://{}/html/login/index.html'.format(
self._host)) as resp:
assert resp.status == 200, "Response status code for fetching index is {}".format(
file, resp.status)
re_res = re.search(r'[0-9a-zA-Z]{64}', await resp.text())
if re_res:
challenge = re_res.group(0)
encrypted_password = hashlib.sha256('{}:{}'.format(
challenge, self._password).encode()).hexdigest()
# noinspection SpellCheckingInspection
request_data = {
'password': encrypted_password,
'csrf_token': 'nulltoken',
'showpw': '0',
'challengev': challenge,
}
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'X-Requested-With': 'XMLHttpRequest',
'Referer': 'http://{}/html/login/index.html'.format(self._host),
}
async with self._session.post('http://{}/data/Login.json'.format(
self._host),
data=request_data,
headers=headers) as resp:
content = await resp.read()
parsed = json.loads(content)
data = self.parse_typed_dict(parsed)
assert data[
'login'] == 'success', "Login wasn't successful: {}".format(
data)
derived_key = hashlib.pbkdf2_hmac(
'sha1',
hashlib.sha256(self._password.encode()).hexdigest().encode(),
challenge[0:16].encode(), 1000, 16).hex()
cookies = SimpleCookie()
cookies['challengev'] = challenge
cookies['challengev']['domain'] = self._host
cookies['derivedk'] = derived_key
cookies['derivedk']['domain'] = self._host
self._session.cookie_jar.update_cookies(cookies)
for cookie in self._session.cookie_jar:
self.logger.info(cookie)
if self._cookie_persistent_path:
try:
self._session.cookie_jar.save(self._cookie_persistent_path)
except Exception as e:
self.logger.error("Storing cookies failed", exc_info=True)
def load_json(fname):
with open(fname, 'r') as read_file:
input_json = read_file.read()
# Keep the javascript preamble to reattach later
preamble = input_json[0:15]
data = dirtyjson.loads(input_json[15:])
return (preamble, data)
def test_ignore_single_line_comments(self):
s = """
// here are some comments
{
// comments inside too
test: 1, // and at the end of lines
'aack': 0x80,
"bar": [ // even inside arrays
1,
2,
3, // and after trailing commas
],
more: { // and inside objects
once: true,
twice: false,
three_times3: null // and at the end
}
}
console.log(b);
"""
rval = dirtyjson.loads(s)
self.assertEqual(
rval, {
'test': 1,
'aack': 0x80,
'bar': [1, 2, 3],
'more': {
'once': True,
'twice': False,
'three_times3': None
}
})
def _invoke_serverless(self, stage='', local=False):
name = _get_serverless_name(self.config, self.target)
command = 'serverless invoke'
if local:
command += ' local'
if stage:
command += ' --stage %s' % stage
logger.debug("Invoking via 'serverless'")
command += ' --log --verbose --function %s' % name
output, retcode = next(run(command, out='oneshot'))
logger.debug("Invocation logs from 'serverless':\n%s", output)
if retcode != 0:
error = RuntimeError('Invocation failed on Serverless: %s' %
command)
try:
details = dirtyjson.loads(output)
except ValueError: # No JSON in the output
details = {}
if isinstance(details, dict):
details = dict(details)
elif isinstance(details, list):
details = list(details)
error.logs = output
error.details = details
raise error
return output, None
def dirty_json_or_none(text):
# type: (Text) -> dict
try:
return dirtyjson.loads(text)
except dirtyjson.error.Error:
# logger.warning('^^^^^^^^^^ malformed json text: %s', text)
malformed_json_text.append(text)
return None
def test_not_at_beginning(self):
s = """
// here are some comments
var a = 1; // here is a line of regular JS
var b = {test: 1, 'aack': 0x80, "bar": [1, 2, 3]};
console.log(b);
"""
first_object_index = s.index('{')
rval = dirtyjson.loads(s, start_index=first_object_index)
self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]})
rval = dirtyjson.loads(s, start_index=first_object_index + 1, search_for_first_object=True)
self.assertEqual(rval, [1, 2, 3])
rval = dirtyjson.loads(s, search_for_first_object=True)
self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]})
def parse(self, lint_data):
messages = set()
for msgdata in json.loads(lint_data).get("Issues", []):
try:
path = msgdata["Pos"]["Filename"]
line = msgdata["Pos"]["Line"]
msgbody = msgdata["FromLinter"] + ": " + msgdata["Text"]
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print("Invalid message: {0}".format(msgdata))
return messages
def test_not_at_beginning(self):
s = """
// here are some comments
var a = 1; // here is a line of regular JS
var b = {test: 1, 'aack': 0x80, "bar": [1, 2, 3]};
console.log(b);
"""
first_object_index = s.index('{')
rval = dirtyjson.loads(s, start_index=first_object_index)
self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]})
rval = dirtyjson.loads(s,
start_index=first_object_index + 1,
search_for_first_object=True)
self.assertEqual(rval, [1, 2, 3])
rval = dirtyjson.loads(s, search_for_first_object=True)
self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]})
def search(self, search_strings, age=0, ep_obj=None, **kwargs):
"""
Search a provider and parse the results.
:param search_strings: A dict with mode (key) and the search value (value)
:param age: Not used
:param ep_obj: Not used
:returns: A list of search results (structure)
"""
results = []
if self.custom_url:
if not validators.url(self.custom_url):
log.warning('Invalid custom url: {0}', self.custom_url)
return results
self.url = self.custom_url
self.urls = {
'login': urljoin(self.url, '/torrents/'),
'search': urljoin(self.url, '/t.json'),
'download': urljoin(self.url, '/download.php/')
}
if not self.login():
return results
for mode in search_strings:
log.debug('Search mode: {0}', mode)
for search_string in search_strings[mode]:
if mode != 'RSS':
log.debug('Search string: {search}',
{'search': search_string})
search_string = '+'.join(search_string.split())
params = dict({'q': search_string}, **self.categories[mode])
response = self.session.get(self.urls['search'], params=params)
if not response or not response.text:
log.debug('No data returned from provider')
continue
try:
jdata = djson.loads(response.text)
except ValueError as error:
log.error(
"Couldn't deserialize JSON document. Error: {0!r}",
error)
continue
results += self.parse(jdata, mode)
return results
def parse(self, lint_data):
messages = set()
for error_stanza in json.loads(lint_data):
messages.add((
error_stanza.get("file", None),
error_stanza.get("line", None),
error_stanza.get("message", None),
))
return messages
def parse(self, lint_data):
messages = set()
for msgdata in json.loads(lint_data):
try:
path = msgdata['path']
line = msgdata['line']
msgbody = msgdata['linter'] + ': ' + msgdata['message']
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print('Invalid message: {0}'.format(msgdata))
return messages
async def heartbeat(self) -> bool:
# We shouldn't have caching issues, but maybe the speedport interprets the params
params = {'_time': int(time.time()), '_rand': random.randint(1, 1000)}
async with self._session.get('http://{}/data/heartbeat.json'.format(
self._host),
params=params) as resp:
assert resp.status == 200, "Response status code for heartbeat is {}".format(
resp.status)
raw_data = json.loads(await resp.text())
data = self.parse_typed_dict(raw_data)
return data['loginstate'] == '1'
def parse(self, lint_data):
messages = set()
for msgdata in json.loads(lint_data):
try:
path = msgdata["path"]
line = msgdata["line"]
msgbody = msgdata["linter"] + ": " + msgdata["message"]
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print("Invalid message: {0}".format(msgdata))
return messages
def fetch_public_key(repo):
"""Download RSA public key Travis will use for this repo.
Travis API docs: http://docs.travis-ci.com/api/#repository-keys
"""
keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo)
data = json.loads(urlopen(keyurl).read().decode())
if 'key' not in data:
errmsg = "Could not find public key for repo: {}.\n".format(repo)
errmsg += "Have you already added your GitHub repo to Travis?"
raise ValueError(errmsg)
return data['key']
def parse(self, lint_data):
messages = set()
for file_path, output in lint_data:
for msgdata in json.loads(output):
try:
path = file_path
line = msgdata['line']
msgbody = msgdata['message']
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print('Invalid message: {0}'.format(msgdata))
return messages
def parse(self, lint_data):
messages = set()
for filedata in json.loads(lint_data):
if filedata.get("file") and filedata.get("messages"):
path = filedata["file"]
for msgdata in filedata["messages"]:
try:
line = msgdata["line"]
msgbody = msgdata["message"]
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print("Invalid message: {0}".format(msgdata))
return messages
def parse(self, lint_data):
messages = set()
for filedata in json.loads(lint_data):
if filedata.get('messages'):
for msgdata in filedata['messages']:
try:
path = filedata['filePath']
line = msgdata['line']
msgbody = msgdata['message']
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print('Invalid message: {0}'.format(msgdata))
return messages
def test_truncated_input(self):
test_cases = [
('', 'Expecting value', 0),
('[', "Expecting value or ']'", 1),
('[42', "Expecting ',' delimiter", 3),
('[42,', 'Expecting value', 4),
('["', 'Unterminated string starting at', 1),
('["spam', 'Unterminated string starting at', 1),
('["spam"', "Expecting ',' delimiter", 7),
('["spam",', 'Expecting value', 8),
('{', 'Expecting property name', 1),
('{"', 'Unterminated string starting at', 1),
('{"spam', 'Unterminated string starting at', 1),
('{"spam"', "Expecting ':' delimiter", 7),
('{"spam":', 'Expecting value', 8),
('{"spam":42', "Expecting ',' delimiter", 10),
('{"spam":42,', 'Expecting property name',
11),
('"', 'Unterminated string starting at', 0),
('"spam', 'Unterminated string starting at', 0),
('[,', "Expecting value", 1),
]
for data, msg, idx in test_cases:
try:
dirtyjson.loads(data)
except dirtyjson.Error:
e = sys.exc_info()[1]
self.assertEqual(
e.msg[:len(msg)],
msg,
"%r doesn't start with %r for %r" % (e.msg, msg, data))
self.assertEqual(
e.pos, idx,
"pos %r != %r for %r" % (e.pos, idx, data))
except Exception:
e = sys.exc_info()[1]
self.fail("Unexpected exception raised %r %s" % (e, e))
else:
self.fail("Unexpected success parsing '%r'" % (data,))
def _extract_dirtyjson_definition(match_html, match_data, r, key, flags=0):
m = re.search(r, match_html, flags)
if m is None:
return
dirtyjson_string = m.group(1)
fixed_dirtyjson_string = fix_dirtyjson(dirtyjson_string)
value = dirtyjson.loads(fixed_dirtyjson_string)
if value is None:
return
match_data[key] = value
def parse(self, lint_data):
messages = set()
msgdata = ''
for line in lint_data.split('\n'):
try:
msgdata = json.loads(line)
path = msgdata['location']['file']
line = msgdata['location']['line']
msgbody = msgdata['message']
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print('Invalid message: {0}'.format(msgdata))
return messages
def parse(self, lint_data):
messages = set()
for filename, msgs in json.loads(lint_data).items():
if msgs:
for msgdata in msgs:
try:
path = filename
line = msgdata['line']
msgbody = msgdata['message']
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print('Invalid message: {0}'.format(msgdata))
return messages
def parse(self, lint_data):
messages = set()
msgdata = ""
for line in lint_data.split("\n"):
try:
msgdata = json.loads(line)
path = msgdata["location"]["file"]
line = msgdata["location"]["line"]
msgbody = msgdata["message"]
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print("({0}) Invalid message: {1}".format(type(self).__name__, msgdata))
return messages
def parse(self, lint_data):
messages = set()
for msgdata in json.loads(lint_data).get('messages'):
try:
path = msgdata['location']['path']
line = msgdata['location']['line']
msgbody = '{0}: {1} ({2})'.format(msgdata['source'],
msgdata['message'],
msgdata['code'])
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print('Invalid message: {0}'.format(msgdata))
return messages
def parse_bad_contents(contents):
"""Recursively attempts to fix JSON until it succeeds.
Args:
contents - String contents to try parsing.
Returns:
Parsed JSON result.
"""
try:
return dirtyjson.loads(contents)
except Exception as e:
print(contents)
return parse_bad_contents(repair(e, contents))
async def fetch_data(self, file: str):
with self.FETCH_EXCEPTIONS.labels(file).count_exceptions():
with self.FETCH_TIME.labels(file).time():
async with self._session.get('http://{}/data/{}.json'.format(
self._host, file)) as resp:
assert resp.status == 200, "Response status code for {} is {}".format(
file, resp.status)
raw = await resp.text()
try:
return json.loads(raw)
except json.error.Error as e:
self.logger.error("Error unmarshalling received data",
exc_info=True)
raise
def parse(self, lint_data):
messages = set()
for file_path, output in lint_data:
try:
for msgdata in json.loads(output):
try:
path = file_path
line = msgdata["line"]
msgbody = msgdata["message"]
messages.add((path, line, msgbody))
except (ValueError, KeyError):
print("Invalid message: {0}".format(msgdata))
except json.error.Error:
print("Invalid message: {0}".format(output))
return messages
def parse(self, lint_data):
messages = set()
for file_path, output in lint_data:
try:
for error_stanza in json.loads(output):
messages.add((
file_path,
error_stanza.get("line", None),
error_stanza.get("message", None),
))
except Exception:
traceback.print_exc()
return messages
def search(self, search_strings, age=0, ep_obj=None, **kwargs):
"""
Search a provider and parse the results.
:param search_strings: A dict with mode (key) and the search value (value)
:param age: Not used
:param ep_obj: Not used
:returns: A list of search results (structure)
"""
results = []
if not self.login():
return results
for mode in search_strings:
log.debug('Search mode: {0}', mode)
for search_string in search_strings[mode]:
if mode != 'RSS':
log.debug('Search string: {search}',
{'search': search_string})
search_string = '+'.join(search_string.split())
params = dict({'q': search_string}, **self.categories[mode])
response = self.session.get(self.urls['search'], params=params)
if not response or not response.text:
log.debug('No data returned from provider')
continue
try:
jdata = djson.loads(response.text)
except ValueError as error:
log.error("Couldn't deserialize JSON document. Error: {0!r}", error)
continue
results += self.parse(jdata, mode)
return results
def test_ignore_single_line_comments(self):
s = """
// here are some comments
{
// comments inside too
test: 1, // and at the end of lines
'aack': 0x80,
"bar": [ // even inside arrays
1,
2,
3, // and after trailing commas
],
more: { // and inside objects
once: true,
twice: false,
three_times3: null // and at the end
}
}
console.log(b);
"""
rval = dirtyjson.loads(s)
self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3], 'more': {'once': True, 'twice': False, 'three_times3': None}})
def test_ignore_inline_comments(self):
s = """
/* here are some comments
* that should all be skipped
* right up until the terminator */ {
/* comments inside too */
test: 1, /* and at the end of lines */
'aack': 0x80,
"bar": [ // even inside arrays
1,
2,
3, // and after trailing commas
],
/* comment this block out
more: { // and inside objects
once: true,
twice: false,
three_times3: null // and at the end
} */
}
console.log(b);
"""
rval = dirtyjson.loads(s)
self.assertEqual(rval, {'test': 1, 'aack': 0x80, 'bar': [1, 2, 3]})
def test_degenerates_allow(self):
for inf in (PosInf, NegInf):
self.assertEqual(dirtyjson.loads(json.dumps(inf)), inf)
# Python 2.5 doesn't have math.isnan
nan = dirtyjson.loads(json.dumps(NaN))
self.assertTrue((0 + nan) != nan)
def test_empty_strings(self):
self.assertEqual(dirtyjson.loads('""'), "")
self.assertEqual(dirtyjson.loads(u'""'), u"")
self.assertEqual(dirtyjson.loads('[""]'), [""])
self.assertEqual(dirtyjson.loads(u'[""]'), [u""])
def test_parse(self):
# test in/out equivalence and parsing
res = dirtyjson.loads(JSON)
out = json.dumps(res)
self.assertEqual(res, dirtyjson.loads(out))
def loads(self, s, **kw):
sio = StringIO(s)
res = dirtyjson.loads(s, **kw)
self.assertEqual(res, dirtyjson.load(sio, **kw))
return res
def test_float(self):
rval = dirtyjson.loads('1', parse_int=float)
self.assertTrue(isinstance(rval, float))
self.assertEqual(rval, 1.0)
def test_decoder_optimizations(self):
rval = dirtyjson.loads('{ "key" : "value" , "k":"v" }')
self.assertEqual(rval, {"key": "value", "k": "v"})
def test_floats(self):
for num in [1617161771.7650001, math.pi, math.pi ** 100,
math.pi ** -100, 3.1]:
self.assertEqual(dirtyjson.loads(json.dumps(num)), num)
self.assertEqual(dirtyjson.loads(text_type(json.dumps(num))), num)
def test_ints(self):
for num in [1, long_type(1), 1 << 32, 1 << 64]:
self.assertEqual(dirtyjson.loads(json.dumps(num)), num)
self.assertEqual(dirtyjson.loads(text_type(json.dumps(num))), num)
def test_decimal(self):
rval = dirtyjson.loads('1.1', parse_float=decimal.Decimal)
self.assertTrue(isinstance(rval, decimal.Decimal))
self.assertEqual(rval, decimal.Decimal('1.1'))
def test_empty_strings_with_single_quotes(self):
self.assertEqual(dirtyjson.loads("''"), "")
self.assertEqual(dirtyjson.loads(u"''"), u"")
self.assertEqual(dirtyjson.loads("['']"), [""])
self.assertEqual(dirtyjson.loads(u"['']"), [u""])
