def is_valid_json(json: str) -> bool:
try:
ujson.decode(json)
except:
return False
else:
return True
def test_encodeDictConversion(self):
input = { "k1": 1, "k2": 2, "k3": 3, "k4": 4 }
output = ujson.encode(input)
self.assertEquals(input, json.loads(output))
self.assertEquals(input, ujson.decode(output))
self.assertEquals(input, ujson.decode(output))
pass
def test_dynamic_by_name(self):
VISITOR_ID_1 = uuid.uuid4().hex
EVENT_1 = "Event 1 %s" % uuid.uuid4().hex
EVENT_2 = "Event 2 %s" % uuid.uuid4().hex
yield self.post_event(VISITOR_ID_1, EVENT_1)
yield self.post_event(VISITOR_ID_1, EVENT_2)
events = yield self.get_event_dict()
event_id_1 = events[EVENT_1]
event_id_2 = events[EVENT_2]
qs = urlencode([("event", EVENT_1), ("event", EVENT_2)])
result = yield request(
"GET",
"%s/funnel?%s" % (self.url, qs),
username=self.username,
password=self.password)
self.assertEqual(result.code, 200)
returned_event_ids = ujson.decode(result.body)["event_ids"]
self.assertTrue(event_id_1 in returned_event_ids)
self.assertTrue(event_id_2 in returned_event_ids)
funnel = ujson.decode(result.body)["funnel"]
unique_funnel = ujson.decode(result.body)["unique_funnel"]
self.assertEqual(funnel[0][0], event_id_1)
self.assertEqual(funnel[1][0], event_id_2)
self.assertEqual(funnel[0][1], 1)
self.assertEqual(funnel[1][1], 1)
self.assertEqual(unique_funnel[0][0], event_id_1)
self.assertEqual(unique_funnel[1][0], event_id_2)
self.assertEqual(unique_funnel[0][1], 1)
self.assertEqual(unique_funnel[1][1], 1)
def test_decodeArrayOnlyCommaFail(self):
input = "[,]"
try:
ujson.decode(input)
except ValueError:
pass
else:
assert False, "expected ValueError"
def test_decodeArrayUnmatchedBracketFail(self):
input = "[]]"
try:
ujson.decode(input)
except ValueError:
pass
else:
assert False, "expected ValueError"
def test_decodeVeryTooBigValue(self):
try:
input = "18446744073709551616"
ujson.decode(input)
except ValueError:
pass
else:
assert False, "expected ValueError"
def test_decodeVeryTooSmallValue(self):
try:
input = "-90223372036854775809"
ujson.decode(input)
except ValueError:
pass
else:
assert False, "expected ValueError"
def test_decodeObjectDepthTooBig(self):
input = '{' * (1024 * 1024)
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
return
assert False, "Wrong exception"
def test_decodeWithTrailingNonWhitespaces(self):
try:
input = "{}\n\t a"
ujson.decode(input)
except ValueError:
pass
else:
assert False, "expected ValueError"
def test_decodeTooBigValue(self):
try:
input = "9223372036854775808"
ujson.decode(input)
except ValueError as e:
pass
else:
assert False, "expected ValueError"
def test_decodeStringUntermEscapeSequence(self):
input = '"TESTING\\"'
try:
ujson.decode(input)
assert False, "Expected exception!"
except (ValueError):
return
assert False, "Wrong exception"
def test_goodreads(self):
def autogenerate_goodreads(*args, **kwargs):
autogenerator = delta.Autogenerator(paths="books", includes="books/id")
return [x.data for x in autogenerator(*args, **kwargs)]
from ujson import decode
old = decode(read("%s/goodreads/old.json" % DATAPATH))
new = decode(read("%s/goodreads/new.json" % DATAPATH))
self.assertEqual(autogenerate_goodreads(old, new), [])
def test_decodeBrokenObjectStart(self):
input = "{"
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
return
assert False, "Wrong exception"
def test_decodeJibberish(self):
input = "fdsa sda v9sa fdsa"
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
return
assert False, "Wrong exception"
def test_decodeStringUnterminated(self):
input = "\"TESTING"
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
return
assert False, "Wrong exception"
def test_decodeStringBadEscape(self):
input = "\"TESTING\\\""
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
return
assert False, "Wrong exception"
def test_decodeNullBroken(self):
input = "n"
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
return
assert False, "Wrong exception"
def test_decodeDictWithNoColonOrValue(self):
input = '{{{{"key"}}}}'
try:
ujson.decode(input)
assert False, "Expected exception!"
except (ValueError):
return
assert False, "Wrong exception"
def test_decodeDictWithNoValue(self):
input = "{{{{\"key\":}}}}"
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
return
assert False, "Wrong exception"
def test_decodeBigEscape(self):
for x in range(10):
if six.PY3:
base = '\u00e5'.encode('utf-8')
quote = "\"".encode()
else:
base = "\xc3\xa5"
quote = "\""
input = quote + (base * 1024 * 1024 * 2) + quote
ujson.decode(input)
def test_encodeLongNegConversion(self):
input = -9223372036854775808
output = ujson.encode(input)
outputjson = json.loads(output)
outputujson = ujson.decode(output)
self.assertEqual(input, json.loads(output))
self.assertEqual(output, json.dumps(input))
self.assertEqual(input, ujson.decode(output))
def test_decodeBrokenListLeakTest(self):
input = '[[[true'
for x in xrange(1000):
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError):
continue
assert False, "Wrong exception"
def test_decodeBrokenDictKeyTypeLeakTest(self):
input = '{{1337:""}}'
for x in xrange(1000):
try:
ujson.decode(input)
assert False, "Expected exception!"
except(ValueError) as e:
continue
assert False, "Wrong exception"
def test_hooks_exceptionHandling(self):
class TestHookException(Exception):
pass
call_counters = {
'pre_encode_hook': 0,
'string_hook': 0,
'object_hook': 0,
}
def pre_encode_hook(_unused_obj):
call_counters['pre_encode_hook'] += 1
raise TestHookException()
def string_hook(_unused_obj):
call_counters['string_hook'] += 1
raise TestHookException()
def object_hook(_unused_obj):
call_counters['object_hook'] += 1
raise TestHookException()
input = {
'foo': 1,
'bar': {
'a': 'a',
'b': 'b',
},
}
json = """
{
"foo": 1,
"bar": {
"a": "a",
"b": "b"
}
}
"""
with self.assertRaises(TestHookException):
ujson.encode(input, pre_encode_hook=pre_encode_hook)
with self.assertRaises(TestHookException):
ujson.decode(json, string_hook=string_hook)
with self.assertRaises(TestHookException):
ujson.decode(json, object_hook=object_hook)
# Test not only that the exception is raised, but also that the hook
# is called only once. I.e. that the low-level code detects the error
# and stops the iteration process after the first call.
self.assertEqual(call_counters['pre_encode_hook'], 1)
self.assertEqual(call_counters['string_hook'], 1)
self.assertEqual(call_counters['object_hook'], 1)
def test_decodeTooSmallValue(self):
if ujson.bigint_supported:
for i in range(30):
input = -90223372036854775809 - i
obj = ujson.decode('%s' % input)
self.assertEqual(input, obj)
for i in range(30):
input = -41255008296960273580 - i
obj = ujson.decode('%s' % input)
self.assertEqual(input, obj)
else:
input = "-90223372036854775809"
self.assertRaises(ValueError, ujson.decode, input)
def test_messages_since(self):
conn = self._make_one()
name = conn.create_queue()
# add test message
r1 = conn.post(url=name, data=u'Hello 1')
key1 = ujson.decode(r1.text)[u'messages'][0][u'key']
r2 = conn.post(url=name, data=u'Hello 2')
key2 = ujson.decode(r2.text)[u'messages'][0][u'key']
messages = conn.messages(name, since=key2)
self.assertEqual(len(messages), 0)
messages = conn.messages(name, since=key1)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0][u'body'], u'Hello 2')
def test_doublePrecisionTest(self):
input = 30.012345678901234
output = ujson.encode(input, double_precision = 15)
self.assertEqual(input, json.loads(output))
self.assertEqual(input, ujson.decode(output))
output = ujson.encode(input, double_precision = 9)
self.assertEqual(round(input, 9), json.loads(output))
self.assertEqual(round(input, 9), ujson.decode(output))
output = ujson.encode(input, double_precision = 3)
self.assertEqual(round(input, 3), json.loads(output))
self.assertEqual(round(input, 3), ujson.decode(output))
def action():
redisqueue = j.clients.credis.getRedisQueue("127.0.0.1", 9999, "logs")
redisqueueEco = j.clients.credis.getRedisQueue("127.0.0.1", 9999, "eco")
OSISclient = j.core.osis.client
OSISclientLogger = j.core.osis.getClientForCategory(OSISclient, "system", "log")
OSISclientEco = j.core.osis.getClientForCategory(OSISclient, "system", "eco")
log = None
path = "%s/apps/processmanager/loghandling/"%j.dirs.baseDir
if j.system.fs.exists(path=path):
loghandlingTE = j.core.taskletengine.get(path)
log=redisqueue.get_nowait()
# j.core.grid.logger.osis = OSISclientLogger
else:
loghandlingTE = None
eco = None
path = "%s/apps/processmanager/eventhandling"%j.dirs.baseDir
if j.system.fs.exists(path=path):
eventhandlingTE = j.core.taskletengine.get(path)
eco=redisqueueEco.get_nowait()
else:
eventhandlingTE = None
out=[]
while log<>None:
log2=json.decode(log)
log3 = j.logger.getLogObjectFromDict(log2)
log4= loghandlingTE.executeV2(logobj=log3)
if log4<>None:
out.append(log4.__dict__)
if len(out)>500:
OSISclientLogger.set(out)
out=[]
log=redisqueue.get_nowait()
if len(out)>0:
OSISclientLogger.set(out)
while eco<>None:
eco2=json.decode(eco)
eco2["epoch"] = int(time.time())
eco3 = j.errorconditionhandler.getErrorConditionObject(ddict=eco2)
eco4= eventhandlingTE.executeV2(eco=eco3)
if hasattr(eco4,"tb"):
eco4.__dict__.pop("tb")
OSISclientEco.set(eco4.__dict__)
eco=redisqueueEco.get_nowait()
def test_encodeNullCharacter(self):
input = "31337 \x00 1337"
output = ujson.encode(input)
self.assertEqual(input, json.loads(output))
self.assertEqual(output, json.dumps(input))
self.assertEqual(input, ujson.decode(output))
input = "\x00"
output = ujson.encode(input)
self.assertEqual(input, json.loads(output))
self.assertEqual(output, json.dumps(input))
self.assertEqual(input, ujson.decode(output))
self.assertEqual('" \\u0000\\r\\n "', ujson.dumps(u" \u0000\r\n "))
def test_create(self):
VISITOR_ID_1 = uuid.uuid4().hex
EVENT_1 = "Event 1 %s" % uuid.uuid4().hex
EVENT_2 = "Event 2 %s" % uuid.uuid4().hex
yield self.post_event(VISITOR_ID_1, EVENT_1)
yield self.post_event(VISITOR_ID_1, EVENT_2)
events = yield self.get_event_dict()
event_id_1 = events[EVENT_1]
event_id_2 = events[EVENT_2]
FUNNEL_NAME = uuid.uuid4().hex
DESCRIPTION = uuid.uuid4().hex
result = yield request(
"POST",
"%s/funnel/%s" % (self.url, FUNNEL_NAME),
username=self.username,
password=self.password,
data=[
("description", DESCRIPTION),
("event_id",event_id_1),
("event_id",event_id_2)])
self.assertEqual(result.code, 201)
result = yield request(
"GET",
"%s/funnel/%s" % (self.url, FUNNEL_NAME),
username=self.username,
password=self.password)
self.assertEqual(result.code, 200)
returned_description = ujson.decode(result.body)["description"]
self.assertEqual(DESCRIPTION, returned_description)
returned_event_ids = ujson.decode(result.body)["event_ids"]
self.assertTrue(event_id_1 in returned_event_ids)
self.assertTrue(event_id_2 in returned_event_ids)
funnel = ujson.decode(result.body)["funnel"]
unique_funnel = ujson.decode(result.body)["unique_funnel"]
self.assertEqual(funnel[0][0], event_id_1)
self.assertEqual(funnel[1][0], event_id_2)
self.assertEqual(funnel[0][1], 1)
self.assertEqual(funnel[1][1], 1)
self.assertEqual(unique_funnel[0][0], event_id_1)
self.assertEqual(unique_funnel[1][0], event_id_2)
self.assertEqual(unique_funnel[0][1], 1)
self.assertEqual(unique_funnel[1][1], 1)
result = yield request(
"DELETE",
"%s/funnel/%s" % (self.url, FUNNEL_NAME),
username=self.username,
password=self.password)
self.assertEqual(result.code, 200)
def test_encodeDoubleNegConversion(self):
input = -math.pi
output = ujson.encode(input)
self.assertEqual(round(input, 5), round(json.loads(output), 5))
self.assertEqual(round(input, 5), round(ujson.decode(output), 5))
def test_encodeStringConversion2(self):
input = "A string \\ / \b \f \n \r \t"
output = ujson.encode(input)
self.assertEqual(input, json.loads(output))
self.assertEqual(output, '"A string \\\\ \\/ \\b \\f \\n \\r \\t"')
self.assertEqual(input, ujson.decode(output))
def test_decodeFromUnicode(self):
input = "{\"obj\": 31337}"
dec1 = ujson.decode(input)
dec2 = ujson.decode(str(input))
self.assertEqual(dec1, dec2)
def test_decodeNumericIntPos(self):
input = "31337"
self.assertEqual(31337, ujson.decode(input))
def test_encodeLongConversion(self):
input = 9223372036854775807
output = ujson.encode(input)
self.assertEqual(input, json.loads(output))
self.assertEqual(output, json.dumps(input))
self.assertEqual(input, ujson.decode(output))
def test_decodeNumericIntExpe(self):
input = "1337e40"
output = ujson.decode(input)
self.assertEqual(output, json.loads(input))
def test_decodeArrayEmpty(self):
input = "[]"
obj = ujson.decode(input)
self.assertEqual([], obj)
def test_decodeArrayOneItem(self):
input = "[31337]"
ujson.decode(input)
def test_decodeBigValue(self):
input = "9223372036854775807"
ujson.decode(input)
def test_encodeArrayOfNestedArrays(self):
input = [[[[]]]] * 20
output = ujson.encode(input)
self.assertEqual(input, json.loads(output))
#self.assertEqual(output, json.dumps(input))
self.assertEqual(input, ujson.decode(output))
def test_encodeUnicodeConversion1(self):
input = "Räksmörgås اسامة بن محمد بن عوض بن لادن"
enc = ujson.encode(input)
dec = ujson.decode(enc)
self.assertEqual(enc, json_unicode(input))
self.assertEqual(dec, json.loads(enc))
def test_encodeFalseConversion(self):
input = False
output = ujson.encode(input)
self.assertEqual(input, json.loads(output))
self.assertEqual(output, json.dumps(input))
self.assertEqual(input, ujson.decode(output))
def test_encodeUnicodeConversion2(self):
input = "\xe6\x97\xa5\xd1\x88"
enc = ujson.encode(input)
dec = ujson.decode(enc)
self.assertEqual(enc, json_unicode(input))
self.assertEqual(dec, json.loads(enc))
def test_encodeControlEscaping(self):
input = "\x19"
enc = ujson.encode(input)
dec = ujson.decode(enc)
self.assertEqual(input, dec)
self.assertEqual(enc, json_unicode(input))
def test_decodeLongUnsignedValue(self):
input = "18446744073709551615"
ujson.decode(input)
def test_decimalDecodeTestPrecise(self):
sut = {'a': 4.56}
encoded = ujson.encode(sut)
decoded = ujson.decode(encoded, precise_float=True)
self.assertEqual(sut, decoded)
def test_decodeArrayDict(self):
input = "{}"
obj = ujson.decode(input)
self.assertEqual({}, obj)
def format_json(json_data):
actor_identities = {} # Where we're going to store our list of actor ids
actor_identities['items'] = []
actor_quantity = len(json_data) # Get the quantity of actors!
#actor_quantity = 2 # For testing!
widgets = [
Percentage(), # Setting how we wan the progress bar to look
' ',
Bar(),
' ',
ETA()
]
scrape_range_ref = actor_quantity + 1
pbar = ProgressBar(
widgets=widgets,
maxval=scrape_range_ref).start() #Prepare the progress bar
progress_iterator = 0
for actor_number in range(
actor_quantity): # There are approx 979 pages we've scraped
current_actor_data = json_data[
actor_number] # Storing the current page in memory
current_actor_name = current_actor_data[
'name'] # The actor's real name
current_actor_roles = current_actor_data['movie_credits'][
'cast'] # The roles the character played
quantity_actor_roles = len(
current_actor_roles) # How many roles they played
current_actor_role_synonyms = [] # Empty list for each actor
current_actor_role_synonyms.append(
current_actor_name
) # We want the entity synonym to include their real name, not just characters:movies
for actor_role in range(quantity_actor_roles):
if (len(current_actor_role_synonyms) > 98):
continue # Can't go over 99 synonyms per value
current_target = current_actor_roles[
actor_role] # Simple reference
if 'character' not in current_target: # Make sure the character field exists!
continue # Skip if it doesn't exist!
else:
if 'title' not in current_target: # Check that the title exists in json!
continue # Skip if the movie title doesn't exist!
else:
actor_character = current_target[
'character'] # Grab their character name
if (len(actor_character) < 2):
continue # If the character's name is less than 2 characters long, skip (likely a blank field)
elif (actor_character.lower() == 'himself'):
continue # If they know the actor's name, they don't need to l
elif (actor_character.lower() == 'herself'):
continue
elif (actor_character.lower() == 'self'):
continue
else:
actor_character = re.sub(
r'\([^()]*\)', '', actor_character
) # Any parenthesised information (uncredited) (self) (himself) etc will be removed from character strings!
#actor_character = re.sub('".*?"', '', actor_character) # Removing any quotation marks in character name. Future improvement could match character_nickname:actor similar to handling / below
actor_character = re.sub('"', '', actor_character)
actor_movie = re.sub(
r'\([^()]*\)', '',
current_target['title']) # Grab the movie name
if (
"/" in actor_character
): # If the actor plays multiple characters it'll be listed as "character secret identity / character super hero"
multiple_characters = actor_character.split(
"/") # Split string into list of characters
for character in multiple_characters:
if (len(current_actor_role_synonyms) > 98):
continue # Can't go over 99 synonyms per value
character.strip() # Remove whitespace
in_entity = character + " in " + actor_movie # "character in movie"
#from_entity = character + " from " + actor_movie # "character from movie"
current_actor_role_synonyms.append(
in_entity) # Add the 'in' synonym
#current_actor_role_synonyms.append(from_entity) # Add the 'from' synonym
else:
actor_character.strip(
) # Strip whitespace from left/right of the string
in_entity = actor_character + " in " + actor_movie # 'in'
#from_entity = actor_character + " from " + actor_movie # 'from'
current_actor_role_synonyms.append(
in_entity) # Add the 'in' synonym
#current_actor_role_synonyms.append(from_entity) # Add the 'from' synonym
#if (len(current_actor_role_synonyms) > 95):
# print(len(current_actor_role_synonyms))
text_to_json = ujson.dumps({
"value": current_actor_name,
"synonyms": current_actor_role_synonyms
}) # Changing the text into json
actor_identities['items'].append(
ujson.decode(text_to_json)) # Append the synonyms to the list
pbar.update(progress_iterator + 1) # Display incremented progress
progress_iterator += 1 # Iterate the progress bar for next iteration
pbar.finish() #Once we've complete the scraping, end the progress bar.
return actor_identities['items']
def test_decodeNumericIntExpeMinus(self):
input = "1.337e-4"
output = ujson.decode(input)
self.assertEqual(output, json.loads(input))
#if (len(current_actor_role_synonyms) > 95):
# print(len(current_actor_role_synonyms))
text_to_json = ujson.dumps({
"value": current_actor_name,
"synonyms": current_actor_role_synonyms
}) # Changing the text into json
actor_identities['items'].append(
ujson.decode(text_to_json)) # Append the synonyms to the list
pbar.update(progress_iterator + 1) # Display incremented progress
progress_iterator += 1 # Iterate the progress bar for next iteration
pbar.finish() #Once we've complete the scraping, end the progress bar.
return actor_identities['items']
if __name__ == '__main__':
with open('popular_people.json') as data_file:
actor_json_data = ujson.load(data_file) # Load actor data in
formatted_json = format_json(
actor_json_data) # Where the majority of the magic happens
wrapped_json = ujson.decode(
"[{\"entries\":" + ujson.encode(formatted_json) +
", \"name\": \"actors\"}]"
) # Wrapping the JSON with dialogflow's preferred formatting
write_json_to_disk(wrapped_json)
def test_numericIntFrcExp(self):
input = "1.337E40"
output = ujson.decode(input)
self.assertEqual(output, json.loads(input))
def test_decimalDecodeTest(self):
sut = {'a': 4.56}
encoded = ujson.encode(sut)
decoded = ujson.decode(encoded)
self.assertAlmostEqual(sut[u'a'], decoded[u'a'])
def test_decodeNullCharacter(self):
input = "\"31337 \\u0000 31337\""
self.assertEqual(ujson.decode(input), json.loads(input))
def test_encodeDecodeLongDecimal(self):
sut = {'a': -528656961.4399388}
encoded = ujson.dumps(sut)
ujson.decode(encoded)
def test_decodeNumericIntNeg(self):
input = "-31337"
self.assertEqual(-31337, ujson.decode(input))
def test_decodeWithTrailingWhitespaces(self):
input = "{}\n\t "
ujson.decode(input)
def helper(expected_output, **encode_kwargs):
output = ujson.encode(input, **encode_kwargs)
self.assertEqual(output, expected_output)
if encode_kwargs.get('escape_forward_slashes', True):
self.assertEqual(input, json.loads(output))
self.assertEqual(input, ujson.decode(output))
def test_decodeSmallValue(self):
input = "-9223372036854775808"
ujson.decode(input)
def test_encodeDecimal(self):
sut = decimal.Decimal("1337.1337")
encoded = ujson.encode(sut, double_precision=100)
decoded = ujson.decode(encoded)
self.assertEqual(decoded, 1337.1337)
def test_encodeArrayOfDoubles(self):
input = [31337.31337, 31337.31337, 31337.31337, 31337.31337] * 10
output = ujson.encode(input)
self.assertEqual(input, json.loads(output))
#self.assertEqual(output, json.dumps(input))
self.assertEqual(input, ujson.decode(output))
