def testEncodeUnicodeBMP(self):
s = '\U0001f42e\U0001f42e\U0001F42D\U0001F42D' # 🐮🐮🐭🐭
encoded = ultrajson.dumps(s)
encoded_json = json.dumps(s)
if len(s) == 4:
self.assertEqual(len(encoded), len(s) * 12 + 2)
else:
self.assertEqual(len(encoded), len(s) * 6 + 2)
self.assertEqual(encoded, encoded_json)
decoded = ultrajson.loads(encoded)
self.assertEqual(s, decoded)
# ultrajson outputs an UTF-8 encoded str object
if PY3:
encoded = ultrajson.dumps(s, ensure_ascii=False)
else:
encoded = ultrajson.dumps(s, ensure_ascii=False).decode("utf-8")
# json outputs an unicode object
encoded_json = json.dumps(s, ensure_ascii=False)
self.assertEqual(len(encoded), len(s) + 2) # original length + quotes
self.assertEqual(encoded, encoded_json)
decoded = ultrajson.loads(encoded)
self.assertEqual(s, decoded)
def test_ReadBadObjectSyntax(self):
try:
ultrajson.loads('{"age", 44}')
except ValueError:
pass
else:
assert False, "expected ValueError"
def test_decodeArrayFaultyUnicode(self):
try:
ultrajson.loads('[18446744073709551616]')
except ValueError:
pass
else:
assert False, "expected ValueError"
def dataReceived(self, data):
"""
Take data coming from the server and put it into the queue for the
client
We are also doing a bit of checking to combine auth messages.
"""
if self.data_buffer:
data = self.data_buffer + data
self.data_buffer = ''
try:
data = json.loads(data)
except ValueError:
# We probably just didn't get all of it
self.data_buffer = data
return
self.outgoing_queue.put(json.dumps(data))
jData = data
if "auth" in jData and not self.authPacket:
self.authPacket = jData
elif "auth" in jData and self.authPacket:
self.authPacket.update(jData['auth'])
self.hud_queue.put(json.dumps(self.authPacket))
else:
self.hud_queue.put(json.dumps(data))
def dataFromTeeReceived(self, data):
jData = json.loads(data)
if "auth" in jData:
if 'username' in jData['auth']:
self.user = jData['auth']['username']
Controller.send_message(self.user, data)
self.hud_queue.get().addCallback(self.dataFromTeeReceived)
def testEncodeSymbols(self):
s = '\u273f\u2661\u273f' # ✿♡✿
encoded = ultrajson.dumps(s)
encoded_json = json.dumps(s)
self.assertEqual(len(encoded), len(s) * 6 + 2) # 6 characters + quotes
self.assertEqual(encoded, encoded_json)
decoded = ultrajson.loads(encoded)
self.assertEqual(s, decoded)
# ultrajson outputs an UTF-8 encoded str object
if PY3:
encoded = ultrajson.dumps(s, ensure_ascii=False)
else:
encoded = ultrajson.dumps(s, ensure_ascii=False).decode("utf-8")
# json outputs an unicode object
encoded_json = json.dumps(s, ensure_ascii=False)
self.assertEqual(len(encoded), len(s) + 2) # original length + quotes
self.assertEqual(encoded, encoded_json)
decoded = ultrajson.loads(encoded)
self.assertEqual(s, decoded)
def send_message(user, msg):
data = json.loads(msg)
if 'display' in data.keys():
current = Controller.cached_details[user]
current.update(data['display'])
Controller.cached_details[user] = current
if user in Controller.users:
try:
Controller.users[user](msg)
except Exception as e:
print e
client.captureException(sys.exc_info())
def test_encodeBlist(self):
try:
from blist import blist
except ImportError:
return
b = blist(list(range(10)))
c = ultrajson.dumps(b)
d = ultrajson.loads(c)
self.assertEqual(10, len(d))
for x in range(10):
self.assertEqual(x, d[x])
def handle_data(self, json_data):
if self.data_buffer:
data = self.data_buffer + json_data
self.data_buffer = ''
try:
data = json.loads(json_data)
except ValueError:
# We probably just didn't get all of it
self.data_buffer = json_data
return
for ret_key in data:
if ret_key in self.input_handlers:
self.input_handlers[ret_key](data[ret_key])
def read_file(battle_ids):
for battle_id in battle_ids:
with gzip.open("./data/{0}.json.gz".format(battle_id), "rt", encoding="utf-8") as f:
try:
raw = f.read(-1)
if len(raw) == 0:
continue
battle = json.loads(raw)
yield battle
except Exception as e:
print('=========================')
print(e)
print(battle)
traceback.print_exc()
def create_config_dir(self):
config_dirname = os.path.dirname(self.config_path)
if not os.path.exists(config_dirname):
os.makedirs(os.path.basename(config_dirname))
# Real configuration file:
if not os.path.exists(self.config_path):
self.config = {}
with open(self.config_path, 'w') as file_obj:
file_obj.write(ultrajson.dumps(self.config))
else:
with open(self.config_path) as file_obj:
self.config = ultrajson.loads(file_obj.read())
# Virtual dir:
cfg_dir = ConfigDir(self.control_dir, 'config', self.config, self)
self.control_dir.add_child(cfg_dir)
self.config_dir = cfg_dir
return cfg_dir
def get_params():
parser = argparse.ArgumentParser(description='store graph')
parser.add_argument("--redis-service", type=str, default='localhost:6379')
return parser.parse_args()
if __name__ == "__main__":
args = get_params()
graph_api = ginf.GinfGraph(args.redis_service)
for i,line in enumerate(sys.stdin):
# Error handling for bad records in GNIP data
if line.strip() == '':
continue
if json.loads(line).keys() == ['info']:
continue
try:
obj = ginf.io.gnip(json.loads(line))
graph_api.update(obj)
except KeyboardInterrupt:
raise
except:
print >> sys.stderr, 'error @ %s' % line
if not i % 1000:
print >> sys.stderr, '%d records completed' % i
graph_api.execute(force=True)
def test_ReadFalse(self):
self.assertEqual(False, ultrajson.loads("false"))
def test_ReadTrue(self):
self.assertEqual(True, ultrajson.loads("true"))
from ginf import GinfAPI
from ginf.helpers import safeget
def get_params():
parser = argparse.ArgumentParser(description='predict location')
parser.add_argument("--redis-service", type=str, default='localhost:6379')
parser.add_argument("--always-predict", action='store_true')
parser.add_argument("--always-dirty", action='store_true')
return parser.parse_args()
if __name__ == "__main__":
args = get_params()
ginf_api = GinfAPI(args.redis_service)
for i,line in enumerate(sys.stdin):
try:
source = re.sub('id:twitter.com:', '', safeget(json.loads(line), 'actor.id'))
loc, mode = ginf_api.get_user_loc(source, always_predict=args.always_predict, always_dirty=args.always_dirty)
if loc:
loc.update({"source" : source, "mode" : mode})
print json.dumps(loc)
except:
print >> sys.stderr, 'error: %s' % line
if not i % 1000:
print >> sys.stderr, '%d records complete' % i
def test_ReadNull(self):
self.assertEqual(None, ultrajson.loads("null"))
def ultrajsonDec():
x = ultrajson.loads(decodeData)
def test_ReadArrayOfSymbols(self):
self.assertEqual([True, False, None], ultrajson.loads(" [ true, false,null] "))
def test_decodeFloatingPointAdditionalTests(self):
self.assertEqual(-1.1234567893, ultrajson.loads("-1.1234567893"))
self.assertEqual(-1.234567893, ultrajson.loads("-1.234567893"))
self.assertEqual(-1.34567893, ultrajson.loads("-1.34567893"))
self.assertEqual(-1.4567893, ultrajson.loads("-1.4567893"))
self.assertEqual(-1.567893, ultrajson.loads("-1.567893"))
self.assertEqual(-1.67893, ultrajson.loads("-1.67893"))
self.assertEqual(-1.7893, ultrajson.loads("-1.7893"))
self.assertEqual(-1.893, ultrajson.loads("-1.893"))
self.assertEqual(-1.3, ultrajson.loads("-1.3"))
self.assertEqual(1.1234567893, ultrajson.loads("1.1234567893"))
self.assertEqual(1.234567893, ultrajson.loads("1.234567893"))
self.assertEqual(1.34567893, ultrajson.loads("1.34567893"))
self.assertEqual(1.4567893, ultrajson.loads("1.4567893"))
self.assertEqual(1.567893, ultrajson.loads("1.567893"))
self.assertEqual(1.67893, ultrajson.loads("1.67893"))
self.assertEqual(1.7893, ultrajson.loads("1.7893"))
self.assertEqual(1.893, ultrajson.loads("1.893"))
self.assertEqual(1.3, ultrajson.loads("1.3"))
