def main(args):
logging.basicConfig(level=logging.INFO)
model = json.load(args.ifile)
fn = fn_from_args(args)
if fn:
weight_layers = [layer for layer in model
if layer['layerName'] in weight_first_list]
if fn.needs_two_step:
for weights in [layer['parameters'][0] for layer in weight_layers]:
fn.consume(weights)
fn.done()
for i, layer in enumerate(weight_layers):
layer['parameters'][0] = transform(layer['parameters'][0], fn)
if fn.needs_two_step:
model = {
'codebook' : fn.serialize_codebook(),
'model' : model
}
if args.ubjson_format:
args.ofile.write(simpleubjson.encode(model))
elif args.json:
args.ofile.write(json.dumps(model).encode('utf8'))
else:
msgpack.dump(model, args.ofile, use_bin_type=True,
use_single_float=args.single_precision_float)
args.ofile.close()
args.ifile.close()
def inspect_draft9(decoder, level, *args):
while 1:
try:
tag, length, value = decoder.next_tlv()
utag = tag.decode()
except EarlyEndOfStreamError:
break
# standalone markers
if length is None and value is None:
if utag in ']}':
level -= 1
maybe_write('[%s]\n' % (utag, ), level)
if utag in '{[':
level += 1
# plane values
elif length is None and value is not None:
value = decoder.dispatch[tag](decoder, tag, length, value)
maybe_write('[%s] [%s]\n' % (utag, value), level)
# sized values
else:
value = decoder.dispatch[tag](decoder, tag, length, value)
pattern = '[%s] [%s] [%s] [%s]\n'
# very dirty hack to show size as marker and value
_decoder = Draft9Decoder(simpleubjson.encode(length,
spec=spec))
tlv = _decoder.next_tlv()
args = tuple([utag, tlv[0].decode(), tlv[2], value])
maybe_write(pattern % args, level)
def inspect_draft9(decoder, level, *args):
while 1:
try:
tag, length, value = decoder.next_tlv()
utag = tag.decode()
except EarlyEndOfStreamError:
break
# standalone markers
if length is None and value is None:
if utag in ']}':
level -= 1
maybe_write('[%s]\n' % (utag,), level)
if utag in '{[':
level += 1
# plane values
elif length is None and value is not None:
value = decoder.dispatch[tag](decoder, tag, length, value)
maybe_write('[%s] [%s]\n' % (utag, value), level)
# sized values
else:
value = decoder.dispatch[tag](decoder, tag, length, value)
pattern = '[%s] [%s] [%s] [%s]\n'
# very dirty hack to show size as marker and value
_decoder = Draft9Decoder(simpleubjson.encode(length, spec=spec))
tlv = _decoder.next_tlv()
args = tuple([utag, tlv[0].decode(), tlv[2], value])
maybe_write(pattern % args, level)
def main(args):
logging.basicConfig(level=logging.INFO)
model = json.load(args.ifile)
fn = fn_from_args(args)
if fn:
weight_layers = [
layer for layer in model if layer['layerName'] in weight_first_list
]
if fn.needs_two_step:
for weights in [layer['parameters'][0] for layer in weight_layers]:
fn.consume(weights)
fn.done()
for i, layer in enumerate(weight_layers):
layer['parameters'][0] = transform(layer['parameters'][0], fn)
if fn.needs_two_step:
model = {'codebook': fn.serialize_codebook(), 'model': model}
if args.ubjson_format:
args.ofile.write(simpleubjson.encode(model))
elif args.json:
args.ofile.write(json.dumps(model).encode('utf8'))
else:
msgpack.dump(model,
args.ofile,
use_bin_type=True,
use_single_float=args.single_precision_float)
args.ofile.close()
args.ifile.close()
def compress_file_ub_json(self, json_file_location):
parsed_json = self.get_json_as_object(json_file_location)
compr_started_time = time.time()
ubjson_bytes = simpleubjson.encode(parsed_json)
duration_compression = time.time() - compr_started_time
CompressorThread.TOTAL_COMPRESSION_DURATION += duration_compression
return ubjson_bytes
def setUp(self):
self.decode = lambda *a, **k: simpleubjson.decode(
spec='draft-9', *a, **k)
self.encode = lambda *a, **k: simpleubjson.encode(
spec='draft-9', *a, **k)
import simpleubjson
bytestring = b'Dr8@\xbbnR\x8c\xe4\xe1+\x82e\xc0\xea?\x99\x83YWH\xb6\xc1\xfa\xab\x93\xe5\xbat}'
document = {
"Key1": "Value1",
"Key2": bytestring,
}
ubjdata = simpleubjson.encode(document)
decoded = simpleubjson.decode(ubjdata)
for key in document:
if document[key] != decoded[key]:
print("Error, non matching key:", key)
def setUp(self):
self.decode = lambda *a, **k: simpleubjson.decode(spec='draft-9', *a, **k)
self.encode = lambda *a, **k: simpleubjson.encode(spec='draft-9', *a, **k)
def make_benchmark(name, count):
data = load_case(name)
src = simpleubjson.encode(data, spec='draft-8')
total = run_test(simpleubjson.decode, count, src, spec='draft-8')
print(
format_results('simpleubjson', simpleubjson.__version__,
'Decoded Draft-8', total, count))
total = run_test(simpleubjson.encode, count, data, spec='draft-8')
print(
format_results('simpleubjson', simpleubjson.__version__,
'Encoded Draft-8', total, count))
print
src = simpleubjson.encode(data, spec='draft-9')
# func = lambda *a, **k: list(simpleubjson.decode(*a, **k))
def func(*a, **k):
return list(simpleubjson.decode(*a, **k))
total = run_test(func, count, src, spec='draft-9')
print(
format_results('simpleubjson', simpleubjson.__version__,
'Decoded Draft-9', total, count))
total = run_test(simpleubjson.encode, count, data, spec='draft-9')
print(
format_results('simpleubjson', simpleubjson.__version__,
'Encoded Draft-9', total, count))
if json:
print
total = run_test(json.loads, count, json.dumps(data))
print(
format_results('json_stdlib', json.__version__, 'Decoded', total,
count))
total = run_test(json.dumps, count, data)
print(
format_results('json_stdlib', json.__version__, 'Encoded', total,
count))
if simplejson:
print
simplejson._toggle_speedups(True)
total = run_test(simplejson.loads, count, simplejson.dumps(data))
print(
format_results('simplejson_c', simplejson.__version__, 'Decoded',
total, count))
simplejson._toggle_speedups(True)
total = run_test(simplejson.dumps, count, data)
print(
format_results('simplejson_c', simplejson.__version__, 'Encoded',
total, count))
print
simplejson._toggle_speedups(False)
total = run_test(simplejson.loads, count, simplejson.dumps(data))
print(
format_results('simplejson_py', simplejson.__version__, 'Decoded',
total, count))
simplejson._toggle_speedups(False)
total = run_test(simplejson.dumps, count, data)
print(
format_results('simplejson_py', simplejson.__version__, 'Encoded',
total, count))
if ujson:
print
total = run_test(ujson.decode, count, ujson.encode(data))
print(
format_results('ujson', ujson.__version__, 'Decoded', total,
count))
total = run_test(ujson.encode, count, data)
print(
format_results('ujson', ujson.__version__, 'Encoded', total,
count))
if erlport:
print
total = run_test(erlport.decode, count, erlport.encode(data))
print(
format_results('erlport', erlport.__version__, 'Decoded', total,
count))
total = run_test(erlport.encode, count, data)
print(
format_results('erlport', erlport.__version__, 'Encoded', total,
count))
print
total = run_test(pickle.loads, count, pickle.dumps(data))
print(format_results('pickle', pickle.__version__, 'Decoded', total,
count))
total = run_test(pickle.dumps, count, data)
print(format_results('pickle', pickle.__version__, 'Encoded', total,
count))
def make_benchmark(name, count):
data = load_case(name)
src = simpleubjson.encode(data, spec='draft-8')
total = run_test(simpleubjson.decode, count, src, spec='draft-8')
print(format_results('simpleubjson', simpleubjson.__version__,
'Decoded Draft-8', total, count))
total = run_test(simpleubjson.encode, count, data, spec='draft-8')
print(format_results('simpleubjson', simpleubjson.__version__,
'Encoded Draft-8', total, count))
print
src = simpleubjson.encode(data, spec='draft-9')
# func = lambda *a, **k: list(simpleubjson.decode(*a, **k))
def func(*a, **k): return list(simpleubjson.decode(*a, **k))
total = run_test(func, count, src, spec='draft-9')
print(format_results('simpleubjson', simpleubjson.__version__,
'Decoded Draft-9', total, count))
total = run_test(simpleubjson.encode, count, data, spec='draft-9')
print(format_results('simpleubjson', simpleubjson.__version__,
'Encoded Draft-9', total, count))
if json:
print
total = run_test(json.loads, count, json.dumps(data))
print(format_results('json_stdlib', json.__version__,
'Decoded', total, count))
total = run_test(json.dumps, count, data)
print(format_results('json_stdlib', json.__version__,
'Encoded', total, count))
if simplejson:
print
simplejson._toggle_speedups(True)
total = run_test(simplejson.loads, count, simplejson.dumps(data))
print(format_results('simplejson_c', simplejson.__version__,
'Decoded', total, count))
simplejson._toggle_speedups(True)
total = run_test(simplejson.dumps, count, data)
print(format_results('simplejson_c', simplejson.__version__,
'Encoded', total, count))
print
simplejson._toggle_speedups(False)
total = run_test(simplejson.loads, count, simplejson.dumps(data))
print(format_results('simplejson_py', simplejson.__version__,
'Decoded', total, count))
simplejson._toggle_speedups(False)
total = run_test(simplejson.dumps, count, data)
print(format_results('simplejson_py', simplejson.__version__,
'Encoded', total, count))
if ujson:
print
total = run_test(ujson.decode, count, ujson.encode(data))
print(format_results('ujson', ujson.__version__,
'Decoded', total, count))
total = run_test(ujson.encode, count, data)
print(format_results('ujson', ujson.__version__,
'Encoded', total, count))
if erlport:
print
total = run_test(erlport.decode, count, erlport.encode(data))
print(format_results('erlport', erlport.__version__,
'Decoded', total, count))
total = run_test(erlport.encode, count, data)
print(format_results('erlport', erlport.__version__,
'Encoded', total, count))
print
total = run_test(pickle.loads, count, pickle.dumps(data))
print(format_results('pickle', pickle.__version__,
'Decoded', total, count))
total = run_test(pickle.dumps, count, data)
print(format_results('pickle', pickle.__version__,
'Encoded', total, count))