def main(args): logging.basicConfig(level=logging.INFO) model = json.load(args.ifile) fn = fn_from_args(args) if fn: weight_layers = [layer for layer in model if layer['layerName'] in weight_first_list] if fn.needs_two_step: for weights in [layer['parameters'][0] for layer in weight_layers]: fn.consume(weights) fn.done() for i, layer in enumerate(weight_layers): layer['parameters'][0] = transform(layer['parameters'][0], fn) if fn.needs_two_step: model = { 'codebook' : fn.serialize_codebook(), 'model' : model } if args.ubjson_format: args.ofile.write(simpleubjson.encode(model)) elif args.json: args.ofile.write(json.dumps(model).encode('utf8')) else: msgpack.dump(model, args.ofile, use_bin_type=True, use_single_float=args.single_precision_float) args.ofile.close() args.ifile.close()
def inspect_draft9(decoder, level, *args): while 1: try: tag, length, value = decoder.next_tlv() utag = tag.decode() except EarlyEndOfStreamError: break # standalone markers if length is None and value is None: if utag in ']}': level -= 1 maybe_write('[%s]\n' % (utag, ), level) if utag in '{[': level += 1 # plane values elif length is None and value is not None: value = decoder.dispatch[tag](decoder, tag, length, value) maybe_write('[%s] [%s]\n' % (utag, value), level) # sized values else: value = decoder.dispatch[tag](decoder, tag, length, value) pattern = '[%s] [%s] [%s] [%s]\n' # very dirty hack to show size as marker and value _decoder = Draft9Decoder(simpleubjson.encode(length, spec=spec)) tlv = _decoder.next_tlv() args = tuple([utag, tlv[0].decode(), tlv[2], value]) maybe_write(pattern % args, level)
def inspect_draft9(decoder, level, *args): while 1: try: tag, length, value = decoder.next_tlv() utag = tag.decode() except EarlyEndOfStreamError: break # standalone markers if length is None and value is None: if utag in ']}': level -= 1 maybe_write('[%s]\n' % (utag,), level) if utag in '{[': level += 1 # plane values elif length is None and value is not None: value = decoder.dispatch[tag](decoder, tag, length, value) maybe_write('[%s] [%s]\n' % (utag, value), level) # sized values else: value = decoder.dispatch[tag](decoder, tag, length, value) pattern = '[%s] [%s] [%s] [%s]\n' # very dirty hack to show size as marker and value _decoder = Draft9Decoder(simpleubjson.encode(length, spec=spec)) tlv = _decoder.next_tlv() args = tuple([utag, tlv[0].decode(), tlv[2], value]) maybe_write(pattern % args, level)
def main(args): logging.basicConfig(level=logging.INFO) model = json.load(args.ifile) fn = fn_from_args(args) if fn: weight_layers = [ layer for layer in model if layer['layerName'] in weight_first_list ] if fn.needs_two_step: for weights in [layer['parameters'][0] for layer in weight_layers]: fn.consume(weights) fn.done() for i, layer in enumerate(weight_layers): layer['parameters'][0] = transform(layer['parameters'][0], fn) if fn.needs_two_step: model = {'codebook': fn.serialize_codebook(), 'model': model} if args.ubjson_format: args.ofile.write(simpleubjson.encode(model)) elif args.json: args.ofile.write(json.dumps(model).encode('utf8')) else: msgpack.dump(model, args.ofile, use_bin_type=True, use_single_float=args.single_precision_float) args.ofile.close() args.ifile.close()
def compress_file_ub_json(self, json_file_location): parsed_json = self.get_json_as_object(json_file_location) compr_started_time = time.time() ubjson_bytes = simpleubjson.encode(parsed_json) duration_compression = time.time() - compr_started_time CompressorThread.TOTAL_COMPRESSION_DURATION += duration_compression return ubjson_bytes
def setUp(self): self.decode = lambda *a, **k: simpleubjson.decode( spec='draft-9', *a, **k) self.encode = lambda *a, **k: simpleubjson.encode( spec='draft-9', *a, **k)
import simpleubjson bytestring = b'Dr8@\xbbnR\x8c\xe4\xe1+\x82e\xc0\xea?\x99\x83YWH\xb6\xc1\xfa\xab\x93\xe5\xbat}' document = { "Key1": "Value1", "Key2": bytestring, } ubjdata = simpleubjson.encode(document) decoded = simpleubjson.decode(ubjdata) for key in document: if document[key] != decoded[key]: print("Error, non matching key:", key)
def setUp(self): self.decode = lambda *a, **k: simpleubjson.decode(spec='draft-9', *a, **k) self.encode = lambda *a, **k: simpleubjson.encode(spec='draft-9', *a, **k)
def make_benchmark(name, count): data = load_case(name) src = simpleubjson.encode(data, spec='draft-8') total = run_test(simpleubjson.decode, count, src, spec='draft-8') print( format_results('simpleubjson', simpleubjson.__version__, 'Decoded Draft-8', total, count)) total = run_test(simpleubjson.encode, count, data, spec='draft-8') print( format_results('simpleubjson', simpleubjson.__version__, 'Encoded Draft-8', total, count)) print src = simpleubjson.encode(data, spec='draft-9') # func = lambda *a, **k: list(simpleubjson.decode(*a, **k)) def func(*a, **k): return list(simpleubjson.decode(*a, **k)) total = run_test(func, count, src, spec='draft-9') print( format_results('simpleubjson', simpleubjson.__version__, 'Decoded Draft-9', total, count)) total = run_test(simpleubjson.encode, count, data, spec='draft-9') print( format_results('simpleubjson', simpleubjson.__version__, 'Encoded Draft-9', total, count)) if json: print total = run_test(json.loads, count, json.dumps(data)) print( format_results('json_stdlib', json.__version__, 'Decoded', total, count)) total = run_test(json.dumps, count, data) print( format_results('json_stdlib', json.__version__, 'Encoded', total, count)) if simplejson: print simplejson._toggle_speedups(True) total = run_test(simplejson.loads, count, simplejson.dumps(data)) print( format_results('simplejson_c', simplejson.__version__, 'Decoded', total, count)) simplejson._toggle_speedups(True) total = run_test(simplejson.dumps, count, data) print( format_results('simplejson_c', simplejson.__version__, 'Encoded', total, count)) print simplejson._toggle_speedups(False) total = run_test(simplejson.loads, count, simplejson.dumps(data)) print( format_results('simplejson_py', simplejson.__version__, 'Decoded', total, count)) simplejson._toggle_speedups(False) total = run_test(simplejson.dumps, count, data) print( format_results('simplejson_py', simplejson.__version__, 'Encoded', total, count)) if ujson: print total = run_test(ujson.decode, count, ujson.encode(data)) print( format_results('ujson', ujson.__version__, 'Decoded', total, count)) total = run_test(ujson.encode, count, data) print( format_results('ujson', ujson.__version__, 'Encoded', total, count)) if erlport: print total = run_test(erlport.decode, count, erlport.encode(data)) print( format_results('erlport', erlport.__version__, 'Decoded', total, count)) total = run_test(erlport.encode, count, data) print( format_results('erlport', erlport.__version__, 'Encoded', total, count)) print total = run_test(pickle.loads, count, pickle.dumps(data)) print(format_results('pickle', pickle.__version__, 'Decoded', total, count)) total = run_test(pickle.dumps, count, data) print(format_results('pickle', pickle.__version__, 'Encoded', total, count))
def make_benchmark(name, count): data = load_case(name) src = simpleubjson.encode(data, spec='draft-8') total = run_test(simpleubjson.decode, count, src, spec='draft-8') print(format_results('simpleubjson', simpleubjson.__version__, 'Decoded Draft-8', total, count)) total = run_test(simpleubjson.encode, count, data, spec='draft-8') print(format_results('simpleubjson', simpleubjson.__version__, 'Encoded Draft-8', total, count)) print src = simpleubjson.encode(data, spec='draft-9') # func = lambda *a, **k: list(simpleubjson.decode(*a, **k)) def func(*a, **k): return list(simpleubjson.decode(*a, **k)) total = run_test(func, count, src, spec='draft-9') print(format_results('simpleubjson', simpleubjson.__version__, 'Decoded Draft-9', total, count)) total = run_test(simpleubjson.encode, count, data, spec='draft-9') print(format_results('simpleubjson', simpleubjson.__version__, 'Encoded Draft-9', total, count)) if json: print total = run_test(json.loads, count, json.dumps(data)) print(format_results('json_stdlib', json.__version__, 'Decoded', total, count)) total = run_test(json.dumps, count, data) print(format_results('json_stdlib', json.__version__, 'Encoded', total, count)) if simplejson: print simplejson._toggle_speedups(True) total = run_test(simplejson.loads, count, simplejson.dumps(data)) print(format_results('simplejson_c', simplejson.__version__, 'Decoded', total, count)) simplejson._toggle_speedups(True) total = run_test(simplejson.dumps, count, data) print(format_results('simplejson_c', simplejson.__version__, 'Encoded', total, count)) print simplejson._toggle_speedups(False) total = run_test(simplejson.loads, count, simplejson.dumps(data)) print(format_results('simplejson_py', simplejson.__version__, 'Decoded', total, count)) simplejson._toggle_speedups(False) total = run_test(simplejson.dumps, count, data) print(format_results('simplejson_py', simplejson.__version__, 'Encoded', total, count)) if ujson: print total = run_test(ujson.decode, count, ujson.encode(data)) print(format_results('ujson', ujson.__version__, 'Decoded', total, count)) total = run_test(ujson.encode, count, data) print(format_results('ujson', ujson.__version__, 'Encoded', total, count)) if erlport: print total = run_test(erlport.decode, count, erlport.encode(data)) print(format_results('erlport', erlport.__version__, 'Decoded', total, count)) total = run_test(erlport.encode, count, data) print(format_results('erlport', erlport.__version__, 'Encoded', total, count)) print total = run_test(pickle.loads, count, pickle.dumps(data)) print(format_results('pickle', pickle.__version__, 'Decoded', total, count)) total = run_test(pickle.dumps, count, data) print(format_results('pickle', pickle.__version__, 'Encoded', total, count))