def decrypt5(in_stream, out_stream, public_key, secret_key, verify_all,
symmetric, force, block_size, metadata_length):
metadata_bytes = in_stream.read(metadata_length)
md = dict(ubjson.loadb(metadata_bytes))
metadata = {}
if public_key is not None:
sign_key_encoded = public_key
else:
sign_key_encoded = md['sign_key']
if md['sign_key'] != public_key:
print(
"WARNING: sign_key in metadata and public key provided don't match",
file=sys.stderr)
metadata['sign_key'] = nacl.public.PublicKey(sign_key_encoded)
metadata['secret_key'] = nacl.public.PrivateKey(secret_key)
decrypt_box = nacl.public.Box(metadata['secret_key'], metadata['sign_key'])
secure_metadata_bytes = decrypt_box.decrypt(md['secure'])
secure_metadata = dict(ubjson.loadb(secure_metadata_bytes))
metadata["key"] = secure_metadata["key"]
metadata["block_size"] = secure_metadata["block_size"]
counter = 0
decrypt_box = nacl.secret.SecretBox(metadata['key'])
block_size = metadata['block_size'] + SECRET_BOX_SIGN_SIZE
block = in_stream.read(block_size)
while len(block) > 0:
# Use network endianness
counternonce = counter.to_bytes(nacl.secret.SecretBox.NONCE_SIZE,
"big")
out_block = decrypt_box.decrypt(block, nonce=counternonce)
out_stream.write(out_block)
block = in_stream.read(block_size)
counter += 1
return
def unserialize(self, payload):
"""
Implements :func:`autobahn.wamp.interfaces.IObjectSerializer.unserialize`
"""
if self._batched:
msgs = []
N = len(payload)
i = 0
while i < N:
# read message length prefix
if i + 4 > N:
raise Exception("batch format error [1]")
l = struct.unpack("!L", payload[i:i + 4])[0]
# read message data
if i + 4 + l > N:
raise Exception("batch format error [2]")
data = payload[i + 4:i + 4 + l]
# append parsed raw message
msgs.append(ubjson.loadb(data))
# advance until everything consumed
i = i + 4 + l
if i != N:
raise Exception("batch format error [3]")
return msgs
else:
unpacked = ubjson.loadb(payload)
return [unpacked]
def unserialize(self, payload):
"""
Implements :func:`autobahn.wamp.interfaces.IObjectSerializer.unserialize`
"""
if self._batched:
msgs = []
N = len(payload)
i = 0
while i < N:
# read message length prefix
if i + 4 > N:
raise Exception("batch format error [1]")
l = struct.unpack("!L", payload[i:i + 4])[0]
# read message data
if i + 4 + l > N:
raise Exception("batch format error [2]")
data = payload[i + 4:i + 4 + l]
# append parsed raw message
msgs.append(ubjson.loadb(data))
# advance until everything consumed
i = i + 4 + l
if i != N:
raise Exception("batch format error [3]")
return msgs
else:
unpacked = ubjson.loadb(payload)
return [unpacked]
def _call(self, method, args={}):
msg = ubjson.dumpb({method: self._flatten_arrays(args)})
msglen = struct.pack("=i", len(msg))
try:
self.callpipe.write(msglen + msg)
if method.startswith("Set"):
return None
self.callpipe.flush()
except OSError as exc:
raise AMSWorkerError('Error while sending a message') from exc
if method == "Exit":
return None
results = []
while True:
try:
msgbuf = self._read_exactly(self.replypipe, 4)
msglen = struct.unpack("=i", msgbuf)[0]
msgbuf = self._read_exactly(self.replypipe, msglen)
except EOFError as exc:
raise AMSWorkerError("Error while trying to read a reply") from exc
try:
msg = ubjson.loadb(msgbuf)
except Exception as exc:
raise AMSWorkerError("Error while decoding a reply") from exc
if "return" in msg:
ret = msg["return"]
if ret["status"] == 0:
return results
else:
raise AMSPipeError.from_message(ret)
else:
results.append(msg)
def send_request(self, request):
"""
Sends the request to the peer.
:param request: The request as dictionary.
:return: The response as dictionary
"""
request = {
"action": "req",
"payload": ESCSEQ_OBJ_START + \
ubjson.dumpb(request).replace(b'\x00', b'\x00\x00') + \
ESCSEQ_OBJ_END,
"cv": threading.Condition()
}
with request['cv']:
self._msgqueue.put(request)
if not request['cv'].wait(timeout=CALL_TIMEOUT):
raise TimeoutError("No response within %d seconds" %
CALL_TIMEOUT)
if 'error' in request:
raise request['error']
try:
return ubjson.loadb(request['response'])
except ubjson.DecoderException:
errmsg = "%s: Couldn't parse UBJSON from lora_controller. Got: %s" % \
(self._name, " ".join(hex(x) for x in request['response']))
logger.error(errmsg, exc_info=True)
raise RuntimeError(errmsg)
def __init__(self,filename,mode = 'rb',logging_level=logging.WARNING):
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.setLevel(logging_level)
self.filename = None
self.mode = mode
self.header = None
self.fill_loggerstreamdata = False
self.loggerstreams = []
try:
self.logger.debug('Opening file with mode ' + mode)
self.f = open(filename,mode)
self.filename = filename
except:
self.logger.warning('Could not open file: ' + filename)
# Init the file and write a file info header
if(mode == 'wb'):
self.logger.debug('Writing header')
self.header = {'desc':'pymds_logger LoggerFile','datatype':'ubjson','created':str(datetime.datetime.now())}
self.write_raw(self.header)
self.sync()
# Read the first line and check if a header was found
elif(mode == 'rb'):
self.fill_loggerstreamdata = True
data = self.f.readline()
try:
data_decode = ubjson.loadb(data)
if(data_decode['desc'] == 'pymds_logger LoggerFile'):
creation_time = data_decode['created']
print('This is a good header! Of a file created at ' + creation_time)
except:
self.logger.warning('Did not find a valid header. Will abort')
raise TypeError('Did not find a valid header. Wrong datatype in file')
def parse_follow_on_issuance_metadata_10007(data):
pos = 0
# Multiple fields follow: field name (null delimited), variable length integer, raw data of field
fields = dict()
while pos < len(data):
# Protocol 10007: there are no special properties for follow on issuance metadata
if data[pos:pos + 1] == "\0":
assetproplen = ord(data[pos + 2:pos + 3])
assetprop = data[pos + 3:pos + 3 + assetproplen]
proptype = ord(data[pos + 1])
if proptype == 0x05: # JSON_DETAILS
fname = "JSON Data"
try:
json_data = ubjson.loadb(assetprop)
for k in ('text', 'json'):
if k in json_data:
json_data = json_data[k]
break
fields[fname] = render_long_data_with_popover(json.dumps(json_data))
except ValueError:
fields[fname] = long_hex(assetprop)
else:
fname = "MultiChain special property at offset {0}".format(pos)
fields[fname] = long_hex(assetprop)
pos = pos + 3 + assetproplen
continue
try:
searchdata = data[pos:]
fname = searchdata[:searchdata.index("\0")]
pos = pos + len(fname) + 1
flen = ord(data[pos:pos + 1])
pos += 1
# print "pos of payload: ", pos
if flen == 253:
(size,) = struct.unpack('<H', data[pos:pos + 2])
flen = size
pos += 2
elif flen == 254:
(size,) = struct.unpack('<I', data[pos:pos + 4])
flen = size
pos += 4
elif flen == 255:
(size,) = struct.unpack('<Q', data[pos:pos + 8])
flen = size
pos += 8
fields[fname] = data[pos:pos + flen]
pos += flen
except:
print('Ignorando erro...')
break
return fields
def __load(self): # pylint: disable=too-many-branches
fsplit = splitext(self.__fname)
if fsplit[1] == '.json':
if exists(self.__fname):
# Migrate from json to ubjson
with self.__stash_lock:
with open(self.__fname, 'r') as f:
self.__stash = json.loads(f.read())
rename(self.__fname, self.__fname + '.old')
if fsplit[1] != '.ubjz':
self.__fname = fsplit[0] + '.ubjz'
if exists(self.__fname):
with self.__stash_lock:
with gzip_open(self.__fname, 'rb') as f:
self.__stash = ubjson.loadb(f.read())
elif self.__stash is None:
self.__stash = {
THINGS: {}, # Current/last state of Things
DIFF: {}, # Diffs not yet updated in Iotic Space
DIFFCOUNT: 0
} # Diff counter
if not exists(self.__pname):
self.__properties = {}
else:
with self.__stash_lock:
with open(self.__pname, 'r') as f:
self.__properties = json.loads(f.read())
with self.__stash_lock:
stash_copy = deepcopy(self.__stash)
self.__stash = {}
# Migrate built-in keys
for key, value in stash_copy.items():
if key in [THINGS, DIFF, DIFFCOUNT]:
logger.debug("--> Migrating built-in %s", key)
self.__stash[key] = stash_copy[key]
# Migrate bad keys
for key, value in stash_copy.items():
if key not in [THINGS, DIFF, DIFFCOUNT]:
if key not in stash_copy[THINGS]:
logger.info("--> Migrating key to THINGS %s", key)
self.__stash[THINGS][key] = value
self.__stash.pop(key, None)
# Remove redundant LAT/LONG (LOCATION used instead)
for el, et in self.__stash[THINGS].items(): # pylint: disable=unused-variable
et.pop(LAT, None)
et.pop(LONG, None)
self.__save()
def Read(fname):
"""
Given a filename or list of filenames, returns the Python dict.
Arguments
---------
fname: str, list
Returns
-------
dict or list of dict
"""
if isinstance(fname, str):
with open(fname, "rb") as f:
ret = ubjson.loadb(f.read())
elif isinstance(fname, list):
ret = []
for ff in fname:
try:
with open(ff, "rb") as f:
ret.append(ubjson.loadb(f.read()))
except:
print("Error with file ", ff)
return ret
def read_message(self):
""" Read an entire message from the registered socket.
Returns None on failure, Dict of data from ubjson on success.
"""
while True:
try:
rd, wr, exc = select.select(self.inp, self.out, self.inp)
if (len(rd) < 1):
continue
except OSError as e:
if (e.args[0] == errno.EBADF):
print("Socket closed, shutting down")
return None
for s in rd:
try:
# The first 4 bytes are the message's length
# read this first
while (len(self.buf) < 4):
self.buf += s.recv(4 - len(self.buf))
message_len = unpack(">L", self.buf[0:4])[0]
# Now read in message_len amount of data
while (len(self.buf) < (message_len + 4)):
self.buf += s.recv((message_len + 4) - len(self.buf))
try:
# Exclude the the message length in the header
msg = ubjson.loadb(self.buf[4:])
# Clear out the old buffer
del self.buf
self.buf = bytearray()
return msg
except DecoderException as e:
print("ERROR: Decode failure in Slippstream")
print(e)
print(hexdump(self.buf[4:]))
self.buf.clear()
return None
except socket.error as e:
if (e.args[0] == errno.EWOULDBLOCK): continue
else:
print("ERROR with socket:", e)
return None
def get_context_data(self, **kwargs):
ctx = super().get_context_data(**kwargs)
api = get_client()
ctx['stream_details'] = api.liststreams(ctx['stream'])[0]
ctx['stream_items'] = list(reversed(api.liststreamitems(
ctx['stream'])))
for key, item in enumerate(ctx['stream_items']):
if 'blocktime' in item:
ctx['stream_items'][key][
'formatted_time'] = datetime.datetime.fromtimestamp(
item['blocktime'])
if item['data']:
try:
ctx['stream_items'][key]['formatted_data'] = ubjson.loadb(
unhexlify(item['data']))
except Exception as e:
ctx['stream_items'][key]['formatted_data'] = item['data']
return ctx
def read_message(self):
""" Read an entire message from the registered socket.
Returns None on failure, Dict of data from ubjson on success.
"""
while True:
try:
# The first 4 bytes are the message's length
# read this first
while len(self.buf) < 4:
self.buf += self.server.recv(4 - len(self.buf))
if len(self.buf) == 0:
return None
message_len = unpack(">L", self.buf[0:4])[0]
# Now read in message_len amount of data
while len(self.buf) < (message_len + 4):
self.buf += self.server.recv((message_len + 4) -
len(self.buf))
try:
# Exclude the the message length in the header
msg = ubjson.loadb(self.buf[4:])
# Clear out the old buffer
del self.buf
self.buf = bytearray()
return msg
except DecoderException as exception:
print("ERROR: Decode failure in SlippiComm")
print(exception)
print(hexdump(self.buf[4:]))
self.buf.clear()
return None
except socket.error as exception:
if exception.args[0] == errno.EWOULDBLOCK:
continue
print("ERROR with socket:", exception)
return None
def connect(self):
with open(self._path, mode='rb') as file:
full = ubjson.loadb(file.read())
raw = full["raw"]
self._contents = raw
try:
self.playedOn = full["metadata"]["playedOn"]
except KeyError:
pass
try:
self.timestamp = full["metadata"]["startAt"]
except KeyError:
pass
try:
self.consoleNick = full["metadata"]["consoleNick"]
except KeyError:
pass
try:
self.players = full["metadata"]["players"]
except KeyError:
pass
return True
def read(self,n = None):
"""
Args:
Returns:
data_decode: List
"""
print('Reading')
if(self.mode == 'rb'):
data_decode_all = []
data = b''
for line in self.f:
data += line
try:
data_decode = ubjson.loadb(data)
data = b''
self.interprete_data(data_decode)
data_decode_all.append(data_decode)
#print(data_decode)
except:
print('Could no decode:' + str(data))
pass
return data_decode_all
def get_timestamps(hash_value: str, stream='timestamp') -> Optional[List]:
client = get_active_rpc_client()
response = client.liststreamkeyitems(stream, hash_value, verbose=True, count=1000, start=-1000)
if response['error'] is not None:
raise RpcResponseError(response['error']['message'])
result = response['result']
timestamps = []
for entry in result:
if entry['data']:
if not isinstance(entry['data'], str):
log.warning('Stream item data is not a string: %s' % entry['data'])
# Todo investigate dict with size, txid, vout in stream item data
continue
data = ubjson.loadb(unhexlify(entry['data']))
comment = data.get('comment', '')
else:
comment = ''
for publisher in entry['publishers']:
timestamps.append((entry['time'], publisher, comment))
return timestamps
def parse_create_stream_10007(data):
pos = 0
# Multiple fields follow: field name (null delimited), variable length integer, raw data of field
fields = dict()
# If the property 'Open to all writers' is not present, we treat it as false.
opentoall = "Open to all writers"
fields[opentoall] = "False"
while pos < len(data):
# Is this a special property with meaning only for MultiChain?
if data[pos:pos + 1] == "\0":
assetproplen = ord(data[pos + 2:pos + 3])
assetprop = data[pos + 3:pos + 3 + assetproplen]
proptype = ord(data[pos + 1])
# Create stream has special properties
if proptype == 0x01:
fields["Name"] = assetprop
elif proptype == 0x04:
fields[opentoall] = str(ord(assetprop) == 1)
elif proptype == 0x05: # JSON_DETAILS
fname = "JSON Data"
try:
json_data = ubjson.loadb(assetprop)
for k in ('text', 'json'):
if k in json_data:
json_data = json_data[k]
break
fields[fname] = render_long_data_with_popover(json.dumps(json_data))
except ValueError:
fields[fname] = long_hex(assetprop)
elif proptype == 0x06: # PERMISSIONS
permission_code, = struct.unpack("<b", assetprop)
fields[opentoall] = str(permission_code == 0)
elif proptype in [0x07]: # RESTRICTIONS
restriction_code, = struct.unpack("<b", assetprop)
if restriction_code != 0:
restrictions = []
if restriction_code == 0x01:
restrictions.append("onchain")
if restriction_code == 0x02:
restrictions.append("offchain")
fields["Restrictions"] = ','.join(restrictions)
# Filter
elif proptype == 0x45: # RESTRICTIONS
pass
elif proptype == 0x46: # FILTER CODE
fields["Code"] = render_long_data_with_popover("<pre>{}</pre>".format(assetprop), hover=False)
elif proptype == 0x47: # TYPE
fname = "Type"
filter_type = ord(assetprop[0])
if filter_type == 0x00:
fields[fname] = "Transaction"
elif filter_type == 0x01:
fields[fname] = "Stream"
else:
fields[fname] = "Invalid ({})".format(filter_type)
else:
fields["Property at offset {0}".format(pos)] = long_hex(assetprop)
pos = pos + 3 + assetproplen
continue
searchdata = data[pos:]
try:
fname = searchdata[:searchdata.index("\0")]
pos = pos + len(fname) + 1
flen = ord(data[pos:pos + 1])
pos += 1
# print "pos of payload: ", pos
if flen == 253:
(size,) = struct.unpack('<H', data[pos:pos + 2])
flen = size
pos += 2
elif flen == 254:
(size,) = struct.unpack('<I', data[pos:pos + 4])
flen = size
pos += 4
elif flen == 255:
(size,) = struct.unpack('<Q', data[pos:pos + 8])
flen = size
pos += 8
fields[fname] = data[pos:pos + flen]
pos += flen
except:
print('Ignorando erro...')
break;
return fields
def connect(self):
with open(self._path, mode='rb') as file:
full = ubjson.loadb(file.read())
raw = full["raw"]
self._contents = raw
return True
def process_inputs_and_outputs(data_db, raw_transaction, pubkeyhash_version,
checksum_value) -> bool: # todo: better name
relevant = False
txid = raw_transaction["txid"]
signers = [] # todo: SIGHASH_ALL
for n, vin in enumerate(raw_transaction["vin"]):
if 'scriptSig' in vin:
public_key = vin['scriptSig']['asm'].split(' ')[1]
signers.append(
public_key_to_address(public_key, pubkeyhash_version,
checksum_value))
for i, vout in enumerate(raw_transaction["vout"]):
for item in vout.get("items", []):
# stream item
if item["type"] == "stream":
publishers = item["publishers"]
for publisher in publishers:
Address.create_if_not_exists(data_db, publisher)
if item["name"] == app.STREAM_TIMESTAMP:
relevant = True
comment = ''
if item['data']:
data = ubjson.loadb(unhexlify(item['data']))
if 'comment' in data:
comment += data.get('comment', '')
data_db.add(
Timestamp(txid=txid,
pos_in_tx=i,
hash=item["keys"][0],
comment=comment,
address=publishers[0]))
# flush for the primary key
data_db.flush()
elif item['name'] == app.STREAM_ALIAS:
alias = item["keys"][0]
# Sanity checks
if item["data"] or not is_valid_username(
alias) or len(publishers) != 1:
continue
relevant = True
data_db.add(
Alias(txid=txid,
pos_in_tx=i,
address=publishers[0],
alias=alias))
# flush for the primary key
data_db.flush()
elif item['name'] == app.STREAM_ISCC:
iscc = item["keys"]
if len(iscc) != 4:
continue
meta_id, content_id, data_id, instance_id = iscc
if ISCC.already_exists(data_db, meta_id, content_id,
data_id, instance_id):
continue
data = item['data']
if 'json' not in data or 'title' not in data['json']:
continue
relevant = True
data_db.add(
ISCC(txid=txid,
address=publishers[0],
meta_id=meta_id,
content_id=content_id,
data_id=data_id,
instance_id=instance_id,
title=data['json']['title']))
# flush for the primary key
data_db.flush()
# vote
for perm in vout.get('permissions', []):
relevant = True
for perm_type, changed in perm.items():
if changed and perm_type in permission_candidates:
for address in vout['scriptPubKey']['addresses']:
Address.create_if_not_exists(data_db, address)
Address.create_if_not_exists(data_db,
signers[vout['n']])
data_db.add(
Vote(txid=txid,
pos_in_tx=i,
from_address=signers[vout['n']],
to_address=address,
start_block=perm['startblock'],
end_block=perm['endblock'],
perm_type=perm_type))
# flush for the primary key
data_db.flush()
return relevant
def read_dbson(fname):
"""
Reads dbson
"""
with open(fname, 'rb') as f:
return to_dbson(ubjson.loadb(f.read()))
# Replace with IP and port shown in the RIOT console
target_if = socket.getaddrinfo('fd00::1337:0001' if len(sys.argv)<2 else sys.argv[1], 9000, socket.AF_INET6, socket.SOCK_STREAM)
(family, socktype, proto, canonname, sockaddr) = target_if[0]
# Wrap the command in OBJ_BEGIN and OBJ_END sequences
bincmd = b'\x00\x01' + ubjson.dumpb(command) + b'\x00\x02'
# Open socket and send command
s = socket.socket(family, socktype, proto)
s.connect(sockaddr)
s.send(bincmd)
# Collect the response. Will loop until a OBJ_END sequence arrives
data = b''
try:
prefix = s.recv(2)
if prefix == b'\x00\x01':
while data[-2:] != b'\x00\x02':
data += s.recv(1)
# Unescape bianry zeroes
if data[-2:]==b'\x00\x00':
data = data[:-1]
time.sleep(1/10000)
res = ubjson.loadb(data[:-2])
print(res)
else:
print("Wrong prefix")
except KeyboardInterrupt:
pass
s.close()
def handle(self):
"""Override the handle method"""
if UDP:
data = self.request[0].strip()
socket = self.request[1]
sockAddr = socket, self.client_address
if data == b'ACK':
with send_UDP_Lock: # we need to wait when sending is done
printdd(f'Got ACK from {self.client_address}')
if (sockAddr) not in self.server.ackCounts:
#DNPprintw('no ACK to delete from '+str(self.client_address))
pass
else:
#printi(croppedText(f'deleting {sockAddr}: {self.server.ackCounts[sockAddr][1].keys()}'))
with ackCount_Lock:
del self.server.ackCounts[sockAddr]
return
else:
data = self.request.recv(1024).strip()
try:
cmd = ubjson.loadb(data)
except:
msg = f'ERR.LS: wrong command format (not ubjson): {data}'
if UDP:
#_send_UDP(msg.encode('utf-8'), *sockAddr)
socket.sendto(b'\x00\x00\x00\x00', self.client_address)
return
#printi((f'Client {self.client_address} wrote:\n{cmd}'))
# retrieve previous source to server.lastPID LDO
try:
Device.server.lastPID.value[0] = _LDO_Handler.lastPID
# remember current source
_LDO_Handler.lastPID = '%s;%i %s %s'%(*self.client_address\
,cmd['pid'], cmd['username'], )
#print('lastPID now',_LDO_Handler.lastPID)
except:
pass
printd(f'Got command {cmd} from {self.client_address}')
try:
cmdArgs = cmd['cmd']
except:
raise KeyError("'cmd' key missing in request")
if cmdArgs[0] == 'unsubscribe':
#print(f'cmdArgs: {cmdArgs} from {self.client_address}')
for devName, dev in Server.DevDict.items():
printi(f'unsubscribing {self.client_address} from {devName}')
dev.unsubscribe(self.client_address)
return
if cmdArgs[0] == 'retransmit':
Server.Perf['Retransmits'] += 1
#print(f'Retransmit {cmdArgs} from {sockAddr}, ackCount:{_myUDPServer.ackCounts.keys()}')
if not sockAddr in _myUDPServer.ackCounts:
printw(f'sockaddr wrong\n{sockAddr}')
#for key in _myUDPServer.ackCounts:
# print(key)
#printw(croppedText(f'Retransmitting: {cmd}'))#: {_myUDPServer.ackCounts[sockAddr][0],_myUDPServer.ackCounts[sockAddr][1].keys()}'))
offsetSize = tuple(cmdArgs[1])
try:
chunk = _myUDPServer.ackCounts[sockAddr][1][offsetSize]
except Exception as e:
msg = f'in LDO_Handle: {e}, sa:{sockAddr[1]}, os:{offsetSize}'
printe(msg)
raise RuntimeError(msg)
#DNTprint(f'sending {len(chunk)} bytes of chunk {offsetSize} to {sockAddr[1]}')
socket.sendto(chunk, sockAddr[1])
return
try:
devName = cmdArgs[1][0][0].split(NSDelimiter)[1]
#print('subscriber for device '+devName)
if devName != '*':
dev = Server.DevDict[devName]
except KeyError:
printe(f'Device not supported: {devName}')
except:
printe(f'unexpected exception, cmdArgs: {cmdArgs}')
raise NameError(('Subscription should be of the form:'\
"[['host,dev1', [parameters]]]\ngot: "+str(cmdArgs[1])))
if cmdArgs[0] == 'subscribe':
#print(f'>register_subscriber {self.client_address} for cmd {cmdArgs}')
dev.register_subscriber(self.client_address, socket, cmdArgs[1])
return
r = _reply(cmdArgs, *sockAddr)
def load(self, data: bytes) -> Any:
if data[0] != 2:
raise psycopg.DataError(f"bad ubjson version number: {data[0]}")
return ubjson.loadb(data[1:])
# Testing ubjson library.
bin_file_name = 'test-{}.ubjson.bin'
with open(bin_file_name.format(i), 'wb') as bin_file:
start = time.time()
ubs = ubjson.dumpb(json_data)
end = time.time()
ubjson_pack_time += [end - start]
bin_file.write(ubs)
ubjson_file_size = os.stat(bin_file_name.format(i)).st_size
ubjson_files_sizes += [ubjson_file_size]
with open(bin_file_name.format(i), 'rb') as bin_file:
ubs = bin_file.read()
start = time.time()
ubjson.loadb(ubs)
end = time.time()
ubjson_unpack_time += [end - start]
ubjson_compression += [ubjson_file_size / json_file_size]
# Testing cbor library.
bin_file_name = 'test-{}.cbor.bin'
with open(bin_file_name.format(i), 'wb') as bin_file:
start = time.time()
cb = cbor.dumps(json_data)
end = time.time()
cbor_pack_time += [end - start]
bin_file.write(cb)
cbor_file_size = os.stat(bin_file_name.format(i)).st_size
def parse_new_issuance_metadata_10007(data):
pos = 0
searchdata = data[pos:]
# Multiple fields follow: field name (null delimited), variable length integer, raw data of field
fields = dict()
# If the property 'Open to follow-on issuance' is not present, we treat it as false.
opentoissuance = "Open to follow-on issuance"
fields[opentoissuance] = False
while pos < len(data):
# Is this a special property with meaning only for MultiChain?
if data[pos:pos + 1] == "\0":
assetproplen = ord(data[pos + 2:pos + 3])
assetprop = data[pos + 3:pos + 3 + assetproplen]
proptype = ord(data[pos + 1])
# Create stream has special properties
if proptype == 0x01:
fname = "Asset Name"
fields[fname] = assetprop
elif proptype == 0x02:
fname = opentoissuance
fields[fname] = bool(ord(assetprop) == 1)
elif proptype == 0x05: # JSON_DETAILS
fname = "JSON Data"
try:
json_data = ubjson.loadb(assetprop)
for k in ('text', 'json'):
if k in json_data:
json_data = json_data[k]
break
fields[fname] = render_long_data_with_popover(json.dumps(json_data))
except ValueError:
fields[fname] = long_hex(assetprop)
except:
break
elif proptype == 0x06: # RESTRICTIONS
restriction_code, = struct.unpack("<b", assetprop)
if restriction_code != 0:
fname = "Restrictions"
restrictions = []
if restriction_code & 0x02:
restrictions.append("send")
if restriction_code & 0x04:
restrictions.append("receive")
fields[fname] = ','.join(restrictions)
elif proptype == 0x41:
fname = "Quantity Multiple"
(multiplier,) = struct.unpack("<L", assetprop)
fields[fname] = multiplier
else:
fname = "MultiChain special property at offset {0}".format(pos)
fields[fname] = long_hex(assetprop)
pos = pos + 3 + assetproplen
continue
searchdata = data[pos:]
try:
fname = searchdata[:searchdata.index("\0")]
# print "field name: ", fname, " field name len: ", len(fname)
pos = pos + len(fname) + 1
# print "pos of vle: ", pos
# subdata = subdata[len(fname):]
flen = ord(data[pos:pos + 1])
pos += 1
# print "pos of payload: ", pos
if flen == 253:
(size,) = struct.unpack('<H', data[pos:pos + 2])
flen = size
pos += 2
elif flen == 254:
(size,) = struct.unpack('<I', data[pos:pos + 4])
flen = size
pos += 4
elif flen == 255:
(size,) = struct.unpack('<Q', data[pos:pos + 8])
flen = size
pos += 8
# print "pos of payload: ", pos
# print "payload length: ", flen
fields[fname] = data[pos:pos + flen]
pos += flen
except:
print('Ignorando erro...')
break
return fields
import ubjson
with open("/tmp/data.json", "rb") as f:
serialized = f.read()
data = ubjson.loadb(serialized)
print(data)
assert(data["name"] == "python-ubjson")
assert(data["versions"] == ["1", "2"])
assert(data["group"]["is_a_package"] is True)
assert(data["group"]["value"] == 42)
import ubjson
with open("/tmp/data.json", "rb") as f:
serialized = f.read()
data = ubjson.loadb(serialized)
print(data)
assert (data["name"] == "python-ubjson")
assert (data["versions"] == ["1", "2"])
assert (data["group"]["is_a_package"] is True)
assert (data["group"]["value"] == 42)
def _receive_dictio(sock, hostPort: tuple):
'''Receive and decode message from associated socket'''
# _printd('\n>receive_dictio')
if UDP:
# _printd('>_recvUdp')
data, addr = _recvUdp(sock, socketSize)
# _printd('<_recvUdp')
# acknowledge the receiving
try:
sock.sendto(b'ACK', hostPort)
except OSError as e:
_printw(f'OSError: {e}')
# _printd(f'ACK sent to {hostPort}')
#self.sock.sendto(b'ACK', self.hostPort)
#_printd('ACK2 sent to '+str(self.hostPort))
# else:
# if True:#try:
# data = self.sock.recv(self.recvMax)
# self.sock.close()
# addr = (self.lHost,self.lPort)
# else:#except Exception as e:
# _printw('in sock.recv:'+str(e))
# return {}
#print('received %i bytes'%(len(data)))
#_printd('received %i of '%len(data)+str(type(data))+' from '+str(addr)':')
# decode received data
# allow exception here, it will be caught in execute_cmd
if len(data) == 0:
_printw(f'empty reply for: {LastDictio}')
return {}
try:
decoded = ubjson.loadb(data)
except Exception as e:
_printw(f'exception in ubjson.load Data[{len(data)}]: {e}')
#print(str(data)[:150])
#raise ValueError('in _receive_dictio: '+msg)
return {}
#for key,val in decoded.items():
# print(f'received from {key}: {val.keys()}')
if not isinstance(decoded, dict):
#print('decoded is not dict')
return decoded
for cnsDev in decoded:
# items could by numpy arrays, the following should decode everything:
parDict = decoded[cnsDev]
for parName, item in list(parDict.items()):
# _printd(f'parName {parName}: {parDict[parName].keys()}')
# check if it is numpy array
shapeDtype = parDict[parName].get('numpy')
if shapeDtype is not None:
#print(f'par {parName} is numpy {shapeDtype}')
shape, dtype = shapeDtype
v = parDict[parName]['value']
# it is numpy array
from numpy import frombuffer
parDict[parName]['value'] =\
frombuffer(v,dtype).reshape(shape)
del parDict[parName]['numpy']
else:
#print(f'not numpy {parName}')
pass
# _printd(f'<receive_dictio')
return decoded
