def unserialize(self, payload):
"""
Implements :func:`autobahn.wamp.interfaces.IObjectSerializer.unserialize`
"""
if self._batched:
msgs = []
N = len(payload)
i = 0
while i < N:
# read message length prefix
if i + 4 > N:
raise Exception("batch format error [1]")
l = struct.unpack("!L", payload[i:i + 4])[0]
# read message data
if i + 4 + l > N:
raise Exception("batch format error [2]")
data = payload[i + 4:i + 4 + l]
# append parsed raw message
msgs.append(cbor.loads(data))
# advance until everything consumed
i = i + 4 + l
if i != N:
raise Exception("batch format error [3]")
return msgs
else:
unpacked = cbor.loads(payload)
return [unpacked]
def unpack(self, row, key):
data = {
'key': key,
'url': row['f:url'],
'timestamp': int(row['f:timestamp']),
'request': {
'method': row['f:request.method'],
'client': cbor.loads(zlib.decompress(row['f:request.client'])),
'headers': cbor.loads(zlib.decompress(row['f:request.headers'])),
'body': cbor.loads(zlib.decompress(row['f:request.body'])),
},
'response': {
'status': row['f:response.status'],
'server': {
'hostname': row['f:response.server.hostname'],
'address': row['f:response.server.address'],
},
'headers': cbor.loads(zlib.decompress(row['f:response.headers'])),
'body': cbor.loads(zlib.decompress(row['f:response.body'])),
},
'indices': [],
}
for kk, vv in row.items():
mm = re.match(r"^f:index\.(?P<key>.*)\.[0-9]+$", kk)
if mm is not None:
data['indices'].append({'key':mm.group('key'), 'value': vv})
return data
def compare_set_request(req1, req2):
if len(req1.entries) != len(req2.entries):
return False
entries1 = [(e.address, cbor.loads(e.data)) for e in req1.entries]
entries2 = [(e.address, cbor.loads(e.data)) for e in req2.entries]
return entries1 == entries2
def test_sa_proposal_unsupported_macs(self):
msg = b'\x01' + cbor.dumps({'macs': ['hmac-md5']})
mac = self.get_mac(msg, self.R_a)
response = self.get_response(msg + mac).msg
sa = cbor.loads(response[1:-self.mac_len])
self.assertEqual(sa['mac'], 'sha3_256')
self.assertEqual(sa['mac_len'], 8)
def pull_req(self):
try:
data = cbor.dumps({
enums.TransferFields.AppName: self.appname,
enums.TransferFields.RequestType: enums.RequestType.Pull,
enums.TransferFields.Versions: self.parent_version
})
req_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# req_socket.settimeout(2)
self.logger.debug("Connecting to parent.")
req_socket.connect(self.parent)
self.logger.debug("Client Connection successful, sending data (pull req)")
req_socket.send(pack("!L", len(data)))
send_all(req_socket, data)
self.logger.debug("Data sent (pull req)")
content_length = unpack("!L", req_socket.recv(4))[0]
data = cbor.loads(receive_data(req_socket, content_length))
self.logger.debug("Data received (pull req).")
# Versions
new_versions = data[enums.TransferFields.Versions]
# Actual payload
package = data[enums.TransferFields.Data]
# Send bool status back.
req_socket.send(pack("!?", True))
self.logger.debug("Ack sent (pull req)")
req_socket.close()
self.parent_version = self.get_new_version(new_versions)
return package, new_versions
except Exception as e:
print ("PULL", e)
print(traceback.format_exc())
raise
def require_store(self, blockid):
"""Ensure that the store for this block (including all dependent
blocks) is loaded into the _blockmap
:param str blockid: identifier to associate with the block
"""
# this is all about removing recursion... yes, recursion is a useful
# thing... however python is not really very friendly to deep recursion
# and since this might go through the entire chain of blocks... seems
# like avoiding recursion is a very useful thing
# pass 1... build the list of blocks that we need to load in order
# to load the current block
blocklist = []
while blockid not in self._blockmap:
logger.info('add block %s to the queue for loading', blockid)
blocklist.insert(0, blockid)
if blockid not in self._persistmap:
raise KeyError('unknown block', blockid)
blockinfo = cbor2dict(self._persistmap[blockid])
blockid = blockinfo['PreviousBlockID']
# pass 2... starting with the oldest block, begin to load
# the stores
for blockid in blocklist:
logger.info('load block %s from storage', blockid)
blockinfo = cbor.loads(self._persistmap[blockid])
prevstore = self._blockmap[blockinfo['PreviousBlockID']]
blockstore = prevstore.clone_block(blockinfo, True)
blockstore.commit_block(blockid)
self._blockmap[blockid] = blockstore
def esIndex(ccaDir, team, crawler, index, docType, url=None, outPath=None, storeprefix=None):
if not url and not outPath:
raise Exception("Either Elastic Url or output path must be specified.")
ccaJsonList = list_files(ccaDir)
print "Processing ["+str(len(ccaJsonList))+"] files."
procCount = 0
failedList=[]
failedReasons=[]
CDRVersion = 2.0
outFile = codecs.open(outPath, 'w', 'utf-8') if outPath else None
for f in ccaJsonList:
with open(f, 'r') as fd:
try:
newDoc = {}
c = fd.read()
# fix for no request body out of Nutch CCA
c.replace("\"body\" : null", "\"body\" : \"null\"")
ccaDoc = json.loads(cbor.loads(c).value, encoding='utf8')
newDoc["url"] = ccaDoc["url"]
newDoc["timestamp"] = ccaDoc["imported"]
newDoc["team"] = team
newDoc["crawler"] = crawler
newDoc["raw_content"] = ccaDoc["response"]["body"]
newDoc["content_type"] = getContentType(ccaDoc)
parsed = parser.from_buffer(newDoc["raw_content"].encode("utf-8"))
newDoc["crawl_data"] = {}
if "content" in parsed:
newDoc["crawl_data"]["content"] = parsed["content"]
newDoc["extracted_text"] = parsed["content"]
# CDR version 2.0 additions
newDoc["_id"] = ccaDoc["key"]
newDoc["obj_original_url"] = ccaDoc["url"]
# newDoc["obj_parent"] = ??? Missing # TODO: get this field some how!
newDoc["obj_stored_url"] = url_to_nutch_dump_path(ccaDoc["url"], prefix=storeprefix)
newDoc["extracted_metadata"] = parsed["metadata"] if 'metadata' in parsed else {}
newDoc["version"] = CDRVersion
verboseLog("Indexing ["+f+"] to Elasticsearch.")
if url:
indexDoc(url, newDoc, index, docType)
if outFile:
outFile.write(json.dumps(newDoc))
outFile.write("\n")
procCount += 1
except Exception as err:
failedList.append(f)
failedReasons.append(str(err))
traceback.print_exc()
if outFile:
print("Output Stored at %s" % outPath)
outFile.close()
print "Processed " + str(procCount) + " CBOR files successfully."
print "Failed files: " + str(len(failedList))
if _verbose:
for i in range(len(failedList)):
verboseLog("File: "+failedList[i]+" failed because "+failedReasons[i])
def _get_intkey_data():
state = _get_intkey_state()
# state is a list of dictionaries: { data: ..., address: ... }
dicts = [cbor.loads(b64decode(entry['data'])) for entry in state]
LOGGER.debug(dicts)
data = {k:v for d in dicts for k, v in d.items()} # merge dicts
return data
def pull_req(self):
try:
data = cbor.dumps({
enums.TransferFields.AppName: self.appname,
enums.TransferFields.RequestType: enums.RequestType.Pull,
enums.TransferFields.Versions: self.parent_version
})
self.logger.debug(
"Client Connection successful, sending data (pull req)")
self.server_connection.send(pack("!L", len(data)))
send_all(self.server_connection, data)
self.logger.debug("Data sent (pull req)")
content_length = unpack("!L", self.server_connection.recv(4))[0]
resp = receive_data(self.server_connection, content_length)
data = cbor.loads(resp)
self.logger.debug("Data received (pull req).")
# Versions
new_versions = data[enums.TransferFields.Versions]
# Actual payload
package = data[enums.TransferFields.Data]
# Send bool status back.
self.server_connection.send(pack("!?", True))
self.logger.debug("Ack sent (pull req)")
self.parent_version = self.get_new_version(new_versions)
return package, new_versions
except Exception as e:
print ("PULL", e)
print(traceback.format_exc())
raise
def test_sa_proposal_macs(self):
msg = b'\x01' + cbor.dumps({'macs': ['sha3_512']})
mac = self.get_mac(msg, self.R_a)
response = self.get_response(msg + mac).msg
sa = cbor.loads(response[1:-8])
self.assertEqual(sa['mac'], 'sha3_512')
self.assertEqual(sa['mac_len'], 8)
def artifact_get(data, attributes, timestamp):
data = cbor.loads(zlib.decompress(data))
data['attributes'] = attributes
data['timestamp'] = timestamp
data['request']['body'] = artifact_decode(data['request']['body'])
data['response']['body'] = artifact_decode(data['response']['body'])
return data
def get_statuses(self, work_unit_keys):
# return [{'status':int, 'expration':bigint, 'worker_id':, 'traceback':}, ...]
out = []
with self.mutex:
with self._cursor() as cursor:
self._expire_stale_leases(cursor)
for wuk in work_unit_keys:
# TODO: many queries is slow, make big query, but
# can't return results from cursor.executemany()
cursor.execute('SELECT wudata, status, timeout, worker '
'FROM {schema}.wu '
'WHERE spec = %s AND wukey = %s'
.format(schema=self._schema),
(self.name, psycopg2.Binary(wuk)))
found = False
for row in cursor:
found = True
data = cbor.loads(bytes(row[0]))
status = row[1]
timeout = row[2]
worker = row[3] and bytes(row[3])
sdict = {'status': status, 'expration': timeout}
if worker:
sdict['worker_id'] = worker
if 'traceback' in data:
sdict['traceback'] = data['traceback']
out.append(sdict)
if not found:
out.append(None)
return out
def get_work(self, worker_id, lease_time, max_jobs):
'''Get an available work unit from this work spec.
If there are no available work units to do, returns
:const:`None`. This can also return :const:`None` if
there are already :attr:`max_running` pending work units.
If there is no work but this work spec is :attr:`continuous`,
could return a new synthetic work unit.
In all cases, if a work unit is returned, `worker_id` will
own it and it will move to :data:`PENDING` state.
:param str worker_id: worker that will own the work unit
:param long lease_time: latest time the worker will own the
work unit
:return: list of :class:`WorkUnit`, maybe empty
'''
out = []
max_jobs = nmin(max_jobs, self.max_getwork)
with self.mutex:
with self._cursor() as cursor:
self._expire_stale_leases(cursor)
cursor.execute('SELECT * '
'FROM {schema}.get_work(%s, %s, %s, %s)'
.format(schema=self._schema),
(self.name, max_jobs,
psycopg2.Binary(worker_id), long(lease_time)))
for row in cursor:
wukey = bytes(row[0])
wudata = cbor.loads(bytes(row[1]))
prio = row[2]
wu = WorkUnit(wukey, wudata, priority=prio)
out.append(wu)
return out
def esIndex(ccaDir, team, crawler, url, index, docType):
ccaJsonList = list_files(ccaDir)
print "Processing ["+str(len(ccaJsonList))+"] files."
procList=[]
failedList=[]
failedReasons=[]
for f in ccaJsonList:
ccaDoc = None
newDoc = {}
with open(f, 'r') as fd:
try:
c = fd.read()
# fix for no request body out of Nutch CCA
c.replace("\"body\" : null", "\"body\" : \"null\"")
ccaDoc = json.loads(cbor.loads(c), encoding='utf8')
newDoc["url"] = ccaDoc["url"]
newDoc["timestamp"] = ccaDoc["imported"]
newDoc["team"] = team
newDoc["crawler"] = crawler
newDoc["raw_content"] = ccaDoc["response"]["body"]
newDoc["content_type"] = getContentType(ccaDoc)
parsed = parser.from_buffer(newDoc["raw_content"].encode("utf-8"))
newDoc["crawl_data"] = {}
newDoc["crawl_data"]["content"] = parsed["content"]
verboseLog("Indexing ["+f+"] to Elasticsearch.")
indexDoc(url, newDoc, index, docType)
procList.append(f)
except ValueError, err:
failedList.append(f)
failedReasons.append(str(err))
def __call__(self, buf):
"""
CBOR Multicodec encoder functionality.
:params buf: CBOR string.
:return: CBOR Multicodec buffer.
"""
try:
cbor.loads(buf)
except (ValueError, EOFError):
raise exceptions.MalformedBufferError
encoded = header.add_header(buf, FORMAT)
return encoded
def get_batch(self, keys):
with self._lmdb.begin() as txn:
result = []
for key in keys:
packed = txn.get(key.encode())
if packed is not None:
result.append((key, cbor.loads(packed)))
return result
def get(self, key, index=None):
with self._lmdb.begin() as txn:
packed = txn.get(key.encode())
try:
return cbor.loads(packed)
except ValueError:
return None
def object_for_bytes(obj_bytes):
try:
decoded = cbor.loads(obj_bytes)
if isinstance(decoded, dict):
return decoded
except ValueError:
pass
return obj_bytes
def __cborsenml_set(self, payload, request_uri):
try:
new = cbor.loads(payload)
if len(new) != 1 or new[0].get(-2, "") + new[0].get(0, "") != request_uri:
raise BadRequest("Not a single record pertaining to this resource")
self.value = self.valuetype(new[self.cborsenml_key])
except (KeyError, ValueError):
raise BadRequest()
def from_kvlayer(key, data, path=None):
data = cbor.loads(data)
meta_data = data['meta_data']
user_data = data['user_data']
factory = Item
if data.get('is_folder', False):
factory = Folder
return factory(namespace=uni(key[0]), owner=uni(key[1]), inode=key[2],
path=path, meta_data=meta_data, data=user_data)
def worker_in(self, buf):
[name, arg, flags, serial, timeout] = cbor.loads(buf)
allow_reply = serial != 0
logging.debug('got rpc call {} with serial {}, allow_reply is {}, timeout_ms is {}'.format(name, serial, allow_reply, timeout))
if timeout != 0 and timeout < time.time() * 1000:
logging.debug('queue timeout')
else:
return self.worker_call(name, arg, serial, allow_reply)
async def recv_data(reader):
try:
con_succ = await reader.read(n=4)
content_length = unpack("!L", con_succ)[0]
data = await read_all(reader, content_length)
return cbor.loads(data)
except Exception as e:
print (e)
print (con_succ)
raise
def _collect_wu_kdps(self, out, cursor):
# Common iteration over query results for _get_work_units_* below
for row in cursor:
key = bytes(row[0])
data = cbor.loads(bytes(row[1]))
prio = row[2]
status = row[3]
nwu = WorkUnit(key, data, priority=prio)
nwu.status = status
out.append( (key, nwu) )
async def recv_objects(self):
while True:
# XXX this transacton affects the concurrent send task! is it a problem?
with self.db.ensure_transaction():
for i in range(5000):
data = await self.recv_sized()
if not data: break
obj = cbor.loads(data)
self.object_received(obj)
if not data: break
def get(self, key):
"""Retrieves a value associated with a key from the database
Args:
key (str): The key to retrieve
"""
with self._lmdb.begin() as txn:
packed = txn.get(key.encode())
if packed is not None:
return cbor.loads(packed)
def test_load_full(app, label_store, four_item_dump, tmpdir):
fn = tmpdir.join('dump.cbor')
fn.write(cbor.dumps(four_item_dump), mode='wb')
app.runcmd('load', [str(fn)])
assert (len(list(label_store.everything(include_deleted=True))) ==
len(four_item_dump))
assert (len(list(label_store.everything(include_deleted=False))) == 2)
assert app.stdout.getvalue() == ''
app.runcmd('dump_all', [])
assert cbor.loads(app.stdout.getvalue()) == four_item_dump
def cbor2dict(dictionary):
"""Deserializes CBOR into a dictionary.
Args:
dictionary (bytes): the CBOR object to deserialize.
Returns:
dict: a dictionary object reflecting the structure of the CBOR.
"""
return _ascii_encode_dict(cbor.loads(dictionary))
def _get_work_units(self, work_spec_name, num):
"yields (key, prio, data)"
assert isinstance(work_spec_name, basestring), 'bad work_spec_name type={} {!r}'.format(type(work_spec_name), work_spec_name)
assert isinstance(num, (int, long)), 'bad limit num type={} {!r}'.format(type(num), num)
db = self.db
c = db.cursor()
c.execute('SELECT key, prio, data FROM jobs WHERE name = ? ORDER BY name, prio DESC, key ASC LIMIT ?', (work_spec_name, num))
x = c.fetchone()
while x is not None:
yield x[0], x[1], cbor.loads(x[2])
x = c.fetchone()
def _get_state_data(name, context):
address = make_intkey_address(name)
state_entries = context.get_state([address])
try:
return cbor.loads(state_entries[0].data)
except IndexError:
return {}
except:
raise InternalError('Failed to load state data')
def from_cbor_bytes(cls, encoded):
import cbor
try:
# FIXME: don't silently accept trailing bytes
array = cbor.loads(encoded)
except: # FIXME in library: give one subclassable "failure to decode" error
raise link_header.ParseException("Not valid CBOR")
if not isinstance(array, list):
raise link_header.ParseException("Not a CBOR array")
return cls(Link.from_cbor_data(x) for x in array)
def undos(self, hdr):
db = self.open_epoch_db(hdr.slot()[0], readonly=True)
buf = db.get(b'u/' + hdr.hash())
if buf:
return cbor.loads(buf)
def __init__(self, payload=""):
try:
data = cbor.loads(payload)
action = data['action']
org_name = data['org_name']
creator_id = data['creator_id']
org_id = data['org_id']
new_member_id = ""
new_org_id = ""
curr_member_id = ""
new_org_creator = ""
certificate_hash = ""
can_mark_attendance = "False"
bdb_pub_key = ""
bdb_priv_key = ""
frmodel = ""
except ValueError:
raise InvalidTransaction("Incorrect Data Serialization")
if not action:
raise InvalidTransaction("Action not specified")
if not org_name:
raise InvalidTransaction("Organization Name not specified")
if not creator_id:
raise InvalidTransaction(
"Organization creator's DigitalID not specified")
if not org_id:
raise InvalidTransaction(
"Organization creator's Organizatin ID not specified")
if "new_member_id" in data:
new_member_id = data['new_member_id']
if "new_org_id" in data:
new_org_id = data['new_org_id']
if "curr_member_id" in data:
curr_member_id = data['curr_member_id']
if "new_org_creator" in data:
new_org_creator = data['new_org_creator']
if "certificate_hash" in data:
certificate_hash = data['certificate_hash']
if "can_mark_attendance" in data:
can_mark_attendance = data['can_mark_attendance']
if "bdb_pub_key" in data:
bdb_pub_key = data['bdb_pub_key']
if "bdb_priv_key" in data:
bdb_priv_key = data['bdb_priv_key']
if "frmodel" in data:
frmodel = data['frmodel']
self._action = action
self._org_name = org_name
self._creator_id = creator_id
self._org_id = org_id
self._new_member_id = new_member_id
self._new_org_id = new_org_id
self._curr_member_id = curr_member_id
self._new_org_creator = new_org_creator
self._certificate_hash = certificate_hash
self._can_mark_attendance = can_mark_attendance
self._bdb_pub_key = bdb_pub_key
self._bdb_priv_key = bdb_priv_key
self._frmodel = frmodel
def main(): # pragma: no cover
global client
op = None
path = None
payload = None
content_type = None
#ct = {'content_type': defines.Content_types["application/link-format"]}
ct = {}
ct['accept'] = 0
ct['ocf_accept_content_format_version'] = int(2048)
ct['ocf_content_format_version'] = int(2048)
try:
opts, args = getopt.getopt(sys.argv[1:], "ho:p:P:f:c:", [
"help", "operation=", "path=", "payload=", "payload_file=",
"content-type"
])
except getopt.GetoptError as err:
# print help information and exit:
print(
(str(err))) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
if o in ("-o", "--operation"):
op = a
elif o in ("-p", "--path"):
path = a
elif o in ("-P", "--payload"):
payload = a
elif o in ("-c", "--content-type"):
ct['accept'] = a
print("content type request : ", ct)
elif o in ("-f", "--payload-file"):
with open(a, 'r') as f:
payload = f.read()
elif o in ("-h", "--help"):
usage()
sys.exit()
else:
usage()
sys.exit(2)
if op is None:
print("Operation must be specified")
usage()
sys.exit(2)
if path is None:
print("Path must be specified")
usage()
sys.exit(2)
if not path.startswith("coap://"):
print("Path must be conform to coap://host[:port]/path")
usage()
sys.exit(2)
host, port, path = parse_uri(path)
try:
tmp = socket.gethostbyname(host)
host = tmp
except socket.gaierror:
pass
client = HelperClient(server=(host, port))
if op == "GET":
if path is None:
print("Path cannot be empty for a GET request")
usage()
sys.exit(2)
response = client.get(path, None, None, **ct)
print((response.pretty_print()))
if response.content_type == defines.Content_types["application/json"]:
json_data = json.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print("JSON ::")
print(json_string)
if response.content_type == defines.Content_types["application/cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print("JSON ::")
print(json_string)
if response.content_type == defines.Content_types[
"application/vnd.ocf+cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print("JSON ::")
print(json_string)
client.stop()
elif op == "GETNONE":
if path is None:
print("Path cannot be empty for a GET-None request")
usage()
sys.exit(2)
response = client.get_non(path, None, None, **ct)
print((response.pretty_print()))
if response.content_type == defines.Content_types["application/json"]:
json_data = json.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print("JSON ::")
print(json_string)
if response.content_type == defines.Content_types["application/cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print("JSON ::")
print(json_string)
if response.content_type == defines.Content_types[
"application/vnd.ocf+cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print("JSON ::")
print(json_string)
client.stop()
elif op == "OBSERVE":
if path is None:
print("Path cannot be empty for a GET request")
usage()
sys.exit(2)
client.observe(path, client_callback_observe)
elif op == "DELETE":
if path is None:
print("Path cannot be empty for a DELETE request")
usage()
sys.exit(2)
response = client.delete(path)
print((response.pretty_print()))
client.stop()
elif op == "POST":
if path is None:
print("Path cannot be empty for a POST request")
usage()
sys.exit(2)
if payload is None:
print("Payload cannot be empty for a POST request")
usage()
sys.exit(2)
print("payload for POST (ascii):", payload)
print(ct['accept'])
if ct['accept'] == str(defines.Content_types["application/cbor"]):
json_data = json.loads(payload)
cbor_data = cbor.dumps(json_data)
payload = bytes(cbor_data)
if ct['accept'] == str(
defines.Content_types["application/vnd.ocf+cbor"]):
json_data = json.loads(payload)
cbor_data = cbor.loads(json_data)
payload = cbor_data
response = client.post(path, payload, None, None, **ct)
print((response.pretty_print()))
if response.content_type == defines.Content_types["application/cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print(json_string)
if response.content_type == defines.Content_types[
"application/vnd.ocf+cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print(json_string)
client.stop()
elif op == "PUT":
if path is None:
print("Path cannot be empty for a PUT request")
usage()
sys.exit(2)
if payload is None:
print("Payload cannot be empty for a PUT request")
usage()
sys.exit(2)
response = client.put(path, payload)
print((response.pretty_print()))
client.stop()
elif op == "DISCOVER":
response = client.discover(path, client_callback, None, **ct)
#response = client.discover( path, None, None, **ct)
if response is not None:
print(response.pretty_print())
if response.content_type == defines.Content_types[
"application/cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print(json_string)
if response.content_type == defines.Content_types[
"application/vnd.ocf+cbor"]:
json_data = cbor.loads(response.payload)
json_string = json.dumps(json_data, indent=2, sort_keys=True)
print(json_string)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print("Client Shutdown")
client.stop()
else:
print("Operation not recognized")
usage()
sys.exit(2)
def _extract_extenal_aad(message, i_am_sender, request_partiv=None):
result = original_extract_external_aad(message, i_am_sender, request_partiv)
print("Verify: External AAD: bytes.fromhex(%r), %r"%(result.hex(), cbor.loads(result)))
return result
def from_suit(self, d):
self.v = c().from_suit(cbor.loads(d))
return self
def from_cbor(self, event): # Read in an Event from cbor format (parameter is bytes()), creates a new Event object
meta, signature, content = cbor.loads(event)
meta = Meta.from_cbor(meta)
content = Content.from_cbor(content)
return Event(meta, signature, content)
f.write(cbor_data)
f.close()
if (args.cbor):
cbor_raw = None
try:
cbor_raw = open(args.cbor, 'rb').read()
except:
print("cbor_to_txt: failed:", args.cbor)
traceback.print_exc()
if args.verbose:
print("cbor data:")
print(cbor_raw)
json_data = None
try:
json_data = cbor.loads(cbor_raw)
except:
print("cbor_to_txt: failed:", cbor_raw)
traceback.print_exc()
if args.verbose:
print("json data:")
print(json_data)
fp = open(str(args.cbor) + ".json", "w")
json_string = json.dumps(json_data, indent=2, sort_keys=True)
fp.write(json_string)
fp.close()
def get_file():
"""Get key file"""
global keys_obj
global failct
global private_key
#look for quadski
err, results = grasp.get_flood(asa_nonce, flood_obj)
if not err:
# results contains all the unexpired tagged objectives
# but really there should only be one...
if results == []:
grasp.tprint("Found no value for",flood_obj.name)
time.sleep(10) #pause for something to change...
return
else:
grasp.tprint("Found value for",flood_obj.name)
# recover quadski's public key
quadski_pub_key = serialization.load_pem_public_key(
results[0].objective.value,
backend=default_backend() )
else:
grasp.tprint("get_flood failed", grasp.etext[err])
return #not this time
#set up objective value
ciphertext = quadski_pub_key.encrypt(
password,
padding.OAEP(
mgf=padding.MGF1(algorithm=hashes.SHA256()),
algorithm=hashes.SHA256(),
label=None))
keys_obj.value = [ciphertext,pem]
#start of negotiating session
grasp.tprint("Asking for keys")
#discover a peer
if failct > 3:
failct = 0
grasp.tprint("Flushing", keys_obj.name, "discovery")
_, ll = grasp.discover(asa_nonce, keys_obj, 1000, flush = True)
else:
_, ll = grasp.discover(asa_nonce, keys_obj, 1000)
if ll==[]:
grasp.tprint("Discovery failed")
failct += 1
return
grasp.ttprint("Discovered locator", ll[0].locator)
#attempt to negotiate
err, snonce, received_obj = grasp.req_negotiate(asa_nonce, keys_obj, ll[0], 5000)
if err:
if err==grasp.errors.declined and received_obj!="":
_e = received_obj
else:
_e = grasp.etext[err]
grasp.tprint("Negotiation error:", _e)
failct += 1
grasp.tprint("Fail count", failct)
if _e == "Incorrect password":
get_pass()
return
elif (not err) and snonce:
grasp.ttprint("requested, session_nonce:",snonce,"received_obj",received_obj)
grasp.ttprint("Received raw reply",received_obj.value)
grasp.ttprint("received_obj is", received_obj.value)
if received_obj.value == b'':
grasp.tprint("Keys not found")
else:
grasp.ttprint("Starting transfer")
#decrypt block
plaintext = private_key.decrypt(
received_obj.value,
padding.OAEP(
mgf=padding.MGF1(algorithm=hashes.SHA256()),
algorithm=hashes.SHA256(),
label=None))
keys = cbor.loads(plaintext)
key = keys[0]
iv = keys[1]
file = open(r"quadsk.py","w")
file.write("key="+str(key)+"\n")
file.write("iv="+str(iv)+"\n")
file.close()
grasp.tprint("quadsk.py saved OK")
err = grasp.end_negotiate(asa_nonce, snonce, True)
if err:
grasp.tprint("end_negotiate error:",grasp.etext[err])
else:
#immediate end, strange
grasp.tprint("Unexpected reply", received_obj.value)
#end of a negotiating session
time.sleep(10) #pause for something to change...
return
def from_cbor(cls, content): # Read in a Content() object from cbor format
identifier, parameters = cbor.loads(content)
return Content(identifier, parameters)
def _decode(encoded):
return cbor.loads(encoded)
def _loads(self, data):
return cbor.loads(data)
def load_ndarray(path):
data = cbor.loads(open(path, 'br').read())
return np.reshape(data['data'], data['dim'])
def test_cbor_format(self):
cbored = cbor.dumps(self.expected_value)
key = 'format_demo_cbor'
self.db.set(key, cbored)
self.assertEqual(self.expected_value, cbor.loads(self.db.get(key)))
def deepcopy(the_dict):
# This is faster than actual deepcopy.
#return the_dict
return cbor.loads(cbor.dumps(the_dict))
# Testing cbor library.
bin_file_name = 'test-{}.cbor.bin'
with open(bin_file_name.format(i), 'wb') as bin_file:
start = time.time()
cb = cbor.dumps(json_data)
end = time.time()
cbor_pack_time += [end - start]
bin_file.write(cb)
cbor_file_size = os.stat(bin_file_name.format(i)).st_size
cbor_files_sizes += [cbor_file_size]
with open(bin_file_name.format(i), 'rb') as bin_file:
cb = bin_file.read()
start = time.time()
cbor.loads(cb)
end = time.time()
cbor_unpack_time += [end - start]
cbor_compression += [cbor_file_size / json_file_size]
# Зависимость степени сжатия от исходного объема файла JSON.
# pylab.plot(json_files_sizes, msgpack_compression, '-', label='msgpack')
# pylab.plot(json_files_sizes, bson_compression, '-', label='BSON')
# pylab.plot(json_files_sizes, ubjson_compression, '-', label='ubjson')
# pylab.plot(json_files_sizes, cbor_compression, '-', label='CBOR')
#
# legend = pylab.legend(fancybox=True, loc='upper left', ncol=4)
# legend.get_frame().set_alpha(1)
#
# pylab.xscale('log')
def run(self):
time.sleep(1) # avoid printing glitch
grasp.tprint("WARNING: you can't run this test suite more than once without restarting the Python context; it leaves GRASP data structures dirty!\n")
grasp.skip_dialogue(testing=True, selfing=True, diagnosing=True)
time.sleep(1) # just to avoid mixed up print output
####################################
# Test code: register ASA/objective#
####################################
test_obj = grasp.objective("EX1")
test_obj.loop_count = 2
test_obj.synch=True
err,test_nonce = grasp.register_asa("ASA-1")
if not err:
err = grasp.register_obj(test_nonce,test_obj,ttl=10000)
if not err:
grasp.tprint("ASA-1 and EX1 Registered OK")
####################################
# Test code: discover EX1 #
####################################
err, test_ll = grasp.discover(test_nonce, test_obj, 10000)
if len(test_ll)>0:
grasp.tprint("EX1 discovery result", test_ll[0].locator)
else:
grasp.tprint("No EX1 discovery response")
####################################
# Test code: register and discover #
# Boot ASA/objective #
####################################
err, boot_nonce = grasp.register_asa("Boot")
if err:
#we've got a problem...
raise RuntimeError("Can't register Boot as ASA")
boot_obj = grasp.objective("Boot")
boot_obj.loop_count = 2
boot_obj.synch = True
err = grasp.register_obj(boot_nonce, boot_obj,discoverable=True)
if err:
#we've got a different problem...
raise RuntimeError("Can't register Boot objective")
for i in range(3):
#test discovery 3 times, including artificial Divert
grasp.tprint("Test ASA: grasp._test_divert",grasp._test_divert)
err, boot_ll = grasp.discover(boot_nonce, boot_obj, 5000)
if len(boot_ll)>0:
grasp.tprint("Boot discovery result", boot_ll[0].locator)
grasp._test_divert = True
else:
grasp.tprint("No Boot discovery response")
time.sleep(5)
grasp._test_divert = False
####################################
# Test code: send Flood messages
####################################
obj1 = grasp.objective("Money")
obj1.synch = True
obj1.loop_count=2
err = grasp.register_obj(test_nonce,obj1)
obj1.value = [100,"NZD"]
obj2 = grasp.objective("Bling")
obj2.synch = True
obj2.loop_count=2
err = grasp.register_obj(test_nonce,obj2)
obj2.value = ["Diamonds", "Rubies"]
err = grasp.register_obj(test_nonce,obj2)
if err:
grasp.tprint(grasp.etext[err])
obj3 = grasp.objective("Intent.PrefixManager")
obj3.synch = True
obj3.loop_count=2
err = grasp.register_obj(test_nonce,obj3)
#obj3.value = '{"autonomic_intent":[{"model_version": "1.0"},{"intent_type": "Network management"},{"autonomic_domain": "Customer_X_intranet"},{"intent_name": "Prefix management"},{"intent_version": 73},{"Timestamp": "20150606 00:00:00"},{"Lifetime": "Permanent"},{"signature":"XXXXXXXXXXXXXXXXXXX"},{"content":[{"role": [{"role_name": "RSG"},{"role_characteristic":[{"prefix_length":"34"}]}]},{"role": [{"role_name": "ASG"},{"role_characteristic":[{"prefix_length": "44"}]}]},{"role": [{"role_name": "CSG"},{"role_characteristic":[{"prefix_length": "56"}]}]}]}]}'
#obj3.value = '{"autonomic_intent dummy text"}'
#obj3.value = bytes.fromhex('48deadbeefdeadbeef') #dummy CBOR
obj3.value = cbor.dumps(["Some","embedded","CBOR",[1,2,3]])
grasp.flood(test_nonce, 0, grasp.tagged_objective(obj1,None),
grasp.tagged_objective(obj2,None),
grasp.tagged_objective(obj3,None))
###################################
# Test code: Listen Synchronize as from Boot ASA
###################################
boot_obj.value = [1,"two",3]
boot_obj.synch = True
err = grasp.listen_synchronize(boot_nonce, boot_obj)
grasp.tprint("Listen synch", grasp.etext[err])
grasp.tprint(grasp._obj_registry[1].objective.name, "value", grasp._obj_registry[1].objective.value)
###################################
# Test code: call Synchronize as from EX1
###################################
time.sleep(5)
err, result = grasp.synchronize(test_nonce, obj2, None, 5000)
if not err:
grasp.tprint("Flooded synch obj2", result.value)
else:
grasp.tprint("Synch fail obj2", grasp.etext[err])
err, result = grasp.synchronize(test_nonce, obj3, None, 5000)
if not err:
grasp.tprint("Flooded synch obj3", result.value)
else:
grasp.tprint("Synch fail obj3", grasp.etext[err])
#this should fail as test_obj was neither flooded or listened for.
err, result = grasp.synchronize(test_nonce, test_obj, None, 5000)
if not err:
grasp.tprint("Synch test_obj (should fail)", result.value)
else:
grasp.tprint("Synch fail test_obj (should fail)", grasp.etext[err])
boot2_obj = grasp.objective("Boot")
boot2_obj.synch = True
err, result = grasp.synchronize(test_nonce, boot2_obj, None, 5000)
if not err:
grasp.tprint("Synch boot2_obj", result.name, result.value)
else:
grasp.tprint("Synch fail boot2_obj", grasp.etext[err])
###################################
# Test code: print obj_registry
# and flood cache
###################################
grasp.tprint("Objective registry contents:")
for x in grasp._obj_registry:
o= x.objective
grasp.tprint(o.name,"ASA:",x.asa_id,"Listen:",x.listening,"Neg:",o.neg,
"Synch:",o.synch,"Count:",o.loop_count,"Value:",o.value)
grasp.tprint("Flood cache contents:")
for x in grasp._flood_cache:
grasp.tprint(x.objective.name,"count:",x.objective.loop_count,"value:",
x.objective.value,"source:",x.source)
time.sleep(5)
###################################
# Test code: check flood cache for Tag 24 case
###################################
flobj = grasp.objective("Intent.PrefixManager")
flobj.synch = True
err, tobs = grasp.get_flood(test_nonce,flobj)
if not err:
for x in tobs:
try:
grasp.tprint("Flooded CBOR",x.objective.name,cbor.loads(x.objective.value))
except:
grasp.tprint("Flooded raw",x.objective.name,x.objective.value)
###################################
# Test code: deregister and then
# attempt synch
###################################
err = grasp.deregister_obj(boot_nonce, boot_obj)
if not err:
grasp.tprint("Deregistered Boot",)
else:
grasp.tprint("Deregister failure",grasp.etext[err])
err, result = grasp.synchronize(test_nonce, boot2_obj, None, 5000)
if not err:
grasp.tprint("Synch boot2_obj (should fail)", result.name, result.value)
else:
grasp.tprint("Synch fail boot2_obj (should fail)", grasp.etext[err])
###################################
# Test code: start in-host negotiate test
###################################
cheat_nonce = None
Neg1().start()
time.sleep(10)
Neg2().start()
#####################################
# Test code:
# test deregistering an ASA,
# and exit the thread
#####################################
grasp.deregister_asa(test_nonce,"ASA-1")
grasp.tprint("Exiting ASA test thread")
def is_valid(url, uniqueURLs):
try:
parsed = urlparse(url)
if parsed.scheme not in set(["http", "https"]):
return False
#Checks for right file types
p = not re.match(
r".*\.(css|js|bmp|gif|jpe?g|ico"
+ r"|png|tiff?|mid|mp2|mp3|mp4"
+ r"|wav|avi|mov|mpeg|ram|m4v|mkv|ogg|ogv|pdf"
+ r"|ps|eps|tex|ppt|pptx|doc|docx|xls|xlsx|names"
+ r"|data|dat|exe|bz2|tar|msi|bin|7z|psd|dmg|iso"
+ r"|epub|dll|cnf|tgz|sha1"
+ r"|thmx|mso|arff|rtf|jar|csv"
+ r"|rm|smil|wmv|swf|wma|zip|rar|gz)$", parsed.path.lower())
if not p:
return False
#Checks for the right domains
r = not not re.match(
r"((.*\.|)ics\.uci\.edu)|"
+ r"((.*\.|)cs\.uci\.edu)|"
+ r"((.*\.|)informatics\.uci\.edu)|"
+ r"((.*\.|)stat\.uci\.edu)|"
+ r"((www\.|)today\.uci\.edu\/department\/information_computer_sciences)",parsed.netloc.lower())
if not r:
return False
#Checks pages to avoid
for phrase in badPhrases:
if phrase in url:
return False
for l in blacklist:
if l in url:
return False
try:
#Reads robots.txt to check for disallows
robotPage = parsed.scheme + "://" + parsed.netloc.lower() + "/robots.txt"
config = uniqueURLs["@config"]
if rpDict.get(robotPage) == None:
#taken from the download method given to us but modified so we don't bypass the cache
#a time delay added so we keep with politeness
time.sleep(config.time_delay)
host, port = config.cache_server
resp = requests.get(
f"http://{host}:{port}/",
params=[("q", f"{robotPage}"), ("u", f"{config.user_agent}")],timeout=5)
if resp:
rResp = Response(cbor.loads(resp.content))
else:
rResp = Response({
"error": f"Spacetime Response error {resp} with url {robotPage}.",
"status": resp.status_code,
"url": robotPage})
if not (399 < rResp.status < 609):
#make sure the page was able to be opened
rString = rResp.raw_response.content.decode("utf-8")
linesList = rString.split("\n")
rp = urllib.robotparser.RobotFileParser()
rp.parse(linesList)
rpDict[robotPage] = rp
else:
rp = rpDict.get(robotPage)
if rp != None:
#make sure this page is allowed to be visited from this robots.txt page
if rp.can_fetch("*",url) == False:
return False
except:
#print("Timeout error (5 seconds):",robotPage)
pass
if url[-5:] == "/feed":
return False
return True
except TypeError:
print ("TypeError for ", parsed)
raise
def get_i_want_list(i_have_list):
list_of_extensions = sync.compare_feeds(cbor.loads(i_have_list))
return cbor.dumps(list_of_extensions), list_of_extensions
def _get_data():
state = _get_state()
# state is a list of dictionaries: { data: ..., address: ... }
dicts = [cbor.loads(b64decode(entry['data'])) for entry in state]
data = {k: v for d in dicts for k, v in d.items()} # merge dicts
return data
def hex_cbor(s):
"from hex-digits to parsed cbor"
return cbor.loads(bytes.fromhex(s.replace('\n', '').replace(' ', '')))
def tohex(b):
"binary string to hex-digits"
return binascii.hexlify(b).decode()
cose = hex_cbor(txt)
assert cose.tag == 18 # 18 = cose-sign1 type message
protected = cbor.loads(cose.value[0])
key = ". alg".split()
protected = {key[k]: protected[k] for k in protected}
unprotected = cose.value[1]
key = ". . . . kid".split()
unprotected = {key[k]: unprotected[k] for k in unprotected}
# now thanks to value[1] say kid = AsymmetricECDSA256
# we know that there are two more values, the
# protected data and the signature
payload = cbor.loads(cose.value[2])
# rename keys from 1,2,3,... to 'iss', sub','aud', ...
key = ". iss sub aud exp nbf iat cti".split()
payload = {key[k]: payload[k] for k in payload}
del key
async def get_global_transactions(self, request):
"""
DONT USE now instead list_transactions handler
"""
LOGGER.debug('list_transactions for validator')
paging_controls = self._get_paging_controls(request)
validator_query = client_transaction_pb2.ClientTransactionListRequest(
head_id=self._get_head_id(request),
transaction_ids=self._get_filter_ids(request),
sorting=self._get_sorting_message(request, "default"),
paging=self._make_paging_message(paging_controls))
response = await self._query_validator(
Message.CLIENT_TRANSACTION_LIST_REQUEST,
client_transaction_pb2.ClientTransactionListResponse,
validator_query)
data = [self._expand_transaction(t) for t in response['transactions']]
transactions = [cbor.loads(base64.b64decode(tx['payload'])) for tx in data]
# transactions = list(filter(lambda tx: 'Verb' in tx and tx['Verb'] == 'transfer',
# [cbor.loads(base64.b64decode(tx['payload'])) for tx in data]))
result_list = []
for tx in transactions:
if not isinstance(tx, dict) or \
'Verb' not in tx or \
tx['Verb'] != 'transfer':
continue
result_tx = {
'timestamp': datetime.now().__str__(),
'status': True,
'tx_payload': tx['num_bgt'],
'currency': tx['group_id'],
'address_from': tx['Name'],
'address_to': tx['to_addr'],
'fee': 0.1,
'extra': 'extra information'
}
try:
result_tx['address_from'] = _public2base64url(result_tx['address_from'])
except:
pass
try:
result_tx['address_to'] = _public2base64url(result_tx['address_to'])
except:
pass
result_list.append(result_tx)
LOGGER.debug('get_global_transactions result_list=%s',result_list)
return self._wrap_paginated_response(
request=request,
response=response,
controls=paging_controls,
data = None,
usermeta={
'transactions': result_list
})
def hex_cbor(s):
"from hex-digits to parsed cbor"
return cbor.loads(bytes.fromhex(s.replace('\n', '').replace(' ', '')))
def get_keys(self):
return {
key: value
for entry in self.get_data()
for key, value in cbor.loads(entry).items()
}
async def _send_data_request(self, appname, version):
reader, _ = await asyncio.open_connection()
self.read_queue.put((reader, appname, version))
data = cbor.loads(await reader.read())
await reader.close()
return data
def get_output(self, txin):
data = self.db.get(b'ut/t/' + cbor.dumps(txin))
if data:
return cbor.loads(data)
import time
import csv
import cbor
# Get a source dataset
places = {}
with open("kx-nz-place-names-nzgb-CSV/nz-place-names-nzgb.csv") as f:
src_data = csv.DictReader(f)
for row in src_data:
places[row['name']] = {
'id': int(row['feat_id']),
'X': float(row['ref_point_X']),
'Y': float(row['ref_point_Y'])
}
print("Places: %d" % len(places))
# Encode into cbor
start = time.time()
cbor_data = cbor.dumps(places)
print('Encoding time: %f' % (time.time() - start))
# Write out (for C++)
with open("cbor-python-wrote", 'wb') as f:
f.write(cbor_data)
# Parse back to source
start = time.time()
cbor_original = cbor.loads(cbor_data)
print('Parse time: %f' % (time.time() - start))
def iter_utxo(self):
from .wallet import TxIn, TxOut
prefix = b'ut/t/'
for k, v in iter_prefix(self.db, prefix):
yield TxIn(*cbor.loads(k[len(prefix):])), TxOut(*cbor.loads(v))
def parse_from_bytes(self, buffer):
"""Returns a consensus state object re-created from the serialized
consensus state provided.
Args:
buffer (bytes): A byte string representing the serialized
version of a consensus state to re-create. This was created
by a previous call to serialize_to_bytes
Returns:
ConsensusState: object representing the serialized byte string
provided
Raises:
ValueError: failure to parse into a valid ConsensusState object
"""
try:
# Deserialize the CBOR back into a dictionary and set the simple
# fields, doing our best to check validity.
self_dict = cbor.loads(buffer)
if not isinstance(self_dict, dict):
raise \
ValueError(
'buffer is not a valid serialization of a '
'ConsensusState object')
self._aggregate_local_mean = \
float(self_dict['_aggregate_local_mean'])
self._local_mean = None
self._population_samples = collections.deque()
for sample in self_dict['_population_samples']:
(duration, local_mean) = [float(value) for value in sample]
if not math.isfinite(duration) or duration < 0:
raise \
ValueError(
'duration ({}) is invalid'.format(duration))
if not math.isfinite(local_mean) or local_mean < 0:
raise \
ValueError(
'local_mean ({}) is invalid'.format(local_mean))
self._population_samples.append(
ConsensusState._PopulationSample(duration=duration,
local_mean=local_mean))
self._total_block_claim_count = \
int(self_dict['_total_block_claim_count'])
validators = self_dict['_validators']
if not math.isfinite(self.aggregate_local_mean) or \
self.aggregate_local_mean < 0:
raise \
ValueError(
'aggregate_local_mean ({}) is invalid'.format(
self.aggregate_local_mean))
if self.total_block_claim_count < 0:
raise \
ValueError(
'total_block_claim_count ({}) is invalid'.format(
self.total_block_claim_count))
if not isinstance(validators, dict):
raise ValueError('_validators is not a dict')
# Now walk through all of the key/value pairs in the the
# validators dictionary and reconstitute the validator state from
# them, again trying to validate the data the best we can. The
# only catch is that because the validator state objects are named
# tuples, cbor.dumps() treated them as tuples and so we lost the
# named part. When re-creating the validator state, are going to
# leverage the namedtuple's _make method.
self._validators = {}
for key, value in validators.items():
validator_state = ValidatorState._make(value)
self._check_validator_state(validator_state)
self._validators[str(key)] = validator_state
except (LookupError, ValueError, KeyError, TypeError) as error:
raise \
ValueError(
'Error parsing ConsensusState buffer: {}'.format(error))
def from_cbor(cls, meta): # Read in a Meta() object from cbor format
feed_id, seq_no, hash_of_prev, signature_info, hash_of_content = cbor.loads(meta)
return Meta(feed_id, seq_no, hash_of_prev, signature_info, hash_of_content)
def ParseFromString(self, str_value):
self.ParseFromDict(cbor.loads(str_value))
