def test_generic():
comp = b'\x08\x0andn-python'
assert Component.get_type(comp) == Component.TYPE_GENERIC
assert Component.to_str(comp) == 'ndn-python'
assert Component.from_str('ndn-python') == comp
assert Component.from_str('8=ndn-python') == comp
comp = b"\x08\x07foo%bar"
assert Component.to_str(comp) == "foo%25bar"
assert Component.from_str('foo%25bar') == comp
assert Component.from_str('8=foo%25bar') == comp
comp = b'\x08\x04-._~'
assert Component.to_str(comp) == "-._~"
assert Component.from_str('-._~') == comp
assert Component.from_str('8=-._~') == comp
with pytest.raises(ValueError):
Component.from_str(":/?#[]@")
comp = Component.from_bytes(b':/?#[]@')
assert Component.to_str(comp) == "%3A%2F%3F%23%5B%5D%40"
assert Component.from_str("%3a%2f%3f%23%5b%5d%40") == comp
with pytest.raises(ValueError):
Component.from_str("/")
assert Component.from_str('') == b'\x08\x00'
assert Component.from_bytes(b'') == b'\x08\x00'
def test_compare():
comps = [
Component.from_hex('0000000000000000000000000000000000000000000000000000000000000000', 1),
Component.from_hex('0000000000000000000000000000000000000000000000000000000000000001', 1),
Component.from_hex('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 1),
Component.from_hex('0000000000000000000000000000000000000000000000000000000000000000', 2),
Component.from_hex('0000000000000000000000000000000000000000000000000000000000000001', 2),
Component.from_hex('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 2),
Component.from_bytes(b'', 0x03),
Component.from_bytes(b'\x44', 0x03),
Component.from_bytes(b'\x46', 0x03),
Component.from_bytes(b'\x41\x41', 0x03),
Component.from_str(''),
Component.from_str('D'),
Component.from_str('F'),
Component.from_str('AA'),
Component.from_str('21426='),
Component.from_str('21426=%44'),
Component.from_str('21426=%46'),
Component.from_str('21426=%41%41')
]
for i, lhs in enumerate(comps):
for j, rhs in enumerate(comps):
assert (lhs == rhs) == (i == j)
assert (lhs != rhs) == (i != j)
assert (lhs < rhs) == (i < j)
assert (lhs <= rhs) == (i <= j)
assert (lhs > rhs) == (i > j)
assert (lhs >= rhs) == (i >= j)
async def fetch(self, obj_type: str, obj_name: bytes):
# Return if it exists
if self.repo.has_obj(obj_name) and obj_name not in self.incomplete_list:
return False
self.incomplete_list[obj_name] = obj_type
# Fetch object
packet_name = self.prefix + [Component.from_bytes(obj_name)]
wire = b''.join([bytes(seg) async for seg in segment_fetcher(self.app, packet_name, must_be_fresh=False)])
pack = SyncObject.parse(wire, ignore_critical=True)
fetched_obj_type = bytes(pack.obj_type).decode()
# Check type
if obj_type and obj_type != fetched_obj_type:
raise ValueError(f'{obj_type} is expected but get {fetched_obj_type}')
# Write into repo TODO: Transfer compressed data
h = hashlib.sha1(obj_type.encode() + b' ' + f'{len(pack.obj_data)}'.encode() + b'\x00')
h.update(pack.obj_data)
if h.digest() != obj_name:
raise ValueError(f'{obj_name} has a different digest')
self.repo.store_obj(bytes(pack.obj_type), bytes(pack.obj_data))
# Trigger recurisve fetching
if obj_type == "commit":
await self.traverse_commit(bytes(pack.obj_data))
elif obj_type == "tree":
await self.traverse_tree(bytes(pack.obj_data))
elif obj_type != "blob":
raise ValueError(f'Unknown data type {obj_type}')
del self.incomplete_list[obj_name]
def on_interest(self, name: FormalName, _param: InterestParam, _app_param: typing.Optional[BinaryStr]):
# Get the name and segment number
if Component.get_type(name[-1]) == Component.TYPE_SEGMENT:
obj_name = Component.get_value(name[-2])
seg_no = Component.to_number(name[-1])
else:
obj_name = Component.get_value(name[-1])
seg_no = 0
# Read the data
# Git objects are small so we can read the whole object
try:
obj_type, data = self.repo.read_obj(bytes(obj_name))
except ValueError:
logging.warning(f'Requested file {obj_name} does not exist in repo {self.repo.repo_name}')
return
# Extract the segment and calculate Name
data_name = self.prefix + [Component.from_bytes(obj_name), Component.from_segment(seg_no)]
start_pos = seg_no * SEGMENTATION_SIZE
data_seg = data[start_pos:start_pos + SEGMENTATION_SIZE]
packet_obj = SyncObject()
packet_obj.obj_type = obj_type.encode()
packet_obj.obj_data = data_seg
wire = packet_obj.encode()
final_block = (len(data) + SEGMENTATION_SIZE - 1) // SEGMENTATION_SIZE
self.app.put_data(data_name, wire,
freshness_period=3600000,
final_block_id=Component.from_segment(final_block))
logging.debug(f'Responded {obj_name} segment {final_block} in repo {self.repo.repo_name}')
async def query_face_id(app, uri, fuzzy_query=False):
query_filter = FaceQueryFilter()
query_filter.face_query_filter = FaceQueryFilterValue()
if not fuzzy_query:
query_filter.face_query_filter.uri = uri.encode('utf-8')
else:
query_filter.face_query_filter.uri_scheme = uri.encode('utf-8')
query_filter_msg = query_filter.encode()
name = Name.from_str("/localhost/nfd/faces/query") + [
Component.from_bytes(query_filter_msg)
]
try:
_, _, data = await app.express_interest(name,
lifetime=1000,
can_be_prefix=True,
must_be_fresh=True)
except (InterestCanceled, InterestTimeout, InterestNack, ValidationFailure,
NetworkError):
logging.error(f'Query failed')
return None
ret = FaceStatusMsg.parse(data)
logging.info(ret)
if not fuzzy_query:
return ret.face_status[0].face_id
else:
return ret
async def _check(self, method: str, repo_name, process_id: int) -> RepoCommandResponse:
"""
Return parsed insert check response message.
# TODO: Use command interests instead of regular interests
"""
cmd_param = RepoCommandParameter()
cmd_param.process_id = process_id
cmd_param_bytes = cmd_param.encode()
name = repo_name[:]
name.append(method + ' check')
name.append(Component.from_bytes(cmd_param_bytes))
try:
print(f'Expressing interest: {Name.to_str(name)}')
data_name, meta_info, content = await self.app.express_interest(
name, must_be_fresh=True, can_be_prefix=False, lifetime=1000)
print(f'Received data name: {Name.to_str(data_name)}')
except InterestNack as e:
print(f'Nacked with reason={e.reason}')
return None
except InterestTimeout:
print(f'Timeout: {Name.to_str(name)}')
return None
try:
cmd_response = RepoCommandResponse.parse(content)
except DecodeError as exc:
logging.warning('Response blob decoding failed')
return None
except Exception as e:
print(e)
return cmd_response
def encode(self) -> Component:
model = MetaDataModel()
model.source = self.source
model.tseqno = self.tseqno
model.nulled = self.nulled
model.nopcks = self.nopcks
return Component.from_bytes(model.encode())
def test_state_vector_decode():
enc_sv = b'\xCA\x03\x6F\x6E\x65\xCB\x01\x01\xCA\x03\x74\x77\x6F\xCB\x01\x02'
enc_sv = Component.from_bytes(enc_sv, StateVectorModelTypes.VECTOR.value)
sv = StateVector(enc_sv)
assert sv.get("one") == 1
assert sv.get("two") == 2
def test_basic_encode():
uri = ('/Emid/25042=P3//./%1C%9F'
'/sha256digest=0415e3624a151850ac686c84f155f29808c0dd73819aa4a4c20be73a4d8a874c')
name = Name.from_str(uri)
assert len(name) == 6
assert name[0] == Component.from_bytes(b'Emid')
assert name[1] == b'\xfd\x61\xd2\x02\x50\x33'
assert name[2] == Component.from_bytes(b'')
assert name[3] == Component.from_bytes(b'.')
assert name[4] == Component.from_bytes(b'\x1C\x9F')
assert Component.get_type(name[5]) == Component.TYPE_IMPLICIT_SHA256
assert Name.encoded_length(name) == 57
assert (Name.encode(name) ==
b'\x07\x37\x08\x04Emid\xfda\xd2\x02P3\x08\x00\x08\x01.\x08\x02\x1c\x9f'
b'\x01 \x04\x15\xe3bJ\x15\x18P\xachl\x84\xf1U\xf2\x98\x08\xc0\xdds\x81'
b'\x9a\xa4\xa4\xc2\x0b\xe7:M\x8a\x87L')
def test_state_vector_decode() -> None:
# hard coded bytes of component vector based on SVS protocol
enc_sv = b'\xCA\x03\x6F\x6E\x65\xCB\x01\x01\xCA\x03\x74\x77\x6F\xCB\x01\x02'
enc_sv = Component.from_bytes(enc_sv, SVSyncTlvTypes.VECTOR.value)
sv: StateVector = StateVector(enc_sv)
assert sv.get("one") == 1
assert sv.get("two") == 2
def _unsubscribe_inactive_processes(self):
for process_id, last_check_tp in self.process_id_to_last_check_tp.items():
if last_check_tp > int(time.time()) + 10:
topic = process_id_to_check_prefix[process_id] + ['check', Component.from_bytes(process_id)]
self.pb.unsubscribe(topic)
logging.info('CommandChecker unsubscribed from {}'.format(Name.to_str(topic)))
del self.process_id_to_response[process_id]
del self.process_id_to_last_check_tp[process_id]
del self.process_id_to_check_prefix[process_id]
async def publish(self, topic: NonStrictName, msg: bytes):
"""
Publish ``msg`` to ``topic``. Make several attempts until the subscriber returns a\
response.
:param topic: NonStrictName. The topic to publish ``msg`` to.
:param msg: bytes. The message to publish. The pub-sub API does not make any assumptions on\
the format of this message.
:return: Return true if received response from a subscriber.
"""
logging.info(f'publishing a message to topic: {Name.to_str(topic)}')
# generate a nonce for each message. Nonce is a random sequence of bytes
nonce = os.urandom(4)
# wrap msg in a data packet named /<publisher_prefix>/msg/<topic>/nonce
data_name = Name.normalize(self.prefix + ['msg'] + topic +
[Component.from_bytes(nonce)])
self.published_data[data_name] = self.app.prepare_data(data_name, msg)
# prepare notify interest
int_name = topic + ['notify']
app_param = NotifyAppParam()
app_param.publisher_prefix = self.prefix
app_param.notify_nonce = nonce
if self.forwarding_hint:
app_param.publisher_fwd_hint = ForwardingHint()
app_param.publisher_fwd_hint.name = self.forwarding_hint
aio.ensure_future(self._erase_publisher_state_after(data_name, 5))
# express notify interest
n_retries = 3
is_success = False
while n_retries > 0:
try:
logging.debug(
f'sending notify interest: {Name.to_str(int_name)}')
_, _, _ = await self.app.express_interest(int_name,
app_param.encode(),
must_be_fresh=False,
can_be_prefix=False)
is_success = True
break
except InterestNack as e:
logging.debug(f'Nacked with reason: {e.reason}')
await aio.sleep(1)
n_retries -= 1
except InterestTimeout:
logging.debug(f'Timeout')
n_retries -= 1
# if receiving notify response, the subscriber has finished fetching msg
if is_success:
logging.debug(f'received notify response for: {data_name}')
else:
logging.debug(f'did not receive notify response for: {data_name}')
await self._erase_publisher_state_after(data_name, 0)
return is_success
def sha256_tester(typ, uri_prefix):
hex_text = '%28%ba%d4%b5%27%5b%d3%92%db%b6%70%c7%5c%f0%b6%6f%13%f7%94%2b%21%e8%0f%55%c0%e8%6b%37%47%53%a5%48'
hex_lower = ''.join(hex_text.split('%'))
hex_upper = hex_lower.upper()
comp = Component.from_bytes(bytes.fromhex(hex_upper), typ=typ)
assert Component.get_type(comp) == typ
assert Component.to_str(comp) == uri_prefix + hex_lower
assert Component.from_str(uri_prefix + hex_lower) == comp
assert Component.from_str(uri_prefix + hex_upper) == comp
async def _process_notify_interest(self, int_name, int_param, app_param):
"""
Async helper for ``_on_notify_interest()``.
"""
logging.debug(f'received notify interest: {Name.to_str(int_name)}')
topic = int_name[:-2] # remove digest and `notify`
# parse notify interest
app_param = NotifyAppParam.parse(app_param)
publisher_prefix = app_param.publisher_prefix
notify_nonce = app_param.notify_nonce
publisher_fwd_hint = app_param.publisher_fwd_hint
int_param = InterestParam()
if publisher_fwd_hint:
# support only 1 forwarding hint now
int_param.forwarding_hint = [(0x0, publisher_fwd_hint.name)]
# send msg interest, retransmit 3 times
msg_int_name = publisher_prefix + ['msg'] + topic + [
Component.from_bytes(notify_nonce)
]
n_retries = 3
# de-duplicate notify interests of the same nonce
if notify_nonce in self.nonce_processed:
logging.info(
f'Received duplicate notify interest for nonce {notify_nonce}')
return
self.nonce_processed.add(notify_nonce)
aio.ensure_future(self._erase_subsciber_state_after(notify_nonce, 60))
while n_retries > 0:
try:
logging.debug(
f'sending msg interest: {Name.to_str(msg_int_name)}')
data_name, meta_info, msg = await self.app.express_interest(
msg_int_name, int_param=int_param)
break
except InterestNack as e:
logging.debug(f'Nacked with reason: {e.reason}')
await aio.sleep(1)
n_retries -= 1
except InterestTimeout:
logging.debug(f'Timeout')
n_retries -= 1
if msg == None:
return
# pass msg to application
logging.info(f'received subscribed msg: {Name.to_str(msg_int_name)}')
self.topic_to_cb[topic](bytes(msg))
# acknowledge notify interest with an empty data packet
logging.debug(f'acknowledging notify interest {Name.to_str(int_name)}')
self.app.put_data(int_name, None)
def _announce_process_status(self):
"""
Announce the status of all active processes over PubSub. Each process status is published\
to topic /<check_prefix>/check/<process_id>.
"""
for process_id, status in self.m_process_id_to_status.items():
check_prefix = self.m_process_id_to_check_prefix[process_id]
topic = check_prefix + ['check', Component.from_bytes(process_id)]
msg = status.encode()
# do not care about whether the subscriber acknowledges
aio.ensure_future(self.pb.publish(topic, msg))
def main():
parser = argparse.ArgumentParser(description='python client.py')
parser.add_argument('-n',
'--node_prefix',
required=True,
help='Prefix of catalog ("/217B/repo/node/A")')
parser.add_argument('-c',
'--command',
default='insert',
choices=['insert', 'delete', 'recall'],
help='Command Verb')
parser.add_argument('-d',
'--data_name',
required=True,
help='data name ("/foo/bar/1.txt")')
parser.add_argument('-s',
'--hash',
required=True,
help='data hash ("1bd109fe")')
parser.add_argument('-o',
'--desired_copies',
type=int,
default=3,
help='desired copies')
args = parser.parse_args()
logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
app = NDNApp(face=None, keychain=KeychainDigest())
name = Name.from_str(args.node_prefix)
name.append(Component.from_str(args.command))
datainfo_tlv_model = DatainfoTlvModel()
datainfo_tlv_model.data_name = args.data_name.encode()
datainfo_tlv_model.hash = args.hash.encode()
datainfo_tlv_model.desired_copies = args.desired_copies
datainfo_name_component = Component.from_bytes(datainfo_tlv_model.encode())
name.append(datainfo_name_component)
# logging.info(name)
try:
app.run_forever(after_start=send(app, name))
except FileNotFoundError:
logging.error('Error: could not connect to NFD.\n')
return 0
def check(self, check_prefix: NonStrictName, process_id: bytes) -> Optional[RepoCommandResponse]:
"""
Check the status of process ``process_id`` published under ``check_prefix``. This function\
returns the in-memory process status, therefore it returns immediately.
:return: Optional[RepoCommandResponse]. The last known status of ``process_id``. Therefore,\
the first call to a process, it returns None.
"""
# If process_id is not seen before, subscribe to its status with PubSub
if process_id not in self.process_id_to_response:
topic = check_prefix + ['check', Component.from_bytes(process_id)]
cb = self.make_on_msg(process_id)
self.pb.subscribe(topic, cb)
self.process_id_to_response[process_id] = None
self.process_id_to_check_prefix[process_id] = check_prefix
logging.info('CommandChecker subscribing to {}'.format(Name.to_str(topic)))
# Remember when this process is last checked
self.process_id_to_last_check_tp[process_id] = int(time.time())
return self.process_id_to_response[process_id]
async def query_face_id(self, uri):
query_filter = FaceQueryFilter()
query_filter.face_query_filter = FaceQueryFilterValue()
query_filter.face_query_filter.uri = uri
query_filter_msg = query_filter.encode()
name = Name.from_str("/localhost/nfd/faces/query") + [
Component.from_bytes(query_filter_msg)
]
try:
_, _, data = await self.app.express_interest(name,
lifetime=1000,
can_be_prefix=True,
must_be_fresh=True)
except (InterestCanceled, InterestTimeout, InterestNack,
ValidationFailure, NetworkError):
logging.error(f'Query failed')
return None
msg = FaceStatusMsg.parse(data)
if len(msg.face_status) <= 0:
return None
return msg.face_status[0].face_id
async def send_repo_command(self, node_prefix : NonStrictName, verb : str , datainfo : DatainfoTlvModel):
# "/a/b" -> list of bytearrays
name = Name.normalize(node_prefix)
# "/a/b" -> "/a/b/insert"
name.append(Component.from_str(verb))
datainfo_name_component = Component.from_bytes(datainfo.encode())
name.append(datainfo_name_component)
logging.info('Interest Sent: {}'.format(Name.to_str(name)))
try:
data_name, meta_info, content = await self.app.express_interest(name, must_be_fresh=True, can_be_prefix=False, nonce=gen_nonce(), lifetime=1000)
logging.info('Data Received: {}\n'.format(Name.to_str(data_name)))
# print(meta_info)
# print(bytes(content) if content else None)
except InterestNack as e:
# A NACK is received
logging.warning(f'Interest Nacked with reason={e.reason}\n')
return 0
except InterestTimeout:
# Interest times out
logging.warning(f'Interest Timeout\n')
return 0
# results = self.parse_results(content)
# logging.info(results)
return 1
def encode(self) -> Component:
model:MetaDataModel = MetaDataModel()
model.source, model.tseqno, model.nopcks = self.source, self.tseqno, self.nopcks
return Component.from_bytes(model.encode())
async def delete_file(self, prefix, start_block_id: int,
end_block_id: int):
"""
Delete data packets between [<name_at_repo>/<start_block_id>, <name_at_repo>/<end_block_id>]
from the remote repo.
:param prefix: NonStrictName. The name with which this file is stored in the repo.
:param start_block_id: int.
:param end_block_id: int.
"""
# Send command interest
cmd_param = RepoCommandParameter()
cmd_param.name = prefix
cmd_param.start_block_id = start_block_id
cmd_param.end_block_id = end_block_id
cmd_param_bytes = cmd_param.encode()
# Send cmd interests to repo
name = self.repo_name[:]
name.append('delete')
name.append(Component.from_bytes(cmd_param_bytes))
try:
logging.info(f'Expressing interest: {Name.to_str(name)}')
data_name, meta_info, content = await self.app.express_interest(
name, must_be_fresh=True, can_be_prefix=False, lifetime=1000)
logging.info(f'Received data name: {Name.to_str(data_name)}')
except InterestNack as e:
logging.warning(f'Nacked with reason: {e.reason}')
return
except InterestTimeout:
logging.warning(f'Timeout')
return
# Parse response from repo
try:
cmd_response = RepoCommandResponse.parse(content)
except DecodeError as exc:
logging.warning('Response blob decoding failed')
return
process_id = cmd_response.process_id
status_code = cmd_response.status_code
logging.info(
f'cmd_response process {process_id} accepted: status code {status_code}'
)
# Send delete check interest wait until delete process completes
checker = CommandChecker(self.app)
while True:
response = await checker.check_delete(self.repo_name, process_id)
if response is None:
logging.info(f'Response code is None')
await aio.sleep(1)
elif response.status_code == 300:
logging.info(f'Response code is {response.status_code}')
await aio.sleep(1)
elif response.status_code == 200:
logging.info(
'Delete process {} status: {}, delete_num: {}'.format(
process_id, response.status_code, response.delete_num))
break
else:
# Shouldn't get here
assert False
async def insert_file(self, file_path: str, name_at_repo):
"""
Insert a file to remote repo.
:param file_path: Local FS path to file to insert
:param name_at_repo: Name used to store file at repo
"""
num_packets = self._prepare_data(file_path, name_at_repo)
if num_packets == 0:
return
# Register prefix for responding interests from repo
# self.app.route(name_at_repo)(self._on_interest)
await self.app.register(name_at_repo, self._on_interest)
cmd_param = RepoCommandParameter()
cmd_param.name = name_at_repo
cmd_param.start_block_id = 0
cmd_param.end_block_id = num_packets - 1
cmd_param_bytes = cmd_param.encode()
# Send cmd interest to repo
name = self.repo_name[:]
name.append('insert')
name.append(Component.from_bytes(cmd_param_bytes))
try:
logging.info(f'Expressing interest: {Name.to_str(name)}')
data_name, meta_info, content = await self.app.express_interest(
name, must_be_fresh=True, can_be_prefix=False, lifetime=1000)
logging.info(f'Received data name: {Name.to_str(data_name)}')
except InterestNack as e:
logging.warning(f'Nacked with reason: {e.reason}')
return
except InterestTimeout:
logging.warning(f'Timeout')
return
# Parse response from repo
try:
cmd_response = RepoCommandResponse.parse(content)
except DecodeError as exc:
logging.warning('Response blob decoding failed')
return
process_id = cmd_response.process_id
status_code = cmd_response.status_code
logging.info(f'cmd_response process {process_id} accepted: status code {status_code}')
# Send insert check interest to wait until insert process completes
checker = CommandChecker(self.app)
while True:
response = await checker.check_insert(self.repo_name, process_id)
if response is None:
logging.info(f'Response code is None')
await aio.sleep(1)
elif response.status_code == 300:
logging.info(f'Response code is {response.status_code}')
await aio.sleep(1)
elif response.status_code == 200:
logging.info('Insert process {} status: {}, insert_num: {}'
.format(process_id,
response.status_code,
response.insert_num))
break
else:
# Shouldn't get here
assert False
