def setUp(self):
self.parser: DefaultNFNParser = DefaultNFNParser()
synced_data_struct_factory = PiCNSyncDataStructFactory()
synced_data_struct_factory.register("cs", ContentStoreMemoryExact)
synced_data_struct_factory.register(
"fib", ForwardingInformationBaseMemoryPrefix)
synced_data_struct_factory.register("pit",
PendingInterstTableMemoryExact)
synced_data_struct_factory.register("computation_table",
NFNComputationList)
synced_data_struct_factory.register("faceidtable", FaceIDDict)
synced_data_struct_factory.create_manager()
cs = synced_data_struct_factory.manager.cs()
fib = synced_data_struct_factory.manager.fib()
pit = synced_data_struct_factory.manager.pit()
faceidtable = synced_data_struct_factory.manager.faceidtable()
self.r2cclient = TimeoutR2CHandler()
parser = DefaultNFNParser()
comp_table = synced_data_struct_factory.manager.computation_table(
self.r2cclient, parser)
self.optimizer: MapReduceOptimizer = MapReduceOptimizer(
cs, fib, pit, faceidtable)
def handle_interest_from_lower(self, face_id: int, interest: Interest, to_lower: multiprocessing.Queue):
self.logger.info("Incoming interest: " + interest.name.to_string())
# incoming interest is nfn expression
if interest.name.string_components[-1] == "NFN":
try:
parser = DefaultNFNParser()
nfn_str, prepended_name = parser.network_name_to_nfn_str(interest.name)
ast = parser.parse(nfn_str)
# assert that valid publish expression
if is_publish_expression(ast):
# store to database
data_name = ast.params[0]._element
payload = ast.params[1]._element
try:
payload = base64.b64decode(payload[7:])
self.logger.info("Payload is base64 encoded. Decoded.")
except:
self.logger.info("Invalid publish expression. The payload could not be decoded.")
nack = Nack(interest.name, reason=NackReason.COMP_NOT_PARSED, interest=interest)
self.queue_to_lower.put([face_id, nack])
self.cs.add_content_object(Content(data_name, payload))
self.logger.info("Add to database: " + data_name)
# reply confirmation
confirmation = Content(interest.name, "ok")
to_lower.put([face_id, confirmation])
else:
self.logger.info("Invalid publish expression. Wrong format.")
nack = Nack(interest.name, reason=NackReason.COMP_NOT_PARSED, interest=interest)
self.queue_to_lower.put([face_id, nack])
except:
self.logger.info("Invalid publish expression.")
nack = Nack(interest.name, reason=NackReason.COMP_NOT_PARSED, interest=interest)
self.queue_to_lower.put([face_id, nack])
# incoming interest is data request
else:
db_entry = self.cs.find_content_object(interest.name)
if db_entry is not None:
self.logger.info("Found in database")
to_lower.put([face_id, db_entry.content])
return
else:
self.logger.info("Not found in database")
nack = Nack(interest.name, NackReason.NO_CONTENT, interest)
to_lower.put([face_id, nack])
return
def test_handle_r2c_request(self):
"""test the handling of r2c messages"""
name = Name("/test/NFN")
comp_list = NFNComputationList(self.r2cClient, DefaultNFNParser())
comp_list.add_computation(name, 1, Interest(name))
r2c_request = self.r2cClient.R2C_create_message(name)
c = self.r2cClient.R2C_handle_request(r2c_request, comp_list)
self.assertEqual(c, Content(r2c_request, "Running"))
def parse_nfn_str(name: str) -> Name:
name = name.replace("""'""", "")
parser = DefaultNFNParser()
optimizer = BaseNFNOptimizer(None, None, None, None)
if '/NFN' in name:
name = name.replace("/NFN", "")
ast = parser.parse(name)
if ast is None:
return None
names = optimizer._get_names_from_ast(ast)
if names is None or names == []:
names = optimizer._get_functions_from_ast(ast)
if names is None or names == []:
return None
prepend_name = Name(names[0])
if prepend_name is None:
return None
name_str = optimizer._set_prepended_name(ast, prepend_name, ast)
if name_str is None:
return None
res = parser.nfn_str_to_network_name(name_str)
return res
def __init__(self, name: Name, id: int=0, interest: Interest=None, ast: AST=None, r2cclient: BaseR2CHandler=None,
parser: DefaultNFNParser=DefaultNFNParser()):
self.original_name: Name = name # original name of the computation
self.id = id
self.interest = interest
self.ast: AST = ast
self.r2cclient: BaseR2CHandler = r2cclient if r2cclient is not None else TimeoutR2CHandler() # r2c clients used for ageing
self.awaiting_data: List[NFNAwaitListEntry] = [] # data that are awaited by the computation
self.available_data: Dict[Name, Content] = {} # data that are required and now available
self.rewrite_list: List[Name] = [] # list of all possible rewrites
self.parser = parser
self.comp_state: NFNComputationState = NFNComputationState.START # marker where to continue this computation after requests
self.time_stamp = time.time() # time at which the computation was started
self.timeout = 4.0 #timeout before a request expires
def setUp(self):
#setup icn_layer
synced_data_struct_factory = PiCNSyncDataStructFactory()
synced_data_struct_factory.register("cs", ContentStoreMemoryExact)
synced_data_struct_factory.register(
"fib", ForwardingInformationBaseMemoryPrefix)
synced_data_struct_factory.register("pit",
PendingInterstTableMemoryExact)
synced_data_struct_factory.register("computation_table",
NFNComputationList)
synced_data_struct_factory.register("faceidtable", FaceIDDict)
synced_data_struct_factory.create_manager()
cs = synced_data_struct_factory.manager.cs()
fib = synced_data_struct_factory.manager.fib()
pit = synced_data_struct_factory.manager.pit()
faceidtable = synced_data_struct_factory.manager.faceidtable()
self.r2cclient = TimeoutR2CHandler()
parser = DefaultNFNParser()
comp_table = synced_data_struct_factory.manager.computation_table(
self.r2cclient, parser)
self.executor = {"PYTHON": NFNPythonExecutor()}
self.nfn_layer = BasicNFNLayer(cs,
fib,
pit,
faceidtable,
comp_table,
self.executor,
parser,
self.r2cclient,
log_level=255)
self.nfn_layer.queue_to_lower = multiprocessing.Queue()
self.nfn_layer.queue_from_lower = multiprocessing.Queue()
def setUp(self):
self.r2cclient = TimeoutR2CHandler()
self.computationList: NFNComputationList = NFNComputationList(self.r2cclient, DefaultNFNParser())
def __init__(self,
port=9000,
log_level=255,
encoder: BasicEncoder = None,
interfaces: List[BaseInterface] = None,
executors: BaseNFNExecutor = None,
ageing_interval: int = 3,
use_thunks=False):
# debug level
logger = Logger("NFNForwarder", log_level)
logger.info("Start PiCN NFN Forwarder on port " + str(port))
# packet encoder
if encoder is None:
self.encoder = SimpleStringEncoder(log_level=log_level)
else:
encoder.set_log_level(log_level)
self.encoder = encoder
# setup data structures
synced_data_struct_factory = PiCNSyncDataStructFactory()
synced_data_struct_factory.register("cs", ContentStoreMemoryExact)
synced_data_struct_factory.register(
"fib", ForwardingInformationBaseMemoryPrefix)
synced_data_struct_factory.register("pit",
PendingInterstTableMemoryExact)
synced_data_struct_factory.register("faceidtable", FaceIDDict)
synced_data_struct_factory.register("computation_table",
NFNComputationList)
synced_data_struct_factory.register("timeoutprevention_dict",
TimeoutPreventionMessageDict)
if use_thunks:
synced_data_struct_factory.register("thunktable", ThunkList)
synced_data_struct_factory.register("plantable", PlanTable)
synced_data_struct_factory.create_manager()
cs = synced_data_struct_factory.manager.cs()
fib = synced_data_struct_factory.manager.fib()
pit = synced_data_struct_factory.manager.pit()
faceidtable = synced_data_struct_factory.manager.faceidtable()
self.parser = DefaultNFNParser()
if use_thunks:
thunktable = synced_data_struct_factory.manager.thunktable()
plantable = synced_data_struct_factory.manager.plantable(
self.parser)
#setup chunkifier
self.chunkifier = SimpleContentChunkifyer()
# default interface
if interfaces is not None:
self.interfaces = interfaces
mgmt_port = port
else:
interfaces = [UDP4Interface(port)]
mgmt_port = interfaces[0].get_port()
# initialize layers
self.linklayer = BasicLinkLayer(interfaces,
faceidtable,
log_level=log_level)
self.packetencodinglayer = BasicPacketEncodingLayer(
self.encoder, log_level=log_level)
self.icnlayer = BasicICNLayer(log_level=log_level,
ageing_interval=ageing_interval)
self.chunklayer = BasicChunkLayer(self.chunkifier, log_level=log_level)
# setup nfn
self.icnlayer._interest_to_app = True
if executors is None:
self.executors = {"PYTHON": NFNPythonExecutor()}
else:
self.executors = executors
self.r2cclient = TimeoutR2CHandler()
comp_table = synced_data_struct_factory.manager.computation_table(
self.r2cclient, self.parser)
self.nfnlayer = BasicNFNLayer(cs,
fib,
pit,
faceidtable,
comp_table,
self.executors,
self.parser,
self.r2cclient,
log_level=log_level)
if use_thunks:
self.thunk_layer = BasicThunkLayer(cs,
fib,
pit,
faceidtable,
thunktable,
plantable,
self.parser,
log_level=log_level)
self.nfnlayer.optimizer = ThunkPlanExecutor(
cs, fib, pit, faceidtable, plantable)
timeoutprevention_dict = synced_data_struct_factory.manager.timeoutprevention_dict(
)
self.timeoutpreventionlayer = BasicTimeoutPreventionLayer(
timeoutprevention_dict, comp_table, pit=pit, log_level=log_level)
if use_thunks:
self.lstack: LayerStack = LayerStack([
self.nfnlayer, self.chunklayer, self.timeoutpreventionlayer,
self.thunk_layer, self.icnlayer, self.packetencodinglayer,
self.linklayer
])
else:
self.lstack: LayerStack = LayerStack([
self.nfnlayer, self.chunklayer, self.timeoutpreventionlayer,
self.icnlayer, self.packetencodinglayer, self.linklayer
])
self.icnlayer.cs = cs
self.icnlayer.fib = fib
self.icnlayer.pit = pit
# mgmt
self.mgmt = Mgmt(self.icnlayer.cs,
self.icnlayer.fib,
self.icnlayer.pit,
self.linklayer,
mgmt_port,
self.stop_forwarder,
log_level=log_level)
class test_DefaultNFNParser(unittest.TestCase):
"""Test the default Parser"""
def setUp(self):
self.parser = DefaultNFNParser()
pass
def tearDown(self):
pass
def test_parser_parsing_name(self):
"""Test the parser, parse a simple name"""
string = "/test/data"
ast = AST_Name("/test/data")
res = self.parser.parse(string)
self.assertTrue(isinstance(res, AST_Name))
self.assertEqual(ast._element, res._element)
def test_parser_parsing_simple_call(self):
"""Test the parser, parsing a simple call"""
string = "/call/func(/test/data)"
fc1 = AST_FuncCall("/call/func")
param1 = AST_Name("/test/data")
res: AST = self.parser.parse(string)
self.assertTrue(isinstance(res, AST_FuncCall))
self.assertEqual(fc1._element, res._element)
self.assertEqual(len(res.params), 1)
self.assertTrue(isinstance(res.params[0], AST_Name))
self.assertEqual(param1._element, res.params[0]._element)
def test_parser_parsing_simple_call_multiple_parameter(self):
"""Test the parser, parsing a simple call with multiple parameter"""
string = "/call/func(/test/data,2,X)"
fc1 = AST_FuncCall("/call/func")
param1 = AST_Name("/test/data")
param2 = AST_Int("2")
param3 = AST_Var("X")
res: AST = self.parser.parse(string)
self.assertTrue(isinstance(res, AST_FuncCall))
self.assertEqual(fc1._element, res._element)
self.assertEqual(len(res.params), 3)
self.assertTrue(isinstance(res.params[0], AST_Name))
self.assertEqual(param1._element, res.params[0]._element)
self.assertTrue(isinstance(res.params[1], AST_Int))
self.assertEqual(param2._element, res.params[1]._element)
self.assertTrue(isinstance(res.params[2], AST_Var))
self.assertEqual(param3._element, res.params[2]._element)
def test_parser_parsing_multiple_calls(self):
"""Test the parser, parsing multiple calls"""
string = '/call/func(/lib/wcount("HelloWorld"))'
fc1 = AST_FuncCall("/call/func")
fc2 = AST_FuncCall("/lib/wcount")
param1 = AST_String('"HelloWorld"')
res: AST = self.parser.parse(string)
self.assertTrue(isinstance(res, AST_FuncCall))
self.assertEqual(fc1._element, res._element)
self.assertEqual(len(res.params), 1)
self.assertTrue(isinstance(res.params[0], AST_FuncCall))
self.assertEqual(fc2._element, res.params[0]._element)
self.assertEqual(len(res.params[0].params), 1)
self.assertTrue(isinstance(res.params[0].params[0], AST_String))
self.assertEqual(param1._element, res.params[0].params[0]._element)
def test_parser_parsing_multiple_calls_multiple_parameter(self):
"""Test the parser, parsing multiple calls with multiple parameters"""
string = '/call/func(3,/lib/wcount(/call/libfun(/test/data),"HelloWorld"))'
fc1 = AST_FuncCall("/call/func")
fc2 = AST_FuncCall("/lib/wcount")
fc3 = AST_FuncCall("/call/libfun")
param1 = AST_Int("3")
param2 = AST_Int("/test/data")
param3 = AST_String('"HelloWorld"')
res: AST = self.parser.parse(string)
self.assertTrue(isinstance(res, AST_FuncCall))
self.assertEqual(fc1._element, res._element)
self.assertEqual(len(res.params), 2)
self.assertTrue(isinstance(res.params[0], AST_Int))
self.assertEqual(param1._element, res.params[0]._element)
self.assertTrue(isinstance(res.params[1], AST_FuncCall))
self.assertEqual(fc2._element, res.params[1]._element)
self.assertEqual(len(res.params[1].params), 2)
self.assertTrue(isinstance(res.params[1].params[0], AST_FuncCall))
self.assertEqual(fc3._element, res.params[1].params[0]._element)
self.assertEqual(len(res.params[1].params[0].params), 1)
self.assertTrue(isinstance(res.params[1].params[0].params[0], AST_Name))
self.assertEqual(param2._element, res.params[1].params[0].params[0]._element)
self.assertTrue(isinstance(res.params[1].params[1], AST_String))
self.assertEqual(param3._element, res.params[1].params[1]._element)
def test_parser_parsing_error1(self):
"""Test the parser, parsing call with syntax error 1"""
string = "/call/func(/test/data"
res: AST = self.parser.parse(string)
self.assertEqual(res, None)
def test_parser_parsing_error2(self):
"""Test the parser, parsing call with syntax error 2"""
string = "/call/func(/test/data,/lib/call(X)"
res: AST = self.parser.parse(string)
self.assertEqual(res, None)
def test_parser_parsing_error3(self):
"""Test the parser, parsing call with syntax error 3"""
string = "call/func(/test/data)"
res: AST = self.parser.parse(string)
self.assertEqual(res, None)
def test_ast_to_string(self):
"""Test converting the ast back to a string"""
string = '/call/func(3,/lib/wcount(/call/libfun(/test/data),"HelloWorld"))'
ast: AST = self.parser.parse(string)
res = str(ast)
self.assertEqual(string, res)
def test_sub_ast_to_string(self):
"""Test converting the ast back to a string"""
string = '/call/func(3,/lib/wcount(/call/libfun(/test/data),"HelloWorld"))'
part_str = '/lib/wcount(/call/libfun(/test/data),"HelloWorld")'
ast: AST = self.parser.parse(string)
part_ast = ast.params[1]
res = str(part_ast)
self.assertEqual(part_str, res)
def test_ast_to_string_nfn_marker(self):
"""Test converting the ast back to a string using nfn marker"""
string = '/call/func(3,/lib/wcount(/call/libfun(/test/data),"HelloWorld"))'
marked_str = '/call/func(3,%/lib/wcount%(/call/libfun(/test/data),"HelloWorld"))'
ast: AST = self.parser.parse(string)
ast.params[1]._prepend = True
res = str(ast)
self.assertEqual(marked_str, res)
def test_network_name_to_nfn_str(self):
"""Test transforming network name to nfn str"""
n = Name("/test/data")
function_str = '/call/func(3,/lib/wcount(/call/libfun(_),"HelloWorld"))'
cmp_str = '/call/func(3,/lib/wcount(/call/libfun(/test/data),"HelloWorld"))'
n += function_str
n += "NFN"
res, prepended = self.parser.network_name_to_nfn_str(n)
self.assertEqual(res, cmp_str)
self.assertEqual(prepended, Name("/test/data"))
def test_nfn_str_to_network_name(self):
"Test transforming nfn str to network name"
nfn_str = '/call/func(3,/lib/wcount(/call/libfun(%/test/data%),"HelloWorld"))'
compname = Name("/test/data")
function_str = '/call/func(3,/lib/wcount(/call/libfun(_),"HelloWorld"))'
compname += function_str
compname += "NFN"
res = self.parser.nfn_str_to_network_name(nfn_str)
self.assertEqual(res, compname)
def setUp(self):
self.parser = DefaultNFNParser()
pass
def __init__(self, parser: DefaultNFNParser = DefaultNFNParser()):
self.parser = parser
pass
def __init__(self, foldername: Optional[str], prefix: Name,
port=9000, log_level=255, encoder: BasicEncoder = None,
autoconfig: bool = False, autoconfig_routed: bool = False, interfaces: List[BaseInterface] = None,
use_thunks=False):
"""
:param foldername: If None, use an in-memory repository. Else, use a file system repository.
"""
logger = Logger("ICNRepoSession", log_level)
logger.info("Start PiCN Data Repository with Sessions")
# packet encoder
if encoder is None:
self.encoder = SimpleStringEncoder(log_level=log_level)
else:
encoder.set_log_level(log_level)
self.encoder = encoder
# chunkifyer
self.chunkifyer = SimpleContentChunkifyer()
# repo
manager = multiprocessing.Manager()
if foldername is None:
self.repo: BaseRepository = SimpleMemoryRepository(prefix, manager, logger)
else:
self.repo: BaseRepository = SimpleFileSystemRepository(foldername, prefix, manager, logger)
# initialize layers
synced_data_struct_factory = PiCNSyncDataStructFactory()
synced_data_struct_factory.register("faceidtable", FaceIDDict)
if use_thunks:
synced_data_struct_factory.register("thunktable", ThunkList)
synced_data_struct_factory.register("plantable", PlanTable)
synced_data_struct_factory.create_manager()
faceidtable = synced_data_struct_factory.manager.faceidtable()
if use_thunks:
self.parser = DefaultNFNParser()
thunktable = synced_data_struct_factory.manager.thunktable()
plantable = synced_data_struct_factory.manager.plantable(self.parser)
if interfaces is not None:
self.interfaces = interfaces
mgmt_port = port
else:
interfaces = [UDP4Interface(port)]
mgmt_port = interfaces[0].get_port()
self.linklayer = BasicLinkLayer(interfaces, faceidtable, log_level=log_level)
self.packetencodinglayer = BasicPacketEncodingLayer(self.encoder, log_level=log_level)
self.chunklayer = BasicChunkLayer(self.chunkifyer, log_level=log_level)
self.repolayer = SessionRepositoryLayer(self.repo, log_level=log_level)
if use_thunks:
self.thunklayer = BasicThunkLayer(None, None, None, faceidtable, thunktable, plantable, self.parser, self.repo, log_level=log_level)
logger.info("Using Thunks")
if use_thunks:
self.lstack: LayerStack = LayerStack([
self.repolayer,
self.chunklayer,
self.thunklayer,
self.packetencodinglayer,
self.linklayer
])
else:
self.lstack: LayerStack = LayerStack([
self.repolayer,
self.chunklayer,
self.packetencodinglayer,
self.linklayer
])
if autoconfig:
self.autoconfiglayer = AutoconfigRepoLayer(name=prefix.string_components[-1],
addr='127.0.0.1',
linklayer=self.linklayer, repo=self.repo,
register_global=autoconfig_routed, log_level=log_level)
self.lstack.insert(self.autoconfiglayer, below_of=self.chunklayer)
# mgmt
self.mgmt = Mgmt(None, None, None, self.linklayer, mgmt_port,
self.start_repo, repo_path=foldername,
repo_prfx=prefix, log_level=log_level)
def setUp(self):
self.planTable = PlanTable(DefaultNFNParser())
def __init__(self, parser: DefaultNFNParser = DefaultNFNParser()):
super().__init__(parser)