def __init__(self, dht_addr): self.my_addr = dht_addr self.my_id = identifier.RandomId() self.my_node = Node(self.my_addr, self.my_id) self.tracker = tracker.Tracker() self.token_m = token_manager.TokenManager() self.reactor = ThreadedReactor() self.rpc_m = RPCManager(self.reactor, self.my_addr[1]) self.querier = Querier(self.rpc_m, self.my_id) self.routing_m = RoutingManager(self.my_node, self.querier, bootstrap_nodes) self.responder = Responder(self.my_id, self.routing_m, self.tracker, self.token_m) self.responder.set_on_query_received_callback( self.routing_m.on_query_received) self.querier.set_on_response_received_callback( self.routing_m.on_response_received) self.querier.set_on_error_received_callback( self.routing_m.on_error_received) self.querier.set_on_timeout_callback(self.routing_m.on_timeout) self.querier.set_on_nodes_found_callback(self.routing_m.on_nodes_found) self.routing_m.do_bootstrap() self.rpc_m.add_msg_callback(QUERY, self.responder.on_query_received) self.lookup_m = LookupManager(self.my_id, self.querier, self.routing_m)
def __init__(self, dht_addr): my_addr = dht_addr my_id = identifier.RandomId() my_node = Node(my_addr, my_id) tracker_ = tracker.Tracker() token_m = token_manager.TokenManager() self.reactor = ThreadedReactor() rpc_m = RPCManager(self.reactor, my_addr[1]) querier_ = Querier(rpc_m, my_id) routing_m = RoutingManager(my_node, querier_, bootstrap_nodes) responder_ = Responder(my_id, routing_m, tracker_, token_m) responder_.set_on_query_received_callback(routing_m.on_query_received) querier_.set_on_response_received_callback( routing_m.on_response_received) querier_.set_on_error_received_callback(routing_m.on_error_received) querier_.set_on_timeout_callback(routing_m.on_timeout) querier_.set_on_nodes_found_callback(routing_m.on_nodes_found) routing_m.do_bootstrap() rpc_m.add_msg_callback(QUERY, responder_.on_query_received) self.lookup_m = LookupManager(my_id, querier_, routing_m) self._routing_m = routing_m
def __init__(self, version_label, my_node, conf_path, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name, bootstrap_mode): self.bootstrapper = bootstrap.OverlayBootstrapper(conf_path) my_addr = my_node.addr self._my_id = my_node.id # id indicated by user if not self._my_id: self._my_id = self._my_id = identifier.RandomId() # random id self._my_node = Node(my_addr, self._my_id, version=version_label) self.msg_f = message.MsgFactory(version_label, self._my_id, private_dht_name) self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager(self._my_node, self.msg_f, self.bootstrapper) self._responder = responder.Responder(self._my_id, self._routing_m, self.msg_f, bootstrap_mode) self._tracker = self._responder._tracker self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f, self.bootstrapper) self._experimental_m = experimental_m_mod.ExperimentalManager( self._my_node.id, self.msg_f) current_ts = time.time() self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._cached_lookups = []
def getid(self): """getid() get modem id data""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(IMInfoMsgHandler("getid")) msg = Msg.s_makeMessage("GetIMInfo") self.querier.sendMsg(msg)
def __init__(self, dht_addr, state_path, routing_m_mod, lookup_m_mod): self.state_filename = os.path.join(state_path, STATE_FILENAME) self.load_state() if not self._my_id: self._my_id = identifier.RandomId() self._my_node = Node(dht_addr, self._my_id) self._tracker = tracker.Tracker() self._token_m = token_manager.TokenManager() self._reactor = ThreadedReactor() self._reactor.listen_udp(self._my_node.addr[1], self._on_datagram_received) #self._rpc_m = RPCManager(self._reactor) self._querier = Querier(self._my_id) bootstrap_nodes = self.loaded_nodes or BOOTSTRAP_NODES del self.loaded_nodes self._routing_m = routing_m_mod.RoutingManager(self._my_node, bootstrap_nodes) # self._responder = Responder(self._my_id, self._routing_m, # self._tracker, self._token_m) self._lookup_m = lookup_m_mod.LookupManager(self._my_id) current_time = time.time() self._next_maintenance_ts = current_time self._next_save_state_ts = current_time + SAVE_STATE_DELAY self._running = False
def getIMConfig(self): """getIMConfig() get modem configuration flags byte""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(IMConfigMsgHandler("getIMConfig")) msg = Msg.s_makeMessage("GetIMConfig") self.querier.sendMsg(msg)
def __init__(self, dht_addr): my_addr = dht_addr my_id = identifier.RandomId() my_node = Node(my_addr, my_id) tracker_ = tracker.Tracker() token_m = token_manager.TokenManager() self.reactor = ThreadedReactor() rpc_m = RPCManager(self.reactor, my_addr[1]) querier_ = Querier(rpc_m, my_id) routing_m = RoutingManager(my_node, querier_, bootstrap_nodes) responder_ = Responder(my_id, routing_m, tracker_, token_m) responder_.set_on_query_received_callback( routing_m.on_query_received) querier_.set_on_response_received_callback( routing_m.on_response_received) querier_.set_on_error_received_callback( routing_m.on_error_received) querier_.set_on_timeout_callback(routing_m.on_timeout) querier_.set_on_nodes_found_callback(routing_m.on_nodes_found) routing_m.do_bootstrap() rpc_m.add_msg_callback(QUERY, responder_.on_query_received) self.lookup_m = LookupManager(my_id, querier_, routing_m) self._routing_m = routing_m
def do_query(self, query): """ Renders the results page """ q = Querier() results = None try: # extracts only the bindings from the result dictionary bindings = [r['results']['bindings'] for r in q.query(str(query)) if r['results']['bindings'] != []] results = self.unpack_results(bindings) except: # in case of any exception should render an error page results = "ERROR" return self.serve_template('results.txt', results=results)
def cancelLinking(self): """cancelLinking() takes modem out of linking or unlinking mode""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(DefaultMsgHandler("cancel linking")) msg = Msg.s_makeMessage("CancelALLLinking") self.querier.sendMsg(msg)
def __init__(self, version_label, my_node, conf_path, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name, bootstrap_mode): self.bootstrapper = bootstrap.OverlayBootstrapper(conf_path) my_addr = my_node.addr self._my_id = my_node.id # id indicated by user if not self._my_id: self._my_id = self._my_id = identifier.RandomId() # random id self._my_node = Node(my_addr, self._my_id, version=version_label) self.msg_f = message.MsgFactory(version_label, self._my_id, private_dht_name) self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager( self._my_node, self.msg_f, self.bootstrapper) self._responder = responder.Responder(self._my_id, self._routing_m, self.msg_f, bootstrap_mode) self._tracker = self._responder._tracker self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f, self.bootstrapper) self._experimental_m = experimental_m_mod.ExperimentalManager( self._my_node.id, self.msg_f) current_ts = time.time() self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._cached_lookups = []
def __init__(self, dht_addr, state_filename, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name): #TODO: don't do this evil stuff!!! message.private_dht_name = private_dht_name if size_estimation: self._size_estimation_file = open('size_estimation.dat', 'w') self.state_filename = state_filename saved_id, saved_bootstrap_nodes = state.load(self.state_filename) if saved_id: self._my_id = saved_id else: self._my_id = identifier.RandomId() self._my_node = Node(dht_addr, self._my_id) self._tracker = tracker.Tracker() self._token_m = token_manager.TokenManager() self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager(self._my_node, saved_bootstrap_nodes) self._lookup_m = lookup_m_mod.LookupManager(self._my_id) self._experimental_m = experimental_m_mod.ExperimentalManager(self._my_node.id) current_ts = time.time() self._next_save_state_ts = current_ts + SAVE_STATE_DELAY self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._pending_lookups = []
def test_querier(self): """ If the number of entries for Fe2O3 in the MP is larger or equal to 12. The larger case is for in case more have been added in the future. """ self.assertTrue(len(Querier(API_KEY, 'Fe2O3').mp_all) >= 12, msg='Number of entries for Fe2O3 is not correct')
def __init__(self, dht_addr, state_path, routing_m_mod, lookup_m_mod, private_dht_name): #TODO: don't do this evil stuff!!! message.private_dht_name = private_dht_name self.state_filename = os.path.join(state_path, STATE_FILENAME) self.load_state() if not self._my_id: self._my_id = identifier.RandomId() self._my_node = Node(dht_addr, self._my_id) self._tracker = tracker.Tracker() self._token_m = token_manager.TokenManager() self._reactor = ThreadedReactor() self._reactor.listen_udp(self._my_node.addr[1], self._on_datagram_received) #self._rpc_m = RPCManager(self._reactor) self._querier = Querier(self._my_id) bootstrap_nodes = self.loaded_nodes or BOOTSTRAP_NODES del self.loaded_nodes self._routing_m = routing_m_mod.RoutingManager(self._my_node, bootstrap_nodes) # self._responder = Responder(self._my_id, self._routing_m, # self._tracker, self._token_m) self._lookup_m = lookup_m_mod.LookupManager(self._my_id) current_time = time.time() self._next_maintenance_ts = current_time self._next_save_state_ts = current_time + SAVE_STATE_DELAY self._running = False
def do_query(self, query): """ Renders the results page """ q = Querier() results = None try: # extracts only the bindings from the result dictionary bindings = [ r['results']['bindings'] for r in q.query(str(query)) if r['results']['bindings'] != [] ] results = self.unpack_results(bindings) except: # in case of any exception should render an error page results = "ERROR" return self.serve_template('results.txt', results=results)
def linkAsResponder(self, otherDevice, group): """linkAsResponder(otherDevice, group) puts modem in link mode to respond to device "otherDevice" on group "group" """ addr = InsteonAddress(otherDevice) self.querier = Querier(addr) self.querier.setMsgHandler(DefaultMsgHandler("start linking")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0x00) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg)
def unlinkAsController(self, otherDevice, group): """unlinkAsController(otherDevice, group) puts modem in unlink mode to unlink as controller from device "otherDevice" on group "group" """ addr = InsteonAddress(otherDevice) self.querier = Querier(addr) self.querier.setMsgHandler(DefaultMsgHandler("unlink as controller")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0xFF) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg)
def __init__(self, version_label, my_node, state_filename, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name, bootstrap_mode): if size_estimation: self._size_estimation_file = open('size_estimation.dat', 'w') self.state_filename = state_filename saved_id, saved_bootstrap_nodes = state.load(self.state_filename) my_addr = my_node.addr self._my_id = my_node.id # id indicated by user if not self._my_id: self._my_id = saved_id # id loaded from file if not self._my_id: self._my_id = self._my_id = identifier.RandomId() # random id self._my_node = Node(my_addr, self._my_id, version=version_label) self.msg_f = message.MsgFactory(version_label, self._my_id, private_dht_name) self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager( self._my_node, saved_bootstrap_nodes, self.msg_f) self._responder = responder.Responder(self._my_id, self._routing_m, self.msg_f, bootstrap_mode) self._tracker = self._responder._tracker self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f) self._experimental_m = experimental_m_mod.ExperimentalManager( self._my_node.id, self.msg_f) current_ts = time.time() self._next_save_state_ts = current_ts + SAVE_STATE_DELAY self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._pending_lookups = [] self._cached_lookups = {}
class Controller: def __init__(self, dht_addr): self.my_addr = dht_addr self.my_id = identifier.RandomId() self.my_node = Node(self.my_addr, self.my_id) self.tracker = tracker.Tracker() self.token_m = token_manager.TokenManager() self.reactor = ThreadedReactor() self.rpc_m = RPCManager(self.reactor, self.my_addr[1]) self.querier = Querier(self.rpc_m, self.my_id) self.routing_m = RoutingManager(self.my_node, self.querier, bootstrap_nodes) self.responder = Responder(self.my_id, self.routing_m, self.tracker, self.token_m) self.responder.set_on_query_received_callback( self.routing_m.on_query_received) self.querier.set_on_response_received_callback( self.routing_m.on_response_received) self.querier.set_on_error_received_callback( self.routing_m.on_error_received) self.querier.set_on_timeout_callback(self.routing_m.on_timeout) self.querier.set_on_nodes_found_callback(self.routing_m.on_nodes_found) self.routing_m.do_bootstrap() self.rpc_m.add_msg_callback(QUERY, self.responder.on_query_received) self.lookup_m = LookupManager(self.my_id, self.querier, self.routing_m) def start(self): self.reactor.start() def stop(self): #TODO2: stop each manager self.reactor.stop() def get_peers(self, info_hash, callback_f, bt_port=None): return self.lookup_m.get_peers(info_hash, callback_f, bt_port)
def __init__(self, version_label, my_node, state_filename, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name, bootstrap_mode): if size_estimation: self._size_estimation_file = open('size_estimation.dat', 'w') self.state_filename = state_filename saved_id, saved_bootstrap_nodes = state.load(self.state_filename) my_addr = my_node.addr self._my_id = my_node.id # id indicated by user if not self._my_id: self._my_id = saved_id # id loaded from file if not self._my_id: self._my_id = self._my_id = identifier.RandomId() # random id self._my_node = Node(my_addr, self._my_id, version=version_label) self.msg_f = message.MsgFactory(version_label, self._my_id, private_dht_name) self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager( self._my_node, saved_bootstrap_nodes, self.msg_f) self._responder = responder.Responder(self._my_id, self._routing_m, self.msg_f, bootstrap_mode) self._tracker = self._responder._tracker self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f) self._experimental_m = experimental_m_mod.ExperimentalManager( self._my_node.id, self.msg_f) current_ts = time.time() self._next_save_state_ts = current_ts + SAVE_STATE_DELAY self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._pending_lookups = [] self._cached_lookups = []
class Device: name = "" address = InsteonAddress() db = None dbbuilder = None querier = None def __init__(self, name, addr): self.name = name self.address = addr self.db = DB() self.querier = Querier(addr) addDev(self) def modifyDB(self, listener): self.dbbuilder.setListener(listener) # after db download complete, listener will perform action self.getdb() def setRecord(self, offset, laddr, group, linkType, data): msg = self.makeMsg(offset, laddr, group, linkType,data) self.querier.setMsgHandler(MsgHandler("got set record")) self.querier.sendMsg(msg) def makeMsg(self, offset, laddr, group, linkType, data): msg = Msg.s_makeMessage("SendExtendedMessage") msg.setAddress("toAddress", InsteonAddress(self.getAddress())) msg.setByte("messageFlags", 0x1f) msg.setByte("command1", 0x2f) msg.setByte("command2", 0x00) msg.setByte("userData1", 0x00) # don't care info msg.setByte("userData2", 0x02) # set database msg.setByte("userData3", offset >> 8) # high byte msg.setByte("userData4", offset & 0xff) # low byte msg.setByte("userData5", 8) # number of bytes set: 1...8 msg.setByte("userData6", linkType) msg.setByte("userData7", group) msg.setByte("userData8", laddr.getHighByte()) msg.setByte("userData9", laddr.getMiddleByte()) msg.setByte("userData10", laddr.getLowByte()) # depends on mode: could be e.g. trigger point msg.setByte("userData11", data[0]) msg.setByte("userData12", data[1]) # unused? msg.setByte("userData13", data[2]) # unused? rb = msg.getBytes("command1", 15); checksum = (~sum(rb) + 1) & 0xFF msg.setByte("userData14", checksum) return msg def getName(self): return self.name def getAddress(self): return self.address def getdb(self): """getdb() download the device database and print it on the console""" iofun.out("getting db, be patient!") self.dbbuilder.clear() self.dbbuilder.start() def printdb(self): """printdb() print the downloaded link database to the console""" self.dbbuilder.printdb() def getId(self): """getId() get category, subcategory, firmware, hardware version""" self.querier.setMsgHandler(IdMsgHandler("id")) self.querier.querysd(0x10, 0x00) # # link database management # def addController(self, addr, group, data = None): """addController(addr, group[, data]) add device with "addr" as controller for group "group", with link data "data" """ data = data if data else [00, 00, group]; self.modifyDB(LinkRecordAdder(self, addr, group, data, True)) def removeController(self, addr, group): """removeController(addr, group) remove device with "addr" as controller for group "group", with link data "data" """ self.modifyDB(LinkRecordRemover(self, addr, group, True)) def addResponder(self, addr, group, data = None): """addResponder(addr, group[, data]) add device with "addr" as responder for group "group", with link data "data" """ data = data if data else [00, 00, group]; self.modifyDB(LinkRecordAdder(self, addr, group, data, False)) def removeResponder(self, addr, group): """removeResponder(addr, group) remove device with "addr" as responder for group "group" """ self.modifyDB(LinkRecordRemover(self, addr, group, False)) def removeDevice(self, addr): """removeDevice(addr): removes all links to device with address "addr" from device database""" self.modifyDB(DeviceRemover(self, addr)) def replaceDevice(self, oldAddr, newAddr): """replaceDevice(oldAddr, newAddr): replaces all linkdb occurrences of oldAddr with newAddr """ self.dbbuilder.setListener(AddressReplacer(self, oldAddr, newAddr)) # after db download is complete, listener will perform action self.getdb() def removeLastRecord(self): """removeLastRecord() removes the last device in the link database""" self.modifyDB(LastRecordRemover(self)) def nukeDB(self): """nukeDB() really WIPES OUT all records in the device's database!""" self.modifyDB(LastNRecordRemover(self, -1)) def setOnLevelResponder(self, addr, group, level, ramprate = 28, button = 1): """setOnLevelResponder(addr, group, level, ramprate = 28, button = 1) sets (on level, ramp rate, button) for controller with "addr" and group "group" """ self.modifyDB(OnLevelModifier(self, addr, group, level, ramprate, button, False)) def enterLinkingMode(self, group): """enterLinkingMode(group) causes the device to enter linking mode""" self.querier.setMsgHandler(MsgHandler("enter linking mode")) self.querier.querysd(0x09, group); def exitLinkingMode(self): """exitLinkingMode() causes the device to exit linking mode""" self.querier.setMsgHandler(MsgHandler("exit linking mode")) self.querier.querysd(0x08, 0x01); def enterUnlinkingMode(self, group): """enterUnlinkingMode(group) causes the device to enter unlinking mode""" self.querier.setMsgHandler(MsgHandler("enter unlinking mode")) self.querier.querysd(0x0A, group);
def __init__(self, name, addr): self.name = name self.address = addr self.db = DB() self.querier = Querier(addr) addDev(self)
class Device: name = "" address = InsteonAddress() db = None dbbuilder = None querier = None def __init__(self, name, addr): self.name = name self.address = addr self.db = DB() self.querier = Querier(addr) addDev(self) def modifyDB(self, listener): self.dbbuilder.setListener(listener) # after db download complete, listener will perform action self.getdb() def setRecord(self, offset, laddr, group, linkType, data): msg = self.makeMsg(offset, laddr, group, linkType,data) self.querier.setMsgHandler(MsgHandler("got set record")) self.querier.sendMsg(msg) def makeMsg(self, offset, laddr, group, linkType, data): msg = Msg.s_makeMessage("SendExtendedMessage") msg.setAddress("toAddress", InsteonAddress(self.getAddress())) msg.setByte("messageFlags", 0x1f) msg.setByte("command1", 0x2f) msg.setByte("command2", 0x00) msg.setByte("userData1", 0x00) # don't care info msg.setByte("userData2", 0x02) # set database msg.setByte("userData3", offset >> 8) # high byte msg.setByte("userData4", offset & 0xff) # low byte msg.setByte("userData5", 8) # number of bytes set: 1...8 msg.setByte("userData6", linkType) msg.setByte("userData7", group) msg.setByte("userData8", laddr.getHighByte()) msg.setByte("userData9", laddr.getMiddleByte()) msg.setByte("userData10", laddr.getLowByte()) # depends on mode: could be e.g. trigger point msg.setByte("userData11", data[0]) msg.setByte("userData12", data[1]) # unused? msg.setByte("userData13", data[2]) # unused? rb = msg.getBytes("command1", 15); checksum = (~sum(rb) + 1) & 0xFF msg.setByte("userData14", checksum) return msg def getName(self): return self.name def getAddress(self): return self.address def getdb(self): """getdb() download the device database and print it on the console""" iofun.out("getting db, be patient!") self.dbbuilder.clear() self.dbbuilder.start() def printdb(self): """printdb() print the downloaded link database to the console""" self.dbbuilder.printdb() def getId(self): """getId() get category, subcategory, firmware, hardware version""" self.querier.setMsgHandler(IdMsgHandler("id")) self.querier.querysd(0x10, 0x00) # # link database management # def addController(self, addr, group, data = None): """addController(addr, group, data) add device with "addr" as controller for group "group", with link data "data" """ data = data if data else [00, 00, group]; self.modifyDB(LinkRecordAdder(self, addr, group, data, True)) def removeController(self, addr, group): """removeController(addr, group) remove device with "addr" as controller for group "group", with link data "data" """ self.modifyDB(LinkRecordRemover(self, addr, group, True)) def addResponder(self, addr, group, data = None): """addResponder(addr, group, data) add device with "addr" as responder for group "group", with link data "data" """ data = data if data else [00, 00, group]; self.modifyDB(LinkRecordAdder(self, addr, group, data, False)) def removeResponder(self, addr, group): """removeResponder(addr, group) remove device with "addr" as responder for group "group" """ self.modifyDB(LinkRecordRemover(self, addr, group, False)) def removeDevice(self, addr): """removeDevice(addr): removes all links to device with address "addr" from device database""" self.modifyDB(DeviceRemover(self, addr)) def replaceDevice(self, oldAddr, newAddr): """replaceDevice(oldAddr, newAddr): replaces all linkdb occurrences of oldAddr with newAddr """ self.dbbuilder.setListener(AddressReplacer(self, oldAddr, newAddr)) # after db download is complete, listener will perform action self.getdb() def removeLastRecord(self): """removeLastRecord() removes the last device in the link database""" self.modifyDB(LastRecordRemover(self)) def nukeDB(self): """nukeDB() really WIPES OUT all records in the device's database!""" self.modifyDB(LastNRecordRemover(self, -1)) def setOnLevelResponder(self, addr, group, level, ramprate = 28, button = 1): """setOnLevelResponder(addr, group, level, ramprate = 28, button = 1) sets (on level, ramp rate, button) for controller with "addr" and group "group" """ self.modifyDB(OnLevelModifier(self, addr, group, level, ramprate, button, False))
class TestQuerier: def setup(self): global time time = querier.time = MockTime() self.querier = Querier(tc.CLIENT_ID) def test_generate_tids(self): #TODO: move to message num_tids = 1000 if RUN_CPU_INTENSIVE_TESTS: num_tids = pow(2, 16) + 2 #CPU intensive for i in xrange(num_tids): eq_(self.querier._next_tid(), chr(i % 256) + chr((i / 256) % 256)) def test_ping_with_reponse(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = message.OutgoingPingResponse(tc.SERVER_ID) bencoded_r = ping_r_msg_out.encode(q.tid) # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_response_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is q def test_ping_with_timeout(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(3) # The server never responds and the timeout is triggered stored_q = self.querier.on_timeout(tc.SERVER_ADDR) assert stored_q is q def test_unsolicited_response(self): # Server creates unsolicited response # It might well be that the server responds using another port, # and therefore, the addr is not matched # TODO: consider accepting responses from a different port ping_r_msg_out = message.OutgoingPingResponse(tc.SERVER_ID) bencoded_r = ping_r_msg_out.encode('zz') # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_response_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is None def test_response_with_different_tid(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = message.OutgoingPingResponse(tc.SERVER_ID) bencoded_r = ping_r_msg_out.encode('zz') # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_response_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is None def test_error(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = message.OutgoingErrorMsg(message.GENERIC_E) bencoded_r = ping_r_msg_out.encode(tc.TID) # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_error_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is None def tear_down(self): global time time.unmock() time = querier.time = time.actual_time
class Controller: def __init__(self, dht_addr, state_path, routing_m_mod, lookup_m_mod, private_dht_name): #TODO: don't do this evil stuff!!! message.private_dht_name = private_dht_name self.state_filename = os.path.join(state_path, STATE_FILENAME) self.load_state() if not self._my_id: self._my_id = identifier.RandomId() self._my_node = Node(dht_addr, self._my_id) self._tracker = tracker.Tracker() self._token_m = token_manager.TokenManager() self._reactor = ThreadedReactor() self._reactor.listen_udp(self._my_node.addr[1], self._on_datagram_received) #self._rpc_m = RPCManager(self._reactor) self._querier = Querier(self._my_id) bootstrap_nodes = self.loaded_nodes or BOOTSTRAP_NODES del self.loaded_nodes self._routing_m = routing_m_mod.RoutingManager(self._my_node, bootstrap_nodes) # self._responder = Responder(self._my_id, self._routing_m, # self._tracker, self._token_m) self._lookup_m = lookup_m_mod.LookupManager(self._my_id) current_time = time.time() self._next_maintenance_ts = current_time self._next_save_state_ts = current_time + SAVE_STATE_DELAY self._running = False def start(self): assert not self._running self._running = True self._reactor.start() self._main_loop() def stop(self): assert self._running #TODO2: stop each manager self._reactor.stop() def save_state(self): rnodes = self._routing_m.get_main_rnodes() f = open(self.state_filename, 'w') f.write('%r\n' % self._my_id) for rnode in rnodes: f.write('%d\t%r\t%s\t%d\t%f\n' % ( self._my_id.log_distance(rnode.id), rnode.id, rnode.addr[0], rnode.addr[1], rnode.rtt * 1000)) f.close() def load_state(self): self._my_id = None self.loaded_nodes = [] try: f = open(self.state_filename) except(IOError): return # the first line contains this node's identifier hex_id = f.readline().strip() self._my_id = Id(hex_id) # the rest of the lines contain routing table nodes # FORMAT # log_distance hex_id ip port rtt for line in f: _, hex_id, ip, port, _ = line.split() addr = (ip, int(port)) node_ = Node(addr, Id(hex_id)) self.loaded_nodes.append(node_) f.close def get_peers(self, lookup_id, info_hash, callback_f, bt_port=0): assert self._running # look if I'm tracking this info_hash local_peers = self._tracker.get(info_hash) # do the lookup log_distance = info_hash.log_distance(self._my_id) bootstrap_rnodes = self._routing_m.get_closest_rnodes(log_distance, None, True) lookup_obj = self._lookup_m.get_peers(lookup_id, info_hash, callback_f, bt_port) lookup_queries_to_send = lookup_obj.start(bootstrap_rnodes) self._send_queries(lookup_queries_to_send) if not lookup_queries_to_send: # There are no nodes in my routing table, announce to myself self._announce(lookup_obj) # NOTICE: the callback is NOT triggered, zero is returned. return len(lookup_queries_to_send), local_peers def print_routing_table_stats(self): self._routing_m.print_stats() def _main_loop(self): current_time = time.time() # Routing table if current_time > self._next_maintenance_ts: (maintenance_delay, queries_to_send, maintenance_lookup_target) = self._routing_m.do_maintenance() self._send_queries(queries_to_send) if maintenance_lookup_target: log_distance = maintenance_lookup_target.log_distance( self._my_id) bootstrap_nodes = self._routing_m.get_closest_rnodes( log_distance, None, True) lookup_obj = self._lookup_m.maintenance_lookup( maintenance_lookup_target) lookup_queries_to_send = lookup_obj.start(bootstrap_nodes) self._send_queries(lookup_queries_to_send) self._next_maintenance_ts = (current_time + maintenance_delay) # Auto-save routing table if current_time > self._next_save_state_ts: self.save_state() self._next_save_state_ts = current_time + SAVE_STATE_DELAY # Schedule next call delay = (min(self._next_maintenance_ts, self._next_save_state_ts) - current_time) self._reactor.call_later(delay, self._main_loop) def _maintenance_lookup(self, target): self._lookup_m.maintenance_lookup(target) def _on_datagram_received(self, data, addr): try: msg = message.IncomingMsg(data, addr) except(message.MsgError): return # ignore message if msg.sender_id == self._my_id: logger.debug('Got a msg from myself:\n%r', msg) return if msg.type == message.QUERY: response_msg = self._get_response(msg) if response_msg: bencoded_response = response_msg.encode(msg.tid) self._reactor.sendto(bencoded_response, addr) maintenance_queries_to_send = self._routing_m.on_query_received( msg.sender_node) elif msg.type in (message.RESPONSE, message.ERROR): related_query = self._querier.on_response_received(msg, addr) if not related_query: # Query timed out or unrequested response return # lookup related tasks if related_query.lookup_obj: if msg.type == message.RESPONSE: (lookup_queries_to_send, peers, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_response_received( msg, msg.sender_node) else: #ERROR peers = None # an error msg doesn't have peers (lookup_queries_to_send, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_error_received( msg, msg.sender_node) self._send_queries(lookup_queries_to_send) if related_query.lookup_obj.callback_f: lookup_id = related_query.lookup_obj.lookup_id if peers: related_query.lookup_obj.callback_f(lookup_id, peers) if lookup_done: self._announce(related_query.lookup_obj) related_query.lookup_obj.callback_f(lookup_id, None) # maintenance related tasks if msg.type == message.RESPONSE: maintenance_queries_to_send = \ self._routing_m.on_response_received( msg.sender_node, related_query.rtt, msg.all_nodes) else: maintenance_queries_to_send = \ self._routing_m.on_error_received( msg.sender_node) else: # unknown type return self._send_queries(maintenance_queries_to_send) def _get_response(self, msg): if msg.query == message.PING: return message.OutgoingPingResponse(self._my_id) elif msg.query == message.FIND_NODE: log_distance = msg.target.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes(log_distance, NUM_NODES, False) return message.OutgoingFindNodeResponse(self._my_id, rnodes) elif msg.query == message.GET_PEERS: token = self._token_m.get() log_distance = msg.info_hash.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes(log_distance, NUM_NODES, False) peers = self._tracker.get(msg.info_hash) if peers: logger.debug('RESPONDING with PEERS:\n%r' % peers) return message.OutgoingGetPeersResponse(self._my_id, token, nodes=rnodes, peers=peers) elif msg.query == message.ANNOUNCE_PEER: peer_addr = (msg.sender_addr[0], msg.bt_port) self._tracker.put(msg.info_hash, peer_addr) return message.OutgoingAnnouncePeerResponse(self._my_id) else: logger.debug('Invalid QUERY: %r' % (msg.query)) #TODO: maybe send an error back? def _on_response_received(self, msg): pass def _on_timeout(self, addr): related_query = self._querier.on_timeout(addr) if not related_query: return # timeout cancelled (got response/error already) if related_query.lookup_obj: (lookup_queries_to_send, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_timeout(related_query.dstnode) self._send_queries(lookup_queries_to_send) if lookup_done and related_query.lookup_obj.callback_f: self._announce(related_query.lookup_obj) lookup_id = related_query.lookup_obj.lookup_id related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_timeout( related_query.dstnode) self._send_queries(maintenance_queries_to_send) def _announce(self, lookup_obj): queries_to_send, announce_to_myself = lookup_obj.announce() self._send_queries(queries_to_send) ''' if announce_to_myself: self._tracker.put(lookup_obj._info_hash, (self._my_node.addr[0], lookup_obj._bt_port)) ''' def _send_queries(self, queries_to_send, lookup_obj=None): if queries_to_send is None: return for query in queries_to_send: timeout_task = self._reactor.call_later(TIMEOUT_DELAY, self._on_timeout, query.dstnode.addr) bencoded_query = self._querier.register_query(query, timeout_task) self._reactor.sendto(bencoded_query, query.dstnode.addr)
class Modem2413U(Device): """============== Insteon PowerLinc modem (PLM) ===============""" def __init__(self, name, addr): Device.__init__(self, name, addr) self.dbbuilder = ModemDBBuilder(addr, self.db) def __modifyModemDB(self, listener): self.dbbuilder.setListener(listener) # after db download complete, listener will perform action iofun.out("getting db, be patient!") self.dbbuilder.clear() self.dbbuilder.start() def getdb(self): """getdb() download the modem database and print it on the console""" self.dbbuilder.start() self.dbbuilder.wait() self.dbbuilder.dumpDB() out("Modem Link DB complete") def readdb(self): """readdb() download the modem database and return it""" self.dbbuilder.start() self.dbbuilder.wait() out("Modem Link DB complete!") return self.dbbuilder.getdb() def startWatch(self): """startWatch() modem will print all incoming messages on terminal""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(MsgDumper("modem")) self.querier.startWait(10000) def stopWatch(self): """stopWatch() stop modem from printing all incoming messages on terminal""" if (self.querier): self.querier.cancel() def getid(self): """getid() get modem id data""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(IMInfoMsgHandler("getid")) msg = Msg.s_makeMessage("GetIMInfo") self.querier.sendMsg(msg) def getIMConfig(self): """getIMConfig() get modem configuration flags byte""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(IMConfigMsgHandler("getIMConfig")) msg = Msg.s_makeMessage("GetIMConfig") self.querier.sendMsg(msg) def sendOn(self, group): """sendOn(group) sends ALLLink broadcast ON message to group "group" """ msg = message.createStdMsg(InsteonAddress("00.00.00"), 0x0f, 0x11, 0xFF, group) iofun.writeMsg(msg) iofun.out("sent msg: " + msg.toString()) def sendOff(self, group): """sendOff(group) sends ALLLink broadcast OFF message to group "group" """ msg = message.createStdMsg(InsteonAddress("00.00.00"), 0x0f, 0x13, 0xFF, group) iofun.writeMsg(msg) iofun.out("sent msg: " + msg.toString()) def linkAsController(self, group): """linkAsController(group) puts modem in link mode on group "group" """ self.querier.setMsgHandler(DefaultMsgHandler("link as controller")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0x01) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def linkAsResponder(self, group): """linkAsResponder(group) puts modem in link mode on group "group" """ self.querier.setMsgHandler(DefaultMsgHandler("start linking")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0x00) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def linkAsEither(self, group): """linkAsEither(group) puts modem in link mode to link as controller or responder on group "group" """ self.querier.setMsgHandler( DefaultMsgHandler("link/unlink as controller or responder")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0x03) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def respondToUnlink(self, group): """respondToUnlink(group) make modem respond to unlink message""" # could not get 0xFF to unlink self.linkAsEither(group) def unlinkAsController(self, group): """unlinkAsController(group) puts modem in unlink mode to unlink as controller on group "group" """ self.querier.setMsgHandler(DefaultMsgHandler("unlink as controller")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0xFF) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def cancelLinking(self): """cancelLinking() takes modem out of linking or unlinking mode""" self.querier.setMsgHandler(DefaultMsgHandler("cancel linking")) msg = Msg.s_makeMessage("CancelALLLinking") self.querier.sendMsg(msg) def addController(self, addr, group): """addController(addr, group): adds device with address "addr" to modem link database as controller for group "group" """ self.modifyRecord(addr, group, 0x40, 0xa2, [0, 0, group], "addController") def addResponder(self, addr, group, data=None): """addResponder(addr, group[, data]): adds device with address "addr" to modem link database as responder to group "group" """ data = data if data else [00, 00, group] self.modifyRecord(addr, group, 0x41, 0xa2, data, "addResponder") def addSoftwareResponder(self, addr): """addSoftwareResponder(addr): adds device with address "addr" to modem link database as software responder""" self.modifyRecord(addr, 0xef, 0x41, 0xa2, [0, 0, 0xef], "addSoftwareController") def removeResponderOrController(self, addr, group): """removeResponderOrController(addr, group) removes device with address "addr" and group "group" from modem link database""" self.__deleteFirstRecord(addr, group, "removeResponderOrController") def removeResponder(self, addr, group): """removeResponder(addr, group) could not be implemented for the modem. Use removeResponderOrController() instead!""" iofun.out("removeResponder(addr, group) could not be implemented" + " for the modem. Use removeResponderOrController() instead!") def removeController(self, addr, group): """removeController(addr, group) could not be implemented for the modem. Use removeResponderOrController() instead!""" iofun.out("removeController(addr, group) could not be implemented" + " for the modem. Use removeResponderOrController() instead!") def removeDevice(self, addr): """removeDevice(addr): removes all links to device with address "addr" from modem database""" self.__modifyModemDB(DeviceRemover(self, addr)) def __deleteFirstRecord(self, addr, group, text="delete record"): self.modifyRecord(addr, group, 0x80, 0x00, [0, 0, 0], text) def modifyFirstOrAdd(self, addr, group, recordFlags, data): if (recordFlags & (1 << 6)): # controller self.modifyRecord(addr, group, 0x40, recordFlags, data, "modify first or add") else: self.modifyRecord(addr, group, 0x41, recordFlags, data, "modify first or add") def modifyFirstControllerOrAdd(self, addr, group, data): self.modifyRecord(addr, group, 0x40, 0xe2, data, "modify first ctrl found or add") def modifyFirstResponderOrAdd(self, addr, group, data): self.modifyRecord(addr, group, 0x41, 0xa2, data, "modify first resp found or add") def modifyRecord(self, addr, group, controlCode, recordFlags, data, txt): msg = self.__makeModMsg(addr, group, controlCode, recordFlags, data, txt) self.querier = Querier(self.address) self.querier.setMsgHandler(DefaultMsgHandler(txt)) self.querier.sendMsg(msg) def __makeModMsg(self, addr, group, controlCode, recordFlags, data, txt): msg = Msg.s_makeMessage("ManageALLLinkRecord") msg.setByte("controlCode", controlCode) # mod. first ctrl found or add msg.setByte("recordFlags", recordFlags) msg.setByte("ALLLinkGroup", group) msg.setAddress("linkAddress", InsteonAddress(addr)) msg.setByte("linkData1", data[0] & 0xFF) msg.setByte("linkData2", data[1] & 0xFF) msg.setByte("linkData3", data[2] & 0xFF) return msg def saveDB(self, filename): """saveDB(filename) save modem database to file "filename" """ self.dbbuilder.start() self.dbbuilder.wait() self.dbbuilder.saveDB(filename) def loadDB(self, filename): """loadDB(filename) load modem database from file "filename" (note: this will not change the actual modem db) """ self.dbbuilder.loadDB(filename) self.dbbuilder.dumpDB() def nukeDB(self): """nukeDB() delete complete modem database! """ self.dbbuilder.start() self.dbbuilder.wait() self.dbbuilder.nukeDB(self) def restoreDB(self, filename): """restoreDB(filename) restore modem database from file "filename" """ self.loadDB(filename) self.dbbuilder.restoreDB(self, filename)
from csv import DictReader from rbm import RBM, np from querier import Querier from propertyFinder import PropertyFinder TRAINING_SAMPLE_SIZE = 100 localQuerier = Querier('http://127.0.0.1:9999/bigdata/sparql') localPropertyFinder = PropertyFinder(localQuerier) effectsList = [{'effect': "", 'disease': ""}] csvFile = open("result.csv") trainingSet = DictReader(csvFile) trainingRows = [] trainingData = [[0 for i in range(len(effectsList))] for j in range(TRAINING_SAMPLE_SIZE)] index1=0 for row in trainingSet: if index1 < TRAINING_SAMPLE_SIZE : geneProperties = DictReader(localPropertyFinder.findGeneProperties(row['gene'])) for prop in geneProperties: for index2, item in effectsList: if prop == item: trainingData[index1][index2] = 1 drugProperties = DictReader(localPropertyFinder.findDrugProperties(row['drug'])) for prop in drugProperties: for index2, item in effectsList: if prop == item:
class TestQuerier: def setup(self): global time time = querier.time = MockTime() self.querier = Querier(tc.CLIENT_ID) def test_generate_tids(self): #TODO: move to message num_tids = 1000 if RUN_CPU_INTENSIVE_TESTS: num_tids = pow(2, 16) + 2 #CPU intensive for i in xrange(num_tids): eq_(self.querier._next_tid(), chr(i%256)+chr((i/256)%256)) def test_ping_with_reponse(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = message.OutgoingPingResponse(tc.SERVER_ID) bencoded_r = ping_r_msg_out.encode(q.tid) # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_response_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is q def test_ping_with_timeout(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(3) # The server never responds and the timeout is triggered stored_q = self.querier.on_timeout(tc.SERVER_ADDR) assert stored_q is q def test_unsolicited_response(self): # Server creates unsolicited response # It might well be that the server responds using another port, # and therefore, the addr is not matched # TODO: consider accepting responses from a different port ping_r_msg_out = message.OutgoingPingResponse(tc.SERVER_ID) bencoded_r = ping_r_msg_out.encode('zz') # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_response_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is None def test_response_with_different_tid(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = message.OutgoingPingResponse(tc.SERVER_ID) bencoded_r = ping_r_msg_out.encode('zz') # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_response_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is None def test_error(self): # Client creates a query ping_msg = message.OutgoingPingQuery(tc.CLIENT_ID) q = Query(ping_msg, tc.SERVER_NODE) timeout_task = minitwisted.Task(TIMEOUT_DELAY, None) # Client registers query bencoded_msg = self.querier.register_query(q, timeout_task) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = message.OutgoingErrorMsg(message.GENERIC_E) bencoded_r = ping_r_msg_out.encode(tc.TID) # The client receives the bencoded message ping_r_in = message.IncomingMsg(bencoded_r, tc.SERVER_ADDR) stored_q = self.querier.on_error_received(ping_r_in, tc.SERVER_ADDR) assert stored_q is None def tear_down(self): global time time.unmock() time = querier.time = time.actual_time
def modifyRecord(self, addr, group, controlCode, recordFlags, data, txt): msg = self.__makeModMsg(addr, group, controlCode, recordFlags, data, txt) self.querier = Querier(self.address) self.querier.setMsgHandler(DefaultMsgHandler(txt)) self.querier.sendMsg(msg)
class Controller: def __init__(self, version_label, my_node, state_filename, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name, bootstrap_mode): if size_estimation: self._size_estimation_file = open('size_estimation.dat', 'w') self.state_filename = state_filename saved_id, saved_bootstrap_nodes = state.load(self.state_filename) my_addr = my_node.addr self._my_id = my_node.id # id indicated by user if not self._my_id: self._my_id = saved_id # id loaded from file if not self._my_id: self._my_id = self._my_id = identifier.RandomId() # random id self._my_node = Node(my_addr, self._my_id, version=version_label) self.msg_f = message.MsgFactory(version_label, self._my_id, private_dht_name) self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager( self._my_node, saved_bootstrap_nodes, self.msg_f) self._responder = responder.Responder(self._my_id, self._routing_m, self.msg_f, bootstrap_mode) self._tracker = self._responder._tracker self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f) self._experimental_m = experimental_m_mod.ExperimentalManager( self._my_node.id, self.msg_f) current_ts = time.time() self._next_save_state_ts = current_ts + SAVE_STATE_DELAY self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._pending_lookups = [] self._cached_lookups = [] def on_stop(self): self._experimental_m.on_stop() def get_peers(self, lookup_id, info_hash, callback_f, bt_port, use_cache): """ Start a get\_peers lookup whose target is 'info\_hash'. The handler 'callback\_f' will be called with two arguments ('lookup\_id' and a 'peer list') whenever peers are discovered. Once the lookup is completed, the handler will be called with 'lookup\_id' and None as arguments. This method is designed to be used as minitwisted's external handler. """ datagrams_to_send = [] logger.debug('get_peers %d %r' % (bt_port, info_hash)) if use_cache: peers = self._get_cached_peers(info_hash) if peers and callable(callback_f): callback_f(lookup_id, peers, None) callback_f(lookup_id, None, None) return datagrams_to_send self._pending_lookups.append(self._lookup_m.get_peers(lookup_id, info_hash, callback_f, bt_port)) queries_to_send = self._try_do_lookup() datagrams_to_send = self._register_queries(queries_to_send) return datagrams_to_send def _get_cached_peers(self, info_hash): oldest_valid_ts = time.time() - CACHE_VALID_PERIOD for ts, cached_info_hash, peers in self._cached_lookups: if ts > oldest_valid_ts and info_hash == cached_info_hash: return peers def _add_cache_peers(self, info_hash, peers): oldest_valid_ts = time.time() - CACHE_VALID_PERIOD while self._cached_lookups and self._cached_lookups[0][0] < oldest_valid_ts: # clean up old entries del self._cached_lookups[0] if self._cached_lookups and self._cached_lookups[-1][1] == info_hash: self._cached_lookups[-1][2].extend(peers) else: self._cached_lookups.append((time.time(), info_hash, peers)) def _try_do_lookup(self): queries_to_send = [] if self._pending_lookups: lookup_obj = self._pending_lookups[0] else: return queries_to_send distance = lookup_obj.info_hash.distance(self._my_id) bootstrap_rnodes = self._routing_m.get_closest_rnodes(distance.log, 0, True) #TODO: get the full bucket if bootstrap_rnodes: del self._pending_lookups[0] # look if I'm tracking this info_hash peers = self._tracker.get(lookup_obj.info_hash) callback_f = lookup_obj.callback_f if peers: self._add_cache_peers(lookup_obj.info_hash, peers) if callable(callback_f): callback_f(lookup_obj.lookup_id, peers, None) # do the lookup queries_to_send = lookup_obj.start(bootstrap_rnodes) else: next_lookup_attempt_ts = time.time() + .2 self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, next_lookup_attempt_ts) return queries_to_send def print_routing_table_stats(self): self._routing_m.print_stats() def main_loop(self): """ Perform maintenance operations. The main operation is routing table maintenance where staled nodes are added/probed/replaced/removed as needed. The routing management module specifies the implementation details. This includes keeping track of queries that have not been responded for a long time (timeout) with the help of querier.Querier. The routing manager and the lookup manager will be informed of those timeouts. This method is designed to be used as minitwisted's heartbeat handler. """ queries_to_send = [] current_ts = time.time() #TODO: I think this if should be removed # At most, 1 second between calls to main_loop after the first call if current_ts >= self._next_main_loop_call_ts: self._next_main_loop_call_ts = current_ts + 1 else: # It's too early return self._next_main_loop_call_ts, [] # Retry failed lookup (if any) queries_to_send.extend(self._try_do_lookup()) # Take care of timeouts if current_ts >= self._next_timeout_ts: (self._next_timeout_ts, timeout_queries) = self._querier.get_timeout_queries() for query in timeout_queries: queries_to_send.extend(self._on_timeout(query)) # Routing table maintenance if time.time() >= self._next_maintenance_ts: (maintenance_delay, queries, maintenance_lookup) = self._routing_m.do_maintenance() self._next_maintenance_ts = current_ts + maintenance_delay self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, self._next_maintenance_ts) queries_to_send.extend(queries) if maintenance_lookup: target, rnodes = maintenance_lookup lookup_obj = self._lookup_m.maintenance_lookup(target) queries_to_send.extend(lookup_obj.start(rnodes)) # Auto-save routing table if current_ts >= self._next_save_state_ts: state.save(self._my_id, self._routing_m.get_main_rnodes(), self.state_filename) self._next_save_state_ts = current_ts + SAVE_STATE_DELAY self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, self._next_maintenance_ts, self._next_timeout_ts, self._next_save_state_ts) # Return control to reactor datagrams_to_send = self._register_queries(queries_to_send) return self._next_main_loop_call_ts, datagrams_to_send def _maintenance_lookup(self, target): self._lookup_m.maintenance_lookup(target) def on_datagram_received(self, datagram): """ Perform the actions associated to the arrival of the given datagram. The datagram will be ignored in cases such as invalid format. Otherwise, the datagram will be decoded and different modules will be informed to take action on it. For instance, if the datagram contains a response to a lookup query, both routing and lookup manager will be informed. Additionally, if that response contains peers, the lookup's handler will be called (see get\_peers above). This method is designed to be used as minitwisted's networking handler. """ exp_queries_to_send = [] data = datagram.data addr = datagram.addr datagrams_to_send = [] try: msg = self.msg_f.incoming_msg(datagram) except(message.MsgError): # ignore message return self._next_main_loop_call_ts, datagrams_to_send if msg.type == message.QUERY: if msg.src_node.id == self._my_id: logger.debug('Got a msg from myself:\n%r', msg) return self._next_main_loop_call_ts, datagrams_to_send #zinat: inform experimental_module exp_queries_to_send = self._experimental_m.on_query_received(msg) response_msg = self._responder.get_response(msg) if response_msg: bencoded_response = response_msg.stamp(msg.tid) datagrams_to_send.append( message.Datagram(bencoded_response, addr)) maintenance_queries_to_send = self._routing_m.on_query_received( msg.src_node) elif msg.type == message.RESPONSE: related_query = self._querier.get_related_query(msg) if not related_query: # Query timed out or unrequested response return self._next_main_loop_call_ts, datagrams_to_send ## zinat: if related_query.experimental_obj: exp_queries_to_send = self._experimental_m.on_response_received( msg, related_query) #TODO: you need to get datagrams to be able to send messages (raul) # lookup related tasks if related_query.lookup_obj: (lookup_queries_to_send, peers, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_response_received( msg, msg.src_node) datagrams = self._register_queries(lookup_queries_to_send) datagrams_to_send.extend(datagrams) lookup_obj = related_query.lookup_obj lookup_id = lookup_obj.lookup_id callback_f = lookup_obj.callback_f if peers: self._add_cache_peers(lookup_obj.info_hash, peers) if callable(callback_f): callback_f(lookup_id, peers, msg.src_node) if lookup_done: if callable(callback_f): callback_f(lookup_id, None, msg.src_node) queries_to_send = self._announce( related_query.lookup_obj) datagrams = self._register_queries( queries_to_send) datagrams_to_send.extend(datagrams) # Size estimation if size_estimation and lookup_done: line = '%d %d\n' % ( related_query.lookup_obj.get_number_nodes_within_region()) self._size_estimation_file.write(line) self._size_estimation_file.flush() # maintenance related tasks maintenance_queries_to_send = \ self._routing_m.on_response_received( msg.src_node, related_query.rtt, msg.all_nodes) elif msg.type == message.ERROR: related_query = self._querier.get_related_query(msg) if not related_query: # Query timed out or unrequested response return self._next_main_loop_call_ts, datagrams_to_send #TODO: zinat: same as response exp_queries_to_send = self._experimental_m.on_error_received( msg, related_query) # lookup related tasks if related_query.lookup_obj: peers = None # an error msg doesn't have peers (lookup_queries_to_send, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_error_received(msg) datagrams = self._register_queries(lookup_queries_to_send) datagrams_to_send.extend(datagrams) if lookup_done: # Size estimation if size_estimation: line = '%d %d\n' % ( related_query.lookup_obj.get_number_nodes_within_region()) self._size_estimation_file.write(line) self._size_estimation_file.flush() datagrams = self._announce(related_query.lookup_obj) datagrams_to_send.extend(datagrams) callback_f = related_query.lookup_obj.callback_f if callback_f and callable(callback_f): lookup_id = related_query.lookup_obj.lookup_id if lookup_done: callback_f(lookup_id, None, msg.src_node) # maintenance related tasks maintenance_queries_to_send = \ self._routing_m.on_error_received(addr) else: # unknown type return self._next_main_loop_call_ts, datagrams_to_send # we are done with the plugins # now we have maintenance_queries_to_send, let's send them! datagrams = self._register_queries(maintenance_queries_to_send) datagrams_to_send.extend(datagrams) if exp_queries_to_send: datagrams = self._register_queries(exp_queries_to_send) datagrams_to_send.extend(datagrams) return self._next_main_loop_call_ts, datagrams_to_send def _on_query_received(self): return def _on_response_received(self): return def _on_error_received(self): return def _on_timeout(self, related_query): queries_to_send = [] #TODO: on_timeout should return queries (raul) exp_queries_to_send = self._experimental_m.on_timeout(related_query) if related_query.lookup_obj: (lookup_queries_to_send, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_timeout(related_query.dst_node) queries_to_send.extend(lookup_queries_to_send) callback_f = related_query.lookup_obj.callback_f if lookup_done: # Size estimation if size_estimation: line = '%d %d\n' % ( related_query.lookup_obj.get_number_nodes_within_region()) self._size_estimation_file.write(line) self._size_estimation_file.flush() queries_to_send.extend(self._announce( related_query.lookup_obj)) lookup_id = related_query.lookup_obj.lookup_id if callback_f and callable(callback_f): related_query.lookup_obj.callback_f(lookup_id, None, None) maintenance_queries_to_send = self._routing_m.on_timeout( related_query.dst_node) if maintenance_queries_to_send: queries_to_send.extend(maintenance_queries_to_send) if exp_queries_to_send: datagrams = self._register_queries(exp_queries_to_send) datagrams_to_send.extend(datagrams) return queries_to_send def _announce(self, lookup_obj): queries_to_send, announce_to_myself = lookup_obj.announce() return queries_to_send ''' if announce_to_myself: self._tracker.put(lookup_obj._info_hash, (self._my_node.addr[0], lookup_obj._bt_port)) ''' def _register_queries(self, queries_to_send, lookup_obj=None): if not queries_to_send: return [] timeout_call_ts, datagrams_to_send = self._querier.register_queries( queries_to_send) self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, timeout_call_ts) return datagrams_to_send
?clinvarVariant clinvarv:Variant_Phenotype ?disease ?clinvarVariant ?effect ?disease } """ # Vide a priori query7 = """ SELECT DISTINCT ?effect ?disease WHERE { ?clinvarVariant clinvar:x-medgen ?disease ?clinvarVariant ?effect ?disease } """ localQuerier = Querier(url) # resultQuery1 = localQuerier.query(prefix+query1) # csvFile1 = open("csvFile1.csv", 'w') # csvFile1.write(resultQuery1) # print "[Running Time] %s sec" % (time.time() - start_time) # resultQuery2 = localQuerier.query(prefix+query2) # csvFile2 = open("csvFile2.csv", 'w') # csvFile2.write(resultQuery2) # print "[Running Time] %s sec" % (time.time() - start_time) # resultQuery3 = localQuerier.query(prefix+query3) # csvFile3 = open("csvFile3.csv", 'w') # csvFile3.write(resultQuery3) # print "[Running Time] %s sec" % (time.time() - start_time)
class Modem2413U(Device): """============== Insteon PowerLinc modem (PLM) ===============""" def __init__(self, name, addr): Device.__init__(self, name, addr) self.dbbuilder = ModemDBBuilder(addr, self.db) def __modifyModemDB(self, listener): self.dbbuilder.setListener(listener) # after db download complete, listener will perform action iofun.out("getting db, be patient!") self.dbbuilder.clear() self.dbbuilder.start() def getdb(self): """getdb() download the modem database and print it on the console""" self.dbbuilder.start() self.dbbuilder.wait() self.dbbuilder.dumpDB() out("Modem Link DB complete") def startWatch(self): """startWatch() modem will print all incoming messages on terminal""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(MsgDumper("modem")) self.querier.startWait(10000) def stopWatch(self): """stopWatch() stop modem from printing all incoming messages on terminal""" if (self.querier): self.querier.cancel() def getid(self): """getid() get modem id data""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(IMInfoMsgHandler("getid")) msg = Msg.s_makeMessage("GetIMInfo") self.querier.sendMsg(msg) def sendOn(self, group): """sendOn(group) sends ALLLink broadcast ON message to group "group" """ msg = message.createStdMsg(InsteonAddress("00.00.00"), 0x0f, 0x11, 0xFF, group) iofun.writeMsg(msg) iofun.out("sent msg: " + msg.toString()) def sendOff(self, group): """sendOff(group) sends ALLLink broadcast OFF message to group "group" """ msg = message.createStdMsg(InsteonAddress("00.00.00"), 0x0f, 0x13, 0xFF, group) iofun.writeMsg(msg) iofun.out("sent msg: " + msg.toString()) def linkAsController(self, otherDevice, group): """linkAsController(otherDevice, group) puts modem in link mode to control device "otherDevice" on group "group" """ addr = InsteonAddress(otherDevice) self.querier = Querier(addr) self.querier.setMsgHandler(DefaultMsgHandler("link as controller")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0x01) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def linkAsResponder(self, otherDevice, group): """linkAsResponder(otherDevice, group) puts modem in link mode to respond to device "otherDevice" on group "group" """ addr = InsteonAddress(otherDevice) self.querier = Querier(addr) self.querier.setMsgHandler(DefaultMsgHandler("start linking")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0x00) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def linkAsEither(self, otherDevice, group): """linkAsEither(otherDevice, group) puts modem in link mode to link as controller or responder to device "otherDevice" on group "group" """ addr = InsteonAddress(otherDevice) self.querier = Querier(addr) self.querier.setMsgHandler( DefaultMsgHandler("link/unlink as controller or responder")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0x03) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def respondToUnlink(self, otherDevice, group): """respondToUnlink(otherDevice, group) make modem respond to unlink message from other device""" # could not get 0xFF to unlink self.linkAsEither(otherDevice, group) def unlinkAsController(self, otherDevice, group): """unlinkAsController(otherDevice, group) puts modem in unlink mode to unlink as controller from device "otherDevice" on group "group" """ addr = InsteonAddress(otherDevice) self.querier = Querier(addr) self.querier.setMsgHandler(DefaultMsgHandler("unlink as controller")) msg = Msg.s_makeMessage("StartALLLinking") msg.setByte("LinkCode", 0xFF) msg.setByte("ALLLinkGroup", group) self.querier.sendMsg(msg) def cancelLinking(self): """cancelLinking() takes modem out of linking or unlinking mode""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(DefaultMsgHandler("cancel linking")) msg = Msg.s_makeMessage("CancelALLLinking") self.querier.sendMsg(msg) def addController(self, addr, group): """addController(addr, group): adds device with address "addr" to modem link database as controller for group "group" """ self.modifyRecord(addr, group, 0x40, 0xa2, [0,0,group], "addController") def addResponder(self, addr, group): """addResponder(addr, group): adds device with address "addr" to modem link database as responder to group "group" """ self.modifyRecord(addr, group, 0x41, 0xa2, [0,0,group], "addResponder") def addSoftwareResponder(self, addr): """addSoftwareResponder(addr): adds device with address "addr" to modem link database as software responder""" self.modifyRecord(addr, 0xef, 0x41, 0xa2, [0,0,0xef], "addSoftwareController") def removeResponderOrController(self, addr, group): """removeResponderOrController(addr, group) removes device with address "addr" and group "group" from modem link database""" self.__deleteFirstRecord(addr, group, "removeResponderOrController") def removeResponder(self, addr, group): """removeResponder(addr, group) could not be implemented for the modem. Use removeResponderOrController() instead!""" iofun.out("removeResponder(addr, group) could not be implemented" + " for the modem. Use removeResponderOrController() instead!") def removeController(self, addr, group): """removeController(addr, group) could not be implemented for the modem. Use removeResponderOrController() instead!""" iofun.out("removeController(addr, group) could not be implemented" + " for the modem. Use removeResponderOrController() instead!") def removeDevice(self, addr): """removeDevice(addr): removes all links to device with address "addr" from modem database""" self.__modifyModemDB(DeviceRemover(self, addr)) def __deleteFirstRecord(self, addr, group, text = "delete record"): self.modifyRecord(addr, group, 0x80, 0x00, [0,0,0], text) def modifyFirstOrAdd(self, addr, group, recordFlags, data): if (recordFlags & (1 << 6)): # controller self.modifyRecord(addr, group, 0x40, recordFlags, data, "modify first or add") else: self.modifyRecord(addr, group, 0x41, recordFlags, data, "modify first or add") def modifyFirstControllerOrAdd(self, addr, group, data): self.modifyRecord(addr, group, 0x40, 0xe2, data, "modify first ctrl found or add") def modifyFirstResponderOrAdd(self, addr, group, data): self.modifyRecord(addr, group, 0x41, 0xa2, data, "modify first resp found or add") def modifyRecord(self, addr, group, controlCode, recordFlags, data, txt): msg = self.__makeModMsg(addr, group, controlCode, recordFlags, data, txt) self.querier = Querier(self.address) self.querier.setMsgHandler(DefaultMsgHandler(txt)) self.querier.sendMsg(msg) def __makeModMsg(self, addr, group, controlCode, recordFlags, data, txt): msg = Msg.s_makeMessage("ManageALLLinkRecord"); msg.setByte("controlCode", controlCode); # mod. first ctrl found or add msg.setByte("recordFlags", recordFlags); msg.setByte("ALLLinkGroup", group); msg.setAddress("linkAddress", InsteonAddress(addr)); msg.setByte("linkData1", data[0] & 0xFF) msg.setByte("linkData2", data[1] & 0xFF) msg.setByte("linkData3", data[2] & 0xFF) return msg; def saveDB(self, filename): """saveDB(filename) save modem database to file "filename" """ self.dbbuilder.start() self.dbbuilder.wait() self.dbbuilder.saveDB(filename) def loadDB(self, filename): """loadDB(filename) restore modem database from file "filename" """ self.dbbuilder.loadDB(filename) self.dbbuilder.dumpDB() def nukeDB(self): """nukeDB() delete complete modem database! """ self.dbbuilder.start() self.dbbuilder.wait() self.dbbuilder.nukeDB(self) def restoreDB(self, filename): """restoreDB() restore modem database from file "filename" """ self.loadDB(filename) self.dbbuilder.restoreDB(self, filename)
def startWatch(self): """startWatch() modem will print all incoming messages on terminal""" self.querier = Querier(InsteonAddress("00.00.00")) self.querier.setMsgHandler(MsgDumper("modem")) self.querier.startWait(10000)
class TestQuerier: def setup(self): time.mock_mode() self.querier = Querier()#tc.CLIENT_ID) def test_generate_tids(self): #TODO: move to message if RUN_CPU_INTENSIVE_TESTS: num_tids = pow(2, 16) + 2 #CPU intensive else: num_tids = 1000 for i in xrange(num_tids): eq_(self.querier._next_tid(), chr(i%256)+chr((i/256)%256)) def test_ping_with_reponse(self): # Client creates a query ping_msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = ping_msg # Client registers query timeout_ts, bencoded_msgs = self.querier.register_queries([q]) # Client sends bencoded_msg # Server gets bencoded_msg and creates response ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp(ping_msg.tid) time.sleep(1) eq_(self.querier.get_timeout_queries()[1], []) # The client receives the bencoded message (after 1 second) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is ping_msg def test_ping_with_timeout(self): # Client creates a query ping_msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = ping_msg # Client registers query bencoded_msg = self.querier.register_queries([q]) # Client sends bencoded_msg time.sleep(3) # The server never responds and the timeout is triggered timeout_queries = self.querier.get_timeout_queries() eq_(len(timeout_queries[1]), 1) assert timeout_queries[1][0] is ping_msg def test_unsolicited_response(self): # Server creates unsolicited response # It might well be that the server responds using another port, # and therefore, the addr is not matched # TODO: consider accepting responses from a different port ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp('zz') # The client receives the bencoded message ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None def test_response_with_different_tid(self): # Client creates a query ping_msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = ping_msg # Client registers query bencoded_msg = self.querier.register_queries([q]) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp('zz') # The client receives the bencoded message ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None def test_error_received(self): # Client creates a query msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = msg # Client registers query bencoded_msg = self.querier.register_queries([q]) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = servers_msg_f.outgoing_error(tc.CLIENT_NODE, message.GENERIC_E) bencoded_r = ping_r_msg_out.stamp(msg.tid) # The client receives the bencoded message ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is msg def test_many_queries(self): # Client creates a query msgs = [clients_msg_f.outgoing_ping_query( tc.SERVER_NODE) for i in xrange(10)] queries = msgs # Client registers query bencoded_msg = self.querier.register_queries(queries) # Client sends bencoded_msg time.sleep(1) # response for queries[3] ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp(msgs[3].tid) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is msgs[3] # error for queries[2] ping_r_msg_out = servers_msg_f.outgoing_error(tc.CLIENT_NODE, message.GENERIC_E) bencoded_r = ping_r_msg_out.stamp(msgs[2].tid) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is msgs[2] # response to wrong addr ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp(msgs[5].tid) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER2_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None # response with wrong tid ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp('ZZ') ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None # Still no time to trigger timeouts eq_(self.querier.get_timeout_queries()[1], []) time.sleep(1) # Now, the timeouts can be triggered timeout_queries = self.querier.get_timeout_queries() expected_msgs = msgs[:2] + msgs[4:] eq_(len(timeout_queries[1]), len(expected_msgs)) for related_query, expected_msg in zip( timeout_queries[1], expected_msgs): assert related_query is expected_msg def teardown(self): time.normal_mode()
class Controller(): def __init__(self, dht_addr, state_path, routing_m_mod, lookup_m_mod, private_dht_name): message.private_dht_name = private_dht_name self.state_filename = os.path.join(state_path, STATE_FILENAME) self.load_state() if not self._my_id: self._my_id = identifier.RandomId() self._my_node = Node(dht_addr, self._my_id) self._tracker = tracker.Tracker() self._token_m = token_manager.TokenManager() self._reactor = ThreadedReactor() self._reactor.listen_udp(self._my_node.addr[1], self._on_datagram_received) self._querier = Querier(self._my_id) bootstrap_nodes = self.loaded_nodes or BOOTSTRAP_NODES del self.loaded_nodes self._routing_m = routing_m_mod.RoutingManager(self._my_node, bootstrap_nodes) self._lookup_m = lookup_m_mod.LookupManager(self._my_id) current_time = time.time() self._next_maintenance_ts = current_time self._next_save_state_ts = current_time + SAVE_STATE_DELAY self._running = False def start(self): self._running = True self._reactor.start() self._main_loop() def stop(self): self._reactor.stop() def save_state(self): rnodes = self._routing_m.get_main_rnodes() f = open(self.state_filename, 'w') f.write('%r\n' % self._my_id) for rnode in rnodes: f.write('%d\t%r\t%s\t%d\t%f\n' % (self._my_id.log_distance(rnode.id), rnode.id, rnode.addr[0], rnode.addr[1], rnode.rtt * 1000)) f.close() def load_state(self): self._my_id = None self.loaded_nodes = [] try: f = open(self.state_filename) except IOError: return try: hex_id = f.readline().strip() self._my_id = Id(hex_id) for line in f: _, hex_id, ip, port, _ = line.split() addr = (ip, int(port)) node_ = Node(addr, Id(hex_id)) self.loaded_nodes.append(node_) f.close() except: self._my_id = None self.loaded_nodes = [] logger.error('state.dat is corrupted') def get_peers(self, lookup_id, info_hash, callback_f, bt_port=0): logger.critical('get_peers %d %r' % (bt_port, info_hash)) if time.time() > self._next_maintenance_ts + 1: logger.critical('minitwisted crashed or stopped!') return peers = self._tracker.get(info_hash) if peers: callback_f(lookup_id, peers) log_distance = info_hash.log_distance(self._my_id) bootstrap_rnodes = self._routing_m.get_closest_rnodes( log_distance, None, True) lookup_obj = self._lookup_m.get_peers(lookup_id, info_hash, callback_f, bt_port) lookup_queries_to_send = lookup_obj.start(bootstrap_rnodes) self._send_queries(lookup_queries_to_send) return len(lookup_queries_to_send) def print_routing_table_stats(self): self._routing_m.print_stats() def _main_loop(self): current_time = time.time() if current_time > self._next_maintenance_ts: maintenance_delay, queries_to_send, maintenance_lookup_target = self._routing_m.do_maintenance( ) self._send_queries(queries_to_send) if maintenance_lookup_target: log_distance = maintenance_lookup_target.log_distance( self._my_id) bootstrap_nodes = self._routing_m.get_closest_rnodes( log_distance, None, True) lookup_obj = self._lookup_m.maintenance_lookup( maintenance_lookup_target) lookup_queries_to_send = lookup_obj.start(bootstrap_nodes) self._send_queries(lookup_queries_to_send) self._next_maintenance_ts = current_time + maintenance_delay if current_time > self._next_save_state_ts: self.save_state() self._next_save_state_ts = current_time + SAVE_STATE_DELAY delay = min(self._next_maintenance_ts, self._next_save_state_ts) - current_time self._reactor.call_later(delay, self._main_loop) def _maintenance_lookup(self, target): self._lookup_m.maintenance_lookup(target) def _on_datagram_received(self, data, addr): try: msg = message.IncomingMsg(data, addr) except message.MsgError: return if msg.type == message.QUERY: if msg.sender_id == self._my_id: logger.debug('Got a msg from myself:\n%r', msg) return response_msg = self._get_response(msg) if response_msg: bencoded_response = response_msg.encode(msg.tid) self._reactor.sendto(bencoded_response, addr) maintenance_queries_to_send = self._routing_m.on_query_received( msg.sender_node) elif msg.type == message.RESPONSE: related_query = self._querier.on_response_received(msg, addr) if not related_query: return if related_query.lookup_obj: if msg.type == message.RESPONSE: lookup_queries_to_send, peers, num_parallel_queries, lookup_done = related_query.lookup_obj.on_response_received( msg, msg.sender_node) self._send_queries(lookup_queries_to_send) if related_query.lookup_obj.callback_f: lookup_id = related_query.lookup_obj.lookup_id if peers: related_query.lookup_obj.callback_f(lookup_id, peers) if lookup_done: self._announce(related_query.lookup_obj) related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_response_received( msg.sender_node, related_query.rtt, msg.all_nodes) elif msg.type == message.ERROR: related_query = self._querier.on_error_received(msg, addr) if not related_query: return if related_query.lookup_obj: peers = None lookup_queries_to_send, num_parallel_queries, lookup_done = related_query.lookup_obj.on_error_received( msg, addr) self._send_queries(lookup_queries_to_send) if related_query.lookup_obj.callback_f: lookup_id = related_query.lookup_obj.lookup_id if lookup_done: self._announce(related_query.lookup_obj) related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_error_received( addr) else: return self._send_queries(maintenance_queries_to_send) def _get_response(self, msg): if msg.query == message.PING: return message.OutgoingPingResponse(self._my_id) if msg.query == message.FIND_NODE: log_distance = msg.target.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes( log_distance, NUM_NODES, False) return message.OutgoingFindNodeResponse(self._my_id, rnodes) if msg.query == message.GET_PEERS: token = self._token_m.get() log_distance = msg.info_hash.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes( log_distance, NUM_NODES, False) peers = self._tracker.get(msg.info_hash) if peers: logger.debug('RESPONDING with PEERS:\n%r' % peers) return message.OutgoingGetPeersResponse(self._my_id, token, nodes=rnodes, peers=peers) if msg.query == message.ANNOUNCE_PEER: peer_addr = (msg.sender_addr[0], msg.bt_port) self._tracker.put(msg.info_hash, peer_addr) return message.OutgoingAnnouncePeerResponse(self._my_id) logger.debug('Invalid QUERY: %r' % msg.query) def _on_response_received(self, msg): pass def _on_timeout(self, addr): related_query = self._querier.on_timeout(addr) if not related_query: return if related_query.lookup_obj: lookup_queries_to_send, num_parallel_queries, lookup_done = related_query.lookup_obj.on_timeout( related_query.dstnode) self._send_queries(lookup_queries_to_send) if lookup_done and related_query.lookup_obj.callback_f: self._announce(related_query.lookup_obj) lookup_id = related_query.lookup_obj.lookup_id related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_timeout( related_query.dstnode) self._send_queries(maintenance_queries_to_send) def _announce(self, lookup_obj): queries_to_send, announce_to_myself = lookup_obj.announce() self._send_queries(queries_to_send) def _send_queries(self, queries_to_send, lookup_obj=None): if queries_to_send is None: return for query in queries_to_send: timeout_task = self._reactor.call_later(TIMEOUT_DELAY, self._on_timeout, query.dstnode.addr) bencoded_query = self._querier.register_query(query, timeout_task) self._reactor.sendto(bencoded_query, query.dstnode.addr)
class TestQuerier: def setup(self): time.mock_mode() self.querier = Querier() #tc.CLIENT_ID) def test_generate_tids(self): #TODO: move to message if RUN_CPU_INTENSIVE_TESTS: num_tids = pow(2, 16) + 2 #CPU intensive else: num_tids = 1000 for i in xrange(num_tids): eq_(self.querier._next_tid(), chr(i % 256) + chr((i / 256) % 256)) def test_ping_with_reponse(self): # Client creates a query ping_msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = ping_msg # Client registers query timeout_ts, bencoded_msgs = self.querier.register_queries([q]) # Client sends bencoded_msg # Server gets bencoded_msg and creates response ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp(ping_msg.tid) time.sleep(1) eq_(self.querier.get_timeout_queries()[1], []) # The client receives the bencoded message (after 1 second) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is ping_msg def test_ping_with_timeout(self): # Client creates a query ping_msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = ping_msg # Client registers query bencoded_msg = self.querier.register_queries([q]) # Client sends bencoded_msg time.sleep(3) # The server never responds and the timeout is triggered timeout_queries = self.querier.get_timeout_queries() eq_(len(timeout_queries[1]), 1) assert timeout_queries[1][0] is ping_msg def test_unsolicited_response(self): # Server creates unsolicited response # It might well be that the server responds using another port, # and therefore, the addr is not matched # TODO: consider accepting responses from a different port ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp('zz') # The client receives the bencoded message ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None def test_response_with_different_tid(self): # Client creates a query ping_msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = ping_msg # Client registers query bencoded_msg = self.querier.register_queries([q]) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp('zz') # The client receives the bencoded message ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None def test_error_received(self): # Client creates a query msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) q = msg # Client registers query bencoded_msg = self.querier.register_queries([q]) # Client sends bencoded_msg time.sleep(1) # Server gets bencoded_msg and creates response ping_r_msg_out = servers_msg_f.outgoing_error(tc.CLIENT_NODE, message.GENERIC_E) bencoded_r = ping_r_msg_out.stamp(msg.tid) # The client receives the bencoded message ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is msg def test_many_queries(self): # Client creates a query msgs = [ clients_msg_f.outgoing_ping_query(tc.SERVER_NODE) for i in xrange(10) ] queries = msgs # Client registers query bencoded_msg = self.querier.register_queries(queries) # Client sends bencoded_msg time.sleep(1) # response for queries[3] ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp(msgs[3].tid) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is msgs[3] # error for queries[2] ping_r_msg_out = servers_msg_f.outgoing_error(tc.CLIENT_NODE, message.GENERIC_E) bencoded_r = ping_r_msg_out.stamp(msgs[2].tid) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is msgs[2] # response to wrong addr ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp(msgs[5].tid) ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER2_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None # response with wrong tid ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE) bencoded_r = ping_r_msg_out.stamp('ZZ') ping_r_in = clients_msg_f.incoming_msg( Datagram(bencoded_r, tc.SERVER_ADDR)) related_query = self.querier.get_related_query(ping_r_in) assert related_query is None # Still no time to trigger timeouts eq_(self.querier.get_timeout_queries()[1], []) time.sleep(1) # Now, the timeouts can be triggered timeout_queries = self.querier.get_timeout_queries() expected_msgs = msgs[:2] + msgs[4:] eq_(len(timeout_queries[1]), len(expected_msgs)) for related_query, expected_msg in zip(timeout_queries[1], expected_msgs): assert related_query is expected_msg def teardown(self): time.normal_mode()
from dash.dependencies import Input, Output # Neo4j Setup from py2neo import Graph import json # Import class to make queries to Neo4j from querier import Querier # Dash Setup external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] app = dash.Dash(__name__, external_stylesheets=external_stylesheets) server = app.server # Connect to Neo4j querier = Querier() # This function returns all components for refresh def serve_layout(): return html.Div(children=[dark_theme()], style={'padding': '0px'}) # Set the website refresh to call a function app.layout = serve_layout # Colors for dark theme theme = { 'dark': True, 'detail': '#149414', 'primary': '#00EA64',
def setup(self): global time time = querier.time = MockTime() self.querier = Querier(tc.CLIENT_ID)
def setup(self): time.mock_mode() self.querier = Querier()#tc.CLIENT_ID)
class Controller(): def __init__(self, dht_addr, state_path, routing_m_mod, lookup_m_mod, private_dht_name): message.private_dht_name = private_dht_name self.state_filename = os.path.join(state_path, STATE_FILENAME) self.load_state() if not self._my_id: self._my_id = identifier.RandomId() self._my_node = Node(dht_addr, self._my_id) self._tracker = tracker.Tracker() self._token_m = token_manager.TokenManager() self._reactor = ThreadedReactor() self._reactor.listen_udp(self._my_node.addr[1], self._on_datagram_received) self._querier = Querier(self._my_id) bootstrap_nodes = self.loaded_nodes or BOOTSTRAP_NODES del self.loaded_nodes self._routing_m = routing_m_mod.RoutingManager(self._my_node, bootstrap_nodes) self._lookup_m = lookup_m_mod.LookupManager(self._my_id) current_time = time.time() self._next_maintenance_ts = current_time self._next_save_state_ts = current_time + SAVE_STATE_DELAY self._running = False def start(self): self._running = True self._reactor.start() self._main_loop() def stop(self): self._reactor.stop() def save_state(self): rnodes = self._routing_m.get_main_rnodes() f = open(self.state_filename, 'w') f.write('%r\n' % self._my_id) for rnode in rnodes: f.write('%d\t%r\t%s\t%d\t%f\n' % (self._my_id.log_distance(rnode.id), rnode.id, rnode.addr[0], rnode.addr[1], rnode.rtt * 1000)) f.close() def load_state(self): self._my_id = None self.loaded_nodes = [] try: f = open(self.state_filename) except IOError: return try: hex_id = f.readline().strip() self._my_id = Id(hex_id) for line in f: _, hex_id, ip, port, _ = line.split() addr = (ip, int(port)) node_ = Node(addr, Id(hex_id)) self.loaded_nodes.append(node_) f.close() except: self._my_id = None self.loaded_nodes = [] logger.error('state.dat is corrupted') def get_peers(self, lookup_id, info_hash, callback_f, bt_port = 0): logger.critical('get_peers %d %r' % (bt_port, info_hash)) if time.time() > self._next_maintenance_ts + 1: logger.critical('minitwisted crashed or stopped!') return peers = self._tracker.get(info_hash) if peers: callback_f(lookup_id, peers) log_distance = info_hash.log_distance(self._my_id) bootstrap_rnodes = self._routing_m.get_closest_rnodes(log_distance, None, True) lookup_obj = self._lookup_m.get_peers(lookup_id, info_hash, callback_f, bt_port) lookup_queries_to_send = lookup_obj.start(bootstrap_rnodes) self._send_queries(lookup_queries_to_send) return len(lookup_queries_to_send) def print_routing_table_stats(self): self._routing_m.print_stats() def _main_loop(self): current_time = time.time() if current_time > self._next_maintenance_ts: maintenance_delay, queries_to_send, maintenance_lookup_target = self._routing_m.do_maintenance() self._send_queries(queries_to_send) if maintenance_lookup_target: log_distance = maintenance_lookup_target.log_distance(self._my_id) bootstrap_nodes = self._routing_m.get_closest_rnodes(log_distance, None, True) lookup_obj = self._lookup_m.maintenance_lookup(maintenance_lookup_target) lookup_queries_to_send = lookup_obj.start(bootstrap_nodes) self._send_queries(lookup_queries_to_send) self._next_maintenance_ts = current_time + maintenance_delay if current_time > self._next_save_state_ts: self.save_state() self._next_save_state_ts = current_time + SAVE_STATE_DELAY delay = min(self._next_maintenance_ts, self._next_save_state_ts) - current_time self._reactor.call_later(delay, self._main_loop) def _maintenance_lookup(self, target): self._lookup_m.maintenance_lookup(target) def _on_datagram_received(self, data, addr): try: msg = message.IncomingMsg(data, addr) except message.MsgError: return if msg.type == message.QUERY: if msg.sender_id == self._my_id: logger.debug('Got a msg from myself:\n%r', msg) return response_msg = self._get_response(msg) if response_msg: bencoded_response = response_msg.encode(msg.tid) self._reactor.sendto(bencoded_response, addr) maintenance_queries_to_send = self._routing_m.on_query_received(msg.sender_node) elif msg.type == message.RESPONSE: related_query = self._querier.on_response_received(msg, addr) if not related_query: return if related_query.lookup_obj: if msg.type == message.RESPONSE: lookup_queries_to_send, peers, num_parallel_queries, lookup_done = related_query.lookup_obj.on_response_received(msg, msg.sender_node) self._send_queries(lookup_queries_to_send) if related_query.lookup_obj.callback_f: lookup_id = related_query.lookup_obj.lookup_id if peers: related_query.lookup_obj.callback_f(lookup_id, peers) if lookup_done: self._announce(related_query.lookup_obj) related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_response_received(msg.sender_node, related_query.rtt, msg.all_nodes) elif msg.type == message.ERROR: related_query = self._querier.on_error_received(msg, addr) if not related_query: return if related_query.lookup_obj: peers = None lookup_queries_to_send, num_parallel_queries, lookup_done = related_query.lookup_obj.on_error_received(msg, addr) self._send_queries(lookup_queries_to_send) if related_query.lookup_obj.callback_f: lookup_id = related_query.lookup_obj.lookup_id if lookup_done: self._announce(related_query.lookup_obj) related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_error_received(addr) else: return self._send_queries(maintenance_queries_to_send) def _get_response(self, msg): if msg.query == message.PING: return message.OutgoingPingResponse(self._my_id) if msg.query == message.FIND_NODE: log_distance = msg.target.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes(log_distance, NUM_NODES, False) return message.OutgoingFindNodeResponse(self._my_id, rnodes) if msg.query == message.GET_PEERS: token = self._token_m.get() log_distance = msg.info_hash.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes(log_distance, NUM_NODES, False) peers = self._tracker.get(msg.info_hash) if peers: logger.debug('RESPONDING with PEERS:\n%r' % peers) return message.OutgoingGetPeersResponse(self._my_id, token, nodes=rnodes, peers=peers) if msg.query == message.ANNOUNCE_PEER: peer_addr = (msg.sender_addr[0], msg.bt_port) self._tracker.put(msg.info_hash, peer_addr) return message.OutgoingAnnouncePeerResponse(self._my_id) logger.debug('Invalid QUERY: %r' % msg.query) def _on_response_received(self, msg): pass def _on_timeout(self, addr): related_query = self._querier.on_timeout(addr) if not related_query: return if related_query.lookup_obj: lookup_queries_to_send, num_parallel_queries, lookup_done = related_query.lookup_obj.on_timeout(related_query.dstnode) self._send_queries(lookup_queries_to_send) if lookup_done and related_query.lookup_obj.callback_f: self._announce(related_query.lookup_obj) lookup_id = related_query.lookup_obj.lookup_id related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_timeout(related_query.dstnode) self._send_queries(maintenance_queries_to_send) def _announce(self, lookup_obj): queries_to_send, announce_to_myself = lookup_obj.announce() self._send_queries(queries_to_send) def _send_queries(self, queries_to_send, lookup_obj = None): if queries_to_send is None: return for query in queries_to_send: timeout_task = self._reactor.call_later(TIMEOUT_DELAY, self._on_timeout, query.dstnode.addr) bencoded_query = self._querier.register_query(query, timeout_task) self._reactor.sendto(bencoded_query, query.dstnode.addr)
def setup(self): time.mock_mode() self.querier = Querier() #tc.CLIENT_ID)
""" This program queries the Materials Project database with specified materials using user's API key. """ __author__ = "Erpan Arkin" __email__ = "*****@*****.**" description = 'Materials Project Querying Tool' parser = ArgumentParser(description=description) parser.add_argument('API_KEY', metavar='API_KEY', type=str, help="User's API Key to connect to the MP") parser.add_argument('MATERIAL', metavar='MATERIAL', type=str, help='Chemical formula of the target material, e.g. Fe2O3') parser.add_argument('-p', action='store_true', help="plot all entries' id verse energy per atom") args = parser.parse_args() my_query = Querier(args.API_KEY, args.MATERIAL) my_query.GS_finder() my_query.print_results() if args.p: my_query.plot()
class Controller: def __init__(self, version_label, my_node, conf_path, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name, bootstrap_mode): self.bootstrapper = bootstrap.OverlayBootstrapper(conf_path) my_addr = my_node.addr self._my_id = my_node.id # id indicated by user if not self._my_id: self._my_id = self._my_id = identifier.RandomId() # random id self._my_node = Node(my_addr, self._my_id, version=version_label) self.msg_f = message.MsgFactory(version_label, self._my_id, private_dht_name) self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager(self._my_node, self.msg_f, self.bootstrapper) self._responder = responder.Responder(self._my_id, self._routing_m, self.msg_f, bootstrap_mode) self._tracker = self._responder._tracker self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f, self.bootstrapper) self._experimental_m = experimental_m_mod.ExperimentalManager( self._my_node.id, self.msg_f) current_ts = time.time() self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._cached_lookups = [] def on_stop(self): self._experimental_m.on_stop() self.bootstrapper.save_to_file() def get_peers(self, lookup_id, info_hash, callback_f, bt_port, use_cache): """ Start a get\_peers lookup whose target is 'info\_hash'. The handler 'callback\_f' will be called with three arguments ('lookup\_id', 'peers', 'node') whenever peers are discovered. Once the lookup is completed, the handler will be called with arguments: ('lookup\_id', None, None). This method is called by minitwisted, using the minitwisted thread. """ datagrams_to_send = [] logger.debug('get_peers %d %r' % (bt_port, info_hash)) if use_cache: peers = self._get_cached_peers(info_hash) if peers and callback_f and callable(callback_f): callback_f(lookup_id, peers, None) callback_f(lookup_id, None, None) return datagrams_to_send lookup_obj = self._lookup_m.get_peers(lookup_id, info_hash, callback_f, bt_port) queries_to_send = [] distance = lookup_obj.info_hash.distance(self._my_id) bootstrap_rnodes = self._routing_m.get_closest_rnodes( distance.log, 0, True) #TODO: get the full bucket # look if I'm tracking this info_hash peers = self._tracker.get(lookup_obj.info_hash) callback_f = lookup_obj.callback_f if peers: self._add_cache_peers(lookup_obj.info_hash, peers) if callback_f and callable(callback_f): callback_f(lookup_obj.lookup_id, peers, None) # do the lookup # NOTE: if bootstrap_rnodes is empty, a OVERLAY BOOTSTRAP will be # done. queries_to_send = lookup_obj.start(bootstrap_rnodes, self.bootstrapper) datagrams_to_send = self._register_queries(queries_to_send) return datagrams_to_send def _get_cached_peers(self, info_hash): oldest_valid_ts = time.time() - CACHE_VALID_PERIOD for ts, cached_info_hash, peers in self._cached_lookups: if ts > oldest_valid_ts and info_hash == cached_info_hash: return peers def _add_cache_peers(self, info_hash, peers): oldest_valid_ts = time.time() - CACHE_VALID_PERIOD while self._cached_lookups and self._cached_lookups[0][ 0] < oldest_valid_ts: # clean up old entries del self._cached_lookups[0] if self._cached_lookups and self._cached_lookups[-1][1] == info_hash: self._cached_lookups[-1][2].extend(peers) else: self._cached_lookups.append((time.time(), info_hash, peers)) def print_routing_table_stats(self): self._routing_m.print_stats() def print_routing_table(self): self._routing_m.print_table() def main_loop(self): """ Perform maintenance operations. The main operation is routing table maintenance where staled nodes are added/probed/replaced/removed as needed. The routing management module specifies the implementation details. This includes keeping track of queries that have not been responded for a long time (timeout) with the help of querier.Querier. The routing manager and the lookup manager will be informed of those timeouts. This method is designed to be used as minitwisted's heartbeat handler. """ queries_to_send = [] current_ts = time.time() #TODO: I think this if should be removed # At most, 1 second between calls to main_loop after the first call if current_ts >= self._next_main_loop_call_ts: self._next_main_loop_call_ts = current_ts + 1 else: # It's too early return self._next_main_loop_call_ts, [] # Take care of timeouts if current_ts >= self._next_timeout_ts: (self._next_timeout_ts, timeout_queries) = self._querier.get_timeout_queries() for query in timeout_queries: queries_to_send.extend(self._on_timeout(query)) # Routing table maintenance if time.time() >= self._next_maintenance_ts: (maintenance_delay, queries, maintenance_lookup) = self._routing_m.do_maintenance() self._next_maintenance_ts = current_ts + maintenance_delay self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, self._next_maintenance_ts) queries_to_send.extend(queries) if maintenance_lookup: target, rnodes = maintenance_lookup lookup_obj = self._lookup_m.maintenance_lookup(target) queries_to_send.extend(lookup_obj.start(rnodes)) # Return control to reactor datagrams_to_send = self._register_queries(queries_to_send) return self._next_main_loop_call_ts, datagrams_to_send def _maintenance_lookup(self, target): self._lookup_m.maintenance_lookup(target) def on_datagram_received(self, datagram): """ Perform the actions associated to the arrival of the given datagram. The datagram will be ignored in cases such as invalid format. Otherwise, the datagram will be decoded and different modules will be informed to take action on it. For instance, if the datagram contains a response to a lookup query, both routing and lookup manager will be informed. Additionally, if that response contains peers, the lookup's handler will be called (see get\_peers above). This method is designed to be used as minitwisted's networking handler. """ exp_queries_to_send = [] data = datagram.data addr = datagram.addr datagrams_to_send = [] try: msg = self.msg_f.incoming_msg(datagram) except (message.MsgError): # ignore message return self._next_main_loop_call_ts, datagrams_to_send if msg.type == message.QUERY: if msg.src_node.id == self._my_id: logger.debug('Got a msg from myself:\n%r', msg) return self._next_main_loop_call_ts, datagrams_to_send #zinat: inform experimental_module exp_queries_to_send = self._experimental_m.on_query_received(msg) response_msg = self._responder.get_response(msg) if response_msg: bencoded_response = response_msg.stamp(msg.tid) datagrams_to_send.append( message.Datagram(bencoded_response, addr)) maintenance_queries_to_send = self._routing_m.on_query_received( msg.src_node) elif msg.type == message.RESPONSE: related_query = self._querier.get_related_query(msg) if not related_query: # Query timed out or unrequested response return self._next_main_loop_call_ts, datagrams_to_send ## zinat: if related_query.experimental_obj: exp_queries_to_send = self._experimental_m.on_response_received( msg, related_query) #TODO: you need to get datagrams to be able to send messages (raul) # lookup related tasks if related_query.lookup_obj: (lookup_queries_to_send, peers, num_parallel_queries, lookup_done) = related_query.lookup_obj.on_response_received( msg, msg.src_node) datagrams = self._register_queries(lookup_queries_to_send) datagrams_to_send.extend(datagrams) lookup_obj = related_query.lookup_obj lookup_id = lookup_obj.lookup_id callback_f = lookup_obj.callback_f if peers: self._add_cache_peers(lookup_obj.info_hash, peers) if callback_f and callable(callback_f): callback_f(lookup_id, peers, msg.src_node) if lookup_done: if callback_f and callable(callback_f): callback_f(lookup_id, None, msg.src_node) queries_to_send = self._announce(related_query.lookup_obj) datagrams = self._register_queries(queries_to_send) datagrams_to_send.extend(datagrams) # maintenance related tasks maintenance_queries_to_send = \ self._routing_m.on_response_received( msg.src_node, related_query.rtt, msg.all_nodes) elif msg.type == message.ERROR: related_query = self._querier.get_related_query(msg) if not related_query: # Query timed out or unrequested response return self._next_main_loop_call_ts, datagrams_to_send #TODO: zinat: same as response exp_queries_to_send = self._experimental_m.on_error_received( msg, related_query) # lookup related tasks if related_query.lookup_obj: peers = None # an error msg doesn't have peers (lookup_queries_to_send, num_parallel_queries, lookup_done) = related_query.lookup_obj.on_error_received( msg, addr) datagrams = self._register_queries(lookup_queries_to_send) datagrams_to_send.extend(datagrams) callback_f = related_query.lookup_obj.callback_f if callback_f and callable(callback_f): lookup_id = related_query.lookup_obj.lookup_id if lookup_done: callback_f(lookup_id, None, msg.src_node) if lookup_done: datagrams = self._announce(related_query.lookup_obj) datagrams_to_send.extend(datagrams) # maintenance related tasks maintenance_queries_to_send = \ self._routing_m.on_error_received(addr) else: # unknown type return self._next_main_loop_call_ts, datagrams_to_send # we are done with the plugins # now we have maintenance_queries_to_send, let's send them! datagrams = self._register_queries(maintenance_queries_to_send) datagrams_to_send.extend(datagrams) if exp_queries_to_send: datagrams = self._register_queries(exp_queries_to_send) datagrams_to_send.extend(datagrams) return self._next_main_loop_call_ts, datagrams_to_send def _on_query_received(self): return def _on_response_received(self): return def _on_error_received(self): return def _on_timeout(self, related_query): queries_to_send = [] #TODO: on_timeout should return queries (raul) exp_queries_to_send = self._experimental_m.on_timeout(related_query) if related_query.lookup_obj: (lookup_queries_to_send, num_parallel_queries, lookup_done) = related_query.lookup_obj.on_timeout( related_query.dst_node) queries_to_send.extend(lookup_queries_to_send) callback_f = related_query.lookup_obj.callback_f if lookup_done: lookup_id = related_query.lookup_obj.lookup_id if callback_f and callable(callback_f): related_query.lookup_obj.callback_f(lookup_id, None, None) queries_to_send.extend(self._announce( related_query.lookup_obj)) maintenance_queries_to_send = self._routing_m.on_timeout( related_query.dst_node) if maintenance_queries_to_send: queries_to_send.extend(maintenance_queries_to_send) if exp_queries_to_send: datagrams = self._register_queries(exp_queries_to_send) datagrams_to_send.extend(datagrams) return queries_to_send def _announce(self, lookup_obj): queries_to_send, announce_to_myself = lookup_obj.announce() return queries_to_send ''' if announce_to_myself: self._tracker.put(lookup_obj._info_hash, (self._my_node.addr[0], lookup_obj._bt_port)) ''' def _register_queries(self, queries_to_send, lookup_obj=None): if not queries_to_send: return [] timeout_call_ts, datagrams_to_send = self._querier.register_queries( queries_to_send) self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, timeout_call_ts) return datagrams_to_send
class Controller: def __init__(self, dht_addr, state_path, routing_m_mod, lookup_m_mod): self.state_filename = os.path.join(state_path, STATE_FILENAME) self.load_state() if not self._my_id: self._my_id = identifier.RandomId() self._my_node = Node(dht_addr, self._my_id) self._tracker = tracker.Tracker() self._token_m = token_manager.TokenManager() self._reactor = ThreadedReactor() self._reactor.listen_udp(self._my_node.addr[1], self._on_datagram_received) #self._rpc_m = RPCManager(self._reactor) self._querier = Querier(self._my_id) bootstrap_nodes = self.loaded_nodes or BOOTSTRAP_NODES del self.loaded_nodes self._routing_m = routing_m_mod.RoutingManager(self._my_node, bootstrap_nodes) # self._responder = Responder(self._my_id, self._routing_m, # self._tracker, self._token_m) self._lookup_m = lookup_m_mod.LookupManager(self._my_id) current_time = time.time() self._next_maintenance_ts = current_time self._next_save_state_ts = current_time + SAVE_STATE_DELAY self._running = False def start(self): assert not self._running self._running = True self._reactor.start() self._main_loop() def stop(self): assert self._running #TODO2: stop each manager self._reactor.stop() def save_state(self): rnodes = self._routing_m.get_main_rnodes() f = open(self.state_filename, 'w') f.write('%r\n' % self._my_id) for rnode in rnodes: f.write('%d\t%r\t%s\t%d\t%f\n' % ( self._my_id.log_distance(rnode.id), rnode.id, rnode.addr[0], rnode.addr[1], rnode.rtt * 1000)) f.close() def load_state(self): self._my_id = None self.loaded_nodes = [] try: f = open(self.state_filename) except(IOError): return # the first line contains this node's identifier hex_id = f.readline().strip() self._my_id = Id(hex_id) # the rest of the lines contain routing table nodes # FORMAT # log_distance hex_id ip port rtt for line in f: _, hex_id, ip, port, _ = line.split() addr = (ip, int(port)) node_ = Node(addr, Id(hex_id)) self.loaded_nodes.append(node_) f.close def get_peers(self, lookup_id, info_hash, callback_f, bt_port=0): assert self._running log_distance = info_hash.log_distance(self._my_id) bootstrap_rnodes = self._routing_m.get_closest_rnodes(log_distance, None, True) lookup_obj = self._lookup_m.get_peers(info_hash, callback_f, bt_port) #TODO: propagate lookup_id to the lookup plugin lookup_obj.lookup_id = lookup_id ################################################ lookup_queries_to_send = lookup_obj.start(bootstrap_rnodes) self._send_queries(lookup_queries_to_send) return len(lookup_queries_to_send) def print_routing_table_stats(self): self._routing_m.print_stats() def _main_loop(self): current_time = time.time() # Routing table if current_time > self._next_maintenance_ts: (maintenance_delay, queries_to_send, maintenance_lookup_target) = self._routing_m.do_maintenance() self._send_queries(queries_to_send) if maintenance_lookup_target: log_distance = maintenance_lookup_target.log_distance( self._my_id) bootstrap_nodes = self._routing_m.get_closest_rnodes( log_distance, None, True) lookup_obj = self._lookup_m.maintenance_lookup( maintenance_lookup_target) lookup_queries_to_send = lookup_obj.start(bootstrap_nodes) self._send_queries(lookup_queries_to_send) self._next_maintenance_ts = (current_time + maintenance_delay) # Auto-save routing table if current_time > self._next_save_state_ts: self.save_state() self._next_save_state_ts = current_time + SAVE_STATE_DELAY # Schedule next call delay = (min(self._next_maintenance_ts, self._next_save_state_ts) - current_time) self._reactor.call_later(delay, self._main_loop) def _maintenance_lookup(self, target): self._lookup_m.maintenance_lookup(target) def _on_datagram_received(self, data, addr): try: msg = message.IncomingMsg(data, addr) except(message.MsgError): return # ignore message if msg.type == message.QUERY: response_msg = self._get_response(msg) if response_msg: bencoded_response = response_msg.encode(msg.tid) self._reactor.sendto(bencoded_response, addr) maintenance_queries_to_send = self._routing_m.on_query_received( msg.sender_node) elif msg.type in (message.RESPONSE, message.ERROR): related_query = self._querier.on_response_received(msg, addr) if not related_query: # Query timed out or unrequested response return # lookup related tasks if related_query.lookup_obj: if msg.type == message.RESPONSE: (lookup_queries_to_send, peers, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_response_received( msg, msg.sender_node) else: #ERROR peers = None # an error msg doesn't have peers (lookup_queries_to_send, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_error_received( msg, msg.sender_node) self._send_queries(lookup_queries_to_send) if related_query.lookup_obj.callback_f: lookup_id = related_query.lookup_obj.lookup_id if peers: related_query.lookup_obj.callback_f(lookup_id, peers) if lookup_done: related_query.lookup_obj.callback_f(lookup_id, None) # maintenance related tasks if msg.type == message.RESPONSE: maintenance_queries_to_send = \ self._routing_m.on_response_received( msg.sender_node, related_query.rtt, msg.all_nodes) else: maintenance_queries_to_send = \ self._routing_m.on_error_received( msg.sender_node) else: # unknown type return self._send_queries(maintenance_queries_to_send) def _get_response(self, msg): if msg.query == message.PING: return message.OutgoingPingResponse(self._my_id) elif msg.query == message.FIND_NODE: log_distance = msg.target.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes(log_distance, NUM_NODES, False) return message.OutgoingFindNodeResponse(self._my_id, rnodes) elif msg.query == message.GET_PEERS: token = self._token_m.get() log_distance = msg.info_hash.log_distance(self._my_id) rnodes = self._routing_m.get_closest_rnodes(log_distance, NUM_NODES, False) peers = self._tracker.get(msg.info_hash) if peers: logger.debug('RESPONDING with PEERS:\n%r' % peers) return message.OutgoingGetPeersResponse(self._my_id, token, nodes=rnodes, peers=peers) elif msg.query == message.ANNOUNCE_PEER: peer_addr = (msg.sender_addr[0], msg.bt_port) self._tracker.put(msg.info_hash, peer_addr) return message.OutgoingAnnouncePeerResponse(self._my_id) else: logger.debug('Invalid QUERY: %r' % (msg.query)) #TODO: maybe send an error back? def _on_response_received(self, msg): pass def _on_timeout(self, addr): related_query = self._querier.on_timeout(addr) if not related_query: return # timeout cancelled (got response/error already) if related_query.lookup_obj: (lookup_queries_to_send, num_parallel_queries, lookup_done ) = related_query.lookup_obj.on_timeout(related_query.dstnode) self._send_queries(lookup_queries_to_send) if lookup_done and related_query.lookup_obj.callback_f: lookup_id = related_query.lookup_obj.lookup_id related_query.lookup_obj.callback_f(lookup_id, None) maintenance_queries_to_send = self._routing_m.on_timeout( related_query.dstnode) self._send_queries(maintenance_queries_to_send) def _send_queries(self, queries_to_send, lookup_obj=None): if queries_to_send is None: return for query in queries_to_send: timeout_task = self._reactor.call_later(TIMEOUT_DELAY, self._on_timeout, query.dstnode.addr) bencoded_query = self._querier.register_query(query, timeout_task) self._reactor.sendto(bencoded_query, query.dstnode.addr)
class Controller: def __init__(self, version_label, my_node, state_filename, routing_m_mod, lookup_m_mod, experimental_m_mod, private_dht_name, bootstrap_mode): if size_estimation: self._size_estimation_file = open('size_estimation.dat', 'w') self.state_filename = state_filename saved_id, saved_bootstrap_nodes = state.load(self.state_filename) my_addr = my_node.addr self._my_id = my_node.id # id indicated by user if not self._my_id: self._my_id = saved_id # id loaded from file if not self._my_id: self._my_id = self._my_id = identifier.RandomId() # random id self._my_node = Node(my_addr, self._my_id, version=version_label) self.msg_f = message.MsgFactory(version_label, self._my_id, private_dht_name) self._querier = Querier() self._routing_m = routing_m_mod.RoutingManager(self._my_node, saved_bootstrap_nodes, self.msg_f) self._responder = responder.Responder(self._my_id, self._routing_m, self.msg_f, bootstrap_mode) self._tracker = self._responder._tracker self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f) self._experimental_m = experimental_m_mod.ExperimentalManager( self._my_node.id, self.msg_f) current_ts = time.time() self._next_save_state_ts = current_ts + SAVE_STATE_DELAY self._next_maintenance_ts = current_ts self._next_timeout_ts = current_ts self._next_main_loop_call_ts = current_ts self._pending_lookups = [] self._cached_lookups = {} def on_stop(self): self._experimental_m.on_stop() def get_peers(self, lookup_id, info_hash, callback_f, bt_port, use_cache): """ Start a get\_peers lookup whose target is 'info\_hash'. The handler 'callback\_f' will be called with two arguments ('lookup\_id' and a 'peer list') whenever peers are discovered. Once the lookup is completed, the handler will be called with 'lookup\_id' and None as arguments. This method is designed to be used as minitwisted's external handler. """ datagrams_to_send = [] logger.debug('get_peers %d %r' % (bt_port, info_hash)) if use_cache: peers = self._get_cached_peers(info_hash) if peers and callback_f and callable(callback_f): callback_f(lookup_id, peers, None) callback_f(lookup_id, None, None) return datagrams_to_send self._pending_lookups.append( self._lookup_m.get_peers(lookup_id, info_hash, callback_f, bt_port)) queries_to_send = self._try_do_lookup() datagrams_to_send = self._register_queries(queries_to_send) return datagrams_to_send def _clean_peer_caches(self): oldest_valid_ts = time.time() - CACHE_VALID_PERIOD for key, values in self._cached_lookups.items(): ts, _ = values if ts < oldest_valid_ts: del self._cached_lookups[key] def _get_cached_peers(self, info_hash): self._clean_peer_caches() if info_hash in self._cached_lookups: return self._cached_lookups[info_hash][1] def _add_cache_peers(self, info_hash, peers): self._clean_peer_caches() if info_hash not in self._cached_lookups: self._cached_lookups[info_hash] = (time.time(), []) self._cached_lookups[info_hash][1].extend(peers) def _try_do_lookup(self): queries_to_send = [] current_time = time.time() while self._pending_lookups: pending_lookup = self._pending_lookups[0] # Drop all pending lookups older than PENDING_LOOKUP_TIMEOUT if time.time() > pending_lookup.start_ts + PENDING_LOOKUP_TIMEOUT: del self._pending_lookups[0] else: break if self._pending_lookups: lookup_obj = self._pending_lookups[0] else: return queries_to_send distance = lookup_obj.info_hash.distance(self._my_id) bootstrap_rnodes = self._routing_m.get_closest_rnodes( distance.log, 0, True) # TODO: get the full bucket if bootstrap_rnodes: del self._pending_lookups[0] # look if I'm tracking this info_hash peers = self._tracker.get(lookup_obj.info_hash) callback_f = lookup_obj.callback_f if peers: self._add_cache_peers(lookup_obj.info_hash, peers) if callback_f and callable(callback_f): callback_f(lookup_obj.lookup_id, peers, None) # do the lookup queries_to_send = lookup_obj.start(bootstrap_rnodes) else: next_lookup_attempt_ts = time.time() + .2 self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, next_lookup_attempt_ts) return queries_to_send def print_routing_table_stats(self): self._routing_m.print_stats() def main_loop(self): """ Perform maintenance operations. The main operation is routing table maintenance where staled nodes are added/probed/replaced/removed as needed. The routing management module specifies the implementation details. This includes keeping track of queries that have not been responded for a long time (timeout) with the help of querier.Querier. The routing manager and the lookup manager will be informed of those timeouts. This method is designed to be used as minitwisted's heartbeat handler. """ if prctlimported: prctl.set_name("Tribler" + currentThread().getName()) queries_to_send = [] current_ts = time.time() # TODO: I think this if should be removed # At most, 1 second between calls to main_loop after the first call if current_ts >= self._next_main_loop_call_ts: self._next_main_loop_call_ts = current_ts + 1 else: # It's too early return self._next_main_loop_call_ts, [] # Retry failed lookup (if any) queries_to_send.extend(self._try_do_lookup()) # Take care of timeouts if current_ts >= self._next_timeout_ts: (self._next_timeout_ts, timeout_queries) = self._querier.get_timeout_queries() for query in timeout_queries: queries_to_send.extend(self._on_timeout(query)) # Routing table maintenance if time.time() >= self._next_maintenance_ts: (maintenance_delay, queries, maintenance_lookup) = self._routing_m.do_maintenance() self._next_maintenance_ts = current_ts + maintenance_delay self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, self._next_maintenance_ts) queries_to_send.extend(queries) if maintenance_lookup: target, rnodes = maintenance_lookup lookup_obj = self._lookup_m.maintenance_lookup(target) queries_to_send.extend(lookup_obj.start(rnodes)) # Auto-save routing table if current_ts >= self._next_save_state_ts: state.save(self._my_id, self._routing_m.get_main_rnodes(), self.state_filename) self._next_save_state_ts = current_ts + SAVE_STATE_DELAY self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, self._next_maintenance_ts, self._next_timeout_ts, self._next_save_state_ts) # Return control to reactor datagrams_to_send = self._register_queries(queries_to_send) return self._next_main_loop_call_ts, datagrams_to_send def _maintenance_lookup(self, target): self._lookup_m.maintenance_lookup(target) def on_datagram_received(self, datagram): """ Perform the actions associated to the arrival of the given datagram. The datagram will be ignored in cases such as invalid format. Otherwise, the datagram will be decoded and different modules will be informed to take action on it. For instance, if the datagram contains a response to a lookup query, both routing and lookup manager will be informed. Additionally, if that response contains peers, the lookup's handler will be called (see get\_peers above). This method is designed to be used as minitwisted's networking handler. """ exp_queries_to_send = [] data = datagram.data addr = datagram.addr datagrams_to_send = [] try: msg = self.msg_f.incoming_msg(datagram) except (message.MsgError): # ignore message return self._next_main_loop_call_ts, datagrams_to_send if msg.type == message.QUERY: if msg.src_node.id == self._my_id: logger.debug('Got a msg from myself:\n%r', msg) return self._next_main_loop_call_ts, datagrams_to_send # zinat: inform experimental_module exp_queries_to_send = self._experimental_m.on_query_received(msg) response_msg = self._responder.get_response(msg) if response_msg: bencoded_response = response_msg.stamp(msg.tid) datagrams_to_send.append( message.Datagram(bencoded_response, addr)) maintenance_queries_to_send = self._routing_m.on_query_received( msg.src_node) elif msg.type == message.RESPONSE: related_query = self._querier.get_related_query(msg) if not related_query: # Query timed out or unrequested response return self._next_main_loop_call_ts, datagrams_to_send # zinat: if related_query.experimental_obj: exp_queries_to_send = self._experimental_m.on_response_received( msg, related_query) # TODO: you need to get datagrams to be able to send messages (raul) # lookup related tasks if related_query.lookup_obj: (lookup_queries_to_send, peers, num_parallel_queries, lookup_done) = related_query.lookup_obj.on_response_received( msg, msg.src_node) datagrams = self._register_queries(lookup_queries_to_send) datagrams_to_send.extend(datagrams) lookup_obj = related_query.lookup_obj lookup_id = lookup_obj.lookup_id callback_f = lookup_obj.callback_f if peers: self._add_cache_peers(lookup_obj.info_hash, peers) if callback_f and callable(callback_f): callback_f(lookup_id, peers, msg.src_node) # Size estimation if size_estimation and lookup_done: line = '%d %d\n' % (related_query.lookup_obj. get_number_nodes_within_region()) self._size_estimation_file.write(line) self._size_estimation_file.flush() if lookup_done: if callback_f and callable(callback_f): callback_f(lookup_id, None, msg.src_node) queries_to_send = self._announce(related_query.lookup_obj) datagrams = self._register_queries(queries_to_send) datagrams_to_send.extend(datagrams) # maintenance related tasks maintenance_queries_to_send = \ self._routing_m.on_response_received( msg.src_node, related_query.rtt, msg.all_nodes) elif msg.type == message.ERROR: related_query = self._querier.get_related_query(msg) if not related_query: # Query timed out or unrequested response return self._next_main_loop_call_ts, datagrams_to_send # TODO: zinat: same as response exp_queries_to_send = self._experimental_m.on_error_received( msg, related_query) # lookup related tasks if related_query.lookup_obj: peers = None # an error msg doesn't have peers (lookup_queries_to_send, num_parallel_queries, lookup_done) = related_query.lookup_obj.on_error_received( msg, addr) datagrams = self._register_queries(lookup_queries_to_send) datagrams_to_send.extend(datagrams) callback_f = related_query.lookup_obj.callback_f if callback_f and callable(callback_f): lookup_id = related_query.lookup_obj.lookup_id if lookup_done: callback_f(lookup_id, None, msg.src_node) if lookup_done: # Size estimation if size_estimation: line = '%d %d\n' % (related_query.lookup_obj. get_number_nodes_within_region()) self._size_estimation_file.write(line) self._size_estimation_file.flush() datagrams = self._announce(related_query.lookup_obj) datagrams_to_send.extend(datagrams) # maintenance related tasks maintenance_queries_to_send = \ self._routing_m.on_error_received(addr) else: # unknown type return self._next_main_loop_call_ts, datagrams_to_send # we are done with the plugins # now we have maintenance_queries_to_send, let's send them! datagrams = self._register_queries(maintenance_queries_to_send) datagrams_to_send.extend(datagrams) if exp_queries_to_send: datagrams = self._register_queries(exp_queries_to_send) datagrams_to_send.extend(datagrams) return self._next_main_loop_call_ts, datagrams_to_send def _on_query_received(self): return def _on_response_received(self): return def _on_error_received(self): return def _on_timeout(self, related_query): queries_to_send = [] # TODO: on_timeout should return queries (raul) exp_queries_to_send = self._experimental_m.on_timeout(related_query) if related_query.lookup_obj: (lookup_queries_to_send, num_parallel_queries, lookup_done) = related_query.lookup_obj.on_timeout( related_query.dst_node) queries_to_send.extend(lookup_queries_to_send) callback_f = related_query.lookup_obj.callback_f if lookup_done: # Size estimation if size_estimation: line = '%d %d\n' % (related_query.lookup_obj. get_number_nodes_within_region()) self._size_estimation_file.write(line) self._size_estimation_file.flush() lookup_id = related_query.lookup_obj.lookup_id if callback_f and callable(callback_f): related_query.lookup_obj.callback_f(lookup_id, None, None) queries_to_send.extend(self._announce( related_query.lookup_obj)) maintenance_queries_to_send = self._routing_m.on_timeout( related_query.dst_node) if maintenance_queries_to_send: queries_to_send.extend(maintenance_queries_to_send) if exp_queries_to_send: datagrams = self._register_queries(exp_queries_to_send) queries_to_send.extend(datagrams) return queries_to_send def _announce(self, lookup_obj): queries_to_send, announce_to_myself = lookup_obj.announce() return queries_to_send ''' if announce_to_myself: self._tracker.put(lookup_obj._info_hash, (self._my_node.addr[0], lookup_obj._bt_port)) ''' def _register_queries(self, queries_to_send, lookup_obj=None): if not queries_to_send: return [] timeout_call_ts, datagrams_to_send = self._querier.register_queries( queries_to_send) self._next_main_loop_call_ts = min(self._next_main_loop_call_ts, timeout_call_ts) return datagrams_to_send