def start(self): from twisted.internet.defer import Deferred, succeed from logs import lg from dht import dht_records from dht import dht_service from dht import known_nodes from main import settings from main.config import conf conf().addConfigNotifier('services/entangled-dht/udp-port', self._on_udp_port_modified) known_seeds = known_nodes.nodes() dht_layers = list(dht_records.LAYERS_REGISTRY.keys()) dht_service.init( udp_port=settings.getDHTPort(), dht_dir_path=settings.ServiceDir('service_entangled_dht'), open_layers=dht_layers, ) lg.info('DHT known seed nodes are : %r DHT layers are : %r' % (known_seeds, dht_layers, )) self.starting_deferred = Deferred() d = dht_service.connect( seed_nodes=known_seeds, layer_id=0, attach=True, ) d.addCallback(self._on_connected) d.addErrback(self._on_connect_failed) return self.starting_deferred or succeed(True)
def _on_connected(self, ok): from twisted.internet.defer import DeferredList from logs import lg from dht import dht_service from dht import known_nodes from main.config import conf lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok)) dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store) dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request) known_seeds = known_nodes.nodes() dl = [] attached_layers = conf().getData('services/entangled-dht/attached-layers', default='') if attached_layers: lg.info('more DHT layers to be attached: %r' % attached_layers) for layer_id in attached_layers.split(','): if layer_id.strip(): dl.append(dht_service.open_layer( layer_id=int(layer_id.strip()), seed_nodes=known_seeds, connect_now=True, attach=True, )) if dl: d = DeferredList(dl) d.addCallback(self._on_layers_attached) d.addErrback(self._on_connect_failed) else: if self.starting_deferred and not self.starting_deferred.called: self.starting_deferred.callback(True) return ok
def connect(): result = Deferred() if not node().listener: node().listenUDP() if node().refresher and node().refresher.active(): node().refresher.reset(0) if _Debug: lg.out( _DebugLevel, 'dht_service.connect DHT already active : SKIP but RESET refresher task' ) result.callback(True) return result def _on_join_success(ok): if _Debug: lg.out( _DebugLevel, 'dht_service.connect DHT JOIN SUCCESS !!!!!!!!!!!!!!!!!!!!!!!') result.callback(True) def _on_join_failed(x): if _Debug: lg.out(_DebugLevel, 'dht_service.connect DHT JOIN FAILED : %s' % x) result.callback(False) def _on_hosts_resolved(live_nodes): if _Debug: lg.out( _DebugLevel, 'dht_service.connect RESOLVED %d live nodes' % (len(live_nodes))) for onenode in live_nodes: lg.out(_DebugLevel, ' %s:%s' % onenode) node().joinNetwork(live_nodes) node()._joinDeferred.addCallback(_on_join_success) node()._joinDeferred.addErrback(_on_join_failed) return live_nodes def _on_hosts_resolve_failed(x): if _Debug: lg.out(_DebugLevel, 'dht_service.connect ERROR : hosts not resolved: %s' % x) result.callback(False) return x _known_nodes = known_nodes.nodes() if _Debug: lg.out( _DebugLevel, 'dht_service.connect STARTING with %d known nodes:' % (len(_known_nodes))) for onenode in _known_nodes: lg.out(_DebugLevel, ' %s:%s' % onenode) d = resolve_hosts(_known_nodes) d.addCallback(_on_hosts_resolved) d.addErrback(_on_hosts_resolve_failed) return result
def _do_connect_proxy_routers_dht_layer(self): from logs import lg from dht import dht_service from dht import dht_records from dht import known_nodes known_seeds = known_nodes.nodes() d = dht_service.connect( seed_nodes=known_seeds, layer_id=dht_records.LAYER_PROXY_ROUTERS, attach=True, ) d.addCallback(self._on_dht_proxy_routers_layer_connected) d.addErrback(lambda *args: lg.err(str(args)))
def _do_join_message_brokers_dht_layer(self): from logs import lg from dht import dht_service from dht import dht_records from dht import known_nodes lg.info('going to join message brokers DHT layer: %d' % dht_records.LAYER_MESSAGE_BROKERS) known_seeds = known_nodes.nodes() dht_service.open_layer( seed_nodes=known_seeds, layer_id=dht_records.LAYER_MESSAGE_BROKERS, connect_now=True, attach=False, )
def _do_join_suppliers_dht_layer(self): from logs import lg from dht import dht_service from dht import dht_records from dht import known_nodes lg.info('going to join suppliers DHT layer: %d' % dht_records.LAYER_SUPPLIERS) known_seeds = known_nodes.nodes() dht_service.open_layer( seed_nodes=known_seeds, layer_id=dht_records.LAYER_SUPPLIERS, connect_now=True, attach=False, )
def _do_connect_suppliers_dht_layer(self): from logs import lg from dht import dht_service from dht import dht_records from dht import known_nodes known_seeds = known_nodes.nodes() d = dht_service.open_layer( layer_id=dht_records.LAYER_SUPPLIERS, seed_nodes=known_seeds, connect_now=True, attach=True, ) d.addCallback(self._on_suppliers_dht_layer_connected) d.addErrback(lambda *args: lg.err(str(args)))
def _do_join_proxy_routers_dht_layer(self): from logs import lg from dht import dht_service from dht import dht_records from dht import known_nodes lg.info('going to join proxy routers DHT layer: %d' % dht_records.LAYER_PROXY_ROUTERS) known_seeds = known_nodes.nodes() d = dht_service.open_layer( layer_id=dht_records.LAYER_PROXY_ROUTERS, seed_nodes=known_seeds, connect_now=True, attach=False, ) d.addCallback(self._on_proxy_routers_dht_layer_connected) d.addErrback(lambda *args: lg.err(str(args)))
def start(self): from logs import lg from dht import dht_service from dht import known_nodes from main import settings from main.config import conf conf().addCallback('services/entangled-dht/udp-port', self._on_udp_port_modified) dht_service.init(udp_port=settings.getDHTPort(), db_file_path=settings.DHTDBFile()) known_seeds = known_nodes.nodes() lg.info('known seed nodes are : %r' % known_seeds) d = dht_service.connect(seed_nodes=known_seeds) d.addCallback(self._on_connected) d.addErrback(self._on_connect_failed) return d
def _on_connected(self, ok): from twisted.internet.defer import DeferredList from logs import lg from dht import dht_service from dht import known_nodes from main.config import conf from services import driver lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok)) dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store) dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request) known_seeds = known_nodes.nodes() dl = [] attached_layers = conf().getData('services/entangled-dht/attached-layers', default='') if attached_layers: attached_layers = list(filter(None, map(lambda v: int(str(v).strip()), attached_layers.split(',')))) else: attached_layers = [] lg.info('reading attached DHT layers from configuration: %r' % attached_layers) all_services_attached_layers = driver.get_attached_dht_layers().values() combined_services_attached_layers = set() count_combined = len(list(map(combined_services_attached_layers.update, all_services_attached_layers))) services_attached_layers = list(combined_services_attached_layers) lg.info('combined attached DHT layers from %d services: %r' % (count_combined, services_attached_layers, )) attached_layers = list(set(attached_layers + services_attached_layers)) lg.info('DHT layers to be attached at startup: %r' % attached_layers) for layer_id in attached_layers: dl.append(dht_service.open_layer( layer_id=layer_id, seed_nodes=known_seeds, connect_now=True, attach=True, )) if dl: d = DeferredList(dl) d.addCallback(self._on_layers_attached) d.addErrback(self._on_connect_failed) else: if self.starting_deferred and not self.starting_deferred.called: self.starting_deferred.callback(True) return ok
def connect(): if not node().listener: node().listenUDP() if node().refresher and node().refresher.active(): node().refresher.reset(0) if _Debug: lg.out(_DebugLevel, 'dht_service.connect did RESET refresher task') return True def _on_hosts_resolved(live_nodes): node().joinNetwork(live_nodes) if _Debug: lg.out( _DebugLevel, 'dht_service.connect RESOLVED %d live nodes' % (len(live_nodes))) _known_nodes = known_nodes.nodes() if _Debug: lg.out( _DebugLevel, 'dht_service.connect STARTING with %d known nodes' % (len(_known_nodes))) resolve_hosts(_known_nodes).addBoth(_on_hosts_resolved) return True
def connect(seed_nodes=[]): result = Deferred() if node().refresher and node().refresher.active(): node().refresher.reset(0) if _Debug: lg.out( _DebugLevel, 'dht_service.connect DHT already active : SKIP but RESET refresher task' ) result.callback(True) return result if not seed_nodes: from dht import known_nodes seed_nodes = known_nodes.nodes() if not node().listener: node().listenUDP() if _Debug: lg.out( _DebugLevel, 'dht_service.connect opened a new listener : %r' % node().listener) if _Debug: lg.out( _DebugLevel, 'dht_service.connect STARTING with %d known nodes:' % (len(seed_nodes))) for onenode in seed_nodes: lg.out(_DebugLevel, ' %s:%s' % onenode) def _on_join_success(live_contacts, resolved_seed_nodes): if _Debug: if isinstance(live_contacts, dict): lg.warn('Unexpected result from joinNetwork: %s' % pprint.pformat(live_contacts)) else: if len(live_contacts) > 0 and live_contacts[0]: lg.out( _DebugLevel, 'dht_service.connect DHT JOIN SUCCESS !!!!!!!!!!!!!!!!!!!!!!!' ) else: lg.out( _DebugLevel, 'dht_service.connect DHT JOINED, but still OFFLINE !!!!!!!!!!' ) lg.warn( 'No live DHT contacts found... your node is NOT CONNECTED TO DHT NETWORK' ) lg.out(_DebugLevel, 'alive DHT nodes: %s' % pprint.pformat(live_contacts)) lg.out(_DebugLevel, 'resolved SEED nodes: %r' % resolved_seed_nodes) lg.out(_DebugLevel, 'DHT node is active, ID=[%s]' % base64.b64encode(node().id)) result.callback(resolved_seed_nodes) return live_contacts def _on_join_failed(x): if _Debug: lg.out(_DebugLevel, 'dht_service.connect DHT JOIN FAILED : %s' % x) result.callback(False) return None def _on_hosts_resolved(resolved_seed_nodes): if _Debug: lg.out( _DebugLevel, 'dht_service.connect RESOLVED %d live nodes' % (len(resolved_seed_nodes))) for onenode in resolved_seed_nodes: lg.out(_DebugLevel, ' %s:%s' % onenode) d = node().joinNetwork(resolved_seed_nodes) d.addCallback(_on_join_success, resolved_seed_nodes) d.addErrback(_on_join_failed) node().expire_task.start(int(KEY_EXPIRE_MIN_SECONDS / 2), now=True) return resolved_seed_nodes def _on_hosts_resolve_failed(x): if _Debug: lg.out(_DebugLevel, 'dht_service.connect ERROR : hosts not resolved: %s' % x) result.callback(False) return x d = resolve_hosts(seed_nodes) d.addCallback(_on_hosts_resolved) d.addErrback(_on_hosts_resolve_failed) return result
def main(options=None, args=None): from dht import dht_relations if options is None and args is None: (options, args) = parseCommandLine() else: (_options, _args) = parseCommandLine() if options is None: options = _options if args is None: args = _args init(udp_port=options.udpport, db_file_path=options.dhtdb) lg.out(_DebugLevel, 'Init udpport=%d dhtdb=%s node=%r' % (options.udpport, options.dhtdb, node())) def _go(nodes): # lg.out(_DebugLevel, 'Connected nodes: %r' % nodes) # lg.out(_DebugLevel, 'DHT node is active, ID=[%s]' % base64.b64encode(node().id)) try: if len(args) == 0: pass elif len(args) > 0: def _r(x): lg.info(x) # reactor.stop() #@UndefinedVariable cmd = args[0] if cmd == 'get': get_value(args[1]).addBoth(_r) elif cmd == 'set': set_value(args[1], args[2], expire=int(args[3])).addBoth(_r) elif cmd == 'get_json': get_json_value(args[1]).addBoth(_r) elif cmd == 'set_json': set_json_value(args[1], jsn.loads(args[2]), expire=(int(args[3]) if len(args)>=4 else 9999)).addBoth(_r) elif cmd == 'get_valid_data': get_valid_data(args[1], rules=jsn.loads(args[2]), return_details=True).addBoth(_r) elif cmd == 'set_valid_data': set_valid_data(args[1], jsn.loads(args[2]), expire=(int(args[3]) if len(args)>=4 else 9999), rules=jsn.loads(args[4])).addBoth(_r) elif cmd == 'read_customer_suppliers': dht_relations.read_customer_suppliers(args[1]).addBoth(_r) elif cmd == 'write_customer_suppliers': dht_relations.write_customer_suppliers(args[1], args[2].split(',')).addBoth(_r) elif cmd == 'write_verify_republish': write_verify_republish_data(args[1], args[2], expire=int(args[3])).addBoth(_r) elif cmd == 'find': find_node(key_to_hash(args[1])).addBoth(_r) elif cmd == 'ping': find_node(random_key()).addBoth(_r) elif cmd == 'get_node_data': pprint.pprint(get_node_data(args[1])) elif cmd == 'observe_data': def _p(val, n): print('observed', n, val) def _o(result): for n in result: d = n.request(args[2]) d.addCallback(_p, n) d = find_node(key_to_hash(args[1])) d.addErrback(_r) d.addCallback(_o) elif cmd == 'discover': def _l(x): lg.info(x) find_node(random_key()).addBoth(_l) _l('') elif cmd == 'dump_db': pprint.pprint(dump_local_db(value_as_json=True)) except: lg.exc() seeds = [] if options.seeds in ['genesis', 'root', b'genesis', b'root', ]: lg.out(_DebugLevel, 'Starting genesis node!!!!!!!!!!!!!!!!!!!!') else: for dht_node_str in options.seeds.split(','): if dht_node_str.strip(): try: dht_node = dht_node_str.strip().split(':') dht_node_host = dht_node[0].strip() dht_node_port = int(dht_node[1].strip()) except: continue seeds.append((dht_node_host, dht_node_port, )) if not seeds: from dht import known_nodes seeds = known_nodes.nodes() lg.out(_DebugLevel, 'Seed nodes: %s' % seeds) if options.delayed: lg.out(_DebugLevel, 'Wait %d seconds before join the network' % options.delayed) import time time.sleep(options.delayed) connect(seeds).addBoth(_go) reactor.run() #@UndefinedVariable