def start(self):
from twisted.internet.defer import Deferred, succeed
from logs import lg
from dht import dht_records
from dht import dht_service
from dht import known_nodes
from main import settings
from main.config import conf
conf().addConfigNotifier('services/entangled-dht/udp-port', self._on_udp_port_modified)
known_seeds = known_nodes.nodes()
dht_layers = list(dht_records.LAYERS_REGISTRY.keys())
dht_service.init(
udp_port=settings.getDHTPort(),
dht_dir_path=settings.ServiceDir('service_entangled_dht'),
open_layers=dht_layers,
)
lg.info('DHT known seed nodes are : %r DHT layers are : %r' % (known_seeds, dht_layers, ))
self.starting_deferred = Deferred()
d = dht_service.connect(
seed_nodes=known_seeds,
layer_id=0,
attach=True,
)
d.addCallback(self._on_connected)
d.addErrback(self._on_connect_failed)
return self.starting_deferred or succeed(True)
def _on_connected(self, ok):
from twisted.internet.defer import DeferredList
from logs import lg
from dht import dht_service
from dht import known_nodes
from main.config import conf
lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok))
dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store)
dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request)
known_seeds = known_nodes.nodes()
dl = []
attached_layers = conf().getData('services/entangled-dht/attached-layers', default='')
if attached_layers:
lg.info('more DHT layers to be attached: %r' % attached_layers)
for layer_id in attached_layers.split(','):
if layer_id.strip():
dl.append(dht_service.open_layer(
layer_id=int(layer_id.strip()),
seed_nodes=known_seeds,
connect_now=True,
attach=True,
))
if dl:
d = DeferredList(dl)
d.addCallback(self._on_layers_attached)
d.addErrback(self._on_connect_failed)
else:
if self.starting_deferred and not self.starting_deferred.called:
self.starting_deferred.callback(True)
return ok
def connect():
result = Deferred()
if not node().listener:
node().listenUDP()
if node().refresher and node().refresher.active():
node().refresher.reset(0)
if _Debug:
lg.out(
_DebugLevel,
'dht_service.connect DHT already active : SKIP but RESET refresher task'
)
result.callback(True)
return result
def _on_join_success(ok):
if _Debug:
lg.out(
_DebugLevel,
'dht_service.connect DHT JOIN SUCCESS !!!!!!!!!!!!!!!!!!!!!!!')
result.callback(True)
def _on_join_failed(x):
if _Debug:
lg.out(_DebugLevel, 'dht_service.connect DHT JOIN FAILED : %s' % x)
result.callback(False)
def _on_hosts_resolved(live_nodes):
if _Debug:
lg.out(
_DebugLevel, 'dht_service.connect RESOLVED %d live nodes' %
(len(live_nodes)))
for onenode in live_nodes:
lg.out(_DebugLevel, ' %s:%s' % onenode)
node().joinNetwork(live_nodes)
node()._joinDeferred.addCallback(_on_join_success)
node()._joinDeferred.addErrback(_on_join_failed)
return live_nodes
def _on_hosts_resolve_failed(x):
if _Debug:
lg.out(_DebugLevel,
'dht_service.connect ERROR : hosts not resolved: %s' % x)
result.callback(False)
return x
_known_nodes = known_nodes.nodes()
if _Debug:
lg.out(
_DebugLevel, 'dht_service.connect STARTING with %d known nodes:' %
(len(_known_nodes)))
for onenode in _known_nodes:
lg.out(_DebugLevel, ' %s:%s' % onenode)
d = resolve_hosts(_known_nodes)
d.addCallback(_on_hosts_resolved)
d.addErrback(_on_hosts_resolve_failed)
return result
def _do_connect_proxy_routers_dht_layer(self):
from logs import lg
from dht import dht_service
from dht import dht_records
from dht import known_nodes
known_seeds = known_nodes.nodes()
d = dht_service.connect(
seed_nodes=known_seeds,
layer_id=dht_records.LAYER_PROXY_ROUTERS,
attach=True,
)
d.addCallback(self._on_dht_proxy_routers_layer_connected)
d.addErrback(lambda *args: lg.err(str(args)))
def _do_join_message_brokers_dht_layer(self):
from logs import lg
from dht import dht_service
from dht import dht_records
from dht import known_nodes
lg.info('going to join message brokers DHT layer: %d' % dht_records.LAYER_MESSAGE_BROKERS)
known_seeds = known_nodes.nodes()
dht_service.open_layer(
seed_nodes=known_seeds,
layer_id=dht_records.LAYER_MESSAGE_BROKERS,
connect_now=True,
attach=False,
)
def _do_join_suppliers_dht_layer(self):
from logs import lg
from dht import dht_service
from dht import dht_records
from dht import known_nodes
lg.info('going to join suppliers DHT layer: %d' %
dht_records.LAYER_SUPPLIERS)
known_seeds = known_nodes.nodes()
dht_service.open_layer(
seed_nodes=known_seeds,
layer_id=dht_records.LAYER_SUPPLIERS,
connect_now=True,
attach=False,
)
def _do_connect_suppliers_dht_layer(self):
from logs import lg
from dht import dht_service
from dht import dht_records
from dht import known_nodes
known_seeds = known_nodes.nodes()
d = dht_service.open_layer(
layer_id=dht_records.LAYER_SUPPLIERS,
seed_nodes=known_seeds,
connect_now=True,
attach=True,
)
d.addCallback(self._on_suppliers_dht_layer_connected)
d.addErrback(lambda *args: lg.err(str(args)))
def _do_join_proxy_routers_dht_layer(self):
from logs import lg
from dht import dht_service
from dht import dht_records
from dht import known_nodes
lg.info('going to join proxy routers DHT layer: %d' % dht_records.LAYER_PROXY_ROUTERS)
known_seeds = known_nodes.nodes()
d = dht_service.open_layer(
layer_id=dht_records.LAYER_PROXY_ROUTERS,
seed_nodes=known_seeds,
connect_now=True,
attach=False,
)
d.addCallback(self._on_proxy_routers_dht_layer_connected)
d.addErrback(lambda *args: lg.err(str(args)))
def start(self):
from logs import lg
from dht import dht_service
from dht import known_nodes
from main import settings
from main.config import conf
conf().addCallback('services/entangled-dht/udp-port',
self._on_udp_port_modified)
dht_service.init(udp_port=settings.getDHTPort(),
db_file_path=settings.DHTDBFile())
known_seeds = known_nodes.nodes()
lg.info('known seed nodes are : %r' % known_seeds)
d = dht_service.connect(seed_nodes=known_seeds)
d.addCallback(self._on_connected)
d.addErrback(self._on_connect_failed)
return d
def _on_connected(self, ok):
from twisted.internet.defer import DeferredList
from logs import lg
from dht import dht_service
from dht import known_nodes
from main.config import conf
from services import driver
lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok))
dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store)
dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request)
known_seeds = known_nodes.nodes()
dl = []
attached_layers = conf().getData('services/entangled-dht/attached-layers', default='')
if attached_layers:
attached_layers = list(filter(None, map(lambda v: int(str(v).strip()), attached_layers.split(','))))
else:
attached_layers = []
lg.info('reading attached DHT layers from configuration: %r' % attached_layers)
all_services_attached_layers = driver.get_attached_dht_layers().values()
combined_services_attached_layers = set()
count_combined = len(list(map(combined_services_attached_layers.update, all_services_attached_layers)))
services_attached_layers = list(combined_services_attached_layers)
lg.info('combined attached DHT layers from %d services: %r' % (count_combined, services_attached_layers, ))
attached_layers = list(set(attached_layers + services_attached_layers))
lg.info('DHT layers to be attached at startup: %r' % attached_layers)
for layer_id in attached_layers:
dl.append(dht_service.open_layer(
layer_id=layer_id,
seed_nodes=known_seeds,
connect_now=True,
attach=True,
))
if dl:
d = DeferredList(dl)
d.addCallback(self._on_layers_attached)
d.addErrback(self._on_connect_failed)
else:
if self.starting_deferred and not self.starting_deferred.called:
self.starting_deferred.callback(True)
return ok
def connect():
if not node().listener:
node().listenUDP()
if node().refresher and node().refresher.active():
node().refresher.reset(0)
if _Debug:
lg.out(_DebugLevel, 'dht_service.connect did RESET refresher task')
return True
def _on_hosts_resolved(live_nodes):
node().joinNetwork(live_nodes)
if _Debug:
lg.out(
_DebugLevel, 'dht_service.connect RESOLVED %d live nodes' %
(len(live_nodes)))
_known_nodes = known_nodes.nodes()
if _Debug:
lg.out(
_DebugLevel, 'dht_service.connect STARTING with %d known nodes' %
(len(_known_nodes)))
resolve_hosts(_known_nodes).addBoth(_on_hosts_resolved)
return True
def connect(seed_nodes=[]):
result = Deferred()
if node().refresher and node().refresher.active():
node().refresher.reset(0)
if _Debug:
lg.out(
_DebugLevel,
'dht_service.connect DHT already active : SKIP but RESET refresher task'
)
result.callback(True)
return result
if not seed_nodes:
from dht import known_nodes
seed_nodes = known_nodes.nodes()
if not node().listener:
node().listenUDP()
if _Debug:
lg.out(
_DebugLevel, 'dht_service.connect opened a new listener : %r' %
node().listener)
if _Debug:
lg.out(
_DebugLevel, 'dht_service.connect STARTING with %d known nodes:' %
(len(seed_nodes)))
for onenode in seed_nodes:
lg.out(_DebugLevel, ' %s:%s' % onenode)
def _on_join_success(live_contacts, resolved_seed_nodes):
if _Debug:
if isinstance(live_contacts, dict):
lg.warn('Unexpected result from joinNetwork: %s' %
pprint.pformat(live_contacts))
else:
if len(live_contacts) > 0 and live_contacts[0]:
lg.out(
_DebugLevel,
'dht_service.connect DHT JOIN SUCCESS !!!!!!!!!!!!!!!!!!!!!!!'
)
else:
lg.out(
_DebugLevel,
'dht_service.connect DHT JOINED, but still OFFLINE !!!!!!!!!!'
)
lg.warn(
'No live DHT contacts found... your node is NOT CONNECTED TO DHT NETWORK'
)
lg.out(_DebugLevel,
'alive DHT nodes: %s' % pprint.pformat(live_contacts))
lg.out(_DebugLevel,
'resolved SEED nodes: %r' % resolved_seed_nodes)
lg.out(_DebugLevel,
'DHT node is active, ID=[%s]' % base64.b64encode(node().id))
result.callback(resolved_seed_nodes)
return live_contacts
def _on_join_failed(x):
if _Debug:
lg.out(_DebugLevel, 'dht_service.connect DHT JOIN FAILED : %s' % x)
result.callback(False)
return None
def _on_hosts_resolved(resolved_seed_nodes):
if _Debug:
lg.out(
_DebugLevel, 'dht_service.connect RESOLVED %d live nodes' %
(len(resolved_seed_nodes)))
for onenode in resolved_seed_nodes:
lg.out(_DebugLevel, ' %s:%s' % onenode)
d = node().joinNetwork(resolved_seed_nodes)
d.addCallback(_on_join_success, resolved_seed_nodes)
d.addErrback(_on_join_failed)
node().expire_task.start(int(KEY_EXPIRE_MIN_SECONDS / 2), now=True)
return resolved_seed_nodes
def _on_hosts_resolve_failed(x):
if _Debug:
lg.out(_DebugLevel,
'dht_service.connect ERROR : hosts not resolved: %s' % x)
result.callback(False)
return x
d = resolve_hosts(seed_nodes)
d.addCallback(_on_hosts_resolved)
d.addErrback(_on_hosts_resolve_failed)
return result
def main(options=None, args=None):
from dht import dht_relations
if options is None and args is None:
(options, args) = parseCommandLine()
else:
(_options, _args) = parseCommandLine()
if options is None:
options = _options
if args is None:
args = _args
init(udp_port=options.udpport, db_file_path=options.dhtdb)
lg.out(_DebugLevel, 'Init udpport=%d dhtdb=%s node=%r' % (options.udpport, options.dhtdb, node()))
def _go(nodes):
# lg.out(_DebugLevel, 'Connected nodes: %r' % nodes)
# lg.out(_DebugLevel, 'DHT node is active, ID=[%s]' % base64.b64encode(node().id))
try:
if len(args) == 0:
pass
elif len(args) > 0:
def _r(x):
lg.info(x)
# reactor.stop() #@UndefinedVariable
cmd = args[0]
if cmd == 'get':
get_value(args[1]).addBoth(_r)
elif cmd == 'set':
set_value(args[1], args[2], expire=int(args[3])).addBoth(_r)
elif cmd == 'get_json':
get_json_value(args[1]).addBoth(_r)
elif cmd == 'set_json':
set_json_value(args[1], jsn.loads(args[2]),
expire=(int(args[3]) if len(args)>=4 else 9999)).addBoth(_r)
elif cmd == 'get_valid_data':
get_valid_data(args[1], rules=jsn.loads(args[2]), return_details=True).addBoth(_r)
elif cmd == 'set_valid_data':
set_valid_data(args[1], jsn.loads(args[2]),
expire=(int(args[3]) if len(args)>=4 else 9999),
rules=jsn.loads(args[4])).addBoth(_r)
elif cmd == 'read_customer_suppliers':
dht_relations.read_customer_suppliers(args[1]).addBoth(_r)
elif cmd == 'write_customer_suppliers':
dht_relations.write_customer_suppliers(args[1], args[2].split(',')).addBoth(_r)
elif cmd == 'write_verify_republish':
write_verify_republish_data(args[1], args[2], expire=int(args[3])).addBoth(_r)
elif cmd == 'find':
find_node(key_to_hash(args[1])).addBoth(_r)
elif cmd == 'ping':
find_node(random_key()).addBoth(_r)
elif cmd == 'get_node_data':
pprint.pprint(get_node_data(args[1]))
elif cmd == 'observe_data':
def _p(val, n):
print('observed', n, val)
def _o(result):
for n in result:
d = n.request(args[2])
d.addCallback(_p, n)
d = find_node(key_to_hash(args[1]))
d.addErrback(_r)
d.addCallback(_o)
elif cmd == 'discover':
def _l(x):
lg.info(x)
find_node(random_key()).addBoth(_l)
_l('')
elif cmd == 'dump_db':
pprint.pprint(dump_local_db(value_as_json=True))
except:
lg.exc()
seeds = []
if options.seeds in ['genesis', 'root', b'genesis', b'root', ]:
lg.out(_DebugLevel, 'Starting genesis node!!!!!!!!!!!!!!!!!!!!')
else:
for dht_node_str in options.seeds.split(','):
if dht_node_str.strip():
try:
dht_node = dht_node_str.strip().split(':')
dht_node_host = dht_node[0].strip()
dht_node_port = int(dht_node[1].strip())
except:
continue
seeds.append((dht_node_host, dht_node_port, ))
if not seeds:
from dht import known_nodes
seeds = known_nodes.nodes()
lg.out(_DebugLevel, 'Seed nodes: %s' % seeds)
if options.delayed:
lg.out(_DebugLevel, 'Wait %d seconds before join the network' % options.delayed)
import time
time.sleep(options.delayed)
connect(seeds).addBoth(_go)
reactor.run() #@UndefinedVariable
