def _go(nodes):
# lg.out(_DebugLevel, 'Connected nodes: %r' % nodes)
# lg.out(_DebugLevel, 'DHT node is active, ID=[%s]' % base64.b64encode(node().id))
try:
if len(args) == 0:
pass
elif len(args) > 0:
def _r(x):
lg.info(x)
# reactor.stop() #@UndefinedVariable
cmd = args[0]
if cmd == 'get':
get_value(args[1]).addBoth(_r)
elif cmd == 'set':
set_value(args[1], args[2], expire=int(args[3])).addBoth(_r)
elif cmd == 'get_json':
get_json_value(args[1]).addBoth(_r)
elif cmd == 'set_json':
set_json_value(args[1], jsn.loads(args[2]),
expire=(int(args[3]) if len(args)>=4 else 9999)).addBoth(_r)
elif cmd == 'get_valid_data':
get_valid_data(args[1], rules=jsn.loads(args[2]), return_details=True).addBoth(_r)
elif cmd == 'set_valid_data':
set_valid_data(args[1], jsn.loads(args[2]),
expire=(int(args[3]) if len(args)>=4 else 9999),
rules=jsn.loads(args[4])).addBoth(_r)
elif cmd == 'read_customer_suppliers':
dht_relations.read_customer_suppliers(args[1]).addBoth(_r)
elif cmd == 'write_customer_suppliers':
dht_relations.write_customer_suppliers(args[1], args[2].split(',')).addBoth(_r)
elif cmd == 'write_verify_republish':
write_verify_republish_data(args[1], args[2], expire=int(args[3])).addBoth(_r)
elif cmd == 'find':
find_node(key_to_hash(args[1])).addBoth(_r)
elif cmd == 'ping':
find_node(random_key()).addBoth(_r)
elif cmd == 'get_node_data':
pprint.pprint(get_node_data(args[1]))
elif cmd == 'observe_data':
def _p(val, n):
print('observed', n, val)
def _o(result):
for n in result:
d = n.request(args[2])
d.addCallback(_p, n)
d = find_node(key_to_hash(args[1]))
d.addErrback(_r)
d.addCallback(_o)
elif cmd == 'discover':
def _l(x):
lg.info(x)
find_node(random_key()).addBoth(_l)
_l('')
elif cmd == 'dump_db':
pprint.pprint(dump_local_db(value_as_json=True))
except:
lg.exc()
def BytesToDict(inp,
encoding='latin1',
errors='strict',
keys_to_text=False,
values_to_text=False,
unpack_types=False):
"""
A smart way to extract input bytes into python dictionary object.
All input bytes will be decoded into text and then loaded via `json.loads()` method.
Finally every text key and value in result dict will be encoded back to bytes if `values_to_text` is False.
Smart feature `unpack_types` can be used to "extract" real types of keys and values from input bytes.
Can be used to extract dictionaries of mixed types - binary and text values.
"""
if not inp:
return {}
_t = strng.to_text(inp, encoding=encoding)
if values_to_text:
return jsn.loads_text(_t, encoding=encoding)
if unpack_types:
return jsn.unpack_dict(jsn.loads(_t, encoding=encoding),
encoding=encoding,
errors=errors)
if keys_to_text:
return jsn.dict_keys_to_text(jsn.loads(_t, encoding=encoding))
return jsn.loads(_t, encoding=encoding)
def _on_rotate_broker_connected(self, response_info, broker_pos, event,
*args, **kwargs):
try:
# skip leading "accepted:" marker
cooperated_brokers = jsn.loads(
strng.to_text(response_info[0].Payload)[9:])
cooperated_brokers.pop('archive_folder_path', None)
cooperated_brokers = {
int(k): id_url.field(v)
for k, v in cooperated_brokers.items()
}
except:
lg.exc()
self.automat('broker-rotate-failed')
return
if _Debug:
lg.args(_DebugLevel,
cooperated=cooperated_brokers,
pos=broker_pos,
e=event)
if id_url.is_the_same(cooperated_brokers.get(broker_pos),
self.my_broker_idurl):
self.automat('broker-rotate-accepted',
cooperated_brokers=cooperated_brokers)
return
self.automat('broker-rotate-rejected',
cooperated_brokers=cooperated_brokers)
def ReadIndex(text_data, encoding='utf-8'):
"""
Read index data base, ``input`` is a ``StringIO.StringIO`` object which
keeps the data.
This is a simple text format, see ``p2p.backup_fs.Serialize()``
method. The first line keeps revision number.
"""
global _LoadingFlag
if _LoadingFlag:
return False
_LoadingFlag = True
backup_fs.Clear()
count = 0
try:
json_data = jsn.loads(
text_data,
encoding=encoding,
)
except:
lg.exc()
json_data = text_data
if _Debug:
lg.args(_DebugLevel, json_data=json_data)
for customer_id in json_data.keys():
if customer_id == 'items':
try:
count = backup_fs.Unserialize(json_data,
from_json=True,
decoding=encoding)
except:
lg.exc()
return False
else:
customer_idurl = global_id.GlobalUserToIDURL(customer_id)
if not id_url.is_cached(customer_idurl):
lg.warn(
'identity %r is not yet cached, skip reading related catalog items'
% customer_idurl)
identitycache.immediatelyCaching(customer_idurl,
try_other_sources=False,
ignore_errors=True)
continue
try:
count = backup_fs.Unserialize(
json_data[customer_id],
iter=backup_fs.fs(customer_idurl),
iterID=backup_fs.fsID(customer_idurl),
from_json=True,
decoding=encoding,
)
except:
lg.exc()
return False
if _Debug:
lg.out(_DebugLevel, 'backup_control.ReadIndex %d items loaded' % count)
# local_site.update_backup_fs(backup_fs.ListAllBackupIDsSQL())
# commit(new_revision)
_LoadingFlag = False
return True
def read_json_response(response, key, result_defer=None, as_bytes=False):
if _Debug:
lg.out(_DebugLevel + 6,
'dht_service.read_json_response [%r] : %r' % (key, response))
value = None
if isinstance(response, list):
if _Debug:
lg.out(_DebugLevel, ' response is a list, value not found')
if result_defer:
result_defer.callback(response)
return None
if isinstance(response, dict):
if response.get('values'):
try:
latest = 0
if as_bytes:
value = jsn.loads(response['values'][0][0])
else:
value = jsn.loads_text(response['values'][0][0])
for v in response['values']:
if v[1] > latest:
latest = v[1]
value = jsn.loads(v[0])
except:
lg.exc()
if _Debug:
lg.out(
_DebugLevel,
' invalid json value found in DHT, return None')
if result_defer:
result_defer.errback(
Exception('invalid json value found in DHT'))
return None
else:
if _Debug:
lg.out(
_DebugLevel,
' response is a dict, "values" field is empty, value not found'
)
if result_defer:
result_defer.callback(response.get('activeContacts', []))
return None
if _Debug:
lg.out(_DebugLevel, ' response is a dict, value is OK')
if result_defer:
result_defer.callback(value)
return value
def BytesToDict(inp, encoding='latin1'):
"""
"""
return jsn.loads(
strng.to_text(
inp,
encoding=encoding,
),
encoding=encoding,
)
def test_jsn(self):
data1 = os.urandom(1024)
dct1 = {
'd': {
'data': data1,
},
}
raw = jsn.dumps(dct1, encoding='latin1')
dct2 = jsn.loads(raw, encoding='latin1')
data2 = dct2['d']['data']
self.assertEqual(data1, data2)
def ReadIndex(text_data, encoding='utf-8'):
"""
Read index data base, ``input`` is a ``StringIO.StringIO`` object which
keeps the data.
This is a simple text format, see ``p2p.backup_fs.Serialize()``
method. The first line keeps revision number.
"""
global _LoadingFlag
if _LoadingFlag:
return False
_LoadingFlag = True
backup_fs.Clear()
count = 0
try:
json_data = jsn.loads(
text_data,
encoding=encoding,
)
except:
lg.exc()
json_data = text_data
if _Debug:
import pprint
lg.out(_DebugLevel, pprint.pformat(json_data))
for customer_id in json_data.keys():
if customer_id == 'items':
try:
count = backup_fs.Unserialize(json_data,
from_json=True,
decoding=encoding)
except:
lg.exc()
return False
else:
customer_idurl = global_id.GlobalUserToIDURL(customer_id)
try:
count = backup_fs.Unserialize(
json_data[customer_id],
iter=backup_fs.fs(customer_idurl),
iterID=backup_fs.fsID(customer_idurl),
from_json=True,
decoding=encoding,
)
except:
lg.exc()
return False
if _Debug:
lg.out(_DebugLevel, 'backup_control.ReadIndex %d items loaded' % count)
# local_site.update_backup_fs(backup_fs.ListAllBackupIDsSQL())
# commit(new_revision)
_LoadingFlag = False
return True
def BytesToDict(inp, encoding='latin1'):
"""
A smart way to extract input bytes into python dictionary object.
bytes will be encoded into text and then loaded via `json.loads()` method.
Finally every text value in result dict will be encoded back to bytes.
"""
return jsn.loads(
strng.to_text(
inp,
encoding=encoding,
),
encoding=encoding,
)
def load_customers(path=None):
"""
Load customers list from disk.
"""
global _CustomersMetaInfo
if path is None:
path = settings.CustomerIDsFilename()
lst = bpio._read_list(path)
if lst is None:
lst = list()
lst = list(map(strng.to_bin, lst))
set_customers(lst)
_CustomersMetaInfo = jsn.loads(
local_fs.ReadTextFile(settings.CustomersMetaInfoFilename()) or '{}')
lg.out(4, 'contactsdb.load_customers %d items' % len(lst))
def read_json_response(response, key, result_defer=None):
value = None
if isinstance(response, list):
if result_defer:
result_defer.callback(response)
return None
if isinstance(response, dict):
try:
value = jsn.loads(response[key])
except:
lg.exc()
if result_defer:
result_defer.errback(
Exception('invalid json value found in DHT'))
return None
if result_defer:
result_defer.callback(value)
return value
def load_customers(path=None):
"""
Load customers list from disk.
"""
global _CustomersMetaInfo
if path is None:
path = settings.CustomerIDsFilename()
lst = bpio._read_list(path)
if lst is None:
lst = list()
lst = list(filter(id_url.is_cached, lst))
set_customers(lst)
_CustomersMetaInfo = jsn.loads(
local_fs.ReadTextFile(settings.CustomersMetaInfoFilename()) or '{}',
keys_to_bin=True,
)
_CustomersMetaInfo = id_url.to_bin_dict(_CustomersMetaInfo)
_CustomersMetaInfo = jsn.dict_values_to_text(_CustomersMetaInfo)
if _Debug:
lg.out(_DebugLevel, 'contactsdb.load_customers %d items' % len(lst))
def _on_new_broker_hired(self, response_info, broker_pos, my_pos,
desired_pos, *args, **kwargs):
try:
# skip leading "accepted:" marker
cooperated_brokers = jsn.loads(
strng.to_text(response_info[0].Payload)[9:])
cooperated_brokers.pop('archive_folder_path', None)
cooperated_brokers = {
int(k): id_url.field(v)
for k, v in cooperated_brokers.items()
}
except:
lg.exc()
self.automat('hire-broker-failed')
return
if _Debug:
lg.args(_DebugLevel,
cooperated=cooperated_brokers,
target=broker_pos,
my=my_pos,
desired=desired_pos,
args=args,
kwargs=kwargs)
if my_pos >= 0:
if id_url.is_the_same(cooperated_brokers.get(my_pos),
self.my_broker_idurl):
self.automat('hire-broker-ok',
cooperated_brokers=cooperated_brokers)
return
if desired_pos >= 0:
if id_url.is_the_same(cooperated_brokers.get(desired_pos),
self.my_broker_idurl):
self.automat('hire-broker-ok',
cooperated_brokers=cooperated_brokers)
return
lg.warn(
'new broker is not cooperative, my idurl is not found in the cooperation on the right place'
)
self.automat('new-broker-rejected',
cooperated_brokers=cooperated_brokers)
def validate_before_store(key, value, originalPublisherID, age, expireSeconds, **kwargs):
"""
Will be executed on receiver side for each (key,value) set request on DHT
"""
try:
json_new_value = jsn.loads(value)
except:
# not a json data to be written - this is not valid
lg.exc()
raise ValueError('input data is not a json value: %r' % value)
if _Debug:
lg.out(_DebugLevel + 8, 'dht_service.validate_before_store key=[%s] json=%r' % (key, json_new_value, ))
new_record_type = json_new_value.get('type')
if not new_record_type:
if _Debug:
lg.out(_DebugLevel, ' new json data do not have "type" field present, store operation FAILED')
raise ValueError('input data do not have "type" field present: %r' % json_new_value)
if key not in node()._dataStore:
if _Debug:
lg.out(_DebugLevel, ' previous value not exists yet, store OK')
return True
prev_value = node()._dataStore[key]
try:
json_prev_value = jsn.loads(prev_value)
except:
if _Debug:
lg.out(_DebugLevel, ' current value in DHT is not a json data, will be overwritten, store OK')
return True
prev_record_type = json_prev_value.get('type')
if prev_record_type and prev_record_type != new_record_type:
if _Debug:
lg.out(_DebugLevel, ' new json data type did not match to existing record type, store operation FAILED')
raise ValueError('new json data type do not match to existing record type: %r' % json_prev_value)
# TODO: need to include "key" field into DHT record and validate it as well
# new_record_key = json_new_value.get('key')
# if not new_record_key:
# if _Debug:
# lg.out(_DebugLevel, ' new json data do not have "key" field present, store operation FAILED')
# return False
# if new_record_key != key:
# if _Debug:
# lg.out(_DebugLevel, ' new json data do not have "key" field set properly, store operation FAILED')
# return False
# prev_record_key = json_prev_value.get('key')
# if prev_record_key and prev_record_key != new_record_key:
# if _Debug:
# lg.out(_DebugLevel, ' new json data "key" field do not match to existing record "key", store operation FAILED')
# return False
try:
prev_revision = int(json_prev_value['revision'])
except:
prev_revision = -1
try:
new_revision = int(json_new_value['revision'])
except:
new_revision = -1
if prev_revision >= 0:
if new_revision < 0:
if _Debug:
lg.out(_DebugLevel, ' new json data must have a revision, store operation FAILED')
raise ValueError('new json data must have a revision')
if new_revision < prev_revision:
if _Debug:
lg.out(_DebugLevel, ' new json data must increment revision number, store operation FAILED')
raise ValueError('new json data must increment revision number, current revision is %d ' % prev_revision)
if new_revision == prev_revision:
if prev_record_type == 'suppliers':
prev_ecc_map = json_prev_value.get('ecc_map')
new_ecc_map = json_new_value.get('ecc_map')
if prev_ecc_map and new_ecc_map != prev_ecc_map:
if _Debug:
lg.out(_DebugLevel, ' new json data have same revision but different ecc_map, store operation FAILED')
raise ValueError('new json data have same revision but different ecc_map, current revision is %d ' % prev_revision)
prev_suppliers = [strng.to_bin(idurl.strip()) for idurl in json_prev_value.get('suppliers', [])]
new_suppliers = [strng.to_bin(idurl.strip()) for idurl in json_new_value.get('suppliers', [])]
if prev_suppliers != new_suppliers:
if _Debug:
lg.out(_DebugLevel, ' new json data have same revision but different suppliers list, store operation FAILED')
raise ValueError('new json data have same revision but different suppliers list, current revision is %d ' % prev_revision)
if _Debug:
lg.out(_DebugLevel, ' new json data is valid and matching existing DHT record, store OK')
return True
