def save_key(key_id, keys_folder=None):
"""
"""
key_object = known_keys()[key_id]
if key_object is None:
lg.warn('can not save key %s because it is not loaded yet' % key_id)
return False
if not keys_folder:
keys_folder = settings.KeyStoreDir()
if key_object.isPublic():
key_filepath = os.path.join(keys_folder, key_id + '.public')
key_dict = key_object.toDict(include_private=False)
key_string = jsn.dumps(key_dict, indent=1, separators=(',', ':'))
else:
key_filepath = os.path.join(keys_folder, key_id + '.private')
key_dict = key_object.toDict(include_private=True)
key_string = jsn.dumps(key_dict, indent=1, separators=(',', ':'))
if not bpio.WriteTextFile(key_filepath, key_string):
lg.warn('failed saving key %r to %r' % (
key_id,
key_filepath,
))
return False
if _Debug:
lg.out(
_DebugLevel, 'my_keys.save_key stored key %r locally in %r' % (
key_id,
key_filepath,
))
return True
def WriteIndex(filepath=None, encoding='utf-8'):
"""
Write index data base to the local file .bitdust/metadata/index.
"""
global _LoadingFlag
if _LoadingFlag:
return
if filepath is None:
filepath = settings.BackupIndexFilePath()
json_data = {}
# json_data = backup_fs.Serialize(to_json=True, encoding=encoding)
for customer_idurl in backup_fs.known_customers():
# customer_id = global_id.UrlToGlobalID(customer_idurl)
customer_id = customer_idurl.to_id()
json_data[customer_id] = backup_fs.Serialize(
iterID=backup_fs.fsID(customer_idurl),
to_json=True,
encoding=encoding,
)
src = '%d\n' % revision()
src += jsn.dumps(
json_data,
indent=1,
separators=(',', ':'),
encoding=encoding,
)
if _Debug:
lg.args(_DebugLevel, json_data=json_data)
return bpio.WriteTextFile(filepath, src)
def save_customers(path=None, save_meta_info=False):
"""
Write current customers list on the disk, ``path`` is a file path to save.
"""
global _CustomersMetaInfo
if path is None:
path = settings.CustomerIDsFilename()
lst = customers()
lst = list(map(strng.to_text, lst))
bpio._write_list(path, lst)
if save_meta_info:
json_info = id_url.to_bin_dict(_CustomersMetaInfo)
local_fs.WriteTextFile(
settings.CustomersMetaInfoFilename(),
jsn.dumps(
json_info,
indent=2,
sort_keys=True,
keys_to_text=True,
))
if _Debug:
lg.out(
_DebugLevel, 'contactsdb.save_customers save_meta_info=%r : %r' % (
save_meta_info,
lst,
))
def DictToBytes(dct,
encoding='latin1',
errors='strict',
keys_to_text=False,
values_to_text=False,
pack_types=False):
"""
Calls `json.dupms()` method for input dict to build bytes output.
Uses encoding to decode every byte string to text and ensure ascii output.
If `keys_to_text` is True it will also convert dict keys from binary strings to text strings.
If `values_to_text` is True it will also convert dict values from binary strings to text strings.
Another smart feature is `pack_types` - it will remember types of keys and values in the input dict.
Respective feature `unpack_types` of `BytesToDict()` method can be used to "extract" exactly same dict from bytes.
Can be used to serialize dictionaries of mixed types - with binary and text values.
"""
return strng.to_bin(
jsn.dumps(
jsn.pack_dict(dct, encoding=encoding, errors=errors)
if pack_types else dct,
separators=(',', ':'),
indent=None,
sort_keys=True,
ensure_ascii=True,
encoding=encoding,
keys_to_text=keys_to_text,
values_to_text=values_to_text,
),
encoding=encoding,
errors=errors,
)
def write_state(customer_id, broker_id, json_value):
service_dir = settings.ServiceDir('service_message_broker')
keepers_dir = os.path.join(service_dir, 'keepers')
broker_dir = os.path.join(keepers_dir, broker_id)
keeper_state_file_path = os.path.join(broker_dir, customer_id)
if json_value is None:
if os.path.isfile(keeper_state_file_path):
try:
os.remove(keeper_state_file_path)
except:
lg.exc()
if _Debug:
lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id)
return None
if not os.path.isdir(broker_dir):
try:
os.makedirs(broker_dir)
except:
lg.exc()
return None
if not local_fs.WriteTextFile(keeper_state_file_path, jsn.dumps(json_value)):
lg.err('failed writing queue_keeper state for customer %r of broker %r to %r' % (
customer_id, broker_id, keeper_state_file_path, ))
return None
if _Debug:
lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id, json_value=json_value)
return json_value
def add_customer_meta_info(customer_idurl, info):
"""
"""
global _CustomersMetaInfo
customer_idurl = strng.to_bin(customer_idurl.strip())
if customer_idurl not in _CustomersMetaInfo:
if _Debug:
lg.out(
_DebugLevel,
'contactsdb.add_customer_meta_info store new meta info for customer %r: %r'
% (
customer_idurl,
info,
))
_CustomersMetaInfo[customer_idurl] = {}
else:
if _Debug:
lg.out(
_DebugLevel,
'contactsdb.add_customer_meta_info update existing meta info for customer %r: %r'
% (
customer_idurl,
info,
))
_CustomersMetaInfo[customer_idurl].update(info)
local_fs.WriteTextFile(
settings.CustomersMetaInfoFilename(),
jsn.dumps(
_CustomersMetaInfo,
indent=2,
sort_keys=True,
))
return _CustomersMetaInfo
def save_customers_list(customers_list):
create_home_dir()
for customer_info in customers_list:
if not WriteTextFile(
customer_info_filepath(customer_info['customer_id']),
jsn.dumps(customer_info, indent=2)):
return False
return True
def add_customer_meta_info(customer_idurl, info):
"""
"""
global _CustomersMetaInfo
customer_idurl = id_url.field(customer_idurl)
if not customer_idurl.is_latest():
if customer_idurl.original() in _CustomersMetaInfo:
if customer_idurl.to_bin() not in _CustomersMetaInfo:
_CustomersMetaInfo[
customer_idurl.to_bin()] = _CustomersMetaInfo.pop(
customer_idurl.original())
lg.info(
'detected and processed idurl rotate for customer meta info : %r -> %r'
% (customer_idurl.original(), customer_idurl.to_bin()))
customer_idurl = id_url.to_bin(customer_idurl)
if 'family_snapshot' in info:
info['family_snapshot'] = id_url.to_bin_list(info['family_snapshot'])
if 'ecc_map' in info:
info['ecc_map'] = strng.to_text(info['ecc_map'])
if customer_idurl not in _CustomersMetaInfo:
if _Debug:
lg.out(
_DebugLevel,
'contactsdb.add_customer_meta_info store new meta info for customer %r: %r'
% (
customer_idurl,
info,
))
_CustomersMetaInfo[customer_idurl] = {}
else:
if _Debug:
lg.out(
_DebugLevel,
'contactsdb.add_customer_meta_info update existing meta info for customer %r: %r'
% (
customer_idurl,
info,
))
_CustomersMetaInfo[customer_idurl].update(info)
json_info = {
k: jsn.dict_keys_to_text(v)
for k, v in id_url.to_bin_dict(_CustomersMetaInfo).items()
}
try:
raw_data = jsn.dumps(
json_info,
indent=2,
sort_keys=True,
keys_to_text=True,
values_to_text=True,
)
except:
lg.exc()
return None
local_fs.WriteTextFile(settings.CustomersMetaInfoFilename(), raw_data)
return _CustomersMetaInfo
def set_json_value(key, json_data, age=0, expire=KEY_EXPIRE_MAX_SECONDS, collect_results=True):
if not node():
return fail(Exception('DHT service is off'))
try:
value = jsn.dumps(json_data, indent=0, sort_keys=True, separators=(',', ':'))
except:
return fail(Exception('bad input json data'))
if _Debug:
lg.out(_DebugLevel, 'dht_service.set_json_value key=[%s] with %d bytes' % (key, len(str(value))))
return set_value(key=key, value=value, age=age, expire=expire, collect_results=collect_results)
def test_jsn(self):
data1 = os.urandom(1024)
dct1 = {
'd': {
'data': data1,
},
}
raw = jsn.dumps(dct1, encoding='latin1')
dct2 = jsn.loads(raw, encoding='latin1')
data2 = dct2['d']['data']
self.assertEqual(data1, data2)
def create_new_transaction(details):
create_home_dir()
max_transaction_id = 0
all_transactions = os.listdir(transactions_dir())
for transaction_id in all_transactions:
if int(transaction_id) > max_transaction_id:
max_transaction_id = int(transaction_id)
new_transaction_id = str(max_transaction_id + 1)
details['transaction_id'] = new_transaction_id
WriteTextFile(transaction_filepath(new_transaction_id), jsn.dumps(details, indent=2))
return details
def remove_customer_meta_info(customer_idurl):
"""
"""
global _CustomersMetaInfo
customer_idurl = strng.to_bin(customer_idurl.strip())
if customer_idurl not in _CustomersMetaInfo:
lg.warn('meta info for customer %r not exist' % customer_idurl)
return False
if _Debug:
lg.out(_DebugLevel, 'contactsdb.remove_customer_meta_info erase existing meta info for customer %r' % customer_idurl)
_CustomersMetaInfo.pop(customer_idurl)
local_fs.WriteTextFile(settings.CustomersMetaInfoFilename(), jsn.dumps(
_CustomersMetaInfo, indent=2, sort_keys=True, keys_to_text=True, ))
return True
def DictToBytes(dct, encoding='latin1'):
"""
"""
return strng.to_bin(
jsn.dumps(
dct,
separators=(',', ':'),
indent=0,
sort_keys=True,
encoding=encoding,
),
encoding=encoding,
errors='strict',
)
def save_group_info(group_key_id):
if not is_group_exist(group_key_id):
lg.warn('group %r is not known' % group_key_id)
return False
group_info = known_groups()[group_key_id]
service_dir = settings.ServiceDir('service_private_groups')
groups_dir = os.path.join(service_dir, 'groups')
group_info_path = os.path.join(groups_dir, group_key_id)
if not os.path.isdir(groups_dir):
bpio._dirs_make(groups_dir)
ret = local_fs.WriteTextFile(group_info_path, jsn.dumps(group_info))
if _Debug:
lg.args(_DebugLevel, group_key_id=group_key_id, group_info_path=group_info_path, ret=ret)
return ret
def write_customers_meta_info_all(new_customers_info):
"""
"""
global _CustomersMetaInfo
_CustomersMetaInfo = new_customers_info
json_info = {k: jsn.dict_keys_to_text(v) for k, v in id_url.to_bin_dict(_CustomersMetaInfo).items()}
try:
raw_data = jsn.dumps(
json_info, indent=2, sort_keys=True, keys_to_text=True, values_to_text=True,
)
except:
lg.exc()
return None
local_fs.WriteTextFile(settings.CustomersMetaInfoFilename(), raw_data)
return _CustomersMetaInfo
def save_customers(path=None, save_meta_info=False):
"""
Write current customers list on the disk, ``path`` is a file path to save.
"""
global _CustomersMetaInfo
if path is None:
path = settings.CustomerIDsFilename()
lst = customers()
lst = list(map(strng.to_text, lst))
bpio._write_list(path, lst)
if save_meta_info:
local_fs.WriteTextFile(settings.CustomersMetaInfoFilename(),
jsn.dumps(_CustomersMetaInfo))
if _Debug:
lg.out(_DebugLevel, 'contactsdb.save_customers : %r' % lst)
def DictToBytes(dct, encoding='latin1'):
"""
Calls `json.dupms()` method for input dict to build bytes output.
Uses encoding to translate every byte string to text and ensure ascii output.
"""
return strng.to_bin(
jsn.dumps(
dct,
separators=(',', ':'),
indent=None,
sort_keys=True,
ensure_ascii=True,
encoding=encoding,
),
encoding=encoding,
errors='strict',
)
def set_broker(customer_id, broker_id, position=0):
service_dir = settings.ServiceDir('service_private_groups')
brokers_dir = os.path.join(service_dir, 'brokers')
customer_dir = os.path.join(brokers_dir, customer_id)
broker_path = os.path.join(customer_dir, broker_id)
if os.path.isfile(broker_path):
lg.warn('broker %r already exist for customer %r, overwriting' % (broker_id, customer_id, ))
if not os.path.isdir(customer_dir):
bpio._dirs_make(customer_dir)
broker_info = {
'position': position,
}
if not local_fs.WriteTextFile(broker_path, jsn.dumps(broker_info)):
lg.err('failed to set broker %r at position %d for customer %r' % (broker_id, position, customer_id, ))
return False
known_brokers(customer_id)[position] = broker_id
if _Debug:
lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id, broker_info=broker_info)
return True
def dump_local_db(value_as_json=False):
if not node():
if _Debug:
lg.out(_DebugLevel, 'dht_service.dump_local_db local node is not ready')
return None
l = []
for itm in node()._dataStore.getAllItems():
if value_as_json:
if isinstance(itm['value'], dict):
_j = jsn.dumps(itm['value'], keys_to_text=True, errors='ignore')
itm['value'] = jsn.loads_text(_j, errors='ignore')
else:
try:
itm['value'] = jsn.loads_text(itm['value'], errors='ignore')
except:
itm['value'] = strng.to_text(itm['value'])
itm['scope'] = 'global'
l.append(itm)
for k, v in node().data.items():
l.append({'key': k, 'value': v, 'scope': 'node', })
return l
def remove_customer_meta_info(customer_idurl):
"""
"""
global _CustomersMetaInfo
customer_idurl = id_url.field(customer_idurl)
if not customer_idurl.is_latest():
if customer_idurl.original() in _CustomersMetaInfo:
if customer_idurl.to_bin() not in _CustomersMetaInfo:
_CustomersMetaInfo[
customer_idurl.to_bin()] = _CustomersMetaInfo.pop(
customer_idurl.original())
lg.info(
'detected and processed idurl rotate for customer meta info : %r -> %r'
% (customer_idurl.original(), customer_idurl.to_bin()))
customer_idurl = id_url.to_bin(customer_idurl)
if customer_idurl not in _CustomersMetaInfo:
lg.warn('meta info for customer %r not exist' % customer_idurl)
return False
if _Debug:
lg.out(
_DebugLevel,
'contactsdb.remove_customer_meta_info erase existing meta info for customer %r'
% customer_idurl)
_CustomersMetaInfo.pop(customer_idurl)
json_info = {
k: jsn.dict_keys_to_text(v)
for k, v in id_url.to_bin_dict(_CustomersMetaInfo).items()
}
local_fs.WriteTextFile(
settings.CustomersMetaInfoFilename(),
jsn.dumps(
json_info,
indent=2,
sort_keys=True,
keys_to_text=True,
values_to_text=True,
))
return True
def set_broker(customer_id, broker_id, position=0):
customer_id = global_id.latest_glob_id(customer_id)
broker_id = global_id.latest_glob_id(broker_id)
service_dir = settings.ServiceDir('service_private_groups')
brokers_dir = os.path.join(service_dir, 'brokers')
customer_dir = os.path.join(brokers_dir, customer_id)
broker_path = os.path.join(customer_dir, broker_id)
if not os.path.isdir(customer_dir):
bpio._dirs_make(customer_dir)
if os.path.isfile(broker_path):
if _Debug:
lg.dbg(_DebugLevel, 'broker %r already exist for customer %r, overwriting' % (broker_id, customer_id, ))
broker_info = {
'position': position,
}
prev_borker_id = known_brokers(customer_id)[position]
if prev_borker_id:
if prev_borker_id == broker_id:
if _Debug:
lg.args(_DebugLevel, customer_id=customer_id, position=position, broker_id=broker_id, prev_borker_id=prev_borker_id)
return True
prev_broker_path = os.path.join(customer_dir, prev_borker_id)
if os.path.isfile(prev_broker_path):
lg.info('replacing existing broker for customer %r at position %d : %r -> %r' % (
customer_id, position, prev_borker_id, broker_id, ))
try:
os.remove(prev_broker_path)
except:
lg.exc()
return False
if not local_fs.WriteTextFile(broker_path, jsn.dumps(broker_info)):
lg.err('failed to set broker %r at position %d for customer %r' % (broker_id, position, customer_id, ))
return False
known_brokers(customer_id)[position] = broker_id
if _Debug:
lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id, broker_info=broker_info)
return True
def write_verify_republish_data(key,
json_data,
age=0,
expire=KEY_EXPIRE_MAX_SECONDS,
rules={}):
"""
"""
try:
raw_value = jsn.dumps(json_data,
indent=0,
sort_keys=True,
separators=(',', ':'))
except:
return fail(Exception('bad input json data'))
ret = Deferred()
_found_nodes = None
_write_response = None
_join = Deferred()
_join.addCallback(_do_verify)
_join.addErrback(lg.errback)
def _some_nodes_found(nodes):
global _write_response
global _found_nodes
global _join
if _Debug:
lg.out(_DebugLevel, 'dht_service._some_nodes_found : %r' % nodes)
if len(nodes) > 0:
_found_nodes = nodes
else:
_found_nodes = []
if _write_response:
_join.callback(_write_response, _found_nodes)
return nodes
def _nodes_not_found(err):
global _found_nodes
global _join
if _Debug:
lg.out(_DebugLevel,
'dht_service._nodes_not_found err=%s' % str(err))
_found_nodes = []
_join.cancel()
del _join
ret.errback(err)
return err
def _write_ok(write_result):
global _write_response
global _found_nodes
global _join
if _Debug:
lg.out(_DebugLevel, 'dht_service._write_ok : %r' % write_result)
_write_response = write_result
if _found_nodes is not None:
_join.callback(_write_response, _found_nodes)
return write_result
def _write_failed(err):
global _join
if _Debug:
lg.out(_DebugLevel, 'dht_service._write_failed err=%r' % err)
_join.cancel()
del _join
ret.errback(err)
return err
def _do_verify(write_response, found_nodes):
if _Debug:
lg.out(
_DebugLevel, 'dht_service._do_verify %r via nodes: %r' % (
write_response,
found_nodes,
))
for node in found_nodes:
node.request(b'verify_update', key, raw_value, age, expire)
ret.callback(write_response, found_nodes)
return None
new_key = random_key()
d_observer = find_node(new_key)
d_observer.addCallback(_some_nodes_found)
d_observer.addErrback(_nodes_not_found)
d_write = set_valid_data(key=key,
json_data=json_data,
age=age,
expire=expire,
rules=rules)
d_write.addCallback(_write_ok)
d_write.addErrback(_write_failed)
return ret
def write_transaction(transaction_id, details):
create_home_dir()
return WriteTextFile(transaction_filepath(transaction_id),
jsn.dumps(details, indent=2))
def write_customer_info(customer_info):
create_home_dir()
create_customer_dir(customer_info['customer_id'])
return WriteTextFile(customer_info_filepath(customer_info['customer_id']),
jsn.dumps(customer_info, indent=2))
def write_settings(settings_dict):
create_home_dir()
return WriteTextFile(settings_filepath(), jsn.dumps(settings_dict,
indent=2))