def send(customer_idurl, packet_id, format_type, key_id, remote_idurl):
if not my_keys.is_key_registered(key_id):
lg.warn(
'not able to return Files() for customer %s, key %s not registered'
% (
customer_idurl,
key_id,
))
return p2p_service.SendFailNoRequest(customer_idurl, packet_id)
if _Debug:
lg.out(
_DebugLevel,
"list_files.send to %s, customer_idurl=%s, key_id=%s" % (
remote_idurl,
customer_idurl,
key_id,
))
ownerdir = settings.getCustomerFilesDir(customer_idurl)
plaintext = ''
if os.path.isdir(ownerdir):
for key_alias in os.listdir(ownerdir):
if not misc.ValidKeyAlias(str(key_alias)):
continue
key_alias_dir = os.path.join(ownerdir, key_alias)
plaintext += TreeSummary(key_alias_dir, key_alias)
else:
lg.warn('did not found customer dir: %s' % ownerdir)
if _Debug:
lg.out(_DebugLevel + 8, '\n%s' % plaintext)
raw_list_files = PackListFiles(plaintext, format_type)
block = encrypted.Block(
CreatorID=my_id.getLocalID(),
BackupID=key_id,
Data=raw_list_files,
SessionKey=key.NewSessionKey(),
EncryptKey=key_id,
)
encrypted_list_files = block.Serialize()
newpacket = p2p_service.SendFiles(
idurl=remote_idurl,
raw_list_files_info=encrypted_list_files,
packet_id=packet_id,
callbacks={
commands.Ack(): on_acked,
commands.Fail(): on_failed,
None: on_timeout,
},
)
return newpacket
def send(customer_idurl, packet_id, format_type):
customer_name = nameurl.GetName(customer_idurl)
if _Debug:
lg.out(
_DebugLevel, "list_files.send to %s, format is '%s'" %
(customer_name, format_type))
ownerdir = settings.getCustomerFilesDir(customer_idurl)
plaintext = ''
if os.path.isdir(ownerdir):
for key_alias in os.listdir(ownerdir):
if not misc.ValidKeyAlias(str(key_alias)):
continue
key_alias_dir = os.path.join(ownerdir, key_alias)
plaintext += TreeSummary(key_alias_dir, key_alias)
else:
lg.warn('did not found customer dir: %s' % ownerdir)
if _Debug:
lg.out(_DebugLevel + 8, '\n%s' % plaintext)
return p2p_service.SendFiles(
idurl=customer_idurl,
raw_list_files_info=PackListFiles(plaintext, format_type),
packet_id=packet_id,
)
def report_donated_storage():
space_dict, free_space = read_customers_quotas()
used_space_dict = read_customers_usage()
r = {}
r['customers_num'] = contactsdb.num_customers()
r['customers'] = []
r['old_customers'] = []
r['errors'] = []
r['consumed'] = 0
r['donated'] = settings.getDonatedBytes()
# r['donated_str'] = diskspace.MakeStringFromBytes(r['donated'])
r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
try:
r['free'] = int(free_space)
except:
r['free'] = 0
used = 0
for idurl in id_url.to_bin_list(contactsdb.customers()):
consumed_by_customer = 0
used_by_customer = 0
if idurl not in list(space_dict.keys()):
r['errors'].append('space consumed by customer %r is unknown' %
idurl)
else:
try:
consumed_by_customer = int(space_dict.pop(idurl))
r['consumed'] += consumed_by_customer
except:
r['errors'].append(
'incorrect value of consumed space for customer %r' %
idurl)
continue
if idurl in list(used_space_dict.keys()):
try:
used_by_customer = int(used_space_dict.pop(idurl))
used += used_by_customer
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
if consumed_by_customer < used_by_customer:
r['errors'].append(
'customer %r currently using more space than requested' %
idurl)
c = {}
c['idurl'] = strng.to_text(idurl)
c['used'] = used_by_customer
# c['used_str'] = diskspace.MakeStringFromBytes(c['used'])
c['consumed'] = consumed_by_customer
# c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed'])
c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
# c['real_str'] = diskspace.MakeStringFromBytes(c['real'])
r['customers'].append(c)
r['used'] = used
# r['used_str'] = diskspace.MakeStringFromBytes(r['used'])
# r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed'])
if r['donated'] != r['free'] + r['consumed']:
r['errors'].append(
'total consumed %d and known free %d (%d total) bytes not match with donated %d bytes'
% (r['consumed'], r['free'], r['consumed'] + r['free'],
r['donated']))
if r['used'] > r['donated']:
r['errors'].append(
'total space used by customers exceed the donated limit')
if len(space_dict) > 0:
r['errors'].append('found %d incorrect records of consumed space' %
len(space_dict))
if r['real'] != r['used']:
r['errors'].append(
'current info needs update, known size is %d bytes but real is %d bytes'
% (r['used'], r['real']))
old_customers_used = 0
old_customers_real = 0
for idurl in used_space_dict.keys():
real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
try:
used = int(used_space_dict[idurl])
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
r['old_customers'].append({
'idurl': strng.to_text(idurl),
'used': used,
# 'used_str': diskspace.MakeStringFromBytes(used_space_dict[idurl]),
'real': real,
# 'real_str': diskspace.MakeStringFromBytes(real),
})
old_customers_used += used
old_customers_real += real
r['old_customers_used'] = old_customers_used
r['old_customers_real'] = old_customers_real
try:
r['used_percent'] = misc.value2percent(float(r['used']),
float(r['donated']), 5)
except:
r['used_percent'] = ''
try:
r['consumed_percent'] = misc.value2percent(float(r['consumed']),
float(r['donated']), 5)
except:
r['consumed_percent'] = ''
return r
def send(customer_idurl,
packet_id,
format_type,
key_id,
remote_idurl,
query_items=[]):
if not query_items:
query_items = [
'*',
]
key_id = my_keys.latest_key_id(key_id)
parts = global_id.ParseGlobalID(key_id)
if parts['key_alias'] == 'master' and parts['idurl'] != my_id.getIDURL():
# lg.warn('incoming ListFiles() request with customer "master" key: %r' % key_id)
if not my_keys.is_key_registered(key_id) and identitycache.HasKey(
parts['idurl']):
lg.info(
'customer public key %r to be registered locally for the first time'
% key_id)
known_ident = identitycache.FromCache(parts['idurl'])
if not my_keys.register_key(key_id, known_ident.getPublicKey()):
lg.err(
'failed to register known public key of the customer: %r' %
key_id)
if not my_keys.is_key_registered(key_id):
lg.warn(
'not able to return Files() for customer %s, key %s not registered'
% (
customer_idurl,
key_id,
))
return p2p_service.SendFailNoRequest(customer_idurl,
packet_id,
response='key not registered')
if _Debug:
lg.out(
_DebugLevel,
"list_files.send to %s, customer_idurl=%s, key_id=%s, query_items=%r"
% (
remote_idurl,
customer_idurl,
key_id,
query_items,
))
ownerdir = settings.getCustomerFilesDir(customer_idurl)
plaintext = ''
if os.path.isdir(ownerdir):
try:
for query_path in query_items:
plaintext += process_query_item(query_path, parts['key_alias'],
ownerdir)
except:
lg.exc()
return p2p_service.SendFailNoRequest(
customer_idurl,
packet_id,
response='list files query processing error')
else:
lg.warn('did not found customer folder: %s' % ownerdir)
if _Debug:
lg.out(_DebugLevel, '\n%s' % plaintext)
raw_list_files = PackListFiles(plaintext, format_type)
block = encrypted.Block(
CreatorID=my_id.getIDURL(),
BackupID=key_id,
Data=raw_list_files,
SessionKey=key.NewSessionKey(session_key_type=key.SessionKeyType()),
SessionKeyType=key.SessionKeyType(),
EncryptKey=key_id,
)
encrypted_list_files = block.Serialize()
newpacket = p2p_service.SendFiles(
idurl=remote_idurl,
raw_list_files_info=encrypted_list_files,
packet_id=packet_id,
callbacks={
commands.Ack(): on_acked,
commands.Fail(): on_failed,
None: on_timeout,
},
)
return newpacket
def report_donated_storage():
space_dict = read_customers_quotas()
used_space_dict = read_customers_usage()
r = {}
r['customers_num'] = contactsdb.num_customers()
r['customers'] = []
r['old_customers'] = []
r['errors'] = []
r['consumed'] = 0
r['donated'] = settings.getDonatedBytes()
r['donated_str'] = diskspace.MakeStringFromBytes(r['donated'])
r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
try:
r['free'] = int(space_dict.pop('free'))
except:
r['free'] = 0
used = 0
for idurl in contactsdb.customers():
consumed_by_customer = 0
used_by_customer = 0
if idurl not in space_dict.keys():
r['errors'].append('space consumed by customer %s is unknown' % idurl)
else:
try:
consumed_by_customer = int(space_dict.pop(idurl))
r['consumed'] += consumed_by_customer
except:
r['errors'].append('incorrect value of consumed space for customer %s' % idurl)
if idurl in used_space_dict.keys():
try:
used_by_customer = int(used_space_dict.pop(idurl))
used += used_by_customer
except:
r['errors'].append('incorrect value of used space for customer %s' % idurl)
if consumed_by_customer < used_by_customer:
r['errors'].append('customer %s currently using more space than requested' % idurl)
c = {}
c['idurl'] = idurl
c['used'] = used_by_customer
c['used_str'] = diskspace.MakeStringFromBytes(c['used'])
c['consumed'] = consumed_by_customer
c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed'])
c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
c['real_str'] = diskspace.MakeStringFromBytes(c['real'])
r['customers'].append(c)
r['used'] = used
r['used_str'] = diskspace.MakeStringFromBytes(r['used'])
r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed'])
if r['donated'] != r['free'] + r['consumed']:
r['errors'].append('total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % (
r['consumed'], r['free'],
r['consumed'] + r['free'], r['donated']))
if r['used'] > r['donated']:
r['errors'].append('total space used by customers exceed the donated limit')
if len(space_dict) > 0:
r['errors'].append('found %d incorrect records of consumed space' % len(space_dict))
if r['real'] != r['used']:
r['errors'].append('current info needs update, known size is %d bytes but real is %d bytes' % (
r['used'], r['real']))
for idurl in used_space_dict.keys():
real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
r['old_customers'].append({
'idurl': idurl,
'used': used_space_dict['idurl'],
'used_str': diskspace.MakeStringFromBytes(used_space_dict['idurl']),
'real': real,
'real_str': diskspace.MakeStringFromBytes(real),
})
try:
r['used_percent'] = misc.percent2string(float(r['used']) / float(r['donated']), 5)
except:
r['used_percent'] = ''
try:
r['consumed_percent'] = misc.percent2string(float(r['consumed']) / float(r['donated']), 5)
except:
r['consumed_percent'] = ''
return r