def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(
contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(
settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(
settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(
backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (
contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() /
contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {
'result': result,
}
def doTestMyCapacity(self, arg):
"""
Here are some values.
+ donated_bytes : you set this in the configs
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
lg.out(8, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(
space_dict)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
space_dict['free'] = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
if failed_customers:
lg.out(
8, ' found FAILED Customers:\n%s' %
(' \n'.join(failed_customers)))
for idurl in failed_customers:
current_customers.remove(idurl)
self.automat('space-overflow',
(space_dict, consumed_bytes, current_customers,
failed_customers))
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(
space_dict, used_dict)
customers_sorted = sorted(
current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],
)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.out(8, ' customer %s will be REMOVED' % idurl)
space_dict['free'] = donated_bytes - consumed_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat(
'space-overflow',
(space_dict, consumed_bytes, current_customers, failed_customers))
def check_create_customers_quotas(donated_bytes=None):
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {
'free': donated_bytes or settings.getDonatedBytes(),
})
lg.info('created a new customers quotas file: %s' %
settings.CustomersSpaceFile())
return True
return False
def doTestMyCapacity(self, *args, **kwargs):
"""
Here are some values.
+ donated_bytes : you set this in the settings
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
if _Debug:
lg.out(_DebugLevel, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict, free_space = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(
space_dict, free_space)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
free_space = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict, free_space)
lg.info('storage quota checks succeed, all customers are verified')
self.automat('space-enough')
return
if failed_customers:
for idurl in failed_customers:
lg.warn('customer %r failed storage quota verification' %
idurl)
current_customers.remove(idurl)
self.automat('space-overflow', failed_customers)
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(
space_dict, used_dict)
customers_sorted = sorted(
current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],
)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.warn(
'customer %r will be removed because of storage quota overflow'
% idurl)
free_space = donated_bytes - consumed_bytes
self.automat('space-overflow', failed_customers)
def doTestMyCapacity(self, arg):
"""
Here are some values.
+ donated_bytes : you set this in the configs
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
lg.out(8, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(space_dict)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
space_dict['free'] = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
if failed_customers:
lg.out(8, ' found FAILED Customers:\n%s' % (
' \n'.join(failed_customers)))
for idurl in failed_customers:
current_customers.remove(idurl)
self.automat('space-overflow', (
space_dict, consumed_bytes, current_customers, failed_customers))
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(space_dict, used_dict)
customers_sorted = sorted(current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.out(8, ' customer %s will be REMOVED' % idurl)
space_dict['free'] = donated_bytes - consumed_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat('space-overflow', (
space_dict, consumed_bytes, current_customers, failed_customers))
def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {'result': result, }
def _on_data(self, newpacket):
import os
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from lib import jsn
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
from storage import accounting
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID):
# SECURITY
# TODO: process files from another customer : glob_path['idurl']
lg.warn("skip, %s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
# p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict = accounting.read_customers_quotas()
if newpacket.OwnerID not in list(space_dict.keys()):
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(self.debug_level, "service_supplier._on_data %r" % newpacket)
lg.out(
self.debug_level, " from [ %s | %s ]" % (
newpacket.OwnerID,
newpacket.CreatorID,
))
lg.out(self.debug_level, " saved with %d bytes to %s" % (
sz,
filename,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
if self.publish_event_supplier_file_modified:
from main import events
events.send('supplier-file-modified',
data=dict(
action='write',
glob_path=glob_path['path'],
owner_id=newpacket.OwnerID,
))
return True
def doTestMyCapacity2(self, arg):
"""
Here are some values.
- donated_bytes : you set this in the config
- spent_bytes : how many space is taken from you by other users right now
- free_bytes = donated_bytes - spent_bytes : not yet allocated space
- used_bytes : size of all files, which you store on your disk for your customers
"""
current_customers = contactsdb.customers()
removed_customers = []
spent_bytes = 0
donated_bytes = settings.getDonatedBytes()
if os.path.isfile(settings.CustomersSpaceFile()):
space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {})
else:
space_dict = {'free': donated_bytes}
used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
lg.out(
8,
'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes)
try:
int(space_dict['free'])
for idurl, customer_bytes in space_dict.items():
if idurl != 'free':
spent_bytes += int(customer_bytes)
except:
lg.exc()
space_dict = {'free': donated_bytes}
spent_bytes = 0
removed_customers = list(current_customers)
current_customers = []
self.automat('space-overflow',
(space_dict, spent_bytes, current_customers,
removed_customers))
return
lg.out(8, ' spent=%d' % spent_bytes)
if spent_bytes < donated_bytes:
space_dict['free'] = donated_bytes - spent_bytes
bpio._write_dict(settings.CustomersSpaceFile(), space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
used_space_ratio_dict = {}
for customer_pos in range(contactsdb.num_customers()):
customer_idurl = contactsdb.customer(customer_pos)
try:
allocated_bytes = int(space_dict[customer_idurl])
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated space unknown' % customer_idurl)
continue
if allocated_bytes <= 0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated_bytes==0' % customer_idurl)
continue
try:
files_size = int(used_dict.get(customer_idurl, 0))
ratio = float(files_size) / float(allocated_bytes)
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s used_dict have wrong value' % customer_idurl)
continue
if ratio > 1.0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
spent_bytes -= allocated_bytes
lg.warn('%s space overflow, where is bptester?' %
customer_idurl)
continue
used_space_ratio_dict[customer_idurl] = ratio
customers_sorted = sorted(
current_customers,
key=lambda i: used_space_ratio_dict[i],
)
while len(customers_sorted) > 0:
customer_idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[customer_idurl])
spent_bytes -= allocated_bytes
space_dict.pop(customer_idurl)
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
lg.out(8, ' customer %s REMOVED' % customer_idurl)
if spent_bytes < donated_bytes:
break
space_dict['free'] = donated_bytes - spent_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat(
'space-overflow',
(space_dict, spent_bytes, current_customers, removed_customers))
def Data(request):
"""
This is when we 1) save my requested data to restore the backup 2) or save
the customer file on our local HDD.
"""
# 1. this is our Data!
if request.OwnerID == my_id.getLocalID():
if _Debug:
lg.out(_DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID)))
if driver.is_started("service_backups"):
if request.PacketID in [settings.BackupIndexFileName()]:
from storage import backup_control
backup_control.IncomingSupplierBackupIndex(request)
return True
return False
# 2. this Data is not belong to us
if not driver.is_started("service_supplier"):
return SendFail(request, "supplier service is off")
if not contactsdb.is_customer(request.OwnerID): # SECURITY
lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID))
SendFail(request, "not a customer")
return
filename = makeFilename(request.OwnerID, request.PacketID)
if filename == "":
lg.warn("got empty filename, bad customer or wrong packetID? ")
SendFail(request, "empty filename")
return
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.warn("ERROR can not create sub dir " + dirname)
SendFail(request, "write error")
return
data = request.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {"free": donated_bytes})
if _Debug:
lg.out(_DebugLevel, "p2p_service.Data created a new space file")
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if request.OwnerID not in space_dict.keys():
lg.warn("no info about donated space for %s" % request.OwnerID)
SendFail(request, "no info about donated space")
return
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
if request.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(used_space_dict[request.OwnerID])
bytes_donated_to_customer = int(space_dict[request.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space for %s" % request.OwnerID)
SendFail(request, "no free space")
return
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.warn("ERROR can not write to " + str(filename))
SendFail(request, "write error")
return
SendAck(request, str(len(request.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
del data
if _Debug:
lg.out(
_DebugLevel,
"p2p_service.Data saved from [%s/%s] to %s"
% (nameurl.GetName(request.OwnerID), nameurl.GetName(request.CreatorID), filename),
)
def report_donated_storage():
space_dict = read_customers_quotas()
used_space_dict = read_customers_usage()
r = {}
r['customers_num'] = contactsdb.num_customers()
r['customers'] = []
r['old_customers'] = []
r['errors'] = []
r['consumed'] = 0
r['donated'] = settings.getDonatedBytes()
r['donated_str'] = diskspace.MakeStringFromBytes(r['donated'])
r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
try:
r['free'] = int(space_dict.pop('free'))
except:
r['free'] = 0
used = 0
for idurl in contactsdb.customers():
consumed_by_customer = 0
used_by_customer = 0
if idurl not in space_dict.keys():
r['errors'].append('space consumed by customer %s is unknown' % idurl)
else:
try:
consumed_by_customer = int(space_dict.pop(idurl))
r['consumed'] += consumed_by_customer
except:
r['errors'].append('incorrect value of consumed space for customer %s' % idurl)
if idurl in used_space_dict.keys():
try:
used_by_customer = int(used_space_dict.pop(idurl))
used += used_by_customer
except:
r['errors'].append('incorrect value of used space for customer %s' % idurl)
if consumed_by_customer < used_by_customer:
r['errors'].append('customer %s currently using more space than requested' % idurl)
c = {}
c['idurl'] = idurl
c['used'] = used_by_customer
c['used_str'] = diskspace.MakeStringFromBytes(c['used'])
c['consumed'] = consumed_by_customer
c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed'])
c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
c['real_str'] = diskspace.MakeStringFromBytes(c['real'])
r['customers'].append(c)
r['used'] = used
r['used_str'] = diskspace.MakeStringFromBytes(r['used'])
r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed'])
if r['donated'] != r['free'] + r['consumed']:
r['errors'].append('total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % (
r['consumed'], r['free'],
r['consumed'] + r['free'], r['donated']))
if r['used'] > r['donated']:
r['errors'].append('total space used by customers exceed the donated limit')
if len(space_dict) > 0:
r['errors'].append('found %d incorrect records of consumed space' % len(space_dict))
if r['real'] != r['used']:
r['errors'].append('current info needs update, known size is %d bytes but real is %d bytes' % (
r['used'], r['real']))
for idurl in used_space_dict.keys():
real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
r['old_customers'].append({
'idurl': idurl,
'used': used_space_dict['idurl'],
'used_str': diskspace.MakeStringFromBytes(used_space_dict['idurl']),
'real': real,
'real_str': diskspace.MakeStringFromBytes(real),
})
try:
r['used_percent'] = misc.percent2string(float(r['used']) / float(r['donated']), 5)
except:
r['used_percent'] = ''
try:
r['consumed_percent'] = misc.percent2string(float(r['consumed']) / float(r['donated']), 5)
except:
r['consumed_percent'] = ''
return r
def check_create_customers_quotas():
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': settings.getDonatedBytes()})
return True
return False
def report_donated_storage():
space_dict, free_space = read_customers_quotas()
used_space_dict = read_customers_usage()
r = {}
r['customers_num'] = contactsdb.num_customers()
r['customers'] = []
r['old_customers'] = []
r['errors'] = []
r['consumed'] = 0
r['donated'] = settings.getDonatedBytes()
# r['donated_str'] = diskspace.MakeStringFromBytes(r['donated'])
r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
try:
r['free'] = int(free_space)
except:
r['free'] = 0
used = 0
for idurl in id_url.to_bin_list(contactsdb.customers()):
consumed_by_customer = 0
used_by_customer = 0
if idurl not in list(space_dict.keys()):
r['errors'].append('space consumed by customer %r is unknown' %
idurl)
else:
try:
consumed_by_customer = int(space_dict.pop(idurl))
r['consumed'] += consumed_by_customer
except:
r['errors'].append(
'incorrect value of consumed space for customer %r' %
idurl)
continue
if idurl in list(used_space_dict.keys()):
try:
used_by_customer = int(used_space_dict.pop(idurl))
used += used_by_customer
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
if consumed_by_customer < used_by_customer:
r['errors'].append(
'customer %r currently using more space than requested' %
idurl)
c = {}
c['idurl'] = strng.to_text(idurl)
c['used'] = used_by_customer
# c['used_str'] = diskspace.MakeStringFromBytes(c['used'])
c['consumed'] = consumed_by_customer
# c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed'])
c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
# c['real_str'] = diskspace.MakeStringFromBytes(c['real'])
r['customers'].append(c)
r['used'] = used
# r['used_str'] = diskspace.MakeStringFromBytes(r['used'])
# r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed'])
if r['donated'] != r['free'] + r['consumed']:
r['errors'].append(
'total consumed %d and known free %d (%d total) bytes not match with donated %d bytes'
% (r['consumed'], r['free'], r['consumed'] + r['free'],
r['donated']))
if r['used'] > r['donated']:
r['errors'].append(
'total space used by customers exceed the donated limit')
if len(space_dict) > 0:
r['errors'].append('found %d incorrect records of consumed space' %
len(space_dict))
if r['real'] != r['used']:
r['errors'].append(
'current info needs update, known size is %d bytes but real is %d bytes'
% (r['used'], r['real']))
old_customers_used = 0
old_customers_real = 0
for idurl in used_space_dict.keys():
real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
try:
used = int(used_space_dict[idurl])
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
r['old_customers'].append({
'idurl': strng.to_text(idurl),
'used': used,
# 'used_str': diskspace.MakeStringFromBytes(used_space_dict[idurl]),
'real': real,
# 'real_str': diskspace.MakeStringFromBytes(real),
})
old_customers_used += used
old_customers_real += real
r['old_customers_used'] = old_customers_used
r['old_customers_real'] = old_customers_real
try:
r['used_percent'] = misc.value2percent(float(r['used']),
float(r['donated']), 5)
except:
r['used_percent'] = ''
try:
r['consumed_percent'] = misc.value2percent(float(r['consumed']),
float(r['donated']), 5)
except:
r['consumed_percent'] = ''
return r
def on_data(newpacket):
if id_url.to_bin(newpacket.OwnerID) == my_id.getIDURL().to_bin():
# this Data belong to us, SKIP
return False
# if not contactsdb.is_customer(newpacket.OwnerID):
# # SECURITY
# # TODO: process files from another customer : glob_path['idurl']
# lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID))
# # p2p_service.SendFail(newpacket, 'not a customer')
# return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
# p2p_service.SendFail(newpacket, 'incorrect path')
return False
authorized_idurl = verify_packet_ownership(newpacket)
if authorized_idurl is None:
lg.err("ownership verification failed for %r" % newpacket)
# p2p_service.SendFail(newpacket, 'ownership verification failed')
return False
filename = make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
# p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict, _ = accounting.read_customers_quotas()
if newpacket.OwnerID.to_bin() not in list(space_dict.keys()):
lg.err("customer space is broken, no info about donated space for %s" %
newpacket.OwnerID)
p2p_service.SendFail(
newpacket,
'customer space is broken, no info about donated space',
remote_idurl=authorized_idurl)
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID.to_bin() in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID.to_bin()])
bytes_donated_to_customer = int(
space_dict[newpacket.OwnerID.to_bin()])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space left for customer data: %s" %
newpacket.OwnerID)
p2p_service.SendFail(newpacket,
'no free space left for customer data',
remote_idurl=authorized_idurl)
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
# Here Data() packet was stored as it is on supplier node (current machine)
del data
p2p_service.SendAck(newpacket,
response=strng.to_text(len(newpacket.Payload)),
remote_idurl=authorized_idurl)
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
# if self.publish_event_supplier_file_modified: # TODO: must remove that actually
# from main import events
# events.send('supplier-file-modified', data=dict(
# action='write',
# glob_path=glob_path['path'],
# owner_id=newpacket.OwnerID,
# ))
return True
def doTestMyCapacity2(self, arg):
"""
Here are some values.
- donated_bytes : you set this in the config
- spent_bytes : how many space is taken from you by other users right now
- free_bytes = donated_bytes - spent_bytes : not yet allocated space
- used_bytes : size of all files, which you store on your disk for your customers
"""
current_customers = contactsdb.customers()
removed_customers = []
spent_bytes = 0
donated_bytes = settings.getDonatedBytes()
if os.path.isfile(settings.CustomersSpaceFile()):
space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {})
else:
space_dict = {'free': donated_bytes}
used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
lg.out(8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes)
try:
int(space_dict['free'])
for idurl, customer_bytes in space_dict.items():
if idurl != 'free':
spent_bytes += int(customer_bytes)
except:
lg.exc()
space_dict = {'free': donated_bytes}
spent_bytes = 0
removed_customers = list(current_customers)
current_customers = []
self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
return
lg.out(8, ' spent=%d' % spent_bytes)
if spent_bytes < donated_bytes:
space_dict['free'] = donated_bytes - spent_bytes
bpio._write_dict(settings.CustomersSpaceFile(), space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
used_space_ratio_dict = {}
for customer_pos in xrange(contactsdb.num_customers()):
customer_idurl = contactsdb.customer(customer_pos)
try:
allocated_bytes = int(space_dict[customer_idurl])
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated space unknown' % customer_idurl)
continue
if allocated_bytes <= 0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated_bytes==0' % customer_idurl)
continue
try:
files_size = int(used_dict.get(customer_idurl, 0))
ratio = float(files_size) / float(allocated_bytes)
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s used_dict have wrong value' % customer_idurl)
continue
if ratio > 1.0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
spent_bytes -= allocated_bytes
lg.warn('%s space overflow, where is bptester?' % customer_idurl)
continue
used_space_ratio_dict[customer_idurl] = ratio
customers_sorted = sorted(current_customers,
key=lambda i: used_space_ratio_dict[i],)
while len(customers_sorted) > 0:
customer_idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[customer_idurl])
spent_bytes -= allocated_bytes
space_dict.pop(customer_idurl)
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
lg.out(8, ' customer %s REMOVED' % customer_idurl)
if spent_bytes < donated_bytes:
break
space_dict['free'] = donated_bytes - spent_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
def Data(request):
"""
This is when we 1) save my requested data to restore the backup 2) or save
the customer file on our local HDD.
"""
if _Debug:
lg.out(
_DebugLevel, 'p2p_service.Data %d bytes in [%s] by %s | %s' %
(len(request.Payload), request.PacketID, request.OwnerID,
request.CreatorID))
# 1. this is our Data!
if request.OwnerID == my_id.getLocalID():
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data %r for us from %s" %
(request, nameurl.GetName(request.RemoteID)))
if driver.is_on('service_backups'):
# TODO: move this into callback
settings.BackupIndexFileName()
indexPacketID = global_id.MakeGlobalID(
idurl=my_id.getLocalID(), path=settings.BackupIndexFileName())
if request.PacketID == indexPacketID:
from storage import backup_control
backup_control.IncomingSupplierBackupIndex(request)
return True
return False
# 2. this Data is not belong to us
if not driver.is_on('service_supplier'):
return SendFail(request, 'supplier service is off')
if not contactsdb.is_customer(request.OwnerID): # SECURITY
lg.warn("%s not a customer, packetID=%s" %
(request.OwnerID, request.PacketID))
SendFail(request, 'not a customer')
return
glob_path = global_id.ParseGlobalID(request.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' +
request.PacketID)
if not glob_path['path']:
lg.warn("got incorrect PacketID")
SendFail(request, 'incorrect PacketID')
return
# TODO: process files from another customer : glob_path['idurl']
filename = makeFilename(request.OwnerID, glob_path['path'])
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID? ")
SendFail(request, 'empty filename')
return
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.warn("ERROR can not create sub dir " + dirname)
SendFail(request, 'write error')
return
data = request.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': donated_bytes})
if _Debug:
lg.out(_DebugLevel, 'p2p_service.Data created a new space file')
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if request.OwnerID not in space_dict.keys():
lg.warn("no info about donated space for %s" % request.OwnerID)
SendFail(request, 'no info about donated space')
return
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
if request.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(used_space_dict[request.OwnerID])
bytes_donated_to_customer = int(space_dict[request.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space for %s" % request.OwnerID)
SendFail(request, 'no free space')
return
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.warn("ERROR can not write to " + str(filename))
SendFail(request, 'write error')
return
SendAck(request, str(len(request.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
del data
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data saved from [%s | %s] to %s" % (
request.OwnerID,
request.CreatorID,
filename,
))
def check_create_customers_quotas():
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': settings.getDonatedBytes()})
return True
return False
def _on_data(self, newpacket):
import os
from twisted.internet import reactor
from logs import lg
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID): # SECURITY
lg.err("%s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
# TODO: process files from another customer : glob_path['idurl']
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {
'free': donated_bytes,
})
lg.warn('created a new space file: %s' %
settings.CustomersSpaceFile())
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if newpacket.OwnerID not in space_dict.keys():
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(),
{})
if newpacket.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(
self.debug_level,
"service_supplier._on_data %r saved from [%s | %s] to %s with %d bytes"
% (
newpacket,
newpacket.OwnerID,
newpacket.CreatorID,
filename,
sz,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
# temporary disabled
# from main import events
# events.send('supplier-file-modified', data=dict(
# action='write',
# glob_path=glob_path['path'],
# owner_id=newpacket.OwnerID,
# ))
return True
