def doTestMyCapacity(self, arg):
"""
Here are some values.
+ donated_bytes : you set this in the configs
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
lg.out(8, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(
space_dict)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
space_dict['free'] = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
if failed_customers:
lg.out(
8, ' found FAILED Customers:\n%s' %
(' \n'.join(failed_customers)))
for idurl in failed_customers:
current_customers.remove(idurl)
self.automat('space-overflow',
(space_dict, consumed_bytes, current_customers,
failed_customers))
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(
space_dict, used_dict)
customers_sorted = sorted(
current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],
)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.out(8, ' customer %s will be REMOVED' % idurl)
space_dict['free'] = donated_bytes - consumed_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat(
'space-overflow',
(space_dict, consumed_bytes, current_customers, failed_customers))
def doTestMyCapacity(self, *args, **kwargs):
"""
Here are some values.
+ donated_bytes : you set this in the settings
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
if _Debug:
lg.out(_DebugLevel, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict, free_space = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(
space_dict, free_space)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
free_space = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict, free_space)
lg.info('storage quota checks succeed, all customers are verified')
self.automat('space-enough')
return
if failed_customers:
for idurl in failed_customers:
lg.warn('customer %r failed storage quota verification' %
idurl)
current_customers.remove(idurl)
self.automat('space-overflow', failed_customers)
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(
space_dict, used_dict)
customers_sorted = sorted(
current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],
)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.warn(
'customer %r will be removed because of storage quota overflow'
% idurl)
free_space = donated_bytes - consumed_bytes
self.automat('space-overflow', failed_customers)
def doTestMyCapacity(self, arg):
"""
Here are some values.
+ donated_bytes : you set this in the configs
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
lg.out(8, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(space_dict)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
space_dict['free'] = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
if failed_customers:
lg.out(8, ' found FAILED Customers:\n%s' % (
' \n'.join(failed_customers)))
for idurl in failed_customers:
current_customers.remove(idurl)
self.automat('space-overflow', (
space_dict, consumed_bytes, current_customers, failed_customers))
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(space_dict, used_dict)
customers_sorted = sorted(current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.out(8, ' customer %s will be REMOVED' % idurl)
space_dict['free'] = donated_bytes - consumed_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat('space-overflow', (
space_dict, consumed_bytes, current_customers, failed_customers))
def on_data(newpacket):
if id_url.to_bin(newpacket.OwnerID) == my_id.getIDURL().to_bin():
# this Data belong to us, SKIP
return False
# if not contactsdb.is_customer(newpacket.OwnerID):
# # SECURITY
# # TODO: process files from another customer : glob_path['idurl']
# lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID))
# # p2p_service.SendFail(newpacket, 'not a customer')
# return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
# p2p_service.SendFail(newpacket, 'incorrect path')
return False
authorized_idurl = verify_packet_ownership(newpacket)
if authorized_idurl is None:
lg.err("ownership verification failed for %r" % newpacket)
# p2p_service.SendFail(newpacket, 'ownership verification failed')
return False
filename = make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
# p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict, _ = accounting.read_customers_quotas()
if newpacket.OwnerID.to_bin() not in list(space_dict.keys()):
lg.err("customer space is broken, no info about donated space for %s" %
newpacket.OwnerID)
p2p_service.SendFail(
newpacket,
'customer space is broken, no info about donated space',
remote_idurl=authorized_idurl)
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID.to_bin() in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID.to_bin()])
bytes_donated_to_customer = int(
space_dict[newpacket.OwnerID.to_bin()])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space left for customer data: %s" %
newpacket.OwnerID)
p2p_service.SendFail(newpacket,
'no free space left for customer data',
remote_idurl=authorized_idurl)
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
# Here Data() packet was stored as it is on supplier node (current machine)
del data
p2p_service.SendAck(newpacket,
response=strng.to_text(len(newpacket.Payload)),
remote_idurl=authorized_idurl)
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
# if self.publish_event_supplier_file_modified: # TODO: must remove that actually
# from main import events
# events.send('supplier-file-modified', data=dict(
# action='write',
# glob_path=glob_path['path'],
# owner_id=newpacket.OwnerID,
# ))
return True
def _on_data(self, newpacket):
import os
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from lib import jsn
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
from storage import accounting
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID):
# SECURITY
# TODO: process files from another customer : glob_path['idurl']
lg.warn("skip, %s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
# p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict = accounting.read_customers_quotas()
if newpacket.OwnerID not in list(space_dict.keys()):
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(self.debug_level, "service_supplier._on_data %r" % newpacket)
lg.out(
self.debug_level, " from [ %s | %s ]" % (
newpacket.OwnerID,
newpacket.CreatorID,
))
lg.out(self.debug_level, " saved with %d bytes to %s" % (
sz,
filename,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
if self.publish_event_supplier_file_modified:
from main import events
events.send('supplier-file-modified',
data=dict(
action='write',
glob_path=glob_path['path'],
owner_id=newpacket.OwnerID,
))
return True
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = accounting.read_customers_quotas()
if space is None:
printlog(
'SpaceTime ERROR customers quotas file can not be read or it is empty, skip'
)
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = accounting.read_customers_usage()
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
for key_alias in os.listdir(onecustdir):
if not misc.ValidKeyAlias(key_alias):
remove_list[onecustdir] = 'invalid key alias'
continue
okekeydir = os.path.join(onecustdir, key_alias)
bpio.traverse_dir_recursive(cb, okekeydir)
currentV = 0
for path in sorted(list(timedict.keys()),
key=lambda x: timedict[x],
reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path +
' file removed (cur:%s, max: %s)' %
(str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
accounting.update_customers_usage(used_space)
def doTestMyCapacity2(self, *args, **kwargs):
"""
Here are some values.
- donated_bytes : you set this in the config
- spent_bytes : how many space is taken from you by other users right now
- free_bytes = donated_bytes - spent_bytes : not yet allocated space
- used_bytes : size of all files, which you store on your disk for your customers
"""
current_customers = contactsdb.customers()
removed_customers = []
spent_bytes = 0
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
if not space_dict:
space_dict = {b'free': donated_bytes}
used_dict = accounting.read_customers_usage()
lg.out(
8,
'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes)
try:
int(space_dict[b'free'])
for idurl, customer_bytes in space_dict.items():
if idurl != b'free':
spent_bytes += int(customer_bytes)
except:
lg.exc()
space_dict = {b'free': donated_bytes}
spent_bytes = 0
removed_customers = list(current_customers)
current_customers = []
self.automat('space-overflow',
(space_dict, spent_bytes, current_customers,
removed_customers))
return
lg.out(8, ' spent=%d' % spent_bytes)
if spent_bytes < donated_bytes:
space_dict[b'free'] = donated_bytes - spent_bytes
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
used_space_ratio_dict = {}
for customer_pos in range(contactsdb.num_customers()):
customer_idurl = contactsdb.customer(customer_pos)
try:
allocated_bytes = int(space_dict[customer_idurl])
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated space unknown' % customer_idurl)
continue
if allocated_bytes <= 0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated_bytes==0' % customer_idurl)
continue
try:
files_size = int(used_dict.get(customer_idurl, 0))
ratio = float(files_size) / float(allocated_bytes)
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s used_dict have wrong value' % customer_idurl)
continue
if ratio > 1.0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
spent_bytes -= allocated_bytes
lg.warn('%s space overflow, where is bptester?' %
customer_idurl)
continue
used_space_ratio_dict[customer_idurl] = ratio
customers_sorted = sorted(
current_customers,
key=lambda i: used_space_ratio_dict[i],
)
while len(customers_sorted) > 0:
customer_idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[customer_idurl])
spent_bytes -= allocated_bytes
space_dict.pop(customer_idurl)
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
lg.out(8, ' customer %s REMOVED' % customer_idurl)
if spent_bytes < donated_bytes:
break
space_dict[b'free'] = donated_bytes - spent_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat(
'space-overflow',
(space_dict, spent_bytes, current_customers, removed_customers))