def start(self):
from logs import lg
from transport import callback
from main import events
from contacts import contactsdb
from storage import accounting
from services import driver
from supplier import customer_space
callback.append_inbox_callback(self._on_inbox_packet_received)
events.add_subscriber(customer_space.on_identity_url_changed, 'identity-url-changed')
events.add_subscriber(customer_space.on_customer_accepted, 'existing-customer-accepted')
events.add_subscriber(customer_space.on_customer_accepted, 'new-customer-accepted')
events.add_subscriber(customer_space.on_customer_terminated, 'existing-customer-denied')
events.add_subscriber(customer_space.on_customer_terminated, 'existing-customer-terminated')
space_dict, _ = accounting.read_customers_quotas()
for customer_idurl in contactsdb.customers():
known_customer_meta_info = contactsdb.get_customer_meta_info(customer_idurl)
events.send('existing-customer-accepted', data=dict(
idurl=customer_idurl,
allocated_bytes=space_dict.get(customer_idurl.to_bin()),
ecc_map=known_customer_meta_info.get('ecc_map'),
position=known_customer_meta_info.get('position'),
))
if driver.is_on('service_entangled_dht'):
self._do_connect_suppliers_dht_layer()
else:
lg.warn('service service_entangled_dht is OFF')
events.add_subscriber(self._on_dht_layer_connected, event_id='dht-layer-connected')
return True
def start(self):
from transport import callback
from main import events
from contacts import contactsdb
from storage import accounting
callback.append_inbox_callback(self._on_inbox_packet_received)
events.add_subscriber(self._on_customer_accepted,
'existing-customer-accepted')
events.add_subscriber(self._on_customer_accepted,
'new-customer-accepted')
events.add_subscriber(self._on_customer_terminated,
'existing-customer-denied')
events.add_subscriber(self._on_customer_terminated,
'existing-customer-terminated')
space_dict = accounting.read_customers_quotas()
for customer_idurl in contactsdb.customers():
known_customer_meta_info = contactsdb.get_customer_meta_info(
customer_idurl)
events.send('existing-customer-accepted',
data=dict(
idurl=customer_idurl,
allocated_bytes=space_dict.get(customer_idurl),
ecc_map=known_customer_meta_info.get('ecc_map'),
position=known_customer_meta_info.get('position'),
))
return True
def cancel(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
customer_idurl = newpacket.OwnerID
if not contactsdb.is_customer(customer_idurl):
lg.warn("got packet from %s, but he is not a customer" % customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.cancel created a new space file')
space_dict = accounting.read_customers_quotas()
if customer_idurl not in list(space_dict.keys()):
lg.warn("got packet from %s, but not found him in space dictionary" % customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
try:
free_bytes = int(space_dict[b'free'])
space_dict[b'free'] = free_bytes + int(space_dict[customer_idurl])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(customer_idurl)
contactsdb.update_customers(new_customers)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.save_customers()
space_dict.pop(customer_idurl)
accounting.write_customers_quotas(space_dict)
from supplier import local_tester
reactor.callLater(0, local_tester.TestUpdateCustomers) # @UndefinedVariable
lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!")
events.send('existing-customer-terminated', dict(idurl=customer_idurl))
return p2p_service.SendAck(newpacket, 'accepted')
def cancel(self, request, info):
from main import events
from p2p import p2p_service
if not contactsdb.is_customer(request.OwnerID):
lg.warn(
"got packet from %s, but he is not a customer" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.cancel created a new space file')
space_dict = accounting.read_customers_quotas()
if request.OwnerID not in space_dict.keys():
lg.warn(
"got packet from %s, but not found him in space dictionary" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
try:
free_bytes = int(space_dict['free'])
space_dict['free'] = free_bytes + int(space_dict[request.OwnerID])
except:
lg.exc()
return p2p_service.SendFail(request, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(request.OwnerID)
contactsdb.update_customers(new_customers)
contactsdb.save_customers()
space_dict.pop(request.OwnerID)
accounting.write_customers_quotas(space_dict)
from supplier import local_tester
reactor.callLater(0, local_tester.TestUpdateCustomers)
lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!")
events.send('existing-customer-terminated', dict(idurl=request.OwnerID))
return p2p_service.SendAck(request, 'accepted')
def cancel(self, request, info):
from p2p import p2p_service
if not contactsdb.is_customer(request.OwnerID):
lg.warn(
"got packet from %s, but he is not a customer" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.cancel created a new space file')
space_dict = accounting.read_customers_quotas()
if request.OwnerID not in space_dict.keys():
lg.warn(
"got packet from %s, but not found him in space dictionary" %
request.OwnerID)
return p2p_service.SendFail(request, 'not a customer')
try:
free_bytes = int(space_dict['free'])
space_dict['free'] = free_bytes + int(space_dict[request.OwnerID])
except:
lg.exc()
return p2p_service.SendFail(request, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(request.OwnerID)
contactsdb.update_customers(new_customers)
contactsdb.save_customers()
space_dict.pop(request.OwnerID)
accounting.write_customers_quotas(space_dict)
from supplier import local_tester
reactor.callLater(0, local_tester.TestUpdateCustomers)
return p2p_service.SendAck(request, 'accepted')
def doTestMyCapacity(self, arg):
"""
Here are some values.
+ donated_bytes : you set this in the configs
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
lg.out(8, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(
space_dict)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
space_dict['free'] = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
if failed_customers:
lg.out(
8, ' found FAILED Customers:\n%s' %
(' \n'.join(failed_customers)))
for idurl in failed_customers:
current_customers.remove(idurl)
self.automat('space-overflow',
(space_dict, consumed_bytes, current_customers,
failed_customers))
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(
space_dict, used_dict)
customers_sorted = sorted(
current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],
)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.out(8, ' customer %s will be REMOVED' % idurl)
space_dict['free'] = donated_bytes - consumed_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat(
'space-overflow',
(space_dict, consumed_bytes, current_customers, failed_customers))
def request(self, request, info):
from p2p import p2p_service
words = request.Payload.split(' ')
try:
bytes_for_customer = int(words[1])
except:
lg.exc()
bytes_for_customer = None
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong storage value : %s" % request.Payload)
return p2p_service.SendFail(request, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.request created a new space file')
space_dict = accounting.read_customers_quotas()
try:
free_bytes = int(space_dict['free'])
except:
lg.exc()
return p2p_service.SendFail(request, 'broken space file')
if (request.OwnerID not in current_customers and request.OwnerID in space_dict.keys()):
lg.warn("broken space file")
return p2p_service.SendFail(request, 'broken space file')
if (request.OwnerID in current_customers and request.OwnerID not in space_dict.keys()):
lg.warn("broken customers file")
return p2p_service.SendFail(request, 'broken customers file')
if request.OwnerID in current_customers:
free_bytes += int(space_dict[request.OwnerID])
space_dict['free'] = free_bytes
current_customers.remove(request.OwnerID)
space_dict.pop(request.OwnerID)
new_customer = False
else:
new_customer = True
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
reactor.callLater(0, local_tester.TestUpdateCustomers)
if new_customer:
lg.out(
8, " NEW CUSTOMER - DENIED !!!!!!!!!!! not enough space")
else:
lg.out(
8, " OLD CUSTOMER - DENIED !!!!!!!!!!! not enough space")
return p2p_service.SendAck(request, 'deny')
space_dict['free'] = free_bytes - bytes_for_customer
current_customers.append(request.OwnerID)
space_dict[request.OwnerID] = bytes_for_customer
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
reactor.callLater(0, local_tester.TestUpdateCustomers)
if new_customer:
lg.out(8, " NEW CUSTOMER ACCEPTED !!!!!!!!!!!!!!")
else:
lg.out(8, " OLD CUSTOMER ACCEPTED !!!!!!!!!!!!!!")
return p2p_service.SendAck(request, 'accepted')
def UpdateCustomers():
"""
Test packets after list of customers was changed.
"""
space, _ = accounting.read_customers_quotas()
if space is None:
printlog('UpdateCustomers ERROR space file can not be read')
return False
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('UpdateCustomers ERROR customers folder not exist')
return False
remove_list = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl.to_bin(), None)
if curspace is None:
remove_list[onecustdir] = 'is not a customer'
continue
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('UpdateCustomers %r folder removed (%s)' % (
path,
remove_list[path],
))
except:
printlog('UpdateCustomers ERROR removing %r' % path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('UpdateCustomers %r file removed (%s)' % (
path,
remove_list[path],
))
except:
printlog('UpdateCustomers ERROR removing %r' % path)
printlog('UpdateCustomers %r' %
time.strftime("%a, %d %b %Y %H:%M:%S +0000"))
return True
def doTestMyCapacity(self, *args, **kwargs):
"""
Here are some values.
+ donated_bytes : you set this in the settings
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
if _Debug:
lg.out(_DebugLevel, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict, free_space = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(
space_dict, free_space)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
free_space = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict, free_space)
lg.info('storage quota checks succeed, all customers are verified')
self.automat('space-enough')
return
if failed_customers:
for idurl in failed_customers:
lg.warn('customer %r failed storage quota verification' %
idurl)
current_customers.remove(idurl)
self.automat('space-overflow', failed_customers)
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(
space_dict, used_dict)
customers_sorted = sorted(
current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],
)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.warn(
'customer %r will be removed because of storage quota overflow'
% idurl)
free_space = donated_bytes - consumed_bytes
self.automat('space-overflow', failed_customers)
def doTestMyCapacity(self, arg):
"""
Here are some values.
+ donated_bytes : you set this in the configs
+ consumed_bytes : how many space was taken from you by other users
+ free_bytes = donated_bytes - consumed_bytes : not yet allocated space
+ used_bytes : size of all files, which you store on your disk for your customers
+ ratio : currently used space compared to consumed space
"""
lg.out(8, 'customers_rejector.doTestMyCapacity')
failed_customers = set()
current_customers = contactsdb.customers()
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
used_dict = accounting.read_customers_usage()
unknown_customers, unused_quotas = accounting.validate_customers_quotas(space_dict)
failed_customers.update(unknown_customers)
for idurl in unknown_customers:
space_dict.pop(idurl, None)
for idurl in unused_quotas:
space_dict.pop(idurl, None)
consumed_bytes = accounting.count_consumed_space(space_dict)
space_dict['free'] = donated_bytes - consumed_bytes
if consumed_bytes < donated_bytes and len(failed_customers) == 0:
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
if failed_customers:
lg.out(8, ' found FAILED Customers:\n%s' % (
' \n'.join(failed_customers)))
for idurl in failed_customers:
current_customers.remove(idurl)
self.automat('space-overflow', (
space_dict, consumed_bytes, current_customers, failed_customers))
return
used_space_ratio_dict = accounting.calculate_customers_usage_ratio(space_dict, used_dict)
customers_sorted = sorted(current_customers,
key=lambda idurl: used_space_ratio_dict[idurl],)
while len(customers_sorted) > 0 and consumed_bytes > donated_bytes:
idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[idurl])
consumed_bytes -= allocated_bytes
space_dict.pop(idurl)
failed_customers.add(idurl)
current_customers.remove(idurl)
lg.out(8, ' customer %s will be REMOVED' % idurl)
space_dict['free'] = donated_bytes - consumed_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat('space-overflow', (
space_dict, consumed_bytes, current_customers, failed_customers))
def cancel(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
from crypt import my_keys
customer_idurl = newpacket.OwnerID
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
customer_ecc_map = json_payload.get('ecc_map')
if not contactsdb.is_customer(customer_idurl):
lg.warn("got packet from %s, but he is not a customer" %
customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
if accounting.check_create_customers_quotas():
lg.info('created a new space file')
space_dict, free_space = accounting.read_customers_quotas()
if customer_idurl.to_bin() not in list(space_dict.keys()):
lg.warn(
"got packet from %s, but not found him in space dictionary" %
customer_idurl)
return p2p_service.SendFail(newpacket, 'not a customer')
try:
free_bytes = int(free_space)
free_space = free_bytes + int(space_dict[customer_idurl.to_bin()])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
new_customers = list(contactsdb.customers())
new_customers.remove(customer_idurl)
space_dict.pop(customer_idurl.to_bin())
accounting.write_customers_quotas(space_dict, free_space)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.update_customers(new_customers)
contactsdb.save_customers()
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
# TODO: erase customer's groups keys also
from supplier import local_tester
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
lg.info("OLD CUSTOMER TERMINATED %r" % customer_idurl)
events.send('existing-customer-terminated',
data=dict(idurl=customer_idurl, ecc_map=customer_ecc_map))
return p2p_service.SendAck(newpacket, 'accepted')
def on_identity_url_changed(evt):
old_idurl = id_url.field(evt.data['old_idurl'])
# update customer idurl in "space" file
contacts_changed = False
for customer_idurl in contactsdb.customers():
if old_idurl == customer_idurl:
customer_idurl.refresh()
contacts_changed = True
lg.info('found customer idurl rotated : %r -> %r' % (
evt.data['old_idurl'],
evt.data['new_idurl'],
))
if contacts_changed:
contactsdb.save_customers()
# update meta info for that customer
meta_info_changed = False
all_meta_info = contactsdb.read_customers_meta_info_all()
for customer_idurl_bin in list(all_meta_info.keys()):
if id_url.is_cached(old_idurl) and id_url.is_cached(
customer_idurl_bin):
if old_idurl == id_url.field(customer_idurl_bin):
latest_customer_idurl_bin = id_url.field(
customer_idurl_bin).to_bin()
if latest_customer_idurl_bin != customer_idurl_bin:
all_meta_info[
latest_customer_idurl_bin] = all_meta_info.pop(
customer_idurl_bin)
meta_info_changed = True
lg.info(
'found customer idurl rotated in customers meta info : %r -> %r'
% (
latest_customer_idurl_bin,
customer_idurl_bin,
))
if meta_info_changed:
contactsdb.write_customers_meta_info_all(all_meta_info)
# update customer idurl in "space" file
space_dict, free_space = accounting.read_customers_quotas()
space_changed = False
for customer_idurl_bin in list(space_dict.keys()):
if id_url.is_cached(old_idurl) and id_url.is_cached(
customer_idurl_bin):
if id_url.field(customer_idurl_bin) == old_idurl:
latest_customer_idurl_bin = id_url.field(
customer_idurl_bin).to_bin()
if latest_customer_idurl_bin != customer_idurl_bin:
space_dict[latest_customer_idurl_bin] = space_dict.pop(
customer_idurl_bin)
space_changed = True
lg.info(
'found customer idurl rotated in customer quotas dictionary : %r -> %r'
% (
latest_customer_idurl_bin,
customer_idurl_bin,
))
if space_changed:
accounting.write_customers_quotas(space_dict, free_space)
# rename customer folder where I store all his files
old_customer_dirname = str(global_id.UrlToGlobalID(evt.data['old_idurl']))
new_customer_dirname = str(global_id.UrlToGlobalID(evt.data['new_idurl']))
customers_dir = settings.getCustomersFilesDir()
old_owner_dir = os.path.join(customers_dir, old_customer_dirname)
new_owner_dir = os.path.join(customers_dir, new_customer_dirname)
if os.path.isdir(old_owner_dir):
try:
bpio.move_dir_recursive(old_owner_dir, new_owner_dir)
lg.info('copied %r into %r' % (
old_owner_dir,
new_owner_dir,
))
if os.path.exists(old_owner_dir):
bpio._dir_remove(old_owner_dir)
lg.warn('removed %r' % old_owner_dir)
except:
lg.exc()
# update customer idurl in "spaceused" file
local_tester.TestSpaceTime()
return True
def on_data(newpacket):
if id_url.to_bin(newpacket.OwnerID) == my_id.getIDURL().to_bin():
# this Data belong to us, SKIP
return False
# if not contactsdb.is_customer(newpacket.OwnerID):
# # SECURITY
# # TODO: process files from another customer : glob_path['idurl']
# lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID))
# # p2p_service.SendFail(newpacket, 'not a customer')
# return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
# p2p_service.SendFail(newpacket, 'incorrect path')
return False
authorized_idurl = verify_packet_ownership(newpacket)
if authorized_idurl is None:
lg.err("ownership verification failed for %r" % newpacket)
# p2p_service.SendFail(newpacket, 'ownership verification failed')
return False
filename = make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
# p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict, _ = accounting.read_customers_quotas()
if newpacket.OwnerID.to_bin() not in list(space_dict.keys()):
lg.err("customer space is broken, no info about donated space for %s" %
newpacket.OwnerID)
p2p_service.SendFail(
newpacket,
'customer space is broken, no info about donated space',
remote_idurl=authorized_idurl)
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID.to_bin() in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID.to_bin()])
bytes_donated_to_customer = int(
space_dict[newpacket.OwnerID.to_bin()])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space left for customer data: %s" %
newpacket.OwnerID)
p2p_service.SendFail(newpacket,
'no free space left for customer data',
remote_idurl=authorized_idurl)
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
# Here Data() packet was stored as it is on supplier node (current machine)
del data
p2p_service.SendAck(newpacket,
response=strng.to_text(len(newpacket.Payload)),
remote_idurl=authorized_idurl)
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
# if self.publish_event_supplier_file_modified: # TODO: must remove that actually
# from main import events
# events.send('supplier-file-modified', data=dict(
# action='write',
# glob_path=glob_path['path'],
# owner_id=newpacket.OwnerID,
# ))
return True
def _on_data(self, newpacket):
import os
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from lib import jsn
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
from storage import accounting
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID):
# SECURITY
# TODO: process files from another customer : glob_path['idurl']
lg.warn("skip, %s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
# p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict = accounting.read_customers_quotas()
if newpacket.OwnerID not in list(space_dict.keys()):
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(self.debug_level, "service_supplier._on_data %r" % newpacket)
lg.out(
self.debug_level, " from [ %s | %s ]" % (
newpacket.OwnerID,
newpacket.CreatorID,
))
lg.out(self.debug_level, " saved with %d bytes to %s" % (
sz,
filename,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
if self.publish_event_supplier_file_modified:
from main import events
events.send('supplier-file-modified',
data=dict(
action='write',
glob_path=glob_path['path'],
owner_id=newpacket.OwnerID,
))
return True
def request(self, request, info):
from main import events
from p2p import p2p_service
words = request.Payload.split(' ')
try:
bytes_for_customer = int(words[1])
except:
lg.exc()
bytes_for_customer = None
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong storage value : %s" % request.Payload)
return p2p_service.SendFail(request, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.request created a new space file')
space_dict = accounting.read_customers_quotas()
try:
free_bytes = int(space_dict['free'])
except:
lg.exc()
return p2p_service.SendFail(request, 'broken space file')
if (request.OwnerID not in current_customers and request.OwnerID in space_dict.keys()):
lg.warn("broken space file")
return p2p_service.SendFail(request, 'broken space file')
if (request.OwnerID in current_customers and request.OwnerID not in space_dict.keys()):
lg.warn("broken customers file")
return p2p_service.SendFail(request, 'broken customers file')
if request.OwnerID in current_customers:
free_bytes += int(space_dict[request.OwnerID])
space_dict['free'] = free_bytes
current_customers.remove(request.OwnerID)
space_dict.pop(request.OwnerID)
new_customer = False
else:
new_customer = True
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
reactor.callLater(0, local_tester.TestUpdateCustomers)
if new_customer:
lg.out(8, " NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available")
events.send('new-customer-denied', dict(idurl=request.OwnerID))
else:
lg.out(8, " OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available")
events.send('existing-customer-denied', dict(idurl=request.OwnerID))
return p2p_service.SendAck(request, 'deny')
space_dict['free'] = free_bytes - bytes_for_customer
current_customers.append(request.OwnerID)
space_dict[request.OwnerID] = bytes_for_customer
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
reactor.callLater(0, local_tester.TestUpdateCustomers)
if new_customer:
lg.out(8, " NEW CUSTOMER: ACCEPTED !!!!!!!!!!!!!!")
events.send('new-customer-accepted', dict(idurl=request.OwnerID))
else:
lg.out(8, " OLD CUSTOMER: ACCEPTED !!!!!!!!!!!!!!")
events.send('existing-customer-accepted', dict(idurl=request.OwnerID))
return p2p_service.SendAck(request, 'accepted')
def request(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from crypt import my_keys
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
from userid import global_id
customer_idurl = newpacket.OwnerID
customer_id = global_id.UrlToGlobalID(customer_idurl)
bytes_for_customer = 0
try:
bytes_for_customer = int(json_payload['needed_bytes'])
except:
lg.warn("wrong payload" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong payload')
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
data_owner_idurl = None
target_customer_idurl = None
family_position = json_payload.get('position')
ecc_map = json_payload.get('ecc_map')
family_snapshot = json_payload.get('family_snapshot')
key_id = json_payload.get('key_id')
target_customer_id = json_payload.get('customer_id')
if key_id:
# this is a request from external user to access shared data stored by one of my customers
# this is "second" customer requesting data from "first" customer
if not key_id or not my_keys.is_valid_key_id(key_id):
lg.warn('missed or invalid key id')
return p2p_service.SendFail(newpacket, 'invalid key id')
target_customer_idurl = global_id.GlobalUserToIDURL(
target_customer_id)
if not contactsdb.is_customer(target_customer_idurl):
lg.warn("target user %s is not a customer" %
target_customer_id)
return p2p_service.SendFail(newpacket, 'not a customer')
if target_customer_idurl == customer_idurl:
lg.warn('customer %s requesting shared access to own files' %
customer_idurl)
return p2p_service.SendFail(newpacket, 'invalid case')
if not my_keys.is_key_registered(key_id):
lg.warn('key not registered: %s' % key_id)
p2p_service.SendFail(newpacket, 'key not registered')
return False
data_owner_idurl = my_keys.split_key_id(key_id)[1]
if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl:
# pretty complex scenario:
# external customer requesting access to data which belongs not to that customer
# this is "third" customer accessing data belongs to "second" customer
# TODO: for now just stop it
lg.warn(
'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s'
% (
key_id,
customer_idurl,
target_customer_idurl,
))
p2p_service.SendFail(newpacket, 'under construction')
return False
# do not create connection with that customer, only accept the request
lg.info(
'external customer %s requested access to shared data at %s' %
(
customer_id,
key_id,
))
return p2p_service.SendAck(newpacket, 'accepted')
# key_id is not present in the request:
# this is a request to connect new customer (or reconnect existing one) to that supplier
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong payload : %s" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.request created a new space file')
space_dict = accounting.read_customers_quotas()
try:
free_bytes = int(space_dict[b'free'])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl not in current_customers
and customer_idurl in list(space_dict.keys())):
lg.warn("broken space file")
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl in current_customers
and customer_idurl not in list(space_dict.keys())):
lg.warn("broken customers file")
return p2p_service.SendFail(newpacket, 'broken customers file')
if customer_idurl in current_customers:
free_bytes += int(space_dict.get(customer_idurl, 0))
space_dict[b'free'] = free_bytes
current_customers.remove(customer_idurl)
space_dict.pop(customer_idurl)
new_customer = False
else:
new_customer = True
lg.out(
8, ' new_customer=%s current_allocated_bytes=%s' % (
new_customer,
space_dict.get(customer_idurl),
))
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.update_customers(current_customers)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('new-customer-denied', dict(idurl=customer_idurl))
else:
lg.out(
8,
" OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('existing-customer-denied',
dict(idurl=customer_idurl))
return p2p_service.SendAck(newpacket, 'deny')
space_dict[b'free'] = free_bytes - bytes_for_customer
current_customers.append(customer_idurl)
space_dict[customer_idurl] = bytes_for_customer
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
contactsdb.add_customer_meta_info(
customer_idurl, {
'ecc_map': ecc_map,
'position': family_position,
'family_snapshot': family_snapshot,
})
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
try:
if not my_keys.is_key_registered(customer_public_key_id):
key_id, key_object = my_keys.read_key_info(
customer_public_key)
if not my_keys.register_key(key_id, key_object):
lg.err('failed to register customer public key')
except:
lg.exc()
else:
lg.warn('customer public key was not provided in the request')
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot,
)
events.send(
'new-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
family_snapshot=family_snapshot,
key_id=customer_public_key_id,
))
else:
lg.out(
8,
" OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot)
events.send(
'existing-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
key_id=customer_public_key_id,
family_snapshot=family_snapshot,
))
return p2p_service.SendAck(newpacket, 'accepted')
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = accounting.read_customers_quotas()
if space is None:
printlog(
'SpaceTime ERROR customers quotas file can not be read or it is empty, skip'
)
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = accounting.read_customers_usage()
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
for key_alias in os.listdir(onecustdir):
if not misc.ValidKeyAlias(key_alias):
remove_list[onecustdir] = 'invalid key alias'
continue
okekeydir = os.path.join(onecustdir, key_alias)
bpio.traverse_dir_recursive(cb, okekeydir)
currentV = 0
for path in sorted(list(timedict.keys()),
key=lambda x: timedict[x],
reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path +
' file removed (cur:%s, max: %s)' %
(str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
accounting.update_customers_usage(used_space)
def doTestMyCapacity2(self, *args, **kwargs):
"""
Here are some values.
- donated_bytes : you set this in the config
- spent_bytes : how many space is taken from you by other users right now
- free_bytes = donated_bytes - spent_bytes : not yet allocated space
- used_bytes : size of all files, which you store on your disk for your customers
"""
current_customers = contactsdb.customers()
removed_customers = []
spent_bytes = 0
donated_bytes = settings.getDonatedBytes()
space_dict = accounting.read_customers_quotas()
if not space_dict:
space_dict = {b'free': donated_bytes}
used_dict = accounting.read_customers_usage()
lg.out(
8,
'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes)
try:
int(space_dict[b'free'])
for idurl, customer_bytes in space_dict.items():
if idurl != b'free':
spent_bytes += int(customer_bytes)
except:
lg.exc()
space_dict = {b'free': donated_bytes}
spent_bytes = 0
removed_customers = list(current_customers)
current_customers = []
self.automat('space-overflow',
(space_dict, spent_bytes, current_customers,
removed_customers))
return
lg.out(8, ' spent=%d' % spent_bytes)
if spent_bytes < donated_bytes:
space_dict[b'free'] = donated_bytes - spent_bytes
accounting.write_customers_quotas(space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
used_space_ratio_dict = {}
for customer_pos in range(contactsdb.num_customers()):
customer_idurl = contactsdb.customer(customer_pos)
try:
allocated_bytes = int(space_dict[customer_idurl])
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated space unknown' % customer_idurl)
continue
if allocated_bytes <= 0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated_bytes==0' % customer_idurl)
continue
try:
files_size = int(used_dict.get(customer_idurl, 0))
ratio = float(files_size) / float(allocated_bytes)
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s used_dict have wrong value' % customer_idurl)
continue
if ratio > 1.0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
spent_bytes -= allocated_bytes
lg.warn('%s space overflow, where is bptester?' %
customer_idurl)
continue
used_space_ratio_dict[customer_idurl] = ratio
customers_sorted = sorted(
current_customers,
key=lambda i: used_space_ratio_dict[i],
)
while len(customers_sorted) > 0:
customer_idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[customer_idurl])
spent_bytes -= allocated_bytes
space_dict.pop(customer_idurl)
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
lg.out(8, ' customer %s REMOVED' % customer_idurl)
if spent_bytes < donated_bytes:
break
space_dict[b'free'] = donated_bytes - spent_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat(
'space-overflow',
(space_dict, spent_bytes, current_customers, removed_customers))
def request(self, json_payload, newpacket, info):
from twisted.internet import reactor
from logs import lg
from main import events
from crypt import my_keys
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
bytes_for_customer = None
try:
bytes_for_customer = json_payload['needed_bytes']
except:
lg.warn("wrong payload" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong payload')
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong payload : %s" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.request created a new space file')
space_dict = accounting.read_customers_quotas()
try:
free_bytes = int(space_dict['free'])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
if (newpacket.OwnerID not in current_customers
and newpacket.OwnerID in space_dict.keys()):
lg.warn("broken space file")
return p2p_service.SendFail(newpacket, 'broken space file')
if (newpacket.OwnerID in current_customers
and newpacket.OwnerID not in space_dict.keys()):
lg.warn("broken customers file")
return p2p_service.SendFail(newpacket, 'broken customers file')
if newpacket.OwnerID in current_customers:
free_bytes += int(space_dict[newpacket.OwnerID])
space_dict['free'] = free_bytes
current_customers.remove(newpacket.OwnerID)
space_dict.pop(newpacket.OwnerID)
new_customer = False
else:
new_customer = True
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
reactor.callLater(0, local_tester.TestUpdateCustomers)
if new_customer:
lg.out(
8,
" NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('new-customer-denied',
dict(idurl=newpacket.OwnerID))
else:
lg.out(
8,
" OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('existing-customer-denied',
dict(idurl=newpacket.OwnerID))
return p2p_service.SendAck(newpacket, 'deny')
space_dict['free'] = free_bytes - bytes_for_customer
current_customers.append(newpacket.OwnerID)
space_dict[newpacket.OwnerID] = bytes_for_customer
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
try:
if not my_keys.is_key_registered(customer_public_key_id):
key_id, key_object = my_keys.read_key_info(
customer_public_key)
if not my_keys.register_key(key_id, key_object):
lg.warn('failed to register customer public key')
except:
lg.exc()
else:
lg.warn('customer public key was not provided in the request')
reactor.callLater(0, local_tester.TestUpdateCustomers)
if new_customer:
lg.out(8, " NEW CUSTOMER: ACCEPTED !!!!!!!!!!!!!!")
events.send('new-customer-accepted', dict(idurl=newpacket.OwnerID))
else:
lg.out(8, " OLD CUSTOMER: ACCEPTED !!!!!!!!!!!!!!")
events.send('existing-customer-accepted',
dict(idurl=newpacket.OwnerID))
return p2p_service.SendAck(newpacket, 'accepted')
