def validate_customers_quotas(space_dict=None): unknown_customers = set() unused_quotas = set() if space_dict is None: # space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) space_dict = read_customers_quotas() for idurl in list(space_dict.keys()): idurl = strng.to_bin(idurl) try: space_dict[idurl] = int(space_dict[idurl]) except: if idurl != b'free': unknown_customers.add(idurl) continue if idurl != b'free' and space_dict[idurl] <= 0: unknown_customers.add(idurl) continue for idurl in contactsdb.customers(): if idurl not in list(space_dict.keys()): unknown_customers.add(idurl) for idurl in space_dict.keys(): if idurl != b'free': if idurl not in contactsdb.customers(): unused_quotas.add(idurl) return unknown_customers, unused_quotas
def cancel(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from p2p import p2p_service from contacts import contactsdb from storage import accounting customer_idurl = newpacket.OwnerID if not contactsdb.is_customer(customer_idurl): lg.warn("got packet from %s, but he is not a customer" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if customer_idurl not in list(space_dict.keys()): lg.warn("got packet from %s, but not found him in space dictionary" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') try: free_bytes = int(space_dict[b'free']) space_dict[b'free'] = free_bytes + int(space_dict[customer_idurl]) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(customer_idurl) contactsdb.update_customers(new_customers) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.save_customers() space_dict.pop(customer_idurl) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) # @UndefinedVariable lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!") events.send('existing-customer-terminated', dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'accepted')
def ping_customers(timeout=20): from p2p import online_status l = [] for customer_idurl in contactsdb.customers(): if customer_idurl: l.append(online_status.ping(idurl=customer_idurl, ack_timeout=timeout, channel='ping_customers', keep_alive=True)) return DeferredList(l, consumeErrors=True)
def doTestIdleDays(self, *args, **kwargs): """ Action method. """ dead_customers = [] customer_idle_days = config.conf().getInt( 'services/customer-patrol/customer-idle-days', 0) if not customer_idle_days: self.automat('no-idle-customers') return for customer_idurl in contactsdb.customers(): connected_time = ratings.connected_time(customer_idurl.to_bin()) if connected_time is None: lg.warn( 'last connected_time for customer %r is unknown, rejecting customer' % customer_idurl) dead_customers.append(customer_idurl) continue if utime.get_sec1970( ) - connected_time > customer_idle_days * 24 * 60 * 60: lg.warn( 'customer %r connected last time %r seconds ago, rejecting customer' % ( customer_idurl, utime.get_sec1970() - connected_time, )) dead_customers.append(customer_idurl) if dead_customers: lg.warn('found idle customers: %r' % dead_customers) self.automat('found-idle-customers', dead_customers) else: lg.info( 'all customers has some activity recently, no idle customers found' ) self.automat('no-idle-customers')
def customers(AckHandler=None, wide=False): """ Call ``propagate()`` for all known customers. """ if _Debug: lg.out(_DebugLevel, 'propagate.customers') return propagate(contactsdb.customers(), AckHandler, wide)
def start(self): from logs import lg from transport import callback from main import events from contacts import contactsdb from storage import accounting from services import driver from supplier import customer_space callback.append_inbox_callback(self._on_inbox_packet_received) events.add_subscriber(customer_space.on_identity_url_changed, 'identity-url-changed') events.add_subscriber(customer_space.on_customer_accepted, 'existing-customer-accepted') events.add_subscriber(customer_space.on_customer_accepted, 'new-customer-accepted') events.add_subscriber(customer_space.on_customer_terminated, 'existing-customer-denied') events.add_subscriber(customer_space.on_customer_terminated, 'existing-customer-terminated') space_dict, _ = accounting.read_customers_quotas() for customer_idurl in contactsdb.customers(): known_customer_meta_info = contactsdb.get_customer_meta_info(customer_idurl) events.send('existing-customer-accepted', data=dict( idurl=customer_idurl, allocated_bytes=space_dict.get(customer_idurl.to_bin()), ecc_map=known_customer_meta_info.get('ecc_map'), position=known_customer_meta_info.get('position'), )) if driver.is_on('service_entangled_dht'): self._do_connect_suppliers_dht_layer() else: lg.warn('service service_entangled_dht is OFF') events.add_subscriber(self._on_dht_layer_connected, event_id='dht-layer-connected') return True
def start(self): from logs import lg from main import events from contacts import contactsdb from supplier import family_member from transport import callback # TODO: check all imports.! my_id must be loaded latest as possible! from userid import my_id callback.append_inbox_callback(self._on_inbox_packet_received) for customer_idurl in contactsdb.customers(): if not customer_idurl: continue if customer_idurl == my_id.getLocalIDURL(): lg.warn('skipping my own identity') continue fm = family_member.by_customer_idurl(customer_idurl) if not fm: fm = family_member.create_family(customer_idurl) fm.automat('init') local_customer_meta_info = contactsdb.get_customer_meta_info(customer_idurl) fm.automat('family-join', { 'supplier_idurl': my_id.getLocalIDURL(), 'ecc_map': local_customer_meta_info.get('ecc_map'), 'position': local_customer_meta_info.get('position', -1), 'family_snapshot': local_customer_meta_info.get('family_snapshot'), }) events.add_subscriber(self._on_existing_customer_accepted, 'existing-customer-accepted') events.add_subscriber(self._on_new_customer_accepted, 'new-customer-accepted') events.add_subscriber(self._on_existing_customer_terminated, 'existing-customer-terminated') return True
def start(self): from transport import callback from main import events from contacts import contactsdb from storage import accounting callback.append_inbox_callback(self._on_inbox_packet_received) events.add_subscriber(self._on_customer_accepted, 'existing-customer-accepted') events.add_subscriber(self._on_customer_accepted, 'new-customer-accepted') events.add_subscriber(self._on_customer_terminated, 'existing-customer-denied') events.add_subscriber(self._on_customer_terminated, 'existing-customer-terminated') space_dict = accounting.read_customers_quotas() for customer_idurl in contactsdb.customers(): known_customer_meta_info = contactsdb.get_customer_meta_info( customer_idurl) events.send('existing-customer-accepted', data=dict( idurl=customer_idurl, allocated_bytes=space_dict.get(customer_idurl), ecc_map=known_customer_meta_info.get('ecc_map'), position=known_customer_meta_info.get('position'), )) return True
def cancel(self, request, info): from p2p import p2p_service if not contactsdb.is_customer(request.OwnerID): lg.warn( "got packet from %s, but he is not a customer" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if request.OwnerID not in space_dict.keys(): lg.warn( "got packet from %s, but not found him in space dictionary" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') try: free_bytes = int(space_dict['free']) space_dict['free'] = free_bytes + int(space_dict[request.OwnerID]) except: lg.exc() return p2p_service.SendFail(request, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(request.OwnerID) contactsdb.update_customers(new_customers) contactsdb.save_customers() space_dict.pop(request.OwnerID) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) return p2p_service.SendAck(request, 'accepted')
def SendCustomers(): """ Calls ``SendToIDs()`` to send identity to all my customers. """ if _Debug: lg.out(_DebugLevel, "propagate.SendCustomers") SendToIDs(contactsdb.customers(), ack_handler=HandleCustomersAck, wide=True)
def cancel(self, request, info): from main import events from p2p import p2p_service if not contactsdb.is_customer(request.OwnerID): lg.warn( "got packet from %s, but he is not a customer" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if request.OwnerID not in space_dict.keys(): lg.warn( "got packet from %s, but not found him in space dictionary" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') try: free_bytes = int(space_dict['free']) space_dict['free'] = free_bytes + int(space_dict[request.OwnerID]) except: lg.exc() return p2p_service.SendFail(request, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(request.OwnerID) contactsdb.update_customers(new_customers) contactsdb.save_customers() space_dict.pop(request.OwnerID) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!") events.send('existing-customer-terminated', dict(idurl=request.OwnerID)) return p2p_service.SendAck(request, 'accepted')
def doTestMyCapacity(self, arg): """ Here are some values. + donated_bytes : you set this in the configs + consumed_bytes : how many space was taken from you by other users + free_bytes = donated_bytes - consumed_bytes : not yet allocated space + used_bytes : size of all files, which you store on your disk for your customers + ratio : currently used space compared to consumed space """ lg.out(8, 'customers_rejector.doTestMyCapacity') failed_customers = set() current_customers = contactsdb.customers() donated_bytes = settings.getDonatedBytes() space_dict = accounting.read_customers_quotas() used_dict = accounting.read_customers_usage() unknown_customers, unused_quotas = accounting.validate_customers_quotas( space_dict) failed_customers.update(unknown_customers) for idurl in unknown_customers: space_dict.pop(idurl, None) for idurl in unused_quotas: space_dict.pop(idurl, None) consumed_bytes = accounting.count_consumed_space(space_dict) space_dict['free'] = donated_bytes - consumed_bytes if consumed_bytes < donated_bytes and len(failed_customers) == 0: accounting.write_customers_quotas(space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return if failed_customers: lg.out( 8, ' found FAILED Customers:\n%s' % (' \n'.join(failed_customers))) for idurl in failed_customers: current_customers.remove(idurl) self.automat('space-overflow', (space_dict, consumed_bytes, current_customers, failed_customers)) return used_space_ratio_dict = accounting.calculate_customers_usage_ratio( space_dict, used_dict) customers_sorted = sorted( current_customers, key=lambda idurl: used_space_ratio_dict[idurl], ) while len(customers_sorted) > 0 and consumed_bytes > donated_bytes: idurl = customers_sorted.pop() allocated_bytes = int(space_dict[idurl]) consumed_bytes -= allocated_bytes space_dict.pop(idurl) failed_customers.add(idurl) current_customers.remove(idurl) lg.out(8, ' customer %s will be REMOVED' % idurl) space_dict['free'] = donated_bytes - consumed_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat( 'space-overflow', (space_dict, consumed_bytes, current_customers, failed_customers))
def request(self, request, info): from p2p import p2p_service words = request.Payload.split(' ') try: bytes_for_customer = int(words[1]) except: lg.exc() bytes_for_customer = None if not bytes_for_customer or bytes_for_customer < 0: lg.warn("wrong storage value : %s" % request.Payload) return p2p_service.SendFail(request, 'wrong storage value') current_customers = contactsdb.customers() if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.request created a new space file') space_dict = accounting.read_customers_quotas() try: free_bytes = int(space_dict['free']) except: lg.exc() return p2p_service.SendFail(request, 'broken space file') if (request.OwnerID not in current_customers and request.OwnerID in space_dict.keys()): lg.warn("broken space file") return p2p_service.SendFail(request, 'broken space file') if (request.OwnerID in current_customers and request.OwnerID not in space_dict.keys()): lg.warn("broken customers file") return p2p_service.SendFail(request, 'broken customers file') if request.OwnerID in current_customers: free_bytes += int(space_dict[request.OwnerID]) space_dict['free'] = free_bytes current_customers.remove(request.OwnerID) space_dict.pop(request.OwnerID) new_customer = False else: new_customer = True from supplier import local_tester if free_bytes <= bytes_for_customer: contactsdb.update_customers(current_customers) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) reactor.callLater(0, local_tester.TestUpdateCustomers) if new_customer: lg.out( 8, " NEW CUSTOMER - DENIED !!!!!!!!!!! not enough space") else: lg.out( 8, " OLD CUSTOMER - DENIED !!!!!!!!!!! not enough space") return p2p_service.SendAck(request, 'deny') space_dict['free'] = free_bytes - bytes_for_customer current_customers.append(request.OwnerID) space_dict[request.OwnerID] = bytes_for_customer contactsdb.update_customers(current_customers) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) reactor.callLater(0, local_tester.TestUpdateCustomers) if new_customer: lg.out(8, " NEW CUSTOMER ACCEPTED !!!!!!!!!!!!!!") else: lg.out(8, " OLD CUSTOMER ACCEPTED !!!!!!!!!!!!!!") return p2p_service.SendAck(request, 'accepted')
def start(self): from main import events from contacts import contactsdb from coins import supplier_contract_executor for customer_idurl in contactsdb.customers(): supplier_contract_executor.init_contract(customer_idurl) events.add_subscriber(self._on_customer_modified, 'customer-modified') return True
def doTestMyCapacity(self, *args, **kwargs): """ Here are some values. + donated_bytes : you set this in the settings + consumed_bytes : how many space was taken from you by other users + free_bytes = donated_bytes - consumed_bytes : not yet allocated space + used_bytes : size of all files, which you store on your disk for your customers + ratio : currently used space compared to consumed space """ if _Debug: lg.out(_DebugLevel, 'customers_rejector.doTestMyCapacity') failed_customers = set() current_customers = contactsdb.customers() donated_bytes = settings.getDonatedBytes() space_dict, free_space = accounting.read_customers_quotas() used_dict = accounting.read_customers_usage() unknown_customers, unused_quotas = accounting.validate_customers_quotas( space_dict, free_space) failed_customers.update(unknown_customers) for idurl in unknown_customers: space_dict.pop(idurl, None) for idurl in unused_quotas: space_dict.pop(idurl, None) consumed_bytes = accounting.count_consumed_space(space_dict) free_space = donated_bytes - consumed_bytes if consumed_bytes < donated_bytes and len(failed_customers) == 0: accounting.write_customers_quotas(space_dict, free_space) lg.info('storage quota checks succeed, all customers are verified') self.automat('space-enough') return if failed_customers: for idurl in failed_customers: lg.warn('customer %r failed storage quota verification' % idurl) current_customers.remove(idurl) self.automat('space-overflow', failed_customers) return used_space_ratio_dict = accounting.calculate_customers_usage_ratio( space_dict, used_dict) customers_sorted = sorted( current_customers, key=lambda idurl: used_space_ratio_dict[idurl], ) while len(customers_sorted) > 0 and consumed_bytes > donated_bytes: idurl = customers_sorted.pop() allocated_bytes = int(space_dict[idurl]) consumed_bytes -= allocated_bytes space_dict.pop(idurl) failed_customers.add(idurl) current_customers.remove(idurl) lg.warn( 'customer %r will be removed because of storage quota overflow' % idurl) free_space = donated_bytes - consumed_bytes self.automat('space-overflow', failed_customers)
def start(self): from supplier import customer_assistant from contacts import contactsdb from transport import callback for customer_idurl in contactsdb.customers(): if customer_idurl and not customer_assistant.by_idurl(customer_idurl): ca = customer_assistant.create(customer_idurl) ca.automat('init') callback.add_outbox_callback(self._on_outbox_packet_sent) callback.append_inbox_callback(self._on_inbox_packet_received) return True
def listOfflineCustomers(): """ Loops all customers and check their state, return a list of those with state OFFLINE. """ result = [] for idurl in contactsdb.customers(): if not idurl: result.append(idurl) elif isOffline(idurl): result.append(idurl) return result
def customers(ack_handler=None, wide=False, wait_packets=False): """ Call ``propagate()`` for all known customers. """ if _Debug: lg.out(_DebugLevel, 'propagate.customers') return propagate( selected_contacts=contactsdb.customers(), ack_handler=ack_handler, wide=wide, wait_packets=wait_packets, )
def doTestMyCapacity(self, arg): """ Here are some values. + donated_bytes : you set this in the configs + consumed_bytes : how many space was taken from you by other users + free_bytes = donated_bytes - consumed_bytes : not yet allocated space + used_bytes : size of all files, which you store on your disk for your customers + ratio : currently used space compared to consumed space """ lg.out(8, 'customers_rejector.doTestMyCapacity') failed_customers = set() current_customers = contactsdb.customers() donated_bytes = settings.getDonatedBytes() space_dict = accounting.read_customers_quotas() used_dict = accounting.read_customers_usage() unknown_customers, unused_quotas = accounting.validate_customers_quotas(space_dict) failed_customers.update(unknown_customers) for idurl in unknown_customers: space_dict.pop(idurl, None) for idurl in unused_quotas: space_dict.pop(idurl, None) consumed_bytes = accounting.count_consumed_space(space_dict) space_dict['free'] = donated_bytes - consumed_bytes if consumed_bytes < donated_bytes and len(failed_customers) == 0: accounting.write_customers_quotas(space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return if failed_customers: lg.out(8, ' found FAILED Customers:\n%s' % ( ' \n'.join(failed_customers))) for idurl in failed_customers: current_customers.remove(idurl) self.automat('space-overflow', ( space_dict, consumed_bytes, current_customers, failed_customers)) return used_space_ratio_dict = accounting.calculate_customers_usage_ratio(space_dict, used_dict) customers_sorted = sorted(current_customers, key=lambda idurl: used_space_ratio_dict[idurl],) while len(customers_sorted) > 0 and consumed_bytes > donated_bytes: idurl = customers_sorted.pop() allocated_bytes = int(space_dict[idurl]) consumed_bytes -= allocated_bytes space_dict.pop(idurl) failed_customers.add(idurl) current_customers.remove(idurl) lg.out(8, ' customer %s will be REMOVED' % idurl) space_dict['free'] = donated_bytes - consumed_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat('space-overflow', ( space_dict, consumed_bytes, current_customers, failed_customers))
def validate_customers_quotas(space_dict=None, free_space=None): unknown_customers = set() unused_quotas = set() if space_dict is None or free_space is None: space_dict, free_space = read_customers_quotas() for idurl in list(space_dict.keys()): idurl = strng.to_bin(idurl) try: space_dict[idurl] = int(space_dict[idurl]) except: unknown_customers.add(idurl) continue if space_dict[idurl] <= 0: unknown_customers.add(idurl) continue for idurl in contactsdb.customers(): if idurl.to_bin() not in list(space_dict.keys()): unknown_customers.add(idurl) for idurl in space_dict.keys(): if idurl not in id_url.to_bin_list(contactsdb.customers()): unused_quotas.add(idurl) return unknown_customers, unused_quotas
def cancel(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from p2p import p2p_service from contacts import contactsdb from storage import accounting from crypt import my_keys customer_idurl = newpacket.OwnerID try: customer_public_key = json_payload['customer_public_key'] customer_public_key_id = customer_public_key['key_id'] except: customer_public_key = None customer_public_key_id = None customer_ecc_map = json_payload.get('ecc_map') if not contactsdb.is_customer(customer_idurl): lg.warn("got packet from %s, but he is not a customer" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') if accounting.check_create_customers_quotas(): lg.info('created a new space file') space_dict, free_space = accounting.read_customers_quotas() if customer_idurl.to_bin() not in list(space_dict.keys()): lg.warn( "got packet from %s, but not found him in space dictionary" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') try: free_bytes = int(free_space) free_space = free_bytes + int(space_dict[customer_idurl.to_bin()]) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(customer_idurl) space_dict.pop(customer_idurl.to_bin()) accounting.write_customers_quotas(space_dict, free_space) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.update_customers(new_customers) contactsdb.save_customers() if customer_public_key_id: my_keys.erase_key(customer_public_key_id) # TODO: erase customer's groups keys also from supplier import local_tester reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable lg.info("OLD CUSTOMER TERMINATED %r" % customer_idurl) events.send('existing-customer-terminated', data=dict(idurl=customer_idurl, ecc_map=customer_ecc_map)) return p2p_service.SendAck(newpacket, 'accepted')
def validate_customers_quotas(space_dict=None): unknown_customers = set() unused_quotas = set() if space_dict is None: space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) for idurl in list(space_dict.keys()): try: space_dict[idurl] = int(space_dict[idurl]) except: if idurl != 'free': unknown_customers.add(idurl) continue if idurl != 'free' and space_dict[idurl] <= 0: unknown_customers.add(idurl) continue for idurl in contactsdb.customers(): if idurl not in space_dict.keys(): unknown_customers.add(idurl) for idurl in space_dict.keys(): if idurl != 'free': if idurl not in contactsdb.customers(): unused_quotas.add(idurl) return unknown_customers, unused_quotas
def start(self): from dht import dht_relations from contacts import contactsdb from userid import my_id from main import events events.add_subscriber(self._on_new_customer_accepted, 'new-customer-accepted') events.add_subscriber(self._on_existing_customer_accepted, 'existing-customer-accepted') events.add_subscriber(self._on_existing_customer_terminated, 'existing-customer-terminated') for customer_idurl in contactsdb.customers(): dht_relations.publish_customer_supplier_relation(customer_idurl) dht_relations.scan_customer_supplier_relations(my_id.getLocalID()) return True
def stop(self): from transport import callback from main import events from contacts import contactsdb for customer_idurl in contactsdb.customers(): events.send('existing-customer-terminated', data=dict(idurl=customer_idurl)) events.remove_subscriber(self._on_customer_accepted, 'existing-customer-accepted') events.remove_subscriber(self._on_customer_accepted, 'new-customer-accepted') events.remove_subscriber(self._on_customer_terminated, 'existing-customer-denied') events.remove_subscriber(self._on_customer_terminated, 'existing-customer-terminated') callback.remove_inbox_callback(self._on_inbox_packet_received) return True
def start(self): from twisted.internet import reactor # @UnresolvedImport from userid import id_url from main import events from supplier import customer_assistant from contacts import contactsdb from transport import callback for customer_idurl in contactsdb.customers(): if id_url.is_cached(customer_idurl): if customer_idurl and not customer_assistant.by_idurl( customer_idurl): ca = customer_assistant.create(customer_idurl) reactor.callLater(0, ca.automat, 'init') # @UndefinedVariable events.add_subscriber(self._on_identity_url_changed, 'identity-url-changed') callback.add_outbox_callback(self._on_outbox_packet_sent) callback.append_inbox_callback(self._on_inbox_packet_received) return True
def calculate_customers_usage_ratio(space_dict=None, used_dict=None): if space_dict is None: space_dict = read_customers_quotas() if used_dict is None: used_dict = read_customers_usage() current_customers = contactsdb.customers() used_space_ratio_dict = {} for idurl in current_customers: allocated_bytes = int(space_dict[idurl]) try: files_size = int(used_dict.get(idurl, 0)) except: files_size = 0 try: ratio = float(files_size) / float(allocated_bytes) except: continue used_space_ratio_dict[idurl] = ratio return used_space_ratio_dict
def start(self): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from contacts import contactsdb from userid import id_url from supplier import family_member from transport import callback from userid import my_id callback.append_inbox_callback(self._on_inbox_packet_received) for customer_idurl in contactsdb.customers(): if not customer_idurl: continue if not id_url.is_cached(customer_idurl): continue if customer_idurl == my_id.getLocalID(): lg.warn('skipping my own identity') continue fm = family_member.by_customer_idurl(customer_idurl) if not fm: fm = family_member.create_family(customer_idurl) fm.automat('init') local_customer_meta_info = contactsdb.get_customer_meta_info( customer_idurl) reactor.callLater(0, fm.automat, 'family-join', { # @UndefinedVariable 'supplier_idurl': my_id.getLocalID().to_bin(), 'ecc_map': local_customer_meta_info.get('ecc_map'), 'position': local_customer_meta_info.get('position', -1), 'family_snapshot': id_url.to_bin_list(local_customer_meta_info.get('family_snapshot')), }) events.add_subscriber(self._on_identity_url_changed, 'identity-url-changed') events.add_subscriber(self._on_existing_customer_accepted, 'existing-customer-accepted') events.add_subscriber(self._on_new_customer_accepted, 'new-customer-accepted') events.add_subscriber(self._on_existing_customer_terminated, 'existing-customer-terminated') return True
def report_donated_storage(): space_dict = read_customers_quotas() used_space_dict = read_customers_usage() r = {} r['customers_num'] = contactsdb.num_customers() r['customers'] = [] r['old_customers'] = [] r['errors'] = [] r['consumed'] = 0 r['donated'] = settings.getDonatedBytes() r['donated_str'] = diskspace.MakeStringFromBytes(r['donated']) r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir()) try: r['free'] = int(space_dict.pop('free')) except: r['free'] = 0 used = 0 for idurl in contactsdb.customers(): consumed_by_customer = 0 used_by_customer = 0 if idurl not in space_dict.keys(): r['errors'].append('space consumed by customer %s is unknown' % idurl) else: try: consumed_by_customer = int(space_dict.pop(idurl)) r['consumed'] += consumed_by_customer except: r['errors'].append('incorrect value of consumed space for customer %s' % idurl) if idurl in used_space_dict.keys(): try: used_by_customer = int(used_space_dict.pop(idurl)) used += used_by_customer except: r['errors'].append('incorrect value of used space for customer %s' % idurl) if consumed_by_customer < used_by_customer: r['errors'].append('customer %s currently using more space than requested' % idurl) c = {} c['idurl'] = idurl c['used'] = used_by_customer c['used_str'] = diskspace.MakeStringFromBytes(c['used']) c['consumed'] = consumed_by_customer c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed']) c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) c['real_str'] = diskspace.MakeStringFromBytes(c['real']) r['customers'].append(c) r['used'] = used r['used_str'] = diskspace.MakeStringFromBytes(r['used']) r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed']) if r['donated'] != r['free'] + r['consumed']: r['errors'].append('total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % ( r['consumed'], r['free'], r['consumed'] + r['free'], r['donated'])) if r['used'] > r['donated']: r['errors'].append('total space used by customers exceed the donated limit') if len(space_dict) > 0: r['errors'].append('found %d incorrect records of consumed space' % len(space_dict)) if r['real'] != r['used']: r['errors'].append('current info needs update, known size is %d bytes but real is %d bytes' % ( r['used'], r['real'])) for idurl in used_space_dict.keys(): real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) r['old_customers'].append({ 'idurl': idurl, 'used': used_space_dict['idurl'], 'used_str': diskspace.MakeStringFromBytes(used_space_dict['idurl']), 'real': real, 'real_str': diskspace.MakeStringFromBytes(real), }) try: r['used_percent'] = misc.percent2string(float(r['used']) / float(r['donated']), 5) except: r['used_percent'] = '' try: r['consumed_percent'] = misc.percent2string(float(r['consumed']) / float(r['donated']), 5) except: r['consumed_percent'] = '' return r
def report_donated_storage(): space_dict, free_space = read_customers_quotas() used_space_dict = read_customers_usage() r = {} r['customers_num'] = contactsdb.num_customers() r['customers'] = [] r['old_customers'] = [] r['errors'] = [] r['consumed'] = 0 r['donated'] = settings.getDonatedBytes() # r['donated_str'] = diskspace.MakeStringFromBytes(r['donated']) r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir()) try: r['free'] = int(free_space) except: r['free'] = 0 used = 0 for idurl in id_url.to_bin_list(contactsdb.customers()): consumed_by_customer = 0 used_by_customer = 0 if idurl not in list(space_dict.keys()): r['errors'].append('space consumed by customer %r is unknown' % idurl) else: try: consumed_by_customer = int(space_dict.pop(idurl)) r['consumed'] += consumed_by_customer except: r['errors'].append( 'incorrect value of consumed space for customer %r' % idurl) continue if idurl in list(used_space_dict.keys()): try: used_by_customer = int(used_space_dict.pop(idurl)) used += used_by_customer except: r['errors'].append( 'incorrect value of used space for customer %r' % idurl) continue if consumed_by_customer < used_by_customer: r['errors'].append( 'customer %r currently using more space than requested' % idurl) c = {} c['idurl'] = strng.to_text(idurl) c['used'] = used_by_customer # c['used_str'] = diskspace.MakeStringFromBytes(c['used']) c['consumed'] = consumed_by_customer # c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed']) c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) # c['real_str'] = diskspace.MakeStringFromBytes(c['real']) r['customers'].append(c) r['used'] = used # r['used_str'] = diskspace.MakeStringFromBytes(r['used']) # r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed']) if r['donated'] != r['free'] + r['consumed']: r['errors'].append( 'total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % (r['consumed'], r['free'], r['consumed'] + r['free'], r['donated'])) if r['used'] > r['donated']: r['errors'].append( 'total space used by customers exceed the donated limit') if len(space_dict) > 0: r['errors'].append('found %d incorrect records of consumed space' % len(space_dict)) if r['real'] != r['used']: r['errors'].append( 'current info needs update, known size is %d bytes but real is %d bytes' % (r['used'], r['real'])) old_customers_used = 0 old_customers_real = 0 for idurl in used_space_dict.keys(): real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) try: used = int(used_space_dict[idurl]) except: r['errors'].append( 'incorrect value of used space for customer %r' % idurl) continue r['old_customers'].append({ 'idurl': strng.to_text(idurl), 'used': used, # 'used_str': diskspace.MakeStringFromBytes(used_space_dict[idurl]), 'real': real, # 'real_str': diskspace.MakeStringFromBytes(real), }) old_customers_used += used old_customers_real += real r['old_customers_used'] = old_customers_used r['old_customers_real'] = old_customers_real try: r['used_percent'] = misc.value2percent(float(r['used']), float(r['donated']), 5) except: r['used_percent'] = '' try: r['consumed_percent'] = misc.value2percent(float(r['consumed']), float(r['donated']), 5) except: r['consumed_percent'] = '' return r
def on_identity_url_changed(evt): old_idurl = id_url.field(evt.data['old_idurl']) # update customer idurl in "space" file contacts_changed = False for customer_idurl in contactsdb.customers(): if old_idurl == customer_idurl: customer_idurl.refresh() contacts_changed = True lg.info('found customer idurl rotated : %r -> %r' % ( evt.data['old_idurl'], evt.data['new_idurl'], )) if contacts_changed: contactsdb.save_customers() # update meta info for that customer meta_info_changed = False all_meta_info = contactsdb.read_customers_meta_info_all() for customer_idurl_bin in list(all_meta_info.keys()): if id_url.is_cached(old_idurl) and id_url.is_cached( customer_idurl_bin): if old_idurl == id_url.field(customer_idurl_bin): latest_customer_idurl_bin = id_url.field( customer_idurl_bin).to_bin() if latest_customer_idurl_bin != customer_idurl_bin: all_meta_info[ latest_customer_idurl_bin] = all_meta_info.pop( customer_idurl_bin) meta_info_changed = True lg.info( 'found customer idurl rotated in customers meta info : %r -> %r' % ( latest_customer_idurl_bin, customer_idurl_bin, )) if meta_info_changed: contactsdb.write_customers_meta_info_all(all_meta_info) # update customer idurl in "space" file space_dict, free_space = accounting.read_customers_quotas() space_changed = False for customer_idurl_bin in list(space_dict.keys()): if id_url.is_cached(old_idurl) and id_url.is_cached( customer_idurl_bin): if id_url.field(customer_idurl_bin) == old_idurl: latest_customer_idurl_bin = id_url.field( customer_idurl_bin).to_bin() if latest_customer_idurl_bin != customer_idurl_bin: space_dict[latest_customer_idurl_bin] = space_dict.pop( customer_idurl_bin) space_changed = True lg.info( 'found customer idurl rotated in customer quotas dictionary : %r -> %r' % ( latest_customer_idurl_bin, customer_idurl_bin, )) if space_changed: accounting.write_customers_quotas(space_dict, free_space) # rename customer folder where I store all his files old_customer_dirname = str(global_id.UrlToGlobalID(evt.data['old_idurl'])) new_customer_dirname = str(global_id.UrlToGlobalID(evt.data['new_idurl'])) customers_dir = settings.getCustomersFilesDir() old_owner_dir = os.path.join(customers_dir, old_customer_dirname) new_owner_dir = os.path.join(customers_dir, new_customer_dirname) if os.path.isdir(old_owner_dir): try: bpio.move_dir_recursive(old_owner_dir, new_owner_dir) lg.info('copied %r into %r' % ( old_owner_dir, new_owner_dir, )) if os.path.exists(old_owner_dir): bpio._dir_remove(old_owner_dir) lg.warn('removed %r' % old_owner_dir) except: lg.exc() # update customer idurl in "spaceused" file local_tester.TestSpaceTime() return True
def request(self, json_payload, newpacket, info): from twisted.internet import reactor # @UnresolvedImport from logs import lg from main import events from crypt import my_keys from p2p import p2p_service from contacts import contactsdb from storage import accounting from userid import global_id customer_idurl = newpacket.OwnerID customer_id = global_id.UrlToGlobalID(customer_idurl) bytes_for_customer = 0 try: bytes_for_customer = int(json_payload['needed_bytes']) except: lg.warn("wrong payload" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong payload') try: customer_public_key = json_payload['customer_public_key'] customer_public_key_id = customer_public_key['key_id'] except: customer_public_key = None customer_public_key_id = None data_owner_idurl = None target_customer_idurl = None family_position = json_payload.get('position') ecc_map = json_payload.get('ecc_map') family_snapshot = json_payload.get('family_snapshot') key_id = json_payload.get('key_id') target_customer_id = json_payload.get('customer_id') if key_id: # this is a request from external user to access shared data stored by one of my customers # this is "second" customer requesting data from "first" customer if not key_id or not my_keys.is_valid_key_id(key_id): lg.warn('missed or invalid key id') return p2p_service.SendFail(newpacket, 'invalid key id') target_customer_idurl = global_id.GlobalUserToIDURL( target_customer_id) if not contactsdb.is_customer(target_customer_idurl): lg.warn("target user %s is not a customer" % target_customer_id) return p2p_service.SendFail(newpacket, 'not a customer') if target_customer_idurl == customer_idurl: lg.warn('customer %s requesting shared access to own files' % customer_idurl) return p2p_service.SendFail(newpacket, 'invalid case') if not my_keys.is_key_registered(key_id): lg.warn('key not registered: %s' % key_id) p2p_service.SendFail(newpacket, 'key not registered') return False data_owner_idurl = my_keys.split_key_id(key_id)[1] if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl: # pretty complex scenario: # external customer requesting access to data which belongs not to that customer # this is "third" customer accessing data belongs to "second" customer # TODO: for now just stop it lg.warn( 'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s' % ( key_id, customer_idurl, target_customer_idurl, )) p2p_service.SendFail(newpacket, 'under construction') return False # do not create connection with that customer, only accept the request lg.info( 'external customer %s requested access to shared data at %s' % ( customer_id, key_id, )) return p2p_service.SendAck(newpacket, 'accepted') # key_id is not present in the request: # this is a request to connect new customer (or reconnect existing one) to that supplier if not bytes_for_customer or bytes_for_customer < 0: lg.warn("wrong payload : %s" % newpacket.Payload) return p2p_service.SendFail(newpacket, 'wrong storage value') current_customers = contactsdb.customers() if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.request created a new space file') space_dict = accounting.read_customers_quotas() try: free_bytes = int(space_dict[b'free']) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl not in current_customers and customer_idurl in list(space_dict.keys())): lg.warn("broken space file") return p2p_service.SendFail(newpacket, 'broken space file') if (customer_idurl in current_customers and customer_idurl not in list(space_dict.keys())): lg.warn("broken customers file") return p2p_service.SendFail(newpacket, 'broken customers file') if customer_idurl in current_customers: free_bytes += int(space_dict.get(customer_idurl, 0)) space_dict[b'free'] = free_bytes current_customers.remove(customer_idurl) space_dict.pop(customer_idurl) new_customer = False else: new_customer = True lg.out( 8, ' new_customer=%s current_allocated_bytes=%s' % ( new_customer, space_dict.get(customer_idurl), )) from supplier import local_tester if free_bytes <= bytes_for_customer: contactsdb.update_customers(current_customers) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('new-customer-denied', dict(idurl=customer_idurl)) else: lg.out( 8, " OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available" ) events.send('existing-customer-denied', dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'deny') space_dict[b'free'] = free_bytes - bytes_for_customer current_customers.append(customer_idurl) space_dict[customer_idurl] = bytes_for_customer contactsdb.update_customers(current_customers) contactsdb.save_customers() contactsdb.add_customer_meta_info( customer_idurl, { 'ecc_map': ecc_map, 'position': family_position, 'family_snapshot': family_snapshot, }) accounting.write_customers_quotas(space_dict) if customer_public_key_id: my_keys.erase_key(customer_public_key_id) try: if not my_keys.is_key_registered(customer_public_key_id): key_id, key_object = my_keys.read_key_info( customer_public_key) if not my_keys.register_key(key_id, key_object): lg.err('failed to register customer public key') except: lg.exc() else: lg.warn('customer public key was not provided in the request') reactor.callLater( 0, local_tester.TestUpdateCustomers) # @UndefinedVariable if new_customer: lg.out( 8, " NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot, ) events.send( 'new-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, family_snapshot=family_snapshot, key_id=customer_public_key_id, )) else: lg.out( 8, " OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s" % (customer_idurl, family_position, ecc_map, bytes_for_customer)) lg.out( 8, " family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot) events.send( 'existing-customer-accepted', dict( idurl=customer_idurl, allocated_bytes=bytes_for_customer, ecc_map=ecc_map, position=family_position, key_id=customer_public_key_id, family_snapshot=family_snapshot, )) return p2p_service.SendAck(newpacket, 'accepted')
def SendCustomers(): """ Calls ``SendToIDs()`` to send identity to all my customers. """ lg.out(8, "propagate.SendCustomers") SendToIDs(contactsdb.customers(), HandleCustomersAck)
def _do_refresh_dht_records(self): from dht import dht_relations from contacts import contactsdb for customer_idurl in contactsdb.customers(): dht_relations.publish_customer_supplier_relation(customer_idurl)
def doTestMyCapacity2(self, arg): """ Here are some values. - donated_bytes : you set this in the config - spent_bytes : how many space is taken from you by other users right now - free_bytes = donated_bytes - spent_bytes : not yet allocated space - used_bytes : size of all files, which you store on your disk for your customers """ current_customers = contactsdb.customers() removed_customers = [] spent_bytes = 0 donated_bytes = settings.getDonatedBytes() if os.path.isfile(settings.CustomersSpaceFile()): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) else: space_dict = {'free': donated_bytes} used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) lg.out(8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes) try: int(space_dict['free']) for idurl, customer_bytes in space_dict.items(): if idurl != 'free': spent_bytes += int(customer_bytes) except: lg.exc() space_dict = {'free': donated_bytes} spent_bytes = 0 removed_customers = list(current_customers) current_customers = [] self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers)) return lg.out(8, ' spent=%d' % spent_bytes) if spent_bytes < donated_bytes: space_dict['free'] = donated_bytes - spent_bytes bpio._write_dict(settings.CustomersSpaceFile(), space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return used_space_ratio_dict = {} for customer_pos in xrange(contactsdb.num_customers()): customer_idurl = contactsdb.customer(customer_pos) try: allocated_bytes = int(space_dict[customer_idurl]) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated space unknown' % customer_idurl) continue if allocated_bytes <= 0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated_bytes==0' % customer_idurl) continue try: files_size = int(used_dict.get(customer_idurl, 0)) ratio = float(files_size) / float(allocated_bytes) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s used_dict have wrong value' % customer_idurl) continue if ratio > 1.0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) spent_bytes -= allocated_bytes lg.warn('%s space overflow, where is bptester?' % customer_idurl) continue used_space_ratio_dict[customer_idurl] = ratio customers_sorted = sorted(current_customers, key=lambda i: used_space_ratio_dict[i],) while len(customers_sorted) > 0: customer_idurl = customers_sorted.pop() allocated_bytes = int(space_dict[customer_idurl]) spent_bytes -= allocated_bytes space_dict.pop(customer_idurl) current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) lg.out(8, ' customer %s REMOVED' % customer_idurl) if spent_bytes < donated_bytes: break space_dict['free'] = donated_bytes - spent_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
def doTestMyCapacity2(self, arg): """ Here are some values. - donated_bytes : you set this in the config - spent_bytes : how many space is taken from you by other users right now - free_bytes = donated_bytes - spent_bytes : not yet allocated space - used_bytes : size of all files, which you store on your disk for your customers """ current_customers = contactsdb.customers() removed_customers = [] spent_bytes = 0 donated_bytes = settings.getDonatedBytes() if os.path.isfile(settings.CustomersSpaceFile()): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) else: space_dict = {'free': donated_bytes} used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) lg.out( 8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes) try: int(space_dict['free']) for idurl, customer_bytes in space_dict.items(): if idurl != 'free': spent_bytes += int(customer_bytes) except: lg.exc() space_dict = {'free': donated_bytes} spent_bytes = 0 removed_customers = list(current_customers) current_customers = [] self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers)) return lg.out(8, ' spent=%d' % spent_bytes) if spent_bytes < donated_bytes: space_dict['free'] = donated_bytes - spent_bytes bpio._write_dict(settings.CustomersSpaceFile(), space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return used_space_ratio_dict = {} for customer_pos in range(contactsdb.num_customers()): customer_idurl = contactsdb.customer(customer_pos) try: allocated_bytes = int(space_dict[customer_idurl]) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated space unknown' % customer_idurl) continue if allocated_bytes <= 0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated_bytes==0' % customer_idurl) continue try: files_size = int(used_dict.get(customer_idurl, 0)) ratio = float(files_size) / float(allocated_bytes) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s used_dict have wrong value' % customer_idurl) continue if ratio > 1.0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) spent_bytes -= allocated_bytes lg.warn('%s space overflow, where is bptester?' % customer_idurl) continue used_space_ratio_dict[customer_idurl] = ratio customers_sorted = sorted( current_customers, key=lambda i: used_space_ratio_dict[i], ) while len(customers_sorted) > 0: customer_idurl = customers_sorted.pop() allocated_bytes = int(space_dict[customer_idurl]) spent_bytes -= allocated_bytes space_dict.pop(customer_idurl) current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) lg.out(8, ' customer %s REMOVED' % customer_idurl) if spent_bytes < donated_bytes: break space_dict['free'] = donated_bytes - spent_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat( 'space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
def FetchCustomers(): """ Fetch identity files of all customers. """ return fetch(contactsdb.customers())