def _stats(params): from contacts import contactsdb from p2p import contact_status from lib import diskspace result = {} result['suppliers'] = contactsdb.num_suppliers() result['max_suppliers'] = settings.getSuppliersNumberDesired() result['online_suppliers'] = contact_status.countOnlineAmong( contactsdb.suppliers()) result['customers'] = contactsdb.num_customers() result['bytes_donated'] = settings.getDonatedBytes() result['value_donated'] = diskspace.MakeStringFromBytes( settings.getDonatedBytes()) result['bytes_needed'] = settings.getNeededBytes() result['value_needed'] = diskspace.MakeStringFromBytes( settings.getNeededBytes()) result['bytes_used_total'] = backup_fs.sizebackups() result['value_used_total'] = diskspace.MakeStringFromBytes( backup_fs.sizebackups()) result['bytes_used_supplier'] = 0 if ( contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers())) result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders() result['files_count'] = backup_fs.numberfiles() result['folders_count'] = backup_fs.numberfolders() result['items_count'] = backup_fs.counter() result['timestamp'] = time.time() return { 'result': result, }
def doTestMyCapacity(self, arg): """ Here are some values. + donated_bytes : you set this in the configs + consumed_bytes : how many space was taken from you by other users + free_bytes = donated_bytes - consumed_bytes : not yet allocated space + used_bytes : size of all files, which you store on your disk for your customers + ratio : currently used space compared to consumed space """ lg.out(8, 'customers_rejector.doTestMyCapacity') failed_customers = set() current_customers = contactsdb.customers() donated_bytes = settings.getDonatedBytes() space_dict = accounting.read_customers_quotas() used_dict = accounting.read_customers_usage() unknown_customers, unused_quotas = accounting.validate_customers_quotas( space_dict) failed_customers.update(unknown_customers) for idurl in unknown_customers: space_dict.pop(idurl, None) for idurl in unused_quotas: space_dict.pop(idurl, None) consumed_bytes = accounting.count_consumed_space(space_dict) space_dict['free'] = donated_bytes - consumed_bytes if consumed_bytes < donated_bytes and len(failed_customers) == 0: accounting.write_customers_quotas(space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return if failed_customers: lg.out( 8, ' found FAILED Customers:\n%s' % (' \n'.join(failed_customers))) for idurl in failed_customers: current_customers.remove(idurl) self.automat('space-overflow', (space_dict, consumed_bytes, current_customers, failed_customers)) return used_space_ratio_dict = accounting.calculate_customers_usage_ratio( space_dict, used_dict) customers_sorted = sorted( current_customers, key=lambda idurl: used_space_ratio_dict[idurl], ) while len(customers_sorted) > 0 and consumed_bytes > donated_bytes: idurl = customers_sorted.pop() allocated_bytes = int(space_dict[idurl]) consumed_bytes -= allocated_bytes space_dict.pop(idurl) failed_customers.add(idurl) current_customers.remove(idurl) lg.out(8, ' customer %s will be REMOVED' % idurl) space_dict['free'] = donated_bytes - consumed_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat( 'space-overflow', (space_dict, consumed_bytes, current_customers, failed_customers))
def check_create_customers_quotas(donated_bytes=None): if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), { 'free': donated_bytes or settings.getDonatedBytes(), }) lg.info('created a new customers quotas file: %s' % settings.CustomersSpaceFile()) return True return False
def doTestMyCapacity(self, *args, **kwargs): """ Here are some values. + donated_bytes : you set this in the settings + consumed_bytes : how many space was taken from you by other users + free_bytes = donated_bytes - consumed_bytes : not yet allocated space + used_bytes : size of all files, which you store on your disk for your customers + ratio : currently used space compared to consumed space """ if _Debug: lg.out(_DebugLevel, 'customers_rejector.doTestMyCapacity') failed_customers = set() current_customers = contactsdb.customers() donated_bytes = settings.getDonatedBytes() space_dict, free_space = accounting.read_customers_quotas() used_dict = accounting.read_customers_usage() unknown_customers, unused_quotas = accounting.validate_customers_quotas( space_dict, free_space) failed_customers.update(unknown_customers) for idurl in unknown_customers: space_dict.pop(idurl, None) for idurl in unused_quotas: space_dict.pop(idurl, None) consumed_bytes = accounting.count_consumed_space(space_dict) free_space = donated_bytes - consumed_bytes if consumed_bytes < donated_bytes and len(failed_customers) == 0: accounting.write_customers_quotas(space_dict, free_space) lg.info('storage quota checks succeed, all customers are verified') self.automat('space-enough') return if failed_customers: for idurl in failed_customers: lg.warn('customer %r failed storage quota verification' % idurl) current_customers.remove(idurl) self.automat('space-overflow', failed_customers) return used_space_ratio_dict = accounting.calculate_customers_usage_ratio( space_dict, used_dict) customers_sorted = sorted( current_customers, key=lambda idurl: used_space_ratio_dict[idurl], ) while len(customers_sorted) > 0 and consumed_bytes > donated_bytes: idurl = customers_sorted.pop() allocated_bytes = int(space_dict[idurl]) consumed_bytes -= allocated_bytes space_dict.pop(idurl) failed_customers.add(idurl) current_customers.remove(idurl) lg.warn( 'customer %r will be removed because of storage quota overflow' % idurl) free_space = donated_bytes - consumed_bytes self.automat('space-overflow', failed_customers)
def doTestMyCapacity(self, arg): """ Here are some values. + donated_bytes : you set this in the configs + consumed_bytes : how many space was taken from you by other users + free_bytes = donated_bytes - consumed_bytes : not yet allocated space + used_bytes : size of all files, which you store on your disk for your customers + ratio : currently used space compared to consumed space """ lg.out(8, 'customers_rejector.doTestMyCapacity') failed_customers = set() current_customers = contactsdb.customers() donated_bytes = settings.getDonatedBytes() space_dict = accounting.read_customers_quotas() used_dict = accounting.read_customers_usage() unknown_customers, unused_quotas = accounting.validate_customers_quotas(space_dict) failed_customers.update(unknown_customers) for idurl in unknown_customers: space_dict.pop(idurl, None) for idurl in unused_quotas: space_dict.pop(idurl, None) consumed_bytes = accounting.count_consumed_space(space_dict) space_dict['free'] = donated_bytes - consumed_bytes if consumed_bytes < donated_bytes and len(failed_customers) == 0: accounting.write_customers_quotas(space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return if failed_customers: lg.out(8, ' found FAILED Customers:\n%s' % ( ' \n'.join(failed_customers))) for idurl in failed_customers: current_customers.remove(idurl) self.automat('space-overflow', ( space_dict, consumed_bytes, current_customers, failed_customers)) return used_space_ratio_dict = accounting.calculate_customers_usage_ratio(space_dict, used_dict) customers_sorted = sorted(current_customers, key=lambda idurl: used_space_ratio_dict[idurl],) while len(customers_sorted) > 0 and consumed_bytes > donated_bytes: idurl = customers_sorted.pop() allocated_bytes = int(space_dict[idurl]) consumed_bytes -= allocated_bytes space_dict.pop(idurl) failed_customers.add(idurl) current_customers.remove(idurl) lg.out(8, ' customer %s will be REMOVED' % idurl) space_dict['free'] = donated_bytes - consumed_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat('space-overflow', ( space_dict, consumed_bytes, current_customers, failed_customers))
def _stats(params): from contacts import contactsdb from p2p import contact_status from lib import diskspace result = {} result['suppliers'] = contactsdb.num_suppliers() result['max_suppliers'] = settings.getSuppliersNumberDesired() result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers()) result['customers'] = contactsdb.num_customers() result['bytes_donated'] = settings.getDonatedBytes() result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes()) result['bytes_needed'] = settings.getNeededBytes() result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes()) result['bytes_used_total'] = backup_fs.sizebackups() result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups()) result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers())) result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders() result['files_count'] = backup_fs.numberfiles() result['folders_count'] = backup_fs.numberfolders() result['items_count'] = backup_fs.counter() result['timestamp'] = time.time() return {'result': result, }
def _on_data(self, newpacket): import os from twisted.internet import reactor # @UnresolvedImport from logs import lg from lib import jsn from system import bpio from main import settings from userid import my_id from userid import global_id from contacts import contactsdb from p2p import p2p_service from storage import accounting if newpacket.OwnerID == my_id.getLocalID(): # this Data belong to us, SKIP return False if not contactsdb.is_customer(newpacket.OwnerID): # SECURITY # TODO: process files from another customer : glob_path['idurl'] lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID)) # p2p_service.SendFail(newpacket, 'not a customer') return False glob_path = global_id.ParseGlobalID(newpacket.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID( my_id.getGlobalID('master') + ':' + newpacket.PacketID) if not glob_path['path']: lg.err("got incorrect PacketID") p2p_service.SendFail(newpacket, 'incorrect path') return False filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path) if not filename: lg.warn("got empty filename, bad customer or wrong packetID?") p2p_service.SendFail(newpacket, 'empty filename') return False dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.err("can not create sub dir %s" % dirname) p2p_service.SendFail(newpacket, 'write error') return False data = newpacket.Serialize() donated_bytes = settings.getDonatedBytes() accounting.check_create_customers_quotas(donated_bytes) space_dict = accounting.read_customers_quotas() if newpacket.OwnerID not in list(space_dict.keys()): lg.err("no info about donated space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no info about donated space') return False used_space_dict = accounting.read_customers_usage() if newpacket.OwnerID in list(used_space_dict.keys()): try: bytes_used_by_customer = int( used_space_dict[newpacket.OwnerID]) bytes_donated_to_customer = int(space_dict[newpacket.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len( data): lg.warn("no free space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no free space') return False except: lg.exc() if not bpio.WriteBinaryFile(filename, data): lg.err("can not write to %s" % str(filename)) p2p_service.SendFail(newpacket, 'write error') return False # Here Data() packet was stored as it is on supplier node (current machine) sz = len(data) del data lg.out(self.debug_level, "service_supplier._on_data %r" % newpacket) lg.out( self.debug_level, " from [ %s | %s ]" % ( newpacket.OwnerID, newpacket.CreatorID, )) lg.out(self.debug_level, " saved with %d bytes to %s" % ( sz, filename, )) p2p_service.SendAck(newpacket, str(len(newpacket.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable if self.publish_event_supplier_file_modified: from main import events events.send('supplier-file-modified', data=dict( action='write', glob_path=glob_path['path'], owner_id=newpacket.OwnerID, )) return True
def doTestMyCapacity2(self, arg): """ Here are some values. - donated_bytes : you set this in the config - spent_bytes : how many space is taken from you by other users right now - free_bytes = donated_bytes - spent_bytes : not yet allocated space - used_bytes : size of all files, which you store on your disk for your customers """ current_customers = contactsdb.customers() removed_customers = [] spent_bytes = 0 donated_bytes = settings.getDonatedBytes() if os.path.isfile(settings.CustomersSpaceFile()): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) else: space_dict = {'free': donated_bytes} used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) lg.out( 8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes) try: int(space_dict['free']) for idurl, customer_bytes in space_dict.items(): if idurl != 'free': spent_bytes += int(customer_bytes) except: lg.exc() space_dict = {'free': donated_bytes} spent_bytes = 0 removed_customers = list(current_customers) current_customers = [] self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers)) return lg.out(8, ' spent=%d' % spent_bytes) if spent_bytes < donated_bytes: space_dict['free'] = donated_bytes - spent_bytes bpio._write_dict(settings.CustomersSpaceFile(), space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return used_space_ratio_dict = {} for customer_pos in range(contactsdb.num_customers()): customer_idurl = contactsdb.customer(customer_pos) try: allocated_bytes = int(space_dict[customer_idurl]) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated space unknown' % customer_idurl) continue if allocated_bytes <= 0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated_bytes==0' % customer_idurl) continue try: files_size = int(used_dict.get(customer_idurl, 0)) ratio = float(files_size) / float(allocated_bytes) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s used_dict have wrong value' % customer_idurl) continue if ratio > 1.0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) spent_bytes -= allocated_bytes lg.warn('%s space overflow, where is bptester?' % customer_idurl) continue used_space_ratio_dict[customer_idurl] = ratio customers_sorted = sorted( current_customers, key=lambda i: used_space_ratio_dict[i], ) while len(customers_sorted) > 0: customer_idurl = customers_sorted.pop() allocated_bytes = int(space_dict[customer_idurl]) spent_bytes -= allocated_bytes space_dict.pop(customer_idurl) current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) lg.out(8, ' customer %s REMOVED' % customer_idurl) if spent_bytes < donated_bytes: break space_dict['free'] = donated_bytes - spent_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat( 'space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
def Data(request): """ This is when we 1) save my requested data to restore the backup 2) or save the customer file on our local HDD. """ # 1. this is our Data! if request.OwnerID == my_id.getLocalID(): if _Debug: lg.out(_DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID))) if driver.is_started("service_backups"): if request.PacketID in [settings.BackupIndexFileName()]: from storage import backup_control backup_control.IncomingSupplierBackupIndex(request) return True return False # 2. this Data is not belong to us if not driver.is_started("service_supplier"): return SendFail(request, "supplier service is off") if not contactsdb.is_customer(request.OwnerID): # SECURITY lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID)) SendFail(request, "not a customer") return filename = makeFilename(request.OwnerID, request.PacketID) if filename == "": lg.warn("got empty filename, bad customer or wrong packetID? ") SendFail(request, "empty filename") return dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.warn("ERROR can not create sub dir " + dirname) SendFail(request, "write error") return data = request.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), {"free": donated_bytes}) if _Debug: lg.out(_DebugLevel, "p2p_service.Data created a new space file") space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if request.OwnerID not in space_dict.keys(): lg.warn("no info about donated space for %s" % request.OwnerID) SendFail(request, "no info about donated space") return used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if request.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int(used_space_dict[request.OwnerID]) bytes_donated_to_customer = int(space_dict[request.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len(data): lg.warn("no free space for %s" % request.OwnerID) SendFail(request, "no free space") return except: lg.exc() if not bpio.WriteFile(filename, data): lg.warn("ERROR can not write to " + str(filename)) SendFail(request, "write error") return SendAck(request, str(len(request.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) del data if _Debug: lg.out( _DebugLevel, "p2p_service.Data saved from [%s/%s] to %s" % (nameurl.GetName(request.OwnerID), nameurl.GetName(request.CreatorID), filename), )
def report_donated_storage(): space_dict = read_customers_quotas() used_space_dict = read_customers_usage() r = {} r['customers_num'] = contactsdb.num_customers() r['customers'] = [] r['old_customers'] = [] r['errors'] = [] r['consumed'] = 0 r['donated'] = settings.getDonatedBytes() r['donated_str'] = diskspace.MakeStringFromBytes(r['donated']) r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir()) try: r['free'] = int(space_dict.pop('free')) except: r['free'] = 0 used = 0 for idurl in contactsdb.customers(): consumed_by_customer = 0 used_by_customer = 0 if idurl not in space_dict.keys(): r['errors'].append('space consumed by customer %s is unknown' % idurl) else: try: consumed_by_customer = int(space_dict.pop(idurl)) r['consumed'] += consumed_by_customer except: r['errors'].append('incorrect value of consumed space for customer %s' % idurl) if idurl in used_space_dict.keys(): try: used_by_customer = int(used_space_dict.pop(idurl)) used += used_by_customer except: r['errors'].append('incorrect value of used space for customer %s' % idurl) if consumed_by_customer < used_by_customer: r['errors'].append('customer %s currently using more space than requested' % idurl) c = {} c['idurl'] = idurl c['used'] = used_by_customer c['used_str'] = diskspace.MakeStringFromBytes(c['used']) c['consumed'] = consumed_by_customer c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed']) c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) c['real_str'] = diskspace.MakeStringFromBytes(c['real']) r['customers'].append(c) r['used'] = used r['used_str'] = diskspace.MakeStringFromBytes(r['used']) r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed']) if r['donated'] != r['free'] + r['consumed']: r['errors'].append('total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % ( r['consumed'], r['free'], r['consumed'] + r['free'], r['donated'])) if r['used'] > r['donated']: r['errors'].append('total space used by customers exceed the donated limit') if len(space_dict) > 0: r['errors'].append('found %d incorrect records of consumed space' % len(space_dict)) if r['real'] != r['used']: r['errors'].append('current info needs update, known size is %d bytes but real is %d bytes' % ( r['used'], r['real'])) for idurl in used_space_dict.keys(): real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) r['old_customers'].append({ 'idurl': idurl, 'used': used_space_dict['idurl'], 'used_str': diskspace.MakeStringFromBytes(used_space_dict['idurl']), 'real': real, 'real_str': diskspace.MakeStringFromBytes(real), }) try: r['used_percent'] = misc.percent2string(float(r['used']) / float(r['donated']), 5) except: r['used_percent'] = '' try: r['consumed_percent'] = misc.percent2string(float(r['consumed']) / float(r['donated']), 5) except: r['consumed_percent'] = '' return r
def check_create_customers_quotas(): if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), {'free': settings.getDonatedBytes()}) return True return False
def report_donated_storage(): space_dict, free_space = read_customers_quotas() used_space_dict = read_customers_usage() r = {} r['customers_num'] = contactsdb.num_customers() r['customers'] = [] r['old_customers'] = [] r['errors'] = [] r['consumed'] = 0 r['donated'] = settings.getDonatedBytes() # r['donated_str'] = diskspace.MakeStringFromBytes(r['donated']) r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir()) try: r['free'] = int(free_space) except: r['free'] = 0 used = 0 for idurl in id_url.to_bin_list(contactsdb.customers()): consumed_by_customer = 0 used_by_customer = 0 if idurl not in list(space_dict.keys()): r['errors'].append('space consumed by customer %r is unknown' % idurl) else: try: consumed_by_customer = int(space_dict.pop(idurl)) r['consumed'] += consumed_by_customer except: r['errors'].append( 'incorrect value of consumed space for customer %r' % idurl) continue if idurl in list(used_space_dict.keys()): try: used_by_customer = int(used_space_dict.pop(idurl)) used += used_by_customer except: r['errors'].append( 'incorrect value of used space for customer %r' % idurl) continue if consumed_by_customer < used_by_customer: r['errors'].append( 'customer %r currently using more space than requested' % idurl) c = {} c['idurl'] = strng.to_text(idurl) c['used'] = used_by_customer # c['used_str'] = diskspace.MakeStringFromBytes(c['used']) c['consumed'] = consumed_by_customer # c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed']) c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) # c['real_str'] = diskspace.MakeStringFromBytes(c['real']) r['customers'].append(c) r['used'] = used # r['used_str'] = diskspace.MakeStringFromBytes(r['used']) # r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed']) if r['donated'] != r['free'] + r['consumed']: r['errors'].append( 'total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % (r['consumed'], r['free'], r['consumed'] + r['free'], r['donated'])) if r['used'] > r['donated']: r['errors'].append( 'total space used by customers exceed the donated limit') if len(space_dict) > 0: r['errors'].append('found %d incorrect records of consumed space' % len(space_dict)) if r['real'] != r['used']: r['errors'].append( 'current info needs update, known size is %d bytes but real is %d bytes' % (r['used'], r['real'])) old_customers_used = 0 old_customers_real = 0 for idurl in used_space_dict.keys(): real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl)) try: used = int(used_space_dict[idurl]) except: r['errors'].append( 'incorrect value of used space for customer %r' % idurl) continue r['old_customers'].append({ 'idurl': strng.to_text(idurl), 'used': used, # 'used_str': diskspace.MakeStringFromBytes(used_space_dict[idurl]), 'real': real, # 'real_str': diskspace.MakeStringFromBytes(real), }) old_customers_used += used old_customers_real += real r['old_customers_used'] = old_customers_used r['old_customers_real'] = old_customers_real try: r['used_percent'] = misc.value2percent(float(r['used']), float(r['donated']), 5) except: r['used_percent'] = '' try: r['consumed_percent'] = misc.value2percent(float(r['consumed']), float(r['donated']), 5) except: r['consumed_percent'] = '' return r
def on_data(newpacket): if id_url.to_bin(newpacket.OwnerID) == my_id.getIDURL().to_bin(): # this Data belong to us, SKIP return False # if not contactsdb.is_customer(newpacket.OwnerID): # # SECURITY # # TODO: process files from another customer : glob_path['idurl'] # lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID)) # # p2p_service.SendFail(newpacket, 'not a customer') # return False glob_path = global_id.ParseGlobalID(newpacket.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID( my_id.getGlobalID('master') + ':' + newpacket.PacketID) if not glob_path['path']: lg.err("got incorrect PacketID") # p2p_service.SendFail(newpacket, 'incorrect path') return False authorized_idurl = verify_packet_ownership(newpacket) if authorized_idurl is None: lg.err("ownership verification failed for %r" % newpacket) # p2p_service.SendFail(newpacket, 'ownership verification failed') return False filename = make_valid_filename(newpacket.OwnerID, glob_path) if not filename: lg.warn("got empty filename, bad customer or wrong packetID?") # p2p_service.SendFail(newpacket, 'empty filename') return False dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.err("can not create sub dir %s" % dirname) p2p_service.SendFail(newpacket, 'write error', remote_idurl=authorized_idurl) return False data = newpacket.Serialize() donated_bytes = settings.getDonatedBytes() accounting.check_create_customers_quotas(donated_bytes) space_dict, _ = accounting.read_customers_quotas() if newpacket.OwnerID.to_bin() not in list(space_dict.keys()): lg.err("customer space is broken, no info about donated space for %s" % newpacket.OwnerID) p2p_service.SendFail( newpacket, 'customer space is broken, no info about donated space', remote_idurl=authorized_idurl) return False used_space_dict = accounting.read_customers_usage() if newpacket.OwnerID.to_bin() in list(used_space_dict.keys()): try: bytes_used_by_customer = int( used_space_dict[newpacket.OwnerID.to_bin()]) bytes_donated_to_customer = int( space_dict[newpacket.OwnerID.to_bin()]) if bytes_donated_to_customer - bytes_used_by_customer < len(data): lg.warn("no free space left for customer data: %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no free space left for customer data', remote_idurl=authorized_idurl) return False except: lg.exc() if not bpio.WriteBinaryFile(filename, data): lg.err("can not write to %s" % str(filename)) p2p_service.SendFail(newpacket, 'write error', remote_idurl=authorized_idurl) return False # Here Data() packet was stored as it is on supplier node (current machine) del data p2p_service.SendAck(newpacket, response=strng.to_text(len(newpacket.Payload)), remote_idurl=authorized_idurl) reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable # if self.publish_event_supplier_file_modified: # TODO: must remove that actually # from main import events # events.send('supplier-file-modified', data=dict( # action='write', # glob_path=glob_path['path'], # owner_id=newpacket.OwnerID, # )) return True
def doTestMyCapacity2(self, arg): """ Here are some values. - donated_bytes : you set this in the config - spent_bytes : how many space is taken from you by other users right now - free_bytes = donated_bytes - spent_bytes : not yet allocated space - used_bytes : size of all files, which you store on your disk for your customers """ current_customers = contactsdb.customers() removed_customers = [] spent_bytes = 0 donated_bytes = settings.getDonatedBytes() if os.path.isfile(settings.CustomersSpaceFile()): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) else: space_dict = {'free': donated_bytes} used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) lg.out(8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes) try: int(space_dict['free']) for idurl, customer_bytes in space_dict.items(): if idurl != 'free': spent_bytes += int(customer_bytes) except: lg.exc() space_dict = {'free': donated_bytes} spent_bytes = 0 removed_customers = list(current_customers) current_customers = [] self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers)) return lg.out(8, ' spent=%d' % spent_bytes) if spent_bytes < donated_bytes: space_dict['free'] = donated_bytes - spent_bytes bpio._write_dict(settings.CustomersSpaceFile(), space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return used_space_ratio_dict = {} for customer_pos in xrange(contactsdb.num_customers()): customer_idurl = contactsdb.customer(customer_pos) try: allocated_bytes = int(space_dict[customer_idurl]) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated space unknown' % customer_idurl) continue if allocated_bytes <= 0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated_bytes==0' % customer_idurl) continue try: files_size = int(used_dict.get(customer_idurl, 0)) ratio = float(files_size) / float(allocated_bytes) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s used_dict have wrong value' % customer_idurl) continue if ratio > 1.0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) spent_bytes -= allocated_bytes lg.warn('%s space overflow, where is bptester?' % customer_idurl) continue used_space_ratio_dict[customer_idurl] = ratio customers_sorted = sorted(current_customers, key=lambda i: used_space_ratio_dict[i],) while len(customers_sorted) > 0: customer_idurl = customers_sorted.pop() allocated_bytes = int(space_dict[customer_idurl]) spent_bytes -= allocated_bytes space_dict.pop(customer_idurl) current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) lg.out(8, ' customer %s REMOVED' % customer_idurl) if spent_bytes < donated_bytes: break space_dict['free'] = donated_bytes - spent_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
def Data(request): """ This is when we 1) save my requested data to restore the backup 2) or save the customer file on our local HDD. """ if _Debug: lg.out( _DebugLevel, 'p2p_service.Data %d bytes in [%s] by %s | %s' % (len(request.Payload), request.PacketID, request.OwnerID, request.CreatorID)) # 1. this is our Data! if request.OwnerID == my_id.getLocalID(): if _Debug: lg.out( _DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID))) if driver.is_on('service_backups'): # TODO: move this into callback settings.BackupIndexFileName() indexPacketID = global_id.MakeGlobalID( idurl=my_id.getLocalID(), path=settings.BackupIndexFileName()) if request.PacketID == indexPacketID: from storage import backup_control backup_control.IncomingSupplierBackupIndex(request) return True return False # 2. this Data is not belong to us if not driver.is_on('service_supplier'): return SendFail(request, 'supplier service is off') if not contactsdb.is_customer(request.OwnerID): # SECURITY lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID)) SendFail(request, 'not a customer') return glob_path = global_id.ParseGlobalID(request.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' + request.PacketID) if not glob_path['path']: lg.warn("got incorrect PacketID") SendFail(request, 'incorrect PacketID') return # TODO: process files from another customer : glob_path['idurl'] filename = makeFilename(request.OwnerID, glob_path['path']) if not filename: lg.warn("got empty filename, bad customer or wrong packetID? ") SendFail(request, 'empty filename') return dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.warn("ERROR can not create sub dir " + dirname) SendFail(request, 'write error') return data = request.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), {'free': donated_bytes}) if _Debug: lg.out(_DebugLevel, 'p2p_service.Data created a new space file') space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if request.OwnerID not in space_dict.keys(): lg.warn("no info about donated space for %s" % request.OwnerID) SendFail(request, 'no info about donated space') return used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if request.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int(used_space_dict[request.OwnerID]) bytes_donated_to_customer = int(space_dict[request.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len(data): lg.warn("no free space for %s" % request.OwnerID) SendFail(request, 'no free space') return except: lg.exc() if not bpio.WriteFile(filename, data): lg.warn("ERROR can not write to " + str(filename)) SendFail(request, 'write error') return SendAck(request, str(len(request.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) del data if _Debug: lg.out( _DebugLevel, "p2p_service.Data saved from [%s | %s] to %s" % ( request.OwnerID, request.CreatorID, filename, ))
def _on_data(self, newpacket): import os from twisted.internet import reactor from logs import lg from system import bpio from main import settings from userid import my_id from userid import global_id from contacts import contactsdb from p2p import p2p_service if newpacket.OwnerID == my_id.getLocalID(): # this Data belong to us, SKIP return False if not contactsdb.is_customer(newpacket.OwnerID): # SECURITY lg.err("%s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID)) p2p_service.SendFail(newpacket, 'not a customer') return False glob_path = global_id.ParseGlobalID(newpacket.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID( my_id.getGlobalID('master') + ':' + newpacket.PacketID) if not glob_path['path']: lg.err("got incorrect PacketID") p2p_service.SendFail(newpacket, 'incorrect path') return False # TODO: process files from another customer : glob_path['idurl'] filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path) if not filename: lg.warn("got empty filename, bad customer or wrong packetID?") p2p_service.SendFail(newpacket, 'empty filename') return False dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.err("can not create sub dir %s" % dirname) p2p_service.SendFail(newpacket, 'write error') return False data = newpacket.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), { 'free': donated_bytes, }) lg.warn('created a new space file: %s' % settings.CustomersSpaceFile()) space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if newpacket.OwnerID not in space_dict.keys(): lg.err("no info about donated space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no info about donated space') return False used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if newpacket.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int( used_space_dict[newpacket.OwnerID]) bytes_donated_to_customer = int(space_dict[newpacket.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len( data): lg.warn("no free space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no free space') return False except: lg.exc() if not bpio.WriteFile(filename, data): lg.err("can not write to %s" % str(filename)) p2p_service.SendFail(newpacket, 'write error') return False # Here Data() packet was stored as it is on supplier node (current machine) sz = len(data) del data lg.out( self.debug_level, "service_supplier._on_data %r saved from [%s | %s] to %s with %d bytes" % ( newpacket, newpacket.OwnerID, newpacket.CreatorID, filename, sz, )) p2p_service.SendAck(newpacket, str(len(newpacket.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) # temporary disabled # from main import events # events.send('supplier-file-modified', data=dict( # action='write', # glob_path=glob_path['path'], # owner_id=newpacket.OwnerID, # )) return True