def read_bandwidthIN(): """ Reads today's incoming bandwidth stats from disk. """ global BandInDict lg.out(6, 'bandwidth.read_bandwidthIN ') for idurl, bytesin in bpio._read_dict(filenameIN(), {}).items(): BandInDict[idurl] = int(bytesin)
def read_bandwidthOUT(): """ Reads today's outgoing bandwidth stats from disk. """ global BandOutDict lg.out(6, 'bandwidth.read_bandwidthOUT ') for idurl, bytesout in bpio._read_dict(filenameOUT(), {}).items(): BandOutDict[idurl] = int(bytesout)
def UpdateCustomers(): """ Test packets after list of customers was changed. """ space = bpio._read_dict(settings.CustomersSpaceFile()) if space is None: printlog('UpdateCustomers ERROR space file can not be read') return customers_dir = settings.getCustomersFilesDir() if not os.path.exists(customers_dir): printlog('UpdateCustomers ERROR customers folder not exist') return remove_list = {} for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): remove_list[onecustdir] = 'is not a folder' continue # idurl = nameurl.FilenameUrl(customer_filename) idurl = global_id.GlobalUserToIDURL(customer_filename) if idurl is None: remove_list[onecustdir] = 'wrong folder name' continue curspace = space.get(idurl, None) if curspace is None: remove_list[onecustdir] = 'is not a customer' continue for path in remove_list.keys(): if not os.path.exists(path): continue if os.path.isdir(path): try: bpio._dir_remove(path) printlog('UpdateCustomers ' + path + ' folder removed (%s)' % (remove_list[path])) except: printlog('UpdateCustomers ERROR removing ' + path) continue try: if not os.access(path, os.W_OK): os.chmod(path, 0o600) except: pass try: os.remove(path) printlog('UpdateCustomers ' + path + ' file removed (%s)' % (remove_list[path])) except: printlog('UpdateCustomers ERROR removing ' + path) printlog('UpdateCustomers ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
def validate_customers_quotas(space_dict=None): unknown_customers = set() unused_quotas = set() if space_dict is None: space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) for idurl in list(space_dict.keys()): try: space_dict[idurl] = int(space_dict[idurl]) except: if idurl != 'free': unknown_customers.add(idurl) continue if idurl != 'free' and space_dict[idurl] <= 0: unknown_customers.add(idurl) continue for idurl in contactsdb.customers(): if idurl not in space_dict.keys(): unknown_customers.add(idurl) for idurl in space_dict.keys(): if idurl != 'free': if idurl not in contactsdb.customers(): unused_quotas.add(idurl) return unknown_customers, unused_quotas
def _on_data(self, newpacket): import os from twisted.internet import reactor from logs import lg from system import bpio from main import settings from userid import my_id from userid import global_id from contacts import contactsdb from p2p import p2p_service if newpacket.OwnerID == my_id.getLocalID(): # this Data belong to us, SKIP return False if not contactsdb.is_customer(newpacket.OwnerID): # SECURITY # TODO: process files from another customer : glob_path['idurl'] lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID)) # p2p_service.SendFail(newpacket, 'not a customer') return False glob_path = global_id.ParseGlobalID(newpacket.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID( my_id.getGlobalID('master') + ':' + newpacket.PacketID) if not glob_path['path']: lg.err("got incorrect PacketID") p2p_service.SendFail(newpacket, 'incorrect path') return False filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path) if not filename: lg.warn("got empty filename, bad customer or wrong packetID?") p2p_service.SendFail(newpacket, 'empty filename') return False dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.err("can not create sub dir %s" % dirname) p2p_service.SendFail(newpacket, 'write error') return False data = newpacket.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), { 'free': donated_bytes, }) lg.warn('created a new space file: %s' % settings.CustomersSpaceFile()) space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if newpacket.OwnerID not in space_dict.keys(): lg.err("no info about donated space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no info about donated space') return False used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if newpacket.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int( used_space_dict[newpacket.OwnerID]) bytes_donated_to_customer = int(space_dict[newpacket.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len( data): lg.warn("no free space for %s" % newpacket.OwnerID) p2p_service.SendFail(newpacket, 'no free space') return False except: lg.exc() if not bpio.WriteFile(filename, data): lg.err("can not write to %s" % str(filename)) p2p_service.SendFail(newpacket, 'write error') return False # Here Data() packet was stored as it is on supplier node (current machine) sz = len(data) del data lg.out( self.debug_level, "service_supplier._on_data %r saved from [%s | %s] to %s with %d bytes" % ( newpacket, newpacket.OwnerID, newpacket.CreatorID, filename, sz, )) p2p_service.SendAck(newpacket, str(len(newpacket.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) if self.publish_event_supplier_file_modified: from main import events events.send('supplier-file-modified', data=dict( action='write', glob_path=glob_path['path'], owner_id=newpacket.OwnerID, )) return True
def read_total_rating_dict(idurl): return bpio._read_dict(rating_total_file(idurl))
def read_month_rating_dict(idurl, monthstr=None): if monthstr is None: monthstr = time.strftime('%m%y') return bpio._read_dict(rating_month_file(idurl, monthstr))
def read_customers_quotas(): return bpio._read_dict(settings.CustomersSpaceFile(), {})
def read_customers_usage(): return bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
def Data(request): """ This is when we 1) save my requested data to restore the backup 2) or save the customer file on our local HDD. """ # 1. this is our Data! if request.OwnerID == my_id.getLocalID(): if _Debug: lg.out(_DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID))) if driver.is_started("service_backups"): if request.PacketID in [settings.BackupIndexFileName()]: from storage import backup_control backup_control.IncomingSupplierBackupIndex(request) return True return False # 2. this Data is not belong to us if not driver.is_started("service_supplier"): return SendFail(request, "supplier service is off") if not contactsdb.is_customer(request.OwnerID): # SECURITY lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID)) SendFail(request, "not a customer") return filename = makeFilename(request.OwnerID, request.PacketID) if filename == "": lg.warn("got empty filename, bad customer or wrong packetID? ") SendFail(request, "empty filename") return dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.warn("ERROR can not create sub dir " + dirname) SendFail(request, "write error") return data = request.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), {"free": donated_bytes}) if _Debug: lg.out(_DebugLevel, "p2p_service.Data created a new space file") space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if request.OwnerID not in space_dict.keys(): lg.warn("no info about donated space for %s" % request.OwnerID) SendFail(request, "no info about donated space") return used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if request.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int(used_space_dict[request.OwnerID]) bytes_donated_to_customer = int(space_dict[request.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len(data): lg.warn("no free space for %s" % request.OwnerID) SendFail(request, "no free space") return except: lg.exc() if not bpio.WriteFile(filename, data): lg.warn("ERROR can not write to " + str(filename)) SendFail(request, "write error") return SendAck(request, str(len(request.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) del data if _Debug: lg.out( _DebugLevel, "p2p_service.Data saved from [%s/%s] to %s" % (nameurl.GetName(request.OwnerID), nameurl.GetName(request.CreatorID), filename), )
def SpaceTime(): """ Test all packets for each customer. Check if he use more space than we gave him and if packets is too old. """ printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000"))) space = bpio._read_dict(settings.CustomersSpaceFile()) if space is None: printlog('SpaceTime ERROR can not read file ' + settings.CustomersSpaceFile()) return customers_dir = settings.getCustomersFilesDir() if not os.path.exists(customers_dir): printlog('SpaceTime ERROR customers folder not exist') return remove_list = {} used_space = {} for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): remove_list[onecustdir] = 'is not a folder' continue idurl = nameurl.FilenameUrl(customer_filename) if idurl is None: remove_list[onecustdir] = 'wrong folder name' continue curspace = space.get(idurl, None) if curspace is None: remove_list[onecustdir] = 'not found in space file' continue try: maxspaceV = int(curspace) except: remove_list[onecustdir] = 'wrong space value' continue timedict = {} sizedict = {} def cb(path, subpath, name): # if not os.access(path, os.R_OK | os.W_OK): # return False if not os.path.isfile(path): return True # if name in [settings.BackupIndexFileName(),]: # return False stats = os.stat(path) timedict[path] = stats.st_ctime sizedict[path] = stats.st_size bpio.traverse_dir_recursive(cb, onecustdir) currentV = 0 for path in sorted(timedict.keys(), key=lambda x: timedict[x], reverse=True): filesize = sizedict.get(path, 0) currentV += filesize if currentV < maxspaceV: continue try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (cur:%s, max: %s)' % (str(currentV), str(maxspaceV))) except: printlog('SpaceTime ERROR removing ' + path) # time.sleep(0.01) used_space[idurl] = str(currentV) timedict.clear() sizedict.clear() for path in remove_list.keys(): if not os.path.exists(path): continue if os.path.isdir(path): try: bpio._dir_remove(path) printlog('SpaceTime ' + path + ' dir removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) continue try: if not os.access(path, os.W_OK): os.chmod(path, 0o600) except: pass try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) del remove_list bpio._write_dict(settings.CustomersUsedSpaceFile(), used_space)
def Data(request): """ This is when we 1) save my requested data to restore the backup 2) or save the customer file on our local HDD. """ if _Debug: lg.out( _DebugLevel, 'p2p_service.Data %d bytes in [%s] by %s | %s' % (len(request.Payload), request.PacketID, request.OwnerID, request.CreatorID)) # 1. this is our Data! if request.OwnerID == my_id.getLocalID(): if _Debug: lg.out( _DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID))) if driver.is_on('service_backups'): # TODO: move this into callback settings.BackupIndexFileName() indexPacketID = global_id.MakeGlobalID( idurl=my_id.getLocalID(), path=settings.BackupIndexFileName()) if request.PacketID == indexPacketID: from storage import backup_control backup_control.IncomingSupplierBackupIndex(request) return True return False # 2. this Data is not belong to us if not driver.is_on('service_supplier'): return SendFail(request, 'supplier service is off') if not contactsdb.is_customer(request.OwnerID): # SECURITY lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID)) SendFail(request, 'not a customer') return glob_path = global_id.ParseGlobalID(request.PacketID) if not glob_path['path']: # backward compatible check glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' + request.PacketID) if not glob_path['path']: lg.warn("got incorrect PacketID") SendFail(request, 'incorrect PacketID') return # TODO: process files from another customer : glob_path['idurl'] filename = makeFilename(request.OwnerID, glob_path['path']) if not filename: lg.warn("got empty filename, bad customer or wrong packetID? ") SendFail(request, 'empty filename') return dirname = os.path.dirname(filename) if not os.path.exists(dirname): try: bpio._dirs_make(dirname) except: lg.warn("ERROR can not create sub dir " + dirname) SendFail(request, 'write error') return data = request.Serialize() donated_bytes = settings.getDonatedBytes() if not os.path.isfile(settings.CustomersSpaceFile()): bpio._write_dict(settings.CustomersSpaceFile(), {'free': donated_bytes}) if _Debug: lg.out(_DebugLevel, 'p2p_service.Data created a new space file') space_dict = bpio._read_dict(settings.CustomersSpaceFile()) if request.OwnerID not in space_dict.keys(): lg.warn("no info about donated space for %s" % request.OwnerID) SendFail(request, 'no info about donated space') return used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) if request.OwnerID in used_space_dict.keys(): try: bytes_used_by_customer = int(used_space_dict[request.OwnerID]) bytes_donated_to_customer = int(space_dict[request.OwnerID]) if bytes_donated_to_customer - bytes_used_by_customer < len(data): lg.warn("no free space for %s" % request.OwnerID) SendFail(request, 'no free space') return except: lg.exc() if not bpio.WriteFile(filename, data): lg.warn("ERROR can not write to " + str(filename)) SendFail(request, 'write error') return SendAck(request, str(len(request.Payload))) from supplier import local_tester reactor.callLater(0, local_tester.TestSpaceTime) del data if _Debug: lg.out( _DebugLevel, "p2p_service.Data saved from [%s | %s] to %s" % ( request.OwnerID, request.CreatorID, filename, ))
def read_customers_quotas(): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) space_dict_bin = {strng.to_bin(k): v for k, v in space_dict.items()} return space_dict_bin
def read_customers_usage(): usage_dict = jsn.dict_keys_to_bin( bpio._read_dict(settings.CustomersUsedSpaceFile(), {})) usage_dict = {id_url.field(k).to_bin(): v for k, v in usage_dict.items()} return usage_dict
def read_customers_quotas(): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) free_space = int(space_dict.pop('free', 0)) space_dict = {id_url.field(k).to_bin(): v for k, v in space_dict.items()} return space_dict, free_space
def doTestMyCapacity2(self, arg): """ Here are some values. - donated_bytes : you set this in the config - spent_bytes : how many space is taken from you by other users right now - free_bytes = donated_bytes - spent_bytes : not yet allocated space - used_bytes : size of all files, which you store on your disk for your customers """ current_customers = contactsdb.customers() removed_customers = [] spent_bytes = 0 donated_bytes = settings.getDonatedBytes() if os.path.isfile(settings.CustomersSpaceFile()): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) else: space_dict = {'free': donated_bytes} used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) lg.out( 8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes) try: int(space_dict['free']) for idurl, customer_bytes in space_dict.items(): if idurl != 'free': spent_bytes += int(customer_bytes) except: lg.exc() space_dict = {'free': donated_bytes} spent_bytes = 0 removed_customers = list(current_customers) current_customers = [] self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers)) return lg.out(8, ' spent=%d' % spent_bytes) if spent_bytes < donated_bytes: space_dict['free'] = donated_bytes - spent_bytes bpio._write_dict(settings.CustomersSpaceFile(), space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return used_space_ratio_dict = {} for customer_pos in range(contactsdb.num_customers()): customer_idurl = contactsdb.customer(customer_pos) try: allocated_bytes = int(space_dict[customer_idurl]) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated space unknown' % customer_idurl) continue if allocated_bytes <= 0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated_bytes==0' % customer_idurl) continue try: files_size = int(used_dict.get(customer_idurl, 0)) ratio = float(files_size) / float(allocated_bytes) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s used_dict have wrong value' % customer_idurl) continue if ratio > 1.0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) spent_bytes -= allocated_bytes lg.warn('%s space overflow, where is bptester?' % customer_idurl) continue used_space_ratio_dict[customer_idurl] = ratio customers_sorted = sorted( current_customers, key=lambda i: used_space_ratio_dict[i], ) while len(customers_sorted) > 0: customer_idurl = customers_sorted.pop() allocated_bytes = int(space_dict[customer_idurl]) spent_bytes -= allocated_bytes space_dict.pop(customer_idurl) current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) lg.out(8, ' customer %s REMOVED' % customer_idurl) if spent_bytes < donated_bytes: break space_dict['free'] = donated_bytes - spent_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat( 'space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
def doTestMyCapacity2(self, arg): """ Here are some values. - donated_bytes : you set this in the config - spent_bytes : how many space is taken from you by other users right now - free_bytes = donated_bytes - spent_bytes : not yet allocated space - used_bytes : size of all files, which you store on your disk for your customers """ current_customers = contactsdb.customers() removed_customers = [] spent_bytes = 0 donated_bytes = settings.getDonatedBytes() if os.path.isfile(settings.CustomersSpaceFile()): space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {}) else: space_dict = {'free': donated_bytes} used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {}) lg.out(8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes) try: int(space_dict['free']) for idurl, customer_bytes in space_dict.items(): if idurl != 'free': spent_bytes += int(customer_bytes) except: lg.exc() space_dict = {'free': donated_bytes} spent_bytes = 0 removed_customers = list(current_customers) current_customers = [] self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers)) return lg.out(8, ' spent=%d' % spent_bytes) if spent_bytes < donated_bytes: space_dict['free'] = donated_bytes - spent_bytes bpio._write_dict(settings.CustomersSpaceFile(), space_dict) lg.out(8, ' space is OK !!!!!!!!') self.automat('space-enough') return used_space_ratio_dict = {} for customer_pos in xrange(contactsdb.num_customers()): customer_idurl = contactsdb.customer(customer_pos) try: allocated_bytes = int(space_dict[customer_idurl]) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated space unknown' % customer_idurl) continue if allocated_bytes <= 0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s allocated_bytes==0' % customer_idurl) continue try: files_size = int(used_dict.get(customer_idurl, 0)) ratio = float(files_size) / float(allocated_bytes) except: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) lg.warn('%s used_dict have wrong value' % customer_idurl) continue if ratio > 1.0: if customer_idurl in current_customers: current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) else: lg.warn('%s not customers' % customer_idurl) spent_bytes -= allocated_bytes lg.warn('%s space overflow, where is bptester?' % customer_idurl) continue used_space_ratio_dict[customer_idurl] = ratio customers_sorted = sorted(current_customers, key=lambda i: used_space_ratio_dict[i],) while len(customers_sorted) > 0: customer_idurl = customers_sorted.pop() allocated_bytes = int(space_dict[customer_idurl]) spent_bytes -= allocated_bytes space_dict.pop(customer_idurl) current_customers.remove(customer_idurl) removed_customers.append(customer_idurl) lg.out(8, ' customer %s REMOVED' % customer_idurl) if spent_bytes < donated_bytes: break space_dict['free'] = donated_bytes - spent_bytes lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!') self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))