def save():
"""
Writes today stats on disk.
"""
lg.out(6, 'bandwidth.save')
bpio._write_dict(filenameIN(), getBandwidthIN())
bpio._write_dict(filenameOUT(), getBandwidthOUT())
def save():
"""
Writes today stats on disk.
"""
lg.out(6, 'bandwidth.save')
bpio._write_dict(filenameIN(), getBandwidthIN())
bpio._write_dict(filenameOUT(), getBandwidthOUT())
def check_create_customers_quotas(donated_bytes=None):
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {
'free': donated_bytes or settings.getDonatedBytes(),
})
lg.info('created a new customers quotas file: %s' %
settings.CustomersSpaceFile())
return True
return False
def saveOUT(basename=None):
"""
Writes outgoing stats for today on disk.
"""
if basename is None:
basename = misc.gmtime2str('%d%m%y')
ret = os.path.isfile(filenameOUT(basename))
bpio._write_dict(filenameOUT(basename), getBandwidthOUT())
if not ret:
lg.out(4, 'bandwidth.saveOUT to new file ' + basename)
else:
lg.out(22, 'bandwidth.saveOUT to ' + basename)
return ret
def saveOUT(basename=None):
"""
Writes outgoing stats for today on disk.
"""
if basename is None:
basename = misc.gmtime2str('%d%m%y')
ret = os.path.isfile(filenameOUT(basename))
bpio._write_dict(filenameOUT(basename), getBandwidthOUT())
if not ret:
lg.out(4, 'bandwidth.saveOUT to new file ' + basename)
else:
lg.out(22, 'bandwidth.saveOUT to ' + basename)
return ret
def write_customers_quotas(new_space_dict, free_space):
space_dict = {
id_url.field(k).to_text(): v
for k, v in new_space_dict.items()
}
space_dict['free'] = free_space
return bpio._write_dict(settings.CustomersSpaceFile(), space_dict)
def update_customers_usage(new_space_usage_dict):
usage_dict = {
id_url.field(k).to_bin(): v
for k, v in new_space_usage_dict.items()
}
return bpio._write_dict(settings.CustomersUsedSpaceFile(),
jsn.dict_keys_to_text(usage_dict))
def update_customers_usage(new_space_usage_dict):
return bpio._write_dict(settings.CustomersUsedSpaceFile(),
new_space_usage_dict)
def check_create_customers_quotas():
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': settings.getDonatedBytes()})
return True
return False
def write_total_rating_dict(idurl, rating_dict):
return bpio._write_dict(rating_total_file(idurl), rating_dict)
def _on_data(self, newpacket):
import os
from twisted.internet import reactor
from logs import lg
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID):
# SECURITY
# TODO: process files from another customer : glob_path['idurl']
lg.warn("skip, %s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
# p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {
'free': donated_bytes,
})
lg.warn('created a new space file: %s' %
settings.CustomersSpaceFile())
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if newpacket.OwnerID not in space_dict.keys():
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(),
{})
if newpacket.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(
self.debug_level,
"service_supplier._on_data %r saved from [%s | %s] to %s with %d bytes"
% (
newpacket,
newpacket.OwnerID,
newpacket.CreatorID,
filename,
sz,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
if self.publish_event_supplier_file_modified:
from main import events
events.send('supplier-file-modified',
data=dict(
action='write',
glob_path=glob_path['path'],
owner_id=newpacket.OwnerID,
))
return True
def check_create_customers_quotas():
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': settings.getDonatedBytes()})
return True
return False
def write_month_rating_dict(idurl, rating_dict, monthstr=None):
if monthstr is None:
monthstr = time.strftime('%m%y')
return bpio._write_dict(rating_month_file(idurl, monthstr), rating_dict)
def update_customers_usage(new_space_usage_dict):
return bpio._write_dict(settings.CustomersUsedSpaceFile(), new_space_usage_dict)
def write_customers_quotas(new_space_dict):
return bpio._write_dict(settings.CustomersSpaceFile(), new_space_dict)
def Data(request):
"""
This is when we 1) save my requested data to restore the backup 2) or save
the customer file on our local HDD.
"""
# 1. this is our Data!
if request.OwnerID == my_id.getLocalID():
if _Debug:
lg.out(_DebugLevel, "p2p_service.Data %r for us from %s" % (request, nameurl.GetName(request.RemoteID)))
if driver.is_started("service_backups"):
if request.PacketID in [settings.BackupIndexFileName()]:
from storage import backup_control
backup_control.IncomingSupplierBackupIndex(request)
return True
return False
# 2. this Data is not belong to us
if not driver.is_started("service_supplier"):
return SendFail(request, "supplier service is off")
if not contactsdb.is_customer(request.OwnerID): # SECURITY
lg.warn("%s not a customer, packetID=%s" % (request.OwnerID, request.PacketID))
SendFail(request, "not a customer")
return
filename = makeFilename(request.OwnerID, request.PacketID)
if filename == "":
lg.warn("got empty filename, bad customer or wrong packetID? ")
SendFail(request, "empty filename")
return
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.warn("ERROR can not create sub dir " + dirname)
SendFail(request, "write error")
return
data = request.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(), {"free": donated_bytes})
if _Debug:
lg.out(_DebugLevel, "p2p_service.Data created a new space file")
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if request.OwnerID not in space_dict.keys():
lg.warn("no info about donated space for %s" % request.OwnerID)
SendFail(request, "no info about donated space")
return
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
if request.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(used_space_dict[request.OwnerID])
bytes_donated_to_customer = int(space_dict[request.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space for %s" % request.OwnerID)
SendFail(request, "no free space")
return
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.warn("ERROR can not write to " + str(filename))
SendFail(request, "write error")
return
SendAck(request, str(len(request.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
del data
if _Debug:
lg.out(
_DebugLevel,
"p2p_service.Data saved from [%s/%s] to %s"
% (nameurl.GetName(request.OwnerID), nameurl.GetName(request.CreatorID), filename),
)
def write_customers_quotas(new_space_dict):
return bpio._write_dict(settings.CustomersSpaceFile(), new_space_dict)
def write_customers_quotas(new_space_dict):
new_space_dict_text = {strng.to_text(k): v for k, v in new_space_dict.items()}
return bpio._write_dict(settings.CustomersSpaceFile(), new_space_dict_text)
def Data(request):
"""
This is when we 1) save my requested data to restore the backup 2) or save
the customer file on our local HDD.
"""
if _Debug:
lg.out(
_DebugLevel, 'p2p_service.Data %d bytes in [%s] by %s | %s' %
(len(request.Payload), request.PacketID, request.OwnerID,
request.CreatorID))
# 1. this is our Data!
if request.OwnerID == my_id.getLocalID():
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data %r for us from %s" %
(request, nameurl.GetName(request.RemoteID)))
if driver.is_on('service_backups'):
# TODO: move this into callback
settings.BackupIndexFileName()
indexPacketID = global_id.MakeGlobalID(
idurl=my_id.getLocalID(), path=settings.BackupIndexFileName())
if request.PacketID == indexPacketID:
from storage import backup_control
backup_control.IncomingSupplierBackupIndex(request)
return True
return False
# 2. this Data is not belong to us
if not driver.is_on('service_supplier'):
return SendFail(request, 'supplier service is off')
if not contactsdb.is_customer(request.OwnerID): # SECURITY
lg.warn("%s not a customer, packetID=%s" %
(request.OwnerID, request.PacketID))
SendFail(request, 'not a customer')
return
glob_path = global_id.ParseGlobalID(request.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(my_id.getGlobalID() + ':' +
request.PacketID)
if not glob_path['path']:
lg.warn("got incorrect PacketID")
SendFail(request, 'incorrect PacketID')
return
# TODO: process files from another customer : glob_path['idurl']
filename = makeFilename(request.OwnerID, glob_path['path'])
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID? ")
SendFail(request, 'empty filename')
return
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.warn("ERROR can not create sub dir " + dirname)
SendFail(request, 'write error')
return
data = request.Serialize()
donated_bytes = settings.getDonatedBytes()
if not os.path.isfile(settings.CustomersSpaceFile()):
bpio._write_dict(settings.CustomersSpaceFile(),
{'free': donated_bytes})
if _Debug:
lg.out(_DebugLevel, 'p2p_service.Data created a new space file')
space_dict = bpio._read_dict(settings.CustomersSpaceFile())
if request.OwnerID not in space_dict.keys():
lg.warn("no info about donated space for %s" % request.OwnerID)
SendFail(request, 'no info about donated space')
return
used_space_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
if request.OwnerID in used_space_dict.keys():
try:
bytes_used_by_customer = int(used_space_dict[request.OwnerID])
bytes_donated_to_customer = int(space_dict[request.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space for %s" % request.OwnerID)
SendFail(request, 'no free space')
return
except:
lg.exc()
if not bpio.WriteFile(filename, data):
lg.warn("ERROR can not write to " + str(filename))
SendFail(request, 'write error')
return
SendAck(request, str(len(request.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime)
del data
if _Debug:
lg.out(
_DebugLevel, "p2p_service.Data saved from [%s | %s] to %s" % (
request.OwnerID,
request.CreatorID,
filename,
))
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = bpio._read_dict(settings.CustomersSpaceFile())
if space is None:
printlog('SpaceTime ERROR can not read file ' + settings.CustomersSpaceFile())
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
idurl = nameurl.FilenameUrl(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
# if not os.access(path, os.R_OK | os.W_OK):
# return False
if not os.path.isfile(path):
return True
# if name in [settings.BackupIndexFileName(),]:
# return False
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
bpio.traverse_dir_recursive(cb, onecustdir)
currentV = 0
for path in sorted(timedict.keys(), key=lambda x: timedict[x], reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (cur:%s, max: %s)' % (str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' % (remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' % (remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
bpio._write_dict(settings.CustomersUsedSpaceFile(), used_space)
def doTestMyCapacity2(self, arg):
"""
Here are some values.
- donated_bytes : you set this in the config
- spent_bytes : how many space is taken from you by other users right now
- free_bytes = donated_bytes - spent_bytes : not yet allocated space
- used_bytes : size of all files, which you store on your disk for your customers
"""
current_customers = contactsdb.customers()
removed_customers = []
spent_bytes = 0
donated_bytes = settings.getDonatedBytes()
if os.path.isfile(settings.CustomersSpaceFile()):
space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {})
else:
space_dict = {'free': donated_bytes}
used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
lg.out(
8,
'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes)
try:
int(space_dict['free'])
for idurl, customer_bytes in space_dict.items():
if idurl != 'free':
spent_bytes += int(customer_bytes)
except:
lg.exc()
space_dict = {'free': donated_bytes}
spent_bytes = 0
removed_customers = list(current_customers)
current_customers = []
self.automat('space-overflow',
(space_dict, spent_bytes, current_customers,
removed_customers))
return
lg.out(8, ' spent=%d' % spent_bytes)
if spent_bytes < donated_bytes:
space_dict['free'] = donated_bytes - spent_bytes
bpio._write_dict(settings.CustomersSpaceFile(), space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
used_space_ratio_dict = {}
for customer_pos in range(contactsdb.num_customers()):
customer_idurl = contactsdb.customer(customer_pos)
try:
allocated_bytes = int(space_dict[customer_idurl])
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated space unknown' % customer_idurl)
continue
if allocated_bytes <= 0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated_bytes==0' % customer_idurl)
continue
try:
files_size = int(used_dict.get(customer_idurl, 0))
ratio = float(files_size) / float(allocated_bytes)
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s used_dict have wrong value' % customer_idurl)
continue
if ratio > 1.0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
spent_bytes -= allocated_bytes
lg.warn('%s space overflow, where is bptester?' %
customer_idurl)
continue
used_space_ratio_dict[customer_idurl] = ratio
customers_sorted = sorted(
current_customers,
key=lambda i: used_space_ratio_dict[i],
)
while len(customers_sorted) > 0:
customer_idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[customer_idurl])
spent_bytes -= allocated_bytes
space_dict.pop(customer_idurl)
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
lg.out(8, ' customer %s REMOVED' % customer_idurl)
if spent_bytes < donated_bytes:
break
space_dict['free'] = donated_bytes - spent_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat(
'space-overflow',
(space_dict, spent_bytes, current_customers, removed_customers))
def doTestMyCapacity2(self, arg):
"""
Here are some values.
- donated_bytes : you set this in the config
- spent_bytes : how many space is taken from you by other users right now
- free_bytes = donated_bytes - spent_bytes : not yet allocated space
- used_bytes : size of all files, which you store on your disk for your customers
"""
current_customers = contactsdb.customers()
removed_customers = []
spent_bytes = 0
donated_bytes = settings.getDonatedBytes()
if os.path.isfile(settings.CustomersSpaceFile()):
space_dict = bpio._read_dict(settings.CustomersSpaceFile(), {})
else:
space_dict = {'free': donated_bytes}
used_dict = bpio._read_dict(settings.CustomersUsedSpaceFile(), {})
lg.out(8, 'customers_rejector.doTestMyCapacity donated=%d' % donated_bytes)
try:
int(space_dict['free'])
for idurl, customer_bytes in space_dict.items():
if idurl != 'free':
spent_bytes += int(customer_bytes)
except:
lg.exc()
space_dict = {'free': donated_bytes}
spent_bytes = 0
removed_customers = list(current_customers)
current_customers = []
self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))
return
lg.out(8, ' spent=%d' % spent_bytes)
if spent_bytes < donated_bytes:
space_dict['free'] = donated_bytes - spent_bytes
bpio._write_dict(settings.CustomersSpaceFile(), space_dict)
lg.out(8, ' space is OK !!!!!!!!')
self.automat('space-enough')
return
used_space_ratio_dict = {}
for customer_pos in xrange(contactsdb.num_customers()):
customer_idurl = contactsdb.customer(customer_pos)
try:
allocated_bytes = int(space_dict[customer_idurl])
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated space unknown' % customer_idurl)
continue
if allocated_bytes <= 0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s allocated_bytes==0' % customer_idurl)
continue
try:
files_size = int(used_dict.get(customer_idurl, 0))
ratio = float(files_size) / float(allocated_bytes)
except:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
lg.warn('%s used_dict have wrong value' % customer_idurl)
continue
if ratio > 1.0:
if customer_idurl in current_customers:
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
else:
lg.warn('%s not customers' % customer_idurl)
spent_bytes -= allocated_bytes
lg.warn('%s space overflow, where is bptester?' % customer_idurl)
continue
used_space_ratio_dict[customer_idurl] = ratio
customers_sorted = sorted(current_customers,
key=lambda i: used_space_ratio_dict[i],)
while len(customers_sorted) > 0:
customer_idurl = customers_sorted.pop()
allocated_bytes = int(space_dict[customer_idurl])
spent_bytes -= allocated_bytes
space_dict.pop(customer_idurl)
current_customers.remove(customer_idurl)
removed_customers.append(customer_idurl)
lg.out(8, ' customer %s REMOVED' % customer_idurl)
if spent_bytes < donated_bytes:
break
space_dict['free'] = donated_bytes - spent_bytes
lg.out(8, ' SPACE NOT ENOUGH !!!!!!!!!!')
self.automat('space-overflow', (space_dict, spent_bytes, current_customers, removed_customers))