def report_local_storage():
# TODO
# if customers folder placed outside of BaseDir()
# need to add: total = total + customers
r = {}
r['backups'] = bpio.getDirectorySize(settings.getLocalBackupsDir())
# r['backups_str'] = diskspace.MakeStringFromBytes(r['backups'])
r['temp'] = bpio.getDirectorySize(settings.getTempDir())
# r['temp_str'] = diskspace.MakeStringFromBytes(r['temp'])
r['customers'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
# r['customers_str'] = diskspace.MakeStringFromBytes(r['customers'])
r['total'] = bpio.getDirectorySize(settings.BaseDir())
# r['total_str'] = diskspace.MakeStringFromBytes(r['total'])
dataDriveFreeSpace, dataDriveTotalSpace = diskusage.GetDriveSpace(
settings.getCustomersFilesDir())
if dataDriveFreeSpace is None:
dataDriveFreeSpace = 0
r['disktotal'] = int(dataDriveTotalSpace)
# r['disktotal_str'] = diskspace.MakeStringFromBytes(r['disktotal'])
r['diskfree'] = int(dataDriveFreeSpace)
# r['diskfree_str'] = diskspace.MakeStringFromBytes(r['diskfree'])
try:
r['total_percent'] = misc.value2percent(float(r['total']),
float(r['disktotal']), 5)
except:
r['total_percent'] = ''
try:
r['diskfree_percent'] = misc.value2percent(float(r['diskfree']),
float(r['disktotal']), 5)
except:
r['diskfree_percent'] = ''
return r
def report_local_storage():
# TODO
# if customers folder placed outside of BaseDir()
# need to add: total = total + customers
r = {}
r['backups'] = bpio.getDirectorySize(settings.getLocalBackupsDir())
r['backups_str'] = diskspace.MakeStringFromBytes(r['backups'])
r['temp'] = bpio.getDirectorySize(settings.getTempDir())
r['temp_str'] = diskspace.MakeStringFromBytes(r['temp'])
r['customers'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
r['customers_str'] = diskspace.MakeStringFromBytes(r['customers'])
r['total'] = bpio.getDirectorySize(settings.GetBaseDir())
r['total_str'] = diskspace.MakeStringFromBytes(r['total'])
dataDriveFreeSpace, dataDriveTotalSpace = diskusage.GetDriveSpace(settings.getCustomersFilesDir())
if dataDriveFreeSpace is None:
dataDriveFreeSpace = 0
r['disktotal'] = int(dataDriveTotalSpace)
r['disktotal_str'] = diskspace.MakeStringFromBytes(r['disktotal'])
r['diskfree'] = int(dataDriveFreeSpace)
r['diskfree_str'] = diskspace.MakeStringFromBytes(r['diskfree'])
try:
r['total_percent'] = misc.percent2string(float(r['total']) / float(r['disktotal']), 5)
except:
r['total_percent'] = ''
try:
r['diskfree_percent'] = misc.percent2string(float(r['diskfree']) / float(r['disktotal']), 5)
except:
r['diskfree_percent'] = ''
return r
def send(customer_idurl, packet_id, format_type):
customer_name = nameurl.GetName(customer_idurl)
MyID = my_id.getLocalID()
RemoteID = customer_idurl
PacketID = packet_id
if _Debug:
lg.out(
_DebugLevel, "list_files.send to %s, format is '%s'" %
(customer_name, format_type))
custdir = settings.getCustomersFilesDir()
ownerdir = os.path.join(custdir, nameurl.UrlFilename(customer_idurl))
if not os.path.isdir(ownerdir):
if _Debug:
lg.out(_DebugLevel,
"list_files.send did not found customer dir: " + ownerdir)
src = PackListFiles('', format_type)
result = signed.Packet(commands.Files(), MyID, MyID, PacketID, src,
RemoteID)
gateway.outbox(result)
return result
plaintext = TreeSummary(ownerdir)
if _Debug:
lg.out(_DebugLevel + 8, '\n%s' % (plaintext))
src = PackListFiles(plaintext, format_type)
result = signed.Packet(commands.Files(), MyID, MyID, PacketID, src,
RemoteID)
gateway.outbox(result)
return result
def ListFiles(request):
"""
We will want to use this to see what needs to be resent, and expect normal
case is very few missing.
This is to build the ``Files()`` we are holding for a customer.
"""
if not driver.is_started("service_supplier"):
return SendFail(request, "supplier service is off")
MyID = my_id.getLocalID()
RemoteID = request.OwnerID
PacketID = request.PacketID
Payload = request.Payload
if _Debug:
lg.out(
_DebugLevel, "p2p_service.ListFiles from [%s], format is %s" % (nameurl.GetName(request.OwnerID), Payload)
)
custdir = settings.getCustomersFilesDir()
ownerdir = os.path.join(custdir, nameurl.UrlFilename(request.OwnerID))
if not os.path.isdir(ownerdir):
if _Debug:
lg.out(_DebugLevel, "p2p_service.ListFiles did not find customer dir " + ownerdir)
src = PackListFiles("", Payload)
result = signed.Packet(commands.Files(), MyID, MyID, PacketID, src, RemoteID)
gateway.outbox(result)
return result
plaintext = TreeSummary(ownerdir)
if _Debug:
lg.out(_DebugLevel + 4, "\n%s" % (plaintext))
src = PackListFiles(plaintext, Payload)
result = signed.Packet(commands.Files(), MyID, MyID, PacketID, src, RemoteID)
gateway.outbox(result)
return result
def UpdateCustomers():
"""
Test packets after list of customers was changed.
"""
space, _ = accounting.read_customers_quotas()
if space is None:
printlog('UpdateCustomers ERROR space file can not be read')
return False
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('UpdateCustomers ERROR customers folder not exist')
return False
remove_list = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl.to_bin(), None)
if curspace is None:
remove_list[onecustdir] = 'is not a customer'
continue
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('UpdateCustomers %r folder removed (%s)' % (
path,
remove_list[path],
))
except:
printlog('UpdateCustomers ERROR removing %r' % path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('UpdateCustomers %r file removed (%s)' % (
path,
remove_list[path],
))
except:
printlog('UpdateCustomers ERROR removing %r' % path)
printlog('UpdateCustomers %r' %
time.strftime("%a, %d %b %Y %H:%M:%S +0000"))
return True
def Validate():
"""
Check all packets to be valid.
"""
printlog('Validate %r' % time.strftime("%a, %d %b %Y %H:%M:%S +0000"))
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
return False
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
continue
for key_alias_filename in os.listdir(onecustdir):
onekeydir = os.path.join(onecustdir, key_alias_filename)
if not os.path.isdir(onekeydir):
continue
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
packetsrc = bpio.ReadBinaryFile(path)
if not packetsrc:
try:
os.remove(
path) # if is is no good it is of no use to anyone
printlog('Validate %r removed (empty file)' % path)
except:
printlog('Validate ERROR removing %r' % path)
return False
p = signed.Unserialize(packetsrc)
if p is None:
try:
os.remove(
path) # if is is no good it is of no use to anyone
printlog('Validate %r removed (unserialize error)' %
path)
except:
printlog('Validate ERROR removing %r')
return False
result = p.Valid()
packetsrc = ''
del p
if not result:
try:
os.remove(
path) # if is is no good it is of no use to anyone
printlog('Validate %r removed (invalid packet)' % path)
except:
printlog('Validate ERROR removing %r' % path)
return False
time.sleep(0.1)
return False
bpio.traverse_dir_recursive(cb, onekeydir)
return True
def constructFilename(customerID, packetID):
customerDirName = nameurl.UrlFilename(customerID)
customersDir = settings.getCustomersFilesDir()
if not os.path.exists(customersDir):
bpio._dir_make(customersDir)
ownerDir = os.path.join(customersDir, customerDirName)
if not os.path.exists(ownerDir):
bpio._dir_make(ownerDir)
filename = os.path.join(ownerDir, packetID)
return filename
def Validate():
"""
Check all packets to be valid.
"""
printlog('Validate ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
contactsdb.init()
commands.init()
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
return
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
continue
for key_alias_filename in os.listdir(onecustdir):
onekeydir = os.path.join(onecustdir, key_alias_filename)
if not os.path.isdir(onekeydir):
continue
def cb(path, subpath, name):
# if not os.access(path, os.R_OK | os.W_OK):
# return False
if not os.path.isfile(path):
return True
# if name in [settings.BackupIndexFileName(),]:
# return False
packetsrc = bpio.ReadBinaryFile(path)
if not packetsrc:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog('Validate ' + path + ' removed (empty file)')
except:
printlog('Validate ERROR removing ' + path)
return False
p = signed.Unserialize(packetsrc)
if p is None:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog('Validate ' + path + ' removed (unserialize error)')
except:
printlog('Validate ERROR removing ' + path)
return False
result = p.Valid()
packetsrc = ''
del p
if not result:
try:
os.remove(path) # if is is no good it is of no use to anyone
printlog('Validate ' + path + ' removed (invalid packet)')
except:
printlog('Validate ERROR removing ' + path)
return False
time.sleep(0.1)
return False
bpio.traverse_dir_recursive(cb, onekeydir)
def ListCustomerFiles1(customerNumber):
"""
On the status form when clicking on a customer, find out what files we're
holding for that customer.
"""
idurl = contactsdb.customer(customerNumber)
filename = nameurl.UrlFilename(idurl)
customerDir = os.path.join(settings.getCustomersFilesDir(), filename)
if os.path.exists(customerDir) and os.path.isdir(customerDir):
backupFilesList = os.listdir(customerDir)
if len(backupFilesList) > 0:
return ListSummary(backupFilesList)
return "No files stored for this customer"
def ListCustomerFiles1(customerNumber):
"""
On the status form when clicking on a customer, find out what files we're
holding for that customer.
"""
idurl = contactsdb.customer(customerNumber)
filename = nameurl.UrlFilename(idurl)
customerDir = os.path.join(settings.getCustomersFilesDir(), filename)
if os.path.exists(customerDir) and os.path.isdir(customerDir):
backupFilesList = os.listdir(customerDir)
if len(backupFilesList) > 0:
return ListSummary(backupFilesList)
return "No files stored for this customer"
def OkToShareSpace(desiredSharedSpaceMB):
"""
Make sure a user really has the space they claim they want to share.
"""
dataDir = settings.getCustomersFilesDir()
dataDriveFreeSpace, dataDriveTotalSpace = GetDriveSpace(dataDir)
if dataDriveFreeSpace is None:
return False
currentlySharedSpace = GetDirectorySize(dataDir)
if (currentlySharedSpace + dataDriveFreeSpace /
(1024 * 1024)) < desiredSharedSpaceMB:
return False
else:
return True
def constructFilename(customerIDURL, packetID):
customerGlobID, packetID = packetid.SplitPacketID(packetID)
if customerGlobID:
customerIDURL_packet = global_id.GlobalUserToIDURL(customerGlobID)
if customerIDURL_packet != customerIDURL:
lg.warn('construct filename for another customer: %s != %s' %
(customerIDURL_packet, customerIDURL))
customerDirName = nameurl.UrlFilename(customerIDURL)
customersDir = settings.getCustomersFilesDir()
if not os.path.exists(customersDir):
bpio._dir_make(customersDir)
ownerDir = os.path.join(customersDir, customerDirName)
if not os.path.exists(ownerDir):
bpio._dir_make(ownerDir)
filename = os.path.join(ownerDir, packetID)
return filename
def ListCustomerFiles(customer_idurl):
filename = nameurl.UrlFilename(customer_idurl)
customer_dir = os.path.join(settings.getCustomersFilesDir(), filename)
result = cStringIO.StringIO()
def cb(realpath, subpath, name):
if os.path.isdir(realpath):
result.write('D%s\n' % subpath)
else:
result.write('F%s\n' % subpath)
return True
bpio.traverse_dir_recursive(cb, customer_dir)
src = result.getvalue()
result.close()
return src
def ListCustomerFiles(customer_idurl):
filename = nameurl.UrlFilename(customer_idurl)
customer_dir = os.path.join(settings.getCustomersFilesDir(), filename)
result = cStringIO.StringIO()
def cb(realpath, subpath, name):
if os.path.isdir(realpath):
result.write("D%s\n" % subpath)
else:
result.write("F%s\n" % subpath)
return True
bpio.traverse_dir_recursive(cb, customer_dir)
src = result.getvalue()
result.close()
return src
def UpdateCustomers():
"""
Test packets after list of customers was changed.
"""
space = bpio._read_dict(settings.CustomersSpaceFile())
if space is None:
printlog('UpdateCustomers ERROR space file can not be read')
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('UpdateCustomers ERROR customers folder not exist')
return
remove_list = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
idurl = nameurl.FilenameUrl(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'is not a customer'
continue
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('UpdateCustomers ' + path + ' folder removed (%s)' % (remove_list[path]))
except:
printlog('UpdateCustomers ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('UpdateCustomers ' + path + ' file removed (%s)' % (remove_list[path]))
except:
printlog('UpdateCustomers ERROR removing ' + path)
printlog('UpdateCustomers ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
def main():
"""
This method is for tests.
Need to move all things here to unit tests. TODO.
"""
dataDir = settings.getCustomersFilesDir()
tempDir = settings.TempDir()
dataDriveFreeSpace = 0
dataDriveTotalSpace = 0
tempDriveFreeSpace = 0
tempDriveTotalSpace = 0
dataDriveFreeSpace, dataDriveTotalSpace = GetDriveSpace(dataDir)
tempDriveFreeSpace, tempDriveTotalSpace = GetDriveSpace(tempDir)
print "data dir =", dataDir
print "tep dir =", tempDir
print "data dir: " + str(dataDriveFreeSpace /
(1024 * 1024)) + "MB free/" + str(
dataDriveTotalSpace /
(1024 * 1024)) + "MB total"
print "temp dir: " + str(tempDriveFreeSpace /
(1024 * 1024)) + "MB free/" + str(
tempDriveTotalSpace /
(1024 * 1024)) + "MB total"
print time.time()
print "our temp files: " + str(
GetOurTempFileSizeTotal(tempDir) / (1024 * 1024)) + "MB"
print time.time()
GetDirectorySize(dataDir)
ds = diskspace.DiskSpace()
print ds.getValueBest(dataDriveFreeSpace)
print "at OkToShareSpace ..."
print "ok to share 100MB - should be true"
print OkToShareSpace(100)
print "ok to share 12345678MB - should be false"
print OkToShareSpace(12345678)
def make_filename(customerGlobID, packetID, keyAlias=None):
keyAlias = keyAlias or 'master'
customerDirName = str(customerGlobID)
customersDir = settings.getCustomersFilesDir()
if not os.path.exists(customersDir):
if _Debug:
lg.dbg(_DebugLevel, 'making a new folder: %s' % customersDir)
bpio._dir_make(customersDir)
ownerDir = os.path.join(customersDir, customerDirName)
if not os.path.exists(ownerDir):
if _Debug:
lg.dbg(_DebugLevel, 'making a new folder: %s' % ownerDir)
bpio._dir_make(ownerDir)
keyAliasDir = os.path.join(ownerDir, keyAlias)
if not os.path.exists(keyAliasDir):
if _Debug:
lg.dbg(_DebugLevel, 'making a new folder: %s' % keyAliasDir)
bpio._dir_make(keyAliasDir)
filename = os.path.join(keyAliasDir, packetID)
return filename
def OkToShareSpace(desiredSharedSpaceMB):
"""
Make sure a user really has the space they claim they want to share.
"""
dataDir = settings.getCustomersFilesDir()
dataDriveFreeSpace, dataDriveTotalSpace = GetDriveSpace(dataDir)
if not dataDriveFreeSpace or not dataDriveTotalSpace:
return False
try:
# TODO: say if less than 10% free storage left of your HDD do not share
testFree = (dataDriveFreeSpace / dataDriveTotalSpace) > 0.1
except:
testFree = False
if not testFree:
return False
currentlySharedSpace = GetDirectorySize(dataDir)
if (currentlySharedSpace + dataDriveFreeSpace /
(1024 * 1024)) < desiredSharedSpaceMB:
return False
return True
def _do_construct_filename(self, customerGlobID, packetID, keyAlias=None):
import os
from logs import lg
from main import settings
from system import bpio
keyAlias = keyAlias or 'master'
customerDirName = str(customerGlobID)
customersDir = settings.getCustomersFilesDir()
if not os.path.exists(customersDir):
lg.info('making a new folder: ' + customersDir)
bpio._dir_make(customersDir)
ownerDir = os.path.join(customersDir, customerDirName)
if not os.path.exists(ownerDir):
lg.info('making a new folder: ' + ownerDir)
bpio._dir_make(ownerDir)
keyAliasDir = os.path.join(ownerDir, keyAlias)
if not os.path.exists(keyAliasDir):
lg.info('making a new folder: ' + keyAliasDir)
bpio._dir_make(keyAliasDir)
filename = os.path.join(keyAliasDir, packetID)
return filename
def __init__(self):
self.installer_state_to_page = {
'AT_STARTUP': self.renderSelectPage,
'WHAT_TO_DO?': self.renderSelectPage,
'INPUT_NAME': self.renderInputNamePage,
'REGISTER': self.renderRegisterNewUserPage,
'AUTHORIZED': self.renderRegisterNewUserPage,
'LOAD_KEY': self.renderLoadKeyPage,
'RECOVER': self.renderRestorePage,
'RESTORED': self.renderRestorePage,
'WIZARD': self.renderWizardPage,
'DONE': self.renderLastPage,
}
self.install_wizard_state_to_page = {
'READY': self.renderWizardStartPage,
'STORAGE': self.renderWizardStoragePage,
'CONTACTS': self.renderWizardContactsPage,
'LAST_PAGE': self.renderLastPage,
'DONE': self.renderLastPage,
}
self.data = {
'username': bpio.ReadTextFile(settings.UserNameFilename()).strip(),
'pksize': settings.DefaultPrivateKeySize(),
'needed': str(int(settings.DefaultNeededBytes() / (1024 * 1024))),
'donated': str(int(settings.DefaultDonatedBytes() / (1024 * 1024))),
'suppliers': str(settings.DefaultDesiredSuppliers()),
'customersdir': unicode(settings.getCustomersFilesDir()),
'localbackupsdir': unicode(settings.getLocalBackupsDir()),
'restoredir': unicode(settings.getRestoreDir()),
'idurl': '',
'keysrc': '',
'name': '',
'surname': '',
'nickname': '',
}
installer.A('init')
# This file (treesummary.py) is part of BitDust Software. # # BitDust is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # BitDust Software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with BitDust Software. If not, see <http://www.gnu.org/licenses/>. # # Please contact us if you have any questions at [email protected] import os import sys import os.path as _p sys.path.insert(0, _p.abspath(_p.join(_p.dirname(_p.abspath(sys.argv[0])), '..'))) from logs import lg from system import bpio from p2p import p2p_service from main import settings from lib import nameurl custdir = settings.getCustomersFilesDir() ownerdir = os.path.join(custdir, nameurl.UrlFilename('http://megafaq.ru/e_vps1004.xml')) plaintext = p2p_service.TreeSummary(ownerdir) print plaintext
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = bpio._read_dict(settings.CustomersSpaceFile())
if space is None:
printlog('SpaceTime ERROR can not read file ' + settings.CustomersSpaceFile())
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
idurl = nameurl.FilenameUrl(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
# if not os.access(path, os.R_OK | os.W_OK):
# return False
if not os.path.isfile(path):
return True
# if name in [settings.BackupIndexFileName(),]:
# return False
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
bpio.traverse_dir_recursive(cb, onecustdir)
currentV = 0
for path in sorted(timedict.keys(), key=lambda x: timedict[x], reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (cur:%s, max: %s)' % (str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' % (remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' % (remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
bpio._write_dict(settings.CustomersUsedSpaceFile(), used_space)
def report_donated_storage():
space_dict, free_space = read_customers_quotas()
used_space_dict = read_customers_usage()
r = {}
r['customers_num'] = contactsdb.num_customers()
r['customers'] = []
r['old_customers'] = []
r['errors'] = []
r['consumed'] = 0
r['donated'] = settings.getDonatedBytes()
# r['donated_str'] = diskspace.MakeStringFromBytes(r['donated'])
r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
try:
r['free'] = int(free_space)
except:
r['free'] = 0
used = 0
for idurl in id_url.to_bin_list(contactsdb.customers()):
consumed_by_customer = 0
used_by_customer = 0
if idurl not in list(space_dict.keys()):
r['errors'].append('space consumed by customer %r is unknown' %
idurl)
else:
try:
consumed_by_customer = int(space_dict.pop(idurl))
r['consumed'] += consumed_by_customer
except:
r['errors'].append(
'incorrect value of consumed space for customer %r' %
idurl)
continue
if idurl in list(used_space_dict.keys()):
try:
used_by_customer = int(used_space_dict.pop(idurl))
used += used_by_customer
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
if consumed_by_customer < used_by_customer:
r['errors'].append(
'customer %r currently using more space than requested' %
idurl)
c = {}
c['idurl'] = strng.to_text(idurl)
c['used'] = used_by_customer
# c['used_str'] = diskspace.MakeStringFromBytes(c['used'])
c['consumed'] = consumed_by_customer
# c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed'])
c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
# c['real_str'] = diskspace.MakeStringFromBytes(c['real'])
r['customers'].append(c)
r['used'] = used
# r['used_str'] = diskspace.MakeStringFromBytes(r['used'])
# r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed'])
if r['donated'] != r['free'] + r['consumed']:
r['errors'].append(
'total consumed %d and known free %d (%d total) bytes not match with donated %d bytes'
% (r['consumed'], r['free'], r['consumed'] + r['free'],
r['donated']))
if r['used'] > r['donated']:
r['errors'].append(
'total space used by customers exceed the donated limit')
if len(space_dict) > 0:
r['errors'].append('found %d incorrect records of consumed space' %
len(space_dict))
if r['real'] != r['used']:
r['errors'].append(
'current info needs update, known size is %d bytes but real is %d bytes'
% (r['used'], r['real']))
old_customers_used = 0
old_customers_real = 0
for idurl in used_space_dict.keys():
real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
try:
used = int(used_space_dict[idurl])
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
r['old_customers'].append({
'idurl': strng.to_text(idurl),
'used': used,
# 'used_str': diskspace.MakeStringFromBytes(used_space_dict[idurl]),
'real': real,
# 'real_str': diskspace.MakeStringFromBytes(real),
})
old_customers_used += used
old_customers_real += real
r['old_customers_used'] = old_customers_used
r['old_customers_real'] = old_customers_real
try:
r['used_percent'] = misc.value2percent(float(r['used']),
float(r['donated']), 5)
except:
r['used_percent'] = ''
try:
r['consumed_percent'] = misc.value2percent(float(r['consumed']),
float(r['donated']), 5)
except:
r['consumed_percent'] = ''
return r
def on_identity_url_changed(evt):
old_idurl = id_url.field(evt.data['old_idurl'])
# update customer idurl in "space" file
contacts_changed = False
for customer_idurl in contactsdb.customers():
if old_idurl == customer_idurl:
customer_idurl.refresh()
contacts_changed = True
lg.info('found customer idurl rotated : %r -> %r' % (
evt.data['old_idurl'],
evt.data['new_idurl'],
))
if contacts_changed:
contactsdb.save_customers()
# update meta info for that customer
meta_info_changed = False
all_meta_info = contactsdb.read_customers_meta_info_all()
for customer_idurl_bin in list(all_meta_info.keys()):
if id_url.is_cached(old_idurl) and id_url.is_cached(
customer_idurl_bin):
if old_idurl == id_url.field(customer_idurl_bin):
latest_customer_idurl_bin = id_url.field(
customer_idurl_bin).to_bin()
if latest_customer_idurl_bin != customer_idurl_bin:
all_meta_info[
latest_customer_idurl_bin] = all_meta_info.pop(
customer_idurl_bin)
meta_info_changed = True
lg.info(
'found customer idurl rotated in customers meta info : %r -> %r'
% (
latest_customer_idurl_bin,
customer_idurl_bin,
))
if meta_info_changed:
contactsdb.write_customers_meta_info_all(all_meta_info)
# update customer idurl in "space" file
space_dict, free_space = accounting.read_customers_quotas()
space_changed = False
for customer_idurl_bin in list(space_dict.keys()):
if id_url.is_cached(old_idurl) and id_url.is_cached(
customer_idurl_bin):
if id_url.field(customer_idurl_bin) == old_idurl:
latest_customer_idurl_bin = id_url.field(
customer_idurl_bin).to_bin()
if latest_customer_idurl_bin != customer_idurl_bin:
space_dict[latest_customer_idurl_bin] = space_dict.pop(
customer_idurl_bin)
space_changed = True
lg.info(
'found customer idurl rotated in customer quotas dictionary : %r -> %r'
% (
latest_customer_idurl_bin,
customer_idurl_bin,
))
if space_changed:
accounting.write_customers_quotas(space_dict, free_space)
# rename customer folder where I store all his files
old_customer_dirname = str(global_id.UrlToGlobalID(evt.data['old_idurl']))
new_customer_dirname = str(global_id.UrlToGlobalID(evt.data['new_idurl']))
customers_dir = settings.getCustomersFilesDir()
old_owner_dir = os.path.join(customers_dir, old_customer_dirname)
new_owner_dir = os.path.join(customers_dir, new_customer_dirname)
if os.path.isdir(old_owner_dir):
try:
bpio.move_dir_recursive(old_owner_dir, new_owner_dir)
lg.info('copied %r into %r' % (
old_owner_dir,
new_owner_dir,
))
if os.path.exists(old_owner_dir):
bpio._dir_remove(old_owner_dir)
lg.warn('removed %r' % old_owner_dir)
except:
lg.exc()
# update customer idurl in "spaceused" file
local_tester.TestSpaceTime()
return True
# This file (treesummary.py) is part of BitDust Software. # # BitDust is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # BitDust Software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with BitDust Software. If not, see <http://www.gnu.org/licenses/>. # # Please contact us if you have any questions at [email protected] import os import sys import os.path as _p sys.path.insert(0, _p.abspath(_p.join(_p.dirname(_p.abspath(sys.argv[0])), '..'))) from logs import lg from system import bpio from p2p import p2p_service from main import settings from lib import nameurl custdir = settings.getCustomersFilesDir() ownerdir = os.path.join(custdir, nameurl.UrlFilename('http://megafaq.ru/e_vps1004.xml')) plaintext = p2p_service.TreeSummary(ownerdir) print plaintext
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = accounting.read_customers_quotas()
if space is None:
printlog(
'SpaceTime ERROR customers quotas file can not be read or it is empty, skip'
)
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = accounting.read_customers_usage()
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
for key_alias in os.listdir(onecustdir):
if not misc.ValidKeyAlias(key_alias):
remove_list[onecustdir] = 'invalid key alias'
continue
okekeydir = os.path.join(onecustdir, key_alias)
bpio.traverse_dir_recursive(cb, okekeydir)
currentV = 0
for path in sorted(list(timedict.keys()),
key=lambda x: timedict[x],
reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path +
' file removed (cur:%s, max: %s)' %
(str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
accounting.update_customers_usage(used_space)
def report_donated_storage():
space_dict = read_customers_quotas()
used_space_dict = read_customers_usage()
r = {}
r['customers_num'] = contactsdb.num_customers()
r['customers'] = []
r['old_customers'] = []
r['errors'] = []
r['consumed'] = 0
r['donated'] = settings.getDonatedBytes()
r['donated_str'] = diskspace.MakeStringFromBytes(r['donated'])
r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
try:
r['free'] = int(space_dict.pop('free'))
except:
r['free'] = 0
used = 0
for idurl in contactsdb.customers():
consumed_by_customer = 0
used_by_customer = 0
if idurl not in space_dict.keys():
r['errors'].append('space consumed by customer %s is unknown' % idurl)
else:
try:
consumed_by_customer = int(space_dict.pop(idurl))
r['consumed'] += consumed_by_customer
except:
r['errors'].append('incorrect value of consumed space for customer %s' % idurl)
if idurl in used_space_dict.keys():
try:
used_by_customer = int(used_space_dict.pop(idurl))
used += used_by_customer
except:
r['errors'].append('incorrect value of used space for customer %s' % idurl)
if consumed_by_customer < used_by_customer:
r['errors'].append('customer %s currently using more space than requested' % idurl)
c = {}
c['idurl'] = idurl
c['used'] = used_by_customer
c['used_str'] = diskspace.MakeStringFromBytes(c['used'])
c['consumed'] = consumed_by_customer
c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed'])
c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
c['real_str'] = diskspace.MakeStringFromBytes(c['real'])
r['customers'].append(c)
r['used'] = used
r['used_str'] = diskspace.MakeStringFromBytes(r['used'])
r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed'])
if r['donated'] != r['free'] + r['consumed']:
r['errors'].append('total consumed %d and known free %d (%d total) bytes not match with donated %d bytes' % (
r['consumed'], r['free'],
r['consumed'] + r['free'], r['donated']))
if r['used'] > r['donated']:
r['errors'].append('total space used by customers exceed the donated limit')
if len(space_dict) > 0:
r['errors'].append('found %d incorrect records of consumed space' % len(space_dict))
if r['real'] != r['used']:
r['errors'].append('current info needs update, known size is %d bytes but real is %d bytes' % (
r['used'], r['real']))
for idurl in used_space_dict.keys():
real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
r['old_customers'].append({
'idurl': idurl,
'used': used_space_dict['idurl'],
'used_str': diskspace.MakeStringFromBytes(used_space_dict['idurl']),
'real': real,
'real_str': diskspace.MakeStringFromBytes(real),
})
try:
r['used_percent'] = misc.percent2string(float(r['used']) / float(r['donated']), 5)
except:
r['used_percent'] = ''
try:
r['consumed_percent'] = misc.percent2string(float(r['consumed']) / float(r['donated']), 5)
except:
r['consumed_percent'] = ''
return r
def renderWizardStoragePage(self, request):
template = 'pages/wizard_storage.html'
req = {}
if request is not None:
req = request.REQUEST
self.data['customersdir'] = unicode(req.get('customersdir',
settings.getCustomersFilesDir()))
self.data['localbackupsdir'] = unicode(req.get('localbackupsdir',
settings.getLocalBackupsDir()))
self.data['restoredir'] = unicode(req.get('restoredir',
settings.getRestoreDir()))
self.data['needed'] = req.get('needed', self.data['needed'])
neededV = diskspace.GetBytesFromString(self.data['needed'] + ' Mb',
settings.DefaultNeededBytes())
self.data['donated'] = req.get('donated', self.data['donated'])
donatedV = diskspace.GetBytesFromString(self.data['donated'] + ' Mb',
settings.DefaultDonatedBytes())
self.data['suppliers'] = req.get('suppliers', self.data['suppliers'])
mounts = []
freeSpaceIsOk = True
if bpio.Windows():
for d in bpio.listLocalDrivesWindows():
free, total = diskusage.GetWinDriveSpace(d[0])
if free is None or total is None:
continue
color = '#ffffff'
if self.data['customersdir'][0].upper() == d[0].upper():
color = '#60e060'
if donatedV >= free:
color = '#e06060'
freeSpaceIsOk = False
if self.data['localbackupsdir'][0].upper() == d[0].upper():
color = '#60e060'
if neededV >= free:
color = '#e06060'
freeSpaceIsOk = False
mounts.append((d[0:2],
diskspace.MakeStringFromBytes(free),
diskspace.MakeStringFromBytes(total),
color,))
elif bpio.Linux() or bpio.Mac():
for mnt in bpio.listMountPointsLinux():
free, total = diskusage.GetLinuxDriveSpace(mnt)
if free is None or total is None:
continue
color = '#ffffff'
if bpio.getMountPointLinux(self.data['customersdir']) == mnt:
color = '#60e060'
if donatedV >= free:
color = '#e06060'
freeSpaceIsOk = False
if bpio.getMountPointLinux(self.data['localbackupsdir']) == mnt:
color = '#60e060'
if neededV >= free:
color = '#e06060'
freeSpaceIsOk = False
mounts.append((mnt,
diskspace.MakeStringFromBytes(free),
diskspace.MakeStringFromBytes(total),
color,))
ok = True
out = ''
if not freeSpaceIsOk:
out += '<font color=red>you do not have enough free space on the disk</font><br/>\n'
ok = False
if donatedV < settings.MinimumDonatedBytes():
out += '<font color=red>you must donate at least %f MB</font><br/>\n' % (
round(settings.MinimumDonatedBytes() / (1024.0 * 1024.0), 2))
ok = False
if not os.path.isdir(self.data['customersdir']):
out += '<font color=red>directory %s not exist</font><br/>\n' % self.data['customersdir']
ok = False
if not os.access(self.data['customersdir'], os.W_OK):
out += '<font color=red>folder %s does not have write permissions</font><br/>\n' % self.data['customersdir']
ok = False
if not os.path.isdir(self.data['localbackupsdir']):
out += '<font color=red>directory %s not exist</font><br/>\n' % self.data['localbackupsdir']
ok = False
if not os.access(self.data['localbackupsdir'], os.W_OK):
out += '<font color=red>folder %s does not have write permissions</font><br/>\n' % self.data['localbackupsdir']
ok = False
if int(self.data['suppliers']) not in settings.getECCSuppliersNumbers():
out += '<font color=red>incorrect number of suppliers, correct values are: %s</font><br/>\n' % (
str(settings.getECCSuppliersNumbers()).strip('[]'))
ok = False
context = {'output': out,
'mounts': mounts,
'needed': self.data['needed'],
'donated': self.data['donated'],
'localbackupsdir': self.data['localbackupsdir'],
'customersdir': self.data['customersdir'],
'restoredir': self.data['restoredir'],
'suppliers': self.data['suppliers'],
}
if request is None:
return template, context, request
action = request.REQUEST.get('action', None)
if action == 'next':
if ok:
install_wizard.A(action, self.data)
return None
if action == 'back':
install_wizard.A(action)
return None
return template, context, request
