def process_query_item(query_path, key_alias, ownerdir):
ret = ''
ret += 'Q%s\n' % query_path
if query_path == '*':
for one_key_alias in os.listdir(ownerdir):
if not misc.ValidKeyAlias(strng.to_text(one_key_alias)):
continue
key_alias_dir = os.path.join(ownerdir, one_key_alias)
ret += TreeSummary(key_alias_dir, key_alias=one_key_alias)
if _Debug:
lg.args(_DebugLevel, query_path=query_path, result_bytes=len(ret))
return ret
# TODO: more validations to be added
clean_path = query_path.replace('.', '').replace('~', '').replace(
':', '').replace('\\', '/').lstrip('/')
path_items = clean_path.split('/')
path_items.insert(0, ownerdir)
local_path = os.path.join(*path_items)
if not os.path.exists(local_path):
lg.warn('local file or folder not exist: %r' % local_path)
return ''
if os.path.isdir(local_path):
ret += TreeSummary(local_path, key_alias=key_alias)
if _Debug:
lg.args(_DebugLevel,
query_path=query_path,
local_path=local_path,
result_bytes=len(ret))
return ret
def is_valid_key_id(global_key_id):
parts = global_id.ParseGlobalID(global_key_id)
if not parts['key_alias']:
lg.warn('no key_alias found in the input')
return False
if not parts['idurl']:
lg.warn('no idurl found in the input')
return False
if not misc.ValidKeyAlias(parts['key_alias']):
lg.warn('invalid key alias in the input')
return False
return True
def send(customer_idurl, packet_id, format_type, key_id, remote_idurl):
if not my_keys.is_key_registered(key_id):
lg.warn(
'not able to return Files() for customer %s, key %s not registered'
% (
customer_idurl,
key_id,
))
return p2p_service.SendFailNoRequest(customer_idurl, packet_id)
if _Debug:
lg.out(
_DebugLevel,
"list_files.send to %s, customer_idurl=%s, key_id=%s" % (
remote_idurl,
customer_idurl,
key_id,
))
ownerdir = settings.getCustomerFilesDir(customer_idurl)
plaintext = ''
if os.path.isdir(ownerdir):
for key_alias in os.listdir(ownerdir):
if not misc.ValidKeyAlias(str(key_alias)):
continue
key_alias_dir = os.path.join(ownerdir, key_alias)
plaintext += TreeSummary(key_alias_dir, key_alias)
else:
lg.warn('did not found customer dir: %s' % ownerdir)
if _Debug:
lg.out(_DebugLevel + 8, '\n%s' % plaintext)
raw_list_files = PackListFiles(plaintext, format_type)
block = encrypted.Block(
CreatorID=my_id.getLocalID(),
BackupID=key_id,
Data=raw_list_files,
SessionKey=key.NewSessionKey(),
EncryptKey=key_id,
)
encrypted_list_files = block.Serialize()
newpacket = p2p_service.SendFiles(
idurl=remote_idurl,
raw_list_files_info=encrypted_list_files,
packet_id=packet_id,
callbacks={
commands.Ack(): on_acked,
commands.Fail(): on_failed,
None: on_timeout,
},
)
return newpacket
def send(customer_idurl, packet_id, format_type):
customer_name = nameurl.GetName(customer_idurl)
if _Debug:
lg.out(
_DebugLevel, "list_files.send to %s, format is '%s'" %
(customer_name, format_type))
ownerdir = settings.getCustomerFilesDir(customer_idurl)
plaintext = ''
if os.path.isdir(ownerdir):
for key_alias in os.listdir(ownerdir):
if not misc.ValidKeyAlias(str(key_alias)):
continue
key_alias_dir = os.path.join(ownerdir, key_alias)
plaintext += TreeSummary(key_alias_dir, key_alias)
else:
lg.warn('did not found customer dir: %s' % ownerdir)
if _Debug:
lg.out(_DebugLevel + 8, '\n%s' % plaintext)
return p2p_service.SendFiles(
idurl=customer_idurl,
raw_list_files_info=PackListFiles(plaintext, format_type),
packet_id=packet_id,
)
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = accounting.read_customers_quotas()
if space is None:
printlog(
'SpaceTime ERROR customers quotas file can not be read or it is empty, skip'
)
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = accounting.read_customers_usage()
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
for key_alias in os.listdir(onecustdir):
if not misc.ValidKeyAlias(key_alias):
remove_list[onecustdir] = 'invalid key alias'
continue
okekeydir = os.path.join(onecustdir, key_alias)
bpio.traverse_dir_recursive(cb, okekeydir)
currentV = 0
for path in sorted(list(timedict.keys()),
key=lambda x: timedict[x],
reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path +
' file removed (cur:%s, max: %s)' %
(str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
accounting.update_customers_usage(used_space)
