def process_query_item(query_path, key_alias, ownerdir): ret = '' ret += 'Q%s\n' % query_path if query_path == '*': for one_key_alias in os.listdir(ownerdir): if not misc.ValidKeyAlias(strng.to_text(one_key_alias)): continue key_alias_dir = os.path.join(ownerdir, one_key_alias) ret += TreeSummary(key_alias_dir, key_alias=one_key_alias) if _Debug: lg.args(_DebugLevel, query_path=query_path, result_bytes=len(ret)) return ret # TODO: more validations to be added clean_path = query_path.replace('.', '').replace('~', '').replace( ':', '').replace('\\', '/').lstrip('/') path_items = clean_path.split('/') path_items.insert(0, ownerdir) local_path = os.path.join(*path_items) if not os.path.exists(local_path): lg.warn('local file or folder not exist: %r' % local_path) return '' if os.path.isdir(local_path): ret += TreeSummary(local_path, key_alias=key_alias) if _Debug: lg.args(_DebugLevel, query_path=query_path, local_path=local_path, result_bytes=len(ret)) return ret
def is_valid_key_id(global_key_id): parts = global_id.ParseGlobalID(global_key_id) if not parts['key_alias']: lg.warn('no key_alias found in the input') return False if not parts['idurl']: lg.warn('no idurl found in the input') return False if not misc.ValidKeyAlias(parts['key_alias']): lg.warn('invalid key alias in the input') return False return True
def send(customer_idurl, packet_id, format_type, key_id, remote_idurl): if not my_keys.is_key_registered(key_id): lg.warn( 'not able to return Files() for customer %s, key %s not registered' % ( customer_idurl, key_id, )) return p2p_service.SendFailNoRequest(customer_idurl, packet_id) if _Debug: lg.out( _DebugLevel, "list_files.send to %s, customer_idurl=%s, key_id=%s" % ( remote_idurl, customer_idurl, key_id, )) ownerdir = settings.getCustomerFilesDir(customer_idurl) plaintext = '' if os.path.isdir(ownerdir): for key_alias in os.listdir(ownerdir): if not misc.ValidKeyAlias(str(key_alias)): continue key_alias_dir = os.path.join(ownerdir, key_alias) plaintext += TreeSummary(key_alias_dir, key_alias) else: lg.warn('did not found customer dir: %s' % ownerdir) if _Debug: lg.out(_DebugLevel + 8, '\n%s' % plaintext) raw_list_files = PackListFiles(plaintext, format_type) block = encrypted.Block( CreatorID=my_id.getLocalID(), BackupID=key_id, Data=raw_list_files, SessionKey=key.NewSessionKey(), EncryptKey=key_id, ) encrypted_list_files = block.Serialize() newpacket = p2p_service.SendFiles( idurl=remote_idurl, raw_list_files_info=encrypted_list_files, packet_id=packet_id, callbacks={ commands.Ack(): on_acked, commands.Fail(): on_failed, None: on_timeout, }, ) return newpacket
def send(customer_idurl, packet_id, format_type): customer_name = nameurl.GetName(customer_idurl) if _Debug: lg.out( _DebugLevel, "list_files.send to %s, format is '%s'" % (customer_name, format_type)) ownerdir = settings.getCustomerFilesDir(customer_idurl) plaintext = '' if os.path.isdir(ownerdir): for key_alias in os.listdir(ownerdir): if not misc.ValidKeyAlias(str(key_alias)): continue key_alias_dir = os.path.join(ownerdir, key_alias) plaintext += TreeSummary(key_alias_dir, key_alias) else: lg.warn('did not found customer dir: %s' % ownerdir) if _Debug: lg.out(_DebugLevel + 8, '\n%s' % plaintext) return p2p_service.SendFiles( idurl=customer_idurl, raw_list_files_info=PackListFiles(plaintext, format_type), packet_id=packet_id, )
def SpaceTime(): """ Test all packets for each customer. Check if he use more space than we gave him and if packets is too old. """ printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000"))) space = accounting.read_customers_quotas() if space is None: printlog( 'SpaceTime ERROR customers quotas file can not be read or it is empty, skip' ) return customers_dir = settings.getCustomersFilesDir() if not os.path.exists(customers_dir): printlog('SpaceTime ERROR customers folder not exist') return remove_list = {} used_space = accounting.read_customers_usage() for customer_filename in os.listdir(customers_dir): onecustdir = os.path.join(customers_dir, customer_filename) if not os.path.isdir(onecustdir): remove_list[onecustdir] = 'is not a folder' continue # idurl = nameurl.FilenameUrl(customer_filename) idurl = global_id.GlobalUserToIDURL(customer_filename) if idurl is None: remove_list[onecustdir] = 'wrong folder name' continue curspace = space.get(idurl, None) if curspace is None: remove_list[onecustdir] = 'not found in space file' continue try: maxspaceV = int(curspace) except: remove_list[onecustdir] = 'wrong space value' continue timedict = {} sizedict = {} def cb(path, subpath, name): if not os.path.isfile(path): return True stats = os.stat(path) timedict[path] = stats.st_ctime sizedict[path] = stats.st_size for key_alias in os.listdir(onecustdir): if not misc.ValidKeyAlias(key_alias): remove_list[onecustdir] = 'invalid key alias' continue okekeydir = os.path.join(onecustdir, key_alias) bpio.traverse_dir_recursive(cb, okekeydir) currentV = 0 for path in sorted(list(timedict.keys()), key=lambda x: timedict[x], reverse=True): filesize = sizedict.get(path, 0) currentV += filesize if currentV < maxspaceV: continue try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (cur:%s, max: %s)' % (str(currentV), str(maxspaceV))) except: printlog('SpaceTime ERROR removing ' + path) # time.sleep(0.01) used_space[idurl] = str(currentV) timedict.clear() sizedict.clear() for path in remove_list.keys(): if not os.path.exists(path): continue if os.path.isdir(path): try: bpio._dir_remove(path) printlog('SpaceTime ' + path + ' dir removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) continue try: if not os.access(path, os.W_OK): os.chmod(path, 0o600) except: pass try: os.remove(path) printlog('SpaceTime ' + path + ' file removed (%s)' % (remove_list[path])) except: printlog('SpaceTime ERROR removing ' + path) del remove_list accounting.update_customers_usage(used_space)