def add_from_engine(cls, engine, **kwargs): app = kwargs.get('app') session = kwargs.get('session', db.session) bind_name = kwargs.get('bind_name') backend_version = kwargs.get('backend_version') use_cache = kwargs.get('use_cache', False) and app if use_cache: flush_cache = kwargs.get('flush_cache', False) cache_path = kwargs.get('cache_path', app.config.get('DISKCACHE_PATH')) cache_timeout = kwargs.get('DISKCACHE_DBEX_DATABASE_TIMEOUT') cache = Cache(cache_path) if flush_cache: cache.pop(engine) try: if use_cache: inspector = cache.get(engine) else: inspector = inspect(engine) if use_cache: cache.set(engine, inspector, cache_timeout) except OperationalError as e: logger.error("cant inspect engine %s" % engine) raise e except Exception as e: raise e db_name = db_engine_to_name(engine) try: info = get_discovered_db_engine_info(bind_name) except NoBindNameFoundError: logger.warning("No info found to engine bind name '%s'" % bind_name) info = '' if not backend_version and info: try: backend_version = info['backend_version'] except TypeError as e: pass except KeyError as e: pass except Exception as e: raise e d = Database(name=db_name, bind_name=bind_name, engine=engine.name, driver=engine.driver, backend_version=backend_version) table_names = inspector.get_table_names() for table_name in table_names: logger.debug("table_name: %s" % (table_name,)) t = Table(name=table_name) column_names = inspector.get_columns(table_name) for column in column_names: c = Column(**column) t.columns.append(c) d.tables.append(t) session.add(d) if kwargs.get('db_session_commit_enabled', True): session.commit()
class Cache(object): def __init__(self): try: self.cache = DC('./tmp') except Exception as ex: print('Get an exception with diskcache open: {}'.format(ex)) self.cache = None def __del__(self): try: self.cache.close() except Exception as ex: print('Get an exception with diskcache close: {}'.format(ex)) def set(self, key, value): if self.cache is not None: self.cache.set(key, BytesIO(value), read=True, tag=u'data') def get(self, key): if self.cache is not None: value = self.cache.get(key, default=b'', read=True, tag=True) if value is not None and value != b'': return value return None def pop(self, key): if self.cache is not None: value = self.cache.pop(key, default=b'', read=True, tag=True) if value is not None and value != b'': return value return None def delete(self, key): if self.cache is not None: self.cache.delete(key) def create_index(self): if self.cache is not None: self.cache.create_tag_index() return self.cache.tag_index return None def clear_all(self): if self.cache is not None: self.cache.clear()
class DiskCachePersistence: def __init__(self, dbname, dbpassphrase): self.dbname = dbname self.dbpassphrase = dbpassphrase self.db = Cache(dbname) # TODO: create encrypted Cache with kdf dbpassphrase # TODO: purge expired skippedMessageKey based on store_time def save_conversation(self, conversation): return self.db.set(b'conv:' + conversation.ks['CONVid'], prefix='conv', tag='conv', retry=True) def load_conversation(self, conv_id): return self.db.get(b'conv:' + conv_id, None, retry=True) def delete_conversation(self, conversation): return self.db.pop(b'conv:' + conversation.ks['CONVid'], None, retry=True) def get_other_names(self, name): names = [] for k in self.db: if k.startswith('conv:'): names.append(self.db[k].other_name) return names
class CloudFS(Operations): '''Baidu netdisk filesystem''' def __init__(self, *args, **kw): self.buffer = Cache('./cache/buffer') self.dir_buffer = Cache('./cache/dir_buffer') self.traversed_folder = {} self.disk = PCS() self.createLock = Lock() self.writing_files = {} self.downloading_files = {} # update all folder in other thread dirReaderDaemon.submit(self.readdirAsync, "/", 100, dirReaderDaemon) def _add_file_to_buffer(self, path, file_info): foo = File() foo['st_ctime'] = file_info['local_ctime'] foo['st_mtime'] = file_info['local_mtime'] foo['st_mode'] = ( stat.S_IFDIR | 0x777) if file_info['isdir'] \ else (stat.S_IFREG | 0x777) foo['st_nlink'] = 2 if file_info['isdir'] else 1 foo['st_size'] = file_info['size'] self.buffer[path] = foo def _del_file_from_buffer(self, path): self.buffer.pop(path) def getattr(self, path, fh=None): if path in self.writing_files: return self.writing_files[path] if path.split("/")[-1].startswith("."): raise FuseOSError(errno.ENOENT) st = None if path not in self.buffer or self.buffer[path] is None: jdata = json.loads(self.disk.meta([path])) if 'info' not in jdata: raise FuseOSError(errno.ENOENT) if jdata['errno'] != 0: raise FuseOSError(errno.ENOENT) file_info = jdata['info'][0] self._add_file_to_buffer(path, file_info) st = self.buffer[path].getDict() else: st = self.buffer[path].getDict() # logger.info(f'st: {st}') return st def readdirAsync(self, path, depth=2, threadPool=pool): try: foo = json.loads(self.disk.list_files(path)) except Exception as s: logger.exception(s) files = ['.', '..'] abs_files = [] if 'errno' in foo: logger.error( "maybe token is not right, try re login http://pan.baidu.com in Chrome" ) if "list" not in foo: # logger.info("no list") return for file in foo['list']: if file['server_filename'].startswith("."): continue files.append(file['server_filename']) abs_files.append(file['path']) # logger.debug(file['path']) file_num = len(abs_files) group = int(math.ceil(file_num / 100.0)) # logger.debug(f"group: {group}") # logger.debug(f"abs_files: {abs_files}") for i in range(group): obj = [f for n, f in enumerate(abs_files) if n % group == i] #一组数据 while 1: try: ret = json.loads(self.disk.meta(obj)) # logger.debug(f'{ret}') break except Exception as e: logger.info(ret) logger.exception(e) for file_info in ret['info']: # logger.debug(file_info) self._add_file_to_buffer(file_info['path'], file_info) if depth > 0: depth -= 1 if file_info['isdir']: if file_info['path'] not in self.traversed_folder: self.traversed_folder[path] = True threadPool.submit(self.readdirAsync, file_info['path'], depth, threadPool) self.dir_buffer[path] = files # @funcLog def readdir(self, path, offset): # if path not in self.traversed_folder: self.traversed_folder[path] = True pool.submit(self.readdirAsync, path, 2, pool) if path in self.dir_buffer: # logger.info(f'{path},{self.dir_buffer[path]}') for r in self.dir_buffer[path]: yield r else: files = ['.', '..'] for r in files: yield r # @funcLog def open(self, path, flags): if path in self.writing_files: return 0 # method does not have thread race problem, open by one thread only try: if path not in self.downloading_files: url = self.disk.getRestUrl(path) x = Task(url, path, self.disk) x.start() self.downloading_files[path] = x except Baidu8Secs as e: logger.exception(e) except Exception as e: logger.exception(e) return 0 def read(self, path, size, offset, fh): x = self.downloading_files[path] if x: data = x.get_cache(offset, size) filename = path[path.rfind("/") + 1:] if filename.startswith("enc."): if offset == 0: if data and len(data) > encrpted_length: data = bytes(cipher(data, 0, encrpted_length, 123)) else: print("decrpt failed!") return data raise FuseOSError(errno.EIO) def updateCahe(self, old, new): directory = old[:old.rfind("/")] filename = old[old.rfind("/") + 1:] if len(directory) == 0: directory = "/" if not new: oldCache = self.dir_buffer[directory] if filename in oldCache: oldCache.remove(filename) self.dir_buffer[directory] = oldCache if old in self.buffer: self.buffer.pop(old) else: oldCache = self.dir_buffer[directory] if filename in oldCache: oldCache.remove(filename) newfilename = new[new.rfind("/") + 1:] oldCache.append(newfilename) self.dir_buffer[directory] = oldCache if old in self.buffer: old_info = self.buffer.pop(old) self.buffer[new] = old_info def unlink(self, path): self.disk.delete([path]) self.updateCahe(path, None) def access(self, path, amode): return 0 def rmdir(self, path): self.disk.delete([path]) self.updateCahe(path, None) def rename(self, old, new): self.disk.rename(old, new) self.updateCahe(old, new) # @funcLog def mkdir(self, path, mode): directory = path[:path.rfind("/")] filename = path[path.rfind("/") + 1:] cache = self.dir_buffer[directory] cache.append(filename) self.dir_buffer[directory] = cache self.disk.mkdir(path) def create(self, path, mode, fh=None): with self.createLock: if path not in self.writing_files: t = time.time() self.writing_files[path] = { 'st_atime': t, 'st_ctime': t, 'st_gid': 20, 'st_mode': stat.S_IFREG | 0x777, 'st_mtime': t, 'st_nlink': 1, 'st_size': 0, 'st_uid': 502, 'uploading_tmp': tempfile.NamedTemporaryFile('wb') } return 0 def flush(self, path, fh): with self.createLock: if path in self.writing_files: self.writing_files[path]["uploading_tmp"].flush() return 0 def release(self, path, fh): with self.createLock: if path in self.writing_files: uploading_tmp = self.writing_files[path]['uploading_tmp'] self.disk.upload(uploading_tmp.name, path) self.writing_files[path]['uploading_tmp'].close() if path in self.writing_files: del self.writing_files[path] # why ? prevent accidently read file when uploading still in progress if path in self.downloading_files: del self.downloading_files[path] print("released", path) return # method does not have thread race problem, release by one thread only if path in self.downloading_files: # self.downloading_files[path].terminate() # del self.downloading_files[path] # uploading_tmp = "./uploading_tmp"+path # logger.info("delete uploading_tmp:", uploading_tmp) # os.remove(uploading_tmp) pass def write(self, path, data, offset, fp): filename = path[path.rfind("/") + 1:] if filename.startswith("enc."): if offset == 0 and data and len(data) > encrpted_length: data = bytes(cipher(data, 0, encrpted_length, 123)) length = len(data) self.writing_files[path]["st_size"] += length self.writing_files[path]["uploading_tmp"].write(data) return length def chmod(self, path, mode): pass def statfs(self, path): return { 'f_bavail': int(85533433401 / 4096), 'f_bfree': int(85533433401 / 4096), # 相同的值 block 'f_favail': 4290675908, 'f_ffree': 4290675908, # 相同的值 node 'f_bsize': 104857, # perferd value 'f_blocks': int(5611374772224 / 8), 'f_files': 4294967279, 'f_flag': 0, 'f_frsize': 4096, 'f_namemax': 255 }
class CloudFS(Operations): '''Baidu netdisk filesystem''' def __init__(self, mainArgs, *args, **kw): logger.info(colored("- fuse 4 cloud driver -", 'red')) self.buffer = Cache('./cache/buffer-batchmeta') self.dir_buffer = Cache('./cache/dir_buffer-buffer-batchmeta') self.attr_requesting = Cache('./cache/attr-requesting') self.mainArgs = mainArgs self.traversed_folder = Cache('./cache/traversed-folder') self.disk = PCS(self.mainArgs) self.createLock = Lock() self.attrLock = Lock() self.writing_files = {} self.downloading_files = {} logger.info(f'mainArgs:{mainArgs}') q = json.loads(self.disk.quota()) # only request once try: self.total_size = q['quota'] self.used = q['used'] except Exception as e: self.total_size = 100000000000 self.used = 0 logger.exception(e) logger.debug(f'con`t load quota api, fall back to default') self.avail = self.total_size - self.used if mainArgs.debug: logger.setLevel(logging.DEBUG) logger.debug(colored("- debug mode -", 'red')) logger.debug( colored("- cach would not be the same after restart -", 'red')) self.buffer = Cache('./cache/buffer-batchmeta' + str(time.time())) self.dir_buffer = Cache('./cache/dir_buffer-buffer-batchmeta' + str(time.time())) self.traversed_folder = Cache('./cache/traversed-folder' + str(time.time())) self._readDirAsync("/", 2, dirReaderDaemon) else: logger.setLevel(logging.INFO) # update all folder in other thread self._readDirAsync("/", mainArgs.preload_level, dirReaderDaemon) @staticmethod def add_write_permission(st, permission='u'): """Add `w` (`write`) permission to specified targets.""" mode_map = { 'u': stat.S_IWUSR, 'g': stat.S_IWGRP, 'o': stat.S_IWOTH, 'a': stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, } logger.info( f'-------------------{type(stat.S_IWUSR)} ,{type(st["st_mode"])}') for t in permission: st['st_mode'] |= mode_map[t] return st def _baidu_file_attr_convert(self, path, file_info): foo = fileAttr.copy() try: foo['st_ctime'] = file_info[ 'local_ctime'] if 'local_ctime' in file_info else file_info[ 'ctime'] foo['st_mtime'] = file_info[ 'local_mtime'] if 'local_mtime' in file_info else file_info[ 'mtime'] foo['st_mode'] = 16877 if file_info['isdir'] else 36279 foo['st_nlink'] = 2 if file_info['isdir'] else 1 foo['st_size'] = int( file_info['size']) if 'size' in file_info else 0 self.buffer[path] = foo except Exception as e: logger.debug(f'======================') logger.debug(f'add buffer error {e},{path}:{file_info}') def _del_file_from_buffer(self, path): self.buffer.pop(path) def _getRootAttr(self): path = "/" if path in self.buffer: return self.buffer[path] logger.debug(f'net root: {path}') jdata = json.loads(self.disk.meta([path])) f = fileAttr.copy() f["st_mode"] = 16877 f["st_nlink"] = 2 if 'error_code' in jdata and jdata["error_code"] != 0: logger.debug(f"error_code:{jdata}") # logger.info(f'{error_map[str(jdata["error_code"])]} args: {path}') self.buffer.set(path, f, expire=60) return f if "list" not in jdata or len(jdata["list"]) == 0: logger.debug(f"{path} not list :{jdata}") self.buffer.set(path, f, expire=60) return f file_info = jdata["list"][0] self._baidu_file_attr_convert(path, file_info) return file_info @funcLog def getattr(self, path, fh=None): ''' Returns a dictionary with keys identical to the stat C structure of stat(2). st_atime, st_mtime and st_ctime should be floats. NOTE: There is an incompatibility between Linux and Mac OS X concerning st_nlink of directories. Mac OS X counts all files inside the directory, while Linux counts only the subdirectories. ''' if path in self.writing_files: return self.writing_files[path] if path.split("/")[-1].startswith("."): raise FuseOSError(errno.ENOENT) # special handle root Attr if path == "/": return self._getRootAttr() parentDir = os.path.dirname(path) if parentDir not in self.dir_buffer: self._readDir(parentDir, 1) if path in self.buffer: return self.buffer[path] raise FuseOSError(errno.ENOENT) @funcLog def truncate(self, path, length, fh=None): self.unlink(path) self.create(path, None) self.writing_files[path]["uploading_tmp"].truncate(length) def _readDirAsync(self, path, depth, p): p.submit(self._readDir, path, depth, p) def _readDir(self, path, depth=2, threadPool=pool): if path not in self.traversed_folder: self.traversed_folder.set(path, b'1', expire=self.mainArgs.cache_timeout) logger.debug(f'net dir {depth} - {path} ') try: foo = json.loads(self.disk.list_files(path)) files = ['.', '..'] if 'error_code' in foo and foo["error_code"] != 0: logger.info( f'{error_map[str(foo["error_code"])]} args: {path}') if "list" not in foo: return depth -= 1 for file in foo['list']: if file['server_filename'].startswith("."): continue files.append(file['server_filename']) # logger.debug(f'{file}') self._baidu_file_attr_convert(file['path'], file) if depth > 0: if file['isdir']: self._readDirAsync(file['path'], depth, threadPool) # self._readDir(file['path'],depth,threadPool) self.dir_buffer[path] = files except Exception as s: logger.exception(s) @funcLog def readdir(self, path, offset): self._readDirAsync(path, 1, pool) if path in self.dir_buffer: for r in self.dir_buffer[path]: yield r else: files = ['.', '..'] for r in files: yield r # @funcLog def open(self, path, flags): if path in self.writing_files: return 0 # method does not have thread race problem, open by one thread only try: if path not in self.downloading_files: url = self.disk.getRestUrl(path) x = Task(url, mainArgs, path, self.disk) x.start() self.downloading_files[path] = x except Baidu8Secs as e: logger.exception(e) except Exception as e: logger.exception(e) return 0 def read(self, path, size, offset, fh): x = self.downloading_files[path] if x: data = x.get_cache(offset, size) filename = path[path.rfind("/") + 1:] if filename.startswith("enc."): if offset == 0: if data and len(data) > encrpted_length: data = bytes( cipher(data, 0, encrpted_length, self.mainArgs.key)) else: print("decrpt failed!") return data raise FuseOSError(errno.EIO) def updateCache(self, path, newValue): ''' add updateCache(path,value) delete updateCache(path,None) udpate updateCache(path,value) ''' pass def updateCacheKeyOnly(self, old, new): ''' delete updateCacheKeyOnly(old,None) add/update updateCacheKeyOnly(old,new) ''' try: old_parent_dir = os.path.dirname(old) old_name = os.path.basename(old) if not new: oldCache = self.dir_buffer.get(old_parent_dir) # remove if oldCache: if old_name in oldCache: oldCache.remove(old_name) self.dir_buffer[old_parent_dir] = oldCache if old in self.buffer: self.buffer.pop(old) else: pass else: print("updateCache", old, new) oldCache = self.dir_buffer[old_parent_dir] new_parent_dir = os.path.dirname(new) if old_name in oldCache: # dir old remove oldCache.remove(old_name) self.dir_buffer[old_parent_dir] = oldCache # dir new add newfilename = new[new.rfind("/") + 1:] newCache = self.dir_buffer.get(new_parent_dir, []) newCache.append(newfilename) self.dir_buffer[new_parent_dir] = newCache if old in self.buffer: old_info = self.buffer.pop(old) self.buffer[new] = old_info except Exception as e: logger.info(e) def updateDir(self, old, new): pass def unlink(self, path): ''' will only delete file ''' print("unlink .....................") self.disk.delete([path]) self.updateCacheKeyOnly(path, None) def rmdir(self, path): ''' will only delete directory ''' self.disk.delete([path]) self.updateCacheKeyOnly(path, None) def access(self, path, amode): return 0 def rename(self, old, new): ''' will effect dir and file ''' logger.info(f'rename {old}, {new}') self.disk.rename(old, new) self.updateCacheKeyOnly(old, new) @funcLog def mkdir(self, path, mode): logger.info(f'making dir {path}') r = json.loads(self.disk.mkdir(path)) if 'error_code' in r: logger.info(f'{r}') logger.info( f'{error_map[str(r["error_code"])]} args: {path}, response:{r}' ) return directory = path[:path.rfind("/")] filename = path[path.rfind("/") + 1:] cache = None if directory in self.dir_buffer: cache = self.dir_buffer[directory] cache.append(filename) self.dir_buffer[directory] = cache self._baidu_file_attr_convert(path, r) @funcLog def create(self, path, mode, fh=None): logger.debug(f'create {path}') with self.createLock: if path not in self.writing_files: attr = fileAttr.copy() t = time.time() attr['uploading_tmp'] = tempfile.NamedTemporaryFile('wb') attr['st_mode'] = attr[ 'st_mode'] | stat.S_IFREG | stat.S_ISUID | stat.S_ISGID self.writing_files[path] = attr else: logger.debug(f'{path} is writing on, wait another turn..') return 0 def flush(self, path, fh): with self.createLock: if path in self.writing_files: self.writing_files[path]["uploading_tmp"].flush() return 0 def release(self, path, fh): with self.createLock: if path in self.writing_files: uploading_tmp = self.writing_files[path]['uploading_tmp'] r = json.loads(self.disk.upload(uploading_tmp.name, path)) logger.info(f'================================={r}') self.writing_files[path]['uploading_tmp'].close() # if path in self.buffer: # del self.buffer[path] if path in self.writing_files: del self.writing_files[path] # why ? prevent accidently read file when uploading still in progress if path in self.downloading_files: del self.downloading_files[path] # update file self._baidu_file_attr_convert(path, r) # update parent dir parentDir = os.path.dirname(path) filename = path[path.rfind("/") + 1:] if parentDir in self.dir_buffer: parentDirCache = self.dir_buffer[parentDir] parentDirCache.append(filename) self.dir_buffer[parentDir] = parentDirCache logger.info(f'{self.dir_buffer[parentDir]}') print("released", path) return # method does not have thread race problem, release by one thread only if path in self.downloading_files: # self.downloading_files[path].terminate() # del self.downloading_files[path] # uploading_tmp = "./uploading_tmp"+path # logger.info("delete uploading_tmp:", uploading_tmp) # os.remove(uploading_tmp) pass def write(self, path, data, offset, fp): filename = path[path.rfind("/") + 1:] if filename.startswith("enc."): if offset == 0 and data and len(data) > encrpted_length: data = bytes( cipher(data, 0, encrpted_length, self.mainArgs.key)) length = len(data) self.writing_files[path]["st_size"] += length self.writing_files[path]["uploading_tmp"].write(data) return length def chmod(self, path, mode): pass def statfs(self, path): # TODO read from cloud disk return { 'f_bavail': int((self.avail) / 4096), 'f_bfree': int((self.avail) / 4096), # 相同的值 block 'f_favail': 4290675908, 'f_ffree': 4290675908, # 相同的值 node 'f_bsize': 104857, # perferd value 'f_blocks': int(self.total_size / 8), 'f_files': 4294967279, 'f_flag': 0, 'f_frsize': 4096, 'f_namemax': 255 }