def exists(self, vault: Vault, user_id: str) -> bool: with store.session() as session: return ( session.query(sa_exists().where( and_(self.model.vault_id == vault.id, self.model.user_id == user_id) )).scalar() )
def get_bundle_for_relpath(self, relpath, vault): # check if path should be ignored for filepart in relpath.split( "/"): # TODO: use os.path.split to be windows compliant if any( fnmatch(filepart, ig) for ig in vault.config.ignore_patterns): return None if os.path.isdir(os.path.join(vault.folder, relpath)): return None with store.session() as session: try: bundle = session.query(Bundle).filter( Bundle.vault == vault, Bundle.relpath == relpath.encode()).one() # vault is loaded lazily. We already have the vault # object here, so just set it. bundle.vault = vault return bundle except NoResultFound: bundle = Bundle(relpath=relpath, vault=vault, vault_id=vault.id) bundle = Bundle(relpath=relpath, vault=vault, vault_id=vault.id) bundle.update_store_hash() return bundle
async def reset(self, vault: Vault): with store.session() as session: session.add(vault) vault.revision_count = 0 vault.file_count = 0 vault.user_count = 0 session.commit()
def list_for_vault(self, vault): with store.session() as session: return ( session.query(UserVaultKey) .filter(self.model.vault_id == vault.id) .all() )
async def dispatch_log_list(request): vault_id = request.match_info.get("vault_id", None) limit = int(request.query.get("limit", 100)) with store.session() as session: return JSONResponse( [logitem_to_json(logitem) for logitem in recent_log_items(app, vault_id, limit, session)] )
def find_key(self, vault: Vault, fingerprint) -> UserVaultKey: with store.session() as session: return ( session.query(UserVaultKey) .filter(self.model.vault_id == vault.id) .filter(self.model.fingerprint == fingerprint) .first() )
def remove(self, vault: Vault, user_id: str): with store.session() as session: return ( session .query(self.model) .filter(self.model.vault_id == vault.id, self.model.user_id == user_id) .delete() )
async def update(self): "Update list from remote" logger.info("Updating flying vaults...") try: backend = await self.app.open_backend() except InvalidAuthentification: logger.info("No login information found, skipping update.") return # Make a map from vault id -> vault info # Maybe we can use VaultManager in the future? v_infos = {v["id"].decode(): v for v in (await backend.list_vaults())} with store.session() as session: lst = await backend.list_vaults_for_identity(self.app.identity) for (vault, user_vault_key, encrypted_metadata) in lst: await trio.sleep(0.001) vault_id = vault["id"].decode("utf-8") logger.debug( "Received vault: %s (with%s metadata)", vault_id, "" if encrypted_metadata else "out", ) if encrypted_metadata: metadata = await self.app._decrypt_metadata( encrypted_metadata, user_vault_key) else: metadata = None v_info = v_infos.get(vault_id) if v_info is None: logger.warning("No information for vault: %s, ignoring.", vault_id) continue # list_vaults should not return a dict, but a parsed # instance of vault or similar modification_date_str = v_info.get("modification_date") if isinstance(modification_date_str, bytes): modification_date_str = modification_date_str.decode() modification_date = iso8601.parse_date(modification_date_str) fv = self.get_or_create_by_id(session, vault_id) fv.byte_size = v_info.get("byte_size", 0) fv.user_count = v_info.get("user_count", 0) fv.file_count = v_info.get("file_count", 0) fv.revision_count = v_info.get("revision_count", 0) fv.modification_date = modification_date fv.vault_metadata = metadata session.add(fv)
async def get_bundle_by_hash(self, vault: Vault, store_hash) -> Bundle: with store.session() as session: try: bundle = session.query(Bundle).filter(Bundle.vault==vault, Bundle.store_hash==store_hash).one() # vault is loaded lazily. We already have the vault # object here, so just set it. # bundle.vault = vault return bundle except NoResultFound: raise FileNotFoundError( 'No file with hash "{0}" exists in {1}'.format(store_hash, vault) )
async def update(self): "Update list from remote" backend = await self.app.open_backend() # Make a map from vault id -> vault info # Maybe we can use VaultManager in the future? v_infos = {v["id"].decode(): v for v in (await backend.list_vaults())} with store.session() as session: lst = await backend.list_vaults_for_identity(self.app.identity) for (vault, user_vault_key, encrypted_metadata) in lst: await asyncio.sleep(0.001) vault_id = vault["id"].decode("utf-8") logger.debug( "Received vault: %s (with%s metadata)", vault_id, "" if encrypted_metadata else "out", ) if encrypted_metadata: metadata = await self.app._decrypt_metadata( encrypted_metadata, user_vault_key ) else: metadata = None v_info = v_infos.get(vault_id) if v_info is None: logger.warning("No information for vault: %s, ignoring.", vault_id) continue modification_date = v_info.get("modification_date") if isinstance(modification_date, bytes): modification_date = modification_date.decode() fv = self.get_or_create_by_id(session, vault_id) fv.byte_size = v_info.get("byte_size", 0) fv.user_count = v_info.get("user_count", 0) fv.file_count = v_info.get("file_count", 0) fv.revision_count = v_info.get("revision_count", 0) fv.modification_date = modification_date fv.vault_metadata = metadata session.add(fv) await backend.close()
async def get_bundle(self, vault: Vault, path: str) -> Bundle: relpath = os.path.relpath(path, vault.folder) with store.session() as session: try: bundle = session.query(Bundle).filter(Bundle.vault==vault, Bundle.relpath==relpath.encode()).one() # vault is loaded lazily. We already have the vault # object here, so just set it. bundle.vault = vault return bundle except NoResultFound: raise FileNotFoundError( 'No file with path "{0}" exists in {1}'.format(relpath, vault) )
async def download_bundles_for_vault(self, vault): """ return an iterator of all bundles in the vault that possible require download """ with store.session() as session: lst = list(session.query(Bundle).filter(Bundle.vault==vault).all()) for bundle in lst: bundle.vault = vault await bundle.update() if bundle.remote_hash_differs: session.expunge(bundle) if inspect(vault).session: session.expunge(vault) yield bundle
async def upload_bundles_for_vault(self, vault): """ return an iterator of all bundles in the vault that require upload """ registered_paths = set() if inspect(vault).session: raise ValueError('Vault object is bound to a session') # First, try to find changes from database to disk with store.session() as session: lst = list( session.query(Bundle).filter(Bundle.vault == vault).all()) for bundle in lst: bundle.vault = vault registered_paths.add(bundle.relpath) await bundle.update() if bundle.remote_hash_differs: session.expunge(bundle) if inspect(vault).session: session.expunge(vault) yield bundle if inspect(vault).session: session.expunge(vault) # Next, we will walk the disk to find new bundles async def walk_disk(subfolder=None): folder = vault.folder if subfolder: folder = os.path.join(folder, subfolder) for file in os.listdir(folder): if any( fnmatch(file, ig) for ig in vault.config.ignore_patterns): continue abspath = os.path.join(folder, file) relpath = os.path.relpath(abspath, vault.folder) #logger.debug("%s, %s", abspath, registered_paths) if relpath in registered_paths: continue if os.path.isdir(abspath): async for bundle in walk_disk(subfolder=relpath): yield bundle else: yield self.get_bundle_for_relpath(relpath, vault) async for bundle in walk_disk(): await bundle.update() yield bundle
async def get_bundle_by_hash(self, vault: Vault, store_hash) -> Bundle: with store.session() as session: try: bundle = session.query(Bundle).filter( Bundle.vault == vault, Bundle.store_hash == store_hash).one() # vault is loaded lazily. We already have the vault # object here, so just set it. # bundle.vault = vault return bundle except NoResultFound: raise FileNotFoundError( 'No file with hash "{0}" exists in {1}'.format( store_hash, vault))
async def download_bundles_for_vault(self, vault): """ return an iterator of all bundles in the vault that possible require download """ with store.session() as session: lst = list( session.query(Bundle).filter(Bundle.vault == vault).all()) for bundle in lst: bundle.vault = vault await bundle.update() if bundle.remote_hash_differs: session.expunge(bundle) if inspect(vault).session: session.expunge(vault) yield bundle
async def get_bundle(self, vault: Vault, path: str) -> Bundle: relpath = os.path.relpath(path, vault.folder) with store.session() as session: try: bundle = session.query(Bundle).filter( Bundle.vault == vault, Bundle.relpath == relpath.encode()).one() # vault is loaded lazily. We already have the vault # object here, so just set it. bundle.vault = vault return bundle except NoResultFound: raise FileNotFoundError( 'No file with path "{0}" exists in {1}'.format( relpath, vault))
async def upload_bundles_for_vault(self, vault): """ return an iterator of all bundles in the vault that require upload """ registered_paths = set() if inspect(vault).session: raise ValueError('Vault object is bound to a session') # First, try to find changes from database to disk with store.session() as session: lst = list(session.query(Bundle).filter(Bundle.vault==vault).all()) for bundle in lst: bundle.vault = vault registered_paths.add(bundle.relpath) await bundle.update() if bundle.remote_hash_differs: session.expunge(bundle) if inspect(vault).session: session.expunge(vault) yield bundle if inspect(vault).session: session.expunge(vault) # Next, we will walk the disk to find new bundles async def walk_disk(subfolder=None): folder = vault.folder if subfolder: folder = os.path.join(folder, subfolder) for file in os.listdir(folder): if any(fnmatch(file, ig) for ig in vault.config.ignore_patterns): continue abspath = os.path.join(folder, file) relpath = os.path.relpath(abspath, vault.folder) if relpath in registered_paths: continue if os.path.isdir(abspath): async for bundle in walk_disk(subfolder=relpath): yield bundle else: yield self.get_bundle_for_relpath(relpath, vault) async for bundle in walk_disk(): await bundle.update() yield bundle
def add(self, vault: Vault, user_id: str, identity: Identity): with store.session() as session: fingerprint = identity.get_fingerprint() public_key = identity.public_key.export_key('DER') user_vault_key = session.query(UserVaultKey)\ .filter(self.model.vault_id == vault.id)\ .filter(self.model.fingerprint == fingerprint)\ .first() if not user_vault_key: session.add( self.model(vault_id=vault.id, fingerprint=fingerprint, user_id=user_id, public_key=public_key)) else: if user_vault_key.user_id != user_id or user_vault_key.public_key != public_key: raise ValueError( "Attempted to another UserVaultKey with existing fingerprint" )
async def refresh_vault_info(self): logger.debug('Refreshing vault information (byte_size) from server...') backend = await self.open_backend() with store.session() as session: for v_info in (await backend.list_vaults()): remote_id = v_info['id'].decode() for v in self.vaults: try: if v.config.id == remote_id: byte_size = int(v_info.get('byte_size', 0)) if byte_size != v.byte_size: logger.debug('Updating byte size for vault %s to %d', remote_id, byte_size) v.byte_size = byte_size session.add(v) except SyncryptBaseException: pass
async def initialize(self): await self.identity.init() if self.vault_dirs is None: self.vault_dirs = self.config.vault_dirs # Load cached vault information from config file config_vaults = [] with store.session() as session: for vault_dir in self.vault_dirs: abs_vault_dir = os.path.normpath(os.path.abspath(vault_dir)) try: vault = session.query(Vault).filter(Vault.folder==abs_vault_dir).one() except NoResultFound: vault = Vault(abs_vault_dir) session.add(vault) config_vaults.append(vault) for vault in config_vaults: await self.start_vault(vault)
async def refresh_vault_info(self): logger.debug('Refreshing vault information (byte_size) from server...') backend = await self.open_backend() with store.session() as session: for v_info in (await backend.list_vaults()): remote_id = v_info['id'].decode() for v in self.vaults: if v.config.id == remote_id: v.byte_size = int(v_info.get('byte_size', 0)) #v.file_count = int(v_info.get('file_count', 0)) #v.user_count = int(v_info.get('user_count', 0)) #v.revision_count = int(v_info.get('revision_count', 0)) #modification_date = v_info.get('modification_date') or b'' #v.modification_date = modification_date.decode() session.add(v) await backend.close()
async def ws_stream_log(request, ws, app, vault_id=None, limit=None, filters=None): "Stream Python logs via WebSockets" await ws.prepare(request) with store.session() as session: for logitem in recent_log_items(app, vault_id, 100, session): await ws.send_str(JSONResponse.encode_body(logitem_to_json(logitem)).decode('utf-8')) root_logger = logging.getLogger() queue = asyncio.Queue(maxsize=MAX_ITEMS_LOGGING_QUEUE) # type: asyncio.Queue handler = QueueHandler(queue, JSONFormatter(app)) if vault_id: handler.addFilter(VaultFilter(app.find_vault_by_id(vault_id))) if filters: for fltr in filters: handler.addFilter(fltr) async def writer(): while not ws.closed: item = await queue.get() try: # Send the item and also try to get up to MAX_ITEMS_BEFORE_DRAIN items from the # queue before draining the connection for _ in range(MAX_ITEMS_BEFORE_DRAIN): await ws.send_str(str(item)) item = queue.get_nowait() except asyncio.QueueEmpty: pass async def reader(): while not ws.closed: await ws.receive() root_logger.addHandler(handler) writer_future = asyncio.ensure_future(writer()) await reader() root_logger.removeHandler(handler) writer_future.cancel()
def add(self, vault: Vault, user_id: str, identity: Identity): with store.session() as session: session.add(self.model(vault_id=vault.id, fingerprint=identity.get_fingerprint(), user_id=user_id, public_key=identity.public_key.export_key('DER')))
def add(self, vault: Vault, user_id: str): with store.session() as session: session.add(VaultUser(vault_id=vault.id, user_id=user_id))
async def delete(self, id): with store.session() as session: return session.query(self.model).filter(self.model.id == id).delete()
async def delete_for_vault(self, vault: Vault) -> None: with store.session() as session: session.query(self.model).filter(self.model.vault_id == vault.id).delete()
def find_key(self, vault: Vault, fingerprint) -> VaultUser: with store.session() as session: return (session.query( self.model).filter(self.model.vault_id == vault.id).filter( self.model.fingerprint == fingerprint).first())
def list_for_vault(self, vault: Vault) -> Sequence[Revision]: with store.session() as session: return ( session.query(Revision).filter(Revision.local_vault_id == vault.id).all() )
def remove(self, vault: Vault, user_id: str, identity: Identity) -> None: with store.session() as session: session.query(self.model).filter( self.model.vault_id == vault.id, self.model.fingerprint == identity.get_fingerprint(), self.model.user_id == user_id).delete()
def exists(self, vault: Vault, user_id: str) -> bool: with store.session() as session: return (session.query(sa_exists().where( and_(self.model.vault_id == vault.id, self.model.user_id == user_id))).scalar())
async def delete_for_vault(self, vault: Vault) -> None: with store.session() as session: session.query(Revision).filter( Revision.local_vault_id == vault.id).delete()
async def apply(self, revision: Revision, vault: Vault): if inspect(vault).session: raise ValueError('Vault object is bound to a session') revision.assert_valid() # 1. Check preconditions for this to be a valid revision (current revision must be parent) if vault.revision != revision.parent_id: raise UnexpectedParentInRevision("Expected parent to be {0}, but is {1}"\ .format(revision.parent_id, vault.revision)) smokesignal.emit('pre_apply_revision', vault=vault, revision=revision) with store.session() as session: # 2. Check if signing user's key is in the user vault key list if revision.operation != RevisionOp.CreateVault: signer_key = self.app.user_vault_keys.find_key( vault, revision.user_fingerprint) if not signer_key: raise InvalidRevision( "Key {0} is not allowed to generate revisions for vault {1}" .format(revision.user_fingerprint, vault)) else: # CreateVault is the only operation that is allowed to provide its own key signer_key = UserVaultKey( vault_id=vault.id, user_id=revision.user_id, fingerprint=revision.user_fingerprint, public_key=revision.user_public_key, ) # 3. Verify revision signature revision.verify(signer_key.get_identity(self.app.config)) # 4. Based on the revision type, perform an action to our state of the vault logger.debug( "Applying %s (%s) to %s", revision.operation, revision.revision_id, vault.id, ) if revision.operation == RevisionOp.CreateVault: session.add(vault) session.add(signer_key) session.add( VaultUser(vault_id=vault.id, user_id=revision.user_id)) session.commit() elif revision.operation == RevisionOp.Upload: try: bundle = await self.app.bundles.get_bundle_by_hash( vault, revision.file_hash) session.delete(bundle) except FileNotFoundError: pass bundle = await self.create_bundle_from_revision( revision, vault) session.add(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.SetMetadata: await vault.write_encrypted_metadata( Once(revision.revision_metadata)) elif revision.operation == RevisionOp.RemoveFile: bundle = await self.app.bundles.get_bundle_by_hash( vault, revision.file_hash) session.delete(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.AddUser: self.app.vault_users.add(vault, revision.user_id) elif revision.operation == RevisionOp.RemoveUser: self.app.vault_users.remove(vault, revision.user_id) elif revision.operation == RevisionOp.AddUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.add(vault, revision.user_id, new_identity) elif revision.operation == RevisionOp.RemoveUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.remove(vault, revision.user_id, new_identity) else: raise NotImplementedError(revision.operation) # 5. Store the revision in config and db revision.local_vault_id = vault.id revision.creator_id = signer_key.user_id session.add(revision) session.commit() vault.revision_count = (session.query(Revision).filter( Revision.local_vault_id == vault.id).count()) if revision.operation in (RevisionOp.Upload, RevisionOp.RemoveFile): vault.file_count = (session.query(Bundle).filter( Bundle.vault_id == vault.id).count()) if revision.operation in (RevisionOp.CreateVault, RevisionOp.AddUser, RevisionOp.RemoveUser): vault.user_count = (session.query(VaultUser).filter( VaultUser.vault_id == vault.id).count()) vault.modification_date = revision.created_at logger.debug( "Vault state revision_count=%s file_count=%s user_count=%s", vault.revision_count, vault.file_count, vault.user_count) # vault.revision = revision.id session.add(vault) vault.update_revision(revision) session.commit() smokesignal.emit('post_apply_revision', vault=vault, revision=revision)
def list_for_vault(self, vault: Vault) -> Sequence[Revision]: with store.session() as session: return (session.query(Revision).filter( Revision.local_vault_id == vault.id).all())
async def apply(self, revision: Revision, vault: Vault): if inspect(vault).session: raise ValueError('Vault object is bound to a session') revision.assert_valid() # 1. Check preconditions for this to be a valid revision (current revision must be parent) if vault.revision != revision.parent_id: raise UnexpectedParentInRevision("Expected parent to be {0}, but is {1}"\ .format(revision.parent_id, vault.revision)) smokesignal.emit('pre_apply_revision', vault=vault, revision=revision) with store.session() as session: # 2. Check if signing user's key is in the user vault key list if revision.operation != RevisionOp.CreateVault: signer_key = self.app.user_vault_keys.find_key( vault, revision.user_fingerprint ) if not signer_key: raise InvalidRevision( "Key {0} is not allowed to generate revisions for vault {1}" .format(revision.user_fingerprint, vault) ) else: # CreateVault is the only operation that is allowed to provide its own key signer_key = UserVaultKey( vault_id=vault.id, user_id=revision.user_id, fingerprint=revision.user_fingerprint, public_key=revision.user_public_key, ) # 3. Verify revision signature revision.verify(signer_key.get_identity(self.app.config)) # 4. Based on the revision type, perform an action to our state of the vault logger.debug( "Applying %s (%s) to %s", revision.operation, revision.revision_id, vault.id, ) if revision.operation == RevisionOp.CreateVault: session.add(vault) session.add(signer_key) session.add(VaultUser(vault_id=vault.id, user_id=revision.user_id)) session.commit() elif revision.operation == RevisionOp.Upload: try: bundle = await self.app.bundles.get_bundle_by_hash(vault, revision.file_hash) session.delete(bundle) except FileNotFoundError: pass bundle = await self.create_bundle_from_revision(revision, vault) session.add(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.SetMetadata: await vault.write_encrypted_metadata(Once(revision.revision_metadata)) elif revision.operation == RevisionOp.RemoveFile: bundle = await self.app.bundles.get_bundle_by_hash(vault, revision.file_hash) session.delete(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.AddUser: self.app.vault_users.add(vault, revision.user_id) elif revision.operation == RevisionOp.RemoveUser: self.app.vault_users.remove(vault, revision.user_id) elif revision.operation == RevisionOp.AddUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.add(vault, revision.user_id, new_identity) elif revision.operation == RevisionOp.RemoveUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.remove(vault, revision.user_id, new_identity) else: raise NotImplementedError(revision.operation) # 5. Store the revision in config and db revision.local_vault_id = vault.id revision.creator_id = signer_key.user_id session.add(revision) session.commit() vault.revision_count = ( session.query(Revision) .filter(Revision.local_vault_id == vault.id) .count() ) if revision.operation in (RevisionOp.Upload, RevisionOp.RemoveFile): vault.file_count = ( session.query(Bundle) .filter(Bundle.vault_id == vault.id) .count() ) if revision.operation in (RevisionOp.CreateVault, RevisionOp.AddUser, RevisionOp.RemoveUser): vault.user_count = ( session.query(VaultUser) .filter(VaultUser.vault_id == vault.id) .count() ) vault.modification_date = revision.created_at logger.debug("Vault state revision_count=%s file_count=%s user_count=%s", vault.revision_count, vault.file_count, vault.user_count) # vault.revision = revision.id session.add(vault) vault.update_revision(revision) session.commit() smokesignal.emit('post_apply_revision', vault=vault, revision=revision)
async def delete_for_vault(self, vault: Vault) -> None: with store.session() as session: session.query(Revision).filter(Revision.local_vault_id == vault.id).delete()
def remove(self, vault: Vault, user_id: str): with store.session() as session: return (session.query(self.model).filter( self.model.vault_id == vault.id, self.model.user_id == user_id).delete())
def all(self): with store.session() as session: return session.query(FlyingVault).all()
def list_for_vault(self, vault: Vault): with store.session() as session: return (session.query( self.model).filter(self.model.vault_id == vault.id).all())
def remove(self, vault: Vault, user_id: str, identity: Identity) -> None: with store.session() as session: session.query(self.model).filter( self.model.vault_id == vault.id, self.model.fingerprint==identity.get_fingerprint(), self.model.user_id==user_id).delete()
async def delete_for_vault(self, vault: Vault) -> None: with store.session() as session: session.query( self.model).filter(self.model.vault_id == vault.id).delete()
def get(self, id): with store.session() as session: return session.query(FlyingVault).filter( FlyingVault.id == id).one()
def get(self, id): with store.session() as session: return session.query(FlyingVault).filter(FlyingVault.id == id).one()
async def delete(self, id): with store.session() as session: return session.query( self.model).filter(self.model.id == id).delete()