def test_bad_ca_certificate(self): """ If no CA certificate is found in the service account directory, ``https_policy_from_config`` raises ``ValueError``. """ t = FilePath(self.useFixture(TempDir()).path) t = t.asBytesMode() serviceaccount = t.child(b"serviceaccount") serviceaccount.makedirs() serviceaccount.child(b"ca.crt").setContent( b"-----BEGIN CERTIFICATE-----\n" b"not a cert pem\n" b"-----END CERTIFICATE-----\n") serviceaccount.child(b"token").setContent(b"token") environ = encode_environ({ u"KUBERNETES_SERVICE_HOST": u"example.invalid.", u"KUBERNETES_SERVICE_PORT": u"443", }) self.patch(os, "environ", environ) config = KubeConfig.from_service_account( path=serviceaccount.asTextMode().path) self.assertThat( lambda: https_policy_from_config(config), raises( ValueError( "Invalid certificate authority certificate found.", "[('PEM routines', 'PEM_read_bio', 'bad base64 decode')]", )), )
def test_missing_ca_certificate(self): """ If no CA certificate is found in the service account directory, ``https_policy_from_config`` raises ``ValueError``. """ t = FilePath(self.useFixture(TempDir()).path) t = t.asBytesMode() serviceaccount = t.child(b"serviceaccount") serviceaccount.makedirs() serviceaccount.child(b"ca.crt").setContent(b"not a cert pem") serviceaccount.child(b"token").setContent(b"token") environ = encode_environ({ u"KUBERNETES_SERVICE_HOST": u"example.invalid.", u"KUBERNETES_SERVICE_PORT": u"443", }) self.patch(os, "environ", environ) config = KubeConfig.from_service_account( path=serviceaccount.asTextMode().path) self.assertThat( lambda: https_policy_from_config(config), raises(ValueError("No certificate authority certificate found.")), )
def test_create_local_snapshot(self, relative_target_path, content): """ ``MagicFolder.local_snapshot_service`` can be used to create a new local snapshot for a file in the folder. """ magic_path = FilePath(self.mktemp()) magic_path.asBytesMode("utf-8").makedirs() target_path = magic_path.preauthChild( relative_target_path).asBytesMode("utf-8") target_path.asBytesMode("utf-8").parent().makedirs( ignoreExistingDirectory=True) target_path.setContent(content) local_snapshot_creator = MemorySnapshotCreator() local_snapshot_service = LocalSnapshotService(magic_path, local_snapshot_creator) clock = object() tahoe_client = object() name = u"local-snapshot-service-test" config = object() participants = object() magic_folder = MagicFolder( client=tahoe_client, config=config, name=name, local_snapshot_service=local_snapshot_service, uploader_service=Service(), initial_participants=participants, clock=clock, ) magic_folder.startService() self.addCleanup(magic_folder.stopService) adding = magic_folder.local_snapshot_service.add_file(target_path, ) self.assertThat( adding, succeeded(Always()), ) self.assertThat( local_snapshot_creator.processed, Equals([target_path]), )
def add(fd: int, path: FilePath, mask: int) -> int: """ Add a watch for the given path to the inotify file descriptor, and return the watch descriptor. @param fd: The file descriptor returned by C{libc.inotify_init}. @param path: The path to watch via inotify. @param mask: Bitmask specifying the events that inotify should monitor. """ wd = cast(int, libc.inotify_add_watch(fd, path.asBytesMode().path, mask)) if wd < 0: raise INotifyError("Failed to add watch on '%r' - (%r)" % (path, wd)) return wd
def _authorized_request(self, token, headers, kubernetes_host=b"example.invalid."): """ Get an agent using ``authenticate_with_serviceaccount`` and issue a request with it. :return bytes: The bytes of the request the agent issues. """ server = AccumulatingProtocol() factory = Factory.forProtocol(lambda: server) factory.protocolConnectionMade = None reactor = create_reactor() reactor.listenTCP(80, factory) t = FilePath(self.useFixture(TempDir()).path) t = t.asBytesMode() serviceaccount = t.child(b"serviceaccount") serviceaccount.makedirs() serviceaccount.child(b"ca.crt").setContent(_CA_CERT_PEM) serviceaccount.child(b"token").setContent(token) environ = encode_environ({ u"KUBERNETES_SERVICE_HOST": kubernetes_host.decode("ascii"), u"KUBERNETES_SERVICE_PORT": u"443" }) self.patch(os, "environ", environ) agent = authenticate_with_serviceaccount( reactor, path=serviceaccount.asTextMode().path, ) d = agent.request(b"GET", b"http://" + kubernetes_host, headers) assertNoResult(self, d) [(host, port, factory, _, _)] = reactor.tcpClients addr = HOST_MAP.get(kubernetes_host.decode("ascii"), None) self.expectThat((host, port), Equals((addr, 80))) pump = ConnectionCompleter(reactor).succeedOnce() pump.pump() return server.data
def test_start_uploader_service(self): """ When the ``MagicFolder`` service is started the given uploader service is also started. """ magic_path = FilePath(self.mktemp()) magic_path.asBytesMode("utf-8").makedirs() local_snapshot_creator = MemorySnapshotCreator() local_snapshot_service = LocalSnapshotService(magic_path, local_snapshot_creator) clock = task.Clock() # create RemoteSnapshotCreator and UploaderService uploader_service = Service() tahoe_client = object() name = u"local-snapshot-service-test" config = object() participants = object() magic_folder = MagicFolder( client=tahoe_client, config=config, name=name, local_snapshot_service=local_snapshot_service, uploader_service=uploader_service, initial_participants=participants, clock=clock, ) magic_folder.startService() self.addCleanup(magic_folder.stopService) self.assertThat( uploader_service.running, Equals(True), )
class AnonymousStorageServer(Fixture): """ Supply an instance of allmydata.storage.server.StorageServer which implements anonymous access to Tahoe-LAFS storage server functionality. :ivar FilePath tempdir: The path to the server's storage on the filesystem. :ivar allmydata.storage.server.StorageServer storage_server: The storage server. """ def _setUp(self): self.tempdir = FilePath(self.useFixture(TempDir()).join(b"storage")) self.storage_server = StorageServer( self.tempdir.asBytesMode().path, b"x" * 20, )
def test_policy(self): """ ``https_policy_from_config`` returns a ``ClientCertificatePolicyForHTTPS`` with no credentials but with trust roots taken from the Kubernetes *serviceaccount* directory it is pointed at. It also respects *KUBERNETES_...* environment variables to identify the address of the server. """ t = FilePath(self.useFixture(TempDir()).path) t = t.asBytesMode() serviceaccount = t.child(b"serviceaccount") serviceaccount.makedirs() serviceaccount.child(b"ca.crt").setContent(_CA_CERT_PEM) serviceaccount.child(b"token").setContent(b"token") netloc = NetLocation(host=u"example.invalid", port=443) environ = encode_environ({ u"KUBERNETES_SERVICE_HOST": netloc.host, u"KUBERNETES_SERVICE_PORT": u"{}".format(netloc.port), }) self.patch(os, "environ", environ) config = KubeConfig.from_service_account( path=serviceaccount.asTextMode().path) policy = https_policy_from_config(config) self.expectThat( policy, Equals( ClientCertificatePolicyForHTTPS( credentials={}, trust_roots={ netloc: pem.parse(_CA_CERT_PEM)[0], }, ), ), )
class DirDBM: """ A directory with a DBM interface. This class presents a hash-like interface to a directory of small, flat files. It can only use strings as keys or values. """ def __init__(self, name): """ @type name: str @param name: Base path to use for the directory storage. """ self.dname = os.path.abspath(name) self._dnamePath = FilePath(name) if not self._dnamePath.isdir(): self._dnamePath.createDirectory() else: # Run recovery, in case we crashed. we delete all files ending # with ".new". Then we find all files who end with ".rpl". If a # corresponding file exists without ".rpl", we assume the write # failed and delete the ".rpl" file. If only a ".rpl" exist we # assume the program crashed right after deleting the old entry # but before renaming the replacement entry. # # NOTE: '.' is NOT in the base64 alphabet! for f in glob.glob(self._dnamePath.child("*.new").path): os.remove(f) replacements = glob.glob(self._dnamePath.child("*.rpl").path) for f in replacements: old = f[:-4] if os.path.exists(old): os.remove(f) else: os.rename(f, old) def _encode(self, k): """ Encode a key so it can be used as a filename. """ # NOTE: '_' is NOT in the base64 alphabet! return base64.encodestring(k).replace(b'\n', b'_').replace(b"/", b"-") def _decode(self, k): """ Decode a filename to get the key. """ return base64.decodestring(k.replace(b'_', b'\n').replace(b"-", b"/")) def _readFile(self, path): """ Read in the contents of a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "rb") as f: s = f.read() return s def _writeFile(self, path, data): """ Write data to a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "wb") as f: f.write(data) f.flush() def __len__(self): """ @return: The number of key/value pairs in this Shelf """ return len(self._dnamePath.listdir()) def __setitem__(self, k, v): """ C{dirdbm[k] = v} Create or modify a textfile in this directory @type k: bytes @param k: key to set @type v: bytes @param v: value to associate with C{k} """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") if not type(v) == bytes: raise TypeError("DirDBM value must be bytes") k = self._encode(k) # We create a new file with extension .new, write the data to it, and # if the write succeeds delete the old file and rename the new one. old = self._dnamePath.child(k) if old.exists(): new = old.siblingExtension(".rpl") # Replacement entry else: new = old.siblingExtension(".new") # New entry try: self._writeFile(new, v) except: new.remove() raise else: if (old.exists()): old.remove() new.moveTo(old) def __getitem__(self, k): """ C{dirdbm[k]} Get the contents of a file in this directory as a string. @type k: bytes @param k: key to lookup @return: The value associated with C{k} @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(k)) try: return self._readFile(path) except (EnvironmentError): raise KeyError(k) def __delitem__(self, k): """ C{del dirdbm[foo]} Delete a file in this directory. @type k: bytes @param k: key to delete @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") k = self._encode(k) try: self._dnamePath.child(k).remove() except (EnvironmentError): raise KeyError(self._decode(k)) def keys(self): """ @return: a L{list} of filenames (keys). """ return list(map(self._decode, self._dnamePath.asBytesMode().listdir())) def values(self): """ @return: a L{list} of file-contents (values). """ vals = [] keys = self.keys() for key in keys: vals.append(self[key]) return vals def items(self): """ @return: a L{list} of 2-tuples containing key/value pairs. """ items = [] keys = self.keys() for key in keys: items.append((key, self[key])) return items def has_key(self, key): """ @type key: bytes @param key: The key to test @return: A true value if this dirdbm has the specified key, a false value otherwise. """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") key = self._encode(key) return self._dnamePath.child(key).isfile() def setdefault(self, key, value): """ @type key: bytes @param key: The key to lookup @param value: The value to associate with key if key is not already associated with a value. """ if key not in self: self[key] = value return value return self[key] def get(self, key, default=None): """ @type key: bytes @param key: The key to lookup @param default: The value to return if the given key does not exist @return: The value associated with C{key} or C{default} if not L{DirDBM.has_key(key)} """ if key in self: return self[key] else: return default def __contains__(self, key): """ @see: L{DirDBM.has_key} """ return self.has_key(key) def update(self, dict): """ Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting keys will be overwritten with the values from L{dict}. @type dict: mapping @param dict: A mapping of key/value pairs to add to this dirdbm. """ for key, val in dict.items(): self[key] = val def copyTo(self, path): """ Copy the contents of this dirdbm to the dirdbm at C{path}. @type path: L{str} @param path: The path of the dirdbm to copy to. If a dirdbm exists at the destination path, it is cleared first. @rtype: C{DirDBM} @return: The dirdbm this dirdbm was copied to. """ path = FilePath(path) assert path != self._dnamePath d = self.__class__(path.path) d.clear() for k in self.keys(): d[k] = self[k] return d def clear(self): """ Delete all key/value pairs in this dirdbm. """ for k in self.keys(): del self[k] def close(self): """ Close this dbm: no-op, for dbm-style interface compliance. """ def getModificationTime(self, key): """ Returns modification time of an entry. @return: Last modification date (seconds since epoch) of entry C{key} @raise KeyError: Raised when there is no such key """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(key)) if path.isfile(): return path.getModificationTime() else: raise KeyError(key)
from twisted.web.static import Data from twisted.web.util import Redirect from twisted.internet import reactor, defer, interfaces from twisted.python.filepath import FilePath from twisted.python.log import msg from twisted.protocols.policies import WrappingFactory from twisted.test.proto_helpers import StringTransport try: from twisted.internet import ssl except: ssl = None from twisted import test serverPEM = FilePath(test.__file__).sibling('server.pem') serverPEMPath = serverPEM.asBytesMode().path class ExtendedRedirect(resource.Resource): """ Redirection resource. The HTTP status code is set according to the C{code} query parameter. @type lastMethod: C{bytes} @ivar lastMethod: Last handled HTTP request method """ isLeaf = True lastMethod = None
from twisted.web.static import Data from twisted.web.util import Redirect from twisted.internet import reactor, defer, interfaces from twisted.python.filepath import FilePath from twisted.python.log import msg from twisted.protocols.policies import WrappingFactory from twisted.test.proto_helpers import StringTransport try: from twisted.internet import ssl except: ssl = None from twisted import test serverPEM = FilePath(test.__file__).sibling('server.pem') serverPEMPath = serverPEM.asBytesMode().path class ExtendedRedirect(resource.Resource): """ Redirection resource. The HTTP status code is set according to the C{code} query parameter. @type lastMethod: C{bytes} @ivar lastMethod: Last handled HTTP request method """ isLeaf = True lastMethod = None
class LocalSnapshotServiceTests(SyncTestCase): """ Tests for ``LocalSnapshotService``. """ def setup_example(self): """ Hypothesis-invoked hook to create per-example state. Reset the database before running each test. """ self.magic_path = FilePath(self.mktemp()).asTextMode("utf-8") self.magic_path.asBytesMode("utf-8").makedirs() self.snapshot_creator = MemorySnapshotCreator() self.snapshot_service = LocalSnapshotService( magic_path=self.magic_path, snapshot_creator=self.snapshot_creator, status=WebSocketStatusService(), ) @given(relative_paths(), binary()) def test_add_single_file(self, relative_path, content): """ Start the service, add a file and check if the operation succeeded. """ to_add = self.magic_path.preauthChild(relative_path) to_add.asBytesMode("utf-8").parent().makedirs( ignoreExistingDirectory=True) to_add.asBytesMode("utf-8").setContent(content) self.snapshot_service.startService() self.assertThat( self.snapshot_service.add_file(to_add), succeeded(Always()), ) self.assertThat(self.snapshot_service.stopService(), succeeded(Always())) self.assertThat( self.snapshot_creator.processed, Equals([to_add]), ) @given(lists(path_segments(), unique=True), data()) def test_add_multiple_files(self, filenames, data): """ Add a bunch of files one by one and check whether the operation is successful. """ files = [] for filename in filenames: to_add = self.magic_path.child(filename) content = data.draw(binary()) to_add.asBytesMode("utf-8").setContent(content) files.append(to_add) self.snapshot_service.startService() list_d = [] for file in files: result_d = self.snapshot_service.add_file(file) list_d.append(result_d) d = defer.gatherResults(list_d) self.assertThat( d, succeeded(Always()), ) self.assertThat(self.snapshot_service.stopService(), succeeded(Always())) self.assertThat(sorted(self.snapshot_creator.processed), Equals(sorted(files))) @given(relative_paths()) def test_add_file_not_a_filepath(self, relative_path): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``TypeError`` if called with something other than a ``FilePath``. """ self.assertThat( self.snapshot_service.add_file(relative_path), failed( AfterPreprocessing( lambda f: (f.type, f.value.args), Equals((TypeError, ("argument must be a FilePath", ))), ), ), ) @given(relative_paths()) def test_add_file_directory(self, relative_path): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``ValueError`` if called with a path that refers to a directory. """ to_add = self.magic_path.preauthChild(relative_path) to_add.asBytesMode("utf-8").makedirs() self.assertThat( self.snapshot_service.add_file(to_add), failed( AfterPreprocessing( lambda f: (f.type, f.value.args), MatchesListwise([ Equals(ValueError), Equals(("expected a regular file, {!r} is a directory". format(to_add.path, ), )), ]), ), ), ) @given(absolute_paths()) def test_add_file_outside_magic_directory(self, to_add): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``ValueError`` if called with a path that is not contained by the Magic-Folder's magic directory. """ assume(not to_add.startswith(self.magic_path.path)) self.assertThat( self.snapshot_service.add_file(FilePath(to_add)), failed( AfterPreprocessing( lambda f: (f.type, f.value.args), MatchesListwise([ Equals(ValueError), MatchesPredicate( lambda args: args[0].startswith( "The path being added "), "%r does not start with 'The path being added '.", ), ]), ), ), )
class DirDBM: """ A directory with a DBM interface. This class presents a hash-like interface to a directory of small, flat files. It can only use strings as keys or values. """ def __init__(self, name): """ @type name: str @param name: Base path to use for the directory storage. """ self.dname = os.path.abspath(name) self._dnamePath = FilePath(name) if not self._dnamePath.isdir(): self._dnamePath.createDirectory() else: # Run recovery, in case we crashed. we delete all files ending # with ".new". Then we find all files who end with ".rpl". If a # corresponding file exists without ".rpl", we assume the write # failed and delete the ".rpl" file. If only a ".rpl" exist we # assume the program crashed right after deleting the old entry # but before renaming the replacement entry. # # NOTE: '.' is NOT in the base64 alphabet! for f in glob.glob(self._dnamePath.child("*.new").path): os.remove(f) replacements = glob.glob(self._dnamePath.child("*.rpl").path) for f in replacements: old = f[:-4] if os.path.exists(old): os.remove(f) else: os.rename(f, old) def _encode(self, k): """ Encode a key so it can be used as a filename. """ # NOTE: '_' is NOT in the base64 alphabet! return base64.encodestring(k).replace(b'\n', b'_').replace(b"/", b"-") def _decode(self, k): """ Decode a filename to get the key. """ return base64.decodestring(k.replace(b'_', b'\n').replace(b"-", b"/")) def _readFile(self, path): """ Read in the contents of a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "rb") as f: s = f.read() return s def _writeFile(self, path, data): """ Write data to a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "wb") as f: f.write(data) f.flush() def __len__(self): """ @return: The number of key/value pairs in this Shelf """ return len(self._dnamePath.listdir()) def __setitem__(self, k, v): """ C{dirdbm[k] = v} Create or modify a textfile in this directory @type k: bytes @param k: key to set @type v: bytes @param v: value to associate with C{k} """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") if not type(v) == bytes: raise TypeError("DirDBM value must be bytes") k = self._encode(k) # We create a new file with extension .new, write the data to it, and # if the write succeeds delete the old file and rename the new one. old = self._dnamePath.child(k) if old.exists(): new = old.siblingExtension(".rpl") # Replacement entry else: new = old.siblingExtension(".new") # New entry try: self._writeFile(new, v) except: new.remove() raise else: if (old.exists()): old.remove() new.moveTo(old) def __getitem__(self, k): """ C{dirdbm[k]} Get the contents of a file in this directory as a string. @type k: bytes @param k: key to lookup @return: The value associated with C{k} @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(k)) try: return self._readFile(path) except (EnvironmentError): raise KeyError(k) def __delitem__(self, k): """ C{del dirdbm[foo]} Delete a file in this directory. @type k: bytes @param k: key to delete @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") k = self._encode(k) try: self._dnamePath.child(k).remove() except (EnvironmentError): raise KeyError(self._decode(k)) def keys(self): """ @return: a L{list} of filenames (keys). """ return list(map(self._decode, self._dnamePath.asBytesMode().listdir())) def values(self): """ @return: a L{list} of file-contents (values). """ vals = [] keys = self.keys() for key in keys: vals.append(self[key]) return vals def items(self): """ @return: a L{list} of 2-tuples containing key/value pairs. """ items = [] keys = self.keys() for key in keys: items.append((key, self[key])) return items def has_key(self, key): """ @type key: bytes @param key: The key to test @return: A true value if this dirdbm has the specified key, a false value otherwise. """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") key = self._encode(key) return self._dnamePath.child(key).isfile() def setdefault(self, key, value): """ @type key: bytes @param key: The key to lookup @param value: The value to associate with key if key is not already associated with a value. """ if key not in self: self[key] = value return value return self[key] def get(self, key, default = None): """ @type key: bytes @param key: The key to lookup @param default: The value to return if the given key does not exist @return: The value associated with C{key} or C{default} if not L{DirDBM.has_key(key)} """ if key in self: return self[key] else: return default def __contains__(self, key): """ @see: L{DirDBM.has_key} """ return self.has_key(key) def update(self, dict): """ Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting keys will be overwritten with the values from L{dict}. @type dict: mapping @param dict: A mapping of key/value pairs to add to this dirdbm. """ for key, val in dict.items(): self[key]=val def copyTo(self, path): """ Copy the contents of this dirdbm to the dirdbm at C{path}. @type path: L{str} @param path: The path of the dirdbm to copy to. If a dirdbm exists at the destination path, it is cleared first. @rtype: C{DirDBM} @return: The dirdbm this dirdbm was copied to. """ path = FilePath(path) assert path != self._dnamePath d = self.__class__(path.path) d.clear() for k in self.keys(): d[k] = self[k] return d def clear(self): """ Delete all key/value pairs in this dirdbm. """ for k in self.keys(): del self[k] def close(self): """ Close this dbm: no-op, for dbm-style interface compliance. """ def getModificationTime(self, key): """ Returns modification time of an entry. @return: Last modification date (seconds since epoch) of entry C{key} @raise KeyError: Raised when there is no such key """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(key)) if path.isfile(): return path.getModificationTime() else: raise KeyError(key)