def test_add_participant(self, folder_name, personal_dmd): """ Adding a new participant works. """ local_path = FilePath(self.mktemp()) local_path.makedirs() folder_config = magic_folder_config( create_local_author("iris"), FilePath(self.mktemp()), local_path, ) # we can't add a new participant if their DMD is the same as # one we already have .. and because Hypothesis is 'sneaky' we # have to make sure it's not our collective, either assume(personal_dmd != folder_config["upload-dircap"]) assume(personal_dmd != to_readonly_capability( folder_config["upload-dircap"])) assume(personal_dmd != folder_config["collective-dircap"]) assume(personal_dmd != to_readonly_capability( folder_config["collective-dircap"])) root = create_fake_tahoe_root() # put our Collective DMD into the fake root root._uri.data[folder_config["collective-dircap"]] = dumps([ u"dirnode", { u"children": { "iris": format_filenode(folder_config["upload-dircap"]), }, }, ]) tahoe_client = create_tahoe_client( DecodedURL.from_text(u"http://invalid./"), create_tahoe_treq_client(root), ) treq = treq_for_folders( Clock(), FilePath(self.mktemp()), AUTH_TOKEN, { folder_name: folder_config, }, start_folder_services=False, tahoe_client=tahoe_client, ) # add a participant using the API self.assertThat( authorized_request( treq, AUTH_TOKEN, b"POST", self.url.child(folder_name, "participants"), dumps({ "author": { "name": "kelly" }, "personal_dmd": personal_dmd, }).encode("utf8")), succeeded( matches_response(code_matcher=Equals(CREATED), body_matcher=AfterPreprocessing( loads, Equals({}))))) # confirm that the "list participants" API includes the added # participant self.assertThat( authorized_request( treq, AUTH_TOKEN, b"GET", self.url.child(folder_name, "participants"), ), succeeded( matches_response(code_matcher=Equals(OK), body_matcher=AfterPreprocessing( loads, Equals({ u"iris": { u"personal_dmd": folder_config["upload-dircap"], }, u'kelly': { u'personal_dmd': personal_dmd, } })))))
class StorageStatusElement(Element): """Class to render a storage status page.""" loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml")) def __init__(self, storage, nickname=""): """ :param _StorageServer storage: data about storage. :param string nickname: friendly name for storage. """ super(StorageStatusElement, self).__init__() self._storage = storage self._nickname = nickname @renderer def nickname(self, req, tag): return tag(self._nickname) @renderer def nodeid(self, req, tag): return tag(idlib.nodeid_b2a(self._storage.my_nodeid)) def _get_storage_stat(self, key): """Get storage server statistics. Storage Server keeps a dict that contains various usage and latency statistics. The dict looks like this: { 'storage_server.accepting_immutable_shares': 1, 'storage_server.allocated': 0, 'storage_server.disk_avail': 106539192320, 'storage_server.disk_free_for_nonroot': 106539192320, 'storage_server.disk_free_for_root': 154415284224, 'storage_server.disk_total': 941088460800, 'storage_server.disk_used': 786673176576, 'storage_server.latencies.add-lease.01_0_percentile': None, 'storage_server.latencies.add-lease.10_0_percentile': None, ... } ``StorageServer.get_stats()`` returns the above dict. Storage status page uses a subset of the items in the dict, concerning disk usage. :param str key: storage server statistic we want to know. """ return self._storage.get_stats().get(key) def render_abbrev_space(self, size): if size is None: return u"?" return abbreviate_space(size) def render_space(self, size): if size is None: return u"?" return u"%d" % size @renderer def storage_stats(self, req, tag): # Render storage status table that appears near the top of the page. total = self._get_storage_stat("storage_server.disk_total") used = self._get_storage_stat("storage_server.disk_used") free_root = self._get_storage_stat("storage_server.disk_free_for_root") free_nonroot = self._get_storage_stat( "storage_server.disk_free_for_nonroot") reserved = self._get_storage_stat("storage_server.reserved_space") available = self._get_storage_stat("storage_server.disk_avail") tag.fillSlots( disk_total=self.render_space(total), disk_total_abbrev=self.render_abbrev_space(total), disk_used=self.render_space(used), disk_used_abbrev=self.render_abbrev_space(used), disk_free_for_root=self.render_space(free_root), disk_free_for_root_abbrev=self.render_abbrev_space(free_root), disk_free_for_nonroot=self.render_space(free_nonroot), disk_free_for_nonroot_abbrev=self.render_abbrev_space( free_nonroot), reserved_space=self.render_space(reserved), reserved_space_abbrev=self.render_abbrev_space(reserved), disk_avail=self.render_space(available), disk_avail_abbrev=self.render_abbrev_space(available)) return tag @renderer def accepting_immutable_shares(self, req, tag): accepting = self._get_storage_stat( "storage_server.accepting_immutable_shares") return tag({True: "Yes", False: "No"}[bool(accepting)]) @renderer def last_complete_bucket_count(self, req, tag): s = self._storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: return tag("Not computed yet") return tag(str(count)) @renderer def count_crawler_status(self, req, tag): p = self._storage.bucket_counter.get_progress() return tag(self.format_crawler_progress(p)) def format_crawler_progress(self, p): cycletime = p["estimated-time-per-cycle"] cycletime_s = "" if cycletime is not None: cycletime_s = " (estimated cycle time %s)" % abbreviate_time( cycletime) if p["cycle-in-progress"]: pct = p["cycle-complete-percentage"] soon = p["remaining-sleep-time"] eta = p["estimated-cycle-complete-time-left"] eta_s = "" if eta is not None: eta_s = " (ETA %ds)" % eta return [ "Current crawl %.1f%% complete" % pct, eta_s, " (next work in %s)" % abbreviate_time(soon), cycletime_s, ] else: soon = p["remaining-wait-time"] return ["Next crawl in %s" % abbreviate_time(soon), cycletime_s] @renderer def storage_running(self, req, tag): if self._storage: return tag return T.h1("No Storage Server Running") @renderer def lease_expiration_enabled(self, req, tag): lc = self._storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") else: return tag("Disabled: scan-only mode, no leases will be removed") @renderer def lease_expiration_mode(self, req, tag): lc = self._storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: tag("Leases will expire naturally, probably 31 days after " "creation or renewal.") else: tag("Leases created or last renewed more than %s ago " "will be considered expired." % abbreviate_time(lc.override_lease_duration)) else: assert lc.mode == "cutoff-date" localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date)) isoutcdate = time_format.iso_utc_date(lc.cutoff_date) tag("Leases created or last renewed before %s (%s) UTC " "will be considered expired." % ( isoutcdate, localizedutcdate, )) if len(lc.mode) > 2: tag(" The following sharetypes will be expired: ", " ".join(sorted(lc.sharetypes_to_expire)), ".") return tag @renderer def lease_current_cycle_progress(self, req, tag): lc = self._storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): lc = self._storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: return "" s = lc.get_state() so_far = s["cycle-to-date"] sr = so_far["space-recovered"] er = s["estimated-remaining-cycle"] esr = er["space-recovered"] ec = s["estimated-current-cycle"] ecr = ec["space-recovered"] p = T.ul() def add(*pieces): p(T.li(pieces)) def maybe(d): if d is None: return "?" return "%d" % d add( "So far, this cycle has examined %d shares in %d buckets" % (sr["examined-shares"], sr["examined-buckets"]), " (%d mutable / %d immutable)" % (sr["examined-buckets-mutable"], sr["examined-buckets-immutable"]), " (%s / %s)" % (abbreviate_space(sr["examined-diskbytes-mutable"]), abbreviate_space(sr["examined-diskbytes-immutable"])), ) add("and has recovered: ", self.format_recovered(sr, "actual")) if so_far["expiration-enabled"]: add("The remainder of this cycle is expected to recover: ", self.format_recovered(esr, "actual")) add("The whole cycle is expected to examine %s shares in %s buckets" % (maybe(ecr["examined-shares"]), maybe( ecr["examined-buckets"]))) add("and to recover: ", self.format_recovered(ecr, "actual")) else: add("If expiration were enabled, we would have recovered: ", self.format_recovered(sr, "configured"), " by now") add("and the remainder of this cycle would probably recover: ", self.format_recovered(esr, "configured")) add("and the whole cycle would probably recover: ", self.format_recovered(ecr, "configured")) add( "if we were strictly using each lease's default 31-day lease lifetime " "(instead of our configured behavior), " "this cycle would be expected to recover: ", self.format_recovered(ecr, "original")) if so_far["corrupt-shares"]: add( "Corrupt shares:", T.ul((T.li([ "SI %s shnum %d" % corrupt_share for corrupt_share in so_far["corrupt-shares"] ])))) return tag("Current cycle:", p) @renderer def lease_last_cycle_results(self, req, tag): lc = self._storage.lease_checker h = lc.get_state()["history"] if not h: return "" last = h[max(h.keys())] start, end = last["cycle-start-finish-times"] tag( "Last complete cycle (which took %s and finished %s ago)" " recovered: " % (abbreviate_time(end - start), abbreviate_time(time.time() - end)), self.format_recovered(last["space-recovered"], "actual")) p = T.ul() def add(*pieces): p(T.li(pieces)) saw = self.format_recovered(last["space-recovered"], "examined") add("and saw a total of ", saw) if not last["expiration-enabled"]: rec = self.format_recovered(last["space-recovered"], "configured") add( "but expiration was not enabled. If it had been, " "it would have recovered: ", rec) if last["corrupt-shares"]: add( "Corrupt shares:", T.ul((T.li([ "SI %s shnum %d" % corrupt_share for corrupt_share in last["corrupt-shares"] ])))) return tag(p) @staticmethod def format_recovered(sr, a): def maybe(d): if d is None: return "?" return "%d" % d return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ (maybe(sr["%s-shares" % a]), maybe(sr["%s-buckets" % a]), maybe(sr["%s-buckets-mutable" % a]), maybe(sr["%s-buckets-immutable" % a]), abbreviate_space(sr["%s-diskbytes" % a]), abbreviate_space(sr["%s-diskbytes-mutable" % a]), abbreviate_space(sr["%s-diskbytes-immutable" % a]), )
def getAccessTime(self): """ Return the archive file's last access time. """ return FilePath(self.zipfile.filename).getAccessTime()
def filenameGenerator(originalFileName, outputExtension): name = os.path.splitext(FilePath(originalFileName).basename())[0] return base.child(name + outputExtension).path
def buildDirectory(store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo, serversDB=None, cachingSeconds=0): """ Return a directory without using a config object; suitable for tests which need to have mulitple directory instances. @param store: The store. @param dataRoot: The path to the directory containing xml files for any xml based services. @param servicesInfo: An interable of ConfigDicts mirroring the DirectoryService and ResourceService sections of stdconfig @param augmentServiceInfo: A ConfigDict mirroring the AugmentService section of stdconfig @param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig @param serversDB: A ServersDB object to assign to the directory """ aggregatedServices = [] cachingServices = [] ldapService = None # LDAP DS has extra stats (see augment.py) for serviceValue in servicesInfo: if not serviceValue.Enabled: continue directoryType = serviceValue.type.lower() params = serviceValue.params if "xml" in directoryType: xmlFile = params.xmlFile xmlFile = fullServerPath(dataRoot, xmlFile) fp = FilePath(xmlFile) if not fp.exists(): fp.setContent(DEFAULT_XML_CONTENT) directory = XMLDirectoryService(fp) elif "opendirectory" in directoryType: from txdav.who.opendirectory import (DirectoryService as ODDirectoryService) # We don't want system accounts returned in lookups, so tell # the service to suppress them. node = params.node directory = ODDirectoryService(nodeName=node, suppressSystemRecords=True) elif "ldap" in directoryType: from twext.who.ldap import (DirectoryService as LDAPDirectoryService, FieldName as LDAPFieldName, RecordTypeSchema) if params.credentials.dn and params.credentials.password: creds = UsernamePassword(params.credentials.dn, params.credentials.password) else: creds = None mapping = params.mapping extraFilters = params.extraFilters directory = LDAPDirectoryService( params.uri, params.rdnSchema.base, credentials=creds, fieldNameToAttributesMap=MappingProxyType({ BaseFieldName.uid: mapping.uid, BaseFieldName.guid: mapping.guid, BaseFieldName.shortNames: mapping.shortNames, BaseFieldName.fullNames: mapping.fullNames, BaseFieldName.emailAddresses: mapping.emailAddresses, LDAPFieldName.memberDNs: mapping.memberDNs, CalFieldName.readOnlyProxy: mapping.readOnlyProxy, CalFieldName.readWriteProxy: mapping.readWriteProxy, CalFieldName.hasCalendars: mapping.hasCalendars, CalFieldName.autoScheduleMode: mapping.autoScheduleMode, }), recordTypeSchemas=MappingProxyType({ RecordType.user: RecordTypeSchema( relativeDN=params.rdnSchema.users, attributes=(), ), RecordType.group: RecordTypeSchema( relativeDN=params.rdnSchema.groups, attributes=(), ), CalRecordType.location: RecordTypeSchema( relativeDN=params.rdnSchema.locations, attributes=(), ), CalRecordType.resource: RecordTypeSchema( relativeDN=params.rdnSchema.resources, attributes=(), ), CalRecordType.address: RecordTypeSchema( relativeDN=params.rdnSchema.addresses, attributes=(), ), }), extraFilters={ RecordType.user: extraFilters.get("users", ""), RecordType.group: extraFilters.get("groups", ""), CalRecordType.location: extraFilters.get("locations", ""), CalRecordType.resource: extraFilters.get("resources", ""), CalRecordType.address: extraFilters.get("addresses", ""), }) ldapService = directory elif "inmemory" in directoryType: from txdav.who.test.support import CalendarInMemoryDirectoryService directory = CalendarInMemoryDirectoryService() else: log.error("Invalid DirectoryType: {dt}", dt=directoryType) raise DirectoryConfigurationError # Set the appropriate record types on each service types = [] fieldNames = [] for recordTypeName in params.recordTypes: recordType = { "users": RecordType.user, "groups": RecordType.group, "locations": CalRecordType.location, "resources": CalRecordType.resource, "addresses": CalRecordType.address, }.get(recordTypeName, None) if recordType is None: log.error("Invalid Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError if recordType in types: log.error("Duplicate Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError types.append(recordType) directory.recordType = ConstantsContainer(types) directory.fieldName = ConstantsContainer( (directory.fieldName, CalFieldName)) fieldNames.append(directory.fieldName) if cachingSeconds: directory = CachingDirectoryService(directory, expireSeconds=cachingSeconds) cachingServices.append(directory) aggregatedServices.append(directory) # # Setup the Augment Service # if augmentServiceInfo.type: for augmentFile in augmentServiceInfo.params.xmlFiles: augmentFile = fullServerPath(dataRoot, augmentFile) augmentFilePath = FilePath(augmentFile) if not augmentFilePath.exists(): augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT) augmentClass = namedClass(augmentServiceInfo.type) log.info("Configuring augment service of type: {augmentClass}", augmentClass=augmentClass) try: augmentService = augmentClass(**augmentServiceInfo.params) except IOError: log.error("Could not start augment service") raise else: augmentService = None userDirectory = None for directory in aggregatedServices: if RecordType.user in directory.recordTypes(): userDirectory = directory break else: log.error("No directory service set up for users") raise DirectoryConfigurationError # Delegate service delegateDirectory = DelegateDirectoryService(userDirectory.realmName, store) # (put at front of list so we don't try to ask the actual DS services # about the delegate-related principals, for performance) aggregatedServices.insert(0, delegateDirectory) # Wiki service if wikiServiceInfo.Enabled: aggregatedServices.append( WikiDirectoryService(userDirectory.realmName, wikiServiceInfo.CollabHost, wikiServiceInfo.CollabPort)) # Aggregate service aggregateDirectory = AggregateDirectoryService(userDirectory.realmName, aggregatedServices) # Augment service try: fieldNames.append(CalFieldName) augmented = AugmentedDirectoryService(aggregateDirectory, store, augmentService) augmented.fieldName = ConstantsContainer(fieldNames) # The delegate directory needs a way to look up user/group records # so hand it a reference to the augmented directory. # FIXME: is there a better pattern to use here? delegateDirectory.setMasterDirectory(augmented) # Tell each caching service what method to use when reporting # times and cache stats for cachingService in cachingServices: cachingService.setTimingMethod(augmented._addTiming) # LDAP has additional stats to report augmented._ldapDS = ldapService except Exception as e: log.error("Could not create directory service", error=e) raise if serversDB is not None: augmented.setServersDB(serversDB) return augmented
def templatefilepath(filename): return FilePath(templatefile(filename))
def setUp(self): self.path = FilePath(__file__)
def test_tls_auth_denied(self): """ A MQTT client offering the wrong certificate won't be authenticated. """ reactor, router, server_factory, session_factory = build_mqtt_server() real_reactor = selectreactor.SelectReactor() logger = make_logger() session, pump = connect_application_session( server_factory, ObservingSession, component_config=ComponentConfig(realm=u"mqtt")) endpoint = create_listening_endpoint_from_config( { "type": "tcp", "port": 1099, "interface": "0.0.0.0", "tls": { "certificate": "server.crt", "key": "server.key", "dhparam": "dhparam", "ca_certificates": ["ca.cert.pem", "intermediate.cert.pem"] }, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) client_endpoint = create_connecting_endpoint_from_config( { "type": "tcp", "host": "127.0.0.1", "port": 1099, "tls": { # BAD key: trusted by the CA, but wrong ID "certificate": "client_1.crt", "hostname": u"localhost", "key": "client_1.key", "ca_certificates": ["ca.cert.pem", "intermediate.cert.pem"] }, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) p = [] l = endpoint.listen(server_factory) class TestProtocol(Protocol): data = b"" expected = (ConnACK(session_present=False, return_code=1).serialise()) def dataReceived(self_, data): self_.data = self_.data + data if len(self_.data) == len(self_.expected): self.assertEqual(self_.data, self_.expected) real_reactor.stop() @l.addCallback def _listening(factory): d = client_endpoint.connect(Factory.forProtocol(TestProtocol)) @d.addCallback def _(proto): p.append(proto) proto.transport.write( Connect( client_id=u"test123", flags=ConnectFlags(clean_session=False)).serialise()) proto.transport.write( Publish(duplicate=False, qos_level=1, retain=False, topic_name=u"test", payload=b"{}", packet_identifier=1).serialise()) lc = LoopingCall(pump.flush) lc.clock = real_reactor lc.start(0.01) def timeout(): print("Timing out :(") real_reactor.stop() print(self.logs.log_text.getvalue()) # Timeout, just in case real_reactor.callLater(10, timeout) real_reactor.run() client_protocol = p[0] # We get a CONNECT self.assertEqual( client_protocol.data, ConnACK(session_present=False, return_code=1).serialise()) client_protocol.data = b"" pump.flush() # No events! self.assertEqual(len(session.events), 0)
Tests for C{setup.py}, Twisted's distutils integration file. """ from __future__ import division, absolute_import import os, sys import twisted from twisted.trial.unittest import SynchronousTestCase from twisted.python.filepath import FilePath from twisted.python.dist import getExtensions # Get rid of the UTF-8 encoding and bytes topfiles segment when FilePath # supports unicode. #2366, #4736, #5203. Also #4743, which requires checking # setup.py, not just the topfiles directory. if not FilePath(twisted.__file__.encode('utf-8')).sibling(b'topfiles').child( b'setup.py').exists(): sourceSkip = "Only applies to source checkout of Twisted" else: sourceSkip = None class TwistedExtensionsTests(SynchronousTestCase): if sourceSkip is not None: skip = sourceSkip def setUp(self): """ Change the working directory to the parent of the C{twisted} package so that L{twisted.python.dist.getExtensions} finds Twisted's own extension definitions. """
class MoreInfoElement(Element): """ An ``Element`` HTML template which can be flattened to describe this node. :param Node node: The node to describe. """ loader = XMLFile(FilePath(__file__).sibling("info.xhtml")) def __init__(self, node): super(MoreInfoElement, self).__init__() self.original = node def abbrev(self, storage_index_or_none): if storage_index_or_none: return base32.b2a(storage_index_or_none)[:6] return "LIT file" def get_type(self): node = self.original if IDirectoryNode.providedBy(node): if not node.is_mutable(): return "immutable directory" return "directory" if IFileNode.providedBy(node): si = node.get_storage_index() if si: if node.is_mutable(): ret = "mutable file" if node.get_version() == MDMF_VERSION: ret += " (mdmf)" else: ret += " (sdmf)" return ret return "immutable file" return "immutable LIT file" return "unknown" @renderer def title(self, req, tag): node = self.original si = node.get_storage_index() t = "More Info for %s" % self.get_type() if si: t += " (SI=%s)" % self.abbrev(si) return tag(t) @renderer def header(self, req, tag): return self.title(req, tag) @renderer def type(self, req, tag): return tag(self.get_type()) @renderer def si(self, req, tag): si = self.original.get_storage_index() if not si: return "None" return tag(base32.b2a(si)) @renderer def size(self, req, tag): node = self.original d = node.get_current_size() def _no_size(size): if size is None: return "?" return size d.addCallback(_no_size) def _handle_unrecoverable(f): f.trap(UnrecoverableFileError) return "?" d.addErrback(_handle_unrecoverable) d.addCallback(lambda size: tag(str(size))) return d @renderer def directory_writecap(self, req, tag): node = self.original if not IDirectoryNode.providedBy(node): return "" if node.is_readonly(): return "" return tag(node.get_uri()) @renderer def directory_readcap(self, req, tag): node = self.original if not IDirectoryNode.providedBy(node): return "" return tag(node.get_readonly_uri()) @renderer def directory_verifycap(self, req, tag): node = self.original if not IDirectoryNode.providedBy(node): return "" verifier = node.get_verify_cap() if verifier: return tag(node.get_verify_cap().to_string()) return "" @renderer def file_writecap(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node write_uri = node.get_write_uri() if not write_uri: return "" return tag(write_uri) @renderer def file_readcap(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node read_uri = node.get_readonly_uri() if not read_uri: return "" return tag(read_uri) @renderer def file_verifycap(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node verifier = node.get_verify_cap() if verifier: return tag(node.get_verify_cap().to_string()) return "" def get_root(self, req): # the addSlash=True gives us one extra (empty) segment depth = len(req.prepath) + len(req.postpath) - 1 link = "/".join([".."] * depth) return link @renderer def raw_link(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node elif IFileNode.providedBy(node): pass else: return "" root = self.get_root(req) quoted_uri = urllib.quote(node.get_uri()) text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri) return T.li("Raw data as ", T.a("text/plain", href=text_plain_url)) @renderer def is_checkable(self, req, tag): node = self.original si = node.get_storage_index() if si: return tag # don't show checker button for LIT files return "" @renderer def check_form(self, req, tag): node = self.original quoted_uri = urllib.quote(node.get_uri()) target = self.get_root(req) + "/uri/" + quoted_uri if IDirectoryNode.providedBy(node): target += "/" check = T.form(action=target, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="check"), T.input(type="hidden", name="return_to", value="."), T.legend("Check on this object", class_="freeform-form-label"), T.div( "Verify every bit? (EXPENSIVE):", T.input(type="checkbox", name="verify"), ), T.div("Repair any problems?: ", T.input(type="checkbox", name="repair")), T.div("Add/renew lease on all shares?: ", T.input(type="checkbox", name="add-lease")), T.div("Emit results in JSON format?: ", T.input(type="checkbox", name="output", value="JSON")), T.input(type="submit", value="Check"), )) return tag(check) @renderer def is_mutable_file(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): return "" if (IFileNode.providedBy(node) and node.is_mutable() and not node.is_readonly()): return tag return "" @renderer def overwrite_form(self, req, tag): node = self.original root = self.get_root(req) action = "%s/uri/%s" % (root, urllib.quote(node.get_uri())) done_url = "%s/uri/%s?t=info" % (root, urllib.quote(node.get_uri())) overwrite = T.form(action=action, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="upload"), T.input(type='hidden', name='when_done', value=done_url), T.legend("Overwrite", class_="freeform-form-label"), "Upload new contents: ", T.input(type="file", name="file"), " ", T.input(type="submit", value="Replace Contents") )) return tag(overwrite) @renderer def is_directory(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): return tag return "" @renderer def deep_check_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) deep_check = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-check"), T.input(type="hidden", name="return_to", value="."), T.legend("Run a deep-check operation (EXPENSIVE)", class_="freeform-form-label"), T.div( "Verify every bit? (EVEN MORE EXPENSIVE):", T.input(type="checkbox", name="verify"), ), T.div("Repair any problems?: ", T.input(type="checkbox", name="repair")), T.div("Add/renew lease on all shares?: ", T.input(type="checkbox", name="add-lease")), T.div("Emit results in JSON format?: ", T.input(type="checkbox", name="output", value="JSON")), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Deep-Check"), )) return tag(deep_check) @renderer def deep_size_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) deep_size = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-size"), T.legend("Run a deep-size operation (EXPENSIVE)", class_="freeform-form-label"), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Deep-Size"), )) return tag(deep_size) @renderer def deep_stats_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) deep_stats = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-stats"), T.legend("Run a deep-stats operation (EXPENSIVE)", class_="freeform-form-label"), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Deep-Stats"), )) return tag(deep_stats) @renderer def manifest_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) manifest = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-manifest"), T.legend("Run a manifest operation (EXPENSIVE)", class_="freeform-form-label"), T.div("Output Format: ", T.select(name="output") ( T.option("HTML", value="html", selected="true"), T.option("text", value="text"), T.option("JSON", value="json"), ), ), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Manifest"), )) return tag(manifest)
def get_filename(self): return FilePath(VERSION_PATH)
def cbFinished(ignored): self.assertEqual(FilePath(output).getContent(), "[('foo', 'bar')]")
from twisted.python.compat import _PY3, networkString, nativeString, intToBytes from twisted.trial import unittest from twisted.web import server, client, error, resource from twisted.internet import reactor, defer, interfaces from twisted.python.filepath import FilePath from twisted.python.log import msg from twisted.protocols.policies import WrappingFactory from twisted.test.proto_helpers import StringTransport try: from twisted.internet import ssl except: ssl = None from twisted import test serverPEM = FilePath(test.__file__.encode("utf-8")).sibling(b'server.pem') serverPEMPath = nativeString(serverPEM.path) # Remove this in #6177, when static is ported to Python 3: if _PY3: from twisted.web.test.test_web import Data else: from twisted.web.static import Data # Remove this in #6178, when util is ported to Python 3: if _PY3: class Redirect(resource.Resource): isLeaf = 1 def __init__(self, url):
def run(self, suite, reactor=None, cooperate=cooperate, untilFailure=False): """ Spawn local worker processes and load tests. After that, run them. @param suite: A tests suite to be run. @param reactor: The reactor to use, to be customized in tests. @type reactor: A provider of L{twisted.internet.interfaces.IReactorProcess} @param cooperate: The cooperate function to use, to be customized in tests. @type cooperate: C{function} @param untilFailure: If C{True}, continue to run the tests until they fail. @type untilFailure: C{bool}. @return: The test result. @rtype: L{DistReporter} """ if reactor is None: from twisted.internet import reactor result = self._makeResult() count = suite.countTestCases() self._stream.write("Running %d tests.\n" % (count, )) if not count: # Take a shortcut if there is no test suite.run(result.original) self.writeResults(result) return result testDir, testDirLock = _unusedTestDirectory( FilePath(self._workingDirectory)) workerNumber = min(count, self._workerNumber) ampWorkers = [LocalWorkerAMP() for x in xrange(workerNumber)] workers = self.createLocalWorkers(ampWorkers, testDir.path) processEndDeferreds = [worker.endDeferred for worker in workers] self.launchWorkerProcesses(reactor.spawnProcess, workers, self._workerArguments) def runTests(): testCases = iter(list(_iterateTests(suite))) workerDeferreds = [] for worker in ampWorkers: workerDeferreds.append( self._driveWorker(worker, result, testCases, cooperate=cooperate)) return DeferredList(workerDeferreds, consumeErrors=True, fireOnOneErrback=True) stopping = [] def nextRun(ign): self.writeResults(result) if not untilFailure: return if not result.wasSuccessful(): return d = runTests() return d.addCallback(nextRun) def stop(ign): testDirLock.unlock() if not stopping: stopping.append(None) reactor.stop() def beforeShutDown(): if not stopping: stopping.append(None) d = DeferredList(processEndDeferreds, consumeErrors=True) return d.addCallback(continueShutdown) def continueShutdown(ign): self.writeResults(result) return ign d = runTests() d.addCallback(nextRun) d.addBoth(stop) reactor.addSystemEventTrigger('before', 'shutdown', beforeShutDown) reactor.run() return result
Dataset, Manifestation, Application, DockerImage, Port, AttachedVolume, Link ) from ._config import ( ApplicationMarshaller, FLOCKER_RESTART_POLICY_NAME_TO_POLICY, model_from_configuration, FigConfiguration, FlockerConfiguration, ConfigurationError ) from .. import __version__ # Default port for REST API: REST_API_PORT = 4523 SCHEMA_BASE = FilePath(__file__).parent().child(b'schema') SCHEMAS = { b'/v1/types.json': yaml.safe_load( SCHEMA_BASE.child(b'types.yml').getContent()), b'/v1/endpoints.json': yaml.safe_load( SCHEMA_BASE.child(b'endpoints.yml').getContent()), } CONTAINER_NAME_COLLISION = make_bad_request( code=CONFLICT, description=u"The container name already exists." ) CONTAINER_NOT_FOUND = make_bad_request( code=NOT_FOUND, description=u"Container not found.") CONTAINER_PORT_COLLISION = make_bad_request( code=CONFLICT, description=u"A specified external port is already in use." )
from lae_util import stripe from twisted.python.filepath import FilePath stripe.api_key = FilePath('../../k8s_secrets/stripe-private.key').getContent().strip() amount = 2500 interval = "month" currency = "USD" name = "LeastAuthority Secure Simple Storage Service (S4)" plan_id = "S4_consumer_iteration_2_beta1_2014-05-27" trial_period_days = 30 statement_descriptor = "S4" stripe.Plan.create(amount=amount, interval=interval, name=name, currency=currency, id=plan_id, trial_period_days=trial_period_days, statement_descriptor=statement_descriptor)
def mapPath(self, fsPathString): return FilePath(fsPathString)
def setUp(self): self.userPath = FilePath(self.mktemp()) self.userPath.makedirs() self.addCleanup(self.userPath.remove) self.makeUsers(self.userPath.path)
def setUp(self): FilePath(self.SMTP_PASSWORD_PATH).setContent(self.SMTP_PASSWORD) self.patch(send_email, 'SMTP_PASSWORD_PATH', self.SMTP_PASSWORD_PATH)
from uuid import UUID import yaml from bitmath import GiB from twisted.python.filepath import FilePath from twisted.internet.task import deferLater from klein import Klein from ..restapi import structured from ..control._config import dataset_id_from_name from ..apiclient import DatasetAlreadyExists SCHEMA_BASE = FilePath(__file__).sibling(b'schema') SCHEMAS = { b'/types.json': yaml.safe_load(SCHEMA_BASE.child(b'types.yml').getContent()), b'/endpoints.json': yaml.safe_load(SCHEMA_BASE.child(b'endpoints.yml').getContent()), } # The default size of a created volume: DEFAULT_SIZE = int(GiB(100).to_Byte().value) def _endpoint(name): """ Decorator factory for API endpoints, adding appropriate JSON in/out encoding.
def _parseTest(self, xml): path = FilePath(self.mktemp()) path.setContent(xml) return tree.parseFileAndReport(path.path)
PRecord, PClass, CheckedPSet, CheckedPVector, CheckedPMap, field, ) from twisted.python.filepath import FilePath from twisted.python.reflect import qual as fqpn from .._persistence import ROOT_CLASS from ... import __version__ from ...testtools import TestCase PERSISTED_MODEL = FilePath(__file__).sibling(b"persisted_model.json") def _precord_model(klass): """ Serialize a ``PRecord`` or ``PClass`` model to something JSON-encodable. :param klass: A ``PRecord`` or ``PClass`` subclass. :return: Tuple of (model dictionary, further classes to process). """ further_classes = set() if issubclass(klass, PRecord): attr_name = "_precord_fields" else: attr_name = "_pclass_fields"
def test_runningLoreMultipleFiles(self): tmp = self.makeTemp('lore_index_test.xhtml', 'lore_index_test2.xhtml') templateFilename = sp('template.tpl') inputFilename = os.path.join(tmp, 'lore_index_test.xhtml') inputFilename2 = os.path.join(tmp, 'lore_index_test2.xhtml') indexFilename = 'theIndexFile' bookFilename = os.path.join(tmp, 'lore_test_book.book') bf = open(bookFilename, 'w') bf.write('Chapter(r"%s", None)\n' % inputFilename) bf.write('Chapter(r"%s", None)\n' % inputFilename2) bf.close() options = lore.Options() options.parseOptions([ '--null', '--book=%s' % bookFilename, '--config', 'template=%s' % templateFilename, '--index=%s' % indexFilename ]) result = lore.runGivenOptions(options) self.assertEqual(None, result) self.assertEqual( # XXX This doesn't seem like a very good index file. """\ aahz: <a href="lore_index_test2.html#index03">link</a><br /> aahz2: <a href="lore_index_test2.html#index02">link</a><br /> language of programming: <a href="lore_index_test.html#index02">link</a>, <a href="lore_index_test2.html#index01">link</a><br /> programming language: <a href="lore_index_test.html#index01">link</a><br /> """, file(FilePath(indexFilename + ".html").path).read()) self.assertXMLEqual( """\ <?xml version="1.0" ?><!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Transitional//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'><html lang="en" xmlns="http://www.w3.org/1999/xhtml"> <head><title>Twisted Documentation: The way of the program</title></head> <body bgcolor="white"> <h1 class="title">The way of the program</h1> <div class="content"> <span/> <p>The first paragraph.</p> <h2>The Python programming language<a name="auto0"/></h2> <a name="index01"/> <a name="index02"/> <p>The second paragraph.</p> </div> <a href="theIndexFile.html">Index</a> </body> </html>""", FilePath(tmp).child("lore_index_test.html").getContent()) self.assertXMLEqual( """\ <?xml version="1.0" ?><!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Transitional//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'><html lang="en" xmlns="http://www.w3.org/1999/xhtml"> <head><title>Twisted Documentation: The second page to index</title></head> <body bgcolor="white"> <h1 class="title">The second page to index</h1> <div class="content"> <span/> <p>The first paragraph of the second page.</p> <h2>The Jython programming language<a name="auto0"/></h2> <a name="index01"/> <a name="index02"/> <a name="index03"/> <p>The second paragraph of the second page.</p> </div> <a href="theIndexFile.html">Index</a> </body> </html>""", FilePath(tmp).child("lore_index_test2.html").getContent())
# sphinx-quickstart on Mon Apr 28 14:54:33 2014. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. from twisted.python.filepath import FilePath import sys import os sys.path.insert(0, FilePath(__file__).parent().parent().path) # Check if we are building on readthedocs on_rtd = os.environ.get('READTHEDOCS', None) == 'True' # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.extlinks', 'sphinx.ext.ifconfig', 'sphinx.ext.intersphinx',
def __init__(self, filename): self.filepath = FilePath(filename)
def test_introducer(self): """ The introducer furl is stable across restarts. """ basedir = self.workdir("test_introducer") c1 = os.path.join(basedir, "c1") tahoe = CLINodeAPI(reactor, FilePath(c1)) self.addCleanup(tahoe.stop_and_wait) out, err, rc_or_sig = yield self.run_bintahoe([ "--quiet", "create-introducer", "--basedir", c1, "--hostname", "127.0.0.1", ]) self.assertEqual(rc_or_sig, 0) # This makes sure that node.url is written, which allows us to # detect when the introducer restarts in _node_has_restarted below. config = fileutil.read(tahoe.config_file.path) self.assertIn('{}web.port = {}'.format(linesep, linesep), config) fileutil.write( tahoe.config_file.path, config.replace( '{}web.port = {}'.format(linesep, linesep), '{}web.port = 0{}'.format(linesep, linesep), ) ) p = Expect() tahoe.run(on_stdout(p)) yield p.expect("introducer running") tahoe.active() yield self.poll(tahoe.introducer_furl_file.exists) # read the introducer.furl file so we can check that the contents # don't change on restart furl = fileutil.read(tahoe.introducer_furl_file.path) tahoe.active() # We don't keep track of PIDs in files on Windows. if not platform.isWindows(): self.assertTrue(tahoe.twistd_pid_file.exists()) self.assertTrue(tahoe.node_url_file.exists()) # rm this so we can detect when the second incarnation is ready tahoe.node_url_file.remove() yield tahoe.stop_and_wait() p = Expect() tahoe.run(on_stdout(p)) yield p.expect("introducer running") # Again, the second incarnation of the node might not be ready yet, so # poll until it is. This time introducer_furl_file already exists, so # we check for the existence of node_url_file instead. yield self.poll(tahoe.node_url_file.exists) # The point of this test! After starting the second time the # introducer furl file must exist and contain the same contents as it # did before. self.assertTrue(tahoe.introducer_furl_file.exists()) self.assertEqual(furl, fileutil.read(tahoe.introducer_furl_file.path))
def exists(self): """ Returns true if the underlying archive exists. """ return FilePath(self.zipfile.filename).exists()
def test_client(self): """ Test too many things. 0) Verify that "tahoe create-node" takes a --webport option and writes the value to the configuration file. 1) Verify that "tahoe run" writes a pid file and a node url file (on POSIX). 2) Verify that the storage furl file has a stable value across a "tahoe run" / stop / "tahoe run" sequence. 3) Verify that the pid file is removed after SIGTERM (on POSIX). """ basedir = self.workdir("test_client") c1 = os.path.join(basedir, "c1") tahoe = CLINodeAPI(reactor, FilePath(c1)) # Set this up right now so we don't forget later. self.addCleanup(tahoe.cleanup) out, err, rc_or_sig = yield self.run_bintahoe([ "--quiet", "create-node", "--basedir", c1, "--webport", "0", "--hostname", "localhost", ]) self.failUnlessEqual(rc_or_sig, 0) # Check that the --webport option worked. config = fileutil.read(tahoe.config_file.path) self.assertIn( '{}web.port = 0{}'.format(linesep, linesep), config, ) # After this it's safe to start the node tahoe.active() p = Expect() # This will run until we stop it. tahoe.run(on_stdout(p)) # Wait for startup to have proceeded to a reasonable point. yield p.expect("client running") tahoe.active() # read the storage.furl file so we can check that its contents don't # change on restart storage_furl = fileutil.read(tahoe.storage_furl_file.path) # We don't keep track of PIDs in files on Windows. if not platform.isWindows(): self.assertTrue(tahoe.twistd_pid_file.exists()) # rm this so we can detect when the second incarnation is ready tahoe.node_url_file.remove() yield tahoe.stop_and_wait() p = Expect() # We don't have to add another cleanup for this one, the one from # above is still registered. tahoe.run(on_stdout(p)) yield p.expect("client running") tahoe.active() self.assertEqual( storage_furl, fileutil.read(tahoe.storage_furl_file.path), ) if not platform.isWindows(): self.assertTrue( tahoe.twistd_pid_file.exists(), "PID file ({}) didn't exist when we expected it to. " "These exist: {}".format( tahoe.twistd_pid_file, tahoe.twistd_pid_file.parent().listdir(), ), ) yield tahoe.stop_and_wait() if not platform.isWindows(): # twistd.pid should be gone by now. self.assertFalse(tahoe.twistd_pid_file.exists())
def getModificationTime(self): """ Return the archive file's modification time. """ return FilePath(self.zipfile.filename).getModificationTime()
def test_create_snapshot(self, author, folder_name, path_in_folder, some_content): """ A **POST** to **/v1/magic-folder/:folder-name/snapshot** with a **path** query argument creates a new local snapshot for the file at the given path in the named folder. """ local_path = FilePath(self.mktemp()) local_path.makedirs() some_file = local_path.preauthChild(path_in_folder).asBytesMode( "utf-8") some_file.parent().makedirs(ignoreExistingDirectory=True) some_file.setContent(some_content) treq = treq_for_folders( Clock(), FilePath(self.mktemp()), AUTH_TOKEN, { folder_name: magic_folder_config(author, FilePath(self.mktemp()), local_path) }, # Unlike test_wait_for_completion above we start the folder # services. This will allow the local snapshot to be created and # our request to receive a response. start_folder_services=True, ) self.assertThat( authorized_request( treq, AUTH_TOKEN, b"POST", self.url.child(folder_name, "snapshot").set(u"path", path_in_folder), ), succeeded(matches_response(code_matcher=Equals(CREATED), ), ), ) self.assertThat( authorized_request( treq, AUTH_TOKEN, b"GET", DecodedURL.from_text(u"http://example.invalid./v1/snapshot")), succeeded( matches_response( code_matcher=Equals(OK), headers_matcher=header_contains({ u"Content-Type": Equals([u"application/json"]), }), body_matcher=AfterPreprocessing( loads, MatchesDict({ folder_name: MatchesDict({ path_in_folder: MatchesListwise([ MatchesDict({ u"type": Equals(u"local"), u"identifier": is_hex_uuid(), # XXX It would be nice to see some # parents if there are any. u"parents": Equals([]), u"content-path": AfterPreprocessing( lambda path: FilePath(path). getContent(), Equals(some_content), ), u"author": Equals(author.to_remote_author(). to_json()), }), ]), }), }), ), ), ), )