def test_add_participant(self, folder_name, personal_dmd):
"""
Adding a new participant works.
"""
local_path = FilePath(self.mktemp())
local_path.makedirs()
folder_config = magic_folder_config(
create_local_author("iris"),
FilePath(self.mktemp()),
local_path,
)
# we can't add a new participant if their DMD is the same as
# one we already have .. and because Hypothesis is 'sneaky' we
# have to make sure it's not our collective, either
assume(personal_dmd != folder_config["upload-dircap"])
assume(personal_dmd != to_readonly_capability(
folder_config["upload-dircap"]))
assume(personal_dmd != folder_config["collective-dircap"])
assume(personal_dmd != to_readonly_capability(
folder_config["collective-dircap"]))
root = create_fake_tahoe_root()
# put our Collective DMD into the fake root
root._uri.data[folder_config["collective-dircap"]] = dumps([
u"dirnode",
{
u"children": {
"iris": format_filenode(folder_config["upload-dircap"]),
},
},
])
tahoe_client = create_tahoe_client(
DecodedURL.from_text(u"http://invalid./"),
create_tahoe_treq_client(root),
)
treq = treq_for_folders(
Clock(),
FilePath(self.mktemp()),
AUTH_TOKEN,
{
folder_name: folder_config,
},
start_folder_services=False,
tahoe_client=tahoe_client,
)
# add a participant using the API
self.assertThat(
authorized_request(
treq, AUTH_TOKEN, b"POST",
self.url.child(folder_name, "participants"),
dumps({
"author": {
"name": "kelly"
},
"personal_dmd": personal_dmd,
}).encode("utf8")),
succeeded(
matches_response(code_matcher=Equals(CREATED),
body_matcher=AfterPreprocessing(
loads, Equals({})))))
# confirm that the "list participants" API includes the added
# participant
self.assertThat(
authorized_request(
treq,
AUTH_TOKEN,
b"GET",
self.url.child(folder_name, "participants"),
),
succeeded(
matches_response(code_matcher=Equals(OK),
body_matcher=AfterPreprocessing(
loads,
Equals({
u"iris": {
u"personal_dmd":
folder_config["upload-dircap"],
},
u'kelly': {
u'personal_dmd': personal_dmd,
}
})))))
class StorageStatusElement(Element):
"""Class to render a storage status page."""
loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml"))
def __init__(self, storage, nickname=""):
"""
:param _StorageServer storage: data about storage.
:param string nickname: friendly name for storage.
"""
super(StorageStatusElement, self).__init__()
self._storage = storage
self._nickname = nickname
@renderer
def nickname(self, req, tag):
return tag(self._nickname)
@renderer
def nodeid(self, req, tag):
return tag(idlib.nodeid_b2a(self._storage.my_nodeid))
def _get_storage_stat(self, key):
"""Get storage server statistics.
Storage Server keeps a dict that contains various usage and
latency statistics. The dict looks like this:
{
'storage_server.accepting_immutable_shares': 1,
'storage_server.allocated': 0,
'storage_server.disk_avail': 106539192320,
'storage_server.disk_free_for_nonroot': 106539192320,
'storage_server.disk_free_for_root': 154415284224,
'storage_server.disk_total': 941088460800,
'storage_server.disk_used': 786673176576,
'storage_server.latencies.add-lease.01_0_percentile': None,
'storage_server.latencies.add-lease.10_0_percentile': None,
...
}
``StorageServer.get_stats()`` returns the above dict. Storage
status page uses a subset of the items in the dict, concerning
disk usage.
:param str key: storage server statistic we want to know.
"""
return self._storage.get_stats().get(key)
def render_abbrev_space(self, size):
if size is None:
return u"?"
return abbreviate_space(size)
def render_space(self, size):
if size is None:
return u"?"
return u"%d" % size
@renderer
def storage_stats(self, req, tag):
# Render storage status table that appears near the top of the page.
total = self._get_storage_stat("storage_server.disk_total")
used = self._get_storage_stat("storage_server.disk_used")
free_root = self._get_storage_stat("storage_server.disk_free_for_root")
free_nonroot = self._get_storage_stat(
"storage_server.disk_free_for_nonroot")
reserved = self._get_storage_stat("storage_server.reserved_space")
available = self._get_storage_stat("storage_server.disk_avail")
tag.fillSlots(
disk_total=self.render_space(total),
disk_total_abbrev=self.render_abbrev_space(total),
disk_used=self.render_space(used),
disk_used_abbrev=self.render_abbrev_space(used),
disk_free_for_root=self.render_space(free_root),
disk_free_for_root_abbrev=self.render_abbrev_space(free_root),
disk_free_for_nonroot=self.render_space(free_nonroot),
disk_free_for_nonroot_abbrev=self.render_abbrev_space(
free_nonroot),
reserved_space=self.render_space(reserved),
reserved_space_abbrev=self.render_abbrev_space(reserved),
disk_avail=self.render_space(available),
disk_avail_abbrev=self.render_abbrev_space(available))
return tag
@renderer
def accepting_immutable_shares(self, req, tag):
accepting = self._get_storage_stat(
"storage_server.accepting_immutable_shares")
return tag({True: "Yes", False: "No"}[bool(accepting)])
@renderer
def last_complete_bucket_count(self, req, tag):
s = self._storage.bucket_counter.get_state()
count = s.get("last-complete-bucket-count")
if count is None:
return tag("Not computed yet")
return tag(str(count))
@renderer
def count_crawler_status(self, req, tag):
p = self._storage.bucket_counter.get_progress()
return tag(self.format_crawler_progress(p))
def format_crawler_progress(self, p):
cycletime = p["estimated-time-per-cycle"]
cycletime_s = ""
if cycletime is not None:
cycletime_s = " (estimated cycle time %s)" % abbreviate_time(
cycletime)
if p["cycle-in-progress"]:
pct = p["cycle-complete-percentage"]
soon = p["remaining-sleep-time"]
eta = p["estimated-cycle-complete-time-left"]
eta_s = ""
if eta is not None:
eta_s = " (ETA %ds)" % eta
return [
"Current crawl %.1f%% complete" % pct,
eta_s,
" (next work in %s)" % abbreviate_time(soon),
cycletime_s,
]
else:
soon = p["remaining-wait-time"]
return ["Next crawl in %s" % abbreviate_time(soon), cycletime_s]
@renderer
def storage_running(self, req, tag):
if self._storage:
return tag
return T.h1("No Storage Server Running")
@renderer
def lease_expiration_enabled(self, req, tag):
lc = self._storage.lease_checker
if lc.expiration_enabled:
return tag("Enabled: expired leases will be removed")
else:
return tag("Disabled: scan-only mode, no leases will be removed")
@renderer
def lease_expiration_mode(self, req, tag):
lc = self._storage.lease_checker
if lc.mode == "age":
if lc.override_lease_duration is None:
tag("Leases will expire naturally, probably 31 days after "
"creation or renewal.")
else:
tag("Leases created or last renewed more than %s ago "
"will be considered expired." %
abbreviate_time(lc.override_lease_duration))
else:
assert lc.mode == "cutoff-date"
localizedutcdate = time.strftime("%d-%b-%Y",
time.gmtime(lc.cutoff_date))
isoutcdate = time_format.iso_utc_date(lc.cutoff_date)
tag("Leases created or last renewed before %s (%s) UTC "
"will be considered expired." % (
isoutcdate,
localizedutcdate,
))
if len(lc.mode) > 2:
tag(" The following sharetypes will be expired: ",
" ".join(sorted(lc.sharetypes_to_expire)), ".")
return tag
@renderer
def lease_current_cycle_progress(self, req, tag):
lc = self._storage.lease_checker
p = lc.get_progress()
return tag(self.format_crawler_progress(p))
@renderer
def lease_current_cycle_results(self, req, tag):
lc = self._storage.lease_checker
p = lc.get_progress()
if not p["cycle-in-progress"]:
return ""
s = lc.get_state()
so_far = s["cycle-to-date"]
sr = so_far["space-recovered"]
er = s["estimated-remaining-cycle"]
esr = er["space-recovered"]
ec = s["estimated-current-cycle"]
ecr = ec["space-recovered"]
p = T.ul()
def add(*pieces):
p(T.li(pieces))
def maybe(d):
if d is None:
return "?"
return "%d" % d
add(
"So far, this cycle has examined %d shares in %d buckets" %
(sr["examined-shares"], sr["examined-buckets"]),
" (%d mutable / %d immutable)" %
(sr["examined-buckets-mutable"], sr["examined-buckets-immutable"]),
" (%s / %s)" %
(abbreviate_space(sr["examined-diskbytes-mutable"]),
abbreviate_space(sr["examined-diskbytes-immutable"])),
)
add("and has recovered: ", self.format_recovered(sr, "actual"))
if so_far["expiration-enabled"]:
add("The remainder of this cycle is expected to recover: ",
self.format_recovered(esr, "actual"))
add("The whole cycle is expected to examine %s shares in %s buckets"
% (maybe(ecr["examined-shares"]), maybe(
ecr["examined-buckets"])))
add("and to recover: ", self.format_recovered(ecr, "actual"))
else:
add("If expiration were enabled, we would have recovered: ",
self.format_recovered(sr, "configured"), " by now")
add("and the remainder of this cycle would probably recover: ",
self.format_recovered(esr, "configured"))
add("and the whole cycle would probably recover: ",
self.format_recovered(ecr, "configured"))
add(
"if we were strictly using each lease's default 31-day lease lifetime "
"(instead of our configured behavior), "
"this cycle would be expected to recover: ",
self.format_recovered(ecr, "original"))
if so_far["corrupt-shares"]:
add(
"Corrupt shares:",
T.ul((T.li([
"SI %s shnum %d" % corrupt_share
for corrupt_share in so_far["corrupt-shares"]
]))))
return tag("Current cycle:", p)
@renderer
def lease_last_cycle_results(self, req, tag):
lc = self._storage.lease_checker
h = lc.get_state()["history"]
if not h:
return ""
last = h[max(h.keys())]
start, end = last["cycle-start-finish-times"]
tag(
"Last complete cycle (which took %s and finished %s ago)"
" recovered: " %
(abbreviate_time(end - start), abbreviate_time(time.time() - end)),
self.format_recovered(last["space-recovered"], "actual"))
p = T.ul()
def add(*pieces):
p(T.li(pieces))
saw = self.format_recovered(last["space-recovered"], "examined")
add("and saw a total of ", saw)
if not last["expiration-enabled"]:
rec = self.format_recovered(last["space-recovered"], "configured")
add(
"but expiration was not enabled. If it had been, "
"it would have recovered: ", rec)
if last["corrupt-shares"]:
add(
"Corrupt shares:",
T.ul((T.li([
"SI %s shnum %d" % corrupt_share
for corrupt_share in last["corrupt-shares"]
]))))
return tag(p)
@staticmethod
def format_recovered(sr, a):
def maybe(d):
if d is None:
return "?"
return "%d" % d
return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \
(maybe(sr["%s-shares" % a]),
maybe(sr["%s-buckets" % a]),
maybe(sr["%s-buckets-mutable" % a]),
maybe(sr["%s-buckets-immutable" % a]),
abbreviate_space(sr["%s-diskbytes" % a]),
abbreviate_space(sr["%s-diskbytes-mutable" % a]),
abbreviate_space(sr["%s-diskbytes-immutable" % a]),
)
def getAccessTime(self):
"""
Return the archive file's last access time.
"""
return FilePath(self.zipfile.filename).getAccessTime()
def filenameGenerator(originalFileName, outputExtension):
name = os.path.splitext(FilePath(originalFileName).basename())[0]
return base.child(name + outputExtension).path
def buildDirectory(store,
dataRoot,
servicesInfo,
augmentServiceInfo,
wikiServiceInfo,
serversDB=None,
cachingSeconds=0):
"""
Return a directory without using a config object; suitable for tests
which need to have mulitple directory instances.
@param store: The store.
@param dataRoot: The path to the directory containing xml files for any xml
based services.
@param servicesInfo: An interable of ConfigDicts mirroring the
DirectoryService and ResourceService sections of stdconfig
@param augmentServiceInfo: A ConfigDict mirroring the AugmentService section
of stdconfig
@param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig
@param serversDB: A ServersDB object to assign to the directory
"""
aggregatedServices = []
cachingServices = []
ldapService = None # LDAP DS has extra stats (see augment.py)
for serviceValue in servicesInfo:
if not serviceValue.Enabled:
continue
directoryType = serviceValue.type.lower()
params = serviceValue.params
if "xml" in directoryType:
xmlFile = params.xmlFile
xmlFile = fullServerPath(dataRoot, xmlFile)
fp = FilePath(xmlFile)
if not fp.exists():
fp.setContent(DEFAULT_XML_CONTENT)
directory = XMLDirectoryService(fp)
elif "opendirectory" in directoryType:
from txdav.who.opendirectory import (DirectoryService as
ODDirectoryService)
# We don't want system accounts returned in lookups, so tell
# the service to suppress them.
node = params.node
directory = ODDirectoryService(nodeName=node,
suppressSystemRecords=True)
elif "ldap" in directoryType:
from twext.who.ldap import (DirectoryService as
LDAPDirectoryService, FieldName as
LDAPFieldName, RecordTypeSchema)
if params.credentials.dn and params.credentials.password:
creds = UsernamePassword(params.credentials.dn,
params.credentials.password)
else:
creds = None
mapping = params.mapping
extraFilters = params.extraFilters
directory = LDAPDirectoryService(
params.uri,
params.rdnSchema.base,
credentials=creds,
fieldNameToAttributesMap=MappingProxyType({
BaseFieldName.uid:
mapping.uid,
BaseFieldName.guid:
mapping.guid,
BaseFieldName.shortNames:
mapping.shortNames,
BaseFieldName.fullNames:
mapping.fullNames,
BaseFieldName.emailAddresses:
mapping.emailAddresses,
LDAPFieldName.memberDNs:
mapping.memberDNs,
CalFieldName.readOnlyProxy:
mapping.readOnlyProxy,
CalFieldName.readWriteProxy:
mapping.readWriteProxy,
CalFieldName.hasCalendars:
mapping.hasCalendars,
CalFieldName.autoScheduleMode:
mapping.autoScheduleMode,
}),
recordTypeSchemas=MappingProxyType({
RecordType.user:
RecordTypeSchema(
relativeDN=params.rdnSchema.users,
attributes=(),
),
RecordType.group:
RecordTypeSchema(
relativeDN=params.rdnSchema.groups,
attributes=(),
),
CalRecordType.location:
RecordTypeSchema(
relativeDN=params.rdnSchema.locations,
attributes=(),
),
CalRecordType.resource:
RecordTypeSchema(
relativeDN=params.rdnSchema.resources,
attributes=(),
),
CalRecordType.address:
RecordTypeSchema(
relativeDN=params.rdnSchema.addresses,
attributes=(),
),
}),
extraFilters={
RecordType.user: extraFilters.get("users", ""),
RecordType.group: extraFilters.get("groups", ""),
CalRecordType.location: extraFilters.get("locations", ""),
CalRecordType.resource: extraFilters.get("resources", ""),
CalRecordType.address: extraFilters.get("addresses", ""),
})
ldapService = directory
elif "inmemory" in directoryType:
from txdav.who.test.support import CalendarInMemoryDirectoryService
directory = CalendarInMemoryDirectoryService()
else:
log.error("Invalid DirectoryType: {dt}", dt=directoryType)
raise DirectoryConfigurationError
# Set the appropriate record types on each service
types = []
fieldNames = []
for recordTypeName in params.recordTypes:
recordType = {
"users": RecordType.user,
"groups": RecordType.group,
"locations": CalRecordType.location,
"resources": CalRecordType.resource,
"addresses": CalRecordType.address,
}.get(recordTypeName, None)
if recordType is None:
log.error("Invalid Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
if recordType in types:
log.error("Duplicate Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
types.append(recordType)
directory.recordType = ConstantsContainer(types)
directory.fieldName = ConstantsContainer(
(directory.fieldName, CalFieldName))
fieldNames.append(directory.fieldName)
if cachingSeconds:
directory = CachingDirectoryService(directory,
expireSeconds=cachingSeconds)
cachingServices.append(directory)
aggregatedServices.append(directory)
#
# Setup the Augment Service
#
if augmentServiceInfo.type:
for augmentFile in augmentServiceInfo.params.xmlFiles:
augmentFile = fullServerPath(dataRoot, augmentFile)
augmentFilePath = FilePath(augmentFile)
if not augmentFilePath.exists():
augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT)
augmentClass = namedClass(augmentServiceInfo.type)
log.info("Configuring augment service of type: {augmentClass}",
augmentClass=augmentClass)
try:
augmentService = augmentClass(**augmentServiceInfo.params)
except IOError:
log.error("Could not start augment service")
raise
else:
augmentService = None
userDirectory = None
for directory in aggregatedServices:
if RecordType.user in directory.recordTypes():
userDirectory = directory
break
else:
log.error("No directory service set up for users")
raise DirectoryConfigurationError
# Delegate service
delegateDirectory = DelegateDirectoryService(userDirectory.realmName,
store)
# (put at front of list so we don't try to ask the actual DS services
# about the delegate-related principals, for performance)
aggregatedServices.insert(0, delegateDirectory)
# Wiki service
if wikiServiceInfo.Enabled:
aggregatedServices.append(
WikiDirectoryService(userDirectory.realmName,
wikiServiceInfo.CollabHost,
wikiServiceInfo.CollabPort))
# Aggregate service
aggregateDirectory = AggregateDirectoryService(userDirectory.realmName,
aggregatedServices)
# Augment service
try:
fieldNames.append(CalFieldName)
augmented = AugmentedDirectoryService(aggregateDirectory, store,
augmentService)
augmented.fieldName = ConstantsContainer(fieldNames)
# The delegate directory needs a way to look up user/group records
# so hand it a reference to the augmented directory.
# FIXME: is there a better pattern to use here?
delegateDirectory.setMasterDirectory(augmented)
# Tell each caching service what method to use when reporting
# times and cache stats
for cachingService in cachingServices:
cachingService.setTimingMethod(augmented._addTiming)
# LDAP has additional stats to report
augmented._ldapDS = ldapService
except Exception as e:
log.error("Could not create directory service", error=e)
raise
if serversDB is not None:
augmented.setServersDB(serversDB)
return augmented
def templatefilepath(filename):
return FilePath(templatefile(filename))
def setUp(self):
self.path = FilePath(__file__)
def test_tls_auth_denied(self):
"""
A MQTT client offering the wrong certificate won't be authenticated.
"""
reactor, router, server_factory, session_factory = build_mqtt_server()
real_reactor = selectreactor.SelectReactor()
logger = make_logger()
session, pump = connect_application_session(
server_factory,
ObservingSession,
component_config=ComponentConfig(realm=u"mqtt"))
endpoint = create_listening_endpoint_from_config(
{
"type": "tcp",
"port": 1099,
"interface": "0.0.0.0",
"tls": {
"certificate": "server.crt",
"key": "server.key",
"dhparam": "dhparam",
"ca_certificates":
["ca.cert.pem", "intermediate.cert.pem"]
},
},
FilePath(__file__).sibling('certs').path, real_reactor, logger)
client_endpoint = create_connecting_endpoint_from_config(
{
"type": "tcp",
"host": "127.0.0.1",
"port": 1099,
"tls": {
# BAD key: trusted by the CA, but wrong ID
"certificate": "client_1.crt",
"hostname": u"localhost",
"key": "client_1.key",
"ca_certificates":
["ca.cert.pem", "intermediate.cert.pem"]
},
},
FilePath(__file__).sibling('certs').path,
real_reactor,
logger)
p = []
l = endpoint.listen(server_factory)
class TestProtocol(Protocol):
data = b""
expected = (ConnACK(session_present=False,
return_code=1).serialise())
def dataReceived(self_, data):
self_.data = self_.data + data
if len(self_.data) == len(self_.expected):
self.assertEqual(self_.data, self_.expected)
real_reactor.stop()
@l.addCallback
def _listening(factory):
d = client_endpoint.connect(Factory.forProtocol(TestProtocol))
@d.addCallback
def _(proto):
p.append(proto)
proto.transport.write(
Connect(
client_id=u"test123",
flags=ConnectFlags(clean_session=False)).serialise())
proto.transport.write(
Publish(duplicate=False,
qos_level=1,
retain=False,
topic_name=u"test",
payload=b"{}",
packet_identifier=1).serialise())
lc = LoopingCall(pump.flush)
lc.clock = real_reactor
lc.start(0.01)
def timeout():
print("Timing out :(")
real_reactor.stop()
print(self.logs.log_text.getvalue())
# Timeout, just in case
real_reactor.callLater(10, timeout)
real_reactor.run()
client_protocol = p[0]
# We get a CONNECT
self.assertEqual(
client_protocol.data,
ConnACK(session_present=False, return_code=1).serialise())
client_protocol.data = b""
pump.flush()
# No events!
self.assertEqual(len(session.events), 0)
Tests for C{setup.py}, Twisted's distutils integration file.
"""
from __future__ import division, absolute_import
import os, sys
import twisted
from twisted.trial.unittest import SynchronousTestCase
from twisted.python.filepath import FilePath
from twisted.python.dist import getExtensions
# Get rid of the UTF-8 encoding and bytes topfiles segment when FilePath
# supports unicode. #2366, #4736, #5203. Also #4743, which requires checking
# setup.py, not just the topfiles directory.
if not FilePath(twisted.__file__.encode('utf-8')).sibling(b'topfiles').child(
b'setup.py').exists():
sourceSkip = "Only applies to source checkout of Twisted"
else:
sourceSkip = None
class TwistedExtensionsTests(SynchronousTestCase):
if sourceSkip is not None:
skip = sourceSkip
def setUp(self):
"""
Change the working directory to the parent of the C{twisted} package so
that L{twisted.python.dist.getExtensions} finds Twisted's own extension
definitions.
"""
class MoreInfoElement(Element):
"""
An ``Element`` HTML template which can be flattened to describe this node.
:param Node node: The node to describe.
"""
loader = XMLFile(FilePath(__file__).sibling("info.xhtml"))
def __init__(self, node):
super(MoreInfoElement, self).__init__()
self.original = node
def abbrev(self, storage_index_or_none):
if storage_index_or_none:
return base32.b2a(storage_index_or_none)[:6]
return "LIT file"
def get_type(self):
node = self.original
if IDirectoryNode.providedBy(node):
if not node.is_mutable():
return "immutable directory"
return "directory"
if IFileNode.providedBy(node):
si = node.get_storage_index()
if si:
if node.is_mutable():
ret = "mutable file"
if node.get_version() == MDMF_VERSION:
ret += " (mdmf)"
else:
ret += " (sdmf)"
return ret
return "immutable file"
return "immutable LIT file"
return "unknown"
@renderer
def title(self, req, tag):
node = self.original
si = node.get_storage_index()
t = "More Info for %s" % self.get_type()
if si:
t += " (SI=%s)" % self.abbrev(si)
return tag(t)
@renderer
def header(self, req, tag):
return self.title(req, tag)
@renderer
def type(self, req, tag):
return tag(self.get_type())
@renderer
def si(self, req, tag):
si = self.original.get_storage_index()
if not si:
return "None"
return tag(base32.b2a(si))
@renderer
def size(self, req, tag):
node = self.original
d = node.get_current_size()
def _no_size(size):
if size is None:
return "?"
return size
d.addCallback(_no_size)
def _handle_unrecoverable(f):
f.trap(UnrecoverableFileError)
return "?"
d.addErrback(_handle_unrecoverable)
d.addCallback(lambda size: tag(str(size)))
return d
@renderer
def directory_writecap(self, req, tag):
node = self.original
if not IDirectoryNode.providedBy(node):
return ""
if node.is_readonly():
return ""
return tag(node.get_uri())
@renderer
def directory_readcap(self, req, tag):
node = self.original
if not IDirectoryNode.providedBy(node):
return ""
return tag(node.get_readonly_uri())
@renderer
def directory_verifycap(self, req, tag):
node = self.original
if not IDirectoryNode.providedBy(node):
return ""
verifier = node.get_verify_cap()
if verifier:
return tag(node.get_verify_cap().to_string())
return ""
@renderer
def file_writecap(self, req, tag):
node = self.original
if IDirectoryNode.providedBy(node):
node = node._node
write_uri = node.get_write_uri()
if not write_uri:
return ""
return tag(write_uri)
@renderer
def file_readcap(self, req, tag):
node = self.original
if IDirectoryNode.providedBy(node):
node = node._node
read_uri = node.get_readonly_uri()
if not read_uri:
return ""
return tag(read_uri)
@renderer
def file_verifycap(self, req, tag):
node = self.original
if IDirectoryNode.providedBy(node):
node = node._node
verifier = node.get_verify_cap()
if verifier:
return tag(node.get_verify_cap().to_string())
return ""
def get_root(self, req):
# the addSlash=True gives us one extra (empty) segment
depth = len(req.prepath) + len(req.postpath) - 1
link = "/".join([".."] * depth)
return link
@renderer
def raw_link(self, req, tag):
node = self.original
if IDirectoryNode.providedBy(node):
node = node._node
elif IFileNode.providedBy(node):
pass
else:
return ""
root = self.get_root(req)
quoted_uri = urllib.quote(node.get_uri())
text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri)
return T.li("Raw data as ", T.a("text/plain", href=text_plain_url))
@renderer
def is_checkable(self, req, tag):
node = self.original
si = node.get_storage_index()
if si:
return tag
# don't show checker button for LIT files
return ""
@renderer
def check_form(self, req, tag):
node = self.original
quoted_uri = urllib.quote(node.get_uri())
target = self.get_root(req) + "/uri/" + quoted_uri
if IDirectoryNode.providedBy(node):
target += "/"
check = T.form(action=target, method="post",
enctype="multipart/form-data")(
T.fieldset(
T.input(type="hidden", name="t", value="check"),
T.input(type="hidden", name="return_to", value="."),
T.legend("Check on this object", class_="freeform-form-label"),
T.div(
"Verify every bit? (EXPENSIVE):",
T.input(type="checkbox", name="verify"),
),
T.div("Repair any problems?: ",
T.input(type="checkbox", name="repair")),
T.div("Add/renew lease on all shares?: ",
T.input(type="checkbox", name="add-lease")),
T.div("Emit results in JSON format?: ",
T.input(type="checkbox", name="output", value="JSON")),
T.input(type="submit", value="Check"),
))
return tag(check)
@renderer
def is_mutable_file(self, req, tag):
node = self.original
if IDirectoryNode.providedBy(node):
return ""
if (IFileNode.providedBy(node)
and node.is_mutable() and not node.is_readonly()):
return tag
return ""
@renderer
def overwrite_form(self, req, tag):
node = self.original
root = self.get_root(req)
action = "%s/uri/%s" % (root, urllib.quote(node.get_uri()))
done_url = "%s/uri/%s?t=info" % (root, urllib.quote(node.get_uri()))
overwrite = T.form(action=action, method="post",
enctype="multipart/form-data")(
T.fieldset(
T.input(type="hidden", name="t", value="upload"),
T.input(type='hidden', name='when_done', value=done_url),
T.legend("Overwrite", class_="freeform-form-label"),
"Upload new contents: ",
T.input(type="file", name="file"),
" ",
T.input(type="submit", value="Replace Contents")
))
return tag(overwrite)
@renderer
def is_directory(self, req, tag):
node = self.original
if IDirectoryNode.providedBy(node):
return tag
return ""
@renderer
def deep_check_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16))
deep_check = T.form(action=req.path, method="post",
enctype="multipart/form-data")(
T.fieldset(
T.input(type="hidden", name="t", value="start-deep-check"),
T.input(type="hidden", name="return_to", value="."),
T.legend("Run a deep-check operation (EXPENSIVE)", class_="freeform-form-label"),
T.div(
"Verify every bit? (EVEN MORE EXPENSIVE):",
T.input(type="checkbox", name="verify"),
),
T.div("Repair any problems?: ",
T.input(type="checkbox", name="repair")),
T.div("Add/renew lease on all shares?: ",
T.input(type="checkbox", name="add-lease")),
T.div("Emit results in JSON format?: ",
T.input(type="checkbox", name="output", value="JSON")),
T.input(type="hidden", name="ophandle", value=ophandle),
T.input(type="submit", value="Deep-Check"),
))
return tag(deep_check)
@renderer
def deep_size_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16))
deep_size = T.form(action=req.path, method="post",
enctype="multipart/form-data")(
T.fieldset(
T.input(type="hidden", name="t", value="start-deep-size"),
T.legend("Run a deep-size operation (EXPENSIVE)", class_="freeform-form-label"),
T.input(type="hidden", name="ophandle", value=ophandle),
T.input(type="submit", value="Deep-Size"),
))
return tag(deep_size)
@renderer
def deep_stats_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16))
deep_stats = T.form(action=req.path, method="post",
enctype="multipart/form-data")(
T.fieldset(
T.input(type="hidden", name="t", value="start-deep-stats"),
T.legend("Run a deep-stats operation (EXPENSIVE)", class_="freeform-form-label"),
T.input(type="hidden", name="ophandle", value=ophandle),
T.input(type="submit", value="Deep-Stats"),
))
return tag(deep_stats)
@renderer
def manifest_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16))
manifest = T.form(action=req.path, method="post",
enctype="multipart/form-data")(
T.fieldset(
T.input(type="hidden", name="t", value="start-manifest"),
T.legend("Run a manifest operation (EXPENSIVE)", class_="freeform-form-label"),
T.div("Output Format: ",
T.select(name="output")
( T.option("HTML", value="html", selected="true"),
T.option("text", value="text"),
T.option("JSON", value="json"),
),
),
T.input(type="hidden", name="ophandle", value=ophandle),
T.input(type="submit", value="Manifest"),
))
return tag(manifest)
def get_filename(self):
return FilePath(VERSION_PATH)
def cbFinished(ignored):
self.assertEqual(FilePath(output).getContent(), "[('foo', 'bar')]")
from twisted.python.compat import _PY3, networkString, nativeString, intToBytes
from twisted.trial import unittest
from twisted.web import server, client, error, resource
from twisted.internet import reactor, defer, interfaces
from twisted.python.filepath import FilePath
from twisted.python.log import msg
from twisted.protocols.policies import WrappingFactory
from twisted.test.proto_helpers import StringTransport
try:
from twisted.internet import ssl
except:
ssl = None
from twisted import test
serverPEM = FilePath(test.__file__.encode("utf-8")).sibling(b'server.pem')
serverPEMPath = nativeString(serverPEM.path)
# Remove this in #6177, when static is ported to Python 3:
if _PY3:
from twisted.web.test.test_web import Data
else:
from twisted.web.static import Data
# Remove this in #6178, when util is ported to Python 3:
if _PY3:
class Redirect(resource.Resource):
isLeaf = 1
def __init__(self, url):
def run(self,
suite,
reactor=None,
cooperate=cooperate,
untilFailure=False):
"""
Spawn local worker processes and load tests. After that, run them.
@param suite: A tests suite to be run.
@param reactor: The reactor to use, to be customized in tests.
@type reactor: A provider of
L{twisted.internet.interfaces.IReactorProcess}
@param cooperate: The cooperate function to use, to be customized in
tests.
@type cooperate: C{function}
@param untilFailure: If C{True}, continue to run the tests until they
fail.
@type untilFailure: C{bool}.
@return: The test result.
@rtype: L{DistReporter}
"""
if reactor is None:
from twisted.internet import reactor
result = self._makeResult()
count = suite.countTestCases()
self._stream.write("Running %d tests.\n" % (count, ))
if not count:
# Take a shortcut if there is no test
suite.run(result.original)
self.writeResults(result)
return result
testDir, testDirLock = _unusedTestDirectory(
FilePath(self._workingDirectory))
workerNumber = min(count, self._workerNumber)
ampWorkers = [LocalWorkerAMP() for x in xrange(workerNumber)]
workers = self.createLocalWorkers(ampWorkers, testDir.path)
processEndDeferreds = [worker.endDeferred for worker in workers]
self.launchWorkerProcesses(reactor.spawnProcess, workers,
self._workerArguments)
def runTests():
testCases = iter(list(_iterateTests(suite)))
workerDeferreds = []
for worker in ampWorkers:
workerDeferreds.append(
self._driveWorker(worker,
result,
testCases,
cooperate=cooperate))
return DeferredList(workerDeferreds,
consumeErrors=True,
fireOnOneErrback=True)
stopping = []
def nextRun(ign):
self.writeResults(result)
if not untilFailure:
return
if not result.wasSuccessful():
return
d = runTests()
return d.addCallback(nextRun)
def stop(ign):
testDirLock.unlock()
if not stopping:
stopping.append(None)
reactor.stop()
def beforeShutDown():
if not stopping:
stopping.append(None)
d = DeferredList(processEndDeferreds, consumeErrors=True)
return d.addCallback(continueShutdown)
def continueShutdown(ign):
self.writeResults(result)
return ign
d = runTests()
d.addCallback(nextRun)
d.addBoth(stop)
reactor.addSystemEventTrigger('before', 'shutdown', beforeShutDown)
reactor.run()
return result
Dataset, Manifestation, Application, DockerImage, Port,
AttachedVolume, Link
)
from ._config import (
ApplicationMarshaller, FLOCKER_RESTART_POLICY_NAME_TO_POLICY,
model_from_configuration, FigConfiguration, FlockerConfiguration,
ConfigurationError
)
from .. import __version__
# Default port for REST API:
REST_API_PORT = 4523
SCHEMA_BASE = FilePath(__file__).parent().child(b'schema')
SCHEMAS = {
b'/v1/types.json': yaml.safe_load(
SCHEMA_BASE.child(b'types.yml').getContent()),
b'/v1/endpoints.json': yaml.safe_load(
SCHEMA_BASE.child(b'endpoints.yml').getContent()),
}
CONTAINER_NAME_COLLISION = make_bad_request(
code=CONFLICT, description=u"The container name already exists."
)
CONTAINER_NOT_FOUND = make_bad_request(
code=NOT_FOUND, description=u"Container not found.")
CONTAINER_PORT_COLLISION = make_bad_request(
code=CONFLICT, description=u"A specified external port is already in use."
)
from lae_util import stripe
from twisted.python.filepath import FilePath
stripe.api_key = FilePath('../../k8s_secrets/stripe-private.key').getContent().strip()
amount = 2500
interval = "month"
currency = "USD"
name = "LeastAuthority Secure Simple Storage Service (S4)"
plan_id = "S4_consumer_iteration_2_beta1_2014-05-27"
trial_period_days = 30
statement_descriptor = "S4"
stripe.Plan.create(amount=amount,
interval=interval,
name=name,
currency=currency,
id=plan_id,
trial_period_days=trial_period_days,
statement_descriptor=statement_descriptor)
def mapPath(self, fsPathString):
return FilePath(fsPathString)
def setUp(self):
self.userPath = FilePath(self.mktemp())
self.userPath.makedirs()
self.addCleanup(self.userPath.remove)
self.makeUsers(self.userPath.path)
def setUp(self):
FilePath(self.SMTP_PASSWORD_PATH).setContent(self.SMTP_PASSWORD)
self.patch(send_email, 'SMTP_PASSWORD_PATH', self.SMTP_PASSWORD_PATH)
from uuid import UUID
import yaml
from bitmath import GiB
from twisted.python.filepath import FilePath
from twisted.internet.task import deferLater
from klein import Klein
from ..restapi import structured
from ..control._config import dataset_id_from_name
from ..apiclient import DatasetAlreadyExists
SCHEMA_BASE = FilePath(__file__).sibling(b'schema')
SCHEMAS = {
b'/types.json':
yaml.safe_load(SCHEMA_BASE.child(b'types.yml').getContent()),
b'/endpoints.json':
yaml.safe_load(SCHEMA_BASE.child(b'endpoints.yml').getContent()),
}
# The default size of a created volume:
DEFAULT_SIZE = int(GiB(100).to_Byte().value)
def _endpoint(name):
"""
Decorator factory for API endpoints, adding appropriate JSON in/out
encoding.
def _parseTest(self, xml):
path = FilePath(self.mktemp())
path.setContent(xml)
return tree.parseFileAndReport(path.path)
PRecord,
PClass,
CheckedPSet,
CheckedPVector,
CheckedPMap,
field,
)
from twisted.python.filepath import FilePath
from twisted.python.reflect import qual as fqpn
from .._persistence import ROOT_CLASS
from ... import __version__
from ...testtools import TestCase
PERSISTED_MODEL = FilePath(__file__).sibling(b"persisted_model.json")
def _precord_model(klass):
"""
Serialize a ``PRecord`` or ``PClass`` model to something
JSON-encodable.
:param klass: A ``PRecord`` or ``PClass`` subclass.
:return: Tuple of (model dictionary, further classes to process).
"""
further_classes = set()
if issubclass(klass, PRecord):
attr_name = "_precord_fields"
else:
attr_name = "_pclass_fields"
def test_runningLoreMultipleFiles(self):
tmp = self.makeTemp('lore_index_test.xhtml', 'lore_index_test2.xhtml')
templateFilename = sp('template.tpl')
inputFilename = os.path.join(tmp, 'lore_index_test.xhtml')
inputFilename2 = os.path.join(tmp, 'lore_index_test2.xhtml')
indexFilename = 'theIndexFile'
bookFilename = os.path.join(tmp, 'lore_test_book.book')
bf = open(bookFilename, 'w')
bf.write('Chapter(r"%s", None)\n' % inputFilename)
bf.write('Chapter(r"%s", None)\n' % inputFilename2)
bf.close()
options = lore.Options()
options.parseOptions([
'--null',
'--book=%s' % bookFilename, '--config',
'template=%s' % templateFilename,
'--index=%s' % indexFilename
])
result = lore.runGivenOptions(options)
self.assertEqual(None, result)
self.assertEqual(
# XXX This doesn't seem like a very good index file.
"""\
aahz: <a href="lore_index_test2.html#index03">link</a><br />
aahz2: <a href="lore_index_test2.html#index02">link</a><br />
language of programming: <a href="lore_index_test.html#index02">link</a>, <a href="lore_index_test2.html#index01">link</a><br />
programming language: <a href="lore_index_test.html#index01">link</a><br />
""",
file(FilePath(indexFilename + ".html").path).read())
self.assertXMLEqual(
"""\
<?xml version="1.0" ?><!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Transitional//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'><html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head><title>Twisted Documentation: The way of the program</title></head>
<body bgcolor="white">
<h1 class="title">The way of the program</h1>
<div class="content">
<span/>
<p>The first paragraph.</p>
<h2>The Python programming language<a name="auto0"/></h2>
<a name="index01"/>
<a name="index02"/>
<p>The second paragraph.</p>
</div>
<a href="theIndexFile.html">Index</a>
</body>
</html>""",
FilePath(tmp).child("lore_index_test.html").getContent())
self.assertXMLEqual(
"""\
<?xml version="1.0" ?><!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Transitional//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'><html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head><title>Twisted Documentation: The second page to index</title></head>
<body bgcolor="white">
<h1 class="title">The second page to index</h1>
<div class="content">
<span/>
<p>The first paragraph of the second page.</p>
<h2>The Jython programming language<a name="auto0"/></h2>
<a name="index01"/>
<a name="index02"/>
<a name="index03"/>
<p>The second paragraph of the second page.</p>
</div>
<a href="theIndexFile.html">Index</a>
</body>
</html>""",
FilePath(tmp).child("lore_index_test2.html").getContent())
# sphinx-quickstart on Mon Apr 28 14:54:33 2014.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
from twisted.python.filepath import FilePath
import sys
import os
sys.path.insert(0, FilePath(__file__).parent().parent().path)
# Check if we are building on readthedocs
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.extlinks',
'sphinx.ext.ifconfig',
'sphinx.ext.intersphinx',
def __init__(self, filename):
self.filepath = FilePath(filename)
def test_introducer(self):
"""
The introducer furl is stable across restarts.
"""
basedir = self.workdir("test_introducer")
c1 = os.path.join(basedir, "c1")
tahoe = CLINodeAPI(reactor, FilePath(c1))
self.addCleanup(tahoe.stop_and_wait)
out, err, rc_or_sig = yield self.run_bintahoe([
"--quiet",
"create-introducer",
"--basedir", c1,
"--hostname", "127.0.0.1",
])
self.assertEqual(rc_or_sig, 0)
# This makes sure that node.url is written, which allows us to
# detect when the introducer restarts in _node_has_restarted below.
config = fileutil.read(tahoe.config_file.path)
self.assertIn('{}web.port = {}'.format(linesep, linesep), config)
fileutil.write(
tahoe.config_file.path,
config.replace(
'{}web.port = {}'.format(linesep, linesep),
'{}web.port = 0{}'.format(linesep, linesep),
)
)
p = Expect()
tahoe.run(on_stdout(p))
yield p.expect("introducer running")
tahoe.active()
yield self.poll(tahoe.introducer_furl_file.exists)
# read the introducer.furl file so we can check that the contents
# don't change on restart
furl = fileutil.read(tahoe.introducer_furl_file.path)
tahoe.active()
# We don't keep track of PIDs in files on Windows.
if not platform.isWindows():
self.assertTrue(tahoe.twistd_pid_file.exists())
self.assertTrue(tahoe.node_url_file.exists())
# rm this so we can detect when the second incarnation is ready
tahoe.node_url_file.remove()
yield tahoe.stop_and_wait()
p = Expect()
tahoe.run(on_stdout(p))
yield p.expect("introducer running")
# Again, the second incarnation of the node might not be ready yet, so
# poll until it is. This time introducer_furl_file already exists, so
# we check for the existence of node_url_file instead.
yield self.poll(tahoe.node_url_file.exists)
# The point of this test! After starting the second time the
# introducer furl file must exist and contain the same contents as it
# did before.
self.assertTrue(tahoe.introducer_furl_file.exists())
self.assertEqual(furl, fileutil.read(tahoe.introducer_furl_file.path))
def exists(self):
"""
Returns true if the underlying archive exists.
"""
return FilePath(self.zipfile.filename).exists()
def test_client(self):
"""
Test too many things.
0) Verify that "tahoe create-node" takes a --webport option and writes
the value to the configuration file.
1) Verify that "tahoe run" writes a pid file and a node url file (on POSIX).
2) Verify that the storage furl file has a stable value across a
"tahoe run" / stop / "tahoe run" sequence.
3) Verify that the pid file is removed after SIGTERM (on POSIX).
"""
basedir = self.workdir("test_client")
c1 = os.path.join(basedir, "c1")
tahoe = CLINodeAPI(reactor, FilePath(c1))
# Set this up right now so we don't forget later.
self.addCleanup(tahoe.cleanup)
out, err, rc_or_sig = yield self.run_bintahoe([
"--quiet", "create-node", "--basedir", c1,
"--webport", "0",
"--hostname", "localhost",
])
self.failUnlessEqual(rc_or_sig, 0)
# Check that the --webport option worked.
config = fileutil.read(tahoe.config_file.path)
self.assertIn(
'{}web.port = 0{}'.format(linesep, linesep),
config,
)
# After this it's safe to start the node
tahoe.active()
p = Expect()
# This will run until we stop it.
tahoe.run(on_stdout(p))
# Wait for startup to have proceeded to a reasonable point.
yield p.expect("client running")
tahoe.active()
# read the storage.furl file so we can check that its contents don't
# change on restart
storage_furl = fileutil.read(tahoe.storage_furl_file.path)
# We don't keep track of PIDs in files on Windows.
if not platform.isWindows():
self.assertTrue(tahoe.twistd_pid_file.exists())
# rm this so we can detect when the second incarnation is ready
tahoe.node_url_file.remove()
yield tahoe.stop_and_wait()
p = Expect()
# We don't have to add another cleanup for this one, the one from
# above is still registered.
tahoe.run(on_stdout(p))
yield p.expect("client running")
tahoe.active()
self.assertEqual(
storage_furl,
fileutil.read(tahoe.storage_furl_file.path),
)
if not platform.isWindows():
self.assertTrue(
tahoe.twistd_pid_file.exists(),
"PID file ({}) didn't exist when we expected it to. "
"These exist: {}".format(
tahoe.twistd_pid_file,
tahoe.twistd_pid_file.parent().listdir(),
),
)
yield tahoe.stop_and_wait()
if not platform.isWindows():
# twistd.pid should be gone by now.
self.assertFalse(tahoe.twistd_pid_file.exists())
def getModificationTime(self):
"""
Return the archive file's modification time.
"""
return FilePath(self.zipfile.filename).getModificationTime()
def test_create_snapshot(self, author, folder_name, path_in_folder,
some_content):
"""
A **POST** to **/v1/magic-folder/:folder-name/snapshot** with a **path** query argument
creates a new local snapshot for the file at the given path in the
named folder.
"""
local_path = FilePath(self.mktemp())
local_path.makedirs()
some_file = local_path.preauthChild(path_in_folder).asBytesMode(
"utf-8")
some_file.parent().makedirs(ignoreExistingDirectory=True)
some_file.setContent(some_content)
treq = treq_for_folders(
Clock(),
FilePath(self.mktemp()),
AUTH_TOKEN,
{
folder_name:
magic_folder_config(author, FilePath(self.mktemp()),
local_path)
},
# Unlike test_wait_for_completion above we start the folder
# services. This will allow the local snapshot to be created and
# our request to receive a response.
start_folder_services=True,
)
self.assertThat(
authorized_request(
treq,
AUTH_TOKEN,
b"POST",
self.url.child(folder_name,
"snapshot").set(u"path", path_in_folder),
),
succeeded(matches_response(code_matcher=Equals(CREATED), ), ),
)
self.assertThat(
authorized_request(
treq, AUTH_TOKEN, b"GET",
DecodedURL.from_text(u"http://example.invalid./v1/snapshot")),
succeeded(
matches_response(
code_matcher=Equals(OK),
headers_matcher=header_contains({
u"Content-Type":
Equals([u"application/json"]),
}),
body_matcher=AfterPreprocessing(
loads,
MatchesDict({
folder_name:
MatchesDict({
path_in_folder:
MatchesListwise([
MatchesDict({
u"type":
Equals(u"local"),
u"identifier":
is_hex_uuid(),
# XXX It would be nice to see some
# parents if there are any.
u"parents":
Equals([]),
u"content-path":
AfterPreprocessing(
lambda path: FilePath(path).
getContent(),
Equals(some_content),
),
u"author":
Equals(author.to_remote_author().
to_json()),
}),
]),
}),
}),
),
), ),
)
