def test_issue_concurrently(self, server_name):
"""
Invoking ``issue_cert`` multiple times concurrently for the same name
will not start multiple issuing processes, only wait for the first
process to complete.
"""
with AcmeFixture() as fixture:
fixture.service.startService()
self.assertThat(fixture.cert_store.as_dict(),
succeeded(Not(Contains(server_name))))
fixture.controller.pause()
d1 = fixture.service.issue_cert(server_name)
self.assertThat(d1, has_no_result())
d2 = fixture.service.issue_cert(server_name)
self.assertThat(d2, has_no_result())
self.assertThat(fixture.controller.count(), Equals(1))
fixture.controller.resume()
self.assertThat(d1, succeeded(Always()))
self.assertThat(d2, succeeded(Always()))
self.assertThat(
fixture.cert_store.as_dict(),
succeeded(MatchesDict({server_name: Not(Equals([]))})))
def test_delete_snapshot(self, filename, content):
"""
Create a snapshot and then a deletion snapshot of it.
"""
foo = self.magic.child(filename)
foo.setContent(content)
# make sure the store_local_snapshot() succeeds
self.assertThat(
self.snapshot_creator.store_local_snapshot(foo),
succeeded(Always()),
)
# delete the file
foo.remove()
# store a new snapshot
self.assertThat(
self.snapshot_creator.store_local_snapshot(foo),
succeeded(Always()),
)
stored_snapshot2 = self.db.get_local_snapshot(filename)
self.assertThat(
stored_snapshot2.is_delete(),
Equals(True),
)
def matches_response(code_matcher=Always(),
headers_matcher=Always(),
body_matcher=Always()):
"""
Match a Treq response object with certain code and body.
:param Matcher code_matcher: A matcher to apply to the response code.
:param Matcher headers_matcher: A matcher to apply to the response headers
(a ``twisted.web.http_headers.Headers`` instance).
:param Matcher body_matcher: A matcher to apply to the response body.
:return: A matcher.
"""
return MatchesAll(
MatchesStructure(
code=code_matcher,
headers=headers_matcher,
),
AfterPreprocessing(
lambda response: content(response),
succeeded(body_matcher),
),
)
def matches_response(code_matcher=Always(),
headers_matcher=Always(),
body_matcher=Always()):
"""
Match a Treq response object with certain code and body.
:param Matcher code_matcher: A matcher to apply to the response code.
:param Matcher headers_matcher: A matcher to apply to the response headers
(a ``twisted.web.http_headers.Headers`` instance).
:param Matcher body_matcher: A matcher to apply to the response body.
:return: A matcher.
"""
matchers = [
MatchesStructure(
code=code_matcher,
headers=headers_matcher,
),
]
# see comment in test_web.MagicFolderTests.test_method_not_allowed
# which is one user that wants nothing to try and read the content
# in some cases..
if body_matcher is not None:
matchers.append(
AfterPreprocessing(
lambda response: content(response),
succeeded(body_matcher),
))
return MatchesAll(*matchers)
def test_start_responding(self, token, subdomain, zone_name):
"""
Calling ``start_responding`` causes an appropriate TXT record to be
created.
"""
challenge = self._challenge_factory(token=token)
response = challenge.response(RSA_KEY_512)
responder = self._responder_factory(zone_name=zone_name)
server_name = u'{}.{}'.format(subdomain, zone_name)
zone = responder._driver.list_zones()[0]
self.assertThat(zone.list_records(), HasLength(0))
d = responder.start_responding(server_name, challenge, response)
self._perform()
self.assertThat(d, succeeded(Always()))
self.assertThat(
zone.list_records(),
MatchesListwise([
MatchesStructure(
name=EndsWith(u'.' + subdomain),
type=Equals('TXT'),
)
]))
# Starting twice before stopping doesn't break things
d = responder.start_responding(server_name, challenge, response)
self._perform()
self.assertThat(d, succeeded(Always()))
self.assertThat(zone.list_records(), HasLength(1))
d = responder.stop_responding(server_name, challenge, response)
self._perform()
self.assertThat(d, succeeded(Always()))
self.assertThat(zone.list_records(), HasLength(0))
def test_create_local_snapshots(self, content1, content2, filename):
"""
Create a local snapshot and then change the content of the file
to make another snapshot.
"""
data1 = io.BytesIO(content1)
parents = []
d = create_snapshot(
name=filename,
author=self.alice,
data_producer=data1,
snapshot_stash_dir=self.stash_dir,
)
d.addCallback(parents.append)
self.assertThat(
d,
succeeded(Always()),
)
data2 = io.BytesIO(content2)
d = create_snapshot(
name=filename,
author=self.alice,
data_producer=data2,
snapshot_stash_dir=self.stash_dir,
parents=parents,
)
d.addCallback(parents.append)
self.assertThat(
d,
succeeded(Always()),
)
def test_add_multiple_files(self, filenames, data):
"""
Add a bunch of files one by one and check whether the operation is
successful.
"""
files = []
for filename in filenames:
to_add = self.magic_path.child(filename)
content = data.draw(binary())
to_add.asBytesMode("utf-8").setContent(content)
files.append(to_add)
self.snapshot_service.startService()
list_d = []
for file in files:
result_d = self.snapshot_service.add_file(file)
list_d.append(result_d)
d = defer.gatherResults(list_d)
self.assertThat(
d,
succeeded(Always()),
)
self.assertThat(self.snapshot_service.stopService(),
succeeded(Always()))
self.assertThat(sorted(self.snapshot_creator.processed),
Equals(sorted(files)))
def test_create_snapshot_twice(self, filename, content1, content2):
"""
If a snapshot already exists for a file, adding a new snapshot to it
should refer to the existing snapshot as a parent.
"""
foo = self.magic.child(filename)
foo.asBytesMode("utf-8").setContent(content1)
# make sure the store_local_snapshot() succeeds
self.assertThat(
self.snapshot_creator.store_local_snapshot(foo),
succeeded(Always()),
)
foo_magicname = path2magic(filename)
stored_snapshot1 = self.db.get_local_snapshot(foo_magicname)
# now modify the file with some new content.
foo.asBytesMode("utf-8").setContent(content2)
# make sure the second call succeeds as well
self.assertThat(
self.snapshot_creator.store_local_snapshot(foo),
succeeded(Always()),
)
stored_snapshot2 = self.db.get_local_snapshot(foo_magicname)
self.assertThat(
stored_snapshot2.parents_local[0],
MatchesStructure(
content_path=Equals(stored_snapshot1.content_path)))
class TestAlwaysInterface(TestMatchersInterface, TestCase):
""":py:func:`~testtools.matchers.Always` always matches."""
matches_matcher = Always()
matches_matches = [42, object(), 'hi mom']
matches_mismatches = []
str_examples = [('Always()', Always())]
describe_examples = []
def test_write_snapshot_to_tahoe_fails(self, name, contents):
"""
If any part of a snapshot upload fails then the metadata for that snapshot
is retained in the local database and the snapshot content is retained
in the stash.
"""
broken_root = ErrorPage(500, "It's broken.", "It's broken.")
f = self.useFixture(
RemoteSnapshotCreatorFixture(
temp=FilePath(self.mktemp()),
author=self.author,
root=broken_root,
upload_dircap="URI:DIR2:foo:bar",
))
config = f.config
remote_snapshot_creator = f.remote_snapshot_creator
snapshots = []
parents = []
for content in contents:
data = io.BytesIO(content)
d = create_snapshot(
name=name,
author=self.author,
data_producer=data,
snapshot_stash_dir=config.stash_path,
parents=parents,
)
d.addCallback(snapshots.append)
self.assertThat(
d,
succeeded(Always()),
)
parents = [snapshots[-1]]
local_snapshot = snapshots[-1]
config.store_local_snapshot(snapshots[-1])
d = remote_snapshot_creator.upload_local_snapshots()
self.assertThat(
d,
succeeded(Always()),
)
self.eliot_logger.flushTracebacks(TahoeAPIError)
self.assertEqual(
local_snapshot,
config.get_local_snapshot(name),
)
self.assertThat(
local_snapshot.content_path.getContent(),
Equals(content),
)
def test_get_welcome(self):
"""
We can retrieve the welcome page
"""
self.setup_example()
self.assertThat(
self.tahoe_client.get_welcome(),
succeeded(
ContainsDict({
"introducers": Always(),
"servers": Always(),
})))
def test_snapshot_bad_metadata(self, raw_metadata):
"""
Test error-handling cases when de-serializing a snapshot. If the
snapshot version is missing or wrong we should error.
"""
# arbitrary (but valid) content-cap
contents = []
content_cap_d = self.tahoe_client.create_immutable(b"0" * 256)
content_cap_d.addCallback(contents.append)
self.assertThat(content_cap_d, succeeded(Always()))
content_cap = contents[0]
# invalid metadata cap (we use Hypothesis to give us two
# definitely-invalid versions)
metadata_caps = []
d = self.tahoe_client.create_immutable(
json.dumps(raw_metadata).encode("utf8"))
d.addCallback(metadata_caps.append)
self.assertThat(d, succeeded(Always()))
# create a Snapshot using the wrong metadata
raw_snapshot_data = {
u"content":
format_filenode(content_cap),
u"metadata":
format_filenode(
metadata_caps[0],
{
u"magic_folder": {
u"author_signature": u"not valid",
},
},
),
}
snapshot_cap = []
d = self.tahoe_client.create_immutable_directory(raw_snapshot_data)
d.addCallback(snapshot_cap.append)
self.assertThat(d, succeeded(Always()))
# now when we read back the snapshot with incorrect metadata,
# it should fail
snapshot_d = create_snapshot_from_capability(snapshot_cap[0],
self.tahoe_client)
self.assertThat(
snapshot_d,
failed(
MatchesStructure(value=AfterPreprocessing(
str, Contains("snapshot_version")), )))
def test_snapshot_roundtrip(self, content, filename):
"""
Create a local snapshot, write into tahoe to create a remote snapshot,
then read back the data from the snapshot cap to recreate the remote
snapshot and check if it is the same as the previous one.
"""
data = io.BytesIO(content)
snapshots = []
# create LocalSnapshot
d = create_snapshot(
name=filename,
author=self.alice,
data_producer=data,
snapshot_stash_dir=self.stash_dir,
parents=[],
)
d.addCallback(snapshots.append)
self.assertThat(
d,
succeeded(Always()),
)
# create remote snapshot
d = write_snapshot_to_tahoe(snapshots[0], self.alice,
self.tahoe_client)
d.addCallback(snapshots.append)
self.assertThat(
d,
succeeded(Always()),
)
# snapshots[1] is a RemoteSnapshot
note("remote snapshot: {}".format(snapshots[1]))
# now, recreate remote snapshot from the cap string and compare with the original.
# Check whether information is preserved across these changes.
snapshot_d = create_snapshot_from_capability(snapshots[1].capability,
self.tahoe_client)
snapshot_d.addCallback(snapshots.append)
self.assertThat(snapshot_d, succeeded(Always()))
snapshot = snapshots[-1]
self.assertThat(snapshot, MatchesStructure(name=Equals(filename)))
content_io = io.BytesIO()
self.assertThat(
snapshot.fetch_content(self.tahoe_client, content_io),
succeeded(Always()),
)
self.assertEqual(content_io.getvalue(), content)
def matches_version_dictionary():
"""
Match the dictionary returned by Tahoe-LAFS'
``RIStorageServer.get_version`` which is also the dictionary returned by
our own ``RIPrivacyPassAuthorizedStorageServer.get_version``.
"""
return ContainsDict({
# It has these two top-level keys, at least. Try not to be too
# fragile by asserting much more than that they are present.
b'application-version':
Always(),
b'http://allmydata.org/tahoe/protocols/storage/v1':
Always(),
})
def test_get(self, voucher, num_passes, now):
"""
``IPassFactory.get`` returns an ``IPassGroup`` provider containing the
requested number of passes.
"""
configless = self.useFixture(
ConfiglessMemoryVoucherStore(
DummyRedeemer(),
lambda: now,
), )
# Make sure there are enough tokens for us to extract!
self.assertThat(
configless.redeem(voucher, num_passes),
succeeded(Always()),
)
pass_factory = SpendingController.for_store(
tokens_to_passes=configless.redeemer.tokens_to_passes,
store=configless.store,
)
group = pass_factory.get(u"message", num_passes)
self.assertThat(
group,
MatchesAll(
Provides([IPassGroup]),
MatchesStructure(passes=HasLength(num_passes), ),
),
)
def check_error(reason):
self.assertThat(reason, IsInstance(Failure))
reason.trap(KubernetesError)
self.assertThat(
reason.value,
MatchesStructure(
code=Equals(NOT_FOUND),
status=MatchesStructure(
metadata=Equals(self.model.v1.ListMeta()),
status=Equals(u"Failure"),
# Error text is wonky and changes from version to
# version. It's not a reliable source of
# information so don't bother making any
# particular assertions about its contents.
message=Always(),
reason=Equals(u"NotFound"),
details=MatchesStructure.byEquality(
kind=obj.kind.lower() + u"s",
name=obj.metadata.name,
group=group,
),
code=Equals(NOT_FOUND),
),
),
)
def test_advise_corrupt_share(self, storage_index, renew_secret,
cancel_secret, sharenum, size):
"""
An advisory of corruption in a share can be sent to the server.
"""
# Hypothesis causes our storage server to be used many times. Clean
# up between iterations.
cleanup_storage_server(self.anonymous_storage_server)
# Create a share we can toy with.
write_toy_shares(
self.anonymous_storage_server,
storage_index,
renew_secret,
cancel_secret,
{sharenum},
size,
canary=self.canary,
)
self.assertThat(
self.client.advise_corrupt_share(
b"immutable",
storage_index,
sharenum,
b"the bits look bad",
),
succeeded(Always()),
)
self.assertThat(
FilePath(self.anonymous_storage_server.corruption_advisory_dir).
children(),
HasLength(1),
)
def test_add_lease(self, storage_index, renew_secrets, cancel_secret,
sharenums, size):
"""
A lease can be added to an existing immutable share.
"""
# Hypothesis causes our storage server to be used many times. Clean
# up between iterations.
cleanup_storage_server(self.anonymous_storage_server)
# Use a different secret so that it's a new lease and not an
# implicit renewal.
add_lease_secret, renew_lease_secret = renew_secrets
assume(add_lease_secret != renew_lease_secret)
# Create a share we can toy with.
write_toy_shares(
self.anonymous_storage_server,
storage_index,
add_lease_secret,
cancel_secret,
sharenums,
size,
canary=self.canary,
)
self.assertThat(
self.client.add_lease(
storage_index,
renew_lease_secret,
cancel_secret,
),
succeeded(Always()),
)
leases = list(self.anonymous_storage_server.get_leases(storage_index))
self.assertThat(leases, HasLength(2))
def test_unauthorized(self, get_config, path):
"""
A request for any resource without the required authorization token
receives a 401 response.
"""
tempdir = self.useFixture(TempDir())
config = get_config(tempdir.join(b"tahoe"), b"tub.port")
root = root_from_config(config, datetime.now)
agent = RequestTraversalAgent(root)
requesting = agent.request(
b"GET",
b"http://127.0.0.1/" + b"/".join(path),
)
responses = []
requesting.addCallback(responses.append)
self.assertThat(
requesting,
succeeded(Always()),
)
[response] = responses
self.assertThat(
response.code,
Equals(UNAUTHORIZED),
)
def test_create_and_download(self, content):
"""
Upload some content (via 'PUT /uri') and then download it (via
'GET /uri?uri=...')
"""
http_client = create_tahoe_treq_client()
@inlineCallbacks
def do_test():
resp = yield http_client.put("http://example.com/uri", content)
self.assertThat(resp.code, Equals(201))
cap_raw = yield resp.content()
cap = from_string(cap_raw)
self.assertThat(cap, IsInstance(CHKFileURI))
resp = yield http_client.get(
"http://example.com/uri?uri={}".format(cap.to_string()))
self.assertThat(resp.code, Equals(200))
round_trip_content = yield resp.content()
# using the form "/uri/<cap>" is also valid
resp = yield http_client.get("http://example.com/uri/{}".format(
cap.to_string()))
self.assertEqual(resp.code, 200)
round_trip_content = yield resp.content()
self.assertEqual(content, round_trip_content)
self.assertThat(
do_test(),
succeeded(Always()),
)
def test_timer_errors(self):
"""
If the timed check fails (for example, because registration fails), the
error should be caught and logged.
"""
with AcmeFixture(client=FailingClient()) as fixture:
# Registration is triggered with service starts.
fixture.service.startService()
latest_logs = flush_logged_errors()
self.assertThat(latest_logs, HasLength(1))
self.assertThat(
str(latest_logs[0]), Contains('Failing at "register".'))
# Forcing a check will trigger again the registration.
self.assertThat(
fixture.service._check_certs(),
succeeded(Always()))
latest_logs = flush_logged_errors()
self.assertThat(latest_logs, HasLength(1))
self.assertThat(
str(latest_logs[0]), Contains('Failing at "register".'))
# Manually stop the service to not stop it from the fixture
# and trigger another failure.
self.assertThat(
fixture.service.stopService(),
failed(AfterPreprocessing(
lambda f: f.value.args[0], Equals('Failing at "stop".'))))
latest_logs = flush_logged_errors()
def test_scan_existing_to_directory(self, relpath):
"""
When we scan a path that we have a snapshot for, but it is now a directory,
we ignore it, and report an error.
"""
local = self.magic_path.preauthChild("existing-file")
local.makedirs()
stash_dir = FilePath(self.mktemp())
stash_dir.makedirs()
self.config.store_currentsnapshot_state("existing-file",
OLD_PATH_STATE)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files, Equals([]))
self.assertThat(
self.status_service._folders["default"]["errors"],
MatchesListwise([
MatchesStructure(summary=MatchesRegex(
"File .* was a file, and now is a directory.")),
]),
)
def test_duplicate_upload(self, content):
"""
Upload the same content (via 'PUT /uri') twice
"""
http_client = create_tahoe_treq_client()
@inlineCallbacks
def do_test():
resp = yield http_client.put("http://example.com/uri", content)
self.assertEqual(resp.code, 201)
cap_raw = yield resp.content()
self.assertThat(
cap_raw,
AfterPreprocessing(
from_string,
IsInstance(CHKFileURI)
)
)
resp = yield http_client.put("http://example.com/uri", content)
self.assertThat(resp.code, Equals(200))
self.assertThat(
do_test(),
succeeded(Always()),
)
def test_stop_redeeming_on_error(self, get_config, now, voucher, counter,
extra_tokens):
"""
If an error is encountered on one of the redemption attempts performed by
``IRedeemer.redeem``, the effort is suspended until the normal retry
logic activates.
"""
num_redemption_groups = counter + 1
num_tokens = num_redemption_groups + extra_tokens
redeemer = RecordingRedeemer(UnpaidRedeemer())
store = self.useFixture(TemporaryVoucherStore(get_config,
lambda: now)).store
controller = PaymentController(
store,
redeemer,
default_token_count=num_tokens,
num_redemption_groups=num_redemption_groups,
clock=Clock(),
)
self.assertThat(
controller.redeem(voucher),
succeeded(Always()),
)
self.assertThat(
redeemer.redemptions,
AfterPreprocessing(
len,
Equals(1),
),
)
def test_create_snapshots(self, filenames, data_strategy):
"""
Create a list of filenames and random content as input and for each
of the (filename, content) mapping, create and store the snapshot in
the database.
"""
files = []
for filename in filenames:
file = self.magic.child(filename)
content = data_strategy.draw(binary())
file.setContent(content)
files.append((file, filename, content))
for (file, filename, _unused) in files:
self.assertThat(self.snapshot_creator.store_local_snapshot(file),
succeeded(Always()))
self.assertThat(self.db.get_all_localsnapshot_paths(),
HasLength(len(files)))
for (file, filename, content) in files:
stored_snapshot = self.db.get_local_snapshot(filename)
stored_content = stored_snapshot.content_path.getContent()
path_state = self.db.get_currentsnapshot_pathstate(filename)
self.assertThat(stored_content, Equals(content))
self.assertThat(stored_snapshot.parents_local, HasLength(0))
self.assertThat(
path_state,
MatchesStructure(
size=Equals(len(content)),
mtime_ns=Equals(seconds_to_ns(file.getModificationTime())),
))
def test_create_snapshots(self, filenames, data_strategy):
"""
Create a list of filenames and random content as input and for each
of the (filename, content) mapping, create and store the snapshot in
the database.
"""
files = []
for filename in filenames:
file = self.magic.child(filename)
content = data_strategy.draw(binary())
file.asBytesMode("utf-8").setContent(content)
files.append((file, filename, content))
for (file, filename, _unused) in files:
self.assertThat(self.snapshot_creator.store_local_snapshot(file),
succeeded(Always()))
self.assertThat(self.db.get_all_localsnapshot_paths(),
HasLength(len(files)))
for (file, filename, content) in files:
mangled_filename = path2magic(filename)
stored_snapshot = self.db.get_local_snapshot(mangled_filename)
stored_content = stored_snapshot.content_path.getContent()
self.assertThat(stored_content, Equals(content))
self.assertThat(stored_snapshot.parents_local, HasLength(0))
def test_should_not_redeem(self, get_config, now, voucher, public_key):
"""
``PaymentController.redeem`` raises ``ValueError`` if passed a voucher in
a state when redemption should not be started.
"""
store = self.useFixture(TemporaryVoucherStore(get_config,
lambda: now)).store
controller = PaymentController(
store,
DummyRedeemer(public_key),
default_token_count=100,
clock=Clock(),
)
self.assertThat(
controller.redeem(voucher),
succeeded(Always()),
)
# Sanity check. It should be redeemed now.
voucher_obj = controller.get_voucher(voucher)
self.assertThat(
voucher_obj.state.should_start_redemption(),
Equals(False),
)
self.assertThat(
controller.redeem(voucher),
failed(AfterPreprocessing(
lambda f: f.type,
Equals(ValueError),
), ),
)
def test_scan_nothing(self, relpath):
"""
An existing, non-updated file is not scanned
"""
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
snap = RemoteSnapshot(
relpath,
self.author,
metadata={
"modification_time": int(local.getModificationTime()),
},
capability=random_immutable(directory=True),
parents_raw=[],
content_cap=random_immutable(),
metadata_cap=random_immutable(),
)
self.config.store_downloaded_snapshot(relpath, snap,
get_pathinfo(local).state)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files, Equals([]))
def test_redeemed_after_redeeming(self, get_config, public_key, now,
voucher):
"""
A ``Voucher`` is marked as redeemed after ``IRedeemer.redeem`` succeeds.
"""
store = self.useFixture(TemporaryVoucherStore(get_config,
lambda: now)).store
controller = PaymentController(
store,
DummyRedeemer(public_key),
default_token_count=100,
clock=Clock(),
)
self.assertThat(
controller.redeem(voucher),
succeeded(Always()),
)
persisted_voucher = store.get(voucher)
self.assertThat(
persisted_voucher.state,
Equals(
model_Redeemed(
finished=now,
token_count=100,
public_key=public_key,
)),
)
def test_scan_something_remote(self, relpath):
"""
We scan an update to a file we already know about.
"""
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
snap = RemoteSnapshot(
relpath,
self.author,
metadata={
# this remote is 2min older than our local file
"modification_time": int(local.getModificationTime()) - 120,
},
capability=random_immutable(directory=True),
parents_raw=[],
content_cap=random_immutable(),
metadata_cap=random_immutable(),
)
self.config.store_downloaded_snapshot(
relpath,
snap,
OLD_PATH_STATE,
)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files, Equals([local]))
