def nettest_to_path(path, allow_arbitrary_paths=False):
"""
Takes as input either a path or a nettest name.
The nettest name may either be prefixed by the category of the nettest (
blocking, experimental, manipulation or third_party) or not.
Args:
allow_arbitrary_paths:
allow also paths that are not relative to the nettest_directory.
Returns:
full path to the nettest file.
"""
if allow_arbitrary_paths and os.path.exists(path):
return path
test_name = path.rsplit("/", 1)[-1]
test_categories = ["blocking", "experimental", "manipulation", "third_party"]
nettest_dir = FilePath(config.nettest_directory)
found_path = None
for category in test_categories:
p = nettest_dir.preauthChild(os.path.join(category, test_name) + ".py")
if p.exists():
if found_path is not None:
raise Exception("Found two tests named %s" % test_name)
found_path = p.path
if not found_path:
raise e.NetTestNotFound(path)
return found_path
def main(argv):
# input
posts = FilePath(argv[1])
# output
blog = FilePath(argv[2])
# Since Sphinx gets confused by image paths with "special" characters in
# them, generate new names for all the image paths and a mapping from the
# old names to the new names.
images = FilePath(argv[3])
imagepaths = []
for post in images.children():
if post.isdir():
imagepaths.append(post)
safe = post.sibling(fixpath(post.basename()))
if post != safe and not safe.isdir():
post.moveTo(safe)
safe.linkTo(post)
entries = []
for post in posts.children():
data = post.getContent().decode("utf-8")
ignored, header, body = data.split(b"---", 2)
meta = dict((text.strip() for text in line.split(":", 1)) for line in header.splitlines() if line.strip())
date = datetime.strptime(meta["date"], "%Y/%m/%d %H:%M:%S")
parent = blog.preauthChild(
("%d/%02d/%02d" % (date.year, date.month, date.day)).encode("utf-8"))
title = fixpath(meta["title"].strip().lower().encode("utf-8")).decode("utf-8")
entry = parent.child((title + ".rst").encode("utf-8"))
header = HEADER_TEMPLATE % dict(
author=meta["author"].strip(), categories="none",
tags=meta["categories"].strip(), title=meta["title"].strip(),
underbar="=" * len(meta["title"].strip()))
for path in imagepaths:
body = body.replace(
u"/" + path.basename().decode("utf-8") + u"/",
u"/" + fixpath(path.basename()).decode("utf-8") + u"/")
if not parent.isdir():
parent.makedirs()
entry.setContent((header + html2rst(body)).encode("utf-8"))
entries.append(entry)
entries.sort()
entries.reverse()
sitemap = SITEMAP % dict(
entries="".join([
"\n " + "/".join(entry.segmentsFrom(blog))
for entry in entries]))
blog.child(b"master.rst").setContent(sitemap.encode("utf-8"))
FilePath(b"conf.py").copyTo(blog.child(b"conf.py"))
def nettest_to_path(path, allow_arbitrary_paths=False):
"""
Takes as input either a path or a nettest name.
The nettest name may either be prefixed by the category of the nettest (
blocking, experimental, manipulation or third_party) or not.
Args:
allow_arbitrary_paths:
allow also paths that are not relative to the nettest_directory.
Returns:
full path to the nettest file.
"""
if allow_arbitrary_paths and os.path.exists(path):
return path
test_name = path.rsplit("/", 1)[-1]
test_categories = [
"blocking", "experimental", "manipulation", "third_party"
]
nettest_dir = FilePath(config.nettest_directory)
found_path = None
for category in test_categories:
p = nettest_dir.preauthChild(os.path.join(category, test_name) + '.py')
if p.exists():
if found_path is not None:
raise Exception("Found two tests named %s" % test_name)
found_path = p.path
if not found_path:
raise e.NetTestNotFound(path)
return found_path
def test_create_fails(self, author, folder_name, path_in_folder):
"""
If a local snapshot cannot be created, a **POST** to
**/v1/magic-folder/<folder-name>/snapshot** receives a response with an HTTP error
code.
"""
local_path = FilePath(self.mktemp())
local_path.makedirs()
# You may not create a snapshot of a directory.
not_a_file = local_path.preauthChild(path_in_folder).asBytesMode(
"utf-8")
not_a_file.makedirs(ignoreExistingDirectory=True)
treq = treq_for_folders(
object(),
FilePath(self.mktemp()),
AUTH_TOKEN,
{
folder_name:
magic_folder_config(author, FilePath(self.mktemp()),
local_path)
},
# This test carefully targets a failure mode that doesn't require
# the service to be running.
start_folder_services=False,
)
self.assertThat(
authorized_request(
treq,
AUTH_TOKEN,
b"POST",
self.url.child(folder_name,
"snapshot").set(u"path", path_in_folder),
),
succeeded(
matches_response(
# Maybe this could be BAD_REQUEST instead, sometimes, if
# the path argument was bogus somehow.
code_matcher=Equals(INTERNAL_SERVER_ERROR),
headers_matcher=header_contains({
u"Content-Type":
Equals([u"application/json"]),
}),
body_matcher=AfterPreprocessing(
loads,
ContainsDict({
u"reason": IsInstance(unicode),
}),
),
), ),
)
def create_fake_repository(self, files):
"""
Create files in a directory to mimic a repository of packages.
:param dict source_repo: Dictionary mapping names of files to create to
contents.
:return: FilePath of directory containing fake package files.
"""
source_repo = FilePath(tempfile.mkdtemp())
for key in files:
new_file = source_repo.preauthChild(key)
if not new_file.parent().exists():
new_file.parent().makedirs()
new_file.setContent(files[key])
return 'file://' + source_repo.path
def get(self):
try:
reload(recognize)
r = recognize.Recognizer()
top = FilePath('sounds')
path = top.preauthChild(self.get_argument('path'))
raw = load(path, 8000)
t1 = time.time()
out = r.recognize(raw, rate=8000)
self.write({
'result': out,
'ms': round(1000 * (time.time() - t1), 1)
})
except Exception:
traceback.print_exc()
raise
def test_create_local_snapshot(self, relative_target_path, content):
"""
``MagicFolder.local_snapshot_service`` can be used to create a new local
snapshot for a file in the folder.
"""
magic_path = FilePath(self.mktemp())
magic_path.asBytesMode("utf-8").makedirs()
target_path = magic_path.preauthChild(
relative_target_path).asBytesMode("utf-8")
target_path.asBytesMode("utf-8").parent().makedirs(
ignoreExistingDirectory=True)
target_path.setContent(content)
local_snapshot_creator = MemorySnapshotCreator()
local_snapshot_service = LocalSnapshotService(magic_path,
local_snapshot_creator)
clock = object()
tahoe_client = object()
name = u"local-snapshot-service-test"
config = object()
participants = object()
magic_folder = MagicFolder(
client=tahoe_client,
config=config,
name=name,
local_snapshot_service=local_snapshot_service,
uploader_service=Service(),
initial_participants=participants,
clock=clock,
)
magic_folder.startService()
self.addCleanup(magic_folder.stopService)
adding = magic_folder.local_snapshot_service.add_file(target_path, )
self.assertThat(
adding,
succeeded(Always()),
)
self.assertThat(
local_snapshot_creator.processed,
Equals([target_path]),
)
def createSkeleton(siteDir):
fromDir = FilePath(warp.__file__).sibling("priv").child("skeleton")
for entryName in (l.strip() for l in fromDir.child("MANIFEST").open()):
entry = fromDir.preauthChild(entryName)
destination = siteDir.preauthChild(entryName)
if destination.exists():
continue
print " Copying %s" % entryName
if not destination.parent().exists():
os.makedirs(destination.parent().path)
entry.copyTo(destination)
print "Done! Run with 'twistd -n warp'"
def createSkeleton(siteDir):
fromDir = FilePath(warp.__file__).sibling("priv").child("skeleton")
for entryName in (l.strip() for l in fromDir.child("MANIFEST").open()):
entry = fromDir.preauthChild(entryName)
destination = siteDir.preauthChild(entryName)
if destination.exists():
continue
print " Copying %s" % entryName
if not destination.parent().exists():
destination.parent().makedirs()
entry.copyTo(destination)
print "Done! Run with 'twistd -n warp'"
def test_wait_for_completion(self, author, folder_name, path_in_folder,
some_content):
"""
A **POST** request to **/v1/magic-folder/:folder-name/snapshot** does not receive a
response before the snapshot has been created in the local database.
"""
local_path = FilePath(self.mktemp())
local_path.makedirs()
some_file = local_path.preauthChild(path_in_folder).asBytesMode(
"utf-8")
some_file.parent().makedirs(ignoreExistingDirectory=True)
some_file.setContent(some_content)
treq = treq_for_folders(
object(),
FilePath(self.mktemp()),
AUTH_TOKEN,
{
folder_name:
magic_folder_config(author, FilePath(self.mktemp()),
local_path)
},
# The interesting behavior of this test hinges on this flag. We
# decline to start the folder services here. Therefore, no local
# snapshots will ever be created. This lets us observe the
# request in a state where it is waiting to receive its response.
# This demonstrates that the response is not delivered before the
# local snapshot is created. See test_create_snapshot for the
# alternative case.
start_folder_services=False,
)
self.assertThat(
authorized_request(
treq,
AUTH_TOKEN,
b"POST",
self.url.child(folder_name,
"snapshot").set(u"path", path_in_folder),
),
has_no_result(),
)
def test_uploader_service(
self,
name,
file_path,
relative_magic_path,
relative_state_path,
author,
collective_cap_kind,
poll_interval,
content,
):
"""
``MagicFolder.from_config`` creates an ``UploaderService`` which will
upload snapshots using the given Tahoe client object.
"""
reactor = task.Clock()
root = create_fake_tahoe_root()
http_client = create_tahoe_treq_client(root)
tahoe_client = TahoeClient(
DecodedURL.from_text(U"http://example.invalid./"),
http_client,
)
ignored, upload_dircap = root.add_mutable_data(
b"URI:DIR2:",
json.dumps([
u"dirnode",
{
u"children": {}
},
]),
)
ignored, collective_dircap = root.add_mutable_data(
collective_cap_kind,
json.dumps([
u"dirnode",
{
u"children": {}
},
]),
)
basedir = FilePath(self.mktemp())
global_config = create_global_configuration(
basedir,
u"tcp:-1",
FilePath(u"/non-tahoe-directory"),
u"tcp:127.0.0.1:-1",
)
magic_path = basedir.preauthChild(relative_magic_path)
magic_path.asBytesMode("utf-8").makedirs()
statedir = basedir.child(u"state")
state_path = statedir.asTextMode("utf-8").preauthChild(
relative_state_path)
target_path = magic_path.asTextMode("utf-8").preauthChild(file_path)
target_path.asBytesMode("utf-8").parent().makedirs(
ignoreExistingDirectory=True)
target_path.asBytesMode("utf-8").setContent(content)
global_config.create_magic_folder(
name,
magic_path,
state_path,
author,
collective_dircap,
upload_dircap,
poll_interval,
)
magic_folder = MagicFolder.from_config(
reactor,
tahoe_client,
name,
global_config,
)
magic_folder.startService()
self.addCleanup(magic_folder.stopService)
self.assertThat(
magic_folder.uploader_service.running,
Equals(True),
)
self.assertThat(
magic_folder.uploader_service._remote_snapshot_creator.
_local_author,
Equals(author),
)
self.assertThat(
magic_folder.folder_name,
Equals(name),
)
# add a file. This won't actually add a file until we advance
# the clock.
d = magic_folder.local_snapshot_service.add_file(target_path, )
self.assertThat(
d,
succeeded(Always()),
)
def children():
return json.loads(root._uri.data[upload_dircap])[1][u"children"]
reactor.advance(poll_interval - 1)
self.assertThat(
children(),
Equals({}),
)
reactor.advance(1)
self.assertThat(
children(),
ContainsDict({path2magic(target_path.path): Always()}),
"Children dictionary {!r} did not contain expected path".format(
children, ),
)
class FindUpdatesTests(SyncTestCase):
"""
Tests for ``find_updated_files``
"""
def setup_example(self):
self.author = create_local_author("alice")
self.magic_path = FilePath(self.mktemp())
self.magic_path.makedirs()
self._global_config = create_testing_configuration(
FilePath(self.mktemp()),
FilePath("dummy"),
)
self.collective_cap = random_dircap()
self.personal_cap = random_dircap()
self.clock = Clock()
self.status_service = WebSocketStatusService(self.clock,
self._global_config)
self.folder_status = FolderStatus("default", self.status_service)
self.config = self._global_config.create_magic_folder(
"default",
self.magic_path,
self.author,
self.collective_cap,
self.personal_cap,
1,
None,
)
# Use a cooperator that does not cooperate.
self.cooperator = Cooperator(
terminationPredicateFactory=lambda: lambda: False,
scheduler=lambda f: f(),
)
self.addCleanup(self.cooperator.stop)
self.tahoe_client = create_tahoe_client(
DecodedURL.from_text("http://invalid./"),
create_tahoe_treq_client(create_fake_tahoe_root()),
)
@given(
relative_paths(), )
def test_scan_new(self, relpath):
"""
A completely new file is scanned
"""
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(
files,
Equals([local]),
)
@skipIf(runtime.platformType == "win32",
"windows does not have unprivileged symlinks")
@given(
relative_paths(), )
def test_scan_symlink(self, relpath):
"""
A completely new symlink is ignored
"""
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.linkTo(local)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(
files,
Equals([]),
)
@given(
relative_paths(), )
def test_scan_nothing(self, relpath):
"""
An existing, non-updated file is not scanned
"""
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
snap = RemoteSnapshot(
relpath,
self.author,
metadata={
"modification_time": int(local.getModificationTime()),
},
capability=random_immutable(directory=True),
parents_raw=[],
content_cap=random_immutable(),
metadata_cap=random_immutable(),
)
self.config.store_downloaded_snapshot(relpath, snap,
get_pathinfo(local).state)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files, Equals([]))
@given(
relative_paths(), )
def test_scan_something_remote(self, relpath):
"""
We scan an update to a file we already know about.
"""
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
snap = RemoteSnapshot(
relpath,
self.author,
metadata={
# this remote is 2min older than our local file
"modification_time": int(local.getModificationTime()) - 120,
},
capability=random_immutable(directory=True),
parents_raw=[],
content_cap=random_immutable(),
metadata_cap=random_immutable(),
)
self.config.store_downloaded_snapshot(
relpath,
snap,
OLD_PATH_STATE,
)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files, Equals([local]))
@given(
relative_paths(), )
def test_scan_something_local(self, relpath):
"""
We scan an update to a file we already know about (but only locally).
"""
local = self.magic_path.preauthChild("existing-file")
local.setContent(b"dummy\n")
stash_dir = FilePath(self.mktemp())
stash_dir.makedirs()
self.config.store_currentsnapshot_state("existing-file",
OLD_PATH_STATE)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files, Equals([local]))
@given(
relative_paths(), )
def test_scan_existing_to_directory(self, relpath):
"""
When we scan a path that we have a snapshot for, but it is now a directory,
we ignore it, and report an error.
"""
local = self.magic_path.preauthChild("existing-file")
local.makedirs()
stash_dir = FilePath(self.mktemp())
stash_dir.makedirs()
self.config.store_currentsnapshot_state("existing-file",
OLD_PATH_STATE)
files = []
self.assertThat(
find_updated_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files, Equals([]))
self.assertThat(
self.status_service._folders["default"]["errors"],
MatchesListwise([
MatchesStructure(summary=MatchesRegex(
"File .* was a file, and now is a directory.")),
]),
)
@given(
relative_paths(), )
def test_scan_delete(self, relpath):
"""
Scanning a relpath we know about with no local file is a delete
"""
self.config.store_currentsnapshot_state(relpath, OLD_PATH_STATE)
files = []
self.assertThat(
find_deleted_files(self.cooperator,
self.config,
files.append,
status=self.folder_status),
succeeded(Always()),
)
self.assertThat(files,
Equals([self.config.magic_path.preauthChild(relpath)]))
@given(
relative_paths(), )
def test_scan_once(self, relpath):
"""
The scanner service discovers a_file when a scan is requested.
"""
relpath = "a_file0"
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
files = []
class FakeLocalSnapshot(object):
pass
class SnapshotService(object):
def add_file(self, f, local_parent=None):
files.append(f)
return succeed(FakeLocalSnapshot())
class FakeUploader(object):
def upload_snapshot(self, snapshot):
return Deferred()
file_factory = MagicFileFactory(
self.config,
self.tahoe_client,
self.folder_status,
SnapshotService(),
FakeUploader(),
object(), # write_participant,
object(), # remote_cache,
object(), # filesystem,
synchronous=True,
)
service = ScannerService(
self.config,
file_factory,
object(),
cooperator=self.cooperator,
scan_interval=None,
clock=self.clock,
)
service.startService()
self.addCleanup(service.stopService)
self.assertThat(
service.scan_once(),
succeeded(Always()),
)
self.assertThat(files, Equals([local]))
@given(
relative_paths(), )
def test_scan_periodic(self, relpath):
"""
The scanner service iteself discovers a_file, when a scan interval is
set.
"""
relpath = "a_file1"
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
files = []
class FakeLocalSnapshot(object):
pass
class SnapshotService(object):
def add_file(self, f, local_parent=None):
files.append(f)
return succeed(FakeLocalSnapshot())
class FakeRemoteSnapshot(object):
pass
class FakeUploader(object):
def upload_snapshot(self, snapshot):
return Deferred()
file_factory = MagicFileFactory(
self.config,
self.tahoe_client,
self.folder_status,
SnapshotService(),
FakeUploader(),
object(), # write_participant,
object(), # remote_cache,
object(), # filesystem,
synchronous=True,
)
service = ScannerService(
self.config,
file_factory,
object(),
cooperator=self.cooperator,
scan_interval=1,
clock=self.clock,
)
service.startService()
self.addCleanup(service.stopService)
self.assertThat(files, Equals([local]))
# XXX should use generated author-names, but get encoding errors
# from FilePath on u'0.conflict-\x00' -- that is, a single nul as
# an author-name
@given(
relative_paths(), )
def test_scan_conflict_files(self, relpath):
"""
A completely new file is found but it is a conflict-marker file
and shouldn't be uploaded
"""
local = self.magic_path.preauthChild(relpath + u".conflict-author")
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
files = []
class FakeLocalSnapshot(object):
pass
class SnapshotService(object):
def add_file(self, f, local_parent=None):
files.append(f)
return succeed(FakeLocalSnapshot())
file_factory = MagicFileFactory(
self.config,
self.tahoe_client,
self.folder_status,
SnapshotService(),
object(), # uploader,
object(), # write_participant,
object(), # remote_cache,
object(), # filesystem,
)
service = ScannerService(
self.config,
file_factory,
object(),
cooperator=self.cooperator,
scan_interval=None,
clock=self.clock,
)
service.startService()
self.addCleanup(service.stopService)
self.assertThat(
service.scan_once(),
succeeded(Always()),
)
self.assertThat(files, Equals([]))
@given(
path_segments(), )
def test_scan_preexisting_local_snapshot(self, relpath):
"""
A pre-existing LocalSnapshot is in our database so the scanner
finds it at startup
"""
# make a pre-existing local snapshot
self.setup_example()
local = self.magic_path.preauthChild(relpath)
local.parent().makedirs(ignoreExistingDirectory=True)
local.setContent(b"dummy\n")
# pretend we stashed it, too
stash = FilePath(self.mktemp())
stash.makedirs()
stash.child(relpath).setContent(b"dummy\n")
local_snap = LocalSnapshot(
relpath,
author=self.author,
metadata={
"modification_time": 1234,
},
content_path=stash.child(relpath),
parents_local=[],
)
self.config.store_local_snapshot(
local_snap,
PathState(42, seconds_to_ns(42), seconds_to_ns(42)),
)
files = []
class SnapshotService(object):
def add_file(self, f):
files.append(f)
return succeed(local_snap)
class FakeUploader(object):
def upload_snapshot(self, snapshot):
return Deferred()
file_factory = MagicFileFactory(
self.config,
self.tahoe_client,
self.folder_status,
SnapshotService(),
FakeUploader(),
object(), # write_participant,
object(), # remote_cache,
object(), # filesystem,
synchronous=True,
)
service = ScannerService(
self.config,
file_factory,
object(),
cooperator=self.cooperator,
scan_interval=None,
clock=self.clock,
)
service.startService()
self.addCleanup(service.stopService)
self.assertThat(
service._loop(),
succeeded(Always()),
)
self.assertThat(files, Equals([local]))
class LocalSnapshotServiceTests(SyncTestCase):
"""
Tests for ``LocalSnapshotService``.
"""
def setup_example(self):
"""
Hypothesis-invoked hook to create per-example state.
Reset the database before running each test.
"""
self.magic_path = FilePath(self.mktemp()).asTextMode("utf-8")
self.magic_path.asBytesMode("utf-8").makedirs()
self.snapshot_creator = MemorySnapshotCreator()
self.snapshot_service = LocalSnapshotService(
magic_path=self.magic_path,
snapshot_creator=self.snapshot_creator,
status=WebSocketStatusService(),
)
@given(relative_paths(), binary())
def test_add_single_file(self, relative_path, content):
"""
Start the service, add a file and check if the operation succeeded.
"""
to_add = self.magic_path.preauthChild(relative_path)
to_add.asBytesMode("utf-8").parent().makedirs(
ignoreExistingDirectory=True)
to_add.asBytesMode("utf-8").setContent(content)
self.snapshot_service.startService()
self.assertThat(
self.snapshot_service.add_file(to_add),
succeeded(Always()),
)
self.assertThat(self.snapshot_service.stopService(),
succeeded(Always()))
self.assertThat(
self.snapshot_creator.processed,
Equals([to_add]),
)
@given(lists(path_segments(), unique=True), data())
def test_add_multiple_files(self, filenames, data):
"""
Add a bunch of files one by one and check whether the operation is
successful.
"""
files = []
for filename in filenames:
to_add = self.magic_path.child(filename)
content = data.draw(binary())
to_add.asBytesMode("utf-8").setContent(content)
files.append(to_add)
self.snapshot_service.startService()
list_d = []
for file in files:
result_d = self.snapshot_service.add_file(file)
list_d.append(result_d)
d = defer.gatherResults(list_d)
self.assertThat(
d,
succeeded(Always()),
)
self.assertThat(self.snapshot_service.stopService(),
succeeded(Always()))
self.assertThat(sorted(self.snapshot_creator.processed),
Equals(sorted(files)))
@given(relative_paths())
def test_add_file_not_a_filepath(self, relative_path):
"""
``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a
``Failure`` wrapping ``TypeError`` if called with something other than
a ``FilePath``.
"""
self.assertThat(
self.snapshot_service.add_file(relative_path),
failed(
AfterPreprocessing(
lambda f: (f.type, f.value.args),
Equals((TypeError, ("argument must be a FilePath", ))),
), ),
)
@given(relative_paths())
def test_add_file_directory(self, relative_path):
"""
``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a
``Failure`` wrapping ``ValueError`` if called with a path that refers
to a directory.
"""
to_add = self.magic_path.preauthChild(relative_path)
to_add.asBytesMode("utf-8").makedirs()
self.assertThat(
self.snapshot_service.add_file(to_add),
failed(
AfterPreprocessing(
lambda f: (f.type, f.value.args),
MatchesListwise([
Equals(ValueError),
Equals(("expected a regular file, {!r} is a directory".
format(to_add.path, ), )),
]),
), ),
)
@given(absolute_paths())
def test_add_file_outside_magic_directory(self, to_add):
"""
``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a
``Failure`` wrapping ``ValueError`` if called with a path that is not
contained by the Magic-Folder's magic directory.
"""
assume(not to_add.startswith(self.magic_path.path))
self.assertThat(
self.snapshot_service.add_file(FilePath(to_add)),
failed(
AfterPreprocessing(
lambda f: (f.type, f.value.args),
MatchesListwise([
Equals(ValueError),
MatchesPredicate(
lambda args: args[0].startswith(
"The path being added "),
"%r does not start with 'The path being added '.",
),
]),
), ),
)
def do_peekfs(cmd,path,*args):
global dsklst
# wstring simply writes a string to new file
def _wstring(fp):
def _wrap(path,string):
# Right teh filez LOL -KTHXBYE, LOLCATZ
tehfile=path.open('w')
tehfile.write(string)
tehfile.close()
return lambda *args: _wrap(fp,*args)
def _wget(fp):
def _wrap(path,url):
req=urllib2.urlopen(url).read()
tehfile=path.open('wb')
tehfile.write(req)
tehfile.close()
return lambda *args: _wrap(fp,*args)
# wstring writes a string to file
def _astring(fp):
def _wrap(path,string):
tehfile=path.open('a')
# Pydocs say both that this should be a no-op
# BUT also say that some systems will not seek on their own?
# we're just being careful here...
tehfile.seek(0,os.SEEK_END)
tehfile.write(string)
tehfile.close()
return lambda *args: _wrap(fp,*args)
# Templating engine
def _template(fp):
def _wrap(path,**template):
scratchfile=path.dirname()+"."+path.basename()+".tmp"
fh=path.open('r')
sfp=FilePath(scratchfile)
sfh=sfp.open('w')
seeklast=0
for buffer in fh.readlines():
for line in buffer:
sfh.write(line.format(**template))
sfh.flush()
sfh.close()
fh.close()
sfp.moveTo(path.realpath())
return lambda *args: _wrap(fp,*args)
def _sed(fp):
return lambda rxp: do_sed(rxp,fp)
def _urlextract(fp):
return lambda url: do_urlextract(fp,url)
def _extract(fp):
return lambda *args: do_extract(fp,*args)
def _moveTo(fp):
return lambda path: fp.moveTo(FilePath(path))
def _copyTo(fp):
return lambda path: fp.copyTo(FilePath(path))
def _ls(fp):
def _wrap(fp,glob="*"):
map (lambda f: f.basename(), fp.globChildren(glob))
return lambda *args: _wrap(fp,*args)
def _b64get(fp):
return lambda: base64.b64encode(fp.getContent())
def _b64put(fp):
def _wrap(path,content):
if path.exists and not is_regularf(path):
return False
return path.setContent(base64.b64decode(content))
return lambda *args: _wrap(fp,*args)
mntpnt=dsklst['/'].real_mountpoint()
# Make sure the user mounts us, don't auto-mount
if not os.path.ismount(mntpnt):
return False
os.chdir(mntpnt)
os.chroot(mntpnt)
pp=FilePath('/')
# We're safe /w preauthChild since we're chroot'ed
fp=pp.preauthChild(path)
"""
Mapping the t.p.f.FilePath methods
which we will allow, to human-names
we can accessed via cmd arg
"""
return {
'chmod': fp.chmod,
'getsize': fp.getsize,
'exists': fp.exists,
'isdir': fp.isdir,
'isfile': fp.isfile,
'islink': fp.islink,
'isabs': fp.isabs,
#'listdir': fp.listdir,
'ls': fp.listdir,
'dir': fp.listdir,
'splitext': fp.splitext,
'touch': fp.touch,
'rm': fp.remove,
'makedirs': fp.makedirs,
'basename': fp.basename,
'dirname': fp.dirname,
'parent': fp.parent,
'mkdir': fp.createDirectory,
'sed': _sed(fp),
'cp': _copyTo(fp),
'mv': _moveTo(fp),
'append': _astring(fp),
'put': _wstring(fp),
'apply_template': _template(fp),
#'wget': _wget(fp),
#'urlextract': _urlextract(fp),
#'extract': _extract(fp),
'get': fp.getContent,
'b64get': _b64get(fp),
'b64put': _b64put(fp),
#'ls': _ls(fp),
#'mknod': _mknod(fp)
}[cmd](*args)
def get(self):
reload(speechmodel)
top = FilePath('sounds')
path = top.preauthChild(self.get_argument('path'))
raw = load(path, speechmodel.rate)
self.recognize(raw)
class LocalSnapshotServiceTests(SyncTestCase):
"""
Tests for ``LocalSnapshotService``.
"""
def setup_example(self):
"""
Hypothesis-invoked hook to create per-example state.
Reset the database before running each test.
"""
self._node_dir = FilePath(self.mktemp())
self._node_dir.makedirs()
self._global_config = create_testing_configuration(
FilePath(self.mktemp()),
self._node_dir,
)
self.magic_path = FilePath(self.mktemp())
self.magic_path.makedirs()
self.magic_config = self._global_config.create_magic_folder(
"name",
self.magic_path,
create_local_author("author"),
random_immutable(directory=True),
random_dircap(),
60,
None,
)
self.status = WebSocketStatusService(reactor, self._global_config)
self.snapshot_creator = MemorySnapshotCreator()
self.snapshot_service = LocalSnapshotService(
config=self.magic_config,
snapshot_creator=self.snapshot_creator,
status=FolderStatus(self.magic_config.name, self.status),
)
@given(relative_paths(), binary())
def test_add_single_file(self, relative_path, content):
"""
Start the service, add a file and check if the operation succeeded.
"""
to_add = self.magic_path.preauthChild(relative_path)
to_add.parent().makedirs(ignoreExistingDirectory=True)
to_add.setContent(content)
self.snapshot_service.startService()
self.assertThat(
self.snapshot_service.add_file(to_add),
succeeded(Always()),
)
self.assertThat(self.snapshot_service.stopService(),
succeeded(Always()))
self.assertThat(
self.snapshot_creator.processed,
Equals([to_add]),
)
@given(lists(path_segments(), unique=True), data())
def test_add_multiple_files(self, filenames, data):
"""
Add a bunch of files one by one and check whether the operation is
successful.
"""
files = []
for filename in filenames:
to_add = self.magic_path.child(filename)
content = data.draw(binary())
to_add.setContent(content)
files.append(to_add)
self.snapshot_service.startService()
list_d = []
for file in files:
result_d = self.snapshot_service.add_file(file)
list_d.append(result_d)
self.assertThat(
defer.gatherResults(list_d),
succeeded(Always()),
)
self.assertThat(self.snapshot_service.stopService(),
succeeded(Always()))
self.assertThat(sorted(self.snapshot_creator.processed),
Equals(sorted(files)))
@given(relative_paths())
def test_add_file_not_a_filepath(self, relative_path):
"""
``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a
``Failure`` wrapping ``TypeError`` if called with something other than
a ``FilePath``.
"""
self.assertThat(
self.snapshot_service.add_file(relative_path),
failed(
AfterPreprocessing(
lambda f: (f.type, f.value.args),
Equals((TypeError, ("argument must be a FilePath", ))),
), ),
)
@given(relative_paths())
def test_add_file_directory(self, relative_path):
"""
``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a
``Failure`` wrapping ``ValueError`` if called with a path that refers
to a directory.
"""
to_add = self.magic_path.preauthChild(relative_path)
to_add.makedirs()
self.assertThat(
self.snapshot_service.add_file(to_add),
failed(
matches_failure(
APIError, "expected a regular file, .* is a directory"), ),
)
@given(absolute_paths())
def test_add_file_outside_magic_directory(self, to_add):
"""
``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a
``Failure`` wrapping ``ValueError`` if called with a path that is not
contained by the Magic-Folder's magic directory.
"""
assume(not to_add.startswith(self.magic_path.path))
self.assertThat(
self.snapshot_service.add_file(FilePath(to_add)),
failed(matches_failure(
APIError,
"The path being added .*",
), ),
)
def test_create_snapshot(self, author, folder_name, path_in_folder,
some_content):
"""
A **POST** to **/v1/snapshot/:folder-name** with a **path** query argument
creates a new local snapshot for the file at the given path in the
named folder.
"""
local_path = FilePath(self.mktemp())
local_path.makedirs()
some_file = local_path.preauthChild(path_in_folder).asBytesMode(
"utf-8")
some_file.parent().makedirs(ignoreExistingDirectory=True)
some_file.setContent(some_content)
treq = treq_for_folders(
Clock(),
FilePath(self.mktemp()),
AUTH_TOKEN,
{
folder_name:
magic_folder_config(author, FilePath(self.mktemp()),
local_path)
},
# Unlike test_wait_for_completion above we start the folder
# services. This will allow the local snapshot to be created and
# our request to receive a response.
start_folder_services=True,
)
self.assertThat(
authorized_request(
treq,
AUTH_TOKEN,
b"POST",
self.url.child(folder_name).set(u"path", path_in_folder),
),
succeeded(matches_response(code_matcher=Equals(CREATED), ), ),
)
self.assertThat(
authorized_request(
treq,
AUTH_TOKEN,
b"GET",
self.url,
),
succeeded(
matches_response(
code_matcher=Equals(OK),
headers_matcher=header_contains({
u"Content-Type":
Equals([u"application/json"]),
}),
body_matcher=AfterPreprocessing(
loads,
MatchesDict({
folder_name:
MatchesDict({
path_in_folder:
MatchesListwise([
MatchesDict({
u"type":
Equals(u"local"),
u"identifier":
is_hex_uuid(),
# XXX It would be nice to see some
# parents if there are any.
u"parents":
Equals([]),
u"content-path":
AfterPreprocessing(
lambda path: FilePath(path).
getContent(),
Equals(some_content),
),
u"author":
Equals(author.to_remote_author().
to_json()),
}),
]),
}),
}),
),
), ),
)
def test_create_local_snapshot(self, relative_target_path, content):
"""
``MagicFolder.local_snapshot_service`` can be used to create a new local
snapshot for a file in the folder.
"""
global_config = create_testing_configuration(
FilePath(self.mktemp()),
FilePath(self.mktemp()),
)
magic_path = FilePath(self.mktemp())
magic_path.makedirs()
mf_config = global_config.create_magic_folder(
u"foldername",
magic_path,
create_local_author(u"zara"),
random_immutable(directory=True),
random_dircap(),
60,
None,
)
target_path = magic_path.preauthChild(relative_target_path)
target_path.parent().makedirs(ignoreExistingDirectory=True)
target_path.setContent(content)
clock = task.Clock()
status_service = WebSocketStatusService(clock, global_config)
folder_status = FolderStatus(u"foldername", status_service)
local_snapshot_creator = MemorySnapshotCreator()
clock = task.Clock()
local_snapshot_service = LocalSnapshotService(
mf_config,
local_snapshot_creator,
folder_status,
)
uploader = Service()
tahoe_client = object()
name = u"local-snapshot-service-test"
participants = object()
magic_folder = MagicFolder(
client=tahoe_client,
config=mf_config,
name=name,
local_snapshot_service=local_snapshot_service,
folder_status=folder_status,
scanner_service=Service(),
remote_snapshot_cache=Service(),
downloader=MultiService(),
uploader=uploader,
participants=participants,
clock=clock,
magic_file_factory=MagicFileFactory(
mf_config,
tahoe_client,
folder_status,
local_snapshot_service,
uploader,
object(),
Service(),
InMemoryMagicFolderFilesystem(),
),
)
magic_folder.startService()
self.addCleanup(magic_folder.stopService)
adding = magic_folder.local_snapshot_service.add_file(
target_path,
)
self.assertThat(
adding,
succeeded(Always()),
)
self.assertThat(
local_snapshot_creator.processed,
Equals([target_path]),
)
