def nettest_to_path(path, allow_arbitrary_paths=False): """ Takes as input either a path or a nettest name. The nettest name may either be prefixed by the category of the nettest ( blocking, experimental, manipulation or third_party) or not. Args: allow_arbitrary_paths: allow also paths that are not relative to the nettest_directory. Returns: full path to the nettest file. """ if allow_arbitrary_paths and os.path.exists(path): return path test_name = path.rsplit("/", 1)[-1] test_categories = ["blocking", "experimental", "manipulation", "third_party"] nettest_dir = FilePath(config.nettest_directory) found_path = None for category in test_categories: p = nettest_dir.preauthChild(os.path.join(category, test_name) + ".py") if p.exists(): if found_path is not None: raise Exception("Found two tests named %s" % test_name) found_path = p.path if not found_path: raise e.NetTestNotFound(path) return found_path
def main(argv): # input posts = FilePath(argv[1]) # output blog = FilePath(argv[2]) # Since Sphinx gets confused by image paths with "special" characters in # them, generate new names for all the image paths and a mapping from the # old names to the new names. images = FilePath(argv[3]) imagepaths = [] for post in images.children(): if post.isdir(): imagepaths.append(post) safe = post.sibling(fixpath(post.basename())) if post != safe and not safe.isdir(): post.moveTo(safe) safe.linkTo(post) entries = [] for post in posts.children(): data = post.getContent().decode("utf-8") ignored, header, body = data.split(b"---", 2) meta = dict((text.strip() for text in line.split(":", 1)) for line in header.splitlines() if line.strip()) date = datetime.strptime(meta["date"], "%Y/%m/%d %H:%M:%S") parent = blog.preauthChild( ("%d/%02d/%02d" % (date.year, date.month, date.day)).encode("utf-8")) title = fixpath(meta["title"].strip().lower().encode("utf-8")).decode("utf-8") entry = parent.child((title + ".rst").encode("utf-8")) header = HEADER_TEMPLATE % dict( author=meta["author"].strip(), categories="none", tags=meta["categories"].strip(), title=meta["title"].strip(), underbar="=" * len(meta["title"].strip())) for path in imagepaths: body = body.replace( u"/" + path.basename().decode("utf-8") + u"/", u"/" + fixpath(path.basename()).decode("utf-8") + u"/") if not parent.isdir(): parent.makedirs() entry.setContent((header + html2rst(body)).encode("utf-8")) entries.append(entry) entries.sort() entries.reverse() sitemap = SITEMAP % dict( entries="".join([ "\n " + "/".join(entry.segmentsFrom(blog)) for entry in entries])) blog.child(b"master.rst").setContent(sitemap.encode("utf-8")) FilePath(b"conf.py").copyTo(blog.child(b"conf.py"))
def nettest_to_path(path, allow_arbitrary_paths=False): """ Takes as input either a path or a nettest name. The nettest name may either be prefixed by the category of the nettest ( blocking, experimental, manipulation or third_party) or not. Args: allow_arbitrary_paths: allow also paths that are not relative to the nettest_directory. Returns: full path to the nettest file. """ if allow_arbitrary_paths and os.path.exists(path): return path test_name = path.rsplit("/", 1)[-1] test_categories = [ "blocking", "experimental", "manipulation", "third_party" ] nettest_dir = FilePath(config.nettest_directory) found_path = None for category in test_categories: p = nettest_dir.preauthChild(os.path.join(category, test_name) + '.py') if p.exists(): if found_path is not None: raise Exception("Found two tests named %s" % test_name) found_path = p.path if not found_path: raise e.NetTestNotFound(path) return found_path
def test_create_fails(self, author, folder_name, path_in_folder): """ If a local snapshot cannot be created, a **POST** to **/v1/magic-folder/<folder-name>/snapshot** receives a response with an HTTP error code. """ local_path = FilePath(self.mktemp()) local_path.makedirs() # You may not create a snapshot of a directory. not_a_file = local_path.preauthChild(path_in_folder).asBytesMode( "utf-8") not_a_file.makedirs(ignoreExistingDirectory=True) treq = treq_for_folders( object(), FilePath(self.mktemp()), AUTH_TOKEN, { folder_name: magic_folder_config(author, FilePath(self.mktemp()), local_path) }, # This test carefully targets a failure mode that doesn't require # the service to be running. start_folder_services=False, ) self.assertThat( authorized_request( treq, AUTH_TOKEN, b"POST", self.url.child(folder_name, "snapshot").set(u"path", path_in_folder), ), succeeded( matches_response( # Maybe this could be BAD_REQUEST instead, sometimes, if # the path argument was bogus somehow. code_matcher=Equals(INTERNAL_SERVER_ERROR), headers_matcher=header_contains({ u"Content-Type": Equals([u"application/json"]), }), body_matcher=AfterPreprocessing( loads, ContainsDict({ u"reason": IsInstance(unicode), }), ), ), ), )
def create_fake_repository(self, files): """ Create files in a directory to mimic a repository of packages. :param dict source_repo: Dictionary mapping names of files to create to contents. :return: FilePath of directory containing fake package files. """ source_repo = FilePath(tempfile.mkdtemp()) for key in files: new_file = source_repo.preauthChild(key) if not new_file.parent().exists(): new_file.parent().makedirs() new_file.setContent(files[key]) return 'file://' + source_repo.path
def get(self): try: reload(recognize) r = recognize.Recognizer() top = FilePath('sounds') path = top.preauthChild(self.get_argument('path')) raw = load(path, 8000) t1 = time.time() out = r.recognize(raw, rate=8000) self.write({ 'result': out, 'ms': round(1000 * (time.time() - t1), 1) }) except Exception: traceback.print_exc() raise
def test_create_local_snapshot(self, relative_target_path, content): """ ``MagicFolder.local_snapshot_service`` can be used to create a new local snapshot for a file in the folder. """ magic_path = FilePath(self.mktemp()) magic_path.asBytesMode("utf-8").makedirs() target_path = magic_path.preauthChild( relative_target_path).asBytesMode("utf-8") target_path.asBytesMode("utf-8").parent().makedirs( ignoreExistingDirectory=True) target_path.setContent(content) local_snapshot_creator = MemorySnapshotCreator() local_snapshot_service = LocalSnapshotService(magic_path, local_snapshot_creator) clock = object() tahoe_client = object() name = u"local-snapshot-service-test" config = object() participants = object() magic_folder = MagicFolder( client=tahoe_client, config=config, name=name, local_snapshot_service=local_snapshot_service, uploader_service=Service(), initial_participants=participants, clock=clock, ) magic_folder.startService() self.addCleanup(magic_folder.stopService) adding = magic_folder.local_snapshot_service.add_file(target_path, ) self.assertThat( adding, succeeded(Always()), ) self.assertThat( local_snapshot_creator.processed, Equals([target_path]), )
def createSkeleton(siteDir): fromDir = FilePath(warp.__file__).sibling("priv").child("skeleton") for entryName in (l.strip() for l in fromDir.child("MANIFEST").open()): entry = fromDir.preauthChild(entryName) destination = siteDir.preauthChild(entryName) if destination.exists(): continue print " Copying %s" % entryName if not destination.parent().exists(): os.makedirs(destination.parent().path) entry.copyTo(destination) print "Done! Run with 'twistd -n warp'"
def createSkeleton(siteDir): fromDir = FilePath(warp.__file__).sibling("priv").child("skeleton") for entryName in (l.strip() for l in fromDir.child("MANIFEST").open()): entry = fromDir.preauthChild(entryName) destination = siteDir.preauthChild(entryName) if destination.exists(): continue print " Copying %s" % entryName if not destination.parent().exists(): destination.parent().makedirs() entry.copyTo(destination) print "Done! Run with 'twistd -n warp'"
def test_wait_for_completion(self, author, folder_name, path_in_folder, some_content): """ A **POST** request to **/v1/magic-folder/:folder-name/snapshot** does not receive a response before the snapshot has been created in the local database. """ local_path = FilePath(self.mktemp()) local_path.makedirs() some_file = local_path.preauthChild(path_in_folder).asBytesMode( "utf-8") some_file.parent().makedirs(ignoreExistingDirectory=True) some_file.setContent(some_content) treq = treq_for_folders( object(), FilePath(self.mktemp()), AUTH_TOKEN, { folder_name: magic_folder_config(author, FilePath(self.mktemp()), local_path) }, # The interesting behavior of this test hinges on this flag. We # decline to start the folder services here. Therefore, no local # snapshots will ever be created. This lets us observe the # request in a state where it is waiting to receive its response. # This demonstrates that the response is not delivered before the # local snapshot is created. See test_create_snapshot for the # alternative case. start_folder_services=False, ) self.assertThat( authorized_request( treq, AUTH_TOKEN, b"POST", self.url.child(folder_name, "snapshot").set(u"path", path_in_folder), ), has_no_result(), )
def test_uploader_service( self, name, file_path, relative_magic_path, relative_state_path, author, collective_cap_kind, poll_interval, content, ): """ ``MagicFolder.from_config`` creates an ``UploaderService`` which will upload snapshots using the given Tahoe client object. """ reactor = task.Clock() root = create_fake_tahoe_root() http_client = create_tahoe_treq_client(root) tahoe_client = TahoeClient( DecodedURL.from_text(U"http://example.invalid./"), http_client, ) ignored, upload_dircap = root.add_mutable_data( b"URI:DIR2:", json.dumps([ u"dirnode", { u"children": {} }, ]), ) ignored, collective_dircap = root.add_mutable_data( collective_cap_kind, json.dumps([ u"dirnode", { u"children": {} }, ]), ) basedir = FilePath(self.mktemp()) global_config = create_global_configuration( basedir, u"tcp:-1", FilePath(u"/non-tahoe-directory"), u"tcp:127.0.0.1:-1", ) magic_path = basedir.preauthChild(relative_magic_path) magic_path.asBytesMode("utf-8").makedirs() statedir = basedir.child(u"state") state_path = statedir.asTextMode("utf-8").preauthChild( relative_state_path) target_path = magic_path.asTextMode("utf-8").preauthChild(file_path) target_path.asBytesMode("utf-8").parent().makedirs( ignoreExistingDirectory=True) target_path.asBytesMode("utf-8").setContent(content) global_config.create_magic_folder( name, magic_path, state_path, author, collective_dircap, upload_dircap, poll_interval, ) magic_folder = MagicFolder.from_config( reactor, tahoe_client, name, global_config, ) magic_folder.startService() self.addCleanup(magic_folder.stopService) self.assertThat( magic_folder.uploader_service.running, Equals(True), ) self.assertThat( magic_folder.uploader_service._remote_snapshot_creator. _local_author, Equals(author), ) self.assertThat( magic_folder.folder_name, Equals(name), ) # add a file. This won't actually add a file until we advance # the clock. d = magic_folder.local_snapshot_service.add_file(target_path, ) self.assertThat( d, succeeded(Always()), ) def children(): return json.loads(root._uri.data[upload_dircap])[1][u"children"] reactor.advance(poll_interval - 1) self.assertThat( children(), Equals({}), ) reactor.advance(1) self.assertThat( children(), ContainsDict({path2magic(target_path.path): Always()}), "Children dictionary {!r} did not contain expected path".format( children, ), )
class FindUpdatesTests(SyncTestCase): """ Tests for ``find_updated_files`` """ def setup_example(self): self.author = create_local_author("alice") self.magic_path = FilePath(self.mktemp()) self.magic_path.makedirs() self._global_config = create_testing_configuration( FilePath(self.mktemp()), FilePath("dummy"), ) self.collective_cap = random_dircap() self.personal_cap = random_dircap() self.clock = Clock() self.status_service = WebSocketStatusService(self.clock, self._global_config) self.folder_status = FolderStatus("default", self.status_service) self.config = self._global_config.create_magic_folder( "default", self.magic_path, self.author, self.collective_cap, self.personal_cap, 1, None, ) # Use a cooperator that does not cooperate. self.cooperator = Cooperator( terminationPredicateFactory=lambda: lambda: False, scheduler=lambda f: f(), ) self.addCleanup(self.cooperator.stop) self.tahoe_client = create_tahoe_client( DecodedURL.from_text("http://invalid./"), create_tahoe_treq_client(create_fake_tahoe_root()), ) @given( relative_paths(), ) def test_scan_new(self, relpath): """ A completely new file is scanned """ local = self.magic_path.preauthChild(relpath) local.parent().makedirs(ignoreExistingDirectory=True) local.setContent(b"dummy\n") files = [] self.assertThat( find_updated_files(self.cooperator, self.config, files.append, status=self.folder_status), succeeded(Always()), ) self.assertThat( files, Equals([local]), ) @skipIf(runtime.platformType == "win32", "windows does not have unprivileged symlinks") @given( relative_paths(), ) def test_scan_symlink(self, relpath): """ A completely new symlink is ignored """ local = self.magic_path.preauthChild(relpath) local.parent().makedirs(ignoreExistingDirectory=True) local.linkTo(local) files = [] self.assertThat( find_updated_files(self.cooperator, self.config, files.append, status=self.folder_status), succeeded(Always()), ) self.assertThat( files, Equals([]), ) @given( relative_paths(), ) def test_scan_nothing(self, relpath): """ An existing, non-updated file is not scanned """ local = self.magic_path.preauthChild(relpath) local.parent().makedirs(ignoreExistingDirectory=True) local.setContent(b"dummy\n") snap = RemoteSnapshot( relpath, self.author, metadata={ "modification_time": int(local.getModificationTime()), }, capability=random_immutable(directory=True), parents_raw=[], content_cap=random_immutable(), metadata_cap=random_immutable(), ) self.config.store_downloaded_snapshot(relpath, snap, get_pathinfo(local).state) files = [] self.assertThat( find_updated_files(self.cooperator, self.config, files.append, status=self.folder_status), succeeded(Always()), ) self.assertThat(files, Equals([])) @given( relative_paths(), ) def test_scan_something_remote(self, relpath): """ We scan an update to a file we already know about. """ local = self.magic_path.preauthChild(relpath) local.parent().makedirs(ignoreExistingDirectory=True) local.setContent(b"dummy\n") snap = RemoteSnapshot( relpath, self.author, metadata={ # this remote is 2min older than our local file "modification_time": int(local.getModificationTime()) - 120, }, capability=random_immutable(directory=True), parents_raw=[], content_cap=random_immutable(), metadata_cap=random_immutable(), ) self.config.store_downloaded_snapshot( relpath, snap, OLD_PATH_STATE, ) files = [] self.assertThat( find_updated_files(self.cooperator, self.config, files.append, status=self.folder_status), succeeded(Always()), ) self.assertThat(files, Equals([local])) @given( relative_paths(), ) def test_scan_something_local(self, relpath): """ We scan an update to a file we already know about (but only locally). """ local = self.magic_path.preauthChild("existing-file") local.setContent(b"dummy\n") stash_dir = FilePath(self.mktemp()) stash_dir.makedirs() self.config.store_currentsnapshot_state("existing-file", OLD_PATH_STATE) files = [] self.assertThat( find_updated_files(self.cooperator, self.config, files.append, status=self.folder_status), succeeded(Always()), ) self.assertThat(files, Equals([local])) @given( relative_paths(), ) def test_scan_existing_to_directory(self, relpath): """ When we scan a path that we have a snapshot for, but it is now a directory, we ignore it, and report an error. """ local = self.magic_path.preauthChild("existing-file") local.makedirs() stash_dir = FilePath(self.mktemp()) stash_dir.makedirs() self.config.store_currentsnapshot_state("existing-file", OLD_PATH_STATE) files = [] self.assertThat( find_updated_files(self.cooperator, self.config, files.append, status=self.folder_status), succeeded(Always()), ) self.assertThat(files, Equals([])) self.assertThat( self.status_service._folders["default"]["errors"], MatchesListwise([ MatchesStructure(summary=MatchesRegex( "File .* was a file, and now is a directory.")), ]), ) @given( relative_paths(), ) def test_scan_delete(self, relpath): """ Scanning a relpath we know about with no local file is a delete """ self.config.store_currentsnapshot_state(relpath, OLD_PATH_STATE) files = [] self.assertThat( find_deleted_files(self.cooperator, self.config, files.append, status=self.folder_status), succeeded(Always()), ) self.assertThat(files, Equals([self.config.magic_path.preauthChild(relpath)])) @given( relative_paths(), ) def test_scan_once(self, relpath): """ The scanner service discovers a_file when a scan is requested. """ relpath = "a_file0" local = self.magic_path.preauthChild(relpath) local.parent().makedirs(ignoreExistingDirectory=True) local.setContent(b"dummy\n") files = [] class FakeLocalSnapshot(object): pass class SnapshotService(object): def add_file(self, f, local_parent=None): files.append(f) return succeed(FakeLocalSnapshot()) class FakeUploader(object): def upload_snapshot(self, snapshot): return Deferred() file_factory = MagicFileFactory( self.config, self.tahoe_client, self.folder_status, SnapshotService(), FakeUploader(), object(), # write_participant, object(), # remote_cache, object(), # filesystem, synchronous=True, ) service = ScannerService( self.config, file_factory, object(), cooperator=self.cooperator, scan_interval=None, clock=self.clock, ) service.startService() self.addCleanup(service.stopService) self.assertThat( service.scan_once(), succeeded(Always()), ) self.assertThat(files, Equals([local])) @given( relative_paths(), ) def test_scan_periodic(self, relpath): """ The scanner service iteself discovers a_file, when a scan interval is set. """ relpath = "a_file1" local = self.magic_path.preauthChild(relpath) local.parent().makedirs(ignoreExistingDirectory=True) local.setContent(b"dummy\n") files = [] class FakeLocalSnapshot(object): pass class SnapshotService(object): def add_file(self, f, local_parent=None): files.append(f) return succeed(FakeLocalSnapshot()) class FakeRemoteSnapshot(object): pass class FakeUploader(object): def upload_snapshot(self, snapshot): return Deferred() file_factory = MagicFileFactory( self.config, self.tahoe_client, self.folder_status, SnapshotService(), FakeUploader(), object(), # write_participant, object(), # remote_cache, object(), # filesystem, synchronous=True, ) service = ScannerService( self.config, file_factory, object(), cooperator=self.cooperator, scan_interval=1, clock=self.clock, ) service.startService() self.addCleanup(service.stopService) self.assertThat(files, Equals([local])) # XXX should use generated author-names, but get encoding errors # from FilePath on u'0.conflict-\x00' -- that is, a single nul as # an author-name @given( relative_paths(), ) def test_scan_conflict_files(self, relpath): """ A completely new file is found but it is a conflict-marker file and shouldn't be uploaded """ local = self.magic_path.preauthChild(relpath + u".conflict-author") local.parent().makedirs(ignoreExistingDirectory=True) local.setContent(b"dummy\n") files = [] class FakeLocalSnapshot(object): pass class SnapshotService(object): def add_file(self, f, local_parent=None): files.append(f) return succeed(FakeLocalSnapshot()) file_factory = MagicFileFactory( self.config, self.tahoe_client, self.folder_status, SnapshotService(), object(), # uploader, object(), # write_participant, object(), # remote_cache, object(), # filesystem, ) service = ScannerService( self.config, file_factory, object(), cooperator=self.cooperator, scan_interval=None, clock=self.clock, ) service.startService() self.addCleanup(service.stopService) self.assertThat( service.scan_once(), succeeded(Always()), ) self.assertThat(files, Equals([])) @given( path_segments(), ) def test_scan_preexisting_local_snapshot(self, relpath): """ A pre-existing LocalSnapshot is in our database so the scanner finds it at startup """ # make a pre-existing local snapshot self.setup_example() local = self.magic_path.preauthChild(relpath) local.parent().makedirs(ignoreExistingDirectory=True) local.setContent(b"dummy\n") # pretend we stashed it, too stash = FilePath(self.mktemp()) stash.makedirs() stash.child(relpath).setContent(b"dummy\n") local_snap = LocalSnapshot( relpath, author=self.author, metadata={ "modification_time": 1234, }, content_path=stash.child(relpath), parents_local=[], ) self.config.store_local_snapshot( local_snap, PathState(42, seconds_to_ns(42), seconds_to_ns(42)), ) files = [] class SnapshotService(object): def add_file(self, f): files.append(f) return succeed(local_snap) class FakeUploader(object): def upload_snapshot(self, snapshot): return Deferred() file_factory = MagicFileFactory( self.config, self.tahoe_client, self.folder_status, SnapshotService(), FakeUploader(), object(), # write_participant, object(), # remote_cache, object(), # filesystem, synchronous=True, ) service = ScannerService( self.config, file_factory, object(), cooperator=self.cooperator, scan_interval=None, clock=self.clock, ) service.startService() self.addCleanup(service.stopService) self.assertThat( service._loop(), succeeded(Always()), ) self.assertThat(files, Equals([local]))
class LocalSnapshotServiceTests(SyncTestCase): """ Tests for ``LocalSnapshotService``. """ def setup_example(self): """ Hypothesis-invoked hook to create per-example state. Reset the database before running each test. """ self.magic_path = FilePath(self.mktemp()).asTextMode("utf-8") self.magic_path.asBytesMode("utf-8").makedirs() self.snapshot_creator = MemorySnapshotCreator() self.snapshot_service = LocalSnapshotService( magic_path=self.magic_path, snapshot_creator=self.snapshot_creator, status=WebSocketStatusService(), ) @given(relative_paths(), binary()) def test_add_single_file(self, relative_path, content): """ Start the service, add a file and check if the operation succeeded. """ to_add = self.magic_path.preauthChild(relative_path) to_add.asBytesMode("utf-8").parent().makedirs( ignoreExistingDirectory=True) to_add.asBytesMode("utf-8").setContent(content) self.snapshot_service.startService() self.assertThat( self.snapshot_service.add_file(to_add), succeeded(Always()), ) self.assertThat(self.snapshot_service.stopService(), succeeded(Always())) self.assertThat( self.snapshot_creator.processed, Equals([to_add]), ) @given(lists(path_segments(), unique=True), data()) def test_add_multiple_files(self, filenames, data): """ Add a bunch of files one by one and check whether the operation is successful. """ files = [] for filename in filenames: to_add = self.magic_path.child(filename) content = data.draw(binary()) to_add.asBytesMode("utf-8").setContent(content) files.append(to_add) self.snapshot_service.startService() list_d = [] for file in files: result_d = self.snapshot_service.add_file(file) list_d.append(result_d) d = defer.gatherResults(list_d) self.assertThat( d, succeeded(Always()), ) self.assertThat(self.snapshot_service.stopService(), succeeded(Always())) self.assertThat(sorted(self.snapshot_creator.processed), Equals(sorted(files))) @given(relative_paths()) def test_add_file_not_a_filepath(self, relative_path): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``TypeError`` if called with something other than a ``FilePath``. """ self.assertThat( self.snapshot_service.add_file(relative_path), failed( AfterPreprocessing( lambda f: (f.type, f.value.args), Equals((TypeError, ("argument must be a FilePath", ))), ), ), ) @given(relative_paths()) def test_add_file_directory(self, relative_path): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``ValueError`` if called with a path that refers to a directory. """ to_add = self.magic_path.preauthChild(relative_path) to_add.asBytesMode("utf-8").makedirs() self.assertThat( self.snapshot_service.add_file(to_add), failed( AfterPreprocessing( lambda f: (f.type, f.value.args), MatchesListwise([ Equals(ValueError), Equals(("expected a regular file, {!r} is a directory". format(to_add.path, ), )), ]), ), ), ) @given(absolute_paths()) def test_add_file_outside_magic_directory(self, to_add): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``ValueError`` if called with a path that is not contained by the Magic-Folder's magic directory. """ assume(not to_add.startswith(self.magic_path.path)) self.assertThat( self.snapshot_service.add_file(FilePath(to_add)), failed( AfterPreprocessing( lambda f: (f.type, f.value.args), MatchesListwise([ Equals(ValueError), MatchesPredicate( lambda args: args[0].startswith( "The path being added "), "%r does not start with 'The path being added '.", ), ]), ), ), )
def do_peekfs(cmd,path,*args): global dsklst # wstring simply writes a string to new file def _wstring(fp): def _wrap(path,string): # Right teh filez LOL -KTHXBYE, LOLCATZ tehfile=path.open('w') tehfile.write(string) tehfile.close() return lambda *args: _wrap(fp,*args) def _wget(fp): def _wrap(path,url): req=urllib2.urlopen(url).read() tehfile=path.open('wb') tehfile.write(req) tehfile.close() return lambda *args: _wrap(fp,*args) # wstring writes a string to file def _astring(fp): def _wrap(path,string): tehfile=path.open('a') # Pydocs say both that this should be a no-op # BUT also say that some systems will not seek on their own? # we're just being careful here... tehfile.seek(0,os.SEEK_END) tehfile.write(string) tehfile.close() return lambda *args: _wrap(fp,*args) # Templating engine def _template(fp): def _wrap(path,**template): scratchfile=path.dirname()+"."+path.basename()+".tmp" fh=path.open('r') sfp=FilePath(scratchfile) sfh=sfp.open('w') seeklast=0 for buffer in fh.readlines(): for line in buffer: sfh.write(line.format(**template)) sfh.flush() sfh.close() fh.close() sfp.moveTo(path.realpath()) return lambda *args: _wrap(fp,*args) def _sed(fp): return lambda rxp: do_sed(rxp,fp) def _urlextract(fp): return lambda url: do_urlextract(fp,url) def _extract(fp): return lambda *args: do_extract(fp,*args) def _moveTo(fp): return lambda path: fp.moveTo(FilePath(path)) def _copyTo(fp): return lambda path: fp.copyTo(FilePath(path)) def _ls(fp): def _wrap(fp,glob="*"): map (lambda f: f.basename(), fp.globChildren(glob)) return lambda *args: _wrap(fp,*args) def _b64get(fp): return lambda: base64.b64encode(fp.getContent()) def _b64put(fp): def _wrap(path,content): if path.exists and not is_regularf(path): return False return path.setContent(base64.b64decode(content)) return lambda *args: _wrap(fp,*args) mntpnt=dsklst['/'].real_mountpoint() # Make sure the user mounts us, don't auto-mount if not os.path.ismount(mntpnt): return False os.chdir(mntpnt) os.chroot(mntpnt) pp=FilePath('/') # We're safe /w preauthChild since we're chroot'ed fp=pp.preauthChild(path) """ Mapping the t.p.f.FilePath methods which we will allow, to human-names we can accessed via cmd arg """ return { 'chmod': fp.chmod, 'getsize': fp.getsize, 'exists': fp.exists, 'isdir': fp.isdir, 'isfile': fp.isfile, 'islink': fp.islink, 'isabs': fp.isabs, #'listdir': fp.listdir, 'ls': fp.listdir, 'dir': fp.listdir, 'splitext': fp.splitext, 'touch': fp.touch, 'rm': fp.remove, 'makedirs': fp.makedirs, 'basename': fp.basename, 'dirname': fp.dirname, 'parent': fp.parent, 'mkdir': fp.createDirectory, 'sed': _sed(fp), 'cp': _copyTo(fp), 'mv': _moveTo(fp), 'append': _astring(fp), 'put': _wstring(fp), 'apply_template': _template(fp), #'wget': _wget(fp), #'urlextract': _urlextract(fp), #'extract': _extract(fp), 'get': fp.getContent, 'b64get': _b64get(fp), 'b64put': _b64put(fp), #'ls': _ls(fp), #'mknod': _mknod(fp) }[cmd](*args)
def get(self): reload(speechmodel) top = FilePath('sounds') path = top.preauthChild(self.get_argument('path')) raw = load(path, speechmodel.rate) self.recognize(raw)
class LocalSnapshotServiceTests(SyncTestCase): """ Tests for ``LocalSnapshotService``. """ def setup_example(self): """ Hypothesis-invoked hook to create per-example state. Reset the database before running each test. """ self._node_dir = FilePath(self.mktemp()) self._node_dir.makedirs() self._global_config = create_testing_configuration( FilePath(self.mktemp()), self._node_dir, ) self.magic_path = FilePath(self.mktemp()) self.magic_path.makedirs() self.magic_config = self._global_config.create_magic_folder( "name", self.magic_path, create_local_author("author"), random_immutable(directory=True), random_dircap(), 60, None, ) self.status = WebSocketStatusService(reactor, self._global_config) self.snapshot_creator = MemorySnapshotCreator() self.snapshot_service = LocalSnapshotService( config=self.magic_config, snapshot_creator=self.snapshot_creator, status=FolderStatus(self.magic_config.name, self.status), ) @given(relative_paths(), binary()) def test_add_single_file(self, relative_path, content): """ Start the service, add a file and check if the operation succeeded. """ to_add = self.magic_path.preauthChild(relative_path) to_add.parent().makedirs(ignoreExistingDirectory=True) to_add.setContent(content) self.snapshot_service.startService() self.assertThat( self.snapshot_service.add_file(to_add), succeeded(Always()), ) self.assertThat(self.snapshot_service.stopService(), succeeded(Always())) self.assertThat( self.snapshot_creator.processed, Equals([to_add]), ) @given(lists(path_segments(), unique=True), data()) def test_add_multiple_files(self, filenames, data): """ Add a bunch of files one by one and check whether the operation is successful. """ files = [] for filename in filenames: to_add = self.magic_path.child(filename) content = data.draw(binary()) to_add.setContent(content) files.append(to_add) self.snapshot_service.startService() list_d = [] for file in files: result_d = self.snapshot_service.add_file(file) list_d.append(result_d) self.assertThat( defer.gatherResults(list_d), succeeded(Always()), ) self.assertThat(self.snapshot_service.stopService(), succeeded(Always())) self.assertThat(sorted(self.snapshot_creator.processed), Equals(sorted(files))) @given(relative_paths()) def test_add_file_not_a_filepath(self, relative_path): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``TypeError`` if called with something other than a ``FilePath``. """ self.assertThat( self.snapshot_service.add_file(relative_path), failed( AfterPreprocessing( lambda f: (f.type, f.value.args), Equals((TypeError, ("argument must be a FilePath", ))), ), ), ) @given(relative_paths()) def test_add_file_directory(self, relative_path): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``ValueError`` if called with a path that refers to a directory. """ to_add = self.magic_path.preauthChild(relative_path) to_add.makedirs() self.assertThat( self.snapshot_service.add_file(to_add), failed( matches_failure( APIError, "expected a regular file, .* is a directory"), ), ) @given(absolute_paths()) def test_add_file_outside_magic_directory(self, to_add): """ ``LocalSnapshotService.add_file`` returns a ``Deferred`` that fires with a ``Failure`` wrapping ``ValueError`` if called with a path that is not contained by the Magic-Folder's magic directory. """ assume(not to_add.startswith(self.magic_path.path)) self.assertThat( self.snapshot_service.add_file(FilePath(to_add)), failed(matches_failure( APIError, "The path being added .*", ), ), )
def test_create_snapshot(self, author, folder_name, path_in_folder, some_content): """ A **POST** to **/v1/snapshot/:folder-name** with a **path** query argument creates a new local snapshot for the file at the given path in the named folder. """ local_path = FilePath(self.mktemp()) local_path.makedirs() some_file = local_path.preauthChild(path_in_folder).asBytesMode( "utf-8") some_file.parent().makedirs(ignoreExistingDirectory=True) some_file.setContent(some_content) treq = treq_for_folders( Clock(), FilePath(self.mktemp()), AUTH_TOKEN, { folder_name: magic_folder_config(author, FilePath(self.mktemp()), local_path) }, # Unlike test_wait_for_completion above we start the folder # services. This will allow the local snapshot to be created and # our request to receive a response. start_folder_services=True, ) self.assertThat( authorized_request( treq, AUTH_TOKEN, b"POST", self.url.child(folder_name).set(u"path", path_in_folder), ), succeeded(matches_response(code_matcher=Equals(CREATED), ), ), ) self.assertThat( authorized_request( treq, AUTH_TOKEN, b"GET", self.url, ), succeeded( matches_response( code_matcher=Equals(OK), headers_matcher=header_contains({ u"Content-Type": Equals([u"application/json"]), }), body_matcher=AfterPreprocessing( loads, MatchesDict({ folder_name: MatchesDict({ path_in_folder: MatchesListwise([ MatchesDict({ u"type": Equals(u"local"), u"identifier": is_hex_uuid(), # XXX It would be nice to see some # parents if there are any. u"parents": Equals([]), u"content-path": AfterPreprocessing( lambda path: FilePath(path). getContent(), Equals(some_content), ), u"author": Equals(author.to_remote_author(). to_json()), }), ]), }), }), ), ), ), )
def test_create_local_snapshot(self, relative_target_path, content): """ ``MagicFolder.local_snapshot_service`` can be used to create a new local snapshot for a file in the folder. """ global_config = create_testing_configuration( FilePath(self.mktemp()), FilePath(self.mktemp()), ) magic_path = FilePath(self.mktemp()) magic_path.makedirs() mf_config = global_config.create_magic_folder( u"foldername", magic_path, create_local_author(u"zara"), random_immutable(directory=True), random_dircap(), 60, None, ) target_path = magic_path.preauthChild(relative_target_path) target_path.parent().makedirs(ignoreExistingDirectory=True) target_path.setContent(content) clock = task.Clock() status_service = WebSocketStatusService(clock, global_config) folder_status = FolderStatus(u"foldername", status_service) local_snapshot_creator = MemorySnapshotCreator() clock = task.Clock() local_snapshot_service = LocalSnapshotService( mf_config, local_snapshot_creator, folder_status, ) uploader = Service() tahoe_client = object() name = u"local-snapshot-service-test" participants = object() magic_folder = MagicFolder( client=tahoe_client, config=mf_config, name=name, local_snapshot_service=local_snapshot_service, folder_status=folder_status, scanner_service=Service(), remote_snapshot_cache=Service(), downloader=MultiService(), uploader=uploader, participants=participants, clock=clock, magic_file_factory=MagicFileFactory( mf_config, tahoe_client, folder_status, local_snapshot_service, uploader, object(), Service(), InMemoryMagicFolderFilesystem(), ), ) magic_folder.startService() self.addCleanup(magic_folder.stopService) adding = magic_folder.local_snapshot_service.add_file( target_path, ) self.assertThat( adding, succeeded(Always()), ) self.assertThat( local_snapshot_creator.processed, Equals([target_path]), )