def test_is_ftp_source(self): test_online = DataSet(OSFS("."), "/local/path", "test_id", "ftp://source/to/file", "test dataset") test_local = DataSet(OSFS("."), "/local/path", "test_id", "./source/to/file", "test dataset") self.assertTrue(test_online.is_online_source()) self.assertFalse(test_local.is_online_source())
def set_root(self, path): root, path = path.split("/", 1) if self._root_path == root: return path self._root_path = root # namespaces = ["details"] if root == "temp": # Open the PyFilesystem2 filesystem as source # from fs import open_fs from fs.osfs import OSFS self._root_fs = OSFS("/tmp") elif root == "user": # Open the PyFilesystem2 filesystem as source # from fs import open_fs from fs.osfs import OSFS self._root_fs = OSFS(os.path.expanduser("~")) elif root == "data_fs": from data.datastore_fs import DatastoreFS self._root_fs = DatastoreFS("/") elif root == "dav2fs": # Open the DAV Provider as source from mapper.fs_from_dav_provider import DAVProvider2FS from wsgidav.fs_dav_provider import FilesystemProvider from data.datastore_dav import DatastoreDAVProvider # dav_provider = FilesystemProvider(BASE_DIR) dav_provider = DatastoreDAVProvider() self._root_fs = DAVProvider2FS(dav_provider) elif root == "data_db": # Database explorer with PyFilesystem2 from data.datastore_db import DatastoreDB self._root_fs = DatastoreDB() # namespaces = ["details", "properties"] elif root == "fire_fs_wip": from fire.firestore_fs import FirestoreFS self._root_fs = FirestoreFS("/") elif root == "fire_db": # Database explorer with PyFilesystem2 from fire.firestore_db import FirestoreDB self._root_fs = FirestoreDB() # namespaces = ["details", "properties"] else: raise ValueError("Invalid PyFilesystem2 root '%s'" % root.replace("<", "<")) return path
def test_should_download_zipped_csv(self): os = OSFS("./tests/test_integration/resources/") file_name = "test_csv_zipped" test_zip_file = 'http://localhost:8001/local_data/base_train.zip' test_ds_zip = DataSet(os, file_name, "test_id", test_zip_file, "test dataset", "zip") test_ds_zip.download() test_ds_zip.unzip_file() df = pd.read_csv(test_ds_zip.uri) self.assertEqual((2, 2), df.shape) os.remove(file_name + "/train.csv") os.removedir(file_name) ## only download os = OSFS("./tests/test_integration/resources/") file_name = "train.csv" test_file = 'http://localhost:8001/local_data/train.csv' test_ds = DataSet(os, file_name, "test_id", test_file, "test dataset") test_ds.download() test_ds.unzip_file() df = pd.read_csv(test_ds.uri) self.assertEqual((2, 2), df.shape) os.remove(file_name)
def test_validate_is_zip(self): test_ds_zip = DataSet(OSFS("."), "/local/path", "test_id", "http://source/to/file", "test dataset", "zip") self.assertEquals(True, test_ds_zip.is_zipped()) test_ds = DataSet(OSFS("."), "/local/path", "test_id", "http://source/to/file", "test dataset") self.assertEquals(False, test_ds.is_zipped())
def __(src_path: Path, dst_path: Path, move: bool = False): """Overloaded function to allow checking in within the same OSFS with only Path instances""" src_path = src_path.resolve() src_fs = OSFS(src_path.parent.as_posix()) src_path = src_path.name existing_parent = next(p for p in dst_path.parents if p.exists()) dst_fs = OSFS(existing_parent.as_posix()) dst_path = dst_path.relative_to(existing_parent).as_posix() return _checkin(src_fs, src_path, dst_fs, dst_path, move)
def build_osfs(self, local_path): if platform.system() == "Windows": disk_character = local_path[:3] local_relative = unicode(local_path[2:]) localfs = OSFS(unicode(disk_character)) else: local_relative = unicode(local_path) localfs = OSFS(u"/") return localfs, local_relative
def setUp(self): self.temp_fs = TempFS() self.drive = "K" while os.path.exists(self.drive+":\\") and self.drive <= "Z": self.drive = chr(ord(self.drive) + 1) if self.drive > "Z": raise RuntimeError("no free drive letters") fs_to_mount = OSFS(self.temp_fs.getsyspath("/")) self.mount_proc = dokan.mount(fs_to_mount,self.drive)#,flags=dokan.DOKAN_OPTION_DEBUG|dokan.DOKAN_OPTION_STDERR,numthreads=1) self.fs = OSFS(self.mount_proc.path)
def test_construct_dataset(self): test_ds_zip = DataSet(OSFS("."), "/local/path", "test_id", "http://source/to/file", "test dataset", "zip") self.assertEquals("zip", test_ds_zip.compression) test_ds = DataSet(OSFS("."), "/local/path", "test_id", "http://source/to/file", "test dataset") self.assertEquals(None, test_ds.compression)
def verify_content_existence(self, modulestore, root_dir, location, dirname, category_name, filename_suffix=''): filesystem = OSFS(root_dir / 'test_export') self.assertTrue(filesystem.exists(dirname)) query_loc = Location('i4x', location.org, location.course, category_name, None) items = modulestore.get_items(query_loc) for item in items: filesystem = OSFS(root_dir / ('test_export/' + dirname)) self.assertTrue(filesystem.exists(item.location.name + filename_suffix))
def __init__( self, *, path: Path, # dataset path, e.g. "mydata.n5/mydataset" data_slice: DataSourceSlice, axiskeys: str = "tzyxc", compression_type: str = "raw", tile_shape: Optional[Shape5D] = None, filesystem: Optional[FS] = None, mode: Mode = Mode.CREATE, ): super().__init__(data_slice=data_slice, tile_shape=tile_shape) if not set(data_slice.shape.present_spatial_axes.keys()).issubset(set(axiskeys)): raise ValueError(f"Cannot represent data source {data_slice} using axiskeys '{axiskeys}'") if not self.N5_SIGNATURE.search(path.as_posix()): raise UnsupportedUrlException(path) self.axiskeys = axiskeys self.compression_type = compression_type if mode == self.Mode.OPEN: self.filesystem = filesystem.opendir(path.as_posix()) if filesystem else OSFS(path.as_posix()) return path_components = self.N5_PATH_SPLITTER.split(path.as_posix()) outer_path = "".join(path_components[0:2]) inner_path = "".join(path_components[2:]) root_fs = filesystem or OSFS(path.anchor) n5_root_fs = root_fs.makedirs(outer_path, recreate=True) if not n5_root_fs.isfile("attributes.json"): with n5_root_fs.openbin("attributes.json", "w") as f: f.write(self.N5_ROOT_ATTRIBUTES) self.filesystem = n5_root_fs.makedirs(inner_path, recreate=True) attributes = { "dimensions": self.data_slice.shape.to_tuple(axiskeys[::-1]), "blockSize": self.tile_shape.to_tuple(axiskeys[::-1]), "axes": list(self.axiskeys[::-1]), "dataType": str(data_slice.dtype), "compression": {"type": self.compression_type}, } with self.filesystem.openbin("attributes.json", "w") as f: f.write(json.dumps(attributes).encode("utf-8")) # create all directories in the constructor to avoid races when processing tiles created_dirs = set() for tile in self.data_slice.split(self.tile_shape): dir_path = self.get_tile_dir_dataset_path(global_roi=tile) if dir_path and dir_path not in created_dirs: self.filesystem.makedirs(dir_path) created_dirs.add(dir_path)
class TestFS(unittest.TestCase): copy_fs_src = OSFS(copy_dir_path) copy_file_src = 'src/file.txt' copy_dir_src = 'src/directory' copy_fs_tgt = OSFS(copy_dir_path) copy_file_tgt = 'tgt/file.txt' copy_dir_tgt = 'tgt/directory' move_fs_src = OSFS(move_dir_path) move_file_src = 'src/file.txt' move_dir_src = 'src/directory' move_fs_tgt = OSFS(move_dir_path) move_file_tgt = 'tgt/file.txt' move_dir_tgt = 'tgt/directory' def _checkin_integrity(self, dst_fs, dst_path, type_method_name): self.assertTrue(dst_fs.exists(dst_path)) self.assertTrue(getattr(dst_fs, type_method_name)(dst_path)) _copy_file_integrity = partialmethod(_checkin_integrity, type_method_name='isfile') _copy_dir_integrity = partialmethod(_checkin_integrity, type_method_name='isdir') def _move_file_integrity(self, dst_fs, dst_path, exists_method): self._copy_file_integrity(dst_fs, dst_path) self.assertFalse(exists_method()) def _move_dir_integrity(self, dst_fs, dst_path, exists_method): self._copy_dir_integrity(dst_fs, dst_path) self.assertFalse(exists_method()) def test_copy(self): setup() # file with checking_in(self.copy_fs_src, self.copy_file_src, self.copy_fs_tgt, self.copy_file_tgt) as result: self._copy_file_integrity(*result) # dir with checking_in(self.copy_fs_src, self.copy_dir_src, self.copy_fs_tgt, self.copy_dir_tgt) as result: self._copy_dir_integrity(*result) def test_move(self): setup() # file with checking_in(self.move_fs_src, self.move_file_src, self.move_fs_tgt, self.move_file_tgt, True) as result: self._move_file_integrity(*result, partial(self.move_fs_src.exists, self.move_file_src)) # dir with checking_in(self.move_fs_src, self.move_dir_src, self.move_fs_tgt, self.move_dir_tgt, True) as result: self._move_dir_integrity(*result, partial(self.move_fs_src.exists, self.move_dir_src))
def _copy_sources_from_template(self): from fs.path import relativefrom from fs.copy import copy_file from fs.osfs import OSFS path_source = self.c['source']['directory'] if not path_source.startswith('/'): path_source = self.fs.getsyspath(path_source) with OSFS('/') as fs: if not fs.exists(path_source): raise ValueError( "Source path {} not exists.".format(path_source)) with OSFS(path_source) as fs_sor: for f in self.c['source']['filenames']: copy_file(fs_sor, f, self.fs, f)
def fetch(source, target, filenames, depth, verbose): from ...api import files_in_directories from fs.osfs import OSFS from fs.copy import copy_file from fs.path import dirname with OSFS(source) as sor: with OSFS(target) as tar: files = files_in_directories(sor, ['*'], filenames, depth) for f in files: if not tar.exists(dirname(f)): tar.makedirs(dirname(f)) copy_file(sor, f, tar, f) if verbose: click.echo('[COPY] {} => {}.'.format( sor.getsyspath(f), tar.getsyspath(f)))
def test_write_new_datapackage(): first = load_datapackage( ZipFS(str(fixture_dir / "merging" / "merging_first.zip"))) second = load_datapackage( ZipFS(str(fixture_dir / "merging" / "merging_second.zip"))) with tempfile.TemporaryDirectory() as td: temp_fs = OSFS(td) result = merge_datapackages_with_mask( first_dp=first, first_resource_group_label="sa-data-vector", second_dp=second, second_resource_group_label="sa-data-array", mask_array=np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0], dtype=bool), output_fs=temp_fs, ) result = load_datapackage(OSFS(td)) assert isinstance(result, DatapackageBase) assert not isinstance(result.fs, MemoryFS) assert len(result.resources) == 5 for suffix in {"indices", "data", "distributions", "flip"}: try: d, r = result.get_resource(f"sa-data-vector.{suffix}") except KeyError: continue assert r["name"] == f"sa-data-vector.{suffix}" assert r["path"] == f"sa-data-vector.{suffix}.npy" assert r["group"] == "sa-data-vector" assert r["nrows"] == 5 if suffix == "data": assert np.allclose(d, np.array([0, 2, 4, 6, 8])) try: d, r = result.get_resource(f"sa-data-array.{suffix}") except KeyError: continue assert r["name"] == f"sa-data-array.{suffix}" assert r["path"] == f"sa-data-array.{suffix}.npy" assert r["group"] == "sa-data-array" assert r["nrows"] == 5 if suffix == "data": assert d.shape == (5, 10) assert np.allclose(d[:, 0], np.array([1, 3, 5, 7, 9]) + 10)
class DummySystem(ImportSystem): @patch('xmodule.modulestore.xml.OSFS', lambda dir: OSFS(mkdtemp())) def __init__(self, load_error_modules, library=False): if library: xmlstore = LibraryXMLModuleStore( "data_dir", source_dirs=[], load_error_modules=load_error_modules) else: xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) course_id = CourseKey.from_string('/'.join([ORG, COURSE, RUN])) course_dir = "test_dir" error_tracker = Mock() super(DummySystem, self).__init__( xmlstore=xmlstore, course_id=course_id, course_dir=course_dir, error_tracker=error_tracker, load_error_modules=load_error_modules, mixins=(InheritanceMixin, XModuleMixin), field_data=KvsFieldData(DictKeyValueStore()), ) def render_template(self, _template, _context): raise Exception("Shouldn't be called")
def export(self): """ Perform the export given the parameters handed to this class at init. """ with self.modulestore.bulk_operations(self.courselike_key): fsm = OSFS(self.root_dir) root = lxml.etree.Element('unknown') # export only the published content with self.modulestore.branch_setting( ModuleStoreEnum.Branch.published_only, self.courselike_key): courselike = self.get_courselike() export_fs = courselike.runtime.export_fs = fsm.makeopendir( self.target_dir) # change all of the references inside the course to use the xml expected key type w/o version & branch xml_centric_courselike_key = self.get_key() adapt_references(courselike, xml_centric_courselike_key, export_fs) courselike.add_xml_to_node(root) # Make any needed adjustments to the root node. self.process_root(root, export_fs) # Process extra items-- drafts, assets, etc root_courselike_dir = self.root_dir + '/' + self.target_dir self.process_extra(root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs) # Any last pass adjustments self.post_process(root, export_fs)
def _load_item(self, item, data_cache, apply_cached_metadata=True): """ Load an XModuleDescriptor from item, using the children stored in data_cache """ data_dir = getattr(item, 'data_dir', item['location']['course']) root = self.fs_root / data_dir if not root.isdir(): root.mkdir() resource_fs = OSFS(root) cached_metadata = {} if apply_cached_metadata: cached_metadata = self.get_cached_metadata_inheritance_tree(Location(item['location'])) # TODO (cdodge): When the 'split module store' work has been completed, we should remove # the 'metadata_inheritance_tree' parameter system = CachingDescriptorSystem( self, data_cache, self.default_class, resource_fs, self.error_tracker, self.render_template, cached_metadata, ) return system.load_item(item['location'])
def __init__(self): self.filesys = tempfs.TempFS() #dirs = ["bin", "boot", "cdrom", "dev", "etc", "home", "lib", "lib64", "lost+found", "media", "mnt", "opt", "root", "run", "sbin", "snap", "srv", "sys", "tmp", "usr", "var"] dirs = [ "etc", "boot" ] # smaller filesystem for demo, also we should not load /home/ as that has our ssh keys and stuff for my_dir in dirs: my_fs = OSFS("/" + my_dir) try: fs.copy.copy_fs(my_fs, self.filesys.makedir("/" + my_dir + "/"), walker=None, on_copy=None) print("dir: " + my_dir + " created") except: print("EXCEPTION: " + my_dir) #self.filesys.tree() #pointer = self.filesys.open("/newfile","w+") #pointer.write("Contents of a file.\r\n") #pointer.close() self.filesys.makedir("/home") pointer = self.filesys.open("/home/notes", "w+") pointer.write( "my username for email is bob, my password for email is password") pointer.close() print("File tree initalized:\r\n") self.filesys.tree()
def make_post_sh(target, file_name): with OSFS(target) as t: with t.open(file_name, 'w') as fout: c = ("#!/bin/bash\n" + "#SBATCH -o %j.out\n" + "#SBATCH -e %j.err\n" + "source ~/.zshrc\n" + "date\n" + "pygate merge\n" + "pygate clear_subdirs\n" + "date\n") print(c, file=fout)
def clear_subdirs(target, no_action=False): with OSFS(target) as t: for d in t.filterdir(path='.', exclude_files=['*'], dirs=['sub*']): if no_action: print('TO DELTE: {0}/{1}.'.format(t.getsyspath('.'), d.name)) else: t.removetree(d.name)
def get(self, key, default): fs = None try: fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key)) dirName = os.path.dirname(fullFileName) pkg = os.path.basename(dirName) fileName = os.path.basename(fullFileName) isZip = pkg.lower().endswith('.zip') if os.path.exists(dirName): if isZip: fs = ZipFS(dirName, mode='r', compression='stored') else: fs = OSFS(dirName, create=True) if fs.exists(fileName): try: #log(fileName) #log(cPickle.loads(fs.getcontents(fileName))) return cPickle.loads(fs.getcontents(fileName)) except Exception: if isZip: log('[WARNING] Broken file: %s' % fullFileName) else: log('[WARNING] Remove broken file: %s' % fullFileName) fs.remove(fileName) raise return default except Exception: err(traceback.format_exc()) return default finally: if fs is not None: fs.close()
def __resolve_path_load(self): from fs.osfs import OSFS import re from pathlib import Path path_check_point = str(Path(self.config('model_dir')) / 'checkpoint') pp = re.compile('^.*: "(.*)".*$') ps = re.compile('.*' + self.config('ckpt_name') + '-([0-9]+)-*') paths = [] with OSFS('/') as fs: if not fs.exists(path_check_point): return fs.getsyspath(path_check_point), False with fs.open(path_check_point) as fin: for l in fin.readlines(): mat_path = pp.match(l) if mat_path is not None: path_load = mat_path[1] mat_step = ps.match(path_load) if mat_step is not None: paths.append([path_load, int(mat_step[1])]) step = self.config('step') or -1 if step == -1: step = max(list(zip(*paths))[1]) for p, s in paths: if s == step: return p, True return step, False
def get_cache(cache_name=DEFAULT_CACHE_NAME, clean=False): """Return the path to a file cache""" from fs.osfs import OSFS from fs.appfs import UserDataFS from fs.errors import CreateFailed import os env_var = (cache_name + '_cache').upper() cache_dir = os.getenv(env_var, None) if cache_dir: try: return OSFS(cache_dir) except CreateFailed as e: raise CreateFailed("Failed to create '{}': {} ".format( cache_dir, e)) else: try: return UserDataFS(cache_name.lower()) except CreateFailed as e: raise CreateFailed("Failed to create '{}': {} ".format( cache_name.lower(), e))
def create_free_editorslf(name): """ Creates a BufferedSlfFS exclusively from the contents of the editor directory. The python files inside the editor directory are responsible for creating the STI images and adding them to the SLF. Each file should contain the function: ``` def add_to_free_editorslf(source_fs, target_fs): pass ``` source_fs is the source OSFS (editor directory) target_fs is the target BufferedSlfFS """ target_fs = BufferedSlfFS() target_fs.library_name = name or "Free editor.slf" target_fs.library_path = "editor\\" target_fs.version = 0x0200 # 2.0 target_fs.sort = 0 # BufferedSlfFS does not guarantee that the entries are sorted source_fs = OSFS('editor') for path in source_fs.walkfiles(): if path.endswith(".py"): # run python file inside the editor directory name = ("editor" + path)[:-3].replace("/", ".") spec = spec_from_file_location(name, source_fs.getsyspath(path)) module = module_from_spec(spec) spec.loader.exec_module(module) module.add_to_free_editorslf(source_fs, target_fs) for path in sorted(target_fs.walkfiles()): print(path) return target_fs
def __init__(self, modulestore, course_entry, default_class, module_data, lazy, **kwargs): """ Computes the settings inheritance and sets up the cache. modulestore: the module store that can be used to retrieve additional modules course_entry: the originally fetched enveloped course_structure w/ branch and course id info. Callers to _load_item provide an override but that function ignores the provided structure and only looks at the branch and course id module_data: a dict mapping Location -> json that was cached from the underlying modulestore """ # needed by capa_problem (as runtime.filestore via this.resources_fs) if course_entry.course_key.course: root = modulestore.fs_root / course_entry.course_key.org / course_entry.course_key.course / course_entry.course_key.run else: root = modulestore.fs_root / course_entry.structure['_id'] root.makedirs_p() # create directory if it doesn't exist super(CachingDescriptorSystem, self).__init__(field_data=None, load_item=self._load_item, resources_fs=OSFS(root), **kwargs) self.modulestore = modulestore self.course_entry = course_entry self.lazy = lazy self.module_data = module_data self.default_class = default_class self.local_modules = {}
def assertFS(path, remainder=None): """Assert existence of fs. Step up one path segment until the directory exists, create missing directories and return filesystem. Args: path (basestring) remainder (basestring) Returns: fs """ if not os.path.isdir(path): head, tail = os.path.split(path) return assertFS(head, "/".join(filter(bool, [tail, remainder]))) fs = OSFS(path) if remainder is not None: fs.makedirs(unicode(remainder)) fs = fs.opendir(unicode(remainder)) return fs
def copy_wiki(): ws = OSFS('wiki-small') for file in ws.walkfiles(): print(file) copyfile('wiki-small%s' % file, 'htmls/%s' % file[file.rfind('/') + 1:]) ws.close()
def set(self, key, value): fs = None try: if not key: return key = key.format(accountDBID=utils.getAccountDBID()) fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key)) dirName = os.path.dirname(fullFileName) pkg = os.path.basename(dirName) fileName = os.path.basename(fullFileName) isZip = pkg.lower().endswith('.zip') save = True if isZip: fs = ZipFS(dirName, mode='a', compression='stored') if fs.exists(fileName): log('[WARNING] archive "{}" already contains file "{}". Do not save the new data.' .format(pkg, fileName)) save = False else: fs = OSFS(dirName, create=True) if save: fs.setcontents(fileName, cPickle.dumps(value)) except Exception: err(traceback.format_exc()) finally: if fs is not None: fs.close()
def test_write_to(dir_read, file_name): """Test the `write_to` method by writing a read file to a virtual file system. Parameters ---------- dir_read : str Directory that contains the source file file_name : str Name of the source file """ path_read = f"{dir_read}/{file_name}" ef = ExternalFile(f"{weldx_root_dir}/{path_read}") with OSFS(weldx_root_dir) as file_system: original_hash = file_system.hash(path_read, "md5") # check writing to hard drive with TemporaryDirectory(dir=weldx_root_dir) as td: ef.write_to(td) new_file_path = Path(f"{Path(td).name}/{file_name}").as_posix() assert file_system.isfile(new_file_path) assert file_system.hash(new_file_path, "md5") == original_hash # check writing to a memory file system with MemoryFS() as file_system: file_system.makedir("some_directory") ef.write_to("some_directory", file_system) new_file_path = f"some_directory/{file_name}" assert file_system.isfile(new_file_path) assert file_system.hash(new_file_path, "md5") == original_hash
def get_yahoo_icon(): try: http = urllib3.PoolManager() fetch_data = http.request('GET', 'http://opi.yahoo.com/online?u=%s&m=t&t=1' % request.args[0]) if fetch_data.status == 200: from fs.osfs import OSFS file_server = OSFS('/home/www-data/web2py/applications/cbw/static/images') if len(request.args) != 1: raise HTTP(400) if fetch_data.data == "01": path = "icon-YMonline.png" else: path = "icon-YMoffline.png" if file_server.exists(path): response.headers['Content-Length'] = file_server.getinfo(path)['size'] response.headers['Content-Type'] = 'image/png' response.headers['Content-Disposition'] = "attachment; filename=yahoo.png" return response.stream(file_server.open(path=path, mode='rb')) else: raise HTTP(404) else: raise Exception(fetch_data.status) except: return None