Example #1
0
 def test_is_ftp_source(self):
     test_online = DataSet(OSFS("."), "/local/path", "test_id",
                           "ftp://source/to/file", "test dataset")
     test_local = DataSet(OSFS("."), "/local/path", "test_id",
                          "./source/to/file", "test dataset")
     self.assertTrue(test_online.is_online_source())
     self.assertFalse(test_local.is_online_source())
Example #2
0
    def set_root(self, path):
        root, path = path.split("/", 1)
        if self._root_path == root:
            return path

        self._root_path = root
        # namespaces = ["details"]
        if root == "temp":
            # Open the PyFilesystem2 filesystem as source
            # from fs import open_fs
            from fs.osfs import OSFS

            self._root_fs = OSFS("/tmp")

        elif root == "user":
            # Open the PyFilesystem2 filesystem as source
            # from fs import open_fs
            from fs.osfs import OSFS

            self._root_fs = OSFS(os.path.expanduser("~"))

        elif root == "data_fs":
            from data.datastore_fs import DatastoreFS

            self._root_fs = DatastoreFS("/")

        elif root == "dav2fs":
            # Open the DAV Provider as source
            from mapper.fs_from_dav_provider import DAVProvider2FS
            from wsgidav.fs_dav_provider import FilesystemProvider
            from data.datastore_dav import DatastoreDAVProvider

            # dav_provider = FilesystemProvider(BASE_DIR)
            dav_provider = DatastoreDAVProvider()
            self._root_fs = DAVProvider2FS(dav_provider)

        elif root == "data_db":
            # Database explorer with PyFilesystem2
            from data.datastore_db import DatastoreDB

            self._root_fs = DatastoreDB()
            # namespaces = ["details", "properties"]

        elif root == "fire_fs_wip":
            from fire.firestore_fs import FirestoreFS

            self._root_fs = FirestoreFS("/")

        elif root == "fire_db":
            # Database explorer with PyFilesystem2
            from fire.firestore_db import FirestoreDB

            self._root_fs = FirestoreDB()
            # namespaces = ["details", "properties"]

        else:
            raise ValueError("Invalid PyFilesystem2 root '%s'" %
                             root.replace("<", "&lt;"))

        return path
Example #3
0
    def test_should_download_zipped_csv(self):
        os = OSFS("./tests/test_integration/resources/")
        file_name = "test_csv_zipped"
        test_zip_file = 'http://localhost:8001/local_data/base_train.zip'

        test_ds_zip = DataSet(os, file_name, "test_id", test_zip_file,
                              "test dataset", "zip")
        test_ds_zip.download()
        test_ds_zip.unzip_file()
        df = pd.read_csv(test_ds_zip.uri)
        self.assertEqual((2, 2), df.shape)
        os.remove(file_name + "/train.csv")
        os.removedir(file_name)

        ## only download
        os = OSFS("./tests/test_integration/resources/")
        file_name = "train.csv"
        test_file = 'http://localhost:8001/local_data/train.csv'

        test_ds = DataSet(os, file_name, "test_id", test_file, "test dataset")
        test_ds.download()
        test_ds.unzip_file()
        df = pd.read_csv(test_ds.uri)
        self.assertEqual((2, 2), df.shape)
        os.remove(file_name)
Example #4
0
    def test_validate_is_zip(self):
        test_ds_zip = DataSet(OSFS("."), "/local/path", "test_id",
                              "http://source/to/file", "test dataset", "zip")
        self.assertEquals(True, test_ds_zip.is_zipped())

        test_ds = DataSet(OSFS("."), "/local/path", "test_id",
                          "http://source/to/file", "test dataset")
        self.assertEquals(False, test_ds.is_zipped())
Example #5
0
def __(src_path: Path, dst_path: Path, move: bool = False):
    """Overloaded function to allow checking in within the same OSFS with only Path instances"""
    src_path = src_path.resolve()
    src_fs = OSFS(src_path.parent.as_posix())
    src_path = src_path.name
    existing_parent = next(p for p in dst_path.parents if p.exists())
    dst_fs = OSFS(existing_parent.as_posix())
    dst_path = dst_path.relative_to(existing_parent).as_posix()
    return _checkin(src_fs, src_path, dst_fs, dst_path, move)
Example #6
0
 def build_osfs(self, local_path):
     if platform.system() == "Windows":
         disk_character = local_path[:3]
         local_relative = unicode(local_path[2:])
         localfs = OSFS(unicode(disk_character))
     else:
         local_relative = unicode(local_path)
         localfs = OSFS(u"/")
     return localfs, local_relative
Example #7
0
 def setUp(self):
     self.temp_fs = TempFS()
     self.drive = "K"
     while os.path.exists(self.drive+":\\") and self.drive <= "Z":
         self.drive = chr(ord(self.drive) + 1)
     if self.drive > "Z":
         raise RuntimeError("no free drive letters")
     fs_to_mount = OSFS(self.temp_fs.getsyspath("/"))
     self.mount_proc = dokan.mount(fs_to_mount,self.drive)#,flags=dokan.DOKAN_OPTION_DEBUG|dokan.DOKAN_OPTION_STDERR,numthreads=1)
     self.fs = OSFS(self.mount_proc.path)
Example #8
0
    def test_construct_dataset(self):
        test_ds_zip = DataSet(OSFS("."), "/local/path", "test_id",
                              "http://source/to/file", "test dataset", "zip")

        self.assertEquals("zip", test_ds_zip.compression)

        test_ds = DataSet(OSFS("."), "/local/path", "test_id",
                          "http://source/to/file", "test dataset")

        self.assertEquals(None, test_ds.compression)
Example #9
0
    def verify_content_existence(self, modulestore, root_dir, location, dirname, category_name, filename_suffix=''):
        filesystem = OSFS(root_dir / 'test_export')
        self.assertTrue(filesystem.exists(dirname))

        query_loc = Location('i4x', location.org, location.course, category_name, None)
        items = modulestore.get_items(query_loc)

        for item in items:
            filesystem = OSFS(root_dir / ('test_export/' + dirname))
            self.assertTrue(filesystem.exists(item.location.name + filename_suffix))
Example #10
0
    def __init__(
        self,
        *,
        path: Path,  # dataset path, e.g. "mydata.n5/mydataset"
        data_slice: DataSourceSlice,
        axiskeys: str = "tzyxc",
        compression_type: str = "raw",
        tile_shape: Optional[Shape5D] = None,
        filesystem: Optional[FS] = None,
        mode: Mode = Mode.CREATE,
    ):
        super().__init__(data_slice=data_slice, tile_shape=tile_shape)
        if not set(data_slice.shape.present_spatial_axes.keys()).issubset(set(axiskeys)):
            raise ValueError(f"Cannot represent data source {data_slice} using axiskeys '{axiskeys}'")
        if not self.N5_SIGNATURE.search(path.as_posix()):
            raise UnsupportedUrlException(path)
        self.axiskeys = axiskeys
        self.compression_type = compression_type

        if mode == self.Mode.OPEN:
            self.filesystem = filesystem.opendir(path.as_posix()) if filesystem else OSFS(path.as_posix())
            return

        path_components = self.N5_PATH_SPLITTER.split(path.as_posix())
        outer_path = "".join(path_components[0:2])
        inner_path = "".join(path_components[2:])

        root_fs = filesystem or OSFS(path.anchor)
        n5_root_fs = root_fs.makedirs(outer_path, recreate=True)
        if not n5_root_fs.isfile("attributes.json"):
            with n5_root_fs.openbin("attributes.json", "w") as f:
                f.write(self.N5_ROOT_ATTRIBUTES)

        self.filesystem = n5_root_fs.makedirs(inner_path, recreate=True)
        attributes = {
            "dimensions": self.data_slice.shape.to_tuple(axiskeys[::-1]),
            "blockSize": self.tile_shape.to_tuple(axiskeys[::-1]),
            "axes": list(self.axiskeys[::-1]),
            "dataType": str(data_slice.dtype),
            "compression": {"type": self.compression_type},
        }
        with self.filesystem.openbin("attributes.json", "w") as f:
            f.write(json.dumps(attributes).encode("utf-8"))

        # create all directories in the constructor to avoid races when processing tiles
        created_dirs = set()
        for tile in self.data_slice.split(self.tile_shape):
            dir_path = self.get_tile_dir_dataset_path(global_roi=tile)
            if dir_path and dir_path not in created_dirs:
                self.filesystem.makedirs(dir_path)
                created_dirs.add(dir_path)
Example #11
0
class TestFS(unittest.TestCase):
    copy_fs_src = OSFS(copy_dir_path)
    copy_file_src = 'src/file.txt'
    copy_dir_src = 'src/directory'
    copy_fs_tgt = OSFS(copy_dir_path)
    copy_file_tgt = 'tgt/file.txt'
    copy_dir_tgt = 'tgt/directory'

    move_fs_src = OSFS(move_dir_path)
    move_file_src = 'src/file.txt'
    move_dir_src = 'src/directory'
    move_fs_tgt = OSFS(move_dir_path)
    move_file_tgt = 'tgt/file.txt'
    move_dir_tgt = 'tgt/directory'

    def _checkin_integrity(self, dst_fs, dst_path, type_method_name):
        self.assertTrue(dst_fs.exists(dst_path))
        self.assertTrue(getattr(dst_fs, type_method_name)(dst_path))

    _copy_file_integrity = partialmethod(_checkin_integrity, type_method_name='isfile')
    _copy_dir_integrity = partialmethod(_checkin_integrity, type_method_name='isdir')

    def _move_file_integrity(self, dst_fs, dst_path, exists_method):
        self._copy_file_integrity(dst_fs, dst_path)
        self.assertFalse(exists_method())

    def _move_dir_integrity(self, dst_fs, dst_path, exists_method):
        self._copy_dir_integrity(dst_fs, dst_path)
        self.assertFalse(exists_method())

    def test_copy(self):
        setup()
        # file
        with checking_in(self.copy_fs_src, self.copy_file_src, self.copy_fs_tgt, self.copy_file_tgt) as result:
            self._copy_file_integrity(*result)

        # dir
        with checking_in(self.copy_fs_src, self.copy_dir_src, self.copy_fs_tgt, self.copy_dir_tgt) as result:
            self._copy_dir_integrity(*result)

    def test_move(self):
        setup()
        # file
        with checking_in(self.move_fs_src, self.move_file_src, self.move_fs_tgt, self.move_file_tgt, True) as result:
            self._move_file_integrity(*result, partial(self.move_fs_src.exists, self.move_file_src))

        # dir
        with checking_in(self.move_fs_src, self.move_dir_src, self.move_fs_tgt, self.move_dir_tgt, True) as result:
            self._move_dir_integrity(*result, partial(self.move_fs_src.exists, self.move_dir_src))
Example #12
0
    def _copy_sources_from_template(self):
        from fs.path import relativefrom
        from fs.copy import copy_file
        from fs.osfs import OSFS
        path_source = self.c['source']['directory']
        if not path_source.startswith('/'):
            path_source = self.fs.getsyspath(path_source)
        with OSFS('/') as fs:
            if not fs.exists(path_source):
                raise ValueError(
                    "Source path {} not exists.".format(path_source))
        with OSFS(path_source) as fs_sor:

            for f in self.c['source']['filenames']:
                copy_file(fs_sor, f, self.fs, f)
Example #13
0
def fetch(source, target, filenames, depth, verbose):
    from ...api import files_in_directories
    from fs.osfs import OSFS
    from fs.copy import copy_file
    from fs.path import dirname
    with OSFS(source) as sor:
        with OSFS(target) as tar:
            files = files_in_directories(sor, ['*'], filenames, depth)
            for f in files:
                if not tar.exists(dirname(f)):
                    tar.makedirs(dirname(f))
                copy_file(sor, f, tar, f)
                if verbose:
                    click.echo('[COPY] {} => {}.'.format(
                        sor.getsyspath(f), tar.getsyspath(f)))
Example #14
0
def test_write_new_datapackage():
    first = load_datapackage(
        ZipFS(str(fixture_dir / "merging" / "merging_first.zip")))
    second = load_datapackage(
        ZipFS(str(fixture_dir / "merging" / "merging_second.zip")))
    with tempfile.TemporaryDirectory() as td:
        temp_fs = OSFS(td)
        result = merge_datapackages_with_mask(
            first_dp=first,
            first_resource_group_label="sa-data-vector",
            second_dp=second,
            second_resource_group_label="sa-data-array",
            mask_array=np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0], dtype=bool),
            output_fs=temp_fs,
        )
        result = load_datapackage(OSFS(td))

        assert isinstance(result, DatapackageBase)
        assert not isinstance(result.fs, MemoryFS)
        assert len(result.resources) == 5

        for suffix in {"indices", "data", "distributions", "flip"}:
            try:
                d, r = result.get_resource(f"sa-data-vector.{suffix}")
            except KeyError:
                continue

            assert r["name"] == f"sa-data-vector.{suffix}"
            assert r["path"] == f"sa-data-vector.{suffix}.npy"
            assert r["group"] == "sa-data-vector"
            assert r["nrows"] == 5

            if suffix == "data":
                assert np.allclose(d, np.array([0, 2, 4, 6, 8]))

            try:
                d, r = result.get_resource(f"sa-data-array.{suffix}")
            except KeyError:
                continue

            assert r["name"] == f"sa-data-array.{suffix}"
            assert r["path"] == f"sa-data-array.{suffix}.npy"
            assert r["group"] == "sa-data-array"
            assert r["nrows"] == 5

            if suffix == "data":
                assert d.shape == (5, 10)
                assert np.allclose(d[:, 0], np.array([1, 3, 5, 7, 9]) + 10)
Example #15
0
class DummySystem(ImportSystem):
    @patch('xmodule.modulestore.xml.OSFS', lambda dir: OSFS(mkdtemp()))
    def __init__(self, load_error_modules, library=False):

        if library:
            xmlstore = LibraryXMLModuleStore(
                "data_dir",
                source_dirs=[],
                load_error_modules=load_error_modules)
        else:
            xmlstore = XMLModuleStore("data_dir",
                                      source_dirs=[],
                                      load_error_modules=load_error_modules)
        course_id = CourseKey.from_string('/'.join([ORG, COURSE, RUN]))
        course_dir = "test_dir"
        error_tracker = Mock()

        super(DummySystem, self).__init__(
            xmlstore=xmlstore,
            course_id=course_id,
            course_dir=course_dir,
            error_tracker=error_tracker,
            load_error_modules=load_error_modules,
            mixins=(InheritanceMixin, XModuleMixin),
            field_data=KvsFieldData(DictKeyValueStore()),
        )

    def render_template(self, _template, _context):
        raise Exception("Shouldn't be called")
Example #16
0
    def export(self):
        """
        Perform the export given the parameters handed to this class at init.
        """
        with self.modulestore.bulk_operations(self.courselike_key):

            fsm = OSFS(self.root_dir)
            root = lxml.etree.Element('unknown')

            # export only the published content
            with self.modulestore.branch_setting(
                    ModuleStoreEnum.Branch.published_only,
                    self.courselike_key):
                courselike = self.get_courselike()
                export_fs = courselike.runtime.export_fs = fsm.makeopendir(
                    self.target_dir)

                # change all of the references inside the course to use the xml expected key type w/o version & branch
                xml_centric_courselike_key = self.get_key()
                adapt_references(courselike, xml_centric_courselike_key,
                                 export_fs)
                courselike.add_xml_to_node(root)

            # Make any needed adjustments to the root node.
            self.process_root(root, export_fs)

            # Process extra items-- drafts, assets, etc
            root_courselike_dir = self.root_dir + '/' + self.target_dir
            self.process_extra(root, courselike, root_courselike_dir,
                               xml_centric_courselike_key, export_fs)

            # Any last pass adjustments
            self.post_process(root, export_fs)
Example #17
0
    def _load_item(self, item, data_cache, apply_cached_metadata=True):
        """
        Load an XModuleDescriptor from item, using the children stored in data_cache
        """
        data_dir = getattr(item, 'data_dir', item['location']['course'])
        root = self.fs_root / data_dir

        if not root.isdir():
            root.mkdir()

        resource_fs = OSFS(root)

        cached_metadata = {}
        if apply_cached_metadata:
            cached_metadata = self.get_cached_metadata_inheritance_tree(Location(item['location']))

        # TODO (cdodge): When the 'split module store' work has been completed, we should remove
        # the 'metadata_inheritance_tree' parameter
        system = CachingDescriptorSystem(
            self,
            data_cache,
            self.default_class,
            resource_fs,
            self.error_tracker,
            self.render_template,
            cached_metadata,
        )
        return system.load_item(item['location'])
Example #18
0
    def __init__(self):
        self.filesys = tempfs.TempFS()

        #dirs = ["bin", "boot", "cdrom", "dev", "etc", "home", "lib", "lib64", "lost+found", "media", "mnt", "opt", "root", "run", "sbin", "snap", "srv", "sys", "tmp", "usr", "var"]
        dirs = [
            "etc", "boot"
        ]  # smaller filesystem for demo, also we should not load /home/ as that has our ssh keys and stuff

        for my_dir in dirs:
            my_fs = OSFS("/" + my_dir)
            try:
                fs.copy.copy_fs(my_fs,
                                self.filesys.makedir("/" + my_dir + "/"),
                                walker=None,
                                on_copy=None)
                print("dir: " + my_dir + " created")
            except:
                print("EXCEPTION: " + my_dir)
            #self.filesys.tree()

        #pointer = self.filesys.open("/newfile","w+")
        #pointer.write("Contents of a file.\r\n")
        #pointer.close()

        self.filesys.makedir("/home")
        pointer = self.filesys.open("/home/notes", "w+")
        pointer.write(
            "my username for email is bob, my password for email is password")
        pointer.close()

        print("File tree initalized:\r\n")
        self.filesys.tree()
Example #19
0
def make_post_sh(target, file_name):
    with OSFS(target) as t:
        with t.open(file_name, 'w') as fout:
            c = ("#!/bin/bash\n" + "#SBATCH -o %j.out\n" +
                 "#SBATCH -e %j.err\n" + "source ~/.zshrc\n" + "date\n" +
                 "pygate merge\n" + "pygate clear_subdirs\n" + "date\n")
            print(c, file=fout)
Example #20
0
def clear_subdirs(target, no_action=False):
    with OSFS(target) as t:
        for d in t.filterdir(path='.', exclude_files=['*'], dirs=['sub*']):
            if no_action:
                print('TO DELTE: {0}/{1}.'.format(t.getsyspath('.'), d.name))
            else:
                t.removetree(d.name)
Example #21
0
 def get(self, key, default):
     fs = None
     try:
         fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key))
         dirName = os.path.dirname(fullFileName)
         pkg = os.path.basename(dirName)
         fileName = os.path.basename(fullFileName)
         isZip = pkg.lower().endswith('.zip')
         if os.path.exists(dirName):
             if isZip:
                 fs = ZipFS(dirName, mode='r', compression='stored')
             else:
                 fs = OSFS(dirName, create=True)
             if fs.exists(fileName):
                 try:
                     #log(fileName)
                     #log(cPickle.loads(fs.getcontents(fileName)))
                     return cPickle.loads(fs.getcontents(fileName))
                 except Exception:
                     if isZip:
                         log('[WARNING] Broken file: %s' % fullFileName)
                     else:
                         log('[WARNING] Remove broken file: %s' %
                             fullFileName)
                         fs.remove(fileName)
                     raise
         return default
     except Exception:
         err(traceback.format_exc())
         return default
     finally:
         if fs is not None:
             fs.close()
Example #22
0
 def __resolve_path_load(self):
     from fs.osfs import OSFS
     import re
     from pathlib import Path
     path_check_point = str(Path(self.config('model_dir')) / 'checkpoint')
     pp = re.compile('^.*: "(.*)".*$')
     ps = re.compile('.*' + self.config('ckpt_name') + '-([0-9]+)-*')
     paths = []
     with OSFS('/') as fs:
         if not fs.exists(path_check_point):
             return fs.getsyspath(path_check_point), False
         with fs.open(path_check_point) as fin:
             for l in fin.readlines():
                 mat_path = pp.match(l)
                 if mat_path is not None:
                     path_load = mat_path[1]
                     mat_step = ps.match(path_load)
                     if mat_step is not None:
                         paths.append([path_load, int(mat_step[1])])
     step = self.config('step') or -1
     if step == -1:
         step = max(list(zip(*paths))[1])
     for p, s in paths:
         if s == step:
             return p, True
     return step, False
Example #23
0
def get_cache(cache_name=DEFAULT_CACHE_NAME, clean=False):
    """Return the path to a file cache"""

    from fs.osfs import OSFS
    from fs.appfs import UserDataFS
    from fs.errors import CreateFailed
    import os

    env_var = (cache_name + '_cache').upper()

    cache_dir = os.getenv(env_var, None)

    if cache_dir:
        try:
            return OSFS(cache_dir)
        except CreateFailed as e:
            raise CreateFailed("Failed to create '{}': {} ".format(
                cache_dir, e))
    else:

        try:
            return UserDataFS(cache_name.lower())
        except CreateFailed as e:
            raise CreateFailed("Failed to create '{}': {} ".format(
                cache_name.lower(), e))
Example #24
0
def create_free_editorslf(name):
    """
    Creates a BufferedSlfFS exclusively from the contents of the editor directory.

    The python files inside the editor directory are responsible for creating the STI images and adding them to the SLF.

    Each file should contain the function:
    ```
    def add_to_free_editorslf(source_fs, target_fs):
        pass
    ```
    source_fs is the source OSFS (editor directory)
    target_fs is the target BufferedSlfFS
    """
    target_fs = BufferedSlfFS()
    target_fs.library_name = name or "Free editor.slf"
    target_fs.library_path = "editor\\"
    target_fs.version = 0x0200  # 2.0
    target_fs.sort = 0  # BufferedSlfFS does not guarantee that the entries are sorted
    source_fs = OSFS('editor')
    for path in source_fs.walkfiles():
        if path.endswith(".py"):
            # run python file inside the editor directory
            name = ("editor" + path)[:-3].replace("/", ".")
            spec = spec_from_file_location(name, source_fs.getsyspath(path))
            module = module_from_spec(spec)
            spec.loader.exec_module(module)
            module.add_to_free_editorslf(source_fs, target_fs)
    for path in sorted(target_fs.walkfiles()):
        print(path)
    return target_fs
    def __init__(self, modulestore, course_entry, default_class, module_data,
                 lazy, **kwargs):
        """
        Computes the settings inheritance and sets up the cache.

        modulestore: the module store that can be used to retrieve additional
        modules

        course_entry: the originally fetched enveloped course_structure w/ branch and course id info.
        Callers to _load_item provide an override but that function ignores the provided structure and
        only looks at the branch and course id

        module_data: a dict mapping Location -> json that was cached from the
            underlying modulestore
        """
        # needed by capa_problem (as runtime.filestore via this.resources_fs)
        if course_entry.course_key.course:
            root = modulestore.fs_root / course_entry.course_key.org / course_entry.course_key.course / course_entry.course_key.run
        else:
            root = modulestore.fs_root / course_entry.structure['_id']
        root.makedirs_p()  # create directory if it doesn't exist

        super(CachingDescriptorSystem,
              self).__init__(field_data=None,
                             load_item=self._load_item,
                             resources_fs=OSFS(root),
                             **kwargs)
        self.modulestore = modulestore
        self.course_entry = course_entry
        self.lazy = lazy
        self.module_data = module_data
        self.default_class = default_class
        self.local_modules = {}
Example #26
0
def assertFS(path, remainder=None):
    """Assert existence of fs.
    Step up one path segment until the directory exists, create missing directories and return filesystem.

    Args:
        path (basestring)
        remainder (basestring)

    Returns:
        fs
    
    """

    if not os.path.isdir(path):
        head, tail = os.path.split(path)

        return assertFS(head, "/".join(filter(bool, [tail, remainder])))

    fs = OSFS(path)

    if remainder is not None:
        fs.makedirs(unicode(remainder))

        fs = fs.opendir(unicode(remainder))

    return fs
Example #27
0
def copy_wiki():
    ws = OSFS('wiki-small')
    for file in ws.walkfiles():
        print(file)
        copyfile('wiki-small%s' % file,
                 'htmls/%s' % file[file.rfind('/') + 1:])
    ws.close()
Example #28
0
 def set(self, key, value):
     fs = None
     try:
         if not key:
             return
         key = key.format(accountDBID=utils.getAccountDBID())
         fullFileName = os.path.join(self.cache_dir, '{0}.dat'.format(key))
         dirName = os.path.dirname(fullFileName)
         pkg = os.path.basename(dirName)
         fileName = os.path.basename(fullFileName)
         isZip = pkg.lower().endswith('.zip')
         save = True
         if isZip:
             fs = ZipFS(dirName, mode='a', compression='stored')
             if fs.exists(fileName):
                 log('[WARNING] archive "{}" already contains file "{}". Do not save the new data.'
                     .format(pkg, fileName))
                 save = False
         else:
             fs = OSFS(dirName, create=True)
         if save:
             fs.setcontents(fileName, cPickle.dumps(value))
     except Exception:
         err(traceback.format_exc())
     finally:
         if fs is not None:
             fs.close()
Example #29
0
    def test_write_to(dir_read, file_name):
        """Test the `write_to` method by writing a read file to a virtual file system.

        Parameters
        ----------
        dir_read : str
            Directory that contains the source file
        file_name : str
            Name of the source file

        """
        path_read = f"{dir_read}/{file_name}"
        ef = ExternalFile(f"{weldx_root_dir}/{path_read}")

        with OSFS(weldx_root_dir) as file_system:
            original_hash = file_system.hash(path_read, "md5")

            # check writing to hard drive
            with TemporaryDirectory(dir=weldx_root_dir) as td:
                ef.write_to(td)
                new_file_path = Path(f"{Path(td).name}/{file_name}").as_posix()
                assert file_system.isfile(new_file_path)
                assert file_system.hash(new_file_path, "md5") == original_hash

        # check writing to a memory file system
        with MemoryFS() as file_system:
            file_system.makedir("some_directory")
            ef.write_to("some_directory", file_system)

            new_file_path = f"some_directory/{file_name}"
            assert file_system.isfile(new_file_path)
            assert file_system.hash(new_file_path, "md5") == original_hash
Example #30
0
def get_yahoo_icon():
    try:
        http = urllib3.PoolManager()
        fetch_data = http.request('GET', 'http://opi.yahoo.com/online?u=%s&m=t&t=1' % request.args[0])
        if fetch_data.status == 200:
            from fs.osfs import OSFS

            file_server = OSFS('/home/www-data/web2py/applications/cbw/static/images')

            if len(request.args) != 1:
                raise HTTP(400)
            if fetch_data.data == "01":
                path = "icon-YMonline.png"
            else:
                path = "icon-YMoffline.png"
            if file_server.exists(path):
                response.headers['Content-Length'] = file_server.getinfo(path)['size']
                response.headers['Content-Type'] = 'image/png'
                response.headers['Content-Disposition'] = "attachment; filename=yahoo.png"
                return response.stream(file_server.open(path=path, mode='rb'))
            else:
                raise HTTP(404)
        else:
            raise Exception(fetch_data.status)
    except:
        return None