def _make_filesystem(identifier): choice = os.getenv("FB_FS") if choice == "testing": path = os.path.join(get_build_dir(), "fs") if not os.path.isdir(path): LOG.warning(("Could not find fs directory: %s. Perhaps you need to run manage.py filebrowser_test_setup?") % path) return LocalSubFileSystem(path) else: cluster_conf = conf.HDFS_CLUSTERS[identifier] return webhdfs.WebHdfs.from_config(cluster_conf)
def get_help_fs(app_name): """ Creates a local file system for a given app's help directory. """ app = appmanager.get_desktop_module(app_name) if app is not None: if app.help_dir is None: raise PopupException("No help available for app '%s'." % app_name) return LocalSubFileSystem(app.help_dir) else: raise PopupException( "App '%s' is not loaded, so no help is available for it!" % app_name)
def _make_filesystem(identifier): choice = os.getenv("FB_FS") if choice == "testing": path = os.path.join(get_build_dir(), "fs") if not os.path.isdir(path): logging.warning( ("Could not find fs directory: %s. Perhaps you need to run " + "manage.py filebrowser_test_setup?") % path) return LocalSubFileSystem(path) else: cluster_conf = conf.HDFS_CLUSTERS[identifier] return hadoopfs.HadoopFileSystem.from_config( cluster_conf, hadoop_bin_path=conf.HADOOP_BIN.get())
def _make_filesystem(identifier): choice = os.getenv("FB_FS") if choice == "testing": path = os.path.join(get_build_dir(), "fs") if not os.path.isdir(path): LOG.warning( ("Could not find fs directory: %s. Perhaps you need to run " + "manage.py filebrowser_test_setup?") % path) return LocalSubFileSystem(path) else: cluster_conf = conf.HDFS_CLUSTERS[identifier] # The only way to disable webhdfs is to specify an empty value if cluster_conf.WEBHDFS_URL.get() != '': return webhdfs.WebHdfs.from_config(cluster_conf) else: return hadoopfs.HadoopFileSystem.from_config( cluster_conf, hadoop_bin_path=conf.HADOOP_BIN.get())
def _init_filesystems(): """Initialize the module-scoped filesystem dictionary.""" global _filesystems if _filesystems is not None: return _filesystems = {} if has_hadoop(): # Load HDFSes _filesystems.update(get_all_hdfs()) # Load local for identifier in conf.LOCAL_FILESYSTEMS.keys(): local_fs = LocalSubFileSystem( conf.LOCAL_FILESYSTEMS[identifier].PATH.get()) if identifier in _filesystems: raise Exception(("Filesystem '%s' configured twice. First is " + "%s, second is local FS %s") % (identifier, _filesystems[identifier], local_fs)) _filesystems[identifier] = local_fs
def create(dir=None): """Creates a "filesystem" in a new temp dir and creates one file in it.""" if not dir: dir = tempfile.mkdtemp() logger.info("Using %s as base dir." % dir) fs = LocalSubFileSystem(dir) def write(path, contents): f = fs.open(path, "w") f.write(contents) f.close() write("/hello.txt", "hello world\n") write("/goodbye.txt", "goodbyte\n") write("/xss.html", "<blink>escape!!!</blink><script>alert('hello')</script>\n") write("/evil path%-of&doom?.txt", "annoying, eh?\n") # no </script> tag in filename, since that's a path delimiter. write("/xsspath<i><script>alert('hello').txt", "definitely annoying.\n") # But we can do </script> as a multi-directory thing! fs.mkdir("/<script>alert('hello')<") write("/<script>alert('hello')</script>", "there") fs.mkdir("/sub?dir") write("/sub?dir/howdy.txt", "there\n") fs.mkdir("/bigfiles") write("/bigfiles/loremipsum.txt", "\n\n".join(lorem_ipsum.paragraphs(1000))) # 50K of dev random write("/bigfiles/random_binary.bin", open("/dev/urandom").read(1024 * 50)) write("/count", "0123456789" * 8) write("/chmod-unreadable", "") fs.chmod("/chmod-unreadable", 0000) write("/chown-staff-group", "") try: stats = fs.stats("/chown-staff-group") # Figure out a group id that is different from the one it already has cur_gid = grp.getgrnam(stats["group"]).gr_gid other_groups = [gid for gid in os.getgroups() if gid != cur_gid] new_gid = other_groups[0] fs.chown("/chown-staff-group", fs.stats("/chown-staff-group")["user"], grp.getgrgid(new_gid).gr_name) except OSError: logger.exception("Ignoring error.") return fs
def create(dir=None): """Creates a "filesystem" in a new temp dir and creates one file in it.""" if not dir: dir = tempfile.mkdtemp() logger.info("Using %s as base dir." % dir) fs = LocalSubFileSystem(dir) def write(path, contents): f = fs.open(path, "w") f.write(contents) f.close() write("/hello.txt", "hello world\n") write("/goodbye.txt","goodbyte\n") write("/xss.html","<blink>escape!!!</blink><script>alert('hello')</script>\n") write("/evil path%-of&doom?.txt", "annoying, eh?\n") # no </script> tag in filename, since that's a path delimiter. write("/xsspath<i><script>alert('hello').txt", "definitely annoying.\n") # But we can do </script> as a multi-directory thing! fs.mkdir("/<script>alert('hello')<") write("/<script>alert('hello')</script>", "there") fs.mkdir("/sub?dir") write("/sub?dir/howdy.txt", "there\n") fs.mkdir("/bigfiles") write("/bigfiles/loremipsum.txt", "\n\n".join(lorem_ipsum.paragraphs(1000))) # 50K of dev random write("/bigfiles/random_binary.bin", open("/dev/urandom").read(1024*50)) write("/count", "0123456789"*8) write("/chmod-unreadable", "") fs.chmod("/chmod-unreadable", 0000) write("/chown-staff-group", "") try: stats = fs.stats("/chown-staff-group") # Figure out a group id that is different from the one it already has cur_gid = grp.getgrnam(stats["group"]).gr_gid other_groups = [gid for gid in os.getgroups() if gid != cur_gid] new_gid = other_groups[0] fs.chown("/chown-staff-group", fs.stats("/chown-staff-group")["user"], grp.getgrgid(new_gid).gr_name) except OSError: logger.exception("Ignoring error.") return fs