def _make_filesystem(identifier):
choice = os.getenv("FB_FS")
if choice == "testing":
path = os.path.join(get_build_dir(), "fs")
if not os.path.isdir(path):
LOG.warning(("Could not find fs directory: %s. Perhaps you need to run manage.py filebrowser_test_setup?") % path)
return LocalSubFileSystem(path)
else:
cluster_conf = conf.HDFS_CLUSTERS[identifier]
return webhdfs.WebHdfs.from_config(cluster_conf)
def get_help_fs(app_name):
"""
Creates a local file system for a given app's help directory.
"""
app = appmanager.get_desktop_module(app_name)
if app is not None:
if app.help_dir is None:
raise PopupException("No help available for app '%s'." % app_name)
return LocalSubFileSystem(app.help_dir)
else:
raise PopupException(
"App '%s' is not loaded, so no help is available for it!" %
app_name)
def _make_filesystem(identifier):
choice = os.getenv("FB_FS")
if choice == "testing":
path = os.path.join(get_build_dir(), "fs")
if not os.path.isdir(path):
logging.warning(
("Could not find fs directory: %s. Perhaps you need to run " +
"manage.py filebrowser_test_setup?") % path)
return LocalSubFileSystem(path)
else:
cluster_conf = conf.HDFS_CLUSTERS[identifier]
return hadoopfs.HadoopFileSystem.from_config(
cluster_conf,
hadoop_bin_path=conf.HADOOP_BIN.get())
def _make_filesystem(identifier):
choice = os.getenv("FB_FS")
if choice == "testing":
path = os.path.join(get_build_dir(), "fs")
if not os.path.isdir(path):
LOG.warning(
("Could not find fs directory: %s. Perhaps you need to run " +
"manage.py filebrowser_test_setup?") % path)
return LocalSubFileSystem(path)
else:
cluster_conf = conf.HDFS_CLUSTERS[identifier]
# The only way to disable webhdfs is to specify an empty value
if cluster_conf.WEBHDFS_URL.get() != '':
return webhdfs.WebHdfs.from_config(cluster_conf)
else:
return hadoopfs.HadoopFileSystem.from_config(
cluster_conf, hadoop_bin_path=conf.HADOOP_BIN.get())
def _init_filesystems():
"""Initialize the module-scoped filesystem dictionary."""
global _filesystems
if _filesystems is not None:
return
_filesystems = {}
if has_hadoop():
# Load HDFSes
_filesystems.update(get_all_hdfs())
# Load local
for identifier in conf.LOCAL_FILESYSTEMS.keys():
local_fs = LocalSubFileSystem(
conf.LOCAL_FILESYSTEMS[identifier].PATH.get())
if identifier in _filesystems:
raise Exception(("Filesystem '%s' configured twice. First is " +
"%s, second is local FS %s") % (identifier, _filesystems[identifier], local_fs))
_filesystems[identifier] = local_fs
def create(dir=None):
"""Creates a "filesystem" in a new temp dir and creates one file in it."""
if not dir:
dir = tempfile.mkdtemp()
logger.info("Using %s as base dir." % dir)
fs = LocalSubFileSystem(dir)
def write(path, contents):
f = fs.open(path, "w")
f.write(contents)
f.close()
write("/hello.txt", "hello world\n")
write("/goodbye.txt", "goodbyte\n")
write("/xss.html",
"<blink>escape!!!</blink><script>alert('hello')</script>\n")
write("/evil path%-of&doom?.txt", "annoying, eh?\n")
# no </script> tag in filename, since that's a path delimiter.
write("/xsspath<i><script>alert('hello').txt", "definitely annoying.\n")
# But we can do </script> as a multi-directory thing!
fs.mkdir("/<script>alert('hello')<")
write("/<script>alert('hello')</script>", "there")
fs.mkdir("/sub?dir")
write("/sub?dir/howdy.txt", "there\n")
fs.mkdir("/bigfiles")
write("/bigfiles/loremipsum.txt",
"\n\n".join(lorem_ipsum.paragraphs(1000)))
# 50K of dev random
write("/bigfiles/random_binary.bin", open("/dev/urandom").read(1024 * 50))
write("/count", "0123456789" * 8)
write("/chmod-unreadable", "")
fs.chmod("/chmod-unreadable", 0000)
write("/chown-staff-group", "")
try:
stats = fs.stats("/chown-staff-group")
# Figure out a group id that is different from the one it already has
cur_gid = grp.getgrnam(stats["group"]).gr_gid
other_groups = [gid for gid in os.getgroups() if gid != cur_gid]
new_gid = other_groups[0]
fs.chown("/chown-staff-group",
fs.stats("/chown-staff-group")["user"],
grp.getgrgid(new_gid).gr_name)
except OSError:
logger.exception("Ignoring error.")
return fs
def create(dir=None):
"""Creates a "filesystem" in a new temp dir and creates one file in it."""
if not dir:
dir = tempfile.mkdtemp()
logger.info("Using %s as base dir." % dir)
fs = LocalSubFileSystem(dir)
def write(path, contents):
f = fs.open(path, "w")
f.write(contents)
f.close()
write("/hello.txt", "hello world\n")
write("/goodbye.txt","goodbyte\n")
write("/xss.html","<blink>escape!!!</blink><script>alert('hello')</script>\n")
write("/evil path%-of&doom?.txt", "annoying, eh?\n")
# no </script> tag in filename, since that's a path delimiter.
write("/xsspath<i><script>alert('hello').txt", "definitely annoying.\n")
# But we can do </script> as a multi-directory thing!
fs.mkdir("/<script>alert('hello')<")
write("/<script>alert('hello')</script>", "there")
fs.mkdir("/sub?dir")
write("/sub?dir/howdy.txt", "there\n")
fs.mkdir("/bigfiles")
write("/bigfiles/loremipsum.txt", "\n\n".join(lorem_ipsum.paragraphs(1000)))
# 50K of dev random
write("/bigfiles/random_binary.bin", open("/dev/urandom").read(1024*50))
write("/count", "0123456789"*8)
write("/chmod-unreadable", "")
fs.chmod("/chmod-unreadable", 0000)
write("/chown-staff-group", "")
try:
stats = fs.stats("/chown-staff-group")
# Figure out a group id that is different from the one it already has
cur_gid = grp.getgrnam(stats["group"]).gr_gid
other_groups = [gid for gid in os.getgroups() if gid != cur_gid]
new_gid = other_groups[0]
fs.chown("/chown-staff-group", fs.stats("/chown-staff-group")["user"],
grp.getgrgid(new_gid).gr_name)
except OSError:
logger.exception("Ignoring error.")
return fs
