def __init__(self, fs, path, mode="w", block_size=None):
self.fs = fs
self.closed = False
assert mode == "w"
extra_confs = []
if block_size:
extra_confs.append("-Ddfs.block.size=%d" % block_size)
self.subprocess_cmd = [self.fs.hadoop_bin_path,
"jar",
hadoop.conf.SUDO_SHELL_JAR.get(),
self.fs.user,
"-Dfs.default.name=" + self.fs.uri] + \
extra_confs + \
["-put", "-", encode_fs_path(path)]
self.subprocess_env = i18n.make_utf8_env()
if self.subprocess_env.has_key('HADOOP_CLASSPATH'):
self.subprocess_env['HADOOP_CLASSPATH'] += ':' + hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()
else:
self.subprocess_env['HADOOP_CLASSPATH'] = hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()
if hadoop.conf.HADOOP_CONF_DIR.get():
self.subprocess_env['HADOOP_CONF_DIR'] = hadoop.conf.HADOOP_CONF_DIR.get()
self.path = path
self.putter = subprocess.Popen(self.subprocess_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
env=self.subprocess_env,
bufsize=WRITE_BUFFER_SIZE)
def __init__(self, fs, path, mode="w", block_size=None):
self.fs = fs
self.closed = False
assert mode == "w"
extra_confs = []
if block_size:
extra_confs.append("-Ddfs.block.size=%d" % block_size)
self.subprocess_cmd = [self.fs.hadoop_bin_path,
"jar",
hadoop.conf.SUDO_SHELL_JAR.get(),
self.fs.user,
"-Dfs.default.name=" + self.fs.uri] + \
extra_confs + \
["-put", "-", encode_fs_path(path)]
self.subprocess_env = i18n.make_utf8_env()
if self.subprocess_env.has_key('HADOOP_CLASSPATH'):
self.subprocess_env['HADOOP_CLASSPATH'] += ':' + hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()
else:
self.subprocess_env['HADOOP_CLASSPATH'] = hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()
if hadoop.conf.HADOOP_CONF_DIR.get():
self.subprocess_env['HADOOP_CONF_DIR'] = hadoop.conf.HADOOP_CONF_DIR.get()
self.path = path
self.putter = subprocess.Popen(self.subprocess_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
env=self.subprocess_env,
bufsize=WRITE_BUFFER_SIZE)
def __init__(self, fs, path, mode="w", block_size=None):
self.fs = fs
self.closed = False
assert mode == "w"
extra_confs = []
if block_size:
extra_confs.append("-Ddfs.block.size=%d" % block_size)
self.subprocess_cmd = [self.fs.hadoop_bin_path,
"dfs",
"-Dfs.default.name=" + self.fs._get_hdfs_base(),
"-Dhadoop.job.ugi=" + self.fs.ugi] + \
extra_confs + \
["-put", "-", encode_fs_path(path)]
self.path = path
self.putter = subprocess.Popen(self.subprocess_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
env=i18n.make_utf8_env(),
bufsize=WRITE_BUFFER_SIZE)