def handle_no_opts(*files):
result = None
if len(files) == 1:
if not common.isexist(f):
# 如果该路径不存在，exception
raise Exception('ls: {0}: No such file or directory'.format(f))
if common.isdir(f):
result = os.listdir(f)
else:
#如果该路径不是目录，则直接添加到result中
result = [f]
else:
result = []
for f in files:
if not common.isexist(f):
# 如果该路径不存在，exception
raise Exception('ls: {0}: No such file or directory'.format(f))
if common.isdir(f):
result.append(os.listdir(f))
else:
#如果该路径不是目录，则直接添加到result中
result.append([f])
return result
def stagedir(pathname):
if isdir(pathname):
data = None
path.walk(pathname, stageinvisitor, data)
else:
stcmd = "stagein -A deferred --nowait --rdonly "
stcmd += " -p %s " % stagepool
stcmd += " -M " + path.abspath(pathname)
print stcmd
os.system(stcmd)
def main():
fs = hdfs.hdfs()
try:
root = "%s/%s" % (fs.working_directory(), TEST_ROOT)
if not isdir(fs, root):
sys.exit("%r does not exist" % root)
print "BS(MB)\tBYTES"
for k, v in usage_by_bs(fs, root).iteritems():
print "%.1f\t%d" % (k / float(MB), v)
finally:
fs.close()
def main():
fs = hdfs.hdfs()
try:
root = "%s/%s" % (fs.working_directory(), TEST_ROOT)
if not isdir(fs, root):
sys.exit("%r does not exist" % root)
print "BS(MB)\tBYTES"
for k, v in usage_by_bs(fs, root).iteritems():
print "%.1f\t%d" % (k / float(MB), v)
finally:
fs.close()
def getdirsize(pathname=None, verbose=False, printfunc=humanPrint):
size = 0
if isdir(pathname):
for child in getchildren(pathname):
if (pathname): child = pathname + "/" + child
size += getdirsize(child, verbose, printfunc)
else:
size = getsize(pathname)
if verbose:
if pathname:
printfunc(pathname, size)
else:
printfunc(environ["CASTOR_HOME"], size)
return size
def treegen(fs, root, depth, span):
if isdir(fs, root) and depth > 0:
for i in xrange(span):
path = "%s/%d_%d" % (root, depth, i)
kind = 'file' if i else 'directory'
if kind == 'file':
bs = random.sample(BS_RANGE, 1)[0]
sys.stderr.write("%s %s %d\n" % (kind[0].upper(), path, (bs/MB)))
with fs.open_file(path, "w", blocksize=bs) as f:
f.write(path)
else:
sys.stderr.write("%s %s 0\n" % (kind[0].upper(), path))
fs.create_directory(path)
treegen(fs, path, depth-1, span)
def treegen(fs, root, depth, span):
if isdir(fs, root) and depth > 0:
for i in xrange(span):
path = "%s/%d_%d" % (root, depth, i)
kind = 'file' if i else 'directory'
if kind == 'file':
kwargs = {}
if pydoop.hadoop_version_info().has_deprecated_bs():
bs = hdfs.fs.hdfs().default_block_size()
else:
bs = random.sample(BS_RANGE, 1)[0]
kwargs['blocksize'] = bs
sys.stderr.write("%s %s %d\n" % (kind[0].upper(), path, (bs/MB)))
with fs.open_file(path, "w", **kwargs) as f:
f.write(path)
else:
sys.stderr.write("%s %s 0\n" % (kind[0].upper(), path))
fs.create_directory(path)
treegen(fs, path, depth-1, span)