def renames(old, new):
"""renames(old, new)
Super-rename; create directories as necessary and delete any left
empty. Works like rename, except creation of any intermediate
directories needed to make the new pathname good is attempted
first. After the rename, directories corresponding to rightmost
path segments of the old name will be pruned way until either the
whole path is consumed or a nonempty directory is found.
Note: this function can fail with the new directory structure made
if you lack permissions needed to unlink the leaf directory or
file.
"""
head, tail = path.split(new)
if head and tail and not path.exists(head):
makedirs(head)
rename(old, new)
head, tail = path.split(old)
if head and tail:
try:
removedirs(head)
except error:
pass
def load(self, path):
if not os.path.isfile(path):
subject = path.split('/')[3]
key = path.split('/MNINonLinear/')[1]
temp = key.split('/', 1)
key = temp[0] + '/' + subject + '/' + temp[1]
url = self.base_path + key
r = requests.get(url)
if r.status_code == 200:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'wb') as f:
f.write(r.content)
def move(src, dest, user=None):
"""
Move or rename src to dest.
"""
src_host, src_port, src_path = path.split(src, user)
dest_host, dest_port, dest_path = path.split(dest, user)
src_fs = hdfs(src_host, src_port, user)
dest_fs = hdfs(dest_host, dest_port, user)
try:
retval = src_fs.move(src_path, dest_fs, dest_path)
return retval
finally:
src_fs.close()
dest_fs.close()
def load(self, path):
if not os.path.isfile(path):
subject = path.split('/')[3]
key = path.split('/MNINonLinear/')[1]
url = self.settings['DIRECTORIES']['HCPDir'].format(
subject, subject, subject) + key
r = requests.get(url,
auth=(self.settings['CREDENTIALS']['Username'],
self.settings['CREDENTIALS']['Password']),
stream=True)
if r.status_code == 200:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'wb') as f:
f.write(r.content)
def move(src, dest, user=None):
"""
Move or rename src to dest.
"""
src_host, src_port, src_path = path.split(src, user)
dest_host, dest_port, dest_path = path.split(dest, user)
src_fs = hdfs(src_host, src_port, user)
dest_fs = hdfs(dest_host, dest_port, user)
try:
retval = src_fs.move(src_path, dest_fs, dest_path)
return retval
finally:
src_fs.close()
dest_fs.close()
def _execvpe(file, args, env=None):
if env is not None:
func = execve
argrest = (args, env)
else:
func = execv
argrest = (args,)
env = environ
head, tail = path.split(file)
if head:
func(file, *argrest)
return
if 'PATH' in env:
envpath = env['PATH']
else:
envpath = defpath
PATH = envpath.split(pathsep)
saved_exc = None
saved_tb = None
for dir in PATH:
fullname = path.join(dir, file)
try:
func(fullname, *argrest)
except error, e:
tb = sys.exc_info()[2]
if (e.errno != errno.ENOENT and e.errno != errno.ENOTDIR
and saved_exc is None):
saved_exc = e
saved_tb = tb
def inverse_key_fn(path):
subpaths = path.split('/')
if len(subpaths) == 2:
fn = subpaths[-1]
if len(fn) > 7 and fn[-7:] == '.binvox':
return fn[:-7]
return None
def inverse_key_fn(path):
subpaths = path.split('/')
if len(subpaths) == 3:
i = subpaths[1]
p0 = self.get_example_image_subpath(cat_id, i, view_angle)
if path == p0:
return i
return None
def getParts(path):
"""
Return the slash-separated parts of a given path as a list
"""
if path == os.sep:
return [os.sep]
else:
return path.split(os.sep)
def normalize_domain(path):
"""
"""
if path.startswith('\\\\'):
parts = path.split('\\')
parts[2] = parts[2].lower()
path = '\\'.join(parts)
return path
def rmr(hdfs_path, user=None):
"""
Recursively remove files and directories.
"""
host, port, path_ = path.split(hdfs_path, user)
fs = hdfs(host, port, user)
retval = fs.delete(path_)
fs.close()
return retval
def rmr(hdfs_path, user=None): """ Recursively remove files and directories. """ host, port, path_ = path.split(hdfs_path, user) fs = hdfs(host, port, user) retval = fs.delete(path_) fs.close() return retval
def mkdir(hdfs_path, user=None): """ Create a directory and its parents as needed. """ host, port, path_ = path.split(hdfs_path, user) fs = hdfs(host, port, user) retval = fs.create_directory(path_) fs.close() return retval
def mkdir(hdfs_path, user=None):
"""
Create a directory and its parents as needed.
"""
host, port, path_ = path.split(hdfs_path, user)
fs = hdfs(host, port, user)
retval = fs.create_directory(path_)
fs.close()
return retval
def path_to_command(path):
splited_path = path.split('/')
if (len(splited_path) < 3):
return
i = 0
launch_index = 0
while (i < len(splited_path)):
if (splited_path[i] == 'launch'):
launch_index = i
i += 1
return splited_path[launch_index - 1] + ' ' + splited_path[launch_index + 1]
def cp(src_hdfs_path, dest_hdfs_path, **kwargs):
"""
Copy the contents of ``src_hdfs_path`` to ``dest_hdfs_path``.
Additional keyword arguments, if any, are handled like in
:func:`open`. If ``src_hdfs_path`` is a directory, its contents
will be copied recursively.
"""
src, dest = {}, {}
try:
for d, p in ((src, src_hdfs_path), (dest, dest_hdfs_path)):
d["host"], d["port"], d["path"] = path.split(p)
d["fs"] = hdfs(d["host"], d["port"])
#--- does src exist? ---
try:
src["info"] = src["fs"].get_path_info(src["path"])
except IOError:
raise IOError("no such file or directory: %r" % (src["path"]))
#--- src exists. Does dest exist? ---
try:
dest["info"] = dest["fs"].get_path_info(dest["path"])
except IOError:
if src["info"]["kind"] == "file":
_cp_file(src["fs"], src["path"], dest["fs"], dest["path"],
**kwargs)
return
else:
dest["fs"].create_directory(dest["path"])
dest_hdfs_path = dest["fs"].get_path_info(dest["path"])["name"]
for item in src["fs"].list_directory(src["path"]):
cp(item["name"], dest_hdfs_path, **kwargs)
return
#--- dest exists. Is it a file? ---
if dest["info"]["kind"] == "file":
raise IOError("%r already exists" % (dest["path"]))
#--- dest is a directory ---
dest["path"] = path.join(dest["path"], path.basename(src["path"]))
if dest["fs"].exists(dest["path"]):
raise IOError("%r already exists" % (dest["path"]))
if src["info"]["kind"] == "file":
_cp_file(src["fs"], src["path"], dest["fs"], dest["path"],
**kwargs)
else:
dest["fs"].create_directory(dest["path"])
dest_hdfs_path = dest["fs"].get_path_info(dest["path"])["name"]
for item in src["fs"].list_directory(src["path"]):
cp(item["name"], dest_hdfs_path, **kwargs)
finally:
for d in src, dest:
try:
d["fs"].close()
except KeyError:
pass
def makedirs(name, mode=0777):
"""makedirs(path [, mode=0777])
Super-mkdir; create a leaf directory and all intermediate ones.
Works like mkdir, except that any intermediate path segment (not
just the rightmost) will be created if it does not exist. This is
recursive.
"""
head, tail = path.split(name)
if not tail:
head, tail = path.split(head)
if head and tail and not path.exists(head):
try:
makedirs(head, mode)
except OSError, e:
# be happy if someone already created the path
if e.errno != errno.EEXIST:
raise
if tail == curdir: # xxx/newdir/. exists if xxx/newdir exists
return
def removedirs(name):
"""removedirs(path)
Super-rmdir; remove a leaf directory and all empty intermediate
ones. Works like rmdir except that, if the leaf directory is
successfully removed, directories corresponding to rightmost path
segments will be pruned away until either the whole path is
consumed or an error occurs. Errors during this latter phase are
ignored -- they generally mean that a directory was not empty.
"""
rmdir(name)
head, tail = path.split(name)
if not tail:
head, tail = path.split(head)
while head and tail:
try:
rmdir(head)
except error:
break
head, tail = path.split(head)
def lsl(hdfs_path, user=None): """ Return a list of dictionaries of file properties. If ``hdfs_path`` is a directory, each list item corresponds to a file or directory contained by it; if it is a file, there is only one item corresponding to the file itself. """ host, port, path_ = path.split(hdfs_path, user) fs = hdfs(host, port, user) dir_list = fs.list_directory(path_) fs.close() return dir_list
def open(hdfs_path, mode="r", buff_size=0, replication=0, blocksize=0,
readline_chunk_size=common.BUFSIZE, user=None):
"""
Open a file, returning an :class:`hdfs_file` object.
``hdfs_path`` and ``user`` are passed to
:func:`~path.split`, while the other args are
passed to the :class:`hdfs_file` constructor.
"""
host, port, path_ = path.split(hdfs_path, user)
fs = hdfs(host, port, user)
return fs.open_file(path_, mode, buff_size, replication, blocksize,
readline_chunk_size)
def cp(src_hdfs_path, dest_hdfs_path, **kwargs):
"""
Copy the contents of ``src_hdfs_path`` to ``dest_hdfs_path``.
Additional keyword arguments, if any, are handled like in
:func:`open`. If ``src_hdfs_path`` is a directory, its contents
will be copied recursively.
"""
src, dest = {}, {}
try:
for d, p in ((src, src_hdfs_path), (dest, dest_hdfs_path)):
d["host"], d["port"], d["path"] = path.split(p)
d["fs"] = hdfs(d["host"], d["port"])
#--- does src exist? ---
try:
src["info"] = src["fs"].get_path_info(src["path"])
except IOError:
raise IOError("no such file or directory: %r" % (src["path"]))
#--- src exists. Does dest exist? ---
try:
dest["info"] = dest["fs"].get_path_info(dest["path"])
except IOError:
if src["info"]["kind"] == "file":
_cp_file(src["fs"], src["path"], dest["fs"], dest["path"], **kwargs)
return
else:
dest["fs"].create_directory(dest["path"])
dest_hdfs_path = dest["fs"].get_path_info(dest["path"])["name"]
for item in src["fs"].list_directory(src["path"]):
cp(item["name"], dest_hdfs_path, **kwargs)
return
#--- dest exists. Is it a file? ---
if dest["info"]["kind"] == "file":
raise IOError("%r already exists" % (dest["path"]))
#--- dest is a directory ---
dest["path"] = path.join(dest["path"], path.basename(src["path"]))
if dest["fs"].exists(dest["path"]):
raise IOError("%r already exists" % (dest["path"]))
if src["info"]["kind"] == "file":
_cp_file(src["fs"], src["path"], dest["fs"], dest["path"], **kwargs)
else:
dest["fs"].create_directory(dest["path"])
dest_hdfs_path = dest["fs"].get_path_info(dest["path"])["name"]
for item in src["fs"].list_directory(src["path"]):
cp(item["name"], dest_hdfs_path, **kwargs)
finally:
for d in src, dest:
try:
d["fs"].close()
except KeyError:
pass
def chmod(hdfs_path, mode, user=None): """ Change file mode bits. :type path: string :param path: the path to the file or directory :type mode: int :param mode: the bitmask to set it to (e.g., 0777) """ host, port, path_ = path.split(hdfs_path, user) fs = hdfs(host, port, user) retval = fs.chmod(path_, mode) fs.close() return retval
def chmod(hdfs_path, mode, user=None):
"""
Change file mode bits.
:type path: string
:param path: the path to the file or directory
:type mode: int
:param mode: the bitmask to set it to (e.g., 0777)
"""
host, port, path_ = path.split(hdfs_path, user)
fs = hdfs(host, port, user)
retval = fs.chmod(path_, mode)
fs.close()
return retval
def path_to_rosrun_command(path):
splited_path = path.split('/')
if (len(splited_path) < 3): return
i = 0
result = ''
while (i < len(splited_path)):
if (splited_path[i] == 'scripts'):
result = splited_path[i - 1] + ' ' + splited_path[i + 1]
elif (splited_path[i] == 'lib'):
# To deal with ~/catkin_ws/devel/lib/some_lib.so
if (i + 2 >= len(splited_path)): break
result = splited_path[i + 1] + ' ' + splited_path[i + 2]
i += 1
return result
def open(hdfs_path,
mode="r",
buff_size=0,
replication=0,
blocksize=0,
readline_chunk_size=common.BUFSIZE,
user=None):
"""
Open a file, returning an :class:`hdfs_file` object.
``hdfs_path`` and ``user`` are passed to
:func:`~path.split`, while the other args are
passed to the :class:`hdfs_file` constructor.
"""
host, port, path_ = path.split(hdfs_path, user)
fs = hdfs(host, port, user)
return fs.open_file(path_, mode, buff_size, replication, blocksize,
readline_chunk_size)
def lsl(hdfs_path, user=None, recursive=False):
"""
Return a list of dictionaries of file properties.
If ``hdfs_path`` is a file, there is only one item corresponding to
the file itself; if it is a directory and ``recursive`` is
:obj:`False`, each list item corresponds to a file or directory
contained by it; if it is a directory and ``recursive`` is
:obj:`True`, the list contains one item for every file or directory
in the tree rooted at ``hdfs_path``.
"""
host, port, path_ = path.split(hdfs_path, user)
fs = hdfs(host, port, user)
if not recursive:
dir_list = fs.list_directory(path_)
else:
treewalk = fs.walk(path_)
top = treewalk.next()
if top['kind'] == 'directory':
dir_list = list(treewalk)
else:
dir_list = [top]
fs.close()
return dir_list
def lsl(hdfs_path, user=None, recursive=False):
"""
Return a list of dictionaries of file properties.
If ``hdfs_path`` is a file, there is only one item corresponding to
the file itself; if it is a directory and ``recursive`` is
:obj:`False`, each list item corresponds to a file or directory
contained by it; if it is a directory and ``recursive`` is
:obj:`True`, the list contains one item for every file or directory
in the tree rooted at ``hdfs_path``.
"""
host, port, path_ = path.split(hdfs_path, user)
fs = hdfs(host, port, user)
if not recursive:
dir_list = fs.list_directory(path_)
else:
treewalk = fs.walk(path_)
top = treewalk.next()
if top['kind'] == 'directory':
dir_list = list(treewalk)
else:
dir_list = [top]
fs.close()
return dir_list
__package__ = 'os'
import path
print path.join(*path.split('hello/world'))
__package__ = 'os.path'
from . import join
from ..path import split
print join(*split('hello/world'))
from .. import path
print path.join(*path.split('hello/world'))
def glob(pathname):
if not has_magic(pathname): return [pathname]
dirname, basename = path.split(pathname)
if dirname[-1:] = '/' and dirname <> '/':
dirname = dirname[:-1]
