def pathscrub(dirty_path, os=None):
"""
Strips illegal characters for a given os from a path.
:param dirty_path: Path to be scrubbed.
:param os: Defines which os mode should be used, can be 'windows', 'mac', 'linux', or None to auto-detect
:return: A valid path.
"""
# See if global os_mode has been defined by pathscrub plugin
if os_mode and not os:
os = os_mode
if os:
# If os is defined, use replacements for that os
replace_map = replace_maps[os_mode]
else:
# If os is not defined, try to detect appropriate
drive, path = ntpath.splitdrive(dirty_path)
if sys.platform.startswith('win') or drive:
replace_map = replace_maps['windows']
elif sys.platform.startswith('darwin'):
replace_map = replace_maps['mac']
else:
replace_map = replace_maps['linux']
# Make sure not to mess with windows drive specifications
drive, path = ntpath.splitdrive(dirty_path)
for search, replace in replace_map.iteritems():
path = re.sub(search, replace, path)
return drive + path.strip()
def scrub(self, dirty_path):
"""
Strips illegal characters for a given os from a path.
:param dirty_path: Path to be scrubbed.
:return: A valid path.
"""
if self.config:
# If os is defined, use replacements for that os
replace_map = self.replace_maps[self.config]
else:
# If os is not defined, try to detect appropriate
drive, path = ntpath.splitdrive(dirty_path)
if sys.platform.startswith('win') or drive:
replace_map = self.replace_maps['windows']
elif sys.platform.startswith('darwin'):
replace_map = self.replace_maps['mac']
else:
replace_map = self.replace_maps['linux']
# Make sure not to mess with windows drive specifications
drive, path = ntpath.splitdrive(dirty_path)
for search, replace in replace_map.iteritems():
path = re.sub(search, replace, path)
return drive + path.strip()
def relntpath(path, start):
import ntpath # the windows version of os.path #available in python 2.6
# return ntpath.relpath(path, start)
if start == None:
start = os.getcwd()
path = ntpath.normpath(path)
start = ntpath.normpath(start)
(drivep, tailp) = ntpath.splitdrive(path)
(drives, tails) = ntpath.splitdrive(start)
# if one of the paths has no drive letter, treat both of them so
if drivep == "" or drives == "":
path = tailp
start = tails
elif drivep != drives:
# ntpath.relpath returns error if drive letters differ, but we wont
return path
pathl = path.replace("/", "\\").split("\\")
startl = start.replace("/", "\\").split("\\")
# print "path: %s, start:%s"%(path, start )
while len(pathl) and len(startl) and pathl[0] == startl[0]:
# print "removing "+pathl[0]
del pathl[0]
del startl[0]
for i in range(len(startl)):
pathl.insert(0, "..")
return ntpath.join(".", *pathl)
def test_nt_helpers(self):
# Trivial validation that the helpers do not break, and support both
# unicode and bytes (UTF-8) paths
executable = nt._getfinalpathname(sys.executable)
for path in executable, os.fsencode(executable):
volume_path = nt._getvolumepathname(path)
path_drive = ntpath.splitdrive(path)[0]
volume_path_drive = ntpath.splitdrive(volume_path)[0]
self.assertEqualCI(path_drive, volume_path_drive)
cap, free = nt._getdiskusage(sys.exec_prefix)
self.assertGreater(cap, 0)
self.assertGreater(free, 0)
b_cap, b_free = nt._getdiskusage(sys.exec_prefix.encode())
# Free space may change, so only test the capacity is equal
self.assertEqual(b_cap, cap)
self.assertGreater(b_free, 0)
for path in [sys.prefix, sys.executable]:
final_path = nt._getfinalpathname(path)
self.assertIsInstance(final_path, str)
self.assertGreater(len(final_path), 0)
b_final_path = nt._getfinalpathname(path.encode())
self.assertIsInstance(b_final_path, bytes)
self.assertGreater(len(b_final_path), 0)
def nt_commonpath(paths): # pylint: disable=too-many-locals
"""Given a sequence of NT path names,
return the longest common sub-path."""
from ntpath import splitdrive
if not paths:
raise ValueError('commonpath() arg is an empty sequence')
check_arg_types('commonpath', *paths)
if isinstance(paths[0], bytes):
sep = b'\\'
altsep = b'/'
curdir = b'.'
else:
sep = '\\'
altsep = '/'
curdir = '.'
drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths]
split_paths = [p.split(sep) for d, p in drivesplits]
try:
isabs, = set(p[:1] == sep for d, p in drivesplits)
except ValueError:
raise ValueError("Can't mix absolute and relative paths")
# Check that all drive letters or UNC paths match. The check is made
# only now otherwise type errors for mixing strings and bytes would not
# be caught.
if len(set(d for d, p in drivesplits)) != 1:
raise ValueError("Paths don't have the same drive")
drive, path = splitdrive(paths[0].replace(altsep, sep))
common = path.split(sep)
common = [c for c in common if c and c != curdir]
split_paths = [[c for c in s if c and c != curdir] for s in split_paths]
s_min = min(split_paths)
s_max = max(split_paths)
for i, run_c in enumerate(s_min):
if run_c != s_max[i]:
common = common[:i]
break
else:
common = common[:len(s_min)]
prefix = drive + sep if isabs else drive
return prefix + sep.join(common)
def commonpath(paths):
"""Given a sequence of path names, returns the longest common sub-path."""
if not paths:
raise ValueError('commonpath() arg is an empty sequence')
if isinstance(paths[0], bytes):
sep = b'\\'
altsep = b'/'
curdir = b'.'
else:
sep = '\\'
altsep = '/'
curdir = '.'
try:
drivesplits = [ntpath.splitdrive(p.replace(altsep, sep).lower()) for p in paths]
split_paths = [p.split(sep) for d, p in drivesplits]
try:
isabs, = set(p[:1] == sep for d, p in drivesplits)
except ValueError:
raise ValueError("Can't mix absolute and relative paths")
# Check that all drive letters or UNC paths match. The check is made only
# now otherwise type errors for mixing strings and bytes would not be
# caught.
if len(set(d for d, p in drivesplits)) != 1:
raise ValueError("Paths don't have the same drive")
drive, path = ntpath.splitdrive(paths[0].replace(altsep, sep))
common = path.split(sep)
common = [c for c in common if c and c != curdir]
split_paths = [[c for c in s if c and c != curdir] for s in split_paths]
s1 = min(split_paths)
s2 = max(split_paths)
for i, c in enumerate(s1):
if c != s2[i]:
common = common[:i]
break
else:
common = common[:len(s1)]
prefix = drive + sep if isabs else drive
return prefix + sep.join(common)
except (TypeError, AttributeError):
genericpath._check_arg_types('commonpath', *paths)
raise
def _host_volume_from_bind(bind):
drive, rest = ntpath.splitdrive(bind)
bits = rest.split(':', 1)
if len(bits) == 1 or bits[1] in ('ro', 'rw'):
return drive + bits[0]
else:
return bits[1].rstrip(':ro').rstrip(':rw')
def translate_path_linux(p):
def linux_path_from_unc_path(unc_share, sub_path):
mount = mount_drives.MountPoint(unc_share)
unix_mountpoint = unicode(mount.mountpoint)
if len(sub_path) == 0:
ret = unix_mountpoint
else:
# Remove leading \ in the path part
while len(sub_path) > 0 and sub_path[0] == u'\\':
sub_path = sub_path[1:]
ret = posixpath.join(unix_mountpoint,
sub_path.replace(u'\\', u'/'))
return ret
# UNC path ?
if p.startswith(u'\\\\'):
unc, path = ntpath.splitunc(p)
return linux_path_from_unc_path(unc, path)
# Windows path with drive ?
drive, path = ntpath.splitdrive(p)
if drive != '':
try:
unc = unicode(settings.drives_mapping[drive.upper()])
except KeyError:
pass
else:
return linux_path_from_unc_path(unc, path)
return p
def MakeZipPath(os_path, isdir, iswindows):
"""Changes a path into zipfile format.
# doctest doesn't seem to honor r'' strings, so the backslashes need to be
# escaped.
>>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
'users/foobar/blah'
>>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
'tmp/tmpfoobar/something'
>>> MakeZipPath('./somefile.txt', False, False)
'somefile.txt'
>>> MakeZipPath('somedir', True, False)
'somedir/'
>>> MakeZipPath('../dir/filename.txt', False, False)
'../dir/filename.txt'
>>> MakeZipPath('dir/../filename.txt', False, False)
'filename.txt'
"""
zip_path = os_path
if iswindows:
import ntpath
# zipfile paths are always posix-style. They also have the drive
# letter and leading slashes removed.
zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
if zip_path.startswith('/'):
zip_path = zip_path[1:]
zip_path = posixpath.normpath(zip_path)
# zipfile also always appends a slash to a directory name.
if isdir:
zip_path += '/'
return zip_path
def ismount(path):
"""Test whether a path is a mount point (defined as root of drive)"""
unc, rest = splitunc(path)
if unc:
return rest in ("", "/", "\\")
p = splitdrive(path)[1]
return len(p) == 1 and p[0] in '/\\'
def MakeZipPath(os_path, isdir, iswindows):
"""Changes a path into zipfile format.
# doctest doesn't seem to honor r'' strings, so the backslashes need to be
# escaped.
>>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
'users/foobar/blah'
>>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
'tmp/tmpfoobar/something'
>>> MakeZipPath('./somefile.txt', False, False)
'somefile.txt'
>>> MakeZipPath('somedir', True, False)
'somedir/'
>>> MakeZipPath('../dir/filename.txt', False, False)
'../dir/filename.txt'
>>> MakeZipPath('dir/../filename.txt', False, False)
'filename.txt'
"""
zip_path = os_path
if iswindows:
import ntpath
# zipfile paths are always posix-style. They also have the drive
# letter and leading slashes removed.
zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
if zip_path.startswith('/'):
zip_path = zip_path[1:]
zip_path = posixpath.normpath(zip_path)
# zipfile also always appends a slash to a directory name.
if isdir:
zip_path += '/'
return zip_path
def split(self, fullPath):
if self.pathSep == '/':
path, filename = posixpath.split(fullPath)
result = {
'fullPath': fullPath,
'directory': path,
'file': filename
}
else:
drive, path = ntpath.splitdrive(fullPath)
path, filename = ntpath.split(path)
result = {
'fullPath': fullPath,
'drive': drive,
'directory': ntpath.join(drive, path),
'file': filename
}
dirSplit = path.split(self.pathSep)
if len(dirSplit) > 2:
lastTwoSubdir = dirSplit[-2:]
headSubdir = dirSplit[:len(dirSplit) - 2]
result['root'] = self.pathSep.join(headSubdir)
result['subdir'] = self.pathSep.join(lastTwoSubdir)
return result
def test_ismount(self):
self.assertTrue(ntpath.ismount("c:\\"))
self.assertTrue(ntpath.ismount("C:\\"))
self.assertTrue(ntpath.ismount("c:/"))
self.assertTrue(ntpath.ismount("C:/"))
self.assertTrue(ntpath.ismount("\\\\.\\c:\\"))
self.assertTrue(ntpath.ismount("\\\\.\\C:\\"))
self.assertTrue(ntpath.ismount(b"c:\\"))
self.assertTrue(ntpath.ismount(b"C:\\"))
self.assertTrue(ntpath.ismount(b"c:/"))
self.assertTrue(ntpath.ismount(b"C:/"))
self.assertTrue(ntpath.ismount(b"\\\\.\\c:\\"))
self.assertTrue(ntpath.ismount(b"\\\\.\\C:\\"))
with os_helper.temp_dir() as d:
self.assertFalse(ntpath.ismount(d))
if sys.platform == "win32":
#
# Make sure the current folder isn't the root folder
# (or any other volume root). The drive-relative
# locations below cannot then refer to mount points
#
test_cwd = os.getenv("SystemRoot")
drive, path = ntpath.splitdrive(test_cwd)
with os_helper.change_cwd(test_cwd):
self.assertFalse(ntpath.ismount(drive.lower()))
self.assertFalse(ntpath.ismount(drive.upper()))
self.assertTrue(ntpath.ismount("\\\\localhost\\c$"))
self.assertTrue(ntpath.ismount("\\\\localhost\\c$\\"))
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$"))
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\"))
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
remote_path = remote_path or '~/'
# In order to expand "~" and "~user" we use ntpath.expanduser(),
# but it relies on environment variables being set. This modifies
# the HOME environment variable in order to use that function, and then
# restores it to its previous value.
home = os.environ.get('HOME')
try:
os.environ['HOME'] = self.home_dir
remote_path = ntpath.expanduser(remote_path)
finally:
if home is None:
del os.environ['HOME']
else:
os.environ['HOME'] = home
drive, remote_path = ntpath.splitdrive(remote_path)
remote_drive = (drive or self.system_drive).rstrip(':')
network_drive = '\\\\%s\\%s$' % (self.GetConnectionIp(), remote_drive)
if vm_util.RunningOnWindows():
self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive)
else:
self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def mkdir_p(self, path):
"""Provides mkdir -p type functionality on the remote host. That is,
all missing parent directories are also created. If the directory we are trying to
create already exists, we silently do nothing. If path or any of its parents is not
a directory an exception is raised.
path - directory to create on remote host
"""
_logger.debug('mkdir_p(' + path + ')')
path = self._sftpify(path)
pa = self.file_exists(path)
if pa != None:
#print str(pa)+" "+str(pa.st_mode)
if not util.is_dir(pa):
raise Exception(self.host + ':' + path + ' is not a directory')
return
# Need to user normpath here since dirname of a directory with a trailing slash
# is the directory without a slash (a dirname bug?)
sd = ntpath.splitdrive(path)
_logger.debug('sd=' + str(sd))
if sd[1] == '':
_logger.debug('path=' + path + ' is a drive letter. Returning...')
return
np = self.path_module.normpath(path)
parent = self.path_module.dirname(np)
assert parent != path
self.mkdir_p(parent)
self.sftp.mkdir(np)
def _host_volume_from_bind(bind):
drive, rest = ntpath.splitdrive(bind)
bits = rest.split(':', 1)
if len(bits) == 1 or bits[1] in ('ro', 'rw'):
return drive + bits[0]
else:
return bits[1].rstrip(':ro').rstrip(':rw')
def test_ismount(self):
self.assertTrue(ntpath.ismount("c:\\"))
self.assertTrue(ntpath.ismount("C:\\"))
self.assertTrue(ntpath.ismount("c:/"))
self.assertTrue(ntpath.ismount("C:/"))
self.assertTrue(ntpath.ismount("\\\\.\\c:\\"))
self.assertTrue(ntpath.ismount("\\\\.\\C:\\"))
self.assertTrue(ntpath.ismount(b"c:\\"))
self.assertTrue(ntpath.ismount(b"C:\\"))
self.assertTrue(ntpath.ismount(b"c:/"))
self.assertTrue(ntpath.ismount(b"C:/"))
self.assertTrue(ntpath.ismount(b"\\\\.\\c:\\"))
self.assertTrue(ntpath.ismount(b"\\\\.\\C:\\"))
with support.temp_dir() as d:
self.assertFalse(ntpath.ismount(d))
if sys.platform == "win32":
#
# Make sure the current folder isn't the root folder
# (or any other volume root). The drive-relative
# locations below cannot then refer to mount points
#
drive, path = ntpath.splitdrive(sys.executable)
with support.change_cwd(os.path.dirname(sys.executable)):
self.assertFalse(ntpath.ismount(drive.lower()))
self.assertFalse(ntpath.ismount(drive.upper()))
self.assertTrue(ntpath.ismount("\\\\localhost\\c$"))
self.assertTrue(ntpath.ismount("\\\\localhost\\c$\\"))
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$"))
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\"))
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
remote_path = remote_path or '~/'
# In order to expand "~" and "~user" we use ntpath.expanduser(),
# but it relies on environment variables being set. This modifies
# the HOME environment variable in order to use that function, and then
# restores it to its previous value.
home = os.environ.get('HOME')
try:
os.environ['HOME'] = self.home_dir
remote_path = ntpath.expanduser(remote_path)
finally:
if home is None:
del os.environ['HOME']
else:
os.environ['HOME'] = home
drive, remote_path = ntpath.splitdrive(remote_path)
remote_drive = (drive or self.system_drive).rstrip(':')
network_drive = '\\\\%s\\%s$' % (self.ip_address, remote_drive)
if vm_util.RunningOnWindows():
self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive)
else:
self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def ismount(path):
"""Test whether a path is a mount point (defined as root of drive)"""
unc, rest = splitunc(path)
if unc:
return rest in ("", "/", "\\")
p = splitdrive(path)[1]
return len(p) == 1 and p[0] in '/\\'
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
remote_path = remote_path or '~/'
home = os.environ['HOME']
try:
os.environ['HOME'] = self.home_dir
remote_path = ntpath.expanduser(remote_path)
finally:
os.environ['HOME'] = home
drive, remote_path = ntpath.splitdrive(remote_path)
remote_drive = (drive or self.system_drive).rstrip(':')
network_drive = '\\\\%s\\%s$' % (self.ip_address, remote_drive)
if vm_util.RunningOnWindows():
self._PsDriveRemoteCopy(local_path, remote_path, copy_to,
network_drive)
else:
self._SmbclientRemoteCopy(local_path, remote_path, copy_to,
network_drive)
def test_ismount(self):
self.assertTrue(ntpath.ismount("c:\\"))
self.assertTrue(ntpath.ismount("C:\\"))
self.assertTrue(ntpath.ismount("c:/"))
self.assertTrue(ntpath.ismount("C:/"))
self.assertTrue(ntpath.ismount("\\\\.\\c:\\"))
self.assertTrue(ntpath.ismount("\\\\.\\C:\\"))
self.assertTrue(ntpath.ismount(b"c:\\"))
self.assertTrue(ntpath.ismount(b"C:\\"))
self.assertTrue(ntpath.ismount(b"c:/"))
self.assertTrue(ntpath.ismount(b"C:/"))
self.assertTrue(ntpath.ismount(b"\\\\.\\c:\\"))
self.assertTrue(ntpath.ismount(b"\\\\.\\C:\\"))
with support.temp_dir() as d:
self.assertFalse(ntpath.ismount(d))
if sys.platform == "win32":
#
# Make sure the current folder isn't the root folder
# (or any other volume root). The drive-relative
# locations below cannot then refer to mount points
#
drive, path = ntpath.splitdrive(sys.executable)
with support.change_cwd(os.path.dirname(sys.executable)):
self.assertFalse(ntpath.ismount(drive.lower()))
self.assertFalse(ntpath.ismount(drive.upper()))
self.assertTrue(ntpath.ismount("\\\\localhost\\c$"))
self.assertTrue(ntpath.ismount("\\\\localhost\\c$\\"))
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$"))
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\"))
def absolute_win_path(val=ntpath.join('C:', ntpath.sep)):
"""Check for an absolute windows file path."""
val = str(val)
if ntpath.isabs(val) and ntpath.splitdrive(val)[0]:
return val
raise ValueError('%r is not a valid absolute path (should start with drive'
' name like C:)' % val)
def mkdir_p(self, path):
"""Provides mkdir -p type functionality on the remote host. That is,
all missing parent directories are also created. If the directory we are trying to
create already exists, we silently do nothing. If path or any of its parents is not
a directory an exception is raised.
path - directory to create on remote host
"""
_logger.debug('mkdir_p('+path+')')
path = self._sftpify(path)
pa = self.file_exists(path)
if pa != None:
#print str(pa)+" "+str(pa.st_mode)
if not util.is_dir(pa):
raise Exception(self.host+':'+path+' is not a directory')
return
# Need to user normpath here since dirname of a directory with a trailing slash
# is the directory without a slash (a dirname bug?)
sd = ntpath.splitdrive(path)
_logger.debug('sd='+str(sd))
if sd[1] == '':
_logger.debug('path='+path+' is a drive letter. Returning...')
return
np = self.path_module.normpath(path)
parent = self.path_module.dirname(np)
assert parent != path
self.mkdir_p(parent)
self.sftp.mkdir(np)
def test_nt_helpers(self):
# Trivial validation that the helpers do not break, and support both
# unicode and bytes (UTF-8) paths
drive, path = ntpath.splitdrive(sys.executable)
drive = drive.rstrip(ntpath.sep) + ntpath.sep
self.assertEqual(drive, nt._getvolumepathname(sys.executable))
self.assertEqual(drive.encode(),
nt._getvolumepathname(sys.executable.encode()))
cap, free = nt._getdiskusage(sys.exec_prefix)
self.assertGreater(cap, 0)
self.assertGreater(free, 0)
b_cap, b_free = nt._getdiskusage(sys.exec_prefix.encode())
# Free space may change, so only test the capacity is equal
self.assertEqual(b_cap, cap)
self.assertGreater(b_free, 0)
for path in [sys.prefix, sys.executable]:
final_path = nt._getfinalpathname(path)
self.assertIsInstance(final_path, str)
self.assertGreater(len(final_path), 0)
b_final_path = nt._getfinalpathname(path.encode())
self.assertIsInstance(b_final_path, bytes)
self.assertGreater(len(b_final_path), 0)
def FindMediaTableIDFromUNCSpec(db_con, unc_spec_filename):
"""
Searches media_table for a file with the specified unc_spec_filename and returns the media_id of it (or None if not found)
:param db_con Open database connection to Adobe Elements 6.0 SQLite Catalog database file
:param unc_spec_filename: Full UNC filespec of file to locate [NB: use raw string literals to avoid slash problems]
:return: media_id of located file or None if not found.
"""
#
lc_base_filename = ntpath.normcase(unc_spec_filename)
drive, remainder = ntpath.splitdrive(lc_base_filename)
lc_base_filename = (ntpath.basename(remainder)).lower()
dir = str.replace(ntpath.dirname(remainder), '\\', '/')
if dir[-1] != '/':
# Add trailing forward slash
dir += '/'
with db_con:
cur = db_con.cursor()
# Find tag 'user_ns' first which is at the top of the treee
query = """
SELECT id
FROM media_table
WHERE filepath_search_index = "{}" AND filename_search_index = "{}";
""".format(dir, lc_base_filename)
cur.execute(query)
row = cur.fetchone()
if row:
media_id = row[0]
else:
media_id = None
return media_id
def is_admin_only_path(path):
"""
This is pretty naive, but seems to do the work for now...
"""
path = ntpath.splitdrive(path)[1].lower()
if path.startswith('\\windows') or path.startswith('\\programdata') or path.startswith('\\program files'):
return True
return False
def test_abspath(self):
tester('ntpath.abspath("C:\\")', "C:\\")
with support.temp_cwd(support.TESTFN) as cwd_dir: # bpo-31047
tester('ntpath.abspath("")', cwd_dir)
tester('ntpath.abspath(" ")', cwd_dir + "\\ ")
tester('ntpath.abspath("?")', cwd_dir + "\\?")
drive, _ = ntpath.splitdrive(cwd_dir)
tester('ntpath.abspath("/abc/")', drive + "\\abc")
def test_abspath(self):
tester('ntpath.abspath("C:\\")', "C:\\")
with support.temp_cwd(support.TESTFN) as cwd_dir: # bpo-31047
tester('ntpath.abspath("")', cwd_dir)
tester('ntpath.abspath(" ")', cwd_dir + "\\ ")
tester('ntpath.abspath("?")', cwd_dir + "\\?")
drive, _ = ntpath.splitdrive(cwd_dir)
tester('ntpath.abspath("/abc/")', drive + "\\abc")
def pathscrub(dirty_path: str,
os: Optional[str] = None,
filename: bool = False) -> str:
"""
Strips illegal characters for a given os from a path.
:param dirty_path: Path to be scrubbed.
:param os: Defines which os mode should be used, can be 'windows', 'mac', 'linux', or None to auto-detect
:param filename: If this is True, path separators will be replaced with '-'
:return: A valid path.
"""
# See if global os_mode has been defined by pathscrub plugin
if os_mode and not os:
os = os_mode
if not os:
# If os is not defined, try to detect appropriate
drive, path = ntpath.splitdrive(dirty_path)
if sys.platform.startswith('win') or drive:
os = 'windows'
elif sys.platform.startswith('darwin'):
os = 'mac'
else:
os = 'linux'
replaces = platform_replaces[os]
# Make sure not to mess with windows drive specifications
drive, path = ntpath.splitdrive(dirty_path)
if filename:
path = path.replace('/', ' ').replace('\\', ' ')
# Remove spaces surrounding path components
path = '/'.join(comp.strip() for comp in path.split('/'))
if os == 'windows':
path = '\\'.join(comp.strip() for comp in path.split('\\'))
for search, replace in replaces:
path = re.sub(search, replace, path)
path = path.strip()
# If we stripped everything from a filename, complain
if filename and dirty_path and not path:
raise ValueError(
'Nothing was left after stripping invalid characters from path `%s`!'
% dirty_path)
return drive + path
def replace_win32_incompat(string, repl="_"):
"""Replace win32 filename incompatible characters from ``string`` by
``repl``."""
# Don't replace : with _ for windows drive
if sys.platform == "win32" and os.path.isabs(string):
drive, rest = ntpath.splitdrive(string)
return drive + _re_win32_incompat.sub(repl, rest)
else:
return _re_win32_incompat.sub(repl, string)
def winpath(path):
drive, rest = ntpath.splitdrive(path)
result = []
while rest and rest != ntpath.sep:
rest, part = ntpath.split(rest)
result.insert(0, part)
if rest:
result.insert(0, rest)
return ntpath.join(drive.upper(), *result)
def replace_win32_incompat(string, repl=u"_"):
"""Replace win32 filename incompatible characters from ``string`` by
``repl``."""
# Don't replace : with _ for windows drive
if sys.platform == "win32" and os.path.isabs(string):
drive, rest = ntpath.splitdrive(string)
return drive + _re_win32_incompat.sub(repl, rest)
else:
return _re_win32_incompat.sub(repl, string)
def winpath(path):
drive, rest = ntpath.splitdrive(path)
result = []
while rest and rest != ntpath.sep:
rest, part = ntpath.split(rest)
result.insert(0, part)
if rest:
result.insert(0, rest)
return ntpath.join(drive.upper(), *result)
def cluster(files, threshold):
win_compat = config.setting["windows_compatibility"] or IS_WIN
artist_dict = ClusterDict()
album_dict = ClusterDict()
tracks = []
for file in files:
artist = file.metadata["albumartist"] or file.metadata["artist"]
album = file.metadata["album"]
# Improve clustering from directory structure if no existing tags
# Only used for grouping and to provide cluster title / artist - not added to file tags.
if win_compat:
filename = ntpath.splitdrive(file.filename)[1]
else:
filename = file.filename
album, artist = album_artist_from_path(filename, album, artist)
# For each track, record the index of the artist and album within the clusters
tracks.append((artist_dict.add(artist), album_dict.add(album)))
artist_cluster_engine = ClusterEngine(artist_dict)
artist_cluster_engine.cluster(threshold)
album_cluster_engine = ClusterEngine(album_dict)
album_cluster_engine.cluster(threshold)
# Arrange tracks into albums
albums = {}
for i, track in enumerate(tracks):
cluster = album_cluster_engine.get_cluster_from_id(track[1])
if cluster is not None:
albums.setdefault(cluster, []).append(i)
# Now determine the most prominent names in the cluster and build the
# final cluster list
for album_id, album in albums.items():
album_name = album_cluster_engine.get_cluster_title(album_id)
artist_max = 0
artist_id = None
artist_hist = {}
for track_id in album:
cluster = artist_cluster_engine.get_cluster_from_id(
tracks[track_id][0])
if cluster is not None:
cnt = artist_hist.get(cluster, 0) + 1
if cnt > artist_max:
artist_max = cnt
artist_id = cluster
artist_hist[cluster] = cnt
if artist_id is None:
artist_name = "Various Artists"
else:
artist_name = artist_cluster_engine.get_cluster_title(
artist_id)
yield album_name, artist_name, (files[i] for i in album)
def __normalize(cls, path, expand_user=False):
if expand_user:
path = os.path.expanduser(path)
drive, path = ntpath.splitdrive(path)
if drive and not ntpath.isabs(path):
raise ValueError('relative paths with drives not supported')
drive = drive.replace('\\', '/')
path, isdir = cls.__normpath(path)
return drive, path, isdir
def convert_path(p):
p = ntpath.realpath(p)
#TODO: not exactly save
if '~' in p:
p = non83_path(p)
drive,path = ntpath.splitdrive(p)
drive = drive[0].lower()
drive = '/mnt/' + drive
path = path.replace('\\', '/')
return drive + path
def pathscrub(dirty_path, os=None, filename=False):
"""
Strips illegal characters for a given os from a path.
:param dirty_path: Path to be scrubbed.
:param os: Defines which os mode should be used, can be 'windows', 'mac', 'linux', or None to auto-detect
:param filename: If this is True, path separators will be replaced with '-'
:return: A valid path.
"""
# See if global os_mode has been defined by pathscrub plugin
if os_mode and not os:
os = os_mode
if not os:
# If os is not defined, try to detect appropriate
drive, path = ntpath.splitdrive(dirty_path)
if sys.platform.startswith('win') or drive:
os = 'windows'
elif sys.platform.startswith('darwin'):
os = 'mac'
else:
os = 'linux'
replaces = platform_replaces[os]
# Make sure not to mess with windows drive specifications
drive, path = ntpath.splitdrive(dirty_path)
if filename:
path = path.replace('/', ' ').replace('\\', ' ')
# Remove spaces surrounding path components
path = '/'.join(comp.strip() for comp in path.split('/'))
if os == 'windows':
path = '\\'.join(comp.strip() for comp in path.split('\\'))
for search, replace in replaces:
path = re.sub(search, replace, path)
path = path.strip()
# If we stripped everything from a filename, complain
if filename and dirty_path and not path:
raise ValueError(
'Nothing was left after stripping invalid characters from path `%s`!' % dirty_path
)
return drive + path
def test_splitdrive(self):
tester('ntpath.splitdrive("c:\\foo\\bar")', ('c:', '\\foo\\bar'))
tester('ntpath.splitdrive("c:/foo/bar")', ('c:', '/foo/bar'))
tester('ntpath.splitdrive("\\\\conky\\mountpoint\\foo\\bar")',
('\\\\conky\\mountpoint', '\\foo\\bar'))
tester('ntpath.splitdrive("//conky/mountpoint/foo/bar")',
('//conky/mountpoint', '/foo/bar'))
tester('ntpath.splitdrive("\\\\\\conky\\mountpoint\\foo\\bar")',
('', '\\\\\\conky\\mountpoint\\foo\\bar'))
tester('ntpath.splitdrive("///conky/mountpoint/foo/bar")',
('', '///conky/mountpoint/foo/bar'))
tester('ntpath.splitdrive("\\\\conky\\\\mountpoint\\foo\\bar")',
('', '\\\\conky\\\\mountpoint\\foo\\bar'))
tester('ntpath.splitdrive("//conky//mountpoint/foo/bar")',
('', '//conky//mountpoint/foo/bar'))
# Issue #19911: UNC part containing U+0130
self.assertEqual(ntpath.splitdrive(u'//conky/MOUNTPOİNT/foo/bar'),
(u'//conky/MOUNTPOİNT', '/foo/bar'))
self.assertEqual(ntpath.splitdrive("//"), ("", "//"))
def __sanitize_path_win32(os_dep_path):
r"""Win32-specific implementation of sanitize_path().
>>> __sanitize_path_win32('C:\\Documents and Settings\\host\\Desktop\\.C:')
'C:\\Documents and Settings\\host\\Desktop\\.C_'
"""
drive, path = ntpath.splitdrive(os_dep_path)
# There should not be colons after the disk definition.
path = path.replace(':', '_')
return ntpath.join(drive, path)
def __sanitize_path_win32(os_dep_path):
r"""Win32-specific implementation of sanitize_path().
>>> __sanitize_path_win32('C:\\Documents and Settings\\host\\Desktop\\.C:')
'C:\\Documents and Settings\\host\\Desktop\\.C_'
"""
drive, path = ntpath.splitdrive(os_dep_path)
# There should not be colons after the disk definition.
path = path.replace(':', '_')
return ntpath.join(drive, path)
def resolve_path(self, link_path):
"""
[MS-SMB2] 2.2.2.2.1.1 Handling the Symbolic Link Error Response
https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-smb2/a8da655c-8b0b-415a-b726-16dc33fa5827
Attempts to resolve the link target path. Will fail if the link is pointing to a local path or a UNC path
on another host or share.
:param link_path: The original path to the symbolic link to resolve relative paths from.
:return: The resolved link target path.
"""
substitute_name = self.get_substitute_name()
print_name = self.get_print_name()
unparsed_path_length = self['unparsed_path_length'].get_value()
b_link_path = to_bytes(to_text(link_path), encoding='utf-16-le')
unparsed_idx = len(b_link_path) - unparsed_path_length
base_link_path = to_text(b_link_path[:unparsed_idx],
encoding='utf-16-le')
unparsed_path = to_text(b_link_path[unparsed_idx:],
encoding='utf-16-le')
# Use the common code in SymbolicLinkReparseDataBuffer() to resolve the link target.
symlink_buffer = SymbolicLinkReparseDataBuffer()
symlink_buffer['flags'] = self['flags'].get_value()
symlink_buffer.set_name(substitute_name, print_name)
target_path = symlink_buffer.resolve_link(
base_link_path) + unparsed_path
if not target_path.startswith('\\\\'):
raise SMBLinkRedirectionError(
"Cannot resolve link targets that point to a local path",
link_path, print_name)
link_share = ntpath.splitdrive(link_path)[0]
target_share = ntpath.splitdrive(target_path)[0]
if link_share != target_share:
raise SMBLinkRedirectionError(
"Cannot resolve link targets that point to a different host/share",
link_path, print_name)
return target_path
def test_splitdrive(self):
tester('ntpath.splitdrive("c:\\foo\\bar")', ("c:", "\\foo\\bar"))
tester('ntpath.splitdrive("c:/foo/bar")', ("c:", "/foo/bar"))
tester('ntpath.splitdrive("\\\\conky\\mountpoint\\foo\\bar")', ("\\\\conky\\mountpoint", "\\foo\\bar"))
tester('ntpath.splitdrive("//conky/mountpoint/foo/bar")', ("//conky/mountpoint", "/foo/bar"))
tester('ntpath.splitdrive("\\\\\\conky\\mountpoint\\foo\\bar")', ("", "\\\\\\conky\\mountpoint\\foo\\bar"))
tester('ntpath.splitdrive("///conky/mountpoint/foo/bar")', ("", "///conky/mountpoint/foo/bar"))
tester('ntpath.splitdrive("\\\\conky\\\\mountpoint\\foo\\bar")', ("", "\\\\conky\\\\mountpoint\\foo\\bar"))
tester('ntpath.splitdrive("//conky//mountpoint/foo/bar")', ("", "//conky//mountpoint/foo/bar"))
# Issue #19911: UNC part containing U+0130
self.assertEqual(ntpath.splitdrive("//conky/MOUNTPOİNT/foo/bar"), ("//conky/MOUNTPOİNT", "/foo/bar"))
def _win32_fixdrive(path):
"""Force drive letters to be consistent.
Win32 is inconsistent whether it returns lower or upper case
and even if it was consistent the user might type the other
so we force it to uppercase
running python.exe under cmd.exe return capital C:\\
running Win32 python inside a cygwin shell returns lowercase c:\\
"""
drive, path = ntpath.splitdrive(path)
return drive.lower() + path
def _unixpath(path):
"""
Replace backslashes with forward slashes
For win32 systems only - ensures forward slashes after performing
`os.path` operations (which by default return back-slashes.
"""
_drive_less = ntpath.splitdrive( path )[1]
_forward_slashed = _drive_less .replace ( "\\", "/" )
return _forward_slashed if win32 else _drive_less
def cluster(files, threshold):
artistDict = ClusterDict()
albumDict = ClusterDict()
tracks = []
for file in files:
artist = file.metadata["albumartist"] or file.metadata["artist"]
album = file.metadata["album"]
# Improve clustering from directory structure if no existing tags
# Only used for grouping and to provide cluster title / artist - not added to file tags.
filename = file.filename
if config.setting["windows_compatibility"] or sys.platform == "win32":
filename = ntpath.splitdrive(filename)[1]
album, artist = album_artist_from_path(filename, album, artist)
# For each track, record the index of the artist and album within the clusters
tracks.append((artistDict.add(artist),
albumDict.add(album)))
artist_cluster_engine = ClusterEngine(artistDict)
artist_cluster_engine.cluster(threshold)
album_cluster_engine = ClusterEngine(albumDict)
album_cluster_engine.cluster(threshold)
# Arrange tracks into albums
albums = {}
for i in xrange(len(tracks)):
cluster = album_cluster_engine.getClusterFromId(tracks[i][1])
if cluster is not None:
albums.setdefault(cluster, []).append(i)
# Now determine the most prominent names in the cluster and build the
# final cluster list
for album_id, album in albums.items():
album_name = album_cluster_engine.getClusterTitle(album_id)
artist_max = 0
artist_id = None
artist_hist = {}
for track_id in album:
cluster = artist_cluster_engine.getClusterFromId(
tracks[track_id][0])
if cluster is not None:
cnt = artist_hist.get(cluster, 0) + 1
if cnt > artist_max:
artist_max = cnt
artist_id = cluster
artist_hist[cluster] = cnt
if artist_id is None:
artist_name = u"Various Artists"
else:
artist_name = artist_cluster_engine.getClusterTitle(artist_id)
yield album_name, artist_name, (files[i] for i in album)
def process_files(IsWindows=False, originalpath='data/', flip=False):
lines = []
with open(originalpath + 'driving_log.csv') as csvfile:
reader = csv.reader(csvfile)
next(reader)
for line in reader:
lines.append(line)
car_camera_images = []
steering_angles = []
steering_correction = 0.25
for line in lines:
source_path = line[0]
if IsWindows:
drive, path_and_file = ntpath.splitdrive(source_path)
path, filename = ntpath.split(path_and_file)
else:
filename = source_path.split('/')[-1]
left_filename = filename.replace('center', 'left')
right_filename = filename.replace('center', 'right')
center_path = originalpath + 'IMG/' + filename
left_path = originalpath + 'IMG/' + left_filename
right_path = originalpath + 'IMG/' + right_filename
image_center = cv2.imread(center_path)
image_left = cv2.imread(left_path)
image_right = cv2.imread(right_path)
if flip:
image_center = cv2.flip(image_center, 0)
image_left = cv2.flip(image_left, 0)
image_right = cv2.flip(image_right, 0)
steering_angle = float(line[3])
steering_left = steering_angle + steering_correction
steering_right = steering_angle - steering_correction
if flip:
steering_angle = -steering_angle
steering_left = -steering_left
steering_right = -steering_right
image_center = process_image(image_center)
image_left = process_image(image_left)
image_right = process_image(image_right)
car_camera_images.extend([image_center, image_left, image_right])
steering_angles.extend([steering_angle, steering_left, steering_right])
return car_camera_images, steering_angles
def split_abs_path(path):
windows = False
out = []
if len(path) > 1 and path[1] == ':':
windows = True
drive, path = ntpath.splitdrive(path)
out.append(drive)
extend = os_path_split_asunder(path, windows)
if windows:
extend.pop(0)
out.extend(extend)
return out
def flat_rootname(filename):
"""A base for a flat file name to correspond to this file.
Useful for writing files about the code where you want all the files in
the same directory, but need to differentiate same-named files from
different directories.
For example, the file a/b/c.py will return 'a_b_c_py'
"""
name = ntpath.splitdrive(filename)[1]
return re.sub(r"[\\/.:]", "_", name)
def flat_rootname(filename):
"""A base for a flat file name to correspond to this file.
Useful for writing files about the code where you want all the files in
the same directory, but need to differentiate same-named files from
different directories.
For example, the file a/b/c.py will return 'a_b_c_py'
"""
name = ntpath.splitdrive(filename)[1]
return re.sub(r"[\\/.:]", "_", name)
def add_volume(self, local, remote, flags=None, prefix=None,
local_must_exist=False):
'''
Add a volume to the service
'''
self._validate_volume(local, remote, local_must_exist=local_must_exist)
if self.container_platform == "windows":
path = ntpath
else:
path = posixpath
# WCOW
if self.container_platform == "windows":
if prefix:
remote_drive, remote = ntpath.splitdrive(remote)
prefix_drive, prefix = ntpath.splitdrive(prefix)
# If prefix drive is unset, copy from remote
if not prefix_drive:
prefix_drive = remote_drive
remote = ntpath.join(prefix_drive, '\\' + prefix.lstrip(r'\/'),
remote.lstrip(r'\/'))
else:
# If LCOW
if os.name == "nt":
# Convert to posix slashed
remote = remote.replace('\\', '/')
# Remove duplicates
remote = re.sub('//+', '/', remote)
# Split drive letter off
drive, remote = ntpath.splitdrive(remote)
if drive:
remote = posixpath.join('/', drive[0], remote.lstrip(r'\/'))
if prefix:
remote = path.join(prefix, remote.lstrip(r'\/'))
self.volumes.append((local, remote))
self.volumes_flags.append(flags)
def make_valid_path(path, windows=None):
"""Removes invalid characters from windows pathnames"""
drive, path = ntpath.splitdrive(path)
if windows is None and drive:
# If a drive is found, this is a windows path
windows = True
if windows:
# Remove invalid characters
for char in ':<>*?"|':
path = path.replace(char, '')
# Windows directories and files cannot end with period
path = re.sub(r'(?<![\./\\])\.+(?=[/\\]|$)', '', path)
return drive + path
def do_get(self, src_path):
try:
newPath = ntpath.normpath(ntpath.join(self._pwd, src_path))
drive, tail = ntpath.splitdrive(newPath)
filename = ntpath.basename(tail)
fh = open(filename, 'wb')
logging.info("Downloading %s\\%s" % (drive, tail))
self.__transferClient.getFile(drive[:-1]+'$', tail, fh.write)
fh.close()
except Exception, e:
logging.error(str(e))
if os.path.exists(filename):
os.remove(filename)
def make_valid_path(path, windows=None):
"""Removes invalid characters from windows pathnames"""
drive, path = ntpath.splitdrive(path)
if windows is None and drive:
# If a drive is found, this is a windows path
windows = True
if windows:
# Remove invalid characters
for char in ':<>*?"|':
path = path.replace(char, "")
# Windows directories and files cannot end with period
path = re.sub(r"(?<![\./\\])\.+(?=[/\\]|$)", "", path)
return drive + path
def _make_posix_filename(fn_or_uri):
if ntpath.splitdrive(fn_or_uri)[0] or ntpath.splitunc(fn_or_uri)[0]:
result = fn_or_uri
else:
parse_result = urlparse(fn_or_uri)
if parse_result.scheme in ('', 'file'):
result = parse_result.path
else:
raise ValueError("unsupported protocol '%s'" % fn_or_uri)
if six.PY2 and isinstance(result, unicode):
# SWIG needs UTF-8
result = result.encode("utf-8")
return result
def do_get(self, src_path):
try:
newPath = ntpath.normpath(ntpath.join(self._pwd, src_path))
drive, tail = ntpath.splitdrive(newPath)
filename = ntpath.basename(tail)
fh = open(filename, 'wb')
logging.info("Downloading %s\\%s" % (drive, tail))
self.__transferClient.getFile(drive[:-1]+'$', tail, fh.write)
fh.close()
except Exception as e:
logging.error(str(e))
if os.path.exists(filename):
os.remove(filename)
def do_get(self, src_path):
try:
import ntpath
newPath = ntpath.normpath(ntpath.join(self.__pwd, src_path))
drive, tail = ntpath.splitdrive(newPath)
filename = ntpath.basename(tail)
fh = open(filename,'wb')
print "[*] Downloading %s\\%s" % (drive, tail)
self.__transferClient.getFile(drive[:-1]+'$', tail, fh.write)
fh.close()
except Exception, e:
print e
os.remove(filename)
pass
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
drive, remote_path = ntpath.splitdrive(remote_path)
drive = (drive or self.system_drive).rstrip(':')
set_error_pref = '$ErrorActionPreference="Stop"'
password = self.password.replace("'", "''")
create_cred = (
'$pw = convertto-securestring -AsPlainText -Force \'%s\';'
'$cred = new-object -typename System.Management.Automation'
'.PSCredential -argumentlist %s,$pw' % (password, self.user_name))
psdrive_name = self.name
root = '\\\\%s\\%s$' % (self.ip_address, drive)
create_psdrive = (
'New-PSDrive -Name %s -PSProvider filesystem -Root '
'%s -Credential $cred' % (psdrive_name, root))
remote_path = '%s:%s' % (psdrive_name, remote_path)
if copy_to:
from_path, to_path = local_path, remote_path
else:
from_path, to_path = remote_path, local_path
copy_item = 'Copy-Item -Path %s -Destination %s' % (from_path, to_path)
delete_connection = 'net use %s /delete' % root
cmd = ';'.join([set_error_pref, create_cred, create_psdrive,
copy_item, delete_connection])
stdout, stderr, retcode = vm_util.IssueCommand(
['powershell', '-Command', cmd], timeout=None)
if retcode:
error_text = ('Got non-zero return code (%s) executing %s\n'
'STDOUT: %sSTDERR: %s' %
(retcode, cmd, stdout, stderr))
raise errors.VirtualMachine.RemoteCommandError(error_text)
def do_get(self, src_path):
try:
import ntpath
newPath = ntpath.normpath(ntpath.join(self.__pwd, src_path))
drive, tail = ntpath.splitdrive(newPath)
filename = ntpath.basename(tail)
fh = open(filename, "wb")
logging.info("Downloading %s\\%s" % (drive, tail))
self.__transferClient.getFile(drive[:-1] + "$", tail, fh.write)
fh.close()
except Exception, e:
logging.error(str(e))
os.remove(filename)
pass
def flat_rootname(filename):
"""A base for a flat file name to correspond to this file.
Useful for writing files about the code where you want all the files in
the same directory, but need to differentiate same-named files from
different directories.
For example, the file a/b/c.py will return 'a_b_c_py'
"""
name = ntpath.splitdrive(filename)[1]
name = re.sub(r"[\\/.:]", "_", name)
if len(name) > MAX_FLAT:
h = hashlib.sha1(name.encode('UTF-8')).hexdigest()
name = name[-(MAX_FLAT-len(h)-1):] + '_' + h
return name
