def get_copy_ancestry(self):
"""Retrieve the list of `(path,rev)` copy ancestors of this node.
Most recent ancestor first. Each ancestor `(path, rev)` corresponds
to the path and revision of the source at the time the copy or move
operation was performed.
"""
ancestors = []
previous = (self._scoped_path_utf8, self.rev, self.root)
while previous:
(previous_path, previous_rev, previous_root) = previous
previous = None
root_path = fs.closest_copy(previous_root, previous_path)
if root_path:
(root, path) = root_path
path = path.lstrip('/')
rev = fs.revision_root_revision(root)
relpath = None
if path != previous_path:
# `previous_path` is a subfolder of `path` and didn't
# change since `path` was copied
relpath = previous_path[len(path):].strip('/')
copied_from = fs.copied_from(root, path)
if copied_from:
(rev, path) = copied_from
path = path.lstrip('/')
root = fs.revision_root(self.fs_ptr, rev, self.pool())
if relpath:
path += '/' + relpath
ui_path = _path_within_scope(self.scope, _from_svn(path))
if ui_path:
ancestors.append((ui_path, rev))
previous = (path, rev, root)
return ancestors
def getFrameRange(input_seq,output_ext):
image_location = os.path.dirname(input_seq)
image_name = os.path.splitext(input_seq)
path = ""
SlateNote=""
if "\\" in frame_seq_string:
frame_seq_string = input_seq.split("\\")
for items in frame_seq_string:
if "." in items:
frameloc=items.split(".")
else:
path = path +"\\"+items
path = path.lstrip("\\")+"\\"
if "/" in frame_seq_string:
frame_seq_string = input_seq.split("/")
for items in frame_seq_string:
if "." in items:
frameloc=items.split(".")
else:
path = path +"/"+items
path = path.lstrip("/")+"/"
extension = frameloc[1]
frame_name = frameloc[0].split("_")
framecounter = len(frame_name)-1
firstFrame = int(frame_name[framecounter])
leading_nums = len(frame_name[len(frame_name)-1])
leading_num_string = str(leading_nums)
item_names = len(frame_name)-1
v=0
prefix=""
while v <= item_names-1:
prefix = prefix+"_"+frame_name[v]
v=v+1
filename = prefix.lstrip("_")
os.system('cd M:\\applications\\utilties\\ffmpeg')
if SlateNote=="":
SlateNote=prefix.lstrip("_")
#print "Caption?"+captiontoggle
ffmpeg_seq = prefix.lstrip("_")+"_%0"+leading_num_string+"d."+extension
# Really want to do a files present vs files detected counter here
# Make it want to spot something with missing frames and/or alternate
# sequences.
#print "Final ffmpeg input:",ffmpeg_seq
if output_ext<>"":
outputFile = prefix.lstrip("_")+"."+output_ext
#print "Output:",outputFile
#return ffmpeg_seq,outputFile
else:
return ffmpeg_seq
def translatePath(url, asURL=False):
"""Returns the translated path."""
try:
scheme, loc, path, x, x, x = urlparse(url)
# print "scheme=%s loc=%s path=%s\nspecial[loc]=%s" % (scheme, loc, path,_special[loc])
if scheme == 'special' and loc in _special:
return ('file://' if asURL else '') \
+ os.path.join(_special[loc], os.path.normpath(path.lstrip('\\/')))
if scheme == 'plugin':
dir = os.path.join(_special['home'], 'addons', loc)
if os.path.isdir(dir):
return ('file://' if asURL else '') \
+ os.path.join(dir, os.path.normpath(path.lstrip('\\/')))
except:
pass
return url
def get_url_rev_and_auth(cls, url):
# type: (str) -> Tuple[str, Optional[str], AuthInfo]
"""
Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
That's required because although they use SSH they sometimes don't
work with a ssh:// scheme (e.g. GitHub). But we need a scheme for
parsing. Hence we remove it again afterwards and return it as a stub.
"""
# Works around an apparent Git bug
# (see https://article.gmane.org/gmane.comp.version-control.git/146500)
scheme, netloc, path, query, fragment = urlsplit(url)
if scheme.endswith('file'):
initial_slashes = path[:-len(path.lstrip('/'))]
newpath = (initial_slashes +
urllib.request.url2pathname(path).replace(
'\\', '/').lstrip('/'))
after_plus = scheme.find('+') + 1
url = scheme[:after_plus] + urlunsplit(
(scheme[after_plus:], netloc, newpath, query, fragment), )
if '://' not in url:
assert 'file:' not in url
url = url.replace('git+', 'git+ssh://')
url, rev, user_pass = super().get_url_rev_and_auth(url)
url = url.replace('ssh://', '')
else:
url, rev, user_pass = super().get_url_rev_and_auth(url)
return url, rev, user_pass
def doctor(link_dir, path):
# Don't mess with paths that just refer to another link:
if path.rstrip()[-1] == '_': return path
path = path.lstrip()
# Don't mess with paths that point somewhere in the outside universe:
if path.startswith('http://'): return ' ' + path
# Prepend link_dir to path
if link_dir.startswith('./'): path = link_dir[2:] + '/' + path
elif link_dir != '.': path = link_dir + '/' + path
# Prepare dir (start_dir, minus initial './')
if start_dir == '.': dir = ''
elif start_dir.startswith('./'): dir = start_dir[2:]
else: dir = start_dir
rest=' '
last_dir = None
while dir and dir != last_dir:
if path.startswith(dir + '/'):
ans = rest + path[len(dir) + 1:]
#print "doctor(%s) abbr:" % (path.rstrip(),), ans
return ans
rest += '../'
last_dir = dir
dir, ignore = os.path.split(dir)
ans = rest + path
#print "doctor(%s) abs:" % (path.rstrip(),), ans
return ans
def isDirectory(self, path):
try:
Tree.fromPath(self, "/" + path.lstrip("/"))
except Exception:
return False
else:
return True
def normalize_path(path):
if sublime.platform() == 'windows':
return os.path.normpath(
path.lstrip('/').replace('/', ':/', 1)
)
else:
return path
def restore(self, to_directory, dry_run):
self.logger.info("restoring generic files")
if not self.locked_manifest:
self.logger.warning("empty or no manifest.json.lock file found, skipping generic files restore")
self.logger.warning("this could be because the backup was not initialize or nothing is in the backup")
return True
for path, data in self.locked_manifest.items():
path = path.lstrip("/")
from_path = os.path.join(self.path, path)
to_path = os.path.join(to_directory, path)
owner, group = data["owner"], data["group"]
owner_id = getpwnam(owner).pw_uid
group_id = getgrnam(group).gr_gid
self.logger.info("copy {} to {} with owner:group of {}({}):{}({})".format(from_path, to_path, owner, owner_id, group, group_id))
if not dry_run:
dirname = os.path.dirname(to_path)
mkdir_p(dirname)
os.chown(dirname, owner_id, group_id)
os.chmod(dirname, int("0700", 8))
shutil.copy2(from_path, to_path)
os.chown(to_path, owner_id, group_id)
os.chmod(to_path, int("0600", 8))
def translatePath(url, asURL=False):
"""Returns the translated path."""
try:
scheme, loc, path, x, x, x = urlparse(url.replace('\\', '/'))
# print "scheme=%s loc=%s path=%s\nspecial[loc]=%s" % (scheme, loc, path,_special[loc])
if scheme == 'special' and loc in _special:
return ('file://' if asURL else '') \
+ os.path.join(_special[loc], os.path.normpath(path.lstrip('\\/')))
if scheme == 'plugin':
dir = xbmcinit.get_addon_path(loc)
if dir:
return ('file://' if asURL else '') \
+ os.path.join(dir, os.path.normpath(path.lstrip('\\/')))
except:
pass
return url
def getpath(root):
path = root.split(basedir)[1]
if path.startswith(os.sep):
path = path.lstrip(os.sep)
return path
def to_url(self, scheme):
super(FileSystemRepository, self).to_url(scheme)
if sys.platform == 'win32':
drive, path = os.path.splitdrive(self.path)
path = '/'.join(path.lstrip(os.path.sep).split(os.path.sep))
return '{0}:///{1}/{2}'.format(scheme, drive, path)
return '{0}://{1}'.format(scheme, self.path)
def post(self, path, params={}, data={}):
xenrt.TEC().logverbose("Posting to %s" % path)
r = self.session.post("%s/%s" % (self.BASE_URL, path.lstrip("/")), params=params, data=data, headers={"sx_csrf": self.sx_csrf})
r.raise_for_status()
if r.json()['result'] != "SUCCESS":
raise xenrt.XRTFailure("Result was %s" % r.json())
return r.json()
def read_bookmarks():
#Check if rc file exists...
check_rc_files();
#Open the filename and read all bookmarks that are in format of:
# BookmarkName : BookmarkSeparator (Note that the ':' is the separator)
try:
bookmarks_file = open(Constants.PATH_FILE_RC);
for bookmark in bookmarks_file.readlines():
bookmark = bookmark.replace("\n", "");
name, path = bookmark.split(Constants.BOOKMARK_SEPARATOR);
#Trim all white spaces.
name = name.replace(" ", "");
path = path.lstrip().rstrip();
Globals.bookmarks[name] = path;
except Exception, e:
#Failed to unpack, this is because the bookmarks aren't in form of
# Name SEPARATOR Path.
#So state it to user, so he could correct manually.
help_msg = "{0} {1} {2} {3}".format("Check if all values are in form of",
C.blue("BookmarkName"),
C.magenta(Constants.BOOKMARK_SEPARATOR),
C.blue("BookmarkPath"));
msg = "{0} ({1})\n{2}".format("Bookmarks file is corrupted.",
C.blue(Constants.PATH_FILE_RC),
help_msg);
print_fatal(msg);
def normalize(self, path):
"""Normalize a path."""
path = re.sub(r"//+", "/", path)
path = path.lstrip("/")
return path
def isIncluded(self, context, path):
""" Decides whether the specified path within the archive should be
unpacked, based on the include/exclude filters
@param path: a relative path within the archive
"""
if not self.__excludes and not self.__includes: return True
if not self.__isResolved:
self.__includes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__includes])
self.__excludes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__excludes])
self.__isResolved = True
assert '\\' not in path
try:
path = path.lstrip('/')
# first check if it matches an exclude
if next( (True for e in self.__excludes if antGlobMatch(e, path)), False): return False
if not self.__includes: # include everything
return True
else:
m = next( (i for i in self.__includes if antGlobMatch(i, path)), None)
if m:
return True
else:
return False
except Exception, e:
raise BuildException('FilteredArchiveContents error for %s'%(self), causedBy=True, location=self.__location)
def getattr(self, path):
components = [name for name in path.lstrip('/').split('/') if name]
entry = self._root
while components:
entry = entry.lookup(components.pop(0))
return entry
def sysroot_path(path):
"""Make the given relative or absolute path "sysrooted"
:param str path: path to be sysrooted
:returns: sysrooted path
:rtype: str
"""
return os.path.join(getSysroot(), path.lstrip(os.path.sep))
def _clarity_county_url(self, path):
url = self._clarity_election_base_url(self.CLARITY_PORTAL_URL +
path.lstrip('/'))
r = requests.get(url)
r.raise_for_status()
redirect_path = self._scrape_county_redirect_path(r.text)
return url + redirect_path
def _serve_request(self, environ, start_response):
path = environ["PATH_INFO"]
m = re.fullmatch(r"/livereload/([0-9]+)/[0-9]+", path)
if m:
epoch = int(m[1])
start_response("200 OK", [("Content-Type", "text/plain")])
def condition():
return self._visible_epoch > epoch
with self._epoch_cond:
if not condition():
# Stall the browser, respond as soon as there's something new.
# If there's not, respond anyway after a minute.
self._log_poll_request(environ.get("HTTP_REFERER"),
request_id=path)
self._epoch_cond.wait_for(
condition, timeout=self.poll_response_timeout)
return [b"%d" % self._visible_epoch]
if path == "/js/livereload.js":
file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "livereload.js")
elif path.startswith(self.mount_path):
if path.endswith("/"):
path += "index.html"
path = path[len(self.mount_path):]
file_path = os.path.join(self.root, path.lstrip("/"))
elif path == "/":
start_response("302 Found", [("Location", self.mount_path)])
return []
else:
return None # Not found
# Wait until the ongoing rebuild (if any) finishes, so we're not serving a half-built site.
with self._epoch_cond:
self._epoch_cond.wait_for(
lambda: self._visible_epoch == self._wanted_epoch)
epoch = self._visible_epoch
try:
file = open(file_path, "rb")
except OSError:
return None # Not found
if path.endswith(".html"):
with file:
content = file.read()
content = self._inject_js_into_html(content, epoch)
file = io.BytesIO(content)
content_length = len(content)
else:
content_length = os.path.getsize(file_path)
content_type = self._guess_type(file_path)
start_response("200 OK", [("Content-Type", content_type),
("Content-Length", str(content_length))])
return wsgiref.util.FileWrapper(file)
def try_static(self, start_response, path):
full_path = os.path.join(self.directory, path.lstrip('/'))
if not full_path.endswith('.html') and os.path.exists(
full_path) and not os.path.isdir(full_path):
with open(full_path, 'rb') as f:
body = f.read()
return self.response(start_response, '200 OK',
self.guess_mime(full_path), body)
def __getitem__ (self, path) :
try :
key, path = path.lstrip("/").split("/", 1)
except :
raise HTTPError(httplib.FORBIDDEN)
if key != self.key :
raise HTTPError(httplib.FORBIDDEN)
return self.root[path]
def get_file_dir_status(self, path):
"""Stats a file or a directoty in hdfs
The superclass expects paths without the leading '/'. This method strips it from the
path to make usage transparent to the caller.
"""
path = path.lstrip('/')
return super(PyWebHdfsClientWithChmod, self).get_file_dir_status(path)
def open(self, path: str) -> Union[datasets.Dataset, posix.Directory]:
paths = path.lstrip('/').split('/')
current = self.root_dataset
if paths == ['']:
return current
for next_dir in paths:
current = current[next_dir]
return current
def url2pathname(urlpath):
""" Takes a file:// URL and turns it into a filename.
"""
path = urllib.url2pathname(urlpath)
if systemType in ('Windows', 'Microsoft'):
path = path.replace('/', '\\')
path = path.lstrip('\\')
return path
def compose_path(root, path):
# Construct the full path, stripping any leading slashes so we
# resolve absolute paths within the root.
full_path = os.path.abspath(os.path.join(
root, path.lstrip(os.path.sep.encode('utf-8'))))
if not full_path.startswith(os.path.abspath(root)):
raise ValueError('Path not contained within root')
return full_path
def open(self, path, wait=True):
"""Helper that prefixes with our local address."""
prefix = "http://%s:%d/" % (self.host, self.port)
url = prefix + path.lstrip('/')
ret = self.s.open(url)
if wait:
self.s.wait_for_page_load()
return ret
def path_to_id(nodepath):
"""Given a path, return a string suitable as an HTML id attribute"""
path = yangpath.strip_namespace(nodepath)
# remove leading slash
path = path.lstrip('/')
path = re.sub(r'\/', r'-', path)
return path.lower()
def addPath(self, path_section, path):
ctxAssert( path_section in REPO_PATH_SECTIONS, "Unknown path section '%s'"%(path_section) )
if os.path.isabs(path):
warningMessage("The absolute path %s will be treated as relative to the repository copy"%(path))
path = path.lstrip('\\/ ')
self.relative_paths[path_section].append( os.path.normpath(path) )
infoMessage("Path '%s' added to section '%s' in repository '%s'"%(path, path_section, self.getID()), 2)
def url2pathname(urlpath):
""" Takes a file:// URL and turns it into a filename.
"""
path = urllib.url2pathname(urlpath)
if systemType in ('Windows', 'Microsoft'):
path = path.replace('/', '\\')
path = path.lstrip('\\')
return path
def build_url(base_url, path):
""" prepend url with base_url unless it's already an absolute URL """
if absolute_http_url_regexp.match(path):
return path
elif base_url:
return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/"))
else:
raise ParamsError("base url missed!")
def path_to_id(nodepath):
"""Given a path, return a string suitable as an HTML id attribute"""
path = yangpath.strip_namespace(nodepath)
# remove leading slash
path = path.lstrip('/')
path = re.sub(r'\/', r'-', path)
return path.lower()
def remove(self, path):
obj = self.connection.Object(self.bucket, path.lstrip('/'))
resp = obj.delete()
if resp['ResponseMetadata']['HTTPStatusCode'] == 204:
return True
else:
return False
def _resolve_path(path):
# If it looks like a path with a drive letter that has leading slashes
# (most likely as artifacts from URL parsing), strip them off. Windows
# doesn't allow colons in paths, so this won't ever mistakenly strip
# off slashes.
if re.match("^/+[a-zA-Z]:", path):
path = path.lstrip("/")
return File._resolve_path(path)
def __getitem__(self, path):
try:
key, path = path.lstrip("/").split("/", 1)
except:
raise HTTPError(httplib.FORBIDDEN)
if key != self.key:
raise HTTPError(httplib.FORBIDDEN)
return self.root[path]
def assert_betty_html(self, path: str) -> str:
file_path = join(
self.site.configuration.www_directory_path, path.lstrip('/'))
self.assertTrue(exists(file_path), '%s does not exist' % file_path)
with open(file_path) as f:
parser = html5lib.HTMLParser(strict=True)
parser.parse(f)
return file_path
def readdir(self, path, fh):
yield '.'
yield '..'
path = path.lstrip('/')
for _, kv in self.client.get_prefix(self._get_meta_key(path),
keys_only=True):
file_path = self._get_path_from_meta_key(kv.key)
if os.path.split(file_path)[0] == path:
yield os.path.split(file_path)[-1]
def _getNode(self, path):
node = self.structure
path = path.lstrip('/')
if path != '':
for n in path.split('/'):
if n not in node:
return None
node = node[n]
return node
def __getitem__ (self, path) :
path = os.path.join(self.root, path.lstrip("./"))
if not os.path.isfile(path) :
raise HTTPError(httplib.NOT_FOUND)
ct = mimetypes.guess_type(path)[0] or "application/octet-stream"
@http(ct)
def handler () :
return open(path).read()
return handler
def __init__(self, zfs, path='/', **opts):
"""
zfs - qmsk.backup.zfs.ZFS
path - str: filesystem path within lvm volume; no leading /
"""
super().__init__(path.lstrip('/'), **opts)
self.zfs = zfs
def request(self, method, path, **kwargs):
version = kwargs.pop('version', self.version)
url = '%s/v%s/%s' % (self.host, version, path.lstrip('/'))
params = kwargs.get('params', {}).copy()
kwargs['params'] = parameters(params)
kwargs.setdefault('connector', self.connector)
response = yield from aiohttp.request(method, url, **kwargs)
return response
def get_path_url(self, path, rev):
"""Retrieve the "native" URL from which this repository is reachable
from Subversion clients.
"""
url = self.params.get('url', '').rstrip('/')
if url:
if not path or path == '/':
return url
return url + '/' + path.lstrip('/')
def get_path_url(self, path, rev):
"""Retrieve the "native" URL from which this repository is reachable
from Subversion clients.
"""
url = self.params.get('url', '').rstrip('/')
if url:
if not path or path == '/':
return url
return url + '/' + path.lstrip('/')
def search_exact(path):
path = path.lstrip("/") # FIXME: this shouldn't be necessary :/
if ctx.filesdb.has_file(path):
(pkg_name, file_info) = ctx.filesdb.get_file(path)
ctx.ui.info(_("Package: %s") % pkg_name)
if ctx.config.options.long:
ctx.ui.info(_("Type: %s, Hash: %s") % (file_info.type, file_info.hash))
else:
ctx.ui.error(_("Path %s does not belong to an installed package") % path)
def dev_sitemap_serve(request, path, insecure=False, **kwargs):
path = path.lstrip('/')
absolute_path = os.path.join(STATIC_SITEMAP_DIR, path)
if not os.path.isfile(absolute_path):
raise Http404("'%s' could not be found" % path)
return static.serve(request,
path,
document_root=STATIC_SITEMAP_DIR,
**kwargs)
def transform(filename):
htmlfile = open(filename, encoding='latin-1')
tree = html.parse(htmlfile, parser=parser)
# Frontpage seems to use <font> tags to indicate headings
for node in tree.xpath('//font'):
size = int(node.get('size')) if 'size' in node.attrib else None
color = node.get('color').lower() if 'color' in node.attrib else ''
if color == '#ff0000':
strong = html.Element('strong')
replace_tag(node, strong)
elif size == 6:
h1 = html.Element('h1')
replace_tag(node, h1)
elif size == 5:
h2 = html.Element('h2')
replace_tag(node, h2)
elif size == 4:
node.drop_tag()
# We rewrite all the urls to point to MD files instead of HTM
for node in tree.xpath('//a[@href]'):
href = node.get('href')
try:
parsed_url = urlparse(href)
path, filename = os.path.split(parsed_url.path)
basename, extension = filename.split('.')
hostname = parsed_url.hostname
except ValueError:
continue
else:
if hostname and hostname.startswith('anastasis'):
hostname = None
if extension.startswith('htm'):
if path:
new_path = '{}{}.{}'.format(path.lstrip('/'), basename,
'md')
else:
new_path = '{}.{}'.format(basename, 'md')
new_url = '', '', new_path, '', '', parsed_url.fragment
node.set('href', urlunparse(new_url))
# Pandoc passes this through, cluttering up the final markdown. Must come
# after footnore rewriting.
for node in tree.xpath('//span[@class="MsoFootnoteReference"]'):
node.drop_tag()
remove_empty(tree)
return etree.tostring(tree,
pretty_print=True,
method='html',
encoding='unicode')
def main(args):
if len(args) != 2:
print('Usage: check-rellinks dir')
sys.exit(1)
os.chdir(args[1])
filenames = glob.glob('**/*.html', recursive=True)
targets = {}
rellinks = {}
for filename in filenames:
with open(filename) as f:
soup = bs4.BeautifulSoup(f, 'html.parser')
links = [
Link(href=e['href'], parsed=urllib.parse.urlparse(e['href']))
for e in soup.find_all('a', href=True)
]
rellinks[filename] = [
link for link in links if link.parsed.scheme == ''
]
targets[filename] = [e['id'] for e in soup.find_all(id=True)]
def check(path, fragment):
if path.endswith('.atom') and fragment == '':
return True
return path in targets and (fragment == ''
or fragment in targets[path])
has_broken = False
for filename in rellinks:
if filename.endswith('-release-notes.html'):
continue
dirname = os.path.dirname(filename)
broken_links = []
for link in rellinks[filename]:
path = link.parsed.path
if path == '':
path = filename
else:
if os.path.splitext(path)[1] == '':
path += '/index.html'
if path.startswith('/'):
path = path.lstrip('/')
else:
path = os.path.normpath(os.path.join(dirname, path))
if not check(path, link.parsed.fragment):
broken_links.append(link.href)
if broken_links:
if not has_broken:
print('Found broken links:')
has_broken = True
print(filename)
for link in broken_links:
print(f' {link}')
if has_broken:
sys.exit(1)
def check_checksum_link(path):
if path.startswith('/'):
path = path.lstrip('/')
current = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
full_path = os.path.join(current, path)
if not os.path.exists(full_path):
print('[MISSING CHECKSUM] %s' % full_path)
return False
print('[OK] %s' % path)
return True
def test_addOrUpdateEntriesFromTarball_path_leading_slash(self):
# Leading slashes are stripped from path names.
path, content = self._makeFile('pot', '/directory')
files = dict(((path, content),))
tarfile_content = LaunchpadWriteTarFile.files_to_string(files)
self.import_queue.addOrUpdateEntriesFromTarball(
tarfile_content, True, self.importer,
productseries=self.productseries)
stripped_path = path.lstrip('/')
self.assertEqual([stripped_path], self._getQueuePaths())
def post(self, path, params={}, data={}):
xenrt.TEC().logverbose("Posting to %s" % path)
r = self.session.post("%s/%s" % (self.BASE_URL, path.lstrip("/")),
params=params,
data=data,
headers={"sx_csrf": self.sx_csrf})
r.raise_for_status()
if r.json()['result'] != "SUCCESS":
raise xenrt.XRTFailure("Result was %s" % r.json())
return r.json()
def get_path_url(self, path, rev):
"""Retrieve the "native" URL from which this repository is reachable
from Subversion clients.
"""
url = self.params.get('url', '').rstrip('/')
if url:
href = Href(url)
if path:
path = path.lstrip('/')
return href(path)
def hdfs_path_to_real_path(hdfs_path, environ):
components = urlparse(hdfs_path)
scheme = components.scheme
path = components.path
if not scheme and not path.startswith('/'):
path = '/user/%s/%s' % (environ['USER'], path)
return os.path.join(environ['MOCK_HDFS_ROOT'], path.lstrip('/'))
def route(self, path):
path = path.lstrip('/')
if path in self.paths:
raise ValueError('Route already definted')
def wrapper(func):
self.paths[path] = func
return func
return wrapper
def split_leading_dir(self, path):
path = str(path)
path = path.lstrip('/').lstrip('\\')
if '/' in path and (('\\' in path and path.find('/') < path.find('\\'))
or '\\' not in path):
return path.split('/', 1)
elif '\\' in path:
return path.split('\\', 1)
else:
return path, ''
def path(self, path):
path = path.lstrip('.').strip('/')
if path:
path = os.path.relpath(path)
# Handle paths for mysql repo where there is no project name
if path.startswith(getattr(self.repo._repo, '_name', '\0')):
path = os.path.join(*(path.split('/')[1:]))
if hasattr(path, 'encode'):
return path.encode('utf-8')
return path
def api_url(path, query=None):
api_path = get_settings().get("api-path").rstrip('/')
path = path.lstrip('/')
query_str = "&".join(
"{}={}".format(url_quote(str(k), safe=''), url_quote(str(v), safe=''))
for k, v in query.items()) if query else None
url = "{}/{}?{}".format(api_path, path,
query_str) if query_str else "{}/{}".format(
api_path, path)
return url
def hdfs_path_to_real_path(hdfs_path, environ):
components = urlparse(hdfs_path)
scheme = components.scheme
path = components.path
if not scheme and not path.startswith('/'):
path = '/user/%s/%s' % (environ['USER'], path)
return os.path.join(environ['MOCK_HDFS_ROOT'], path.lstrip('/'))
def saveFile(self, path, value):
processInputs(self.request)
path = self.request.form.get('path', path)
value = self.request.form.get('value', value)
path = path.lstrip('/').encode('utf-8')
value = value.replace('\r\n', '\n').encode('utf-8')
self.context.writeFile(path, value)
return ' ' # Zope no likey empty responses
def __getitem__(self, path):
path = os.path.join(self.root, path.lstrip("./"))
if not os.path.isfile(path):
raise HTTPError(httplib.NOT_FOUND)
ct = mimetypes.guess_type(path)[0] or "application/octet-stream"
@http(ct)
def handler():
return open(path).read()
return handler
def toLongPathSafe(path, force=False):
"""
Converts the specified path string to a form suitable for passing to API
calls if it exceeds the maximum path length on this OS.
Currently, this is necessary only on Windows, where a unicode string
starting with \\?\ must be used to get correct behaviour for long paths.
Unlike L{normLongPath} which also performs the long path conversion, this
function does NOT convert to a canonical form, normalize slashes or
remove '..' elements (unless required for long path support). It is therefore
faster.
@param path: A path. Must not be a relative path. Can be None/empty. Can
contain ".." sequences, though performance is a lot lower if it does.
@param force: Normally the long path support is added only if this path
exceeds the maximum length on this OS (e.g. 256 chars) or ends with a
directory slash. Set force to True to add long path support regardless of
length, which allows extra characters to be added on to the end of the
string (e.g. ".log" or a directory filename) safely.
@return: The passed-in path, possibly with a "\\?\" prefix added and
forward slashes converted to backslashes on Windows. Any trailing slash
is preserved by this function (though will be converted to a backslash).
"""
if (not __isWindows) or (not path): return path
if (force or len(path)>255 or isDirPath(path)) and not path.startswith('\\\\?\\'):
if path in __longPathCache: return __longPathCache[path]
inputpath = path
# ".." is not permitted in \\?\ paths; normpath is expensive so don't do this unless we have to
if '.' in path:
path = os.path.normpath(path)+('\\' if isDirPath(path) else '')
else:
# path is most likely to contain / so more efficient to conditionalize this
path = path.replace('/','\\')
if '\\\\' in path:
# consecutive \ separators are not permitted in \\?\ paths
path = path.replace('\\\\','\\')
try:
if path.startswith('\\\\'):
path = u'\\\\?\\UNC\\'+path.lstrip('\\') # \\?\UNC\server\share Oh My
else:
path = u'\\\\?\\'+path
except Exception:
# can throw an exception if path is a bytestring containing non-ascii characters
# to be safe, fallback to original string, just hoping it isn't both
# international AND long
# could try converting using a default encoding, but slightly error-prone
pass
__longPathCache[inputpath] = path
return path
def hdfs_uri_to_real_path(hdfs_uri, environ):
"""Map an HDFS URI to a path on the filesystem."""
components = urlparse(hdfs_uri)
scheme = components.scheme
path = components.path
if not scheme and not path.startswith('/'):
path = '/user/%s/%s' % (environ['USER'], path)
return os.path.join(get_mock_hdfs_root(environ=environ), path.lstrip('/'))
