def build_url(self, location, name):
location = location.parent
suffix = None
try:
target = self.pages[str(name)][0]
logger.debug('look up for %s: rendered output: %s', name, target)
suffix = '.html'
except KeyError:
for r in self.config.resources:
t = r.lookup_target(str(name))
if t is not None:
target = t
logger.debug('look up for %s: resources: %s', name, target)
break
else:
logger.warning('look up for %s: nothing found!', name)
return None
n = 0
for a, b in zip(location.parts, target.parts):
if a != b:
break
n += 1
newpath = ('..', ) * (len(location.parts) - n) + target.parts[n:]
p = PurePosixPath(*newpath)
if suffix is not None:
p = p.with_suffix(suffix)
if p.name == 'index.html':
return p.parent / '.'
if p.suffix == '.html' and self.config.htmlless:
p = p.with_suffix('')
return p
def build_url(self, location, name):
location = location.parent
suffix = None
try:
target = self.pages[str(name)][0]
logger.debug('look up for %s: rendered output: %s', name, target)
suffix = '.html'
except KeyError:
for r in self.config.resources:
t = r.lookup_target(str(name))
if t is not None:
target = t
logger.debug('look up for %s: resources: %s', name, target)
break
else:
logger.warning('look up for %s: nothing found!', name)
return None
n = 0
for a, b in zip(location.parts, target.parts):
if a != b:
break
n += 1
newpath = ('..',) * (len(location.parts) - n) + target.parts[n:]
p = PurePosixPath(*newpath)
if suffix is not None:
p = p.with_suffix(suffix)
if p.name == 'index.html':
return p.parent / '.'
if p.suffix == '.html' and self.config.htmlless:
p = p.with_suffix('')
return p
def get(self, path, root=None):
if isinstance(path, str) or isinstance(path, unicode):
path = PurePosixPath(path)
if isinstance(path, PurePath):
path = list(path.parts[1:] if path.is_absolute() else path.parts)
if root is None:
root = self.root
if not isinstance(root, dict):
root = {'?file': True, '?content': root}
if not path: # Check if reached the final location
return root
next_node = path[0]
path.pop(0)
if '?' in next_node:
return None
elif next_node in root:
if not isinstance(root[next_node], dict):
root[next_node] = {'?file': True, '?content': root[next_node]}
root[next_node]['?parent'] = root
return self.get(path, root[next_node])
else:
return None
def __init__(self, *a, indexes, source, root, index_root, **kw):
super().__init__(*a, **kw)
self.source = self._dir / source
self.root = PurePosixPath(root)
self.index_root = PurePosixPath(index_root)
self.indexes = indexes
self.by_name = {}
self.cats = {}
def split_filename_into_tokens(include_dir, filename, is_windows):
# type: (str, str, bool) -> Any
r"""
Tokens are the individual components of paths
Invalid Linux Tokens
'\0' (NUL)
Invalid Windows Tokens
< (less than)
> (greater than)
: (colon - sometimes works, but is actually NTFS Alternate Data Streams)
" (double quote)
/ (forward slash)
\ (backslash)
| (vertical bar or pipe)
? (question mark)
* (asterisk)
All control codes (<= 31)
"""
if is_windows:
inc = PureWindowsPath(include_dir)
pth = PureWindowsPath(filename).parts
# fails if the comment has stripped out the file (e.g., "INCLUDE '$ENV/model.bdf'")
pth0 = pth[0]
# Linux style paths
# /work/model.bdf
if len(pth0) == 1 and pth0[0] == '\\':
# utterly breaks os.path.join
raise SyntaxError('filename=%r cannot start with / on Windows' %
filename)
else:
inc = PurePosixPath(include_dir)
pth = PurePosixPath(filename).parts
# fails if the comment has stripped out the file (e.g., "INCLUDE '$ENV/model.bdf'")
pth0 = pth[0]
if len(pth0) >= 2 and pth0[:2] == r'\\':
# Windows network paths
# \\nas3\work\model.bdf
raise SyntaxError("filename=%r cannot start with \\\\ on Linux" %
filename)
pth2 = split_tokens(pth, is_windows)
if is_windows:
pth3 = ntpath.join(*pth2)
else:
pth3 = posixpath.join(*pth2)
pth_out = inc / pth3
return pth_out
def normalize_path(path):
"""
Convert a Path to a slash-separated string, suitable for use in cloud
object names. (We don't just use ``str(path)`` because our paths might
look different if we ever run on Windows.)
"""
return str(PurePosixPath(path))
def resolveTraversal(self, path):
path = PurePosixPath(path)
if not path.is_absolute():
return path
new_path = []
for node in path.parts:
if node == '.':
pass
elif node == '..':
if len(new_path) > 1:
new_path.pop()
else:
new_path.append(node)
return PurePosixPath(*new_path)
def _search_objects(target_dir, s3_objects, s3_obj_name='', ignore_pattern=''):
log.trace('_search_objects args: %s' % str({
'target_dir': target_dir,
's3_objects len': len(s3_objects),
's3_obj_name': s3_obj_name,
'ignore_pattern': ignore_pattern
}))
matched_objects = set()
for s3_obj in s3_objects:
s3_obj_path = PurePosixPath(s3_obj['Key'])
try:
obj_rel_path = s3_obj_path.relative_to(target_dir)
if (target_dir == s3_obj_path):
continue
except ValueError as e:
log.trace('_search_objects s3_obj not relative_to target_dir: %s' % e.message)
continue
try:
obj_next_path_list = target_dir.joinpath(obj_rel_path.parts[0])
except Exception as e:
log.fatal('Error in _search_objects: "%s"' % e.message)
log.fatal(str({
's3_obj_path': s3_obj_path,
'obj_rel_path': obj_rel_path,
'target_dir': target_dir
}))
exit(1)
obj_next_path_string = str(obj_next_path_list)
log.trace('_search_objects s3_obj_path: ' + str(s3_obj_path))
log.trace('_search_objects obj_next_path_string: ' + obj_next_path_string)
if ignore_pattern:
if re.search(ignore_pattern, s3_obj_path.name):
continue
if s3_obj_name:
# Look for matches to obj_name
if re.search(s3_obj_name, s3_obj_path.name):
matched_objects.add(s3_obj)
else:
matched_objects.add(obj_next_path_string)
results = sorted([obj for obj in matched_objects], reverse=True)
log.trace('_search_objects results: %s' % results)
return results
def update_or_clone(url,
name=None,
branch='master',
remote='origin',
display_status=False,
quiet=False): # pragma: no cover -- TODO commands
if not name:
name = PurePosixPath(url).name
if name.endswith('.git'):
name = name[:-4]
if not Path(name).is_dir():
subprocess.call(['git', 'clone', url, name])
else:
with clckwrkbdgr.fs.CurrentDir(name):
sync(quiet=quiet)
update(quiet=quiet)
return name
def _split_path(abspath, is_windows):
"""
Takes a path and splits it into the various components.
This is a helper method for ``write_include``
"""
if is_windows:
parts = PureWindowsPath(abspath).parts
else:
parts = PurePosixPath(abspath).parts
return parts
def __init__(self, *a, source, root, **kw):
super().__init__(*a, **kw)
self.source = self._dir / source
self.root = PurePosixPath(root)
self.sources = [
p for p in self.source.iterdir() if not self.is_ignored(p)
]
self.log(logging.INFO, 'Found %d sources in %s', len(self.sources),
self.source.resolve())
def do_GET(self): # pragma: no cover
path = self.path
if path.startswith('/'):
path = path[1:]
path_parts = list(map(six.moves.urllib.parse.unquote, PurePosixPath(path).parts))
full_path = self.server.config.root
if path_parts:
full_path = os.path.join(full_path, os.path.join(*path_parts))
if os.path.isfile(full_path):
self._respond(self.mod_file_response(200, full_path))
elif os.path.isdir(full_path):
self.serve_filesystem(full_path)
else:
self._respond(Response(404, 'Not Found'))
def __init__(self, *a, title, description, module, output, **kw):
super().__init__(*a, **kw)
self._info = {}
self.title = title
self.description = description
self.blog = self.config.module_id[module]
if not isinstance(self.blog, Blog):
self.log(
logging.WARNING,
"Configured with a target other than a blog. Expect errors.")
self.output = PurePosixPath(output)
def recursive_index_search(bucket, target_dir, root_dir, obj_name='', ignore_pattern=''):
log.trace('recursive_index_search args: %s' % locals())
s3_objects = _get_list_of_s3_objects(bucket, root_dir)
# Root dir not part of target dir will make pathlib throw
try:
rel_path = target_dir.relative_to(root_dir).parts
except ValueError as e:
log.fatal('Error with root dir and target dir: "%s"\n' % e.message)
exit(1)
objs_in_each_dir = []
objs_in_each_dir.append(
[root_dir, _search_objects(root_dir, s3_objects, obj_name, ignore_pattern)]
)
for index, _ in enumerate(rel_path):
target_dir_to_search = PurePosixPath(consts.aps.join(root_dir.parts + rel_path[:index + 1]))
objs = _search_objects(target_dir_to_search, s3_objects, obj_name, ignore_pattern)
objs_in_each_dir.append([target_dir_to_search, objs])
return objs_in_each_dir
def resolve(self, path_cur, path_next, path_home):
path_cur = PurePosixPath(path_cur)
path_next = PurePosixPath(path_next)
path_next_parts = path_next.parts
if not path_next_parts:
return None
path_new = None
if path_next.is_absolute():
path_new = path_next
elif path_next_parts[0] == '~':
path_new = PurePosixPath(path_home) / PurePosixPath(
*path_next_parts[1:])
else:
path_new = path_cur / path_next
path_new = self.resolveTraversal(path_new)
if not self.get(path_new) is None:
return str(path_new)
else:
return None
def parse_dir_name(raw_path):
path = PurePosixPath(raw_path)
if path.parts[0] == consts.aps:
path = PurePosixPath(consts.aps.join(path.parts[1:]))
return path
def split_tokens(tokens, is_windows):
"""converts a series of path tokens into a joinable path"""
tokens2 = []
is_mac_linux = not is_windows
for itoken, token in enumerate(tokens):
# this is technically legal...
# INCLUDE '/testdir/dir1/dir2/*/myfile.dat'
assert '*' not in token, '* in path not supported; tokens=%s' % tokens
if is_windows:
assert '$' not in token, '$ in path not supported; tokens=%s' % tokens
else:
assert '%' not in token, '%% in path not supported; tokens=%s' % tokens
if itoken == 0 and is_mac_linux and ':' in token:
## no C:/dir/model.bdf on linux/mac
#raise SyntaxError('token cannot include colons (:); token=%r; tokens=%s' % (
#token, str(tokens)))
# this has an environment variable or a drive letter
#print(token)
stokens = token.split(':')
if len(stokens) != 2:
msg = "len(stokens)=%s must be 2; stokens=%s" % (len(stokens), stokens)
raise SyntaxError(msg)
if len(stokens[0]) == 1:
if len(stokens[1]) not in [0, 1]:
raise SyntaxError('tokens=%r token=%r stokens=%s stoken[1]=%r len=%r' % (
tokens, token, stokens, stokens[1], len(stokens[1])))
if len(stokens[0]) < 2:
raise SyntaxError('token cannot include colons (:); token=%r; tokens=%s' % (
token, str(tokens)))
else:
# variables in Windows are not case sensitive; not handled?
token0 = stokens[0]
if is_windows:
assert '$' not in stokens[0], token0
assert '%' not in stokens[0], token0
if '%' in token0:
assert token0[0] == '%', token0
assert token0[-1] == '%', token0
token0 = '%' + token0 + '%'
else:
token0 = '$' + token0
#tokeni = os.path.expandvars('$' + stokens[0])
tokeni = os.path.expandvars(token0)
if '$' in tokeni:
raise SyntaxError('tokeni=%r has a $ in it after expanding (token0=%r)...\n'
'tokens=%s stokens=%s' % (tokeni, token0, tokens, stokens))
tokensi = PureWindowsPath(tokeni).parts
else:
if '$' in token0:
assert token0[0] == '$', token0
else:
token0 = '$' + token0
assert '%' not in stokens[0], token0
tokeni = os.path.expandvars(token0)
tokensi = PurePosixPath(tokeni).parts
tokens2.extend(tokensi)
tokens2.append(stokens[1])
elif ':' in token:
# Windows
# this has an environment variable or a drive letter
stokens = token.split(':')
if len(stokens[0]) == 1:
if len(stokens[1]) not in [0, 1]:
raise SyntaxError('tokens=%r token=%r stokens=%s stoken[1]=%r len=%r' % (
tokens, token, stokens, stokens[1], len(stokens[1])))
# drive letter
if itoken != 0:
raise SyntaxError('the drive letter is in the wrong place; '
'itoken=%s; token=%r; stoken=%s; tokens=%s' % (
itoken, token, stokens, tokens))
tokens2.append(token)
else:
# variables in Windows are not case sensitive; not handled?
environment_vars_to_expand = stokens[:-1]
if len(environment_vars_to_expand) != 1:
raise SyntaxError(
'Only 1 environment variable can be expanded; '
'environment_vars_to_expand = %r' % environment_vars_to_expand)
for env_var in environment_vars_to_expand:
if env_var.strip('$ %') not in os.environ:
environment_variables = list(os.environ.keys())
environment_variables.sort()
raise SyntaxError("Cant find environment variable=%r"
'\nenviron=%s' % (env_var, environment_variables))
env_vari = os.path.expandvars('$' + env_var.strip('%'))
if '$' in env_vari:
raise SyntaxError('env_vari=%r has a $ in it after expanding (token0=%r)...\n'
'tokens=%s stokens=%s' % (env_vari, env_var, tokens, stokens))
if is_windows:
tokensi = PureWindowsPath(env_vari).parts
else:
tokensi = PurePosixPath(env_vari).parts
tokens2.extend(tokensi)
tokens2.append(stokens[-1])
else:
# standard
tokens2.append(token)
return tokens2
def run(self, gen):
for link, template in self.mapping.items():
link = PurePosixPath(link)
def render(gs, template=template):
return gs.render_template(template)
gen.add_render(link, link, render)
