def test_addPyListings(self):
"""
L{tree.addPyListings} accepts a document with nodes with their I{class}
attribute set to I{py-listing} and replaces those nodes with Python
source listings from the file given by the node's I{href} attribute.
"""
listingPath = FilePath(self.mktemp())
listingPath.setContent('def foo():\n pass\n')
parent = dom.Element('div')
listing = dom.Element('a')
listing.setAttribute('href', listingPath.basename())
listing.setAttribute('class', 'py-listing')
parent.appendChild(listing)
tree.addPyListings(parent, listingPath.dirname())
expected = """\
<div><div class="py-listing"><pre><p class="py-linenumber">1
2
</p><span class="py-src-keyword">def</span> <span class="py-src-identifier">foo</span>():
<span class="py-src-keyword">pass</span>
</pre><div class="caption"> - <a href="temp"><span class="filename">temp</span></a></div></div></div>"""
self.assertEqual(parent.toxml(), expected)
def test_addPyListings(self):
"""
L{tree.addPyListings} accepts a document with nodes with their I{class}
attribute set to I{py-listing} and replaces those nodes with Python
source listings from the file given by the node's I{href} attribute.
"""
listingPath = FilePath(self.mktemp())
listingPath.setContent('def foo():\n pass\n')
parent = dom.Element('div')
listing = dom.Element('a')
listing.setAttribute('href', listingPath.basename())
listing.setAttribute('class', 'py-listing')
parent.appendChild(listing)
tree.addPyListings(parent, listingPath.dirname())
expected = """\
<div><div class="py-listing"><pre><p class="py-linenumber">1
2
</p><span class="py-src-keyword">def</span> <span class="py-src-identifier">foo</span>():
<span class="py-src-keyword">pass</span>
</pre><div class="caption"> - <a href="temp"><span class="filename">temp</span></a></div></div></div>"""
self.assertEqual(parent.toxml(), expected)
def _parse_file(self, kind, args):
if args == "-":
get_file = lambda: stdout
else:
path = FilePath(args)
get_file = lambda: LogFile(
path.basename(),
path.dirname(),
rotateLength=1024 * 1024 * 1024,
maxRotatedFiles=10,
)
return lambda reactor: FileDestination(get_file())
def do_urlextract(dest, url):
global dsklst
dest=FilePath(dest)
# Don't do this if not mounted!
mntpnt=dsklst['/'].real_mountpoint()
if not os.path.ismount(mntpnt):
return False
if not dest.isdir():
return False
try:
uh=urllib2.urlopen(url)
tf=tarfile.open(mode='r|*',fileobj=uh)
os.chroot(mntpnt)
os.chdir(os.path.join(dest.dirname(),dest.basename()))
tf.extractall()
except:
traceback.print_exc()
os.chdir('/')
def __init__(self, *argz, **kwz):
super(Logtail, self).__init__(*argz, **kwz)
self.exclude = self.conf.monitor_exclude or list()
if isinstance(self.exclude, types.StringTypes): self.exclude = [self.exclude]
self.exclude = map(re.compile, self.exclude)
paths_watch = self.paths_watch = dict()
self.paths_pos, self.paths_buff = dict(), dict()
masks, paths = self.conf.monitor, list()
if isinstance(masks, bytes): masks = [masks]
for mask in masks:
mask_patterns = self.glob_alter(mask)
for mask_raw in mask_patterns:
mask = FilePath(mask_raw)
# All matched parent dirs - like /x/y/z for /x/*/z/file* - are watched for pattern
# Note that watchers won't be auto-added for /x/m/z, if it'll be created later on
paths_ext = list( (path.realpath(), mask.basename())
for path in it.imap(FilePath, glob(mask.dirname())) )
paths.extend(paths_ext)
# If full pattern already match something, watch it if it's a dir - /x/dir1 for /x/dir*
# Note that watchers won't be auto-added for /x/dir2, if it'll be created later on
if paths_ext: # no point going deeper if parent dirs don't exist
paths.extend( (path.realpath(), '*')
for path in it.imap(FilePath, glob(mask_raw))
if path.realpath().isdir() )
# Aggregate path masks by-realpath
for path, mask in paths:
if not path.isdir():
log.debug('Skipping special path: {}'.format(path))
continue
if path not in paths_watch:
paths_watch[path] = {mask}
else: paths_watch[path].add(mask)
self.notifier_restart()
def test_addPyListingsSkipLines(self):
"""
If a node with the I{py-listing} class also has a I{skipLines}
attribute, that number of lines from the beginning of the source
listing are omitted.
"""
listingPath = FilePath(self.mktemp())
listingPath.setContent('def foo():\n pass\n')
parent = dom.Element('div')
listing = dom.Element('a')
listing.setAttribute('href', listingPath.basename())
listing.setAttribute('class', 'py-listing')
listing.setAttribute('skipLines', 1)
parent.appendChild(listing)
tree.addPyListings(parent, listingPath.dirname())
expected = """\
<div><div class="py-listing"><pre><p class="py-linenumber">1
</p> <span class="py-src-keyword">pass</span>
</pre><div class="caption"> - <a href="temp"><span class="filename">temp</span></a></div></div></div>"""
self.assertEqual(parent.toxml(), expected)
def test_addPyListingsSkipLines(self):
"""
If a node with the I{py-listing} class also has a I{skipLines}
attribute, that number of lines from the beginning of the source
listing are omitted.
"""
listingPath = FilePath(self.mktemp())
listingPath.setContent('def foo():\n pass\n')
parent = dom.Element('div')
listing = dom.Element('a')
listing.setAttribute('href', listingPath.basename())
listing.setAttribute('class', 'py-listing')
listing.setAttribute('skipLines', 1)
parent.appendChild(listing)
tree.addPyListings(parent, listingPath.dirname())
expected = """\
<div><div class="py-listing"><pre><p class="py-linenumber">1
</p> <span class="py-src-keyword">pass</span>
</pre><div class="caption"> - <a href="temp"><span class="filename">temp</span></a></div></div></div>"""
self.assertEqual(parent.toxml(), expected)
def _parse_file(self, kind, arg_text):
# Reserve the possibility of an escape character in the future. \ is
# the standard choice but it's the path separator on Windows which
# pretty much ruins it in this context. Most other symbols already
# have some shell-assigned meaning which makes them treacherous to use
# in a CLI interface. Eliminating all such dangerous symbols leaves
# approximately @.
if u"@" in arg_text:
raise ValueError(
u"Unsupported escape character (@) in destination text ({!r})."
.format(arg_text), )
arg_list = arg_text.split(u",")
path_name = arg_list.pop(0)
if path_name == "-":
get_file = lambda: stdout
else:
path = FilePath(path_name)
rotate_length = int(
self._get_arg(
u"rotate_length",
1024 * 1024 * 1024,
arg_list,
))
max_rotated_files = int(
self._get_arg(
u"max_rotated_files",
10,
arg_list,
))
get_file = lambda: LogFile(
path.basename(),
path.dirname(),
rotateLength=rotate_length,
maxRotatedFiles=max_rotated_files,
)
return lambda reactor: FileDestination(get_file())
class _WordsController(object):
"""
Basic controller that manages registered zones and manages name assignment
by filtering on the registered IP subnet.
"""
def __init__(self, data_dir="data", word_count=3):
self.data_dir = FilePath(data_dir)
self.word_count = word_count
self.separator = b"-"
self.data = dict()
self.proxied = False
self.app = Klein()
# Routing for the HTTP API
@self.app.route("/")
def readme(request):
"""
Public README page
"""
request.setHeader('Content-Type', 'text/plain')
return self.get_readme()
@self.app.route("/register")
def register(request):
"""
A GET request from a registered zone's subnet is sufficient to
trigger a name assignment.
We support mainly the `domain` parameter so that it's easy to reach
from yggdrasil and other networks.
"""
if self.proxied:
# Trust X-Forwarded-For headers if set up in the config
from twisted.web.http import _XForwardedForRequest
request = _XForwardedForRequest(request)
hostname = request.args.get(b'domain', [False])[0]
if not hostname:
hostname = request.getRequestHostname()
try:
ip = request.getClientAddress().host.decode("utf-8")
except:
ip = None
request.setHeader('Content-Type', 'text/plain')
return self.register_ip(hostname, ip)
@lru_cache(10240)
def get_readme(self):
return (
FilePath(self.data_dir.dirname())
.child("README.md")
.getContent()
.decode("utf-8")
)
def register_zone(self, zone, subnet):
"""
Register a zone with the controller along with its allowed subnet.
@param zone: the DNS zone without any trailing or leading dots.
@type zone: C{bytes}
@param subnet: the subnet that will be allowed to register names.
@type subnet: L{ipaddress.ip_network}
"""
assert isinstance(
subnet, (IPv4Network, IPv6Network)
), "'{}' is not ipaddress.ip_network".format(ip_network)
log.debug("Registered Zone {zone} | {subnet}", zone=zone, subnet=subnet)
self.data[zone] = subnet
def register_ip(self, zone, ip):
"""
Actually register a name for a given IP.
@returns: A resource or a byte-string depending on the action being
successful or not.
Possible HTTP codes are:
200 (OK)
400 (Bad request --> no such zone)
403 (Forbidden --> out of subnet)
507 (Insufficient storage --> somehow the name space is kinda full)
"""
if zone not in self.data:
return ErrorPage(
400, "Bad Request", "No such zone, consider hosting your own!"
)
try:
return self.get_assign_name(zone, ip) + b"." + zone + b"\n"
except ValueError:
return ForbiddenResource("Your IP is not allowed to use this resource.")
except LookupError:
return ErrorPage(
507,
"Insufficient Storage",
"It looks like this zone is getting full. Consider hosting your own!",
)
except Exception as ex:
log.error("Error registering {zone} | {ip}", zone=zone, ip=ip)
log.failure(ex)
return ErrorPage(500, "Internal Error", "Something odd happened!")
@lru_cache(maxsize=1024)
def get_assign_name(self, zone, ip):
ipaddr = ip_address(ip)
# collisions should be handled by iterator
it = hash_parts_generator(ip, self.word_count, len(self.all_words))
for h in it:
words = self.separator.join([self.all_words[i] for i in h])
record = self.name_to_record(zone, words)
if record.exists():
try:
record_addr = ip_address(record.getContent().decode("utf-8"))
if record_addr == ipaddr:
# Already registered
return words
except:
# If it contains invalid data, reuse
break
else:
break
else:
raise LookupError("Can't assign name in '{}' for IP '{}'".format(zone, ip))
record.parent().makedirs(ignoreExistingDirectory=True)
record.setContent(ipaddr.compressed.encode("utf-8"))
return words
@property
@lru_cache(maxsize=1024)
def all_words(self):
"""
Return a list of '\n' separated byte-strings ignoring those lines
starting with '#'.
"""
return [
i
for i in self.data_dir.child("word_list").getContent().split(b"\n")
if b"#" not in i
]
def name_to_record(self, zone, words):
"""
Helper class that returns a FilePath object to the file that should
contain the resulting IP.
"""
parts = [zone] + words.split(b"-")
return self.data_dir.descendant(parts)
@lru_cache(maxsize=1024)
def words_to_IP(self, zone, words):
"""
Get the IP associated with certain words in a zone if registered.
"""
assert zone, "Empty zone passed"
record = self.name_to_record(zone, words)
if record.exists() and record.isfile():
return ip_address(record.getContent().decode("utf-8"))
raise ValueError("Name not registered '{}'".format(words))
