def test_addPyListings(self): """ L{tree.addPyListings} accepts a document with nodes with their I{class} attribute set to I{py-listing} and replaces those nodes with Python source listings from the file given by the node's I{href} attribute. """ listingPath = FilePath(self.mktemp()) listingPath.setContent('def foo():\n pass\n') parent = dom.Element('div') listing = dom.Element('a') listing.setAttribute('href', listingPath.basename()) listing.setAttribute('class', 'py-listing') parent.appendChild(listing) tree.addPyListings(parent, listingPath.dirname()) expected = """\ <div><div class="py-listing"><pre><p class="py-linenumber">1 2 </p><span class="py-src-keyword">def</span> <span class="py-src-identifier">foo</span>(): <span class="py-src-keyword">pass</span> </pre><div class="caption"> - <a href="temp"><span class="filename">temp</span></a></div></div></div>""" self.assertEqual(parent.toxml(), expected)
def _parse_file(self, kind, args): if args == "-": get_file = lambda: stdout else: path = FilePath(args) get_file = lambda: LogFile( path.basename(), path.dirname(), rotateLength=1024 * 1024 * 1024, maxRotatedFiles=10, ) return lambda reactor: FileDestination(get_file())
def do_urlextract(dest, url): global dsklst dest=FilePath(dest) # Don't do this if not mounted! mntpnt=dsklst['/'].real_mountpoint() if not os.path.ismount(mntpnt): return False if not dest.isdir(): return False try: uh=urllib2.urlopen(url) tf=tarfile.open(mode='r|*',fileobj=uh) os.chroot(mntpnt) os.chdir(os.path.join(dest.dirname(),dest.basename())) tf.extractall() except: traceback.print_exc() os.chdir('/')
def __init__(self, *argz, **kwz): super(Logtail, self).__init__(*argz, **kwz) self.exclude = self.conf.monitor_exclude or list() if isinstance(self.exclude, types.StringTypes): self.exclude = [self.exclude] self.exclude = map(re.compile, self.exclude) paths_watch = self.paths_watch = dict() self.paths_pos, self.paths_buff = dict(), dict() masks, paths = self.conf.monitor, list() if isinstance(masks, bytes): masks = [masks] for mask in masks: mask_patterns = self.glob_alter(mask) for mask_raw in mask_patterns: mask = FilePath(mask_raw) # All matched parent dirs - like /x/y/z for /x/*/z/file* - are watched for pattern # Note that watchers won't be auto-added for /x/m/z, if it'll be created later on paths_ext = list( (path.realpath(), mask.basename()) for path in it.imap(FilePath, glob(mask.dirname())) ) paths.extend(paths_ext) # If full pattern already match something, watch it if it's a dir - /x/dir1 for /x/dir* # Note that watchers won't be auto-added for /x/dir2, if it'll be created later on if paths_ext: # no point going deeper if parent dirs don't exist paths.extend( (path.realpath(), '*') for path in it.imap(FilePath, glob(mask_raw)) if path.realpath().isdir() ) # Aggregate path masks by-realpath for path, mask in paths: if not path.isdir(): log.debug('Skipping special path: {}'.format(path)) continue if path not in paths_watch: paths_watch[path] = {mask} else: paths_watch[path].add(mask) self.notifier_restart()
def test_addPyListingsSkipLines(self): """ If a node with the I{py-listing} class also has a I{skipLines} attribute, that number of lines from the beginning of the source listing are omitted. """ listingPath = FilePath(self.mktemp()) listingPath.setContent('def foo():\n pass\n') parent = dom.Element('div') listing = dom.Element('a') listing.setAttribute('href', listingPath.basename()) listing.setAttribute('class', 'py-listing') listing.setAttribute('skipLines', 1) parent.appendChild(listing) tree.addPyListings(parent, listingPath.dirname()) expected = """\ <div><div class="py-listing"><pre><p class="py-linenumber">1 </p> <span class="py-src-keyword">pass</span> </pre><div class="caption"> - <a href="temp"><span class="filename">temp</span></a></div></div></div>""" self.assertEqual(parent.toxml(), expected)
def _parse_file(self, kind, arg_text): # Reserve the possibility of an escape character in the future. \ is # the standard choice but it's the path separator on Windows which # pretty much ruins it in this context. Most other symbols already # have some shell-assigned meaning which makes them treacherous to use # in a CLI interface. Eliminating all such dangerous symbols leaves # approximately @. if u"@" in arg_text: raise ValueError( u"Unsupported escape character (@) in destination text ({!r})." .format(arg_text), ) arg_list = arg_text.split(u",") path_name = arg_list.pop(0) if path_name == "-": get_file = lambda: stdout else: path = FilePath(path_name) rotate_length = int( self._get_arg( u"rotate_length", 1024 * 1024 * 1024, arg_list, )) max_rotated_files = int( self._get_arg( u"max_rotated_files", 10, arg_list, )) get_file = lambda: LogFile( path.basename(), path.dirname(), rotateLength=rotate_length, maxRotatedFiles=max_rotated_files, ) return lambda reactor: FileDestination(get_file())
class _WordsController(object): """ Basic controller that manages registered zones and manages name assignment by filtering on the registered IP subnet. """ def __init__(self, data_dir="data", word_count=3): self.data_dir = FilePath(data_dir) self.word_count = word_count self.separator = b"-" self.data = dict() self.proxied = False self.app = Klein() # Routing for the HTTP API @self.app.route("/") def readme(request): """ Public README page """ request.setHeader('Content-Type', 'text/plain') return self.get_readme() @self.app.route("/register") def register(request): """ A GET request from a registered zone's subnet is sufficient to trigger a name assignment. We support mainly the `domain` parameter so that it's easy to reach from yggdrasil and other networks. """ if self.proxied: # Trust X-Forwarded-For headers if set up in the config from twisted.web.http import _XForwardedForRequest request = _XForwardedForRequest(request) hostname = request.args.get(b'domain', [False])[0] if not hostname: hostname = request.getRequestHostname() try: ip = request.getClientAddress().host.decode("utf-8") except: ip = None request.setHeader('Content-Type', 'text/plain') return self.register_ip(hostname, ip) @lru_cache(10240) def get_readme(self): return ( FilePath(self.data_dir.dirname()) .child("README.md") .getContent() .decode("utf-8") ) def register_zone(self, zone, subnet): """ Register a zone with the controller along with its allowed subnet. @param zone: the DNS zone without any trailing or leading dots. @type zone: C{bytes} @param subnet: the subnet that will be allowed to register names. @type subnet: L{ipaddress.ip_network} """ assert isinstance( subnet, (IPv4Network, IPv6Network) ), "'{}' is not ipaddress.ip_network".format(ip_network) log.debug("Registered Zone {zone} | {subnet}", zone=zone, subnet=subnet) self.data[zone] = subnet def register_ip(self, zone, ip): """ Actually register a name for a given IP. @returns: A resource or a byte-string depending on the action being successful or not. Possible HTTP codes are: 200 (OK) 400 (Bad request --> no such zone) 403 (Forbidden --> out of subnet) 507 (Insufficient storage --> somehow the name space is kinda full) """ if zone not in self.data: return ErrorPage( 400, "Bad Request", "No such zone, consider hosting your own!" ) try: return self.get_assign_name(zone, ip) + b"." + zone + b"\n" except ValueError: return ForbiddenResource("Your IP is not allowed to use this resource.") except LookupError: return ErrorPage( 507, "Insufficient Storage", "It looks like this zone is getting full. Consider hosting your own!", ) except Exception as ex: log.error("Error registering {zone} | {ip}", zone=zone, ip=ip) log.failure(ex) return ErrorPage(500, "Internal Error", "Something odd happened!") @lru_cache(maxsize=1024) def get_assign_name(self, zone, ip): ipaddr = ip_address(ip) # collisions should be handled by iterator it = hash_parts_generator(ip, self.word_count, len(self.all_words)) for h in it: words = self.separator.join([self.all_words[i] for i in h]) record = self.name_to_record(zone, words) if record.exists(): try: record_addr = ip_address(record.getContent().decode("utf-8")) if record_addr == ipaddr: # Already registered return words except: # If it contains invalid data, reuse break else: break else: raise LookupError("Can't assign name in '{}' for IP '{}'".format(zone, ip)) record.parent().makedirs(ignoreExistingDirectory=True) record.setContent(ipaddr.compressed.encode("utf-8")) return words @property @lru_cache(maxsize=1024) def all_words(self): """ Return a list of '\n' separated byte-strings ignoring those lines starting with '#'. """ return [ i for i in self.data_dir.child("word_list").getContent().split(b"\n") if b"#" not in i ] def name_to_record(self, zone, words): """ Helper class that returns a FilePath object to the file that should contain the resulting IP. """ parts = [zone] + words.split(b"-") return self.data_dir.descendant(parts) @lru_cache(maxsize=1024) def words_to_IP(self, zone, words): """ Get the IP associated with certain words in a zone if registered. """ assert zone, "Empty zone passed" record = self.name_to_record(zone, words) if record.exists() and record.isfile(): return ip_address(record.getContent().decode("utf-8")) raise ValueError("Name not registered '{}'".format(words))