class Jinja2(Plugin): """Plugin to translate jinja2 templates to html files.""" implements(pagd.interfaces.ITemplate) extensions = ['jinja2', 'j2'] def __init__(self): try: kwargs = { 'loader': FileSystemLoader(self['sitepath']), 'auto_reload': self['siteconfig'].get('jinja2.auto_reload', False), 'cache_size': self['siteconfig'].get('jinja2.cache_size', 50), 'extensions': self['siteconfig'].get('jinja2.extensions', ()), } self.env = Environment(**kwargs) except NameError: self.env = None def render(self, page): tmpl = self._get_template(page.templatefile) return tmpl.render(page.context) if tmpl != None else '' def _get_template(self, template_name, globals=None): if self.env != None: return self.env.get_template(template_name, globals=globals) else: return None
class ExpressionPy( Plugin ): """Plugin evaluates python expression, converts the result into string and supplies escape filtering like url-encode, xml-encode, html-encode, stripping whitespaces on expression substitution.""" implements( ITayraExpression ) def eval( self, mach, text, globals_, locals_ ): """:meth:`tayra.interfaces.ITayraExpression.eval` interface method.""" return str( eval( text, globals_, locals_ )) def filter( self, mach, name, text ): """:meth:`tayra.interfaces.ITayraExpression.filter` interface method.""" handler = getattr( self, name, self.default ) return handler( mach, text ) def u( self, mach, text ): """Assume text as url and quote using urllib.parse.quote()""" return urllib.parse.quote( text ) xmlescapes = { '&' : '&', '>' : '>', '<' : '<', '"' : '"', "'" : ''', } def x( self, mach, text ): """Assume text as XML, and apply escape encoding.""" return re.sub( r'([&<"\'>])', lambda m: self.xmlescapes[m.group()], text ) def h( self, mach, text ): """Assume text as HTML and apply html.escape( quote=True )""" return html.escape( text, quote=True ) def t( self, mach, text ): """Strip whitespaces before and after text using strip()""" return text.strip() def default( self, mach, text ): """Default handler. Return ``None`` so that runtime will try other plugins implementing the filter.""" return None #---- ISettings interface methods @classmethod def default_settings( cls ): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings( cls, sett ): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return sett
class StaticView( Plugin ): """Plugin to serve static files over HTTP.""" implements( IHTTPView ) def __init__( self, viewname, view ): """:meth:`pluggdapps.web.interfaces.IHTTPView.__init__` interface method. """ self.viewname = viewname self.view = view def __call__( self, request, c ): """:meth:`pluggdapps.web.interfaces.IHTTPView.__call__` interface method. """ resp = request.response assetpath = h.abspath_from_asset_spec( self.view['rootloc'] ) docfile = join( assetpath, request.matchdict['path'] ) if docfile and isfile( docfile ) : # Collect information about the document, for response. resp.set_status( b'200' ) stat = os.stat( docfile ) (typ, enc) = mimetypes.guess_type( docfile ) if typ : resp.media_type = typ if enc : resp.content_coding = enc # Populate the context c.etag['body'] = open( docfile, 'rb' ).read() c['last_modified'] = h.http_fromdate( stat.st_mtime ) cc = ('public,max-age=%s' % str(self['max_age']) ).encode('utf-8') resp.set_header( 'cache_control', cc ) # Send Response resp.write( c['body'] ) resp.flush( finishing=True ) else : resp.pa.logwarn( "Not found %r" % docfile ) resp.set_status( b'404' ) resp.flush( finishing=True ) def onfinish( self, request ): """:meth:`pluggdapps.web.interfaces.IHTTPView.__call__` interface method. """ pass #---- ISettings interface methods @classmethod def default_settings( cls ): return _default_settings @classmethod def normalize_settings( cls, sett ): sett['max_age'] = h.asint( sett['max_age'] ) return sett
class ExpressionEvalPy(Plugin): """Plugin evaluates python expression and discards the resulting value. Doesn't supply any filtering rules.""" implements(ITayraExpression) def eval(self, mach, text, globals_, locals_): """:meth:`tayra.interfaces.ITayraExpression.eval` interface method.""" eval(text, globals_, locals_) return ''
class Pandoc(Plugin): """Plugin that can translate different content formats into html format. Under development, contributions are welcome. Make sure that pandoc tool is installed and available through shell command line interface. Supports reStructuredText and Markdown content formats. """ implements(IContent) def __init__(self): self.cmd = join(os.environ['HOME'], '.cabal', 'bin', 'pandoc') #---- IContent interface methods. def articles(self, page): if not isfile(self.cmd): raise Exception('Not found %r' % self.cmd) articles = [] for fpath in page.contentfiles: if not isfile(fpath): continue _, ext = splitext(fpath) ftype = page.context.get('filetype', ext.lstrip('.')) metadata, content = self.parsers[ftype](self, fpath) articles.append((fpath, metadata, content)) return articles def rst2html(self, fpath): return self.pandoc(self.cmd, fpath, 'rst', 'html') def md2html(self, fpath): return self.pandoc(self.cmd, fpath, 'markdown', 'html') def pandoc(self, cmd, fpath, fromm, to): fd, tfile = tempfile.mkstemp() os.system(cmd + ' --highlight-style kate -f %s -t %s -o "%s" "%s"' % (fromm, to, tfile, fpath)) return {}, open(fd).read() parsers = { 'rst': rst2html, 'markdown': md2html, 'mdown': md2html, 'mkdn': md2html, 'md': md2html, 'mkd': md2html, 'mdwn': md2html, 'mdtxt': md2html, 'mdtext': md2html, }
class Hg(Plugin): """Plugin to fetch page's context from Hg, if available, and compose them into a dictionary of page's metadata. """ implements(pagd.interfaces.IXContext) cmd = [ "hg", "log", '--template "{author|person} ; {author|email} ; {date|age}\n"' ] def fetch(self, page): """Provides the following context from git repository log for the file, .. code-block:: python { 'author' : <string>, 'email' : <string>, 'createdon' : <date-string>, 'last_modified' : <date-string> } - `author` will be original author who created the file. - `email` will be email-id of the original author, - `date-string` will be of the format 'Mon Jun 10, 2013' and will refer to author's local-time. """ scale = page.context.get('age_scale') for fpath in page.contentfiles: try: logs = subprocess.check_output(self.cmd + [fpath], stderr=stdout) logs = logs.splitlines() _, _, last_modified = logs[0].decode('utf-8').split(" ; ") author, email, createdon = logs[-1].decode('utf-8').split( " ; ") except: author, email, createdon, last_modified = '', '', '', '' author, email = author.strip(' "\''), email.strip(' "\''), createdon, last_modified = createdon.strip(), last_modified.strip() createdon = age(int(createdon), scale=scale) if createdon else '' last_modified = age( int(last_modified), scale=scale ) \ if last_modified else '' return { 'author': author, 'email': email, 'createdon': createdon, 'last_modified': last_modified }
class Gen( Singleton ): """Sub-command plugin to generate static web site at the given target directory. If a target directory is not specified, it uses layout's default target directory. For more information refer to corresponding layout plugin's documentation. """ implements( ICommand ) cmd = 'gen' description = 'Generate a static site for the give layout and content' #---- ICommand API def subparser( self, parser, subparsers ): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method. """ self.subparser = subparsers.add_parser( self.cmd, description=self.description ) self.subparser.set_defaults( handler=self.handle ) self.subparser.add_argument( '-g', '--config-path', dest='configfile', default='config.json', help='The configuration used to generate the site') self.subparser.add_argument( '-t', '--build-target', dest='buildtarget', default='.', help="Location of target site that contains generated html.") self.subparser.add_argument( '-r', '--regen', dest='regen', action='store_true', default=False, help='Regenerate all site pages.') return parser def handle( self, args ): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method. Instantiate a layout plugin and apply generate() method on the instantiated plugin. ``sitepath`` and ``siteconfig`` references willbe passed as settings dictionary. """ siteconfig = join( args.sitepath, args.configfile ) siteconfig = json2dict( siteconfig ) layoutname = siteconfig.get( 'layout', args.layout ) sett = { 'sitepath' : args.sitepath, 'siteconfig' : siteconfig } layout = self.qp( ILayout, layoutname, settings=sett ) self.pa.loginfo( "Generating site at [%s] with layout [%s] ..." % (args.sitepath, layoutname)) layout.generate( abspath(args.buildtarget), regen=args.regen ) self.pa.loginfo("... complete")
class UnitTest(Singleton): """Sub-command to run available unittests.""" implements(ICommand) description = "Run one or more unittest." cmd = 'unittest' #---- ICommand methods def subparser(self, parser, subparsers): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method.""" self.subparser = subparsers.add_parser(self.cmd, description=self.description) self.subparser.set_defaults(handler=self.handle) self._arguments(self.subparser) def handle(self, args): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method.""" tl = unittest.TestLoader() tr = unittest.TestResult() v = 2 if args.verbosity else 1 runner = unittest.TextTestRunner(verbosity=v) if args.module == 'all': suite = tl.discover(TESTDIR) runner.run(suite) elif args.module: suite = tl.loadTestsFromName('pluggdapps.tests.' + args.module) runner.run(suite) def _arguments(self, parser): parser.add_argument('module', help='unittest module') parser.add_argument("-v", action="store_true", dest="verbosity", help="Verbosity for running test cases") return parser #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return sett
class Tayra(Plugin): """Plugin to translate tayra templates to html files.""" implements(pagd.interfaces.ITemplate) extensions = ['ttl', 'tayra', 'tmpl'] # default template def __init__(self): setts = h.settingsfor('tayra.ttlcompiler.', self['siteconfig']) setts.update(helpers=['pagd.h'], debug=True) self.ttlplugin = self.qp(pluggdapps.interfaces.ITemplate, 'tayra.TTLCompiler', settings=setts) def render(self, page): return self.ttlplugin.render(page.context, file=page.templatefile)
class Mako(Plugin): """Plugin to translate mako templates to html files.""" implements(pagd.interfaces.ITemplate) extensions = ['mako'] def __init__(self): self.kwargs = { 'module_directory' : \ self['siteconfig'].get('mako.module_directory', None), } def render(self, page): try: mytemplate = Template(page.templatefile, **self.kwargs) buf = io.StringIO() mytemplate.render_context(Context(buf, **page.context)) return buf.getvalue() except NameError: return ''
class NewPage(Singleton): """Sub-command plugin to generate a new content page under layout-sitepath. """ implements(ICommand) cmd = 'newpage' description = 'Create a new content page.' #---- ICommand API def subparser(self, parser, subparsers): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method. """ self.subparser = subparsers.add_parser(self.cmd, description=self.description) self.subparser.set_defaults(handler=self.handle) self.subparser.add_argument( '-g', '--config-path', dest='configfile', default='config.json', help='The configuration used to generate the site') self.subparser.add_argument( 'pagename', nargs=1, help='File name, extension not provided defaults to rst') return parser def handle(self, args): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method. Instantiate a layout plugin and apply newpage() method on the instantiated plugin. ``sitepath`` and ``siteconfig`` references willbe passed as settings dictionary. """ siteconfig = join(args.sitepath, args.configfile) sett = {'sitepath': args.sitepath, 'siteconfig': siteconfig} layout = self.qp(ILayout, layoutname, settings=sett) layout.newpage(pagename)
class Native(Plugin): """Plugin that can translate different content formats into html format using python native modules. Uses function APIs defined under :mod:`pagd.contents` module, except for ttl2html. Supports reStructuredText, Markdown, plain-text, plain-html, tayra-templates text. Note that in case of a TTL file, it is interpreted as page content and not as template for this or any other page-contents. """ implements(IContent) def __init__(self): setts = h.settingsfor('tayra.ttlcompiler.', self['siteconfig']) setts.update(debug=True) self.ttlplugin = \ self.qp( pluggdapps.interfaces.ITemplate, 'tayra.TTLCompiler', settings=setts ) #---- IContent interface methods. def articles(self, page): """For ``page``, an instance of :class:`Page` class, using its ``contentfiles`` attribute, translate each file's text to html and return a corresponding list of articles. Where each element in the article is a tuple of, :: ( article's fpath, dictionary-of-metadata, html-text ) """ articles = [] for fpath in page.contentfiles: if not isfile(fpath): continue _, ext = splitext(fpath) ftype = page.context.get('filetype', ext.lstrip('.')) metadata, html = self.parsers[ftype](self, fpath, page) articles.append((fpath, metadata, html)) return articles def rst2html(self, fpath, page): from pagd.contents import rst2html return rst2html(fpath, page) def md2html(self, fpath, page): from pagd.contents import md2html return md2html(fpath, page) def html2html(self, fpath, page): from pagd.contents import html2html return html2html(fpath, page) def text2html(self, fpath, page): from pagd.contents import text2html return text2html(fpath, page) def ttl2html(self, fpath, page): """``fpath`` is identified as a file containing tayra template text. If generated html contains <meta> tag elements, it will be used as source of meta-data information. And return a tuple of (metadata, content). Content is HTML text.""" from pagd.contents import html2metadata html = self.ttlplugin.render(page.context, file=fpath) metadata = html2metadata(html) return metadata, html parsers = { 'rst': rst2html, 'markdown': md2html, 'mdown': md2html, 'mkdn': md2html, 'md': md2html, 'mkd': md2html, 'mdwn': md2html, 'mdtxt': md2html, 'mdtext': md2html, 'txt': text2html, 'text': text2html, 'html': html2html, 'htm': html2html, 'ttl': ttl2html, }
class GZipOutBound(Plugin): """Out-bound transformer to compress response entity using gzip compression technology. Performs gzip encoding if, * b'gzip' is in ``content_encoding`` response header. * if type in ``content_type`` response header is `text` or `application`. * if ``content_type`` response header do not indicate that ``data`` is already compressed variant. If ``data`` successfully gets gzipped, then ``etag`` response header value is suffixed with b';gzip'. If gzip encoding is not applied on ``data``, it is made sure that content_encoding response header does not contain b'gzip' value, """ implements(IHTTPOutBound) #---- IHTTPOutBound method APIs def transform(self, request, data, finishing=True): """:meth:`pluggdapps.web.interfaces.IHTTPOutBound.transform` interface method.""" resp = request.response ctype = resp.headers.get('content_type', b'') cenc = resp.headers.get('content_encoding', b'') etag = resp.headers.get('etag', b'') # Compress only if content-type is 'text/*' or 'application/*' if self._is_gzip(data, cenc, ctype, resp.statuscode): data = self._gzip(data) # etag is always double-quoted. resp.set_header('etag', etag[:-1] + b';gzip"') if etag else None else: enc = cenc.replace(b'gzip', b'') resp.set_header('content_encoding', enc) return data #-- local methods def _is_gzip(self, data, enc, typ, status): return (bool(data) and b'gzip' in enc and (typ.startswith(b'text/') or typ.startswith(b'application/')) and b'zip' not in typ) def _gzip(self, data): buf = BytesIO() gzipper = gzip.GzipFile(mode='wb', compresslevel=self['level'], fileobj=buf) gzipper.write(data) gzipper.close() buf.seek(0) data = buf.getvalue() buf.close() return data #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method. """ return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method. """ sett['level'] = h.asint(sett['level']) return sett
class Ls(Singleton): """Sub-command plugin for pa-script to list internal state of pluggdapps' virtual environment. Instead of using this command, use `sh` sub-command to start a shell and introspect pluggdapps environment. """ implements(ICommand) description = 'list various information about Pluggdapps environment.' cmd = 'ls' #---- ICommand API def subparser(self, parser, subparsers): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method. """ self.subparser = subparsers.add_parser(self.cmd, description=self.description) self.subparser.set_defaults(handler=self.handle) self.subparser.add_argument("-p", dest="plugin", default=None, help="Plugin name") self.subparser.add_argument("-i", dest="interface", default=None, help="Interface name") self.subparser.add_argument("-s", dest="_ls_summary", action="store_true", default=False, help="Summary of pluggdapps environment") self.subparser.add_argument("-e", dest="_ls_settings", default=None, help="list settings") self.subparser.add_argument("-P", dest="_ls_plugins", action="store_true", default=False, help="List plugins defined") self.subparser.add_argument("-I", dest="_ls_interfaces", action="store_true", default=False, help="List interfaces defined") self.subparser.add_argument("-K", dest="_ls_packages", action="store_true", default=False, help="List of pluggdapps packages loaded") self.subparser.add_argument( "-W", dest="_ls_webapps", action="store_true", default=False, help="List all web application and its mount configuration") self.subparser.add_argument( "-m", dest="_ls_implementers", action="store_true", default=False, help="list of interfaces and their plugins") self.subparser.add_argument( "-M", dest="_ls_implementers_r", action="store_true", default=False, help="list of plugins and interfaces they implement") return parser def handle(self, args): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method.""" opts = [ '_ls_summary', '_ls_settings', '_ls_plugins', '_ls_interfaces', '_ls_webapps', '_ls_packages', '_ls_implementers', '_ls_implementers_r' ] for opt in opts: if getattr(args, opt, False): getattr(self, opt)(args) break else: if args.interface: self._ls_interface(args) elif args.plugin: self._ls_plugin(args) #---- Internal functions def _ls_summary(self, args): import pluggdapps webapps_ = webapps() print("Pluggdapps environment") print(" Configuration file : %s" % self.pa.inifile) print(" Erlang Port : %s" % (self.pa.erlport or None)) print(" Loaded packages : %s" % len(pluggdapps.papackages)) print(" Interfaces defined : %s" % len(PluginMeta._interfmap)) print(" Plugins loaded : %s" % len(PluginMeta._pluginmap)) print(" Applications loaded: %s" % len(webapps_)) print("Web-application instances") pprint(webapps_, indent=2) def _ls_settings(self, args): sett = deepcopy(self.pa.settings) if args._ls_settings.startswith('spec'): print("Special sections") pprint({k: sett.pop(k, {}) for k in SPECIAL_SECS + ['DEFAULT']}, indent=2) elif args._ls_settings.startswith('plug'): print("Plugin sections") pprint({k: sett[k] for k in sett if h.is_plugin_section(k)}, indent=2) elif args._ls_settings.startswith('wa') and args.plugin: for instkey, webapp in getattr(self.pa, 'webapps', {}).items(): appsec, netpath, instconfig = instkey if h.sec2plugin(appsec) == args.plugin: print("Settings for %r" % (instkey, )) pprint(webapp.appsettings, indent=2) print() elif args._ls_settings.startswith('def') and args.plugin: print("Default settings for plugin %r" % args.plugin) defaultsett = pa.defaultsettings() pprint(defaultsett().get(h.plugin2sec(args.plugin), {}), indent=2) def _ls_plugins(self, args): l = sorted(list(PluginMeta._pluginmap.items())) for pname, info in l: print((" %-15s in %r" % (pname, info['file']))) def _ls_interfaces(self, args): l = sorted(list(PluginMeta._interfmap.items())) for iname, info in l: print((" %-15s in %r" % (iname, info['assetspec']))) def _ls_interface(self, args): nm = args.interface info = PluginMeta._interfmap.get(nm, None) if info == None: print("Interface %r not defined" % nm) else: print("\nAttribute dictionary : ") pprint(info['attributes'], indent=4) print("\nMethod dictionary : ") pprint(info['methods'], indent=4) print("\nPlugins implementing interface") plugins = PluginMeta._implementers.get(info['cls'], {}) pprint(plugins, indent=4) def _ls_plugin(self, args): from pluggdapps.web.webapp import WebApp for instkey, webapp in self.pa.webapps.items(): appsec, netpath, config = instkey if h.sec2plugin(appsec) == args.plugin: print("Mounted app %r" % appsec) print(" Instkey : ", end='') pprint(webapp.instkey, indent=4) print(" Subdomain : ", webapp.netpath) print(" Router : ", webapp.router) print("Application settings") pprint(webapp.appsettings, indent=4) print() def _ls_webapps(self, args): print("[mountloc]") pprint(self.pa.webapps, indent=2) print("\nWeb-apps mounted") pprint(list(self.pa.webapps.keys()), indent=2) def _ls_packages(self, args): import pluggdapps print("List of loaded packages") pprint(pluggdapps.papackages, indent=2, width=70) def _ls_implementers(self, args): print("List of interfaces and plugins implementing them") print() for i, pmap in PluginMeta._implementers.items(): print(" %-15s" % i.__name__, end='') pprint(list(pmap.keys()), indent=8) def _ls_implementers_r(self, args): print("List of plugins and interfaces implemented by them") for name, info in PluginMeta._pluginmap.items(): intrfs = list( sorted(map(lambda x: x.__name__, info['cls']._interfs))) print(" %-20s" % name, end='') pprint(intrfs, indent=8, width=60) #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return sett
class TTLCompiler(Plugin): """Tayra compiler. Implemented as plugin to leverage on pluggdapps configuration system. Also implement :class:`ITemplate`. Creating a plugin instance can be a costly operation, to avoid this instantiate this plugin once and use :meth:`_init` to initialize it for subsequent uses. """ implements(ITemplate) _memcache = {} ttlloc = None """TemplateLookup object. Encapsulates location information for template file and its intermediate python file.""" ttlfile = '' """Absolute file path pointing to .ttl template script file in.""" ttltext = '' """String of template script either read from the template file or received directly.""" pyfile = '' """If present, absolute file path pointing to compiled template script, .py file.""" pytext = '' """String of python script translated from template script.""" encoding = '' """Character encoding of original template script file.""" ttlparser = None """:class:`tayra.parser.TTLParser` object.""" igen = None """:class:`tayra.codegen.InstrGen` object.""" mach = None """:class:`tayra.runtime.StackMachine` object.""" def __init__(self): from tayra.parser import TTLParser from tayra.codegen import InstrGen from tayra.runtime import StackMachine self.ttlparser = TTLParser(self) self.igen = InstrGen(self) self.mach = StackMachine(self) # Stack machine def __call__(self, **kwargs): """Clone a plugin with the same settings as this one. settings can also be overriden by passing ``settings`` key-word argument as a dictionary.""" settings = {} settings.update({k: self[k] for k in self}) settings.update(kwargs.pop('settings', {})) kwargs['settings'] = settings return self.qp(ISettings, 'tayra.ttlcompiler', **kwargs) def _init(self, file=None, text=None): """Reinitialize the compiler object to compile a different template script.""" from pluggdapps import papackages self.ttlloc = TemplateLookup(self, file, text) self.encoding = self.ttlloc.encoding self.ttlfile = self.ttlloc.ttlfile self.ttltext = self.ttlloc.ttltext self.pyfile = self.ttlloc.pyfile # Compute the module name from ttlfile. asset = h.asset_spec_from_abspath(self.ttlfile, papackages) if asset: n = '.'.join(asset.split(':', 1)[1].split(os.path.sep)) self.modulename = n[:-4] if n.endswith('.ttl') else n else: n = '.'.join(self.ttlfile.split(os.path.sep)) self.modulename = n[:-4] if n.endswith('.ttl') else n self.mach._init(self.ttlfile) self.igen._init() # Check whether pyfile is older than the ttl file. In debug mode is is # always re-generated. self.pytext = '' if self['debug'] == False: if self.ttlfile and isfile(self.ttlfile): if self.pyfile and isfile(self.pyfile): m1 = os.stat(self.ttlfile).st_mtime m2 = os.stat(self.pyfile).st_mtime if m1 <= m2: self.pytext = open(self.pyfile).read() def toast(self, ttltext=None): """Convert template text into abstract-syntax-tree. Return the root node :class:`tayra.ast.Template`.""" ttltext = ttltext or self.ttltext self.ast = self.ttlparser.parse(ttltext, ttlfile=self.ttlfile) return self.ast def topy(self, ast, *args, **kwargs): """Generate intermediate python text from ``ast`` obtained from :meth:`toast` method. If configuration settings allow for persisting the generated python text, then this method will save the intermediate python text in file pointed by :attr:`pyfile`.""" ast.validate() ast.headpass1(self.igen) # Head pass, phase 1 ast.headpass2(self.igen) # Head pass, phase 2 ast.generate(self.igen, *args, **kwargs) # Generation ast.tailpass(self.igen) # Tail pass pytext = self.igen.codetext() if self.pyfile and isdir(dirname(self.pyfile)): open(self.pyfile, 'w', encoding='utf-8').write(pytext) return pytext def compilettl(self, file=None, text=None, args=[], kwargs={}): """Translate a template script text or template script file into intermediate python text. Compile the python text and return the code object. If ``memcache`` configuration is enable, compile code is cached in memory using a hash value generated from template text. ``args`` and ``kwargs``, position arguments and keyword arguments to use during AST.generate() pass. """ self._init(file=file, text=text) if (file or text) else None code = self._memcache.get(self.ttlloc.ttlhash, None) if not code: if not self.pytext: self.pytext = self.topy(self.toast(), *args, **kwargs) code = compile(self.pytext, self.pyfile, 'exec') if self['memcache']: self._memcache.setdefault(self.ttlloc.ttlhash, code) return code def load(self, code, context={}): """Using an optional ``context``, a dictionary of key-value pairs, and the code object obtained from :meth:`compilettl` method, create a new module instance populating it with ``context`` and some standard references.""" from tayra.runtime import Namespace import tayra.h as tmplh # Module instance for the ttl file module = imp.new_module(self.modulename) # Create helper module helper = imp.new_module('template_helper') filterfn = lambda k, v: callable(v) [ helper.__dict__.update(pynamespace(m)) for m in [tmplh] + self['helpers'] ] ctxt = { self.igen.machname: self.mach, '_compiler': self, 'this': Namespace(None, module), 'local': module, 'parent': None, 'next': None, 'h': helper, '__file__': self.pyfile, '_ttlfile': self.ttlfile, '_ttlhash': self.ttlloc.ttlhash, } ctxt.update(context) ctxt['_context'] = ctxt module.__dict__.update(ctxt) # Execute the code in module's context sys.modules.setdefault(self.modulename, module) exec(code, module.__dict__, module.__dict__) return module def generatehtml(self, module, context={}): """Using the ``module`` object obtained from :meth:`load` method, and a context dictionary, call the template module's entry point to generate HTMl text and return the same. """ try: entry = getattr(module.this, self['entry_function']) args = context.get('_bodyargs', []) kwargs = context.get('_bodykwargs', {}) html = entry(*args, **kwargs) if callable(entry) else '' try: from bs4 import BeautifulSoup if self['beautify_html']: html = BeautifulSoup(html).prettify() except: pass return html except: if self['debug']: raise return '' def importlib(self, this, context, file): """Import library ttl files inside the main script's context""" compiler = self() context['_compiler'] = compiler context['this'] = this return compiler.load(compiler.compilettl(file=file), context=context) #---- ITemplate interface methods def render(self, context, **kwargs): """:meth:`pluggdapps.interfaces.ITemplate.render` interface method. Generate HTML string from template script passed either via ``ttext`` or via ``tfile``. ``tfile``, Location of Tayra template file, either as relative directory or as asset specification. ``ttext``, Tayra template text string. """ file, text = kwargs.get('file', None), kwargs.get('text', None) code = self.compilettl(file=file, text=text) module = self.load(code, context=context) html = self.generatehtml(module, context) return html #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _defaultsettings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" sett['nocache'] = h.asbool(sett['nocache']) sett['optimize'] = h.asint(sett['optimize']) sett['lex_debug'] = h.asint(sett['lex_debug']) sett['yacc_debug'] = h.asint(sett['yacc_debug']) sett['strict_undefined'] = h.asbool(sett['strict_undefined']) sett['directories'] = h.parsecsvlines(sett['directories']) sett['tag.plugins'] = h.parsecsvlines(sett['tag.plugins']) sett['beautify_html'] = h.asbool(sett['beautify_html']) sett['memcache'] = h.asbool(sett['memcache']) sett['helpers'] = h.parsecsv(sett['helpers']) return sett
class MatchRouter( Plugin ): """Plugin to resolve HTTP request to a view-callable by matching patterns on request-URL. Refer to :class:`pluggdapps.web.interfaces.IHTTPRouter` interface spec. to understand the general intent and purpose of this plugin. On top of that, this plugin also supports server side HTTP content negotiation. When creating a web application using pluggdapps, developers must implement a router class, a plugin, implementing :class:`pluggdapps.web.interfaces.IHTTPRouter` interfaces. The general thumb rule is to derive their router class from one of the base routers defined under :mod:`pluggdapps.web` package. :class:`MatchRouter` is one such base class. When an application derives its router from this base class, it can override :meth:`onboot` method and call :meth:`add_view` to add view representations for matching resource. This router plugin adheres to HTTP concepts like resource, representation and views. As per the nomenclature, a resource is always identified by request-URL and same resource can have any number of representation. View is a callable entity that is capable of generating the desired representation. Note that a web page that is encoded with `gzip` compression is a different representation of the same resource that does not use `gzip` compression. Instead of programmatically configuring URL routing, it is possible to configure them using a mapper file. Which is a python file containing a list of dictionaries where each dictionary element will be converted to add_view() method call during onboot(). **map file specification,** .. code-block:: python :linenos: [ { 'name' : <variant-name as string>, 'pattern' : <regex pattern-to-match-with-request-URL>, 'view' : <view-callable as string>, 'resource' : <resource-name as string>, 'attr' : <attribute on view callable, as string>, 'method' : <HTTP request method as byte string>, 'media_type' : <content-type as string>, 'language' : <language-string as string>, 'charset' : <charset-string as string>, 'content_coding' : <content-coding as comma separated values>, 'cache_control' : <response header value>, 'rootloc' : <path to root location for static documents>, }, ... ] """ implements( IHTTPRouter ) views = {} """Dictionary of view-names to view-callables and its predicates, typically added via add_view() interface method.""" viewlist = [] """same as views.items() except that this list will maintain the order in which the views where added. The same order will be used while resolving the request to view-callable.""" negotiator = None """:class:`pluggdapps.web.interface.IHTTPNegotiator` plugin to handle HTTP negotiation.""" def onboot( self ): """:meth:`pluggapps.web.interfaces.IHTTPRouter.onboot` interface method. Deriving class must override this method and use :meth:`add_view` to create router mapping.""" self.views = {} self.viewlist = [] self.negotiator = None if self['IHTTPNegotiator'] : self.negotiator = self.qp(IHTTPNegotiator, self['IHTTPNegotiator']) self['defaultview'] = h.string_import( self['defaultview'] ) # Route mapping file is configured, populate view-callables from the # file. mapfile = self['routemapper'] if mapfile and isfile( mapfile ) : for kwargs in eval( open( mapfile ).read() ) : name = kwargs.pop('name') pattern = kwargs.pop('pattern') self.add_view( name, pattern, **kwargs ) elif mapfile : msg = "Wrong configuration for routemapper : %r" % mapfile raise Exception( msg ) def add_view( self, name, pattern, **kwargs ): """Add a router mapping rule. ``name``, The name of the route. This attribute is required and it must be unique among all defined routes in a given web-application. ``pattern``, The pattern of the route. This argument is required. If pattern doesn't match the current URL, route matching continues. For EG, .. code-block:: python :linenos: self.add_view( 'article', 'blog/{year}/{month}/{date}' ) Optional key-word arguments, ``view``, A plugin name or plugin instance implementing :class:`IHTTPView` interface, or just a plain python callable or a string that imports a callable object. What ever the case, please do go through the :class:`IHTTPView` interface specification before authoring a view-callable. ``resource``, A plugin name or plugin instance implementing :class:`IHTTPResource` interface, or just a plain python callable. What ever the case, please do go through the :class:`IHTTPResource` interface specification before authoring a resource-callable. ``attr``, If view-callable is a method on ``view`` object then supply this argument with a valid method name. ``method``, Request predicate. HTTP-method as byte string to be matched with incoming request. ``media_type``, Request predicate. Media type/subtype string specifying the resource variant. If unspecified, will be automatically detected using heuristics. ``language``, Request predicate. Language-range string specifying the resource variant. If unspecified, assumes webapp['language'] from configuration settings. ``charset``, Request predicate. Charset string specifying the resource variant. If unspecified, assumes webapp['encoding'] from configuration settings. ``content_coding``, Comma separated list of content coding that will be applied, in the same order as given, on the resource variant. Defaults to `identity`. ``cache_control``, Cache-Control response header value to be used for the resource's variant. ``rootloc``, To add views for static files, use this attribute. Specifies the root location where static files are located. Note that when using this option, ``pattern`` argument must end with ``*path``. ``media_type``, ``language``, ``content_coding`` and ``charset`` kwargs, if supplied, will be used during content negotiation. """ # Positional arguments. self.views[ name ] = view = {} view['name'] = name view['pattern'] = pattern regex, tmpl, redict = self._compile_pattern( pattern ) view['compiled_pattern'] = re.compile( regex ) view['path_template'] = tmpl view['match_segments'] = redict # Supported key-word arguments view['view'] = kwargs.pop( 'view', None ) view['resource'] = kwargs.pop( 'resource', None ) view['attr'] = kwargs.pop( 'attr', None ) view['method'] = h.strof( kwargs.pop( 'method', None )) # Content Negotiation attributes view['media_type']=kwargs.pop('media_type', 'application/octet-stream') view['content_coding'] = kwargs.pop('content_coding',CONTENT_IDENTITY) view['language'] = kwargs.pop( 'language', self.webapp['language'] ) view['charset'] = kwargs.pop( 'charset', self.webapp['encoding'] ) # Content Negotiation attributes view.update( kwargs ) self.viewlist.append( (name, view) ) def route( self, request ): """:meth:`pluggdapps.web.interfaces.IHTTPRouter.route` interface method. Three phases of request resolution to view-callable, * From configured list of views, filter out views that maps to same request-URL. * From the previous list, filter the variants that match with request predicates. * If content negotiation is enable, apply server-side negotiation algorithm to single out a resource variant. If than one variant remains at the end of all three phases, then pick the first one in the list. And that is why the sequence in which :meth:`add_view` is called for each view representation is important. If ``resource`` attribute is configured on a view, it will be called with ``request`` plugin and ``context`` dictionary. Resource-callable can populate the context with relavant data that will subsequently be used by the view callable, view-template etc. Additionally, if a resource callable populates the context dictionary, it automatically generates the etag for data that was populated through ``c.etag`` dictionary. Populates context with special key `etag` and clears ``c.etag`` before sending the context to view-callable. """ resp = request.response c = resp.context # Three phases of request resolution to view-callable matches = self._match_url( request, self.viewlist ) variants = self._match_predicates( request, matches ) if self.negotiator : variant = self.negotiator.negotiate( request, variants ) elif variants : # First come first served. variant = variants[0] else : variant = None if variant : # If a variant is resolved name, viewd, m = variant['name'], variant, variant['_regexmatch'] resp.media_type = viewd['media_type'] resp.charset = viewd['charset'] resp.language = viewd['language'] resp.content_coding = viewd['content_coding'] request.matchdict = m.groupdict() # Call IHTTPResource plugin configured for this view callable. resource = self._resourceof( request, viewd ) resource( request, c ) if resource else None # If etag is available, compute and subsequently clear them. etag = c.etag.hashout( prefix='res-' ) c.setdefault( 'etag', etag ) if etag else None c.etag.clear() request.view = self._viewof( request, name, viewd ) elif matches : from pluggdapps.web.views import HTTPNotAcceptable request.view = HTTPNotAcceptable else : request.view = self['defaultview'] if callable( request.view ) : # Call the view-callable c['h'] = h request.view( request, c ) def urlpath( self, request, name, **matchdict ): """:meth:`pluggdapps.web.interfaces.IHTTPRouter.route` interface method. Generate url path for request using view-patterns. Return a string of URL-path, with query and anchore elements. ``name``, Name of the view pattern to use for generating this url ``matchdict``, A dictionary of variables in url-patterns and their corresponding value string. Every route definition will have variable (aka dynamic components in path segments) that will be matched with url. If matchdict contains the following keys, `_query`, its value, which must be a dictionary similar to :attr:`pluggdapps.web.interfaces.IHTTPRequest.getparams`, will be interpreted as query parameters and encoded to query string. `_anchor`, its value will be attached at the end of the url as "#<_anchor>". """ view = self.views[ name ] query = matchdict.pop( '_query', None ) fragment = matchdict.pop( '_anchor', None ) path = view['path_template'].format( **matchdict ) return h.make_url( None, path, query, fragment ) def onfinish( self, request ): """:meth:`pluggdapps.web.interfaces.IHTTPRouter.onfinish` interface method. """ pass #-- Local methods. def _viewof( self, request, name, viewd ): """For resolved view ``viewd``, fetch the view-callable.""" v = viewd['view'] self.pa.logdebug( "%r view callable: %r " % (request.uri, v) ) if isinstance(v, str) and isplugin(v) : view = self.qp( IHTTPView, v, name, viewd ) elif isinstance( v, str ): view = h.string_import( v ) else : view = v return getattr( view, viewd['attr'] ) if viewd['attr'] else view def _resourceof( self, request, viewd ): """For resolved view ``viewd``, fetch the resource-callable.""" res = viewd['resource'] self.pa.logdebug( "%r resource callable: %r " % (request.uri, res) ) if isinstance( res, str ) and isplugin( res ) : return self.qp( IHTTPResource, res ) elif isinstance( res, str ) : return h.string_import( res ) else : return res def _match_url( self, request, viewlist ): """Match view pattern with request url and filter out views with matching urls.""" matches = [] for name, viewd in viewlist : # Match urls. m = viewd['compiled_pattern'].match( request.uriparts['path'] ) if m : viewd = dict( viewd.items() ) viewd['_regexmatch'] = m matches.append( viewd ) return matches def _match_predicates( self, request, matches ): """Filter matching views, whose pattern matches with request-url, based on view-predicates. TODO: More predicates to be added.""" variants = [] for viewd in matches : x = True if viewd['method'] != None : x = x and viewd['method'] == h.strof( request.method ) variants.append( viewd ) if x else None return variants def _compile_pattern( self, pattern ): """`pattern` is URL routing pattern. This method compiles the pattern in three different ways and returns them as a tuple of (regex, tmpl, redict) `regex`, A regular expression string that can be used to match incoming request-url to resolve view-callable. `tmpl`, A template formating string that can be used to generate URLs by apps. `redict`, A dictionary of variable components in path segment and optional regular expression that must match its value. This can be used for validation during URL generation. """ regex, tmpl, redict = r'^', '', {} segs = list( filter( None, pattern.split( URLSEP ))) while segs : regex += URLSEP tmpl += URLSEP part = segs.pop(0) if not part : continue if part[0] == '*' : part = URLSEP.join( [part] + segs ) prefx, name, reg, sufx = None, part[1:], r'.*', None segs = [] else : prefx, interp, sufx = re_patt.match( part ).groups() if interp : try : name, reg = interp[1:-1].split(',', 1) except : name, reg = interp[1:-1], None else : name, reg = None, None regex += prefx if prefx else r'' if name and reg and sufx : regex += r'(?P<%s>%s(?=%s))%s' % (name, reg, sufx, sufx) elif name and reg : regex += r'(?P<%s>%s)' % (name, reg) elif name and sufx : regex += r'(?P<%s>.+(?=%s))%s' % (name, sufx, sufx) elif name : regex += r'(?P<%s>.+)' % (name,) elif sufx : regex += sufx tmpl += prefx if prefx else '' tmpl += '{' + name + '}' if name else '' tmpl += sufx if sufx else '' redict[ name ] = reg regex += '$' return regex, tmpl, redict #---- ISettings interface methods @classmethod def default_settings( cls ): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method. """ return _default_settings @classmethod def normalize_settings( cls, sett ): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method. """ x = sett['routemapper'].strip() sett['routemapper'] = h.abspath_from_asset_spec(x) if x else x return sett
class Tags( Plugin ): """Base class for all plugins wanting to handle template tags. Since the base class declares that it implements :class:`tayra.interfaces.ITayraTags` interface, deriving plugins need not do the same. - provides standard specifier syntax for common tag attributes. - gracefully handles undefined tags. """ implements( ITayraTags ) general_toks = { # global attributes 'edit' : ' contenteditable="true"', 'noedit' : ' contenteditable="false"', 'dragcopy' : ' draggable="true" dragzone="copy"', 'dragmove' : ' draggable="true" dragzone="move"', 'draglink' : ' draggable="true" dragzone="link"', 'nodrag' : ' draggable="false"', 'hidden' : ' hidden', 'spellcheck' : ' spellcheck="true"', 'nospellcheck' : ' spellcheck="false"', # dir 'ltr' : ' dir="ltr"', 'rtl' : ' dir="rtl"', # atoms 'disabled' : ' disabled="disabled"', 'checked' : ' checked="checked"', 'readonly' : ' readonly="readonly"', 'selected' : ' selected="selected"', 'multiple' : ' multiple="multiple"', 'defer' : ' defer="defer"', } token_shortcuts = { '#' : lambda tok : \ ' id="%s"' % tok[1:], '.' : lambda tok : \ ' class="%s"' % ' '.join( filter( None, tok.split('.') )), ':' : lambda tok : \ ' name="%s"' % tok[1:], } #---- ITayraTags interface methods def handle( self, mach, tagname, tokens, styles, attributes, content ): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method. This method is expected to be overriden by the deriving plugin class, only for undefined template tags this method will be called, after trying with other plugins in the list of ``tag.plugins``.""" attrs, remtoks = self.parse_specs( tokens, styles, attributes ) l = len(content) - len(content.rstrip()) content, nl = (content[:-l], content[-l:]) if l else (content, '') return ('<%s %s>%s</%s>' % (tagname, attrs, content, tagname)) + nl def parse_specs( self, tokens, styles, attributes ): """The base class provides standard set of tokens and specifiers that are common to most HTML tags. To parse these tokens into tag-attributes, deriving plugins can use this method.""" tagattrs, remtoks = self.parse_tokens( tokens ) tagattrs += (' style="%s"' % ';'.join( styles )) if styles else '' tagattrs += (' ' + ' '.join( attributes )) if attributes else '' return tagattrs, remtoks #-- Local methods. def parse_tokens( self, tokens ): tagattrs, remtoks = '', [] for tok in tokens : attr = self.token_shortcuts.get( tok[0], lambda tok : tok )( tok ) attr = self.general_toks.get( attr, attr ) if tok == attr : remtoks.append( tok ) else : tagattrs += attr return tagattrs, remtoks #---- ISettings interface methods @classmethod def default_settings( cls ): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings( cls, sett ): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return sett
class Commands( Singleton ): """Subcommand plugin for pa-script to list all available sub-commands along with a short description. Like, .. code-block:: bash :linenos: $ pagd commands """ implements( ICommand ) description = 'list of script commands and their short description.' cmd = 'commands' #---- ICommand API def subparser( self, parser, subparsers ): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method. """ self.subparser = subparsers.add_parser( self.cmd, description=self.description ) self.subparser.set_defaults( handler=self.handle ) def handle( self, args ): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method.""" commands = self.qpr(ICommand, 'pagd.*') commands = sorted( commands, key=lambda x : x.caname ) for command in commands : name = command.caname.split('.', 1)[1] rows = self._formatdescr( name, command.description ) for r in rows : print(r) #---- Internal & local functions def _formatdescr( self, name, description ): fmtstr = '%-' + str(self['command_width']) + 's %s' l = self['description_width'] rows, line = [], '' words = ' '.join( description.strip().splitlines() ).split(' ') while words : word = words.pop(0) if len(line) + len(word) >= l : rows.append( fmtstr % (name, line) ) line, name = word, '' else : line = ' '.join([ x for x in [line,word] if x ]) rows.append( fmtstr % (name, line) ) if line else None return rows #---- ISettings interface methods @classmethod def default_settings( cls ): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings( cls, settings ): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" settings['description_width'] = h.asint(settings['description_width']) settings['command_width'] = h.asint(settings['command_width']) return settings
class HTTPNegotiator(Plugin): """Plugin handle server side negotiation. Gather client side negotiable information using following rules, * Use ``accept`` header from http request. If not available assume that any type of media encoding is acceptable by client. * Use ``accept_charset`` from http request. If not available assume that any character encoding is acceptable by client. * Use ``accept_encoding`` from http request. If not available assume `identity` coding. * Use ``accept_language`` from http request. If not available assume any language is acceptable by client. If a configured variant matches any of the combination supported by client, pick that variant and return the same. Otherwise return None. """ implements(IHTTPNegotiator) #---- IHTTPNegotiator interface methods def negotiate(self, request, variants): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" cltbl = self._compile_client_negotiation(request) variants_ = [] for viewd in variants: self._variant_keys(viewd) q = max(cltbl.get(k, 0.0) for k in viewd['_http_negotiator']) variants_.append((viewd, q)) if q else None variants_ = sorted(variants_, key=lambda x: x[1], reverse=True) return variants_[0][0] if variants_ else None #-- local methods. def _variant_keys(self, viewd): if '_http_negotiator' not in viewd: fn = lambda x: (x, ) typ, subtype = viewd['media_type'].split('/', 1) keys = [(viewd['media_type'], ), ('%s/*' % typ, ), ('*/*', )] keys = [ x + y for x in keys for y in map(fn, [viewd['charset'], '*']) ] keys = [ x + y for x in keys for y in map(fn, [viewd['content_coding'], CONTENT_IDENTITY]) ] keys = [ x + y for x in keys for y in map(fn, [viewd['language'], '*']) ] viewd['_http_negotiator'] = keys return viewd['_http_negotiator'] def _compile_client_negotiation(self, request): hs = [ request.headers.get('accept', b''), request.headers.get('accept_charset', b''), request.headers.get('accept_encoding', b''), request.headers.get('accept_language', b''), ] accept = h.parse_accept(hs[0]) or [('*/*', 1.0, b'')] accchr = h.parse_accept_charset(hs[1]) or [('*', 1.0)] accenc = h.parse_accept_encoding(hs[2]) or [(CONTENT_IDENTITY, 1.0)] acclan = h.parse_accept_language(hs[3]) or [('*', 1.0)] ad = {mt: q for mt, q, params in accept} bd = {(a, ch): aq * q for ch, q in accchr for a, aq in ad.items()} cd = {b + (enc, ): bq * q for enc, q in accenc for b, bq in bd.items()} tbl = {} for ln, q in acclan: zd, yd = {}, deepcopy(cd) for part in ln.split('-'): yd = {k + (part, ): cq for k, cq in yd.items()} zd.update(yd) tbl.update({k: cq * q for k, cq in zd.items()}) return tbl #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return sett
class CatchAndDebug(Plugin): """An exception collector that finds traceback information plus supplements. Produces a data structure that can be used by formatters to render them as an interactive web page. Magic variables: If you define one of these variables in your local scope, you can add information to tracebacks that happen in that context. This allows applications to add all sorts of extra information about the context of the error, including URLs, environmental variables, users, hostnames, etc. These are the variables we look for: ``__traceback_info__``: String. This information is added to the traceback, usually fairly literally. ``__traceback_hide__``: Boolean or String. True, this indicates that the frame should be hidden from abbreviated tracebacks. This way you can hide some of the complexity of the larger framework and let the user focus on their own errors. 'before', all frames before this one will be thrown away. By setting it to ``'after'`` then all frames after this will be thrown away until ``'reset'`` is found. In each case the frame where it is set is included, unless you append ``'_and_this'`` to the value (e.g., ``'before_and_this'``). Note that formatters will ignore this entirely if the frame that contains the error wouldn't normally be shown according to these rules. ``__traceback_decorator__``: Callable. Takes frames, a list of ExceptionFrame object, modifies them inplace or return an entirely new object. What ever be the case, it is expected to return a list of frames. This gives the object the ability to manipulate the traceback arbitrarily. The actually interpretation of these values is largely up to the reporters and formatters or the rendering template. The list of frames goes innermost first. Each frame has these attributes; some values may be None if they could not be determined. Each frame is an instance of :class:`ExceptionFrame`. Note that all attributes are optional, and under certain circumstances may be None or may not exist at all -- the collector can only do a best effort, but must avoid creating any exceptions itself. Formatters may want to use ``__traceback_hide__`` as a hint to hide frames that are part of the 'framework' or underlying system. There are a variety of rules about special values for this variables that formatters should be aware of. TODO: More attributes in __traceback_supplement__? Maybe an attribute that gives a list of local variables that should also be collected? Also, attributes that would be explicitly meant for the entire request, not just a single frame. Right now some of the fixed set of attributes (e.g., source_url) are meant for this use, but there's no explicit way for the supplement to indicate new values, e.g., logged-in user, HTTP referrer, environment, etc. Also, the attributes that do exist are Zope/Web oriented. More information on frames? cgitb, for instance, produces extensive information on local variables. There exists the possibility that getting this information may cause side effects, which can make debugging more difficult; but it also provides fodder for post-mortem debugging. However, the collector is not meant to be configurable, but to capture everything it can and let the formatters be configurable. Maybe this would have to be a configuration value, or maybe it could be indicated by another magical variable (which would probably mean 'show all local variables below this frame') """ implements(IHTTPLiveDebug) frame_index = {} # { <identification-code> : ( globals, locals ) ... }: """Each frame has its own globals() and locals() context. To support browser based debugging, expressions need to be evaluated under these context.""" #---- IHTTPLiveDebug method APIs def render(self, request, etype, value, tb): """:meth:`pluggdapps.web.interfaces.IHTTPLiveDebug.render` interface method.""" response = request.response c = self.collectException(request, etype, value, tb) weba = self.pa.findapp(appname='pluggdapps.webadmin') c['url_jquery'] = \ weba.pathfor( request, 'staticfiles', path='jquery-1.8.3.min.js') c['url_css'] = \ weba.pathfor( request, 'staticfiles', path='errorpage.css' ) c['url_palogo150'] = \ weba.pathfor( request, 'staticfiles', path='palogo.150.png' ) html = '' if self['html'] and self['template']: # Must be enable in the configuration and a template_file available html = response.render(request, c, file=self['template']) return html #---- Local methods def getRevision(self, globals): if self['show_revision'] == False: return None rev = globals.get('__revision__', globals.get('__version__', None)) if rev is not None: try: rev = str(rev).strip() except: rev = '???' return rev def collectFrame(self, request, tb, extra_data): """Collect a dictionary of information about a traceback frame.""" if isinstance(tb, tuple): filename, lineno = tb name, globals_, locals_ = '', {}, {} tbid = None else: fr = tb.tb_frame code = fr.f_code filename, lineno = code.co_filename, tb.tb_lineno name, globals_, locals_ = code.co_name, fr.f_globals, fr.f_locals tbid = id(tb) if not isinstance(locals_, dict): # Something weird about this frame; it's not a real dict name = globals_.get('__name__', 'unknown') msg = "Frame %s has an invalid locals(): %r" % (name, locals_) self.pa.logwarn(msg) locals_ = {} try: linetext = open(filename).readlines()[lineno - 1] except: linetext = '' data = { 'modname': globals_.get('__name__', None), 'filename': filename, 'lineno': lineno, 'linetext': linetext, 'revision': self.getRevision(globals_), 'name': name, 'tbid': tbid, # Following are populated further done. 'traceback_info': '', 'traceback_hide': None, 'traceback_decorator': None, 'url_eval': '', } tbi = locals_.get('__traceback_info__', None) if tbi is not None: data['traceback_info'] = str(tbi) for name in ('__traceback_hide__', '__traceback_decorator__'): value = locals_.get(name, globals_.get(name, None)) data.update({name[2:-2]: value}) frameid = md5(str(data).encode('utf-8')).hexdigest() self.frame_index[frameid] = (globals_, locals_) if request: weba = self.pa.findapp(appname='pluggdapps.webadmin') data['url_eval'] = \ weba.urlfor( request, 'framedebug', frameid=frameid ) return data def collectException(self, request, etype, value, tb, limit=None): """``collectException( request, *sys.exc_info() )`` will return an instance of :class:`CollectedException`. Attibutes of this object can be used to render traceback. Use like:: try: blah blah except: exc_data = plugin.collectException(*sys.exc_info()) """ # The next line provides a way to detect recursion. __exception_formatter__ = 1 limit = limit or self['limit'] or getattr(sys, 'tracebacklimit', None) frames, ident_data, extra_data = [], [], {} # Collect all the frames in sys.exc_info's trace-back. n, tbs = 0, [] while tb is not None and (limit is None or n < limit): if tb.tb_frame.f_locals.get('__exception_formatter__'): # Stop recursion. @@: should make a fake ExceptionFrame frames.append('(Recursive formatException() stopped)\n') break ef = ExceptionFrame(**self.collectFrame(request, tb, extra_data)) if bool(ef.traceback_hide) == False and ef.filename: frames.append(ef) tbs.append(tb) n += 1 tb = tb.tb_next if hasattr(value, 'filename'): tb = (value.filename, value.lineno) elif value.__traceback__ not in tbs: tb = value.__traceback__ else: tb = None if tb: ef = ExceptionFrame(**self.collectFrame(request, tb, extra_data)) if bool(ef.traceback_hide) == False and ef.filename: frames.append(ef) decorators = [] for frame in frames: decorators.append(frame.traceback_decorator) ident_data.extend([frame.modname or '?', frame.name or '?']) ident_data.append(str(etype)) ident = hash_identifier(' '.join(ident_data), length=5, upper=True, prefix='E-') for decorator in filter(None, decorators): frames = decorator(frames) kwargs = { 'frames': frames, 'exception_formatted': '\n'.join(traceback.format_exception_only(etype, value)), 'exception_value': str(value), 'exception_type': etype.__name__, 'identification_code': ident, 'date': h.http_fromdate(time.time()), 'extra_data': extra_data, } result = CollectedException(**kwargs) if etype is ImportError: extra_data[('important', 'sys.path')] = [sys.path] return result #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" sett['limit'] = h.asint(sett['limit']) sett['show_revision'] = h.asbool(sett['show_revision']) sett['html'] = h.asbool(sett['html']) return sett
class MyBlog( Plugin ): """A layout plugin to generate personal blog sites. Support create, gen, newpage interfaces APIs for corresponding sub-commands. """ implements(ILayout) layoutpath = join( dirname(__file__), 'myblog') def __init__( self ): self.sitepath = self['sitepath'] if isinstance(self['siteconfig'], dict) : self.siteconfig = self['siteconfig'] else : self.siteconfig = json2dict( join( self['siteconfig'] )) self.plugins = self._plugins( self.sitepath, self.siteconfig ) #---- ILayout interface methods def is_exist(self): """:meth:`padg.interfaces.ILayout.is_exist` interface method.""" xs = [ 'configfile', 'contentdir', 'templatedir' ] x, y, z = [join(self.sitepath, self[x]) for x in xs] return isfile(x) and isdir(y) and isdir(z) def create(self, **kwargs) : """Creates a new layout under ``sitepath``. Uses the directory tree under `pagd:layouts/myblog` as a template for the new layout. Accepts the following variable while creating, ``sitepath``, directory-path under which the new layout had to be created. """ if not isdir( self['sitepath'] ) : os.makedirs( self['sitepath'], exist_ok=True ) _vars = { 'sitepath' : self.sitepath, } overwrite = kwargs.get('overwrite', False) h.template_to_source( self.layoutpath, self.sitepath, _vars, overwrite=overwrite, verbose=True ) def generate(self, buildtarget, **kwargs) : """Generate a static, personal blog site from the layout under ``sitepath``. Note that previously a new-layout must have been created using this plugin and available under `sitepath`. This method, - iterates over each page availabe under the source-layout, - gathers page contexts. - translates page content into html. - locate a template for the page and generate the html for page. Refer to :meth:`pages` method to know how pages are located under layout's content-directory. """ regen = kwargs.get('regen', True) srcfile = kwargs.get('srcfile', None) for page in self.pages() : path = abspath( join( buildtarget, page.relpath )) fname = page.pagename + '.html' self.pa.loginfo(" Generating `%r`" % join(page.relpath, fname) ) # Gather page context page.context.update( self.pagecontext( page )) # Gather page content page.articles = self.pagecontent( page ) # page content can also have context, in the form of metadata # IMPORTANT : myblog will always have one article only. for fpath, metadata, content in page.articles : page.context.update( metadata ) page.context.update( self._fetch_xc( metadata.get('_xcontext', ''), page )) # If skip_context is present then apply them, page = self._skip_context( page ) # Find a template for this page. page.templatefile = self.pagetemplate(page) # Locate the templage if isinstance(page.templatefile, str) : _, ext = splitext(page.templatefile) ttype = page.context.get('templatetype', ext.lstrip('.')) # generate page's html html = self._tmpl2plugin( self.plugins, ttype ).render( page ) os.makedirs(path, exist_ok=True) if not isdir(path) else None open( abspath( join( path, fname )), 'w' ).write(html) SPECIALPAGES = ['_context.json'] def pages(self): """Individual pages are picked based on the relative directory path along with filenames. Note that file extensions are not used to differentite pages, they are only used to detect the file type and apply corresponding translation algorithm to get page's html. """ contentdir = join( self.sitepath, *self['contentdir'].split('/') ) site = Site() site.sitepath = self.sitepath site.siteconfig = self.siteconfig for dirpath, dirs, files in os.walk(contentdir): files = sorted(files) [ files.remove(f) for f in self.SPECIALPAGES if f in files ] while files : pagename, contentfiles, files = \ pagd( join(contentdir, dirpath), files ) page = Page() page.site = site page.pagename = pagename page.relpath = relpath(dirpath, contentdir) page.urlpath = join( relpath(dirpath, contentdir), pagename) page.urlpath = '/'.join( page.urlpath.split( os.sep )) page.contentfiles = contentfiles page.context = self.config2context( self.siteconfig ) page.context.update({ 'site' : page.site, 'page' : page, 'title' : page.pagename, 'summary' : '', 'layout' : self.caname, 'author' : None, 'email' : None, 'createdon' : None, 'last_modified' : None, 'date' : None, }) page.articles = [] yield page def pagecontext( self, page ): """Gathers default context for page. Default context is specified by one or more JSON files by name `_context.json` that is located under every sub-directory that leads to the page-content under layout's content-directory. `_context.json` found one level deeper under content directory will override `_context.json` found in the upper levels. Also, if a pagename has a corresponding JSON file, for eg, ``<layout>/_contents/path/hello-world.rst`` file has a corresponding ``<layout>/_contents/path/hello-world.json``, it will be interepreted as the page's context. This context will override all the default context. If `_xcontext` attribute is found in a default context file, it will be interpreted as plugin name implementing :class:`IXContext` interface. The plugin will be queried, instantiated, to fetch context information from external sources like database. Finally ``last_modified`` time will be gathered from content-file's mtime statistics. """ contentdir = join( self.sitepath, *self['contentdir'].split('/') ) contexts = self.default_context(contentdir, page) # Page's context, if available. page_context_file = join(page.relpath, page.pagename) + '.json' c = json2dict(page_context_file) if isfile(page_context_file) else None contexts.append(c) if c else None context = {} # From the list of context dictionaries in `contexts` check for # `_xcontext` attribute and fetch the context from external source. for c in contexts : context.update(c) context.update( self._fetch_xc( c.get('_xcontext', ''), page )) return context def pagecontent( self, page ): """Pages are located based on filename, and the file extension is not used to differential pages. Hence there can be more than one file by same filename, like, ``_contents/hello-world.rst``, ``_contents/hello-world.md``. In such cases, all files will be considered as part of same page and translated to html based on the extension type. Return a single element list of articles, each article as tuple. Refer to :class:``Page`` class and its ``articles`` attribute to know its data-structure.""" n = page.context.get('IContent', self['IContent']) name = n if n in self.plugins else _default_settings['IContent'] icont = self.plugins.get( name, None ) articles = icont.articles(page) if icont else [] return articles def pagetemplate( self, page ): """For every page that :meth:`pages` method iterates, a corresponding template file should be located. It is located by following steps. - if page's context contain a ``template`` attribute, then its value is interpreted as the template file for page in asset specification format. - join the relative path of the page with ``_template`` sub-directory under the layout, and check whether a template file by pagename is available. For eg, if pagename is ``hello-world`` and its relative path is ``blog/2010``, then a template file ``_templates/blog/2010/hello-world`` will be lookup. Note that the extensio of the template file is immaterial. - If both above steps have failed then will lookup for a ``_default`` template under each sub-directory leading to ``_templates/blog/2010/``. """ tmplpath = join( self.sitepath, *self['templatedir'].split('/') ) tmplfile = None dr = abspath( join( tmplpath, page.relpath )) if page.context.get('template', None) == False : tmplfile = False if tmplfile == None and 'template' in page.context : tmplfile = asset_spec_to_abspath( page.context['template'] ) tmplfile = tmplfile if tmplfile and isfile(tmplfile) else None if tmplfile == None and isdir(dr) : tmplfile = findtemplate(dr, pagename=page.pagename) tmplfile = tmplfile if tmplfile and isfile(tmplfile) else None if tmplfile == None : path = page.relpath while tmplfile == None and path : d = join( tmplpath, path ) if isdir(d) : tmplfile = findtemplate(d, default=self['default_template']) tmplfile = tmplfile \ if tmplfile and isfile(tmplfile) else None path, _ = split( path ) return tmplfile def newpage(self, pagename): contentdir = join( self.sitepath, *self['contentdir'].split('/') ) try : _, ext = splitext(pagename) except : ext = '.rst' filepath = join( self.sitepath, contentdir, pagename+'.rst' ) os.makedirs( dirname(filepath), exist_ok=True ) open(filepath, 'w').write() self.pa.loginfo("New page create - %r", filepath) #---- Local functions def default_context( self, contentdir, page ): """Return a list of context dictionaries from default-context under each sub-directory of content-page's path.""" path = page.relpath.strip(os.sep) contexts = [] fname = self['default_context'] while path : f = join(contentdir, path, fname) contexts.insert(0, json2dict(f)) if isfile(f) else None path, _ = split( path ) return contexts def config2context( self, siteconfig ): xd = { x : siteconfig[x] for x in [ 'disqus', 'show_email', 'social_sharing', 'copyright', 'google_webfonts', 'style', 'age_scale', ] } return xd def _plugins( self, sitepath, siteconfig ): """Instantiate plugins available for :class:`ITemplate`, :class:`IXContext` and :class:`IContent` interfaces. siteconfig and sitepath will be passed as plugin-settings for all instantiated plugins. """ sett = { 'sitepath' : sitepath, 'siteconfig' : siteconfig } plugins = self.qps( ITemplate, settings=sett ) + \ self.qps( IXContext, settings=sett ) + \ self.qps( IContent, settings=sett ) return { p.caname : p for p in plugins } def _tmpl2plugin( self, plugins, tmpl ): """For file type ``tmpl`` return the template plugin.""" for p in plugins.values() : if tmpl in getattr(p, 'extensions', []) : return p else : return None def _skip_context(self, page): attrs = h.parsecsv( page.site.siteconfig.get( 'skip_context', '' )) + \ h.parsecsv( page.context.get( 'skip_context', '' )) [ page.context.update(attr=None) for attr in attrs ] return page def _fetch_xc(self, _xc, page) : ps = h.parsecsv( _xc ) context = {} for s in ps : p = self.plugins.get(s, None) context.update( p.fetch(page) ) if p else None return context #---- ISettings interface methods @classmethod def default_settings( cls ): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings( cls, settings ): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return settings
class HTTPRequest(Plugin): """Plugin encapsulates HTTP request. Refer to :class:`pluggdapps.web.interfaces.IHTTPRequest` interface spec. to understand the general intent and purpose of this plugin. """ implements(IHTTPRequest) content_type = '' """Parsed content type as return from :meth:`parse_content_type`.""" # IHTTPRequest interface methods and attributes def __init__(self, httpconn, method, uri, uriparts, version, headers): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.__init__` interface method.""" self.router = self.cookie = None self.response = self.session = None self.httpconn = httpconn self.method, self.uri, self.uriparts, self.version = \ method, uri, uriparts, version self.headers = headers # Initialize request handler attributes, these attributes will be # valid only after a call to handle() method. self.body = b'' self.chunks = [] self.trailers = {} self.cookies = {} # Only in case of POST and PUT method. self.postparams = {} self.multiparts = {} self.files = {} # Initialize self.params = {} self.getparams = { h.strof(k): list(map(h.strof, vs)) for k, vs in self.uriparts['query'].items() } self.params.update(self.getparams) self.content_type = \ h.parse_content_type( headers.get( 'content_type', None )) self.view = None self.receivedat = time.time() self.finishedat = None def supports_http_1_1(self): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.supports_http_1_1` interface method.""" return self.version == b"HTTP/1.1" def get_ssl_certificate(self): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.get_ssl_certificate` interface method.""" return self.httpconn.get_ssl_certificate() def get_cookie(self, name, default=None): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.get_cookie` interface method.""" return self.cookies[name].value if name in self.cookies else default def get_secure_cookie(self, name, value=None): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.get_secure_cookie` interface method.""" if value is None: value = self.get_cookie(name) return self.cookie.decode_signed_value(name, value) def has_finished(self): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.has_finished` interface method.""" return self.response.has_finished() if self.response else True def ischunked(self): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.ischunked` interface method.""" x = h.parse_transfer_encoding( self.headers.get('transfer_encoding', b'')) return (x[0][0] == 'chunked') if x else False def handle(self, body=None, chunk=None, trailers=None): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.handle` interface method.""" self.cookies = self.cookie.parse_cookies(self.headers) # In case of `chunked` encoding, check whether this is the last chunk. finishing = body or (chunk and trailers and chunk[0] == 0) # Apply IHTTPInBound transformers on this request. data = body if body != None else (chunk[2] if chunk else b'') for tr in self.webapp.in_transformers: data = tr.transform(self, data, finishing=finishing) # Update the request plugin with attributes. if body: self.body = data elif chunk: self.chunks.append((chunk[0], chunk[1], data)) self.trailers = trailers or self.trailers # Process POST and PUT request interpreting multipart content. if self.method in (b'POST', b'PUT'): self.postparams, self.multiparts = \ h.parse_formbody( self.content_type, self.body ) self.postparams = { h.strof(k): list(map(h.strof, vs)) for k, vs in self.postparams.items() } [ self.params.setdefault(name, []).extend(value) for name, value in self.postparams.items() ] [ self.params.setdefault(name, []).extend(value) for name, value in self.multiparts.items() ] [ self.files.setdefault(name, []).extend( (f['filename'], f['value'])) for name, value in self.multiparts.items() ] def onfinish(self): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.onfinish` interface method.""" # Will be callbe by response.onfinish() callback. self.view.onfinish(self) if hasattr(self.view, 'onfinish') else None self.webapp.onfinish(self) self.finishedat = time.time() def urlfor(self, name, **matchdict): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.urlfor` interface method.""" return self.webapp.urlfor(self, name, **matchdict) def pathfor(self, name, **matchdict): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.pathfor` interface method.""" return self.webapp.pathfor(self, name, **matchdict) def appurl(self, webapp, name, **matchdict): """:meth:`pluggdapps.web.interfaces.IHTTPRequest.appurl` interface method.""" return webapp.urlfor(self, name, **matchdict) def __repr__(self): attrs = ("uriparts", "address", "body") args = ", ".join("%s=%r" % (n, getattr(self, n, None)) for n in attrs) return "%s(%s, headers=%s)" % (self.__class__.__name__, args, dict(getattr(self, 'headers', {}))) #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method. """ return _default_settings
class Serve(Singleton): """Sub-command for starting native web server. Configuring this plugin does not control the web server, instead refer to the corresponding web server plugin. By default it uses :class:`HTTPEPollServer`, a single threaded / single process epoll based server. For automatic server restart, when a module or configuration file is modified, pass ``-m`` switch to main script and ``-r`` switch to this sub-command. Typically used in development mode, .. code-block:: bash :linenos: $ pa -w -m -c <master.ini> serve -r .. code-block:: text fork ---> child ------> poll-thread | | | *--------* | monitor *---> pluggdapps-thread """ implements(ICommand) description = "Start epoll based http server." cmd = 'serve' def __init__(self, *args, **kwargs): self.module_mtimes = {} #---- ICommand API methods def subparser(self, parser, subparsers): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method.""" self.subparser = subparsers.add_parser(self.cmd, description=self.description) self.subparser.set_defaults(handler=self.handle) self.subparser.add_argument("-r", dest="mreload", action="store_true", default=False, help="Monitor and reload modules") return parser def handle(self, args): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method.""" self.fork_and_monitor(args) if args.monitor else self.gemini(args) #---- Local function. def gemini(self, args): """Start a poll thread and then start pluggdapps platform.""" server = self.qp('pluggdapps.IHTTPServer', self['IHTTPServer']) if args.mreload: # Launch a thread to poll and then start serving http t = threading.Thread(target=self.pollthread, name='Reloader', args=(args, server)) t.setDaemon(True) t.start() time.sleep(0.5) # To allow the poll-thread to execute first. self.pa.start() # Start pluggdapps server.start() # Blocking call # use os._exit() here and not sys.exit() since within a # thread sys.exit() just closes the given thread and # won't kill the process; note os._exit does not call # any atexit callbacks, nor does it do finally blocks, # flush open files, etc. In otherwords, it is rude. os._exit(3) def fork_and_monitor(self, args): """Fork a child process with same command line arguments except the ``-m`` switch. Monitor and reload the child process until normal exit.""" while True: self.pa.logdebug("Forking monitor ...") pid = os.fork() if pid == 0: # child process cmdargs = sys.argv[:] cmdargs.remove('-m') cmdargs.append(os.environ) h.reseed_random() os.execlpe(sys.argv[0], *cmdargs) else: # parent try: pid, status = os.wait() if status & 0xFF00 != 0x300: sys.exit(status) except KeyboardInterrupt: sys.exit(0) def pollthread(self, args, server): """Thread (daemon) to monitor for changing files.""" self.pa.logdebug("Periodic poll started for module reloader ...") while True: if self.pollthread_checkfiles(args) == True: server.stop() break time.sleep(self['reload.poll_interval']) def pollthread_checkfiles(self, args): """Check whether any of the module files have modified after loading this platform. If so, return True else False.""" from pluggdapps import papackages modfiles = {} for mod in sys.modules.values(): if hasattr(mod, '__file__'): modfiles.setdefault(getattr(mod, '__file__'), mod) inifiles = self.inifiles() if self['reload.config'] else [] files = list(modfiles.keys()) + self.ttlfiles() + inifiles for filename in files: stat = os.stat(filename) mtime = stat.st_mtime if stat else 0 if filename.endswith('.pyc') and os.path.exists(filename[:-1]): mtime = max(os.stat(filename[:-1]).st_mtime, mtime) elif filename.endswith('$py.class') and \ os.path.exists(filename[:-9] + '.py'): mtime = max(os.stat(filename[:-9] + '.py').st_mtime, mtime) if filename not in self.module_mtimes: self.module_mtimes[filename] = mtime elif self.module_mtimes[filename] < mtime: self.pa.logdebug("%r changed, reloading ...\n" % filename) return True return False def inifiles(self): """Return a list of .ini files that are related to this environment.""" if self.pa.inifile: inifiles = [abspath(self.pa.inifile)] inifiles.extend(map(lambda x: x[2], self.pa.webapps.keys())) else: inifiles = [] return inifiles def ttlfiles(self): """Return a list of ttlfile related to this environment.""" from pluggdapps import papackages ttlfiles = h.flatten([ list(map(h.abspath_from_asset_spec, n.get('ttlplugins', []))) for nm, n in papackages.items() ]) return ttlfiles + self.pa._monitoredfiles #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" sett['reload.config'] = h.asbool(sett['reload.config']) sett['reload.poll_interval'] = h.asint(sett['reload.poll_interval']) return sett #---- An alternate implementation for linux platforms. But incomplete !! #---- Only directories can be watched and sub-directories had to be walked #---- and added programmatically. def _watch_handler(signum, frame): # log.info("file changed; reloading...") # use os._exit() here and not sys.exit() since within a thread # sys.exit() just closes the given thread and won't kill the process; # note os._exit does not call any atexit callbacks, nor does it do # finally blocks, flush open files, etc. In otherwords, it is rude. os._exit(3) def _watch_files(self, args): if 'darwin' in sys.platform: log.info("Not supported on `darwin`") os.exit(64) else: _watch_flag = fcntl.DN_MODIFY | fcntl.DN_CREATE | fcntl.DN_MULTISHOT filenames = [args.config] if args.config else [] filenames.extend( getattr(mod, '__file__', None) for mod in sys.modules.values()) filenames = filter(None, filenames) for filename in filenames: fd = os.open(filename, os.O_RDONLY) fcntl.fcntl(fd, fcntl.F_SETSIG, 0) fcntl.fcntl(fd, fcntl.F_NOTIFY, self._watch_flag)
class HTTPCookie(Plugin): """Cookie handling plugin. This plugin uses python standard library's http.cookies module to process request and response cookies. Refer to :class:`pluggdapps.web.interfaces.IHTTPCookie` interface spec. to understand the general intent and purpose this plugin.""" implements(IHTTPCookie) #-- IHTTPCookie interface methods. def parse_cookies(self, headers): """:meth:`pluggdapps.web.interfaces.IHTTPCookie.parse_cookies` interface method.""" cookies = SimpleCookie() cookie = headers.get('cookie', '') try: cookies.load(cookie) return cookies except CookieError: self.pa.logwarn("Unable to parse cookie: %s" % cookie) return None def set_cookie(self, cookies, name, value, **kwargs): """:meth:`pluggdapps.web.interfaces.IHTTPCookie.set_cookie` interface method.""" if re.search(r"[\x00-\x20]", name + value): # Don't let us accidentally inject bad stuff raise ValueError("Invalid cookie %r: %r" % (name, value)) if name in cookies: del cookies[name] cookies[name] = value morsel = cookies[name] domain = kwargs.pop('domain', None) if domain: morsel["domain"] = domain expires_days = kwargs.pop('expires_days', None) expires = kwargs.pop('expires', None) if expires_days is not None and not expires: expires = dt.datetime.utcnow() + dt.timedelta(days=expires_days) if expires: timestamp = calendar.timegm(expires.utctimetuple()) morsel["expires"] = \ email.utils.formatdate(timestamp, localtime=False, usegmt=True) path = kwargs.pop('path', '/') if path: morsel["path"] = path for k, v in kwargs.items(): morsel[k.replace('_', '-')] = v return cookies def create_signed_value(self, name, value): """:meth:`pluggdapps.web.interfaces.IHTTPCookie.set_cookie` interface method.""" parts = [self['secret'], name, value, str(int(time.time()))] parts = [x.encode('utf-8') for x in parts] parts[2] = base64.b64encode(parts[2]) signature = self._create_signature(*parts).encode('utf-8') signedval = b"|".join([parts[2], parts[3], signature]) return signedval.decode(self['value_encoding']) def decode_signed_value(self, name, signedval): """:meth:`pluggdapps.web.interfaces.IHTTPCookie.set_cookie` interface method.""" if not signedval: return None secret = self['secret'].encode('utf-8') name = name.encode('utf-8') value = signedval.encode(self['value_encoding']) parts = value.split(b"|") try: val64, timestamp, signature = parts except: return None args = [name, val64, timestamp] signature_ = self._create_signature(secret, *args) signature_ = signature_.encode('utf-8') if not self._time_independent_equals(signature, signature_): self.pa.logwarn("Invalid cookie signature %r" % value) return None timestamp_val = int(timestamp) if timestamp_val < (time.time() - self['max_age_seconds']): self.pa.logwarn("Expired cookie %r" % value) return None if timestamp_val > (time.time() + self['max_age_seconds']): # _cookie_signature does not hash a delimiter between the # parts of the cookie, so an attacker could transfer trailing # digits from the payload to the timestamp without altering the # signature. For backwards compatibility, sanity-check timestamp # here instead of modifying _cookie_signature. self.pa.logwarn("Cookie timestamp in future %r" % value) return None if timestamp.startswith(b"0"): self.pa.logwarn("Tampered cookie %r" % value) try: return base64.b64decode(val64).decode('utf-8') except Exception: return None def _create_signature(self, secret, *parts): hash = hmac.new(secret, digestmod=hashlib.sha1) [hash.update(part) for part in parts] return hash.hexdigest() def _time_independent_equals(self, a, b): if len(a) != len(b): return False result = 0 for x, y in zip(a, b): if x != y: return False else: return True #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings interface method.""" sett['max_age_seconds'] = h.asint(sett['max_age_seconds']) return sett
class Create(Singleton): """Sub-command plugin to create a new layout at the given sitepath. For example, .. code-block:: bash pagd -l 'pagd.myblog' create uses ``pagd.myblog`` plugin to create a source layout. """ implements(ICommand) cmd = 'create' description = 'Create a source layout.' #---- ICommand API def subparser(self, parser, subparsers): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method. * -g switch can be used to supply a configuration file for layout. * -f switch will overwrite if ``sitepath`` already contains a layout. """ self.subparser = subparsers.add_parser(self.cmd, description=self.description) self.subparser.set_defaults(handler=self.handle) self.subparser.add_argument( '-g', '--config-path', dest='configfile', default='config.json', help='The configuration used to generate the site') self.subparser.add_argument( '-f', '--force', dest='overwrite', action='store_true', default=False, help='Overwrite the source layout if it exists') return parser def handle(self, args): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method. Instantiate a layout plugin and apply create() method on the instantiated plugin. ``sitepath`` and ``siteconfig`` references willbe passed as settings dictionary. """ siteconfig = join(args.sitepath, args.configfile) sett = { 'sitepath': args.sitepath, 'siteconfig': siteconfig, } layout = self.qp(ILayout, args.layout, settings=sett) if not layout: raise Exception( "The given layout is invalid. Please check if you have the " "`layout` in the right place and the environment variable " "has been setup properly if you are using custom path for " "layouts") self.pa.loginfo("Creating site at [%s] with layout [%s] ..." % (args.sitepath, args.layout)) layout.create(overwrite=args.overwrite) self.pa.loginfo("... complete")
class PViews( Singleton ): """Pluggdapps can host many application, and application instances, in the same environment and each application can have any number of view-callables mapped onto url-paths. Use this sub-command, ``pviews``, to print a summary of matching routes and views for a given URL-path Note that the main script must be invoked using `webapps` platform, the ``-w`` switch. .. code-block:: text :linenos: $ pa -w pviews <url-path> mountat = <netpath> urlpath = <url-path> ... view-details .... mountat = <netpath> urlpath = <url-path> ... view-details .... Typically, for the same application instance, there can be many views mapped to same url-path, but differentiated by HTTP request methods and/or modifiers. If that is the case you will be getting more than one list for the same url-path. **netpath** tells the url-prefix (subdomain / host / script-path) on which the application is mounted. By default, if no other sub-command option is specified, matching views will be listed for all mounted application including every instance of the same application. This is useful when each application instance has its router configuration different from one another. There are couple of options that can filter or expand the view listing based on netpath. - **-a** option takes a plugin-name and lists matching views for all instance of application specified by the plugin-name. - **-n** option takes a netpath and lists matching views for application instance mounted onto that netpath. Each view listing might include following information, * route-name, route-pattern, route-path, route-predicates. * view-callable, view-predicates. View-predicates shall include authorisation and authentication. """ implements( ICommand ) description = "List matching views for url-path" cmd = 'pviews' #---- ICommand API methods def subparser( self, parser, subparsers ): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method.""" self.subparser = subparsers.add_parser( self.cmd, description=self.description ) self.subparser.set_defaults( handler=self.handle ) self.subparser.add_argument( "-n", dest="netpath", default=None, help="List view for application mounted on <netpath>" ) self.subparser.add_argument( "-a", dest="appname", default=None, help="List view for all instance of application <appname>" ) return parser def handle( self, args ): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method.""" print( 'hello world' ) #---- ISettings interface methods @classmethod def default_settings( cls ): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings( cls, sett ): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return sett
class Env(Plugin): """Sub-command plugin to generate scaffolding logic for pluggdapps development environment. Can be invoked from pa-script and meant for upstream authors.""" implements(IScaffold, ICommand) description = ("Scaffolding logic to create a new pluggdapps environment") #---- IScaffold API methods. def query_cmdline(self): """:meth:`pluggdapps.interfaces.IScaffold.query_cmdline` interface method.""" if not self['target_dir']: self['target_dir'] = input( "Enter target directory to create environment :") if not self['host_name']: self['host_name'] = input("Enter host name for the environment :") def generate(self): """:meth:`pluggdapps.interfaces.IScaffold.generate` interface method.""" _vars = {'host_name': self['host_name']} target_dir = abspath(self['target_dir']) host_name = abspath(self['host_name']) os.makedirs(target_dir, exist_ok=True) h.template_to_source(self['template_dir'], target_dir, _vars, overwrite=True, verbose=True) def printhelp(self): """:meth:`pluggdapps.interfaces.IScaffold.printhelp` interface method.""" sett = self.default_settings() print(self.description) for name, d in sett.specifications().items(): print(" %20s [%s]" % (name, d['default'])) pprint(d['help'], indent=4) print() #---- ICommand attributes and methods description = "Scaffolding logic to create a new pluggdapps environment." cmd = 'env' def subparser(self, parser, subparsers): """:meth:`pluggdapps.interfaces.ICommand.subparser` interface method.""" self.subparser = subparsers.add_parser(self.cmd, description=self.description) self.subparser.set_defaults(handler=self.handle) self.subparser.add_argument("-n", dest="host_name", default=None, help="Host name") self.subparser.add_argument( "-t", dest="target_dir", default=None, help="Target directory location for web application") return parser def handle(self, args): """:meth:`pluggdapps.interfaces.ICommand.handle` interface method.""" sett = { 'target_dir': args.target_dir or os.getcwd(), 'host_name': args.host_name, } scaff = self.qp(IScaffold, 'pluggdapps.Env', settings=sett) scaff.query_cmdline() print("Generating pluggdapps environment.") scaff.generate() #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" return sett
class FilterBlockPy( Plugin ): """Handle python code blocks. Follows indentation rules as defined by python language. To maintain consistency, it is better to indent the entire python code block by 2 spaces. Each line will be interpreted as a python statement and substituted as is while compiling them into an intermediate .py text. - If filter block is defined inside ``@def`` or ``@interface`` definition, then the filter block will inherit the same local scope and context as applicable to the function/interface definition. - Otherwise, it will be considered as local to the implicitly defined body() function. - To define python code blocks that are global to entire template module, define them outside template tags. .. code-block:: ttl :linenos: <div> :py: print( "hello world" ) :py: """ implements( ITayraFilterBlock ) def headpass1( self, igen, node ): self.filteropen = node.FILTEROPEN.dump(None) + node.NEWLINES1.dump(None) return None def headpass2( self, igen, node, result ): return result def generate( self, igen, node, result, *args, **kwargs ): # Inline self.localfunc = kwargs.get( 'localfunc', False ) self.args, self.kwargs = args, kwargs if self.localfunc : self.genlines( igen, node, *args, **kwargs ) return result def tailpass( self, igen, node, result ): if self.localfunc == False : self.genlines( igen, node, *self.args, **self.kwargs ) return result def genlines( self, igen, node, *args, **kwargs ): indent = len( self.filteropen.rsplit(os.linesep, 1)[-1] ) prefix = '' filtertext = node.filtertext[:] while filtertext : TERM = filtertext.pop(0) term = TERM.dump(None) igen.comment( "lineno:%s" % TERM.lineno ) igen.putstatement( prefix + term.rstrip(' \t') ) prefix = term.rsplit( os.linesep, 1 )[-1][indent:] node.NEWLINES2.generate( igen, *args, **kwargs ) #---- ISettings interface methods @classmethod def default_settings( cls ): return _default_settings @classmethod def normalize_settings( cls, sett ): return sett
class ConfigSqlite3DB(Plugin): """Backend interface to persist configuration information in sqlite3 database. Settings are stored in tables, one table for each mounted application. ``netpath`` name (contains subdomain-hostname / script), on which application is mounted, will be used as table-name. Table structure, <netpath> table : +-------------------+-------------------------------+ | section | settings | +===================+===============================+ | section-name | JSON string of settings | +-------------------+-------------------------------+ where, * section-name can be `special section` or plugin section that starts with ``plugin:`` prefix. * JSON string contains key,value pairs of configuration settings only for those parameters that were updated using web-frontend. * along with netpaths, a special table called ``platform`` will be created. Platform wide settings, overriden by settings from master-ini file, will be stored in this table. * special sections will be present only in case of ``platform`` table. For other `netpath` tables, other that ``[DEFAULT]`` section, no special section will be stored. """ implements(IConfigDB) def __init__(self): self.conn = sqlite3.connect(self['url']) if self['url'] else None def connect(self, *args, **kwargs): """:meth:`pluggdapps.interfaces.IConfigDB.connect` interface method.""" if self.conn == None and self['url']: self.conn = sqlite3.connect(self['url']) def dbinit(self, netpaths=[]): """:meth:`pluggdapps.interfaces.IConfigDB.dbinit` interface method. Optional key-word argument, ``netpaths``, list of web-application mount points. A database table will be created for each netpath. """ if self.conn == None: return None c = self.conn.cursor() # Create the `platform` table if it does not exist. c.execute("CREATE TABLE IF NOT EXISTS platform " "(section TEXT PRIMARY KEY ASC, settings TEXT);") self.conn.commit() for netpath in netpaths: sql = ( "CREATE TABLE IF NOT EXISTS '%s' " "(section TEXT PRIMARY KEY ASC, settings TEXT);" ) %\ netpath c.execute(sql) self.conn.commit() def config(self, **kwargs): """:meth:`pluggdapps.interfaces.IConfigDB.config` interface method. Keyword arguments, ``netpath``, Netpath, including subdomain-hostname and script-path, on which web-application is mounted. Optional. ``section``, Section name to get or set config parameter. Optional. ``name``, Configuration name to get or set for ``section``. Optional. ``value``, If present, this method was invoked for setting configuration ``name`` under ``section``. Optional. - if netpath, section, name and value kwargs are supplied, will update config-parameter `name` under webapp's `section` with `value`. Return the updated value. - if netpath, section, name kwargs are supplied, will return configuration `value` for `name` under webapp's `section`. - if netpath, section kwargs are supplied, will return dictionary of all configuration parameters under webapp's section. - if netpath is supplied, will return the entire table as dictionary of sections and settings. - if netpath is not supplied, will use `section`, `name` and `value` arguments in the context of ``platform`` table. """ if self.conn == None: return None netpath = kwargs.get('netpath', 'platform') section = kwargs.get('section', None) name = kwargs.get('name', None) value = kwargs.get('value', None) c = self.conn.cursor() if section: c.execute("SELECT * FROM '%s' WHERE section='%s'" % (netpath, section)) result = list(c) secsetts = h.json_decode(result[0][1]) if result else {} if name and value: secsetts[name] = value secsetts = h.json_encode(secsetts) c.execute("DELETE FROM '%s' WHERE section='%s'" % (netpath, section)) c.execute("INSERT INTO '%s' VALUES ('%s', '%s')" % (netpath, section, secsetts)) self.conn.commit() rc = value elif name: rc = secsetts[name] else: rc = secsetts else: c.execute("SELECT * FROM '%s'" % (netpath, )) rc = {section: h.json_decode(setts) for section, setts in list(c)} return rc def close(self): """:meth:`pluggdapps.interfaces.IConfigDB.close` interface method.""" if self.conn: self.conn.close() #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method. """ return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method. """ return sett
class WebApp(Plugin): """Base class for all web applications plugins. Implements :class:`pluggdapps.interfaces.IWebApp` interface, refer to interface specification to understand the general intent and purpose of web-application plugins. Every http request enters the application through this plugin class. A comprehensive set of configuration settings are made available by this class. """ implements(IWebApp) def startapp(self): """:meth:`pluggdapps.interfaces.IWebApps.startapp` interface method.""" # Initialize plugins required to handle http request. self.router = self.qp(IHTTPRouter, self['IHTTPRouter']) self.cookie = self.qp(IHTTPCookie, self['IHTTPCookie']) self.in_transformers = [ self.qp(IHTTPInBound, name) for name in self['IHTTPInBound'] ] self.out_transformers = [ self.qp(IHTTPOutBound, name) for name in self['IHTTPOutBound'] ] # Live debug. if self['debug']: self.livedebug = self.qp(IHTTPLiveDebug, self['IHTTPLiveDebug']) else: self.livedebug = None # Initialize plugins. self.router.onboot() def dorequest(self, request, body=None, chunk=None, trailers=None): """:meth:`pluggdapps.interfaces.IWebApps.dorequest` interface method.""" self.pa.logdebug( "[%s] %s %s" % (request.method, request.uri, request.httpconn.address)) try: # Initialize framework attributes request.router = self.router request.cookie = self.cookie # TODO : Initialize session attribute here. request.response = response = \ self.qp( IHTTPResponse, self['IHTTPResponse'], request ) request.handle(body=body, chunk=chunk, trailers=trailers) self.router.route(request) except: self.pa.logerror(h.print_exc()) response.set_header('content_type', b'text/html') if self['debug']: data = self.livedebug.render(request, *sys.exc_info()) response.set_status(b'200') else: response.set_status(b'500') data = ("An error occurred. See the error logs for more " "information. (Turn debug on to display exception " "reports here)") response.write(data) response.flush(finishing=True) def dochunk(self, request, chunk=None, trailers=None): """:meth:`pluggdapps.interfaces.IWebApps.dochunk` interface method.""" request.handle(chunk=chunk, trailers=trailers) self.router.route(request) def onfinish(self, request): """:meth:`pluggdapps.interfaces.IWebApps.onfinish` interface method.""" self.router.onfinish(request) def shutdown(self): """:meth:`pluggdapps.interfaces.IWebApps.shutdown` interface method.""" self.router = None self.cookie = None self.livedebug = None self.in_transformers = [] self.out_transformers = [] def urlfor(self, request, *args, **kwargs): """:meth:`pluggdapps.interfaces.IWebApps.urlfor` interface method.""" return urljoin(self.baseurl, self.pathfor(request, *args, **kwargs)) def pathfor(self, request, *args, **kwargs): """:meth:`pluggdapps.interfaces.IWebApps.pathfor` interface method.""" path = self.router.urlpath(request, *args, **kwargs) if path.startswith(URLSEP): # Prefix uriparts['script'] if request.uriparts['script']: path = request.uriparts['script'] + path return path return self.router.urlpath(request, *args, **kwargs) #---- ISettings interface methods @classmethod def default_settings(cls): """:meth:`pluggdapps.plugin.ISettings.default_settings` interface method.""" return _default_settings @classmethod def normalize_settings(cls, sett): """:meth:`pluggdapps.plugin.ISettings.normalize_settings` interface method.""" sett['encoding'] = sett['encoding'].lower() sett['IHTTPInBound'] = h.parsecsvlines(sett['IHTTPInBound']) sett['IHTTPOutBound'] = h.parsecsvlines(sett['IHTTPOutBound']) return sett