def __init__( self, connection_string=None, connection=None, adaptor=None, minconn=1, maxconn=1, poolkey=None, **args ): conn_str = re.sub(r'\s+', ' ', connection_string) if connection_string else None Dict.__init__( self, connection_string=conn_str, connection=connection, adaptor=adaptor, minconn=minconn, maxconn=maxconn, **args ) if self.connection is None: if self.adaptor is None: self.adaptor = importlib.import_module('sqlite3') elif isinstance(self.adaptor, str): self.adaptor = importlib.import_module(self.adaptor) if self.connection_string is not None: if self.adaptor.__name__ == 'psycopg2': self.pool = importlib.import_module('psycopg2.pool').ThreadedConnectionPool( self.minconn or 1, self.maxconn or 1, self.connection_string or '' ) self.connection = self.pool.getconn(key=self.poolkey) else: self.connection = self.adaptor.connect(self.connection_string or '') if 'sqlite3' in self.adaptor.__name__: self.execute("pragma foreign_keys = ON")
def __init__(self, canon=None): if type(canon)==Canon: self.canon = canon else: Dict.__init__(self, canon=Canon.from_xml(XML(fn=canon))) for book in self.canon.books: book.rexp = re.compile(book.pattern, flags=re.I+re.U)
def __init__(self, **args): Dict.__init__(self, **args) if self.id is not None: self.id = int(self.id) if self.ch is not None: self.ch = int(self.ch) if self.vs is not None: self.vs = int(self.vs)
def __init__(self, fn=None, data=None, ext=None, **args): if type(fn) == str: fn = self.normpath(fn) elif isinstance(fn, self.__class__): fn = str(fn) if ext is not None: fn = os.path.splitext(fn)[0] + ext Dict.__init__(self, fn=fn, data=data, **args)
def __init__(self, loader_class=None, loader_args={}, **Email): """set up an emailer with a particular Email config""" Dict.__init__(self, **Email) if loader_class is None: from tornado.template import Loader as loader_class if Email.get('template_path') is not None: self.loader = loader_class(Email.get('template_path'), **loader_args) if self.default_encoding is None: self.default_encoding = 'UTF-8'
def __init__(self, default=None, **namespaces): Dict.__init__(self) if default is not None: nsmap = {None:default} nsmap.update(**{k:namespaces[k] for k in namespaces.keys() if namespaces[k] != default}) else: nsmap = namespaces self.__dict__['nsmap'] = nsmap for key in namespaces: # each namespace gets its own method. named for its key self[key] = ElementMaker(namespace=namespaces[key], nsmap=nsmap) self._ = ElementMaker(namespace=default, nsmap=nsmap)
def __init__(self, url=None, local=None, parent_path=None, username=None, password=None, trust_server_cert=True, svn=None, svnmucc=None, svnlook=None, access_file=None): Dict.__init__(self, url=URL(url or ''), local=local, parent_path=parent_path, username=username, password=password, trust_server_cert=trust_server_cert, svn=svn or 'svn', svnmucc=svnmucc or 'svnmucc', svnlook=svnlook or 'svnlook') if access_file is not None: # load the access file from .svn_access_file import SVNAccessFile self.access_file = SVNAccessFile(access_file)
def render(self, fn=None, prompt=False, **params): """return a Config with the given params formatted via ``str.format(**params)``. fn=None : If given, will assign this filename to the rendered Config. prompt=False : If True, will prompt for any param that is None. """ from getpass import getpass expected_keys = self.expected_param_keys() compiled_params = Dict(**params) for key in expected_keys: if key not in compiled_params.keys(): if prompt == True: if key == 'password': compiled_params[key] = getpass("%s: " % key) else: compiled_params[key] = input("%s: " % key) if 'path' in key: compiled_params[key] = compiled_params[ key].replace('\\', '') else: compiled_params[key] = "%%(%s)s" % key config = ConfigTemplate(fn=fn, **self) config.__dict__['ordered_keys'] = self.__dict__.get('ordered_keys') for block in config.keys(): for key in config[block].keys(): if type(config[block][key]) == str: config[block][key] = config[block][key] % compiled_params return config
def initialize(c): c.os, c.re, c.time, c.glob, c.datetime = os, re, time, glob, datetime c.String = String c.config = c.settings.get('config') c.url = URL(c.request.full_url(), host=c.settings.host, scheme=c.settings.scheme) c.HTTPError = tornado.web.HTTPError c.messages = Dict()
def __init__(self, path, ingpath=None, outpath=None, errpath=None, **args): """ path [REQ'D] = the filesystem path to the queue directory. Created if it doesn't exist. ingpath = the 'processing' directory, defaults to path+"/ING". outpath = the 'finished' directory, defaults to path+"/OUT". errpath = the 'error' directory, defaults to path+'/ERR'. **args = any other arguments you want to define for your queue class. """ if not os.path.exists(path): os.makedirs(path) if ingpath is None: ingpath = path + '/ING' if not os.path.exists(ingpath): os.makedirs(ingpath) if outpath is None: outpath = path + '/OUT' if not os.path.exists(outpath): os.makedirs(outpath) if errpath is None: errpath = path + '/ERR' if not os.path.exists(errpath): os.makedirs(errpath) Dict.__init__(self, path=path, ingpath=ingpath, outpath=outpath, errpath=errpath, **args)
def encode(C, session, encoding='UTF-8'): d = Dict(**session) for k in d: if type(d[k]) == str: d[k] = d[k].encode(encoding=encoding) elif type(d[k]) == Dict: d[k] = C.encode(d[k], encoding=encoding) return d
def decode(C, session, encoding='UTF-8'): d = Dict(**session) for k in d: if type(d[k]) == bytes: d[k] = d[k].decode(encoding=encoding) elif type(d[k]) == Dict: d[k] = C.decode(d[k], encoding=encoding) return d
def header_data(c): """return header data as a Dict, each key followed by a list of values""" data = Dict() for key, value in c.request.headers.get_all(): if key not in data: data[key] = [] data[key].append(value) return data
def __init__(self, pattern, patterns=Patterns, **args): """Take a given pattern and return a Route object, which is used by RouteMap to create a route map. pattern: a url pattern, with named args in brackets: {var} name can take pattern indicator: {var:int} defaults to {var:slug} args: give default values for other variables handler: the name of the handler action: the name of the handler method to use Example: >>> routes = [Route("/{handler:slug}/"), ... Route("/users/{username:slug}/", handler='users')] >>> for route in routes: print(route.pattern, "(handler=%s)" % route.handler) ^/(?P<handler>[\w\$\-\_\.\+\!\*\(\)\, %%@]+)/?$ (handler=None) ^/users/(?P<username>[\w\$\-\_\.\+\!\*\(\)\, %%@]+)/?$ (handler=users) """ Dict.__init__(self, **args) self.pattern = self.parse(pattern, patterns=patterns) self.regex = self.compile(self.pattern)
def styleProperties(Class, style): """return a properties dict from a given cssutils style """ properties = Dict() for property in style.getProperties(all=True): stylename = property.name + ':' properties[stylename] = property.value if property.priority != '': properties[stylename] = ' !' + property.priority return properties
def finish(self): """record the current stack process as finished""" self.report(fraction=1.0) key = self.stack_key if key is not None: if self.data.get(key) is None: self.data[key] = [] start_time = self.current_times.get(key) or time() self.data[key].append( Dict(runtime=time() - start_time, **self.params))
def __init__(self, routes=None, default_host='', transforms=None, **settings): tornado.web.Application.__init__(self, routes, default_host=default_host, transforms=transforms) self.settings = Dict(**settings)
def parse_config(self, config, split_list=None): self.__dict__['ordered_keys'] = [] for s in config.sections(): self.__dict__['ordered_keys'].append(s) self[s] = Dict() for k, v in config.items(s): # resolve common data types if v.lower() in ['true', 'false', 'yes', 'no']: # boolean self[s][k] = config.getboolean(s, k) elif re.match("^\-?\d+$", v): # integer self[s][k] = int(v) elif re.match("^\-?\d+\.\d*$", v): # float self[s][k] = float(v) elif re.match(LIST_PATTERN, v): # list self[s][k] = eval(v) elif re.match(DICT_PATTERN, v): # dict self[s][k] = Dict(**eval(v)) elif split_list is not None \ and re.search(split_list, v) is not None: self[s][k] = re.split(split_list, v) else: # default: string self[s][k] = v.strip()
def __init__(self, pattern, handler, patterns=Patterns, kwargs=None, name=None): pattern = Route.parse(pattern) tornado.web.URLSpec.__init__(self, pattern, handler, kwargs=kwargs, name=name) self.kwargs = Dict(**self.kwargs)
def load(self, sessionid=None): if os.path.exists(self.session_file(sessionid)): f = open(self.session_file(sessionid)) try: sdata = Dict(**json.loads(f.read())) except: sdata = {} f.close s = Session(self) s.update(**sdata) s.id = sessionid return s else: return Session(self)
def match(self, url): """Given a url, returns a dict indicating how to handle the request.""" for rt in self: # for each route, matchdata = rt.regex.match(url) # see if it matches the url -- CASE-INSENSITIVE if matchdata: # if so, put routing info in a dict and return it d = Dict() # don't operate directly on the route! Bad side-effects would ensue (because Python uses call-by-reference). d.update(**rt) # include everything that is in the route d.pop('regex') # but don't include the compiled regex d.pop('pattern') # or the pattern, for that matter mdd = matchdata.groupdict() for k in mdd: # include named groups from match data if mdd[k] is not None: # except for None values d[k] = mdd[k] return d
def request_data(c): """return request data as a Dict""" return Dict( method=c.request.method, path=c.request.path, query=c.request.query, query_arguments=c.request.query_arguments, # body=c.request.body, body_arguments=c.request.body_arguments, headers=c.header_data, remote_ip=c.request.remote_ip, protocol=c.request.protocol, host=c.request.host, time=c.request.request_time(), # files={n:None for n in c.request.files.keys()}, )
def __init__(self, url='', **kwargs): """create a URL object from the given url cast to str via str(url). The given url can be modified with the following key-word arguments: * scheme = the URL scheme (http, file, mailto, etc.) * host = the host server * path = the path on the host server * params = any URL parameters (begins with ;) * fragment = URL fragment (begins with #) * query = URL query (begins with ?) * qargs = an alternative form of query, with the arguments already parsed into a dict """ # 1. parse the url string # s = str(url).replace('file://', 'file:') # needed for file URLs to parse correctly args = { k: kwargs[k] for k in ['scheme', 'host', 'path', 'params', 'fragment', 'query', 'qargs'] if kwargs.get(k) not in [None, {}, ''] } pr = urllib.parse.urlparse(str(url)) # 2. deal with parameters self.scheme = kwargs.get('scheme') or pr.scheme self.host = kwargs.get('host') or pr.netloc self.path = self.normpath( urllib.parse.unquote(kwargs.get('path') or pr.path)) self.params = kwargs.get('params') or pr.params self.fragment = kwargs.get('fragment') or pr.fragment # 3. deal with query arguments d = Dict(**urllib.parse.parse_qs(kwargs.get('query') or pr.query)) for k in d: d[k] = d[k][-1] # only keep the last instance of an argument if d[k] in [None, '']: _ = d.pop('k') qargs = kwargs.get('qargs') or {} self.qargs = d for k in qargs.keys(): if qargs[k] in ['', None]: if k in self.qargs.keys(): _ = self.qargs.pop(k) else: self.qargs[k] = qargs[k] # 4. deal with file: URL anomalies in urllib if self.scheme == 'file' and self.host != '': self.path, self.host = self.join(self.host, self.path).path, ''
def report(self, fraction=None): """report the total progress for the current stack, optionally given the local fraction completed. fraction=None: if given, used as the fraction of the local method so far completed. runtimes=None: if given, used as the expected runtimes for the current stack. """ r = Dict() local_key = self.stack_key if local_key is None: return {} runtimes = self.runtimes() for key in self.stack_keys: if self.current_times.get(key) is None: self.start(key=key) runtime = runtimes.get(key) or self.runtime(key) if key == local_key and fraction is not None: r[key] = fraction elif runtime is not None: r[key] = (time() - self.current_times[key]) / runtime return r
def from_xml(C, xml): # xml.assertValid() assert xml.root.tag == "{%(bl)s}book" % NS book = C( id=xml.root.get('id'), name=xml.root.get('name'), title=xml.root.find('{%(bl)s}title' % NS).text, pattern=xml.root.find('{%(bl)s}pattern' % NS).text, chapters=[ Dict(**chapter.attrib) for chapter in xml.root.find('{%(bl)s}chapters' % NS).getchildren() ] ) for e in [e for e in xml.root.xpath("*") if e.tag.replace("{%(bl)s}" % NS, "") not in ['title', 'pattern', 'chapters']]: attr = e.tag.replace("{%(bl)s}" % NS, "") book[attr] = e.text return book
def no_qargs(self): u = URL(**self) u.qargs = Dict() return u
def __init__(self, fn=None, mode='r', compression=ZIP_DEFLATED, **args): Dict.__init__(self, fn=fn, mode=mode, compression=compression, **args) if fn is not None: self.zipfile = ZipFile(self.fn, mode=mode, compression=compression)
def __init__(self, fn=None, data=None, **args): if type(fn)==str: fn=fn.strip().replace('\\ ', ' ') Dict.__init__(self, fn=fn, data=data, **args)
def __call__(self, port=None): self.listen(port or (self.settings.Site or Dict()).port or 80) tornado.ioloop.IOLoop.instance().start()
import os from glob import glob from bl.dict import Dict from .canon import Canon from bxml import XML canons = Dict( **{ os.path.basename(fn).split('-')[0]: Canon.from_xml(XML(fn=fn)) for fn in glob( os.path.join(os.path.dirname(__file__), 'resources', 'canons', '*.xml')) }) if __name__ == "__main__": import doctest doctest.testmod()
def __init__(self, db, **args): self.__dict__['db'] = db Dict.__init__(self, **args)
class XSLT(XML): """class for holding, manipulating, and using XSL documents""" NS = {'xsl': 'http://www.w3.org/1999/XSL/Transform'} TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S %Z" # format for timestamp params CACHE = Dict() # XSLT.CACHE = application-level cache. # The keys are digests of the XSLT text, which ensures that # unchanged xslt will cache, while changed xslt will compile. def __init__(self, cache=True, **kwargs): XML.__init__(self, **kwargs) def __call__(self, elem, cache=True, **params): # prepare string parameters -- see http://lxml.de/xpathxslt.html#stylesheet-parameters if 'timestamp' not in params.keys( ): # always include a timestamp param params['timestamp'] = time.strftime(self.TIMESTAMP_FORMAT) for key in params: if type(params[key]) in [str, bytes]: params[key] = etree.XSLT.strparam(params[key]) __xslt = self.make_xslt(cache=cache) return __xslt(elem, **params) def saxon6(self, elem, **params): """Use Saxon6 to process the element. If the XSLT has a filename (fn), use that. Otherwise, make temp. """ java = os.environ.get('java') or 'java' saxon6path = os.path.join( JARS, 'saxon.jar') # saxon 6.5.5, included with jing and trang with tempfile.TemporaryDirectory() as tempdir: if self.fn is None: xslfn = os.path.join(tempdir, "xslt.xsl") self.write(fn=xslfn) else: xslfn = self.fn srcfn = os.path.join(tempdir, "src.xml") outfn = os.path.join(tempdir, "out.xml") XML(fn=srcfn, root=elem).write() cmd = [java, '-jar', saxon6path, '-o', outfn, srcfn, xslfn] \ + ["%s=%r" % (key, params[key]) for key in params.keys()] log.debug("saxon6: %r " % cmd) try: subprocess.check_output(cmd) except subprocess.CalledProcessError as e: error = html.unescape(str(e.output, 'UTF-8')) raise RuntimeError(error).with_traceback( sys.exc_info()[2]) from None if self.find(self.root, "xsl:output") is None or self.find( self.root, "xsl:output").get('method') == 'xml': return etree.parse(outfn) else: return open(outfn, 'rb').read().decode('utf-8') def saxon9(self, elem, **params): """Use Saxon9 to process the element. If the XSLT has a filename (fn), use that. Otherwise, make temp. Returns an lxml.etree._ElementTree (not _Element) """ java = os.environ.get('java') or 'java' saxon9path = os.path.join(JARS, 'saxon9', 'saxon9he.jar') # saxon 9 with tempfile.TemporaryDirectory() as tempdir: if self.fn is None: xslfn = os.path.join(tempdir, "xslt.xsl") self.write(fn=xslfn) else: xslfn = self.fn srcfn = os.path.join(tempdir, "src.xml") outfn = os.path.join(tempdir, "out.xml") XML(fn=srcfn, root=elem).write() cmd = [java, '-jar', saxon9path, '-o:%s' % outfn, '-s:%s' % srcfn, '-xsl:%s' % xslfn] \ + ['%s=%s' % (key, params[key]) for key in params.keys()] log.debug("saxon9: %r " % cmd) try: subprocess.check_output(cmd) except subprocess.CalledProcessError as e: error = html.unescape(str(e.output, 'UTF-8')) raise RuntimeError(error).with_traceback( sys.exc_info()[2]) from None if self.find(self.root, "xsl:output") is None or self.find( self.root, "xsl:output").get('method') == 'xml': return etree.parse(outfn) else: return open(outfn, 'rb').read().decode('utf-8') def append(self, s, *args): if type(s) == etree._Element: elem = s else: elem = XML.Element(s, *args) try: self.root.append(elem) self.xslt = self.make_xslt() except: self.root.remove(elem) raise def make_xslt(self, elem=None, cache=True): # parse the source file here if available, so that the XSLT knows where it is. if elem is None: if self.fn is not None: elem = etree.parse(self.fn) else: elem = self.root if cache == True: digest = String(etree.tounicode(elem)).digest() if self.__class__.CACHE.get(digest) is not None: return self.__class__.CACHE.get(digest) xsl = self.__class__.CACHE[digest] = etree.XSLT(elem) else: xsl = etree.XSLT(elem) return xsl @classmethod def clear_cache(self): XSLT.CACHE = Dict() # == TEMPLATE METHODS == @classmethod def stylesheet(cls, *args, namespaces=None, version='1.0'): if namespaces is not None: nst = ' ' + ' '.join([ "xmlns:%s='%s'" % (k, namespaces[k]) for k in namespaces.keys() if k is not None ]) if None in namespaces.keys(): nst += " xmlns='%s'" % namespaces[None] else: nst = '' xt = XML.Element(XSL_TEMPLATE % (version, XSL_NAMESPACE, nst)) for arg in [a for a in args if a is not None]: xt.append(arg) return xt @classmethod def copy_all(cls): return XML.Element( """<xsl:template match="@*|node()" %s><xsl:copy><xsl:apply-templates select="@*|node()"/></xsl:copy></xsl:template>""", XSL_NAMESPACE) @classmethod def copy(cls, *vals): elem = XML.Element("""<xsl:copy %s></xsl:copy>""", XSL_NAMESPACE) for val in vals: elem.append(val) return elem @classmethod def copy_select(cls, select): elem = XML.Element("""<xsl:copy %s select="%s"></xsl:copy>""", XSL_NAMESPACE, select) for val in vals: elem.append(val) return elem @classmethod def copy_of(cls): return XML.Element("""<xsl:copy-of %s/>""", XSL_NAMESPACE) @classmethod def copy_of_select(cls, select): return XML.Element("""<xsl:copy-of %s select="%s"/>""", XSL_NAMESPACE, select) @classmethod def choose(cls, *args): return XML.Element("<xsl:choose %s/>", XSL_NAMESPACE, *args) @classmethod def when(cls, test, *vals): elem = XML.Element("""<xsl:when %s test="%s"></xsl:when>""", XSL_NAMESPACE, test) for val in vals: elem.append(val) return elem def otherwise(cls, *vals): elem = XML.Element("""<xsl:otherwise %s></xsl:otherwise>""", XSL_NAMESPACE, test) for val in vals: elem.append(val) return elem @classmethod def template_match(cls, match, *vals): elem = XML.Element("""<xsl:template %s match="%s"></xsl:template>""", XSL_NAMESPACE, match) for val in vals: elem.append(val) return elem @classmethod def template_match_mode(cls, match, mode, *vals): elem = XML.Element( """\n<xsl:template %s match="%s" mode="%s"></xsl:template>""", XSL_NAMESPACE, match, mode) for val in vals: elem.append(val) return elem @classmethod def template_name(cls, name, *vals): elem = XML.Element("""<xsl:template %s name="%s"></xsl:template>""", XSL_NAMESPACE, name) for val in vals: elem.append(val) return elem @classmethod def template_match_omission(cls, match): return XML.Element("""<xsl:template %s match="%s"/>""", XSL_NAMESPACE, match) @classmethod def apply_templates(cls): return XML.Element("""<xsl:apply-templates %s/>""", XSL_NAMESPACE) @classmethod def apply_templates_mode(cls, mode): return XML.Element("""<xsl:apply-templates %s mode="%s"/>""", XSL_NAMESPACE, mode) @classmethod def apply_templates_select(cls, select): return XML.Element("""<xsl:apply-templates %s select="%s"/>""", XSL_NAMESPACE, select) @classmethod def apply_templates_select_mode(cls, select, mode): return XML.Element( """<xsl:apply-templates %s select="%s" mode="%s"/>""", XSL_NAMESPACE, select, mode) @classmethod def element(cls, name, *vals): elem = XML.Element("""<xsl:element %s name="%s"></xsl:element>""", XSL_NAMESPACE, name) for val in vals: elem.append(val) return elem @classmethod def attribute(cls, name, *vals): elem = XML.Element("""<xsl:attribute %s name="%s"></xsl:attribute>""", XSL_NAMESPACE, name) for val in vals: elem.append(val) return elem @classmethod def variable(cls, name, *vals): elem = XML.Element("""<xsl:variable %s name="%s"></xsl:variable>""", XSL_NAMESPACE, name) for val in vals: elem.append(val) return elem @classmethod def variable_select(cls, name, select): return XML.Element("""<xsl:variable %s name="%s" select="%s"/>""", XSL_NAMESPACE, name, select) @classmethod def value_of(cls, select): return XML.Element("""<xsl:value-of %s select="%s"/>""", XSL_NAMESPACE, select) @classmethod def text(cls, t): return XML.Element("""<xsl:text %s>%s</xsl:text>""", XSL_NAMESPACE, t) @classmethod def for_each(cls, select, *vals): elem = XML.Element("""<xsl:for-each %s select="%s"></xsl:for-each>""", XSL_NAMESPACE, select) for val in vals: elem.append(val) return elem @classmethod def if_test(cls, test, *vals): return XML.Element("""<xsl:if %s test="%s"></xsl:if>""", XSL_NAMESPACE, test) for val in vals: elem.append(val) return elem @classmethod def output_method(cls, method): return XML.Element("""<xsl:output method="%s" encoding="utf-8" %s/>""", XSL_NAMESPACE, method)
def clear_cache(self): XSLT.CACHE = Dict()
def __init__(self, fn=None, data=None, key=None, **params): # JSON.__init__() loads the stored progress data from the given .json file super().__init__(fn=fn, data=data, params=params) if self.data is None: self.data = Dict() self.current_times = Dict() # start times for current stack processes. self.init_key = key or self.stack_key
def runtimes(self): return Dict(**{key: self.runtime(key) for key in self.data.keys()})