def _csv (self, ET, names, attrs, sep) : import csv a_ref = self._e_type_attr_ref a_typ = self._e_type_attr_typ body = pyk.StringIO () head = pyk.StringIO () bw = csv.DictWriter (body, names) hw = csv.DictWriter (head, names) for ak in attrs : ad = dict \ ( name = a_ref (ET, ak.name) , type = a_typ (ET, ak) , kind = ak.kind , required = ak.is_required , default = ak.raw_default ) bw.writerow (ad) hw.writerow (dict ((n, n) for n in names)) result = \ ( pyk.decoded (head.getvalue ().strip ()) , sep.join ( l.strip () for l in pyk.decoded (body.getvalue ()).split ("\n") ) ) return result
def _violator_values (self) : inv = self.inv bvars = list \ ( x.strip () for x in inv.bvar.replace ("(", "").replace (")", "").split (",") ) for v, d in paired (self.violators, self.violators_attr) : if len (bvars) > 1 and isinstance (v, (list, tuple)) : for k, w in paired (flattened (bvars), flattened (v)) : yield (pyk.decoded (k.strip (), "utf-8"), ui_display (w)) elif isinstance (v, (list, tuple)) : yield \ ( pyk.decoded (inv.bvar, "utf-8") , "[%s]" % (", ".join (map (str, v)), ) ) else : yield (pyk.decoded (inv.bvar, "utf-8"), ui_display (v)) if d : try : items = pyk.iteritems (d) except AttributeError : val = ui_display (d) else : val = sorted \ ( (pyk.decoded (a, "utf-8"), ui_display (x)) for (a, x) in items ) yield ui_display (v), val
def body_lines (self, sep_length = 79) : lines = self.lines if lines is None : if self.body : charset = self.charset or PMA.default_encoding type = self.content_type if type == "text/plain" or type.startswith ("text/x-") : lines = pyk.decoded (self.body, charset).split ("\n") ### XXX put full-quotes-at-end into `self.parts [0]` else : cap = PMA.Mailcap [type] if cap : lines = cap.as_text \ (self._temp_body (), encoding = charset) if lines is None and type.startswith ("text/") : lines = pyk.decoded (self.body, charset).split ("\n") if lines is not None : self.lines = lines if lines is not None : wrapper = textwrap.TextWrapper \ (width = PMA.text_output_width, break_long_words = False) for line in lines : line = line.rstrip ("\r") if line : for l in wrapper.wrap (line or "") : yield l else : ### `wrapper` returns an empty list for an empty string yield line else : hp = Part_Header (self.email, self.headers_to_show) self.lines = lines = list (hp.formatted (sep_length)) for l in lines : yield l
def body_lines(self, sep_length=79): lines = self.lines if lines is None: if self.body: charset = self.charset or PMA.default_encoding type = self.content_type if type == "text/plain" or type.startswith("text/x-"): lines = pyk.decoded(self.body, charset).split("\n") ### XXX put full-quotes-at-end into `self.parts [0]` else: cap = PMA.Mailcap[type] if cap: lines = cap.as_text \ (self._temp_body (), encoding = charset) if lines is None and type.startswith("text/"): lines = pyk.decoded(self.body, charset).split("\n") if lines is not None: self.lines = lines if lines is not None: wrapper = textwrap.TextWrapper \ (width = PMA.text_output_width, break_long_words = False) for line in lines: line = line.rstrip("\r") if line: for l in wrapper.wrap(line or ""): yield l else: ### `wrapper` returns an empty list for an empty string yield line else: hp = Part_Header(self.email, self.headers_to_show) self.lines = lines = list(hp.formatted(sep_length)) for l in lines: yield l
def _violator_values (self) : inv = self.inv bvars = list \ ( x.strip () for x in inv.bvar.replace ("(", "").replace (")", "").split (",") ) for v, d in paired (self.violators, self.violators_attr) : if len (bvars) > 1 and isinstance (v, (list, tuple)) : for k, w in paired (flattened (bvars), flattened (v)) : yield (pyk.decoded (k.strip (), "utf-8"), ui_display (w)) elif isinstance (v, (list, tuple)) : yield \ ( pyk.decoded (inv.bvar, "utf-8") , "[%s]" % (", ".join (map (pyk.text_type, v)), ) ) else : yield (pyk.decoded (inv.bvar, "utf-8"), ui_display (v)) if d : try : items = pyk.iteritems (d) except AttributeError : val = ui_display (d) else : val = sorted \ ( (pyk.decoded (a, "utf-8"), ui_display (x)) for (a, x) in items ) yield ui_display (v), val
def _get_attr (self, name) : result = getattr (self.__fo, name) if result is None : result = "" elif isinstance (result, (tuple, list)) : result = "<br>".join \ ( pyk.decoded (r if r is not None else "", self.__enc) for r in result ) return pyk.decoded (result, self.__enc)
def bindings (self) : if self.entity : FO = self.entity.FO name = self.attribute try : fov = FO (name, self.value) except Exception : value = pyk.decoded (self.value) else : value = pyk.decoded (fov) return ((name, value), )
def __init__ (self, e_type, role, r_obj, epk, * links) : self.e_type = e_type self.role = role self.type_name = e_type.ui_name self.l_ui_display = pyk.decoded (pyk.reprify (epk)) self.r_ui_display = r_obj.ui_repr self.extra_links = tuple \ ( TFL.Record ( pid = getattr (x, "pid", None) , ui_display = pyk.decoded (x.ui_repr) , type_name = _T (x.ui_name) ) for x in links )
def formatted_repr (obj, level = 0, compact = False, indent = " ") : """Return a formatted canonical string representation of `obj`.""" return "\n".join \ ( ( "".join ( ( indent * line.level , pyk.decoded (line.head, pyk.user_config.output_encoding, "ascii") , pyk.decoded (line.body, pyk.user_config.output_encoding, "ascii") , pyk.decoded (line.tail, pyk.user_config.output_encoding, "ascii") ) ) ).replace (" ", " ").rstrip (" ") for line in formatted_repr.iter (obj, level, compact, seen = set ()) )
def load(self): if not self.cachers: return kw = self.kw path = self.cache_path try: with open(path, "rb") as file: cargo = pyk.pickle.load(file) except ValueError as exc: msg = pyk.decoded(exc) if "pickle" in msg: raise IOError(exc) logging.warning \ ( "Loading pickle dump %s failed with exception: %s" % (path, exc) ) raise except Exception as exc: logging.warning \ ( "Loading pickle dump %s failed with exception: %s" % (path, exc) ) raise else: if self.verbose: logging.info("Loaded pickle dump %s successfully" % (path, )) for cp in self._gen_cachers("Unpickling", "Regenerate cache..."): cp.from_pickle_cargo(cargo=cargo, **kw)
def __call__ (self, txt, encoding = "utf8", language = "en", include = None, ** kw) : settings = dict \ ( self.settings , output_encoding = encoding , language_code = language , ** kw ) try : text = pyk.decoded (txt) parts = self._publish_parts \ ( source = text , writer_name = "html4css1" , settings_overrides = settings ) except Exception as exc : msg = \ ( _T ( "Conversion from re-structured text to html " "failed with exception:\n %s" ) % (exc, ) ) logging.exception (msg + "\n\n Offending text:" + text) raise ValueError (msg) if include is None : include = self.include result = "\n".join (p for p in (parts [i] for i in include) if p) for rep in self.replacers : result = rep (result) return result
def str_arg (self, args) : for a in args : try : s = pyk.decoded (a) except Exception as exc : s = "%s --> %s" % (portable_repr (a), exc) yield s
def value (self) : result = self._value if result is None : sig = pyk.decoded (self.signature, self.encoding) result = self._value = self.val_sep.join \ ((self.cargo, self.timestamp, sig)) return result
def handler (self, cmd) : i_enc = cmd.input_encoding tfmt = self.time_fmt = cmd.time_format for arg in cmd.argv : if arg == "-" : arg = "STDIN" output = Filename (".pdf", cmd.Output or arg).name if arg == "STDIN" : file_time = time.localtime () fn = output txt_in = sys.stdin.read () else : file_time = time.localtime (sos.path.getmtime (arg)) fn = self._display_filename (cmd, arg) with open (arg, "rb") as fi : txt_in = fi.read () txt = Untabified \ (pyk.decoded (txt_in, i_enc).lstrip ("\n\f").rstrip ()) ft = time.strftime (tfmt, file_time) options = self._pdf_opts (cmd, arg, fn, ft, output) if cmd.show_options : from _TFL.formatted_repr import formatted_repr print (formatted_repr (options)) ofn = options ["output"] pdf_doc = PDF_Doc (txt, ** options) if cmd.Display : subprocess.run ([cmd.display_program, ofn]) elif cmd.Print : subprocess.run ([cmd.print_program, ofn]) if cmd.purge_pdf and (cmd.Display or cmd.Print) : sos.unlink (ofn) elif cmd.verbose : print ("Created file", ofn)
def value(self): result = self._value if result is None: sig = pyk.decoded(self.signature, self.encoding) result = self._value = self.val_sep.join \ ((self.cargo, self.timestamp, sig)) return result
def minified_js(code): """Return minified javascript `code`. If neither `jsmin`_ nor `rjsmin`_ is installed, `code` is returned unchanged. .. _`jsmin`: https://bitbucket.org/dcs/jsmin/ .. _`rjsmin`: http://opensource.perlig.de/rjsmin/ """ jsmin = None result = code try: from rjsmin import jsmin except ImportError: try: ### https://packages.debian.org/sid/python/python-jsmin from jsmin import jsmin except ImportError: logging.warning("Couldn't import either rjsmin nor jsmin") if jsmin is not None: try: result = jsmin(code) except Exception as exc: logging.error("Exception during minified_js\n %s" % (exc, )) return pyk.encoded \ ( _clean_minified_js (pyk.decoded (result, "utf-8", "latin-1")) , "utf-8" )
def _file_contents(name, encoding="utf-8"): if sos.path.exists(name): with open(name, "rb") as f: result = pyk.decoded(f.read().strip(), encoding) return result else: print("*** *** *** File doesn't exist:", name, "*** *** ***")
def __call__(self, txt, encoding="utf8", language="en", include=None, **kw): settings = dict \ ( self.settings , output_encoding = encoding , language_code = language , ** kw ) try: text = pyk.decoded(txt) parts = self._publish_parts \ ( source = text , writer_name = "html4css1" , settings_overrides = settings ) except Exception as exc: msg = \ ( _T ( "Conversion from re-structured text to html " "failed with exception:\n %s" ) % (exc, ) ) logging.exception(msg + "\n\n Offending text:" + text) raise ValueError(msg) if include is None: include = self.include result = "\n".join(p for p in (parts[i] for i in include) if p) for rep in self.replacers: result = rep(result) return result
def load (self) : if not self.cachers : return kw = self.kw path = self.cache_path try : with open (path, "rb") as file : cargo = pyk.pickle.load (file) except ValueError as exc : msg = pyk.decoded (exc) if "pickle" in msg : raise IOError (exc) logging.warning \ ( "Loading pickle dump %s failed with exception: %s" % (path, exc) ) raise except Exception as exc : logging.warning \ ( "Loading pickle dump %s failed with exception: %s" % (path, exc) ) raise else : if self.verbose : logging.info ("Loaded pickle dump %s successfully" % (path, )) for cp in self._gen_cachers ("Unpickling", "Regenerate cache...") : cp.from_pickle_cargo (cargo = cargo, ** kw)
def csrf_check (self, request, response) : if self.csrf_check_p : error = None if not request.same_origin : error = \ ( _T ("Incorrect origin `%s` for request, expected `%r`") % (request.origin_host, request.server_name) ) else : csrf_token = request.csrf_token if not csrf_token : if self.DEBUG : error = "\n".join \ ( ( pyk.decoded (repr (csrf_token)) , "Client sig : %s" % (csrf_token.x_signature, ) , "Client value: %s" % (csrf_token.x_value, ) , "Server value: %s" % (csrf_token.value, ) , "Session sid : %s" % (request.session.sid, ) ) ) else : error = csrf_token._invalid if error : exc = self.top.Status.See_Other (self.abs_href) self.send_error_email (request, "CSRF", xtra = error) raise exc
def _file_contents (name, encoding = "utf-8") : if sos.path.exists (name) : with open (name, "rb") as f : result = pyk.decoded (f.read ().strip (), encoding) return result else : print ("*** *** *** File doesn't exist:", name, "*** *** ***")
def minified_js (code) : """Return minified javascript `code`. If neither `jsmin`_ nor `rjsmin`_ is installed, `code` is returned unchanged. .. _`jsmin`: https://bitbucket.org/dcs/jsmin/ .. _`rjsmin`: http://opensource.perlig.de/rjsmin/ """ jsmin = None result = code try : from rjsmin import jsmin except ImportError : try : ### https://packages.debian.org/sid/python/python-jsmin from jsmin import jsmin except ImportError : logging.warning ("Couldn't import either rjsmin nor jsmin") if jsmin is not None : try : result = jsmin (code) except Exception as exc : logging.error ("Exception during minified_js\n %s" % (exc, )) return pyk.encoded \ ( _clean_minified_js (pyk.decoded (result, "utf-8", "latin-1")) , "utf-8" )
def py_version (pyx) : try : result = py_version.map [pyx] except KeyError : result = py_version.map [pyx] = subprocess.check_output \ ( [pyx, "--version"], stderr = subprocess.STDOUT ).split (b" ") [-1].strip () return pyk.decoded (result)
def body (self) : result = self._body if result is None : fn = self.file_name with open (fn, "rb") as f : result = self._body = \ pyk.decoded (f.read ().strip (), "utf-8", "iso-8859-1") return result
def body(self): result = self._body if result is None: fn = self.file_name with open(fn, "rb") as f: result = self._body = \ pyk.decoded (f.read ().strip (), "utf-8", "iso-8859-1") return result
def as_text (self, key, filename = "", plist = [], encoding = None) : pipe = sos.popen (self.command (key, filename, plist), "r") body = pipe.read () pipe.close () if encoding is not None : body = pyk.decoded (body, encoding) result = body.split ("\n") return result
def _gen (handler, encoding, scripts, media) : for s in sorted (scripts, key = TFL.Getter.rank) : p = handler.get_path (s.src) if p : with open (p, "rb") as file : yield file.read ().decode (encoding) for s in sorted (media.script_files, key = TFL.Getter.rank) : yield pyk.decoded (s.body, encoding)
def as_static_page (self) : """Return static HTML page for this resource.""" top = self.top with top.LET (dynamic_p = False) : app_iter, status, headers = top.test_client.get \ (self.abs_href_dynamic) if status.startswith ("200") : return "".join (pyk.decoded (chunk) for chunk in app_iter)
def _get_body(self): _pl = self.msg.email while _pl.is_multipart(): _pl = _pl.get_payload(0) result = _pl.get_payload(decode=True) or u"" result = pyk.decoded(result, self.msg.charset) result = _ws_pat.sub(u" ", result.strip()) or u"<empty>" return result
def _get_body (self) : _pl = self.msg.email while _pl.is_multipart () : _pl = _pl.get_payload (0) result = _pl.get_payload (decode = True) or u"" result = pyk.decoded (result, self.msg.charset) result = _ws_pat.sub (u" ", result.strip ()) or u"<empty>" return result
def hashed(cls, clear_password, salt=None): """Hashed value of `clear_password` using `salt`""" if salt is None: salt = bcrypt.gensalt(cls.default_rounds) else: salt = pyk.encoded(salt) result = bcrypt.hashpw(pyk.encoded(clear_password, "ascii"), salt) return pyk.decoded(result, "ascii")
def hashed (cls, clear_password, salt = None) : """Hashed value of `clear_password` using `salt`""" if salt is None : salt = bcrypt.gensalt (cls.default_rounds) else : salt = pyk.encoded (salt) result = bcrypt.hashpw (pyk.encoded (clear_password, "ascii"), salt) return pyk.decoded (result, "ascii")
def bindings (self) : FO = self.FO return sorted \ ( (k, (pyk.decoded (FO (k, v)) if v is not None else v)) for (k, v) in itertools.chain ( (zip (self.e_type.epk_sig, self.epk)) , pyk.iteritems (self.kw) ) )
def _formatted_bindings (self, bindings = None) : if bindings is None : bindings = self.bindings for k, v in bindings : if isinstance (v, (list, tuple)) : v = ", ".join \ ("%s" % (portable_repr (pyk.decoded (x)), ) for x in v) elif v is None or v == "''": v = _T ("None") yield "%s = %s" % (k, v)
def filename (self) : email = self.email if email : result = email.get_param \ ("filename", header = "Content-Disposition") if not result : result = email.get_param ("name") if isinstance (result, tuple) : result = pyk.decoded (result [2], result [0] or "us-ascii") return TFL.Ascii.sanitized_filename (decoded_header (result))
def parse (self) : wlo = \ ( 'Wireless Overview' , pyk.decoded ('Drahtlosübersicht', 'utf-8') , pyk.decoded ('WLAN Übersicht', 'utf-8') ) root = self.tree.getroot () self.wlans = [] for div in root.findall (".//%s" % tag ("div")) : if div.get ('class') != 'cbi-map' : continue if not len (div) or div [0].tag != tag ('h2') : continue if div [0].text not in wlo : continue for tr in div.findall (".//%s" % tag ("tr")) : cls = tr.get ('class') or '' cls = cls.split () if 'cbi-section-table-row' not in cls : continue d = WLAN_Config () self.wlans.append (d) for td in tr : k = td.get ('id') if k : k = k.split ('-') [-1] # special handling of signal picture which has # the necessary info in a title attribute :-( if k == 'signal' and not td.text : if td [0].tag == tag ('img') : title = td [0].get ('title') m = self.title_re.search (title) if m : d.set \ ( signal = m.group (1) , noise = m.group (2) ) continue else : k = 'name' v = td.text d.set (** {k : v}) break
def decoded_header (header) : result = [] if header : for p, c in Lib.decode_header (header) : enc = c or PMA.default_encoding if enc == "unknown" : enc = PMA.default_encoding result.append (pyk.decoded (p, enc)) result = " ".join (result) return result
def decoded_header(header): result = [] if header: for p, c in Lib.decode_header(header): enc = c or PMA.default_encoding if enc == "unknown": enc = PMA.default_encoding result.append(pyk.decoded(p, enc)) result = " ".join(result) return result
def filename(self): email = self.email if email: result = email.get_param \ ("filename", header = "Content-Disposition") if not result: result = email.get_param("name") if isinstance(result, tuple): result = pyk.decoded(result[2], result[0] or "us-ascii") return TFL.Ascii.sanitized_filename(decoded_header(result))
def fallback_repr (self) : try: info = ''.join (format_exception_only (* sys.exc_info () [:2])) except : info = '?' return \ ( '<span class="brokenrepr"><broken repr (%s)>' '</span>' % escape (pyk.decoded (info, "utf-8")).strip () )
def external_module_imports \ (pkg_name, mod_path, ignore_imports = ignore_imports) : """Return the set of external imports of module at `mod_path`.""" result = set () with open (mod_path, "rb") as f : code = pyk.decoded (f.read (), pyk.user_config.input_encoding, "utf-8") for match in import_pat.search_iter (code) : i = match.group ("imported") if not i.startswith (pkg_name) and i not in ignore_imports : result.add (i) return result
def send (self, from_addr, to_addrs, msg, mail_opts = None, rcpt_opts = None) : charset = self.charset msg_s = pyk.decoded (msg, charset) self._log \ ( pyk.as_str ("[%s] Email via %s from %s to %s\n %s", charset) , datetime.datetime.now ().replace (microsecond = 0) , pyk.as_str (self.mail_host, charset) , pyk.as_str (from_addr, charset) , list (pyk.as_str (t, charset) for t in to_addrs) , pyk.as_str ("\n ".join (msg_s.split ("\n")), charset) )
def NEW (cls, app_type, scope = None, ** _kw) : kw = dict (_kw) Version = app_type.Version result = cls \ ( creator = kw.pop ("creator", None) or TFL.Record ( date = pyk.decoded (datetime.datetime.utcnow ()) , tool_version = Version.id , user = getattr (scope, "user", None) or TFL.Environment.username ) , dbid = pyk.decoded (uuid.uuid4 ()) , dbv_hash = app_type.db_version_hash , guid = kw.pop ("guid", None) or getattr (scope, "guid", None) , readonly = kw.pop ("readonly", False) , root_pid = kw.pop ("root_pid", None) or getattr (scope, "_root_pid", None) , ** kw ) return result