def po2csv(po): """Converts a polib.POFile object to a csv string (contained in an instance of StringIO) """ csv_file = StringIO() csv_writer = csv.writer(csv_file) for header_row in csv_header_rows: csv_writer.writerow(header_row) for entry in po: msgstrs = force_plurals(entry) fuzzy = bool2str('fuzzy' in entry.flags, empty_for_false=True) obsolete = bool2str(entry.obsolete, empty_for_false=True) row = [ escape(entry.msgid), escape(entry.msgid_plural), entry.msgctxt, msgstrs, fuzzy, entry.comment, obsolete ] csv_writer.writerow(row) return csv_file
def force_plurals(entry): """A po file's plural handling is slightly weird. If there are no plurals for a translation: msgid "hello" msgstr "hola" If there are plurals however: msgid "%(count)s house" msgid_plural "%(count)s houses" msgstr[0] "%(count)s casa" msgstr[1] "%(count)s casas" So to keep things consistent, we force non-plural translations into the plural form for consistency, and then reading them back, we still have enough information to convert them back into singular translations if that's what they were meant to be. """ plurals = entry.msgstr_plural if not plurals: plurals = {u'0': escape(entry.msgstr)} sorted_plurals = sorted(plurals.items(), key=lambda t: t[0]) return PLURAL_SEPARATOR.join(escape(v) for k, v in sorted_plurals)
def writeFavourites(file, faves): f = sfile.file(file, 'w') f.write('<favourites>') for fave in faves: try: name = utils.escape(fave[0]) thumb = utils.escape(fave[1]) cmd = utils.escape(fave[2]) thumb = convertToHome(thumb) name = 'name="%s" ' % name thumb = 'thumb="%s">' % thumb f.write('\n\t<favourite ') f.write(name) f.write(thumb) f.write(cmd) f.write('</favourite>') except: pass f.write('\n</favourites>') f.close() import xbmcgui try: count = int(xbmcgui.Window(10000).getProperty('Super_Favourites_Count')) except: count = 0 xbmcgui.Window(10000).setProperty('Super_Favourites_Count', str(count+1))
def harvest(text, harvesters=DEFAULT_HARVESTERS): instances = [load_class(namespace) for namespace in harvesters] display_text = '' display_html = '' entities = [] current_text = text for instance in instances: e = instance.harvest(current_text) current_position = 0 for entity in e: entities.append(entity) current_position = entity.start_index l = len(entity.original_text) replacement = repeat_to_length(' ', l) current_text = current_text[:current_position] + \ replacement + current_text[current_position + l:] current_index = 0 for entity in entities: display_html = display_html + escape(text[current_index:entity.start_index]) + entity.display_html display_text = display_text + escape(text[current_index:entity.start_index]) + entity.display_text current_index = entity.end_index display_text = display_text + escape(text[current_index:]) display_html = display_html + escape(text[current_index:]) return { 'display_text': display_text, 'display_html': display_html, }
def set_label_text(self, info, label_text, label_extra=None): label_text = label_text.replace('\n', '').replace('\r', '') if label_extra: self.info_label.set_markup(info % (escape(label_text), escape(label_extra))) else: self.info_label.set_markup(info % escape(label_text))
def build_html_params(**params): params_list = list() for (name, value) in params.items(): if name[0] == u'_': name = name[1:] params_list.append(u' {0}="{1}"'.format( escape(name), escape(value), )) return u''.join(params_list)
def build_signature_base_string(self, oauth_request, consumer, token): sig = ( escape(oauth_request.get_normalized_http_method()), escape(oauth_request.get_normalized_http_url()), escape(oauth_request.get_normalized_parameters()), ) key = '%s&' % escape(consumer.secret) if token: key += escape(token.secret) raw = '&'.join(sig) return key, raw
def get_normalized_parameters(self): """Return a string that contains the parameters that must be signed.""" params = self.parameters try: # Exclude the signature if it exists. del params["oauth_signature"] except: pass # Escape key values before sorting. key_values = [(escape(_utf8_str(k)), escape(_utf8_str(v))) for k, v in params.items()] # Sort lexicographically, first after key, then after value. key_values.sort() # Combine key value pairs into a string. return "&".join(["%s=%s" % (k, v) for k, v in key_values])
def import_experiment(exp_dirname): tasks = check_experiment(exp_dirname) print(exp_dirname) if exp_dirname.endswith("/"): exp_dirname = exp_dirname[:-1] exp_name = os.path.basename(exp_dirname) print(exp_name) i = 0 # while n = exp_name while True: c = con.cursor() s = c.execute("select id from experiments where name = '%s'" % (n)) if not s.fetchall(): break i += 1 n = exp_name + ("-%s" % i) exp_name = n c = con.cursor() print( "insert into experiments (name, url_key, description, visible) values ('%s', '%s', '', 1);" % (exp_name, exp_name)) c.execute( "insert into experiments (name, url_key, description, visible) values ('%s', '%s', '', 1);" % (exp_name, exp_name)) con.commit() s = c.execute("select id from experiments where name = '%s'" % (exp_name)) exp_id, *_ = s.fetchone() source = open(os.path.join(exp_dirname, "source.txt"), "r") ref_path = os.path.join(exp_dirname, "reference.txt") reference = open(ref_path, "r") c = con.cursor() i = 1 for s, r in zip(source, reference): s = escape(s) r = escape(r) # print(i,"""insert into sentences (experiments_id, source, reference) values (%d, '%s', '%s');""" % (exp_id, s,r)) c.execute( """insert into sentences (experiments_id, source, reference) values (%d, '%s', '%s');""" % (exp_id, s, r)) i += 1 con.commit() log("sentences inserted") for t in tasks: import_task(t, exp_id, ref_path)
def search_foods(query): """ Palauttaa hakuehtoja vastaavat elintarvikkeet + id:t. Paluuarvona lista, jossa arvot tyyppiä [id, nimi]. """ query = re.sub("\s+", " ", query).strip() stored = db.get_search(query) if stored: return stored url = URL_ROOT + "/foodsearch.php?name=" + escape(query) print "url=", url root = html.parse(url) foods = [] rows = root.xpath("/html/body/div/table[2]/tr[1]/td[2]/ul/li/a") for row in rows: try: fid = row.attrib["href"].split("=")[1].split("&")[0] except IndexError: logging.error("Scraping food ID failed: " + query) continue foods.append((fid, row.text.strip())) db.add_search({"query": query, "results": foods}) return foods
def main(args): global verbose verbose = args.verbose if verbose: logger.info('Read ' + args.col) mwes = {} counts = 0 for filename in args.path_input: if verbose: logger.info('Read ' + filename) with open(filename) as f: for n_lines, line in enumerate(f, start=1): dat = json.loads(line) dat[args.col] = escape(dat[args.col]) if dat[args.col] in mwes: mwes[dat[args.col]]['freq'].append( (dat['source'], dat['freq'])) continue mwes[dat[args.col]] = dat freq = (mwes[dat[args.col]]['source'], mwes[dat[args.col]]['freq']) mwes[dat[args.col]]['freq'] = [freq] del mwes[dat[args.col]]['source'] counts += n_lines if verbose: logger.info('Read {} entries'.format(counts)) logger.info('Write {} entries to {}'.format(len(mwes), args.path_output)) write_json2xml(args.path_output, mwes) return 0
def to_text(self): text = super(Predicate, self).to_text() if self.neg: text = 'not//' + text if self.prt != '': text += '//' + escape(self.prt) return text
def _parse_urls(self, match): '''Parse URLs.''' mat = match.group(0) # Fix a bug in the regex concerning www...com and www.-foo.com domains # TODO fix this in the regex instead of working around it here domain = match.group(5) if domain[0] in '.-': return mat # Only allow IANA one letter domains that are actually registered if len(domain) == 5 \ and domain[-4:].lower() in ('.com', '.org', '.net') \ and not domain.lower() in IANA_ONE_LETTER_DOMAINS: return mat # Check for urls without http(s) pos = mat.find('http') if pos != -1: pre, url = mat[:pos], mat[pos:] full_url = url # Find the www and force http:// else: pos = mat.lower().find('www') pre, url = mat[:pos], mat[pos:] full_url = 'http://%s' % url self._urls.append(url) if self._html: return '%s%s' % (pre, self.format_url(full_url, self._shorten_url(escape(url))))
def json_data(self): return dict( id = self.id, content = escape(self.content), date_created = self.date_created, user = self.user.json_data() )
def _execute_cmd(self, *args, **kwargs): args = SafeList(args) func_name = kwargs["_name"] properties = CMD_DICT[func_name] default_value = deepcopy(properties.get("default", [])) try: arguments = [] for arg in properties["args"]: kwarg = kwargs.pop(arg, None) if kwarg is None: kwarg = args.pop(0) if kwarg is None: kwarg = default_value.pop(0) arguments.append(kwarg) except IndexError: sub = "" args = properties.get("args") default_args = SafeList(deepcopy(properties.get("default", []))) error_msg = "%%s haven't got enough arguments, need %s argument%s named %%s. " % ( len(args), "" if len(args) == 1 else "s") for arg in reversed(args): sub = "%s%s" % (arg, (": default %s, " % default_args.pop(-1) if default_args else ", ")) + sub raise RedisArgumentError(error_msg % (func_name, sub[:-2])) if args: arguments += args return self._parse_result( FORMAT % (func_name, "%s<->%s" % escape(properties.get("send", default_send)(*arguments))), properties)
def get_content(self): """Returns the content of the message""" text = self.parsed_obj.get_payload() output = '' if isinstance(text, types.ListType) and self.parsed_obj.is_multipart(): for i in text: if i.get_filename(): logging.error('Attachment found!!!') output += "\nSorry but the attachment has been stripped from this message!" continue if (i.get_content_type() == "text/plain" or i.get_content_type() == "text/html"): output+=(i.get_payload()) + '\n\n' #logging.debug("Appending %s to output" % i.get_payload()) else: output = text #logging.error("get_content(): Output before wrap is %s" % output) output = escape(output) output = nl2br(output) output1 = textwrap.wrap(output, 65) output = '' for i in output1: output += i + '<br />' #logging.error("get_content returning %s" % output) return output
def gen_compare_regex(comment, assert_flags, expected_flag): regex = "" for flag in expected_flag: excepted_regex = r"(?:{comment}\s*)({flag}[\t ]*\:[\t ]*.*$)".format( comment=escape("\\$()*+.[]?^{}|", comment), flag=flag) if regex != "": regex = "{}|{}".format(regex, excepted_regex) else: regex = excepted_regex for flag in assert_flags: assert_regex = r"(?:^[\t ]*{comment}\s*)({flag}[\t ]*\:[\t ]*.*$)".format( comment=escape("\\$()*+.[]?^{}|", comment), flag=flag) if regex != "": regex = "{}|{}".format(regex, assert_regex) else: regex = assert_regex return regex
def parse_text(text): text = escape(text) print("Creating request") body = rqt_parse_text.format(text=text).encode("utf-8") print("Request sent to server") response = requests.post(wsdl_file, data=body, headers=headers) print("Response received") return get_result(response)
def get_query_obj(qid): query = queries[qid] query_terms = [ term for term in pyndri.tokenize(escape(query.lower())) if term in token2id ] ids = [str(token2id[term]) for term in query_terms] query_obj = " ".join(ids) return query_obj
def to_tsv(self): ''' Returns the instance's column values as a tab-separated line. A newline is not included. ''' parts = [] for name, field in self._fields: value = field.get_db_prep_value(field.to_python(getattr(self, name))) parts.append(escape(value, quote=False)) return '\t'.join(parts)
def to_header(self, realm=""): """Serialize as a header for an HTTPAuth request.""" auth_header = 'OAuth realm="%s"' % realm # Add the oauth parameters. if self.parameters: for k, v in self.parameters.iteritems(): if k[:6] == "oauth_": auth_header += ', %s="%s"' % (k, escape(str(v))) return {"Authorization": auth_header}
def out(self): res = self.tree.to_string() res = ut.escape(res, ut.TEX_ESCAPE) res = " ".join(res.split()) res = res.replace(" .", ".") res = res.replace(" ,", ",") res = res.replace(" ;", ";") res = res.replace(" :", ":") res = unicode("{{{0}}}", "utf8").format(res) return res
def get_body(self, environ): """Get the HTML body.""" return ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n' '<title>%(code)s %(name)s</title>\n' '<h1>%(name)s</h1>\n' '%(description)s\n') % { 'code': self.code, 'name': escape(self.name), 'description': self.get_description(environ) }
def render_testapp(req): try: import pkg_resources except ImportError: eggs = () else: eggs = list(pkg_resources.working_set) eggs.sort(lambda a, b: cmp(a.project_name.lower(), b.project_name.lower())) python_eggs = [] for egg in eggs: try: version = egg.version except (ValueError, AttributeError): version = 'unknown' python_eggs.append('<li>%s <small>[%s]</small>' % ( escape(egg.project_name), escape(version) )) wsgi_env = [] sorted_environ = req.environ.items() sorted_environ.sort(key=lambda x: repr(x[0]).lower()) for key, value in sorted_environ: wsgi_env.append('<tr><th>%s<td><code>%s</code>' % ( escape(str(key)), ' '.join(wrap(escape(repr(value)))) )) sys_path = [] for item, virtual, expanded in iter_sys_path(): class_ = [] if virtual: class_.append('virtual') if expanded: class_.append('exp') sys_path.append('<li%s>%s' % ( class_ and ' class="%s"' % ' '.join(class_) or '', escape(item) )) return TEMPLATE % { 'python_version': '<br>'.join(escape(sys.version).splitlines()), 'platform': escape(sys.platform), 'os': escape(os.name), 'api_version': sys.api_version, 'byteorder': sys.byteorder, 'werkzeug_version': werkzeug.__version__, 'python_eggs': '\n'.join(python_eggs), 'wsgi_env': '\n'.join(wsgi_env), 'sys_path': '\n'.join(sys_path) }
def post_new_comment(): postid = request.forms.get('postid') name = request.forms.get('name').decode('utf-8') email = request.forms.get('email').decode('utf-8') url = request.forms.get('url').decode('utf-8') content = request.forms.get('content').decode('utf-8') parent = request.forms.get('parentid') check = request.forms.get('check') if check != "123456": post = Post.get(id=postid) redirect("/%s" % post.slug) post = Post.get(id=postid) post.comm_count = post.comm_count + 1 comment = Comment.create(post=post, author=escape(name), email=escape(email), url=escape(url), content=escape(content), parent=parent, published=datetime.datetime.now()) post.save() redirect("/%s#comment-%s" % (post.slug, comment.id))
def writeFavourites(file, faves): kodiFile = os.path.join('special://profile', utils.FILENAME) isKodi = xbmc.translatePath(file) == xbmc.translatePath(kodiFile) f = sfile.file(file, 'w') f.write('<favourites>') for fave in faves: try: name = utils.escape(fave[0]) thumb = utils.escape(fave[1]) cmd = utils.escape(fave[2]) cmd = removeHome(cmd) if isKodi and cmd.lower().startswith('playmedia'): cmd = removeSFOptions(cmd) thumb = utils.convertToHome(thumb) name = 'name="%s" ' % name thumb = 'thumb="%s">' % thumb f.write('\n\t<favourite ') f.write(name) f.write(thumb) f.write(cmd) f.write('</favourite>') except: pass f.write('\n</favourites>') f.close() import xbmcgui try: count = int( xbmcgui.Window(10000).getProperty('Super_Favourites_Count')) except: count = 0 xbmcgui.Window(10000).setProperty('Super_Favourites_Count', str(count + 1))
def get_body(self): return ( u'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n' u'<html>\n<title>%(code)s %(name)s</title>\n<body>\n' u'<h1>%(name)s</h1>\n' u'%(description)s\n' u'</body></html>' ) % { 'code': self.code, 'name': escape(self.name), 'description': self.get_content() }
def get_body(self, environ): """Get the HTML body.""" return ( '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n' '<title>%(code)s %(name)s</title>\n' '<h1>%(name)s</h1>\n' '%(description)s\n' ) % { 'code': self.code, 'name': escape(self.name), 'description': self.get_description(environ) }
def get_cmd_line(self): if not self.cmd_line: cmd_line = 'virsh ' + self.command.name for option in self.options: if option.line is None: continue else: result_line = '--' + option.option.name if option.line: result_line += ' %s' % utils.escape(option.line) cmd_line += ' %s' % result_line self.cmd_line = cmd_line return self.cmd_line
def writeFavourites(file, faves): kodiFile = os.path.join("special://profile", utils.FILENAME) isKodi = xbmc.translatePath(file) == xbmc.translatePath(kodiFile) f = sfile.file(file, "w") f.write("<favourites>") for fave in faves: try: name = utils.escape(fave[0]) thumb = utils.escape(fave[1]) cmd = utils.escape(fave[2]) if isKodi and cmd.lower().startswith("playmedia"): cmd = removeSFOptions(cmd) thumb = utils.convertToHome(thumb) name = 'name="%s" ' % name thumb = 'thumb="%s">' % thumb f.write("\n\t<favourite ") f.write(name) f.write(thumb) f.write(cmd) f.write("</favourite>") except: pass f.write("\n</favourites>") f.close() import xbmcgui try: count = int(xbmcgui.Window(10000).getProperty("Super_Favourites_Count")) except: count = 0 xbmcgui.Window(10000).setProperty("Super_Favourites_Count", str(count + 1))
def create_table_sql(cls, db_name): ''' Returns the SQL command for creating a table for this model. ''' parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db_name, cls.table_name())] cols = [] for name, field in cls._fields: default = field.get_db_prep_value(field.default) cols.append(' %s %s DEFAULT %s' % (name, field.db_type, escape(default))) parts.append(',\n'.join(cols)) parts.append(')') parts.append('ENGINE = ' + cls.engine.create_table_sql()) return '\n'.join(parts)
def visit_Output(self, node, frame): body = [] for child in node.nodes: try: const = child.as_const(frame.eval_ctx) except nodes.Impossible: body.append(child) continue try: if frame.eval_ctx.autoescape: if hasattr(const, '__html__'): const = const.__html__() else: const = escape(const) const = str(const) except Exception: body.append(child) continue if body and isinstance(body[-1], list): body[-1].append(const) else: body.append([const]) # write a format string for the body format = [] arguments = [] for item in body: if isinstance(item, list): format.append(concat(item).replace('%', '%%')) else: format.append('%s') # TOFIX: item is a tuple/list/iterable arguments.append(item) self.writeline('yield ') self.write(repr(concat(format))) if arguments: self.write(' % (') self.indent() for argument in arguments: self.newline(argument) close = 0 if frame.eval_ctx.autoescape: self.write('escape(') close += 1 self.visit(argument, frame) self.write(')' * close) self.outdent() if len(arguments) == 1: # trailing comma for tuple with len 1 self.write(',') self.writeline(')')
def render_testapp(req): try: import pkg_resources except ImportError: eggs = () else: eggs = list(pkg_resources.working_set) eggs.sort( lambda a, b: cmp(a.project_name.lower(), b.project_name.lower())) python_eggs = [] for egg in eggs: try: version = egg.version except (ValueError, AttributeError): version = 'unknown' python_eggs.append('<li>%s <small>[%s]</small>' % (escape(egg.project_name), escape(version))) wsgi_env = [] sorted_environ = req.environ.items() sorted_environ.sort(key=lambda x: repr(x[0]).lower()) for key, value in sorted_environ: wsgi_env.append( '<tr><th>%s<td><code>%s</code>' % (escape(str(key)), ' '.join(wrap(escape(repr(value)))))) sys_path = [] for item, virtual, expanded in iter_sys_path(): class_ = [] if virtual: class_.append('virtual') if expanded: class_.append('exp') sys_path.append( '<li%s>%s' % (class_ and ' class="%s"' % ' '.join(class_) or '', escape(item))) return TEMPLATE % { 'python_version': '<br>'.join(escape(sys.version).splitlines()), 'platform': escape(sys.platform), 'os': escape(os.name), 'api_version': sys.api_version, 'byteorder': sys.byteorder, 'werkzeug_version': werkzeug.__version__, 'python_eggs': '\n'.join(python_eggs), 'wsgi_env': '\n'.join(wsgi_env), 'sys_path': '\n'.join(sys_path) }
def generate(self): """Yields pieces of XML.""" yield u'<url>\n' yield u'<loc>%s</loc>\n' % escape(self.loc) if self.lastmod: if hasattr(self.lastmod, 'strftime'): yield u'<lastmod>%s</lastmod>\n' % self.lastmod.strftime('%Y-%m-%d') elif isinstance(self.lastmod, str): yield u'<lastmod>%s</lastmod>\n' % self.lastmod if self.changefreq and self.changefreq in self.freq_values: yield u'<changefreq>%s</changefreq>\n' % self.changefreq if self.priority and 0.0 <= self.priority <= 1.0: yield u'<priority>%s</priority>\n' % self.priority yield u'</url>\n'
def json_data(self): return { "content": escape(self.content), "title": self.title, "id": self.id, "date_created": self.date_created, "is_active": self.is_active, "user": self.user.json_data(), "tags": self.get_tags(), "points": self.points, "comment_count": self.comment_count, "view_count": self.view_count, "urls": self.get_url() }
def parse_text(text): try: text = escape(text) print("Creating request") body = rqt_parse_text.format(text=text).encode("utf-8") print("Request sent to server") response = requests.post(wsdl_file, data=body, headers=headers) print("Response received") return get_result(response) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print("Error at calling fdg_parser. Reason:", str(e), "File:", fname, "Line:", exc_tb.tb_lineno)
def on_tooltip(self, icon, pos_x, pos_y, key, tip, *args): # Prevent broken tooltips while the context menu is opened if self.popup_open: self.fake_move((0.0, self.mouse_position[1]), True) return True link = self.last_hovered_link if link is not None: if link.startswith('avatar:'): uri = link[7:] num = int(uri[:uri.find(':')]) user = self.get_user(num) img = self.get_image(num) # Set only if something has changed if user != self.tooltip_user \ or img != self.tooltip_img_file: self.set_tooltip(user, img) self.tooltip_user = user tip.set_custom(self.tooltip) return True # Try to expand links elif link.startswith('http://'): if link in self.expanded_links: if self.expanded_links[link] != link: tip.set_markup(lang.html_expanded_tooltip % (escape(link), escape(self.expanded_links[link]))) return True elif not self.is_expanding: self.is_expanding = True URLExpander(link, self.expand_link)
def set_tooltip(self, user, img): self.tooltip_img.show() self.tooltip_label.set_markup(lang.html_avatar_tooltip \ % (escape(user.name), user.statuses_count, user.followers_count, user.friends_count)) # Watch out for malformatted gifs, I'm looking at you @defunkt! if img != self.tooltip_img_file: try: buf = gtk.gdk.pixbuf_new_from_file_at_size(img, 48, 48) self.tooltip_img.set_from_pixbuf(buf) self.tooltip_img_file = img except GLIBError: self.tooltip_img.hide()
def generate(self): """Yield next XML fragment as a string.""" yield u'<url>\n' yield u'<loc>%s</loc>\n' % escape(self.loc) if self.lastmod: if hasattr(self.lastmod, 'strftime'): yield u'<lastmod>%s</lastmod>\n' % \ self.lastmod.strftime('%Y-%m-%d') elif isinstance(self.lastmod, str): yield u'<lastmod>%s</lastmod>\n' % self.lastmod if self.changefreq and self.changefreq in self.freq_values: yield u'<changefreq>%s</changefreq>\n' % self.changefreq if self.priority and 0.0 <= self.priority <= 1.0: yield u'<priority>%s</priority>\n' % self.priority yield u'</url>\n'
def json_data(self): return { "content": escape(self.content), "title": self.title, "id": self.id, "date_created": self.date_created, "user": self.user.json_data(), "tags": [tag.json_data() for tag in self.tags], "points": self.points, "comment_count": self.comment_count, "user_vote": self.user_vote, "user_comment": self.user_comment, "user_answer": self.user_answer, "answer_count": self.answer_count, "urls": self.get_url() }
def process_code_block(self, fd, lines): line = lines[0]; if (not re.match(" *(.*)", line)): return False; fd.write("<pre><code>\n"); while (len(lines) > 0): line = lines[0]; m = re.match(" *(.*)", line); if (not m): break; # we need to escape as code might contain '<' and other stuff fd.write(utils.escape(m.group(1)) + "\n"); lines.popleft(); fd.write("</code></pre>\n"); return True;
def render(self): #resp = '<div class="control-group">' #resp += '<label class="control-label" for="' + self.variable_name + '">' + self.secondary_content + '</label>' #resp += '<div class="controls">' resp = '<div class="row">' resp += '<div class="span1" style="text-align:right;padding-right:10px">' + self.secondary_content + '</div>' resp += '<input type="text" style="input-big" name="' + self.variable_name + '"' if self.answer: resp += 'value="' + escape(str(self.answer)) + '"' resp += '>' resp += '</div>' # resp = '<div class="span1" style="text-align:right">' + self.secondary_content + '</div>' # resp += '<input type="text" style="input-medium" name="' + self.variable_name + '" value="' # if self.answer: # resp += escape(str(self.answer)) # resp += '">' return resp
def gen_course_data(root='out'): """将数据库里的数据导出到本地(导出清晰度最高的数据) :param root: 输出文件的根目录 :return: """ sql_session = SqliteSession().session # 获取所有课程 course_objs = sql_session.query(Course).all() orgin_dir = [ root, ] for course_obj in course_objs: # 获取课程文件路径 course_dir = orgin_dir[:] course_dir.append('{}-{}'.format(course_obj.course_id, course_obj.course_name)) course_dir.append('第{}章-{}'.format(course_obj.chapter_id, course_obj.chapter_name)) course_dir.append('第{}节-{}.m3u8'.format(course_obj.section_id, course_obj.section_name)) course_dir = map(lambda s: escape(s).strip(), course_dir) course_m3u8_path = '/'.join(course_dir) course_dir = os.path.dirname(course_m3u8_path) if not os.path.exists(course_m3u8_path): try: os.makedirs(course_dir, exist_ok=True) except Exception as e: logger.error('{}-{}'.format(course_dir, e)) # 写入数据 section_md5 = course_obj.section_md5 course_detail = sql_session.query( CourseDetail.m3u8, CourseDetail.aes_key).filter( CourseDetail.section_md5 == section_md5).order_by( CourseDetail.width.desc()).first() m3u8_text = course_detail.m3u8 # TRIM_FULL_KEY_URI.sub(r'\1\2',course_detail.m3u8) with open(course_m3u8_path, 'w') as f: f.write(m3u8_text) path_key = json.loads(course_detail.aes_key) key_path, key = path_key.popitem() course_key_path = os.path.join(course_dir, key_path) os.makedirs(os.path.dirname(course_key_path), exist_ok=True) with open(course_key_path, 'w') as f: f.write(key)
def output_text(self, fd, text): # XXX: is this valid ? text = utils.escape(text); # start with a dummy char that is not the escape char result = "X"; start = 0; while(True): m = re.search(self.link_re, text[start:]); if (not m): result += text[start:]; break; result += text[start:start + m.start()]; url = self.image_cb(m.group(2)); result += "<a href=\"" + url + "\">" + m.group(1) + "</a>" start += m.end(); result = re.sub(r"[^\\]\*([^\* ]*[^\\])\*", r"<b>\1</b>", result); result = re.sub(r"[^\\]_([^_ ]*[^\\])_", r"<em>\1</em>", result); # unescape result = re.sub(r"\\(.)", r"\1", result); fd.write(result[1:]);
def text(escape=False, min_len=5, max_len=10): """ Generate a randomized string. """ excludes = "\n\t\r\x0b\x0c" chars = [] for char in string.printable: if char not in excludes: chars.append(char) length = random.randint(min_len, max_len) result_str = ''.join(random.choice(chars) for _ in xrange(length)) if escape: return utils.escape(result_str) else: return result_str
def keys(self, pattern="*", *args): return self._parse_result( FORMAT % ("keys", "%s<->%s" % escape((pattern, ""))), {"recv": json.loads})
def describe(self, uneval=False, predrop=False): describe_first = self.first.describe(uneval=uneval, predrop=(self.is_dropkeepadd() or (predrop and self.is_collection())))\ if self.first != None else "" describe_second = self.second.describe(uneval=uneval)\ if self.second != None else "" if self._kind == "(": # parenthesis group return "(" + describe_first + ")" spacer = " " if self._spaces else "" detail_description = "" if not uneval: if (not self.is_collection()) and ( not self.contains_diceroll()): # No child contains diceroll randomness, so just show the # value; or other repr if valueless like an operator # argument. if self.get_value() == None: return escape(str(self)) return f"{self.get_value()}" # repeated expression special case # This causes details to be hidden from `agg`. if (self._kind == "repeat" or self._kind == "agg") and (not predrop): return ExprResult.description(self.detail) if self.detail: detail_description = " " + \ ExprResult.description(self.detail) if self.is_collection(): # Hide breakout detail description if only one item if len(self.detail.get_all_items() ) < 2 and self.detail.get_value() != None: detail_description = "⇒" + str(self.detail) # Hide details if outer collection already will show them if predrop: detail_description = "" if self._function_like: close_function = ")" if len(describe_second) > 0: close_function = f",{spacer}{describe_second})" return f"{self._kind}({describe_first}" + close_function left = describe_first right = describe_second if self.second == None: # No operands, just describe this node's value. if self.first == None: return str(self) if self._postfix: right = "" else: left = "" right = describe_first op = self._kind if not uneval: op = escape(self._kind) prewrap = f"{left}{spacer}{op}{spacer}{right}" +\ f"{detail_description}" if self._kind == "choose" or self._kind == "C": return "(" + prewrap + ")" elif self.is_collection() and not (predrop or uneval): return "[" + prewrap + "]" return prewrap
def get_description(self): # We could append = (total) for consistency with dice rolls. # Some tweaks to describe() would be needed, as a special case # for `agg` but not `d` causes differences in formatting paths. return "(" + super().get_description( joiner=escape(self.agg_joiner)) + ")"
twitter_status = '{0} {1}'.format(random_post_title, random_post_url) signature_timestamp = datetime.now().strftime('%s') signature_once = str(getrandbits(64)) signature_basestr_auth = {} signature_basestr_auth['status'] = twitter_status signature_basestr_auth['oauth_consumer_key'] = twitter_consumer_key signature_basestr_auth['oauth_nonce'] = signature_once signature_basestr_auth['oauth_signature_method'] = twitter_api_method signature_basestr_auth['oauth_timestamp'] = signature_timestamp signature_basestr_auth['oauth_token'] = twitter_oauth_token signature_basestr_auth['oauth_version'] = twitter_api_version signature_basestr_auth = [ '{0}={1}'.format(escape(k), escape(v)) for k, v in sorted(signature_basestr_auth.items()) ] signature_basestr_auth = '&'.join(signature_basestr_auth) signature_basestr = 'POST&{0}&{1}'.format(escape(twitter_api_end), escape(signature_basestr_auth)) signature_key = '{0}&{1}'.format(twitter_consumer_secret, escape(twitter_oauth_secret)) oauth_hmac_hash = hmac.new(signature_key.encode(), signature_basestr.encode(), hashlib.sha1) oauth_signature = base64.b64encode(oauth_hmac_hash.digest()).decode() oauth_header = {} oauth_header['oauth_consumer_key'] = twitter_consumer_key
def delete(self, key, *args): return self._parse_result(FORMAT % ("delete", "%s<->%s" % escape( (key, ""))))
def expire(self, key, seconds, *args): return self._parse_result(FORMAT % ("expire", "%s<->%s" % escape( (key, seconds))))
def ttl(self, key, *args): return self._parse_result(FORMAT % ("ttl", "%s<->%s" % escape( (key, ""))))
def get_description(self, environ): m = escape(environ.get('REQUEST_METHOD', 'GET')) return '<p>The method %s is not allowed for the requested URL.</p>' % m
def match_single_entity(self, text: str, entity: str, ctx: Context) -> bool: """ 匹配单个entity,如果单tag有规则能拆成多个entity,则转为匹配多个entity """ # print(text, entity, self.known.get_sub(entity)) # 处理entity,将其中的标记标签去除 tag = re.match(r"^([^\[\]]+?)\|([^\[\]]+)$", entity) if tag is not None: entity, tag = tag.group(1), tag.group(2) # 确定匹配到的是否符合上下文要求 ctx_entity = ctx.find(entity, tag) # 此处匹配不会改变Context,所以可以直接返回 if ctx_entity is not None: return (text == ctx_entity) # 编译正则表达式,供后取使用 match_regex = re.compile(r"\[(.+?)\](?=[^\[]|$)") # 遍历需要匹配的entity下所有的子实体 for sub_entity in self.known.get_sub(entity): # 获取子实体 match_text = sub_entity.get_text() # 符号转移,避免文本中原有的符号被当成正则表达式处理 match_escape = utils.escape(match_text) # 构造entity的正则表达式,将需要进行实体判断的部分用通配符取代 match_re = "^%s$" % match_regex.sub(r"(.+?)", match_escape) match = re.match(match_re, text) if match: match_succ = True ctx.new_stack() # 判断实体匹配中是否存在子实体需要匹配 match_entity = match_regex.findall(match_text) # 弱存在子实体,则下述循环会逐一判断每个子实体 for sub_text, sub_entity in zip(match.groups(), match_entity): # 判断是否存在相邻子实体情况 sub_entities = sub_entity.split("][") if len(sub_entities) == 1: # 不存在,可以直接进行单个匹配 match_succ = self.match_single_entity( sub_text, sub_entities[0], ctx) else: # 存在相邻子实体,需要进行分词匹配 match_succ = self.match_multi_entities( sub_text, sub_entities, ctx) # 若无法匹配成功,则不属于此实体 if not match_succ: break if match_succ: # 成功匹配,记录上下文 ctx.append(Instance(entity, text, None, tag=tag)) ctx.apply_stack() return True else: # 匹配失败,抛弃上下文 ctx.aborat_stack() print("aborat") return False
print("Stemming with {a} stemmer without stop words removel".format( a=algo)) print("Please wait while we are stemming the text ... \n") # Processing of the input text before neighbors finding prog = re.compile("[_\-\(]*([A-Z]\.)*[_\-\(]*") for t in toProcess: terms = toProcess[t].split() #stem(algo,toProcess[t]).split() text = "" for w in terms: if (prog.match(w)): w = w.replace('.', '') text = text + " " + w text = ' '.join(escape(text).split()) text = " ".join(nltk.word_tokenize(text)) d = [] text = " ".join([stem(algo, w) for w in text.split()]) d += text.split() toProcess[t] = d #print(toProcess) if bool(args["--stop"]): stopWordsList = set(stopwords.words('english')) print(stopWordsList) for i in toProcess: listWords = [w for w in toProcess[i] if w not in stopWordsList] toProcess[i] = listWords #print(toProcess)