def po2csv(po):
"""Converts a polib.POFile object to a csv string (contained in an instance
of StringIO)
"""
csv_file = StringIO()
csv_writer = csv.writer(csv_file)
for header_row in csv_header_rows:
csv_writer.writerow(header_row)
for entry in po:
msgstrs = force_plurals(entry)
fuzzy = bool2str('fuzzy' in entry.flags, empty_for_false=True)
obsolete = bool2str(entry.obsolete, empty_for_false=True)
row = [
escape(entry.msgid),
escape(entry.msgid_plural),
entry.msgctxt,
msgstrs,
fuzzy,
entry.comment,
obsolete
]
csv_writer.writerow(row)
return csv_file
def force_plurals(entry):
"""A po file's plural handling is slightly weird. If there are no plurals
for a translation:
msgid "hello"
msgstr "hola"
If there are plurals however:
msgid "%(count)s house"
msgid_plural "%(count)s houses"
msgstr[0] "%(count)s casa"
msgstr[1] "%(count)s casas"
So to keep things consistent, we force non-plural translations into the
plural form for consistency, and then reading them back, we still have
enough information to convert them back into singular translations if
that's what they were meant to be.
"""
plurals = entry.msgstr_plural
if not plurals:
plurals = {u'0': escape(entry.msgstr)}
sorted_plurals = sorted(plurals.items(), key=lambda t: t[0])
return PLURAL_SEPARATOR.join(escape(v) for k, v in sorted_plurals)
def writeFavourites(file, faves):
f = sfile.file(file, 'w')
f.write('<favourites>')
for fave in faves:
try:
name = utils.escape(fave[0])
thumb = utils.escape(fave[1])
cmd = utils.escape(fave[2])
thumb = convertToHome(thumb)
name = 'name="%s" ' % name
thumb = 'thumb="%s">' % thumb
f.write('\n\t<favourite ')
f.write(name)
f.write(thumb)
f.write(cmd)
f.write('</favourite>')
except:
pass
f.write('\n</favourites>')
f.close()
import xbmcgui
try: count = int(xbmcgui.Window(10000).getProperty('Super_Favourites_Count'))
except: count = 0
xbmcgui.Window(10000).setProperty('Super_Favourites_Count', str(count+1))
def harvest(text, harvesters=DEFAULT_HARVESTERS):
instances = [load_class(namespace) for namespace in harvesters]
display_text = ''
display_html = ''
entities = []
current_text = text
for instance in instances:
e = instance.harvest(current_text)
current_position = 0
for entity in e:
entities.append(entity)
current_position = entity.start_index
l = len(entity.original_text)
replacement = repeat_to_length(' ', l)
current_text = current_text[:current_position] + \
replacement + current_text[current_position + l:]
current_index = 0
for entity in entities:
display_html = display_html + escape(text[current_index:entity.start_index]) + entity.display_html
display_text = display_text + escape(text[current_index:entity.start_index]) + entity.display_text
current_index = entity.end_index
display_text = display_text + escape(text[current_index:])
display_html = display_html + escape(text[current_index:])
return {
'display_text': display_text,
'display_html': display_html,
}
def set_label_text(self, info, label_text, label_extra=None):
label_text = label_text.replace('\n', '').replace('\r', '')
if label_extra:
self.info_label.set_markup(info % (escape(label_text),
escape(label_extra)))
else:
self.info_label.set_markup(info % escape(label_text))
def build_html_params(**params):
params_list = list()
for (name, value) in params.items():
if name[0] == u'_':
name = name[1:]
params_list.append(u' {0}="{1}"'.format(
escape(name),
escape(value),
))
return u''.join(params_list)
def build_signature_base_string(self, oauth_request, consumer, token):
sig = (
escape(oauth_request.get_normalized_http_method()),
escape(oauth_request.get_normalized_http_url()),
escape(oauth_request.get_normalized_parameters()),
)
key = '%s&' % escape(consumer.secret)
if token:
key += escape(token.secret)
raw = '&'.join(sig)
return key, raw
def get_normalized_parameters(self):
"""Return a string that contains the parameters that must be signed."""
params = self.parameters
try:
# Exclude the signature if it exists.
del params["oauth_signature"]
except:
pass
# Escape key values before sorting.
key_values = [(escape(_utf8_str(k)), escape(_utf8_str(v))) for k, v in params.items()]
# Sort lexicographically, first after key, then after value.
key_values.sort()
# Combine key value pairs into a string.
return "&".join(["%s=%s" % (k, v) for k, v in key_values])
def import_experiment(exp_dirname):
tasks = check_experiment(exp_dirname)
print(exp_dirname)
if exp_dirname.endswith("/"):
exp_dirname = exp_dirname[:-1]
exp_name = os.path.basename(exp_dirname)
print(exp_name)
i = 0
# while
n = exp_name
while True:
c = con.cursor()
s = c.execute("select id from experiments where name = '%s'" % (n))
if not s.fetchall():
break
i += 1
n = exp_name + ("-%s" % i)
exp_name = n
c = con.cursor()
print(
"insert into experiments (name, url_key, description, visible) values ('%s', '%s', '', 1);"
% (exp_name, exp_name))
c.execute(
"insert into experiments (name, url_key, description, visible) values ('%s', '%s', '', 1);"
% (exp_name, exp_name))
con.commit()
s = c.execute("select id from experiments where name = '%s'" % (exp_name))
exp_id, *_ = s.fetchone()
source = open(os.path.join(exp_dirname, "source.txt"), "r")
ref_path = os.path.join(exp_dirname, "reference.txt")
reference = open(ref_path, "r")
c = con.cursor()
i = 1
for s, r in zip(source, reference):
s = escape(s)
r = escape(r)
# print(i,"""insert into sentences (experiments_id, source, reference) values (%d, '%s', '%s');""" % (exp_id, s,r))
c.execute(
"""insert into sentences (experiments_id, source, reference) values (%d, '%s', '%s');"""
% (exp_id, s, r))
i += 1
con.commit()
log("sentences inserted")
for t in tasks:
import_task(t, exp_id, ref_path)
def search_foods(query):
"""
Palauttaa hakuehtoja vastaavat elintarvikkeet + id:t.
Paluuarvona lista, jossa arvot tyyppiä [id, nimi].
"""
query = re.sub("\s+", " ", query).strip()
stored = db.get_search(query)
if stored:
return stored
url = URL_ROOT + "/foodsearch.php?name=" + escape(query)
print "url=", url
root = html.parse(url)
foods = []
rows = root.xpath("/html/body/div/table[2]/tr[1]/td[2]/ul/li/a")
for row in rows:
try:
fid = row.attrib["href"].split("=")[1].split("&")[0]
except IndexError:
logging.error("Scraping food ID failed: " + query)
continue
foods.append((fid, row.text.strip()))
db.add_search({"query": query, "results": foods})
return foods
def main(args):
global verbose
verbose = args.verbose
if verbose:
logger.info('Read ' + args.col)
mwes = {}
counts = 0
for filename in args.path_input:
if verbose:
logger.info('Read ' + filename)
with open(filename) as f:
for n_lines, line in enumerate(f, start=1):
dat = json.loads(line)
dat[args.col] = escape(dat[args.col])
if dat[args.col] in mwes:
mwes[dat[args.col]]['freq'].append(
(dat['source'], dat['freq']))
continue
mwes[dat[args.col]] = dat
freq = (mwes[dat[args.col]]['source'],
mwes[dat[args.col]]['freq'])
mwes[dat[args.col]]['freq'] = [freq]
del mwes[dat[args.col]]['source']
counts += n_lines
if verbose:
logger.info('Read {} entries'.format(counts))
logger.info('Write {} entries to {}'.format(len(mwes),
args.path_output))
write_json2xml(args.path_output, mwes)
return 0
def to_text(self):
text = super(Predicate, self).to_text()
if self.neg:
text = 'not//' + text
if self.prt != '':
text += '//' + escape(self.prt)
return text
def _parse_urls(self, match):
'''Parse URLs.'''
mat = match.group(0)
# Fix a bug in the regex concerning www...com and www.-foo.com domains
# TODO fix this in the regex instead of working around it here
domain = match.group(5)
if domain[0] in '.-':
return mat
# Only allow IANA one letter domains that are actually registered
if len(domain) == 5 \
and domain[-4:].lower() in ('.com', '.org', '.net') \
and not domain.lower() in IANA_ONE_LETTER_DOMAINS:
return mat
# Check for urls without http(s)
pos = mat.find('http')
if pos != -1:
pre, url = mat[:pos], mat[pos:]
full_url = url
# Find the www and force http://
else:
pos = mat.lower().find('www')
pre, url = mat[:pos], mat[pos:]
full_url = 'http://%s' % url
self._urls.append(url)
if self._html:
return '%s%s' % (pre, self.format_url(full_url,
self._shorten_url(escape(url))))
def json_data(self):
return dict(
id = self.id,
content = escape(self.content),
date_created = self.date_created,
user = self.user.json_data()
)
def _execute_cmd(self, *args, **kwargs):
args = SafeList(args)
func_name = kwargs["_name"]
properties = CMD_DICT[func_name]
default_value = deepcopy(properties.get("default", []))
try:
arguments = []
for arg in properties["args"]:
kwarg = kwargs.pop(arg, None)
if kwarg is None:
kwarg = args.pop(0)
if kwarg is None:
kwarg = default_value.pop(0)
arguments.append(kwarg)
except IndexError:
sub = ""
args = properties.get("args")
default_args = SafeList(deepcopy(properties.get("default", [])))
error_msg = "%%s haven't got enough arguments, need %s argument%s named %%s. " % (
len(args), "" if len(args) == 1 else "s")
for arg in reversed(args):
sub = "%s%s" % (arg, (": default %s, " % default_args.pop(-1)
if default_args else ", ")) + sub
raise RedisArgumentError(error_msg % (func_name, sub[:-2]))
if args:
arguments += args
return self._parse_result(
FORMAT %
(func_name, "%s<->%s" %
escape(properties.get("send", default_send)(*arguments))),
properties)
def get_content(self):
"""Returns the content of the message"""
text = self.parsed_obj.get_payload()
output = ''
if isinstance(text, types.ListType) and self.parsed_obj.is_multipart():
for i in text:
if i.get_filename():
logging.error('Attachment found!!!')
output += "\nSorry but the attachment has been stripped from this message!"
continue
if (i.get_content_type() == "text/plain" or i.get_content_type() == "text/html"):
output+=(i.get_payload()) + '\n\n'
#logging.debug("Appending %s to output" % i.get_payload())
else:
output = text
#logging.error("get_content(): Output before wrap is %s" % output)
output = escape(output)
output = nl2br(output)
output1 = textwrap.wrap(output, 65)
output = ''
for i in output1:
output += i + '<br />'
#logging.error("get_content returning %s" % output)
return output
def gen_compare_regex(comment, assert_flags, expected_flag):
regex = ""
for flag in expected_flag:
excepted_regex = r"(?:{comment}\s*)({flag}[\t ]*\:[\t ]*.*$)".format(
comment=escape("\\$()*+.[]?^{}|", comment), flag=flag)
if regex != "":
regex = "{}|{}".format(regex, excepted_regex)
else:
regex = excepted_regex
for flag in assert_flags:
assert_regex = r"(?:^[\t ]*{comment}\s*)({flag}[\t ]*\:[\t ]*.*$)".format(
comment=escape("\\$()*+.[]?^{}|", comment), flag=flag)
if regex != "":
regex = "{}|{}".format(regex, assert_regex)
else:
regex = assert_regex
return regex
def parse_text(text):
text = escape(text)
print("Creating request")
body = rqt_parse_text.format(text=text).encode("utf-8")
print("Request sent to server")
response = requests.post(wsdl_file, data=body, headers=headers)
print("Response received")
return get_result(response)
def get_query_obj(qid):
query = queries[qid]
query_terms = [
term for term in pyndri.tokenize(escape(query.lower()))
if term in token2id
]
ids = [str(token2id[term]) for term in query_terms]
query_obj = " ".join(ids)
return query_obj
def to_tsv(self):
'''
Returns the instance's column values as a tab-separated line. A newline is not included.
'''
parts = []
for name, field in self._fields:
value = field.get_db_prep_value(field.to_python(getattr(self, name)))
parts.append(escape(value, quote=False))
return '\t'.join(parts)
def to_header(self, realm=""):
"""Serialize as a header for an HTTPAuth request."""
auth_header = 'OAuth realm="%s"' % realm
# Add the oauth parameters.
if self.parameters:
for k, v in self.parameters.iteritems():
if k[:6] == "oauth_":
auth_header += ', %s="%s"' % (k, escape(str(v)))
return {"Authorization": auth_header}
def out(self):
res = self.tree.to_string()
res = ut.escape(res, ut.TEX_ESCAPE)
res = " ".join(res.split())
res = res.replace(" .", ".")
res = res.replace(" ,", ",")
res = res.replace(" ;", ";")
res = res.replace(" :", ":")
res = unicode("{{{0}}}", "utf8").format(res)
return res
def get_body(self, environ):
"""Get the HTML body."""
return ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n'
'<title>%(code)s %(name)s</title>\n'
'<h1>%(name)s</h1>\n'
'%(description)s\n') % {
'code': self.code,
'name': escape(self.name),
'description': self.get_description(environ)
}
def render_testapp(req):
try:
import pkg_resources
except ImportError:
eggs = ()
else:
eggs = list(pkg_resources.working_set)
eggs.sort(lambda a, b: cmp(a.project_name.lower(),
b.project_name.lower()))
python_eggs = []
for egg in eggs:
try:
version = egg.version
except (ValueError, AttributeError):
version = 'unknown'
python_eggs.append('<li>%s <small>[%s]</small>' % (
escape(egg.project_name),
escape(version)
))
wsgi_env = []
sorted_environ = req.environ.items()
sorted_environ.sort(key=lambda x: repr(x[0]).lower())
for key, value in sorted_environ:
wsgi_env.append('<tr><th>%s<td><code>%s</code>' % (
escape(str(key)),
' '.join(wrap(escape(repr(value))))
))
sys_path = []
for item, virtual, expanded in iter_sys_path():
class_ = []
if virtual:
class_.append('virtual')
if expanded:
class_.append('exp')
sys_path.append('<li%s>%s' % (
class_ and ' class="%s"' % ' '.join(class_) or '',
escape(item)
))
return TEMPLATE % {
'python_version': '<br>'.join(escape(sys.version).splitlines()),
'platform': escape(sys.platform),
'os': escape(os.name),
'api_version': sys.api_version,
'byteorder': sys.byteorder,
'werkzeug_version': werkzeug.__version__,
'python_eggs': '\n'.join(python_eggs),
'wsgi_env': '\n'.join(wsgi_env),
'sys_path': '\n'.join(sys_path)
}
def post_new_comment():
postid = request.forms.get('postid')
name = request.forms.get('name').decode('utf-8')
email = request.forms.get('email').decode('utf-8')
url = request.forms.get('url').decode('utf-8')
content = request.forms.get('content').decode('utf-8')
parent = request.forms.get('parentid')
check = request.forms.get('check')
if check != "123456":
post = Post.get(id=postid)
redirect("/%s" % post.slug)
post = Post.get(id=postid)
post.comm_count = post.comm_count + 1
comment = Comment.create(post=post, author=escape(name),
email=escape(email), url=escape(url),
content=escape(content), parent=parent,
published=datetime.datetime.now())
post.save()
redirect("/%s#comment-%s" % (post.slug, comment.id))
def writeFavourites(file, faves):
kodiFile = os.path.join('special://profile', utils.FILENAME)
isKodi = xbmc.translatePath(file) == xbmc.translatePath(kodiFile)
f = sfile.file(file, 'w')
f.write('<favourites>')
for fave in faves:
try:
name = utils.escape(fave[0])
thumb = utils.escape(fave[1])
cmd = utils.escape(fave[2])
cmd = removeHome(cmd)
if isKodi and cmd.lower().startswith('playmedia'):
cmd = removeSFOptions(cmd)
thumb = utils.convertToHome(thumb)
name = 'name="%s" ' % name
thumb = 'thumb="%s">' % thumb
f.write('\n\t<favourite ')
f.write(name)
f.write(thumb)
f.write(cmd)
f.write('</favourite>')
except:
pass
f.write('\n</favourites>')
f.close()
import xbmcgui
try:
count = int(
xbmcgui.Window(10000).getProperty('Super_Favourites_Count'))
except:
count = 0
xbmcgui.Window(10000).setProperty('Super_Favourites_Count', str(count + 1))
def get_body(self):
return (
u'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n'
u'<html>\n<title>%(code)s %(name)s</title>\n<body>\n'
u'<h1>%(name)s</h1>\n'
u'%(description)s\n'
u'</body></html>'
) % {
'code': self.code,
'name': escape(self.name),
'description': self.get_content()
}
def get_body(self, environ):
"""Get the HTML body."""
return (
'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n'
'<title>%(code)s %(name)s</title>\n'
'<h1>%(name)s</h1>\n'
'%(description)s\n'
) % {
'code': self.code,
'name': escape(self.name),
'description': self.get_description(environ)
}
def get_cmd_line(self):
if not self.cmd_line:
cmd_line = 'virsh ' + self.command.name
for option in self.options:
if option.line is None:
continue
else:
result_line = '--' + option.option.name
if option.line:
result_line += ' %s' % utils.escape(option.line)
cmd_line += ' %s' % result_line
self.cmd_line = cmd_line
return self.cmd_line
def get_cmd_line(self):
if not self.cmd_line:
cmd_line = 'virsh ' + self.command.name
for option in self.options:
if option.line is None:
continue
else:
result_line = '--' + option.option.name
if option.line:
result_line += ' %s' % utils.escape(option.line)
cmd_line += ' %s' % result_line
self.cmd_line = cmd_line
return self.cmd_line
def writeFavourites(file, faves):
kodiFile = os.path.join("special://profile", utils.FILENAME)
isKodi = xbmc.translatePath(file) == xbmc.translatePath(kodiFile)
f = sfile.file(file, "w")
f.write("<favourites>")
for fave in faves:
try:
name = utils.escape(fave[0])
thumb = utils.escape(fave[1])
cmd = utils.escape(fave[2])
if isKodi and cmd.lower().startswith("playmedia"):
cmd = removeSFOptions(cmd)
thumb = utils.convertToHome(thumb)
name = 'name="%s" ' % name
thumb = 'thumb="%s">' % thumb
f.write("\n\t<favourite ")
f.write(name)
f.write(thumb)
f.write(cmd)
f.write("</favourite>")
except:
pass
f.write("\n</favourites>")
f.close()
import xbmcgui
try:
count = int(xbmcgui.Window(10000).getProperty("Super_Favourites_Count"))
except:
count = 0
xbmcgui.Window(10000).setProperty("Super_Favourites_Count", str(count + 1))
def create_table_sql(cls, db_name):
'''
Returns the SQL command for creating a table for this model.
'''
parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db_name, cls.table_name())]
cols = []
for name, field in cls._fields:
default = field.get_db_prep_value(field.default)
cols.append(' %s %s DEFAULT %s' % (name, field.db_type, escape(default)))
parts.append(',\n'.join(cols))
parts.append(')')
parts.append('ENGINE = ' + cls.engine.create_table_sql())
return '\n'.join(parts)
def visit_Output(self, node, frame):
body = []
for child in node.nodes:
try:
const = child.as_const(frame.eval_ctx)
except nodes.Impossible:
body.append(child)
continue
try:
if frame.eval_ctx.autoescape:
if hasattr(const, '__html__'):
const = const.__html__()
else:
const = escape(const)
const = str(const)
except Exception:
body.append(child)
continue
if body and isinstance(body[-1], list):
body[-1].append(const)
else:
body.append([const])
# write a format string for the body
format = []
arguments = []
for item in body:
if isinstance(item, list):
format.append(concat(item).replace('%', '%%'))
else:
format.append('%s') # TOFIX: item is a tuple/list/iterable
arguments.append(item)
self.writeline('yield ')
self.write(repr(concat(format)))
if arguments:
self.write(' % (')
self.indent()
for argument in arguments:
self.newline(argument)
close = 0
if frame.eval_ctx.autoescape:
self.write('escape(')
close += 1
self.visit(argument, frame)
self.write(')' * close)
self.outdent()
if len(arguments) == 1:
# trailing comma for tuple with len 1
self.write(',')
self.writeline(')')
def render_testapp(req):
try:
import pkg_resources
except ImportError:
eggs = ()
else:
eggs = list(pkg_resources.working_set)
eggs.sort(
lambda a, b: cmp(a.project_name.lower(), b.project_name.lower()))
python_eggs = []
for egg in eggs:
try:
version = egg.version
except (ValueError, AttributeError):
version = 'unknown'
python_eggs.append('<li>%s <small>[%s]</small>' %
(escape(egg.project_name), escape(version)))
wsgi_env = []
sorted_environ = req.environ.items()
sorted_environ.sort(key=lambda x: repr(x[0]).lower())
for key, value in sorted_environ:
wsgi_env.append(
'<tr><th>%s<td><code>%s</code>' %
(escape(str(key)), ' '.join(wrap(escape(repr(value))))))
sys_path = []
for item, virtual, expanded in iter_sys_path():
class_ = []
if virtual:
class_.append('virtual')
if expanded:
class_.append('exp')
sys_path.append(
'<li%s>%s' %
(class_ and ' class="%s"' % ' '.join(class_) or '', escape(item)))
return TEMPLATE % {
'python_version': '<br>'.join(escape(sys.version).splitlines()),
'platform': escape(sys.platform),
'os': escape(os.name),
'api_version': sys.api_version,
'byteorder': sys.byteorder,
'werkzeug_version': werkzeug.__version__,
'python_eggs': '\n'.join(python_eggs),
'wsgi_env': '\n'.join(wsgi_env),
'sys_path': '\n'.join(sys_path)
}
def generate(self):
"""Yields pieces of XML."""
yield u'<url>\n'
yield u'<loc>%s</loc>\n' % escape(self.loc)
if self.lastmod:
if hasattr(self.lastmod, 'strftime'):
yield u'<lastmod>%s</lastmod>\n' % self.lastmod.strftime('%Y-%m-%d')
elif isinstance(self.lastmod, str):
yield u'<lastmod>%s</lastmod>\n' % self.lastmod
if self.changefreq and self.changefreq in self.freq_values:
yield u'<changefreq>%s</changefreq>\n' % self.changefreq
if self.priority and 0.0 <= self.priority <= 1.0:
yield u'<priority>%s</priority>\n' % self.priority
yield u'</url>\n'
def json_data(self):
return {
"content": escape(self.content),
"title": self.title,
"id": self.id,
"date_created": self.date_created,
"is_active": self.is_active,
"user": self.user.json_data(),
"tags": self.get_tags(),
"points": self.points,
"comment_count": self.comment_count,
"view_count": self.view_count,
"urls": self.get_url()
}
def json_data(self):
return {
"content": escape(self.content),
"title": self.title,
"id": self.id,
"date_created": self.date_created,
"is_active": self.is_active,
"user": self.user.json_data(),
"tags": self.get_tags(),
"points": self.points,
"comment_count": self.comment_count,
"view_count": self.view_count,
"urls": self.get_url()
}
def parse_text(text):
try:
text = escape(text)
print("Creating request")
body = rqt_parse_text.format(text=text).encode("utf-8")
print("Request sent to server")
response = requests.post(wsdl_file, data=body, headers=headers)
print("Response received")
return get_result(response)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("Error at calling fdg_parser. Reason:", str(e), "File:", fname,
"Line:", exc_tb.tb_lineno)
def on_tooltip(self, icon, pos_x, pos_y, key, tip, *args):
# Prevent broken tooltips while the context menu is opened
if self.popup_open:
self.fake_move((0.0, self.mouse_position[1]), True)
return True
link = self.last_hovered_link
if link is not None:
if link.startswith('avatar:'):
uri = link[7:]
num = int(uri[:uri.find(':')])
user = self.get_user(num)
img = self.get_image(num)
# Set only if something has changed
if user != self.tooltip_user \
or img != self.tooltip_img_file:
self.set_tooltip(user, img)
self.tooltip_user = user
tip.set_custom(self.tooltip)
return True
# Try to expand links
elif link.startswith('http://'):
if link in self.expanded_links:
if self.expanded_links[link] != link:
tip.set_markup(lang.html_expanded_tooltip
% (escape(link),
escape(self.expanded_links[link])))
return True
elif not self.is_expanding:
self.is_expanding = True
URLExpander(link, self.expand_link)
def set_tooltip(self, user, img):
self.tooltip_img.show()
self.tooltip_label.set_markup(lang.html_avatar_tooltip \
% (escape(user.name), user.statuses_count,
user.followers_count, user.friends_count))
# Watch out for malformatted gifs, I'm looking at you @defunkt!
if img != self.tooltip_img_file:
try:
buf = gtk.gdk.pixbuf_new_from_file_at_size(img, 48, 48)
self.tooltip_img.set_from_pixbuf(buf)
self.tooltip_img_file = img
except GLIBError:
self.tooltip_img.hide()
def generate(self):
"""Yield next XML fragment as a string."""
yield u'<url>\n'
yield u'<loc>%s</loc>\n' % escape(self.loc)
if self.lastmod:
if hasattr(self.lastmod, 'strftime'):
yield u'<lastmod>%s</lastmod>\n' % \
self.lastmod.strftime('%Y-%m-%d')
elif isinstance(self.lastmod, str):
yield u'<lastmod>%s</lastmod>\n' % self.lastmod
if self.changefreq and self.changefreq in self.freq_values:
yield u'<changefreq>%s</changefreq>\n' % self.changefreq
if self.priority and 0.0 <= self.priority <= 1.0:
yield u'<priority>%s</priority>\n' % self.priority
yield u'</url>\n'
def json_data(self):
return {
"content": escape(self.content),
"title": self.title,
"id": self.id,
"date_created": self.date_created,
"user": self.user.json_data(),
"tags": [tag.json_data() for tag in self.tags],
"points": self.points,
"comment_count": self.comment_count,
"user_vote": self.user_vote,
"user_comment": self.user_comment,
"user_answer": self.user_answer,
"answer_count": self.answer_count,
"urls": self.get_url()
}
def process_code_block(self, fd, lines):
line = lines[0];
if (not re.match(" *(.*)", line)):
return False;
fd.write("<pre><code>\n");
while (len(lines) > 0):
line = lines[0];
m = re.match(" *(.*)", line);
if (not m):
break;
# we need to escape as code might contain '<' and other stuff
fd.write(utils.escape(m.group(1)) + "\n");
lines.popleft();
fd.write("</code></pre>\n");
return True;
def json_data(self):
return {
"content": escape(self.content),
"title": self.title,
"id": self.id,
"date_created": self.date_created,
"user": self.user.json_data(),
"tags": [tag.json_data() for tag in self.tags],
"points": self.points,
"comment_count": self.comment_count,
"user_vote": self.user_vote,
"user_comment": self.user_comment,
"user_answer": self.user_answer,
"answer_count": self.answer_count,
"urls": self.get_url()
}
def render(self): #resp = '<div class="control-group">' #resp += '<label class="control-label" for="' + self.variable_name + '">' + self.secondary_content + '</label>' #resp += '<div class="controls">' resp = '<div class="row">' resp += '<div class="span1" style="text-align:right;padding-right:10px">' + self.secondary_content + '</div>' resp += '<input type="text" style="input-big" name="' + self.variable_name + '"' if self.answer: resp += 'value="' + escape(str(self.answer)) + '"' resp += '>' resp += '</div>' # resp = '<div class="span1" style="text-align:right">' + self.secondary_content + '</div>' # resp += '<input type="text" style="input-medium" name="' + self.variable_name + '" value="' # if self.answer: # resp += escape(str(self.answer)) # resp += '">' return resp
def gen_course_data(root='out'):
"""将数据库里的数据导出到本地（导出清晰度最高的数据）
:param root: 输出文件的根目录
:return:
"""
sql_session = SqliteSession().session
# 获取所有课程
course_objs = sql_session.query(Course).all()
orgin_dir = [
root,
]
for course_obj in course_objs:
# 获取课程文件路径
course_dir = orgin_dir[:]
course_dir.append('{}-{}'.format(course_obj.course_id,
course_obj.course_name))
course_dir.append('第{}章-{}'.format(course_obj.chapter_id,
course_obj.chapter_name))
course_dir.append('第{}节-{}.m3u8'.format(course_obj.section_id,
course_obj.section_name))
course_dir = map(lambda s: escape(s).strip(), course_dir)
course_m3u8_path = '/'.join(course_dir)
course_dir = os.path.dirname(course_m3u8_path)
if not os.path.exists(course_m3u8_path):
try:
os.makedirs(course_dir, exist_ok=True)
except Exception as e:
logger.error('{}-{}'.format(course_dir, e))
# 写入数据
section_md5 = course_obj.section_md5
course_detail = sql_session.query(
CourseDetail.m3u8, CourseDetail.aes_key).filter(
CourseDetail.section_md5 == section_md5).order_by(
CourseDetail.width.desc()).first()
m3u8_text = course_detail.m3u8 # TRIM_FULL_KEY_URI.sub(r'\1\2',course_detail.m3u8)
with open(course_m3u8_path, 'w') as f:
f.write(m3u8_text)
path_key = json.loads(course_detail.aes_key)
key_path, key = path_key.popitem()
course_key_path = os.path.join(course_dir, key_path)
os.makedirs(os.path.dirname(course_key_path), exist_ok=True)
with open(course_key_path, 'w') as f:
f.write(key)
def output_text(self, fd, text):
# XXX: is this valid ?
text = utils.escape(text);
# start with a dummy char that is not the escape char
result = "X";
start = 0;
while(True):
m = re.search(self.link_re, text[start:]);
if (not m):
result += text[start:];
break;
result += text[start:start + m.start()];
url = self.image_cb(m.group(2));
result += "<a href=\"" + url + "\">" + m.group(1) + "</a>"
start += m.end();
result = re.sub(r"[^\\]\*([^\* ]*[^\\])\*", r"<b>\1</b>", result);
result = re.sub(r"[^\\]_([^_ ]*[^\\])_", r"<em>\1</em>", result);
# unescape
result = re.sub(r"\\(.)", r"\1", result);
fd.write(result[1:]);
def text(escape=False, min_len=5, max_len=10):
"""
Generate a randomized string.
"""
excludes = "\n\t\r\x0b\x0c"
chars = []
for char in string.printable:
if char not in excludes:
chars.append(char)
length = random.randint(min_len, max_len)
result_str = ''.join(random.choice(chars) for _ in xrange(length))
if escape:
return utils.escape(result_str)
else:
return result_str
def text(escape=False, min_len=5, max_len=10):
"""
Generate a randomized string.
"""
excludes = "\n\t\r\x0b\x0c"
chars = []
for char in string.printable:
if char not in excludes:
chars.append(char)
length = random.randint(min_len, max_len)
result_str = ''.join(random.choice(chars) for _ in xrange(length))
if escape:
return utils.escape(result_str)
else:
return result_str
def keys(self, pattern="*", *args):
return self._parse_result(
FORMAT % ("keys", "%s<->%s" % escape((pattern, ""))),
{"recv": json.loads})
def describe(self, uneval=False, predrop=False):
describe_first = self.first.describe(uneval=uneval,
predrop=(self.is_dropkeepadd() or (predrop and self.is_collection())))\
if self.first != None else ""
describe_second = self.second.describe(uneval=uneval)\
if self.second != None else ""
if self._kind == "(": # parenthesis group
return "(" + describe_first + ")"
spacer = " " if self._spaces else ""
detail_description = ""
if not uneval:
if (not self.is_collection()) and (
not self.contains_diceroll()):
# No child contains diceroll randomness, so just show the
# value; or other repr if valueless like an operator
# argument.
if self.get_value() == None:
return escape(str(self))
return f"{self.get_value()}"
# repeated expression special case
# This causes details to be hidden from `agg`.
if (self._kind == "repeat"
or self._kind == "agg") and (not predrop):
return ExprResult.description(self.detail)
if self.detail:
detail_description = " " + \
ExprResult.description(self.detail)
if self.is_collection():
# Hide breakout detail description if only one item
if len(self.detail.get_all_items()
) < 2 and self.detail.get_value() != None:
detail_description = "⇒" + str(self.detail)
# Hide details if outer collection already will show them
if predrop:
detail_description = ""
if self._function_like:
close_function = ")"
if len(describe_second) > 0:
close_function = f",{spacer}{describe_second})"
return f"{self._kind}({describe_first}" + close_function
left = describe_first
right = describe_second
if self.second == None:
# No operands, just describe this node's value.
if self.first == None:
return str(self)
if self._postfix:
right = ""
else:
left = ""
right = describe_first
op = self._kind
if not uneval:
op = escape(self._kind)
prewrap = f"{left}{spacer}{op}{spacer}{right}" +\
f"{detail_description}"
if self._kind == "choose" or self._kind == "C":
return "(" + prewrap + ")"
elif self.is_collection() and not (predrop or uneval):
return "[" + prewrap + "]"
return prewrap
def get_description(self):
# We could append = (total) for consistency with dice rolls.
# Some tweaks to describe() would be needed, as a special case
# for `agg` but not `d` causes differences in formatting paths.
return "(" + super().get_description(
joiner=escape(self.agg_joiner)) + ")"
twitter_status = '{0} {1}'.format(random_post_title, random_post_url)
signature_timestamp = datetime.now().strftime('%s')
signature_once = str(getrandbits(64))
signature_basestr_auth = {}
signature_basestr_auth['status'] = twitter_status
signature_basestr_auth['oauth_consumer_key'] = twitter_consumer_key
signature_basestr_auth['oauth_nonce'] = signature_once
signature_basestr_auth['oauth_signature_method'] = twitter_api_method
signature_basestr_auth['oauth_timestamp'] = signature_timestamp
signature_basestr_auth['oauth_token'] = twitter_oauth_token
signature_basestr_auth['oauth_version'] = twitter_api_version
signature_basestr_auth = [
'{0}={1}'.format(escape(k), escape(v))
for k, v in sorted(signature_basestr_auth.items())
]
signature_basestr_auth = '&'.join(signature_basestr_auth)
signature_basestr = 'POST&{0}&{1}'.format(escape(twitter_api_end),
escape(signature_basestr_auth))
signature_key = '{0}&{1}'.format(twitter_consumer_secret,
escape(twitter_oauth_secret))
oauth_hmac_hash = hmac.new(signature_key.encode(), signature_basestr.encode(),
hashlib.sha1)
oauth_signature = base64.b64encode(oauth_hmac_hash.digest()).decode()
oauth_header = {}
oauth_header['oauth_consumer_key'] = twitter_consumer_key
def delete(self, key, *args):
return self._parse_result(FORMAT % ("delete", "%s<->%s" % escape(
(key, ""))))
def expire(self, key, seconds, *args):
return self._parse_result(FORMAT % ("expire", "%s<->%s" % escape(
(key, seconds))))
def ttl(self, key, *args):
return self._parse_result(FORMAT % ("ttl", "%s<->%s" % escape(
(key, ""))))
def get_description(self, environ):
m = escape(environ.get('REQUEST_METHOD', 'GET'))
return '<p>The method %s is not allowed for the requested URL.</p>' % m
def match_single_entity(self, text: str, entity: str,
ctx: Context) -> bool:
"""
匹配单个entity，如果单tag有规则能拆成多个entity，则转为匹配多个entity
"""
# print(text, entity, self.known.get_sub(entity))
# 处理entity，将其中的标记标签去除
tag = re.match(r"^([^\[\]]+?)\|([^\[\]]+)$", entity)
if tag is not None:
entity, tag = tag.group(1), tag.group(2)
# 确定匹配到的是否符合上下文要求
ctx_entity = ctx.find(entity, tag)
# 此处匹配不会改变Context，所以可以直接返回
if ctx_entity is not None:
return (text == ctx_entity)
# 编译正则表达式，供后取使用
match_regex = re.compile(r"\[(.+?)\](?=[^\[]|$)")
# 遍历需要匹配的entity下所有的子实体
for sub_entity in self.known.get_sub(entity):
# 获取子实体
match_text = sub_entity.get_text()
# 符号转移，避免文本中原有的符号被当成正则表达式处理
match_escape = utils.escape(match_text)
# 构造entity的正则表达式，将需要进行实体判断的部分用通配符取代
match_re = "^%s$" % match_regex.sub(r"(.+?)", match_escape)
match = re.match(match_re, text)
if match:
match_succ = True
ctx.new_stack()
# 判断实体匹配中是否存在子实体需要匹配
match_entity = match_regex.findall(match_text)
# 弱存在子实体，则下述循环会逐一判断每个子实体
for sub_text, sub_entity in zip(match.groups(), match_entity):
# 判断是否存在相邻子实体情况
sub_entities = sub_entity.split("][")
if len(sub_entities) == 1:
# 不存在，可以直接进行单个匹配
match_succ = self.match_single_entity(
sub_text, sub_entities[0], ctx)
else:
# 存在相邻子实体，需要进行分词匹配
match_succ = self.match_multi_entities(
sub_text, sub_entities, ctx)
# 若无法匹配成功，则不属于此实体
if not match_succ:
break
if match_succ:
# 成功匹配，记录上下文
ctx.append(Instance(entity, text, None, tag=tag))
ctx.apply_stack()
return True
else:
# 匹配失败，抛弃上下文
ctx.aborat_stack()
print("aborat")
return False
print("Stemming with {a} stemmer without stop words removel".format(
a=algo))
print("Please wait while we are stemming the text ... \n")
# Processing of the input text before neighbors finding
prog = re.compile("[_\-\(]*([A-Z]\.)*[_\-\(]*")
for t in toProcess:
terms = toProcess[t].split() #stem(algo,toProcess[t]).split()
text = ""
for w in terms:
if (prog.match(w)):
w = w.replace('.', '')
text = text + " " + w
text = ' '.join(escape(text).split())
text = " ".join(nltk.word_tokenize(text))
d = []
text = " ".join([stem(algo, w) for w in text.split()])
d += text.split()
toProcess[t] = d
#print(toProcess)
if bool(args["--stop"]):
stopWordsList = set(stopwords.words('english'))
print(stopWordsList)
for i in toProcess:
listWords = [w for w in toProcess[i] if w not in stopWordsList]
toProcess[i] = listWords
#print(toProcess)
