def view(tag_name):
tag = db_session.query(Tag).filter_by(name=tag_name).one()
if request.method == "GET":
if not tag:
abort(404)
d["tag"] = tag
d["pastes"] = tag.pastes[:20]
return render_template("tagapp/view.html", **d)
if request.method == "POST":
if not tag:
return json.dump([])
start = request.GET.get("start", 0)
tags = tag.pastes[start - 1 : 20]
return json.decode(tags)
def view(tag_name):
tag = db_session.query(Tag).filter_by(name=tag_name).one()
if request.method == 'GET':
if not tag:
abort(404)
d['tag'] = tag
d['pastes'] = tag.pastes[:20]
return render_template('tagapp/view.html', **d)
if request.method == 'POST':
if not tag:
return json.dump([])
start = request.GET.get('start', 0)
tags = tag.pastes[start - 1:20]
return json.decode(tags)
def extract_jwplayer_setup(data):
"""
Extracts jwplayer setup configuration and returns it as a dictionary.
:param data: A string to extract the setup from
:return: A dictionary containing the setup configuration
"""
data = re.search(r'<script.+?}\(\'(.+)\',\d+,\d+,\'([\w\|]+)\'.*</script>',
data, re.I | re.S)
if data:
replacements = data.group(2).split('|')
data = data.group(1)
for i in reversed(range(len(replacements))):
if len(replacements[i]) > 0:
data = re.sub(r'\b%s\b' % int_to_base(i, 36), replacements[i],
data)
data = re.search(r'\.setup\(([^\)]+?)\);', data)
if data:
return json.decode(data.group(1).decode('string_escape'))
return None
def build_feedparser_cache(update=False):
"""Build a feedparser cache."""
from multiprocessing import Pool, cpu_count
paths = []
for path in glob('feeds/*.dat'):
if not update and os.path.exists(path.replace('dat', 'json')):
continue
paths.append(path)
parser_pool = Pool(cpu_count())
results = []
for path in paths:
results.append(parser_pool.apply_async(feedparse, (path,)))
for result in results:
value = result.get()
if value is None: continue
path, json = value
json = json.decode('base64')
update_cache(path, json)
def request(self, method, params=None):
"""
Creates a new request using the passed method and parameters.
"""
req_id = generate_random_uuid().hex
req = {
'jsonrpc': self.rpc_version,
'method': method,
'id': req_id,
}
if params:
req['params'] = params
enc_req = encode(req)
self._c.send(enc_req)
enc_res = self._c.recv(self.buffer_size)
res = decode(enc_res)
if res.get('id') != req_id:
raise RuntimeError("Received unexpected response: %s" \
% (res, ))
return res.get('result')
def build_feedparser_cache(update=False):
"""Build a feedparser cache."""
from multiprocessing import Pool, cpu_count
paths = []
for path in glob('feeds/*.dat'):
if not update and os.path.exists(path.replace('dat', 'json')):
continue
paths.append(path)
parser_pool = Pool(cpu_count())
results = []
for path in paths:
results.append(parser_pool.apply_async(feedparse, (path,)))
for result in results:
value = result.get()
if value is None: continue
path, json = value
json = json.decode('base64')
update_cache(path, json)
def obfuscate_courseware_entry(self, line, user_profile):
fields = line.rstrip('\r\n').decode('utf8').split('\t')
record = CoursewareRecord(*fields)
# Skip the header.
if record.state == 'state':
return line.rstrip('\r\n')
user_info = {}
if user_profile is not None:
user_id = record.student_id
if user_id != 'NULL':
profile_entry = user_profile.get(user_id)
if profile_entry is None:
self.missing_profile[user_id] += 1
else:
user_info['name'] = [profile_entry.name, ]
# TODO: also read in auth_user, and store username for each user_id.
pass
# Courseware_studentmodule is not processed with the other SQL tables, so it
# is not escaped in the same way. In particular, we will not decode and encode it.
state_str = record.state.replace('\\\\', '\\')
try:
state_dict = simplejson.decode(state_str, all_unicode=True)
except Exception as exc:
log.exception(u"Unable to parse state as JSON for record %s: type = %s, state = %r", record.id, type(state_str), state_str)
return line
# Traverse the dictionary, looking for entries that need to be scrubbed.
updated_state_dict = self.obfuscator.obfuscate_structure(state_dict, u"state", user_info)
if updated_state_dict is not None:
# Can't reset values, so update original fields.
updated_state = json.dumps(updated_state_dict).replace('\\', '\\\\')
fields[4] = updated_state
log.info(u"Obfuscated state for user_id '%s' module_id '%s'", record.student_id, record.module_id)
return u"\t".join(fields).encode('utf-8')
def filter_row(self, row):
user_id = row[3]
user_info = {'user_id': [user_id, ]}
try:
user_id = int(user_id)
entry = self.user_by_id[user_id]
if 'username' in entry:
user_info['username'] = [entry['username'], ]
if 'name' in entry:
user_info['name'] = [entry['name'], ]
except KeyError:
log.error("Unable to find CWSM user_id: %r in the user_by_id map of size %s", user_id, len(self.user_by_id))
row[3] = self.remap_id(user_id) # student_id
# Courseware_studentmodule is not processed with the other SQL tables, so it
# is not escaped in the same way. In particular, we will not decode and encode it,
# but merely transform double backslashes.
state_str = row[4].replace('\\\\', '\\')
try:
state_dict = simplejson.decode(state_str, all_unicode=True)
# Traverse the dictionary, looking for entries that need to be scrubbed.
updated_state_dict = self.obfuscator.obfuscate_structure(state_dict, u"state", user_info)
except Exception: # pylint: disable=broad-except
log.exception(u"Unable to parse state as JSON for record %s: type = %s, state = %r",
row[0], type(state_str), state_str)
updated_state_dict = {}
if updated_state_dict is not None:
# Can't reset values, so update original fields.
updated_state = simplejson.encode(updated_state_dict).replace('\\', '\\\\')
row[4] = updated_state
if self.obfuscator.is_logging_enabled():
log.info(u"Obfuscated state for user_id '%s' module_id '%s'", user_id, row[2])
return row
def obfuscate_forum_entry(self, line, user_profile):
# Round trip does not preserve content. Original had no embedded spaces,
# and entries were in alphabetic order. This is addressed by modifying the
# separators and setting sort_keys, but there are character encodings that
# are also different, as to when \u notation is used for a character as
# opposed to a utf8 encoding of the character.
try:
entry = simplejson.decode(line, all_unicode=True)
except ValueError as exc:
log.error("Failed to parse json for line: %r", line)
return ""
# Get user information:
username = entry.get('author_username')
user_info = {'username': [username, ]}
profile_entry = None
if user_profile is not None:
user_id = entry.get('author_id')
profile_entry = user_profile.get(user_id)
if profile_entry is None:
log.error(u"Missing profile entry for user_id %s username %s", user_id, username)
else:
user_info['name'] = [profile_entry.name, ]
# Clean the body of the forum post.
body = entry['body']
clean_body = self.obfuscator.obfuscate_text(body, user_info)
entry['body'] = clean_body
# Also clean the title, since it also contains username and fullname matches.
if 'title' in entry:
title = entry['title']
clean_title = self.obfuscator.obfuscate_text(title, user_info)
entry['title'] = clean_title
return json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
def finish_round(self):
'''All players have submitted their moves now we sort them and update
the battle stats.
'''
moves = [self.get_moves(i)[-1]
for i in range(1, len(self.players) + 1)]
pools = [Team(self.health[i], self.magic[i], self.defense[i])
for i in range(0, len(self.players))]
tasks = []
for move, team in zip(moves, self.teams):
tuples = simplejson.decode(move)
for hero, tup in zip(team, tuples):
if tup[1] == 1:
call = hero.attack
elif tup[1] == 3:
call = hero.ability
else:
call = hero.defend
tasks.append((tup[0], call, pools[tup[2]]))
tasks.sort()
for task in tasks:
print sys.log >> task
task[1].__call__(task[2])
self.tasks.append(tasks)
def decode_json(line):
"""Wrapper to decode JSON string in an implementation-independent way."""
# TODO: Verify correctness of simplejson
return simplejson.decode(line)
def _load(self, _path, _dict):
_dict.update(decode(open(_path).read()))