Example #1
0
def view(tag_name):
    tag = db_session.query(Tag).filter_by(name=tag_name).one()
    if request.method == "GET":
        if not tag:
            abort(404)
        d["tag"] = tag
        d["pastes"] = tag.pastes[:20]
        return render_template("tagapp/view.html", **d)
    if request.method == "POST":
        if not tag:
            return json.dump([])
        start = request.GET.get("start", 0)
        tags = tag.pastes[start - 1 : 20]
        return json.decode(tags)
Example #2
0
def view(tag_name):
    tag = db_session.query(Tag).filter_by(name=tag_name).one()
    if request.method == 'GET':
        if not tag:
            abort(404)
        d['tag'] = tag
        d['pastes'] = tag.pastes[:20]
        return render_template('tagapp/view.html', **d)
    if request.method == 'POST':
        if not tag:
            return json.dump([])
        start = request.GET.get('start', 0)
        tags = tag.pastes[start - 1:20]
        return json.decode(tags)
Example #3
0
def extract_jwplayer_setup(data):
    """
    Extracts jwplayer setup configuration and returns it as a dictionary.

    :param data: A string to extract the setup from
    :return: A dictionary containing the setup configuration
    """
    data = re.search(r'<script.+?}\(\'(.+)\',\d+,\d+,\'([\w\|]+)\'.*</script>',
                     data, re.I | re.S)
    if data:
        replacements = data.group(2).split('|')
        data = data.group(1)
        for i in reversed(range(len(replacements))):
            if len(replacements[i]) > 0:
                data = re.sub(r'\b%s\b' % int_to_base(i, 36), replacements[i],
                              data)
        data = re.search(r'\.setup\(([^\)]+?)\);', data)
        if data:
            return json.decode(data.group(1).decode('string_escape'))
    return None
def build_feedparser_cache(update=False):
    """Build a feedparser cache."""
    from multiprocessing import Pool, cpu_count

    paths = []
    for path in glob('feeds/*.dat'):
        if not update and os.path.exists(path.replace('dat', 'json')):
            continue
        paths.append(path)

    parser_pool = Pool(cpu_count())
    results = []
    for path in paths:
        results.append(parser_pool.apply_async(feedparse, (path,)))
    for result in results:
        value = result.get()
        if value is None: continue
        path, json = value
        json = json.decode('base64')
        update_cache(path, json)
Example #5
0
 def request(self, method, params=None):
     """
     Creates a new request using the passed method and parameters.
     """
     req_id = generate_random_uuid().hex
     req = {
         'jsonrpc': self.rpc_version,
         'method': method,
         'id': req_id,
     }
     if params:
         req['params'] = params
     enc_req = encode(req)
     self._c.send(enc_req)
     enc_res = self._c.recv(self.buffer_size)
     res = decode(enc_res)
     if res.get('id') != req_id:
         raise RuntimeError("Received unexpected response: %s" \
                            % (res, ))
     return res.get('result')
Example #6
0
def build_feedparser_cache(update=False):
    """Build a feedparser cache."""
    from multiprocessing import Pool, cpu_count

    paths = []
    for path in glob('feeds/*.dat'):
        if not update and os.path.exists(path.replace('dat', 'json')):
            continue
        paths.append(path)

    parser_pool = Pool(cpu_count())
    results = []
    for path in paths:
        results.append(parser_pool.apply_async(feedparse, (path,)))
    for result in results:
        value = result.get()
        if value is None: continue
        path, json = value
        json = json.decode('base64')
        update_cache(path, json)
    def obfuscate_courseware_entry(self, line, user_profile):
        fields = line.rstrip('\r\n').decode('utf8').split('\t')
        record = CoursewareRecord(*fields)

        # Skip the header.
        if record.state == 'state':
            return line.rstrip('\r\n')

        user_info = {}
        if user_profile is not None:
            user_id = record.student_id
            if user_id != 'NULL':
                profile_entry = user_profile.get(user_id)
                if profile_entry is None:
                    self.missing_profile[user_id] += 1
                else:
                    user_info['name'] = [profile_entry.name, ]

        # TODO: also read in auth_user, and store username for each user_id.
        pass

        # Courseware_studentmodule is not processed with the other SQL tables, so it
        # is not escaped in the same way.  In particular, we will not decode and encode it.
        state_str = record.state.replace('\\\\', '\\')
        try:
            state_dict = simplejson.decode(state_str, all_unicode=True)
        except Exception as exc:
            log.exception(u"Unable to parse state as JSON for record %s: type = %s, state = %r", record.id, type(state_str), state_str)
            return line

        # Traverse the dictionary, looking for entries that need to be scrubbed.
        updated_state_dict = self.obfuscator.obfuscate_structure(state_dict, u"state", user_info)

        if updated_state_dict is not None:
            # Can't reset values, so update original fields.
            updated_state = json.dumps(updated_state_dict).replace('\\', '\\\\')
            fields[4] = updated_state
            log.info(u"Obfuscated state for user_id '%s' module_id '%s'", record.student_id, record.module_id)

        return u"\t".join(fields).encode('utf-8')
    def filter_row(self, row):
        user_id = row[3]
        user_info = {'user_id': [user_id, ]}
        try:
            user_id = int(user_id)
            entry = self.user_by_id[user_id]
            if 'username' in entry:
                user_info['username'] = [entry['username'], ]
            if 'name' in entry:
                user_info['name'] = [entry['name'], ]
        except KeyError:
            log.error("Unable to find CWSM user_id: %r in the user_by_id map of size %s", user_id, len(self.user_by_id))

        row[3] = self.remap_id(user_id)  # student_id

        # Courseware_studentmodule is not processed with the other SQL tables, so it
        # is not escaped in the same way.  In particular, we will not decode and encode it,
        # but merely transform double backslashes.
        state_str = row[4].replace('\\\\', '\\')
        try:
            state_dict = simplejson.decode(state_str, all_unicode=True)
            # Traverse the dictionary, looking for entries that need to be scrubbed.
            updated_state_dict = self.obfuscator.obfuscate_structure(state_dict, u"state", user_info)
        except Exception:   # pylint:  disable=broad-except
            log.exception(u"Unable to parse state as JSON for record %s: type = %s, state = %r",
                          row[0], type(state_str), state_str)
            updated_state_dict = {}

        if updated_state_dict is not None:
            # Can't reset values, so update original fields.
            updated_state = simplejson.encode(updated_state_dict).replace('\\', '\\\\')
            row[4] = updated_state
            if self.obfuscator.is_logging_enabled():
                log.info(u"Obfuscated state for user_id '%s' module_id '%s'", user_id, row[2])

        return row
    def obfuscate_forum_entry(self, line, user_profile):
        # Round trip does not preserve content.  Original had no embedded spaces,
        # and entries were in alphabetic order.  This is addressed by modifying the
        # separators and setting sort_keys, but there are character encodings that
        # are also different, as to when \u notation is used for a character as
        # opposed to a utf8 encoding of the character.
        try:
            entry = simplejson.decode(line, all_unicode=True)
        except ValueError as exc:
            log.error("Failed to parse json for line: %r", line)
            return ""

        # Get user information:
        username = entry.get('author_username')
        user_info = {'username': [username, ]}
        profile_entry = None
        if user_profile is not None:
            user_id = entry.get('author_id')
            profile_entry = user_profile.get(user_id)
            if profile_entry is None:
                log.error(u"Missing profile entry for user_id %s username %s", user_id, username)
            else:
                user_info['name'] = [profile_entry.name, ]

        # Clean the body of the forum post.
        body = entry['body']
        clean_body = self.obfuscator.obfuscate_text(body, user_info)
        entry['body'] = clean_body

        # Also clean the title, since it also contains username and fullname matches.
        if 'title' in entry:
            title = entry['title']
            clean_title = self.obfuscator.obfuscate_text(title, user_info)
            entry['title'] = clean_title

        return json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
Example #10
0
 def finish_round(self):
     '''All players have submitted their moves now we sort them and update
     the battle stats.
     '''
     moves = [self.get_moves(i)[-1]
         for i in range(1, len(self.players) + 1)]
     pools = [Team(self.health[i], self.magic[i], self.defense[i])
         for i in range(0, len(self.players))]
     tasks = []
     for move, team in zip(moves, self.teams):
         tuples = simplejson.decode(move)
         for hero, tup in zip(team, tuples):
             if tup[1] == 1:
                 call = hero.attack
             elif tup[1] == 3:
                 call = hero.ability
             else:
                 call = hero.defend
             tasks.append((tup[0], call, pools[tup[2]]))
     tasks.sort()
     for task in tasks:
         print sys.log >> task
         task[1].__call__(task[2])
     self.tasks.append(tasks)
def decode_json(line):
    """Wrapper to decode JSON string in an implementation-independent way."""
    # TODO: Verify correctness of simplejson
    return simplejson.decode(line)
Example #12
0
 def _load(self, _path, _dict):
     _dict.update(decode(open(_path).read()))