Beispiel #1
0
def get_markers(db, client_name, clip_id, congress, chamber):
    api_url = API_PREFIX + client_name + '?type=marker&size=100000'
    data = '{"filter": { "term": { "video_id": %s}}, "sort": [{"offset":{"order":"asc"}}]}' % clip_id
    markers = query_api(db, api_url, data)
    clips = []
    bill_ids = []
    legislators = []
    legislator_ids = []
    roll_ids = []

    if markers:
        for m in markers:
            m_new = m['_source']
            c = {
                'offset': m_new['offset'],
                'events': [htmlentitydecode(m_new['name']).strip(),],
                'time': m_new['datetime']
            }
            if m != markers[-1]:  #if it's not the last one
                c['duration'] = markers[markers.index(m)+1]['_source']['offset'] - m_new['offset']

            year = dateparse(m_new['datetime']).year

            legis, bio_ids = python_utils.extract_legislators(c['events'][0], chamber, db)
            b = python_utils.extract_bills(c['events'][0], congress)
            r = python_utils.extract_rolls(c['events'][0], chamber, year)

            if legis:
                c['legislator_names'] = legis
                for l in legis:
                    if l not in legislators:
                        legislators.append(l)
            if bio_ids:
                c['legislator_ids'] = bio_ids
                for bi in bio_ids:
                    if bi not in legislator_ids:
                        legislator_ids.append(bi)

            if r:
                c['roll_ids'] = r
                for ro in r:
                    if ro not in roll_ids:
                        roll_ids.append(ro)

            if b:
                c['bill_ids'] = b
                for bill in b:
                    if bill not in bill_ids:
                        bill_ids.append(bill)


            clips.append(c)

        return (clips, bill_ids, legislators, legislator_ids, roll_ids)

    else:
        db.warning('There are no markers for video id: %s' % clip_id)
        return (None, None, None, None, None)
def get_markers(db, client_name, clip_id, congress, chamber):
    api_url = API_PREFIX + client_name + '?type=marker&size=100000'
    data = '{"filter": { "term": { "video_id": %s}}, "sort": [{"offset":{"order":"asc"}}]}' % clip_id
    markers = query_api(db, api_url, data)
    clips = []
    bill_ids = []
    legislators = []
    legislator_ids = []
    roll_ids = []

    if markers:
        for m in markers:
            m_new = m['_source']
            c = {
                'offset': m_new['offset'],
                'events': [
                    htmlentitydecode(m_new['name']).strip(),
                ],
                'time': m_new['datetime']
            }
            if m != markers[-1]:  #if it's not the last one
                c['duration'] = markers[markers.index(
                    m) + 1]['_source']['offset'] - m_new['offset']

            year = dateparse(m_new['datetime']).year

            legis, bio_ids = python_utils.extract_legislators(
                c['events'][0], chamber, db)
            b = python_utils.extract_bills(c['events'][0], congress)
            r = python_utils.extract_rolls(c['events'][0], chamber, year)

            if legis:
                c['legislator_names'] = legis
                for l in legis:
                    if l not in legislators:
                        legislators.append(l)
            if bio_ids:
                c['legislator_ids'] = bio_ids
                for bi in bio_ids:
                    if bi not in legislator_ids:
                        legislator_ids.append(bi)

            if r:
                c['roll_ids'] = r
                for ro in r:
                    if ro not in roll_ids:
                        roll_ids.append(ro)

            if b:
                c['bill_ids'] = b
                for bill in b:
                    if bill not in bill_ids:
                        bill_ids.append(bill)

            clips.append(c)

        return (clips, bill_ids, legislators, legislator_ids, roll_ids)

    else:
        db.note('There are no markers for video id: %s' % clip_id)
        return (None, None, None, None, None)
def get_clips_for_senate(db, clip_id, congress, duration, year):
    #go with 5 minute clips?
    chamber = "senate"
    clip_segment = 5 * 60
    clip_number = (duration / clip_segment) + 1

    clips = []
    bill_ids = []
    legislators = []
    legislator_ids = []
    roll_ids = []

    caps = get_captions('floor.senate.gov', clip_id)
    if caps is None:
        print "Server error while fetching captions, skipping."
        return None, None, None, None, None

    offset = 0
    for clip_num in range(1, clip_number + 1):
        start = offset
        if clip_num == clip_number + 1:  #last clip
            dur = duration - offset
        else:
            dur = clip_segment

        c = {'offset': start, 'duration': dur}

        events = ''
        captions = get_senate_clip_captions(caps, start, start + clip_segment)

        legis, bio_ids = python_utils.extract_legislators(
            captions, chamber, db)
        b = python_utils.extract_bills(captions, congress)
        r = python_utils.extract_rolls(captions, chamber, year)

        if legis:
            c['legislator_names'] = legis
            events += 'Legislators mentioned in this clip: '
            for l in legis:
                if l not in legislators:
                    legislators.append(l)
                events += l
                if l != legislators[-1]:
                    events += '; '

        if bio_ids:
            c['legislator_ids'] = bio_ids
            for bi in bio_ids:
                if bi not in legislator_ids:
                    legislator_ids.append(bi)

        if r:
            c['roll_ids'] = r
            for ro in r:
                if ro not in roll_ids:
                    roll_ids.append(ro)

        if b:
            c['bill_ids'] = b
            events += 'Bills mentioned in this clip: '
            for bill in b:
                if bill not in bill_ids:
                    bill_ids.append(bill)

                bill_name = db['bills'].find_one({'bill_id': bill})
                if bill_name and bill_name.has_key(
                        'short_title') and bill_name[
                            'short_title'] and bill_name['short_title'] != '':
                    events += bill_name['short_title'] + '; '
                elif bill_name:
                    events += ("%s%s" % (bill_name['bill_type'],
                                         bill_name['number'])).upper() + '; '

        if events == '':
            events = "No description for clip number %s" % clip_num

        c['events'] = [
            events,
        ]

        clips.append(c)

        offset = offset + clip_segment

    return (clips, bill_ids, legislators, legislator_ids, roll_ids)
Beispiel #4
0
def get_clips_for_senate(db, clip_id, congress, duration, year):
    #go with 5 minute clips?
    chamber = "senate"
    clip_segment = 5 * 60
    clip_number = (duration / clip_segment) + 1

    clips = []
    bill_ids = []
    legislators = []
    legislator_ids = []
    roll_ids = []

    caps = get_captions('floor.senate.gov', clip_id)
    if caps is None:
        print "Server error while fetching captions, skipping."
        return None, None, None, None, None

    offset = 0
    for clip_num in range(1, clip_number + 1):
        start = offset
        if clip_num == clip_number + 1: #last clip
            dur = duration - offset
        else:
            dur = clip_segment

        c = {
            'offset': start,
            'duration': dur
        }

        events = ''
        captions = get_senate_clip_captions(caps, start, start + clip_segment)

        legis, bio_ids = python_utils.extract_legislators(captions, chamber, db)
        b = python_utils.extract_bills(captions, congress)
        r = python_utils.extract_rolls(captions, chamber, year)

        if legis:
            c['legislator_names'] = legis
            events += 'Legislators mentioned in this clip: '
            for l in legis:
                if l not in legislators:
                    legislators.append(l)
                events += l
                if l != legislators[-1]:
                    events += '; '

        if bio_ids:
            c['legislator_ids'] = bio_ids
            for bi in bio_ids:
                if bi not in legislator_ids:
                    legislator_ids.append(bi)

        if r:
            c['roll_ids'] = r
            for ro in r:
                if ro not in roll_ids:
                    roll_ids.append(ro)

        if b:
            c['bill_ids'] = b
            events += 'Bills mentioned in this clip: '
            for bill in b:
                if bill not in bill_ids:
                    bill_ids.append(bill)

                bill_name = db['bills'].find_one({'bill_id':bill })
                if  bill_name and bill_name.has_key('short_title') and bill_name['short_title'] and bill_name['short_title'] != '':
                    events += bill_name['short_title'] + '; '
                elif bill_name:
                    events += ("%s%s" % (bill_name['bill_type'], bill_name['number'])).upper() + '; '

        if events == '':
            events = "No description for clip number %s" % clip_num

        c['events'] = [events,]

        clips.append(c)

        offset = offset + clip_segment

    return (clips, bill_ids, legislators, legislator_ids, roll_ids)