def normalize_huffpo_poll_list(poll_list, poll_type): normal_poll_list = [] huffpo_poll_id_list = [] for poll in poll_list: #print poll normal_poll = {} #copy basic poll info normal_poll['poll_info'] = {} normal_poll['poll_info']['poll_date'] = normalize_date(poll) normal_poll['poll_info']['poll_source_id'] = 3 * int(poll["id"]) normal_poll['poll_info']['poll_source'] = "ofhu" normal_poll['poll_info']['poll_source_uri'] = poll["source"] #copy pollster info if poll['pollster']: normal_poll['pollster_info'] = {'pollster_str' : poll['pollster'], 'pollster_partisan' : poll['partisan'].lower(), 'group_type' : 'pollster'} #remove party from naming normal_poll['pollster_info']['pollster_str'] = normal_poll['pollster_info']['pollster_str'].replace('(D)', '') normal_poll['pollster_info']['pollster_str'] = normal_poll['pollster_info']['pollster_str'].replace('(R)', '') normal_poll['pollster_info']['pollster_list'] = [] normal_poll['pollster_info']['pollster_list_json'] = '' #iterate through "survey houses" and assign as pollster sponsors for house in poll['survey_houses']: house['party'] = house['party'].lower() house['party'] = house['party'].replace('n/a', 'nonpartisan') house['party'] = house['party'].replace('rep', 'gop') house['name'] = house['name'].replace('(D)', '') house['name'] = house['name'].replace('(R)', '') #normalize naming, deprecated if "PPP" in normal_poll['pollster_info']['pollster_str']: normal_poll['pollster_info']['pollster_str'] = "Public Policy Polling" house['group_type'] = 'pollster' normal_poll['pollster_info']['pollster_list'].append(house) for sponsor in poll['sponsors']: sponsor['party'] = sponsor['party'].lower() sponsor['party'] = sponsor['party'].replace('n/a', 'nonpartisan') sponsor['party'] = sponsor['party'].replace('rep', 'gop') sponsor['name'] = sponsor['name'].replace('(D)', '') sponsor['name'] = sponsor['name'].replace('(R)', '') #normalize naming, deprecated if "PPP" in normal_poll['pollster_info']['pollster_str']: normal_poll['pollster_info']['pollster_str'] = "Public Policy Polling" sponsor['group_type'] = 'sponsor' normal_poll['pollster_info']['pollster_list'].append(sponsor) normal_poll['pollster_info']['pollster_list_json'] = json.dumps(normal_poll['pollster_info']['pollster_list']) #create dict for poll stats normal_poll['poll_stats'] = {} normal_poll['poll_stats']['poll_method'] = poll['method'].lower().replace(' ', '_') #assign uuid for parent survey normal_poll['poll_info']['survey_uuid'] = 'dew-uuid-'+str(uuid.uuid1())[0:10] question_num = 0 #iterate through poll questions and generate normalized poll item responses for question in poll['questions']: question_num += 1 normal_poll_question = copy.deepcopy(normal_poll) subpop = 0 poll_topic = question['topic'] normal_poll_question['poll_info']['uuid'] = 'dew-uuid-'+str(uuid.uuid1())[0:10] #normalize region for the poll if question['state'] == 'US': normal_poll_question['poll_info']['poll_region'] = 'US' else: normal_poll_question['poll_info']['poll_region'] = question['state'] #determine party scope for poll question normal_poll_question['poll_info']['poll_name'] = question['name'] if not normal_poll_question['poll_info']['poll_region']: for abv, name in states.iteritems(): if name in normal_poll_question['poll_info']['poll_name']: normal_poll_question['poll_info']['poll_region'] = abv break #copy poll stats to question object normal_poll_question['poll_stats']['poll_sample'] = question['subpopulations'][subpop]['observations'] normal_poll_question['poll_stats']['poll_margin_of_error'] = question['subpopulations'][subpop]['margin_of_error'] #determine screen method for poll if "Likely" in question['subpopulations'][subpop]['name']: normal_poll_question['poll_stats']['poll_screen'] = "likely_voters" if "Registered" in question['subpopulations'][subpop]['name']: normal_poll_question['poll_stats']['poll_screen'] = "registered_voters" if "Adult" in question['subpopulations'][subpop]['name']: normal_poll_question['poll_stats']['poll_screen'] = "adult_respondents" #determine class of eletion (primary/general/etc) #as well as format of poll (head to head, hor race, etc# if 'vs' in question['name']: normal_poll_question['poll_stats']['poll_class'] = 'head_to_head' elif 'Approval' in question['name'] or 'Favorable' in question['name']: normal_poll_question['poll_stats']['poll_class'] = 'favorable' normal_poll_question['poll_info']['poll_office'] = None elif 'Primary' in question['name'] or 'Caucus' in question['name']: normal_poll_question['poll_stats']['poll_class'] = 'horse_race' normal_poll_question['poll_info']['poll_race'] = 'primary' elif 'General' in question['name'] and 'vs' not in question['name'] : normal_poll_question['poll_stats']['poll_class'] = 'horse_race' normal_poll_question['poll_info']['poll_race'] = 'general' elif 'GE' in question['name']: normal_poll_question['poll_stats']['poll_class'] = 'head_to_head' else: normal_poll_question['poll_stats']['poll_class'] = 'unknown' normal_poll_question['poll_info']['poll_office'] = None #determine the election year from topic if poll_topic: if '2016' in poll_topic: normal_poll_question['poll_info']['election_year'] = 2016 elif '2014' in poll_topic: normal_poll_question['poll_info']['election_year'] = 2014 elif '2012' in poll_topic: normal_poll_question['poll_info']['election_year'] = 2012 else: normal_poll_question['poll_info']['election_year'] = None #Determine the political office based on title or topic if 'Presidential' in question['name'] or 'President' in question['name']: normal_poll_question['poll_info']['poll_office'] = 'president' elif 'Senate' in question['name']: normal_poll_question['poll_info']['poll_office'] = 'senate' elif 'President' in question['name']: normal_poll_question['poll_info']['poll_office'] = 'president' else: normal_poll_question['poll_info']['poll_office'] = None if poll_topic: if 'senate' in poll_topic : normal_poll_question['poll_info']['poll_office'] = 'senate' elif 'governor' in poll_topic: normal_poll_question['poll_info']['poll_office'] = 'governor' elif 'president' in poll_topic: normal_poll_question['poll_info']['poll_office'] = 'president' else: normal_poll_question['poll_info']['poll_office'] = None #check if current question is party id, if so, copy party id data to parent survey if 'Identification' in question['name']: for current_poll in normal_poll_list: if current_poll['poll_info']['poll_source_id'] == poll['id']: current_poll['poll_stats']['poll_party_id'] = question['subpopulations'][subpop]['responses'] continue #normalize estimates(PollItem)s for head to head and horse race if normal_poll_question['poll_stats']['poll_class'] == 'head_to_head' or normal_poll_question['poll_stats']['poll_class'] == 'horse_race': normal_poll_question['estimates'] = normalize_horsetrack(question['subpopulations'][subpop]['responses'], normal_poll_question) #normalize estiamtes (PollItem)s for favorable polls elif normal_poll_question['poll_stats']['poll_class'] == 'favorable': normal_poll_question['estimates'] = normalize_favor(question['subpopulations'][subpop]['responses'], normal_poll_question) #normalize unknown class of poll using horserace else: normal_poll_question['estimates'] = normalize_horsetrack(question['subpopulations'][subpop]['responses'], normal_poll_question) #add populated poll question to main list normal_poll_list.append(normal_poll_question) return normal_poll_list
from models import db, Region, PoliticalPollQuestion, Pollster, PollItem, Politician, ElectionSummary, PoliticalPoll from models import get_or_create from normalizer import get_polls from sqlalchemy.exc import IntegrityError from politics import states, senate_class_3_states from datetime import datetime, timedelta from models import app, generate_presidential_snapshot, PollUpdateReport from sqlalchemy import desc import sys import pprint from operator import itemgetter from subprocess import call for abv, name in states.iteritems(): try: reg = Region(national = False, name = name, abv = abv, iso = "US-" + abv) db.session.add(reg) db.session.commit() #print reg except IntegrityError: db.session.rollback() try: reg = Region(national = True, name = "United States", abv = "US", iso = "US") db.session.add(reg) db.session.commit() except IntegrityError: db.session.rollback() #app.debug = False