Exemple #1
0
def normalize_huffpo_poll_list(poll_list, poll_type):
	normal_poll_list = []
	huffpo_poll_id_list = []

	for poll in poll_list:
			#print poll	
			normal_poll = {}

			#copy basic poll info
			normal_poll['poll_info'] = {}
			normal_poll['poll_info']['poll_date'] = normalize_date(poll)
			normal_poll['poll_info']['poll_source_id'] = 3 * int(poll["id"])
			normal_poll['poll_info']['poll_source'] =  "ofhu"
			normal_poll['poll_info']['poll_source_uri'] =  poll["source"]
		

			#copy pollster info
			if poll['pollster']:	
				normal_poll['pollster_info'] = {'pollster_str' : poll['pollster'], 'pollster_partisan' : poll['partisan'].lower(), 'group_type' : 'pollster'}
				
			#remove party from naming
			normal_poll['pollster_info']['pollster_str'] = normal_poll['pollster_info']['pollster_str'].replace('(D)', '')
			normal_poll['pollster_info']['pollster_str'] = normal_poll['pollster_info']['pollster_str'].replace('(R)', '')
			
	
			normal_poll['pollster_info']['pollster_list'] = []
			normal_poll['pollster_info']['pollster_list_json'] = ''
			
			#iterate through "survey houses" and assign as pollster sponsors
			for house in poll['survey_houses']:
				house['party'] = house['party'].lower()
				house['party'] = house['party'].replace('n/a', 'nonpartisan')
				house['party'] = house['party'].replace('rep', 'gop')
				
	
				house['name'] = house['name'].replace('(D)', '')
				house['name'] = house['name'].replace('(R)', '')
				
				#normalize naming, deprecated
				if "PPP" in normal_poll['pollster_info']['pollster_str']:
					normal_poll['pollster_info']['pollster_str'] = "Public Policy Polling"
					
				house['group_type'] = 'pollster'
				
				normal_poll['pollster_info']['pollster_list'].append(house)
				
			for sponsor in poll['sponsors']:
				sponsor['party'] = sponsor['party'].lower()
				sponsor['party'] = sponsor['party'].replace('n/a', 'nonpartisan')
				sponsor['party'] = sponsor['party'].replace('rep', 'gop')
				
	
				sponsor['name'] = sponsor['name'].replace('(D)', '')
				sponsor['name'] = sponsor['name'].replace('(R)', '')
				
				#normalize naming, deprecated
				if "PPP" in normal_poll['pollster_info']['pollster_str']:
					normal_poll['pollster_info']['pollster_str'] = "Public Policy Polling"
					
				sponsor['group_type'] = 'sponsor'
				
				normal_poll['pollster_info']['pollster_list'].append(sponsor)
			
			normal_poll['pollster_info']['pollster_list_json'] = json.dumps(normal_poll['pollster_info']['pollster_list'])
	
			#create dict for poll stats
			normal_poll['poll_stats'] = {}
			normal_poll['poll_stats']['poll_method'] = poll['method'].lower().replace(' ', '_')
					
			#assign uuid for parent survey
			normal_poll['poll_info']['survey_uuid'] =  'dew-uuid-'+str(uuid.uuid1())[0:10]
		
			question_num = 0			
			
			#iterate through poll questions and generate normalized poll item responses 
			for question in poll['questions']:
					question_num += 1
					normal_poll_question = copy.deepcopy(normal_poll)
					subpop = 0
					
					
					poll_topic = question['topic']	

					normal_poll_question['poll_info']['uuid'] =  'dew-uuid-'+str(uuid.uuid1())[0:10]
					
					#normalize region for the poll
					if question['state'] == 'US':
						normal_poll_question['poll_info']['poll_region'] = 'US'
					else:
						normal_poll_question['poll_info']['poll_region'] = question['state']
					 
					#determine party scope for poll question
					
					

					normal_poll_question['poll_info']['poll_name'] = question['name']

					if not normal_poll_question['poll_info']['poll_region']:
						for abv, name in states.iteritems():
							if name in normal_poll_question['poll_info']['poll_name']:
								normal_poll_question['poll_info']['poll_region'] = abv
								break
								
					#copy poll stats to question object	
					normal_poll_question['poll_stats']['poll_sample'] = question['subpopulations'][subpop]['observations']	
					normal_poll_question['poll_stats']['poll_margin_of_error'] = question['subpopulations'][subpop]['margin_of_error']
				
					#determine screen method for poll
					if "Likely" in question['subpopulations'][subpop]['name']:
						normal_poll_question['poll_stats']['poll_screen'] = "likely_voters"
					if "Registered" in question['subpopulations'][subpop]['name']:
						normal_poll_question['poll_stats']['poll_screen'] = "registered_voters"
							
					if "Adult" in question['subpopulations'][subpop]['name']:
						normal_poll_question['poll_stats']['poll_screen'] = "adult_respondents"
					

					#determine class of eletion (primary/general/etc)
					#as well as format of poll (head to head, hor race, etc#
					if 'vs' in question['name']:
						normal_poll_question['poll_stats']['poll_class'] = 'head_to_head'
					elif 'Approval' in question['name'] or 'Favorable' in  question['name']:	
						normal_poll_question['poll_stats']['poll_class'] = 'favorable'	
						normal_poll_question['poll_info']['poll_office'] = None
					elif 'Primary' in question['name'] or 'Caucus' in question['name']:
						normal_poll_question['poll_stats']['poll_class'] = 'horse_race'
						normal_poll_question['poll_info']['poll_race'] = 'primary'
					elif 'General' in question['name'] and 'vs' not in question['name'] :
						normal_poll_question['poll_stats']['poll_class'] = 'horse_race'	
						normal_poll_question['poll_info']['poll_race'] = 'general'
					elif 'GE' in question['name']:
						normal_poll_question['poll_stats']['poll_class'] = 'head_to_head'
					else:
						normal_poll_question['poll_stats']['poll_class'] = 'unknown'	
						normal_poll_question['poll_info']['poll_office'] = None
					
					#determine the election year from topic
					if poll_topic:
						if '2016' in poll_topic:
							normal_poll_question['poll_info']['election_year'] = 2016
						elif '2014' in poll_topic:
							normal_poll_question['poll_info']['election_year'] = 2014
						elif '2012' in poll_topic:
							normal_poll_question['poll_info']['election_year'] = 2012
						else:
							normal_poll_question['poll_info']['election_year'] = None
					

					#Determine the political office based on title or topic	
					if 'Presidential' in question['name'] or 'President' in question['name']:
						normal_poll_question['poll_info']['poll_office'] = 'president'
					elif 'Senate' in question['name']:
						normal_poll_question['poll_info']['poll_office'] = 'senate'
					elif 'President' in question['name']:
						normal_poll_question['poll_info']['poll_office'] = 'president'
					else:	
						normal_poll_question['poll_info']['poll_office'] = None
					if poll_topic:
						if 'senate' in poll_topic :
							normal_poll_question['poll_info']['poll_office'] = 'senate'
						elif 'governor' in poll_topic:
							normal_poll_question['poll_info']['poll_office'] = 'governor'	
						elif 'president' in poll_topic:	
							normal_poll_question['poll_info']['poll_office'] = 'president'
						else:
							normal_poll_question['poll_info']['poll_office'] = None

					#check if current question is party id, if so, copy party id data to parent survey 
					if 'Identification' in question['name']:
						for current_poll in normal_poll_list:
							if current_poll['poll_info']['poll_source_id'] == poll['id']:
								current_poll['poll_stats']['poll_party_id'] = question['subpopulations'][subpop]['responses']	
						continue
					#normalize estimates(PollItem)s for head to head and horse race
					if normal_poll_question['poll_stats']['poll_class'] == 'head_to_head' or normal_poll_question['poll_stats']['poll_class'] == 'horse_race':
						normal_poll_question['estimates'] = normalize_horsetrack(question['subpopulations'][subpop]['responses'], normal_poll_question)
					#normalize estiamtes (PollItem)s for favorable polls
					elif normal_poll_question['poll_stats']['poll_class'] == 'favorable':	
						normal_poll_question['estimates'] = normalize_favor(question['subpopulations'][subpop]['responses'], normal_poll_question)
					#normalize unknown class of poll using horserace
					else:
						normal_poll_question['estimates'] = normalize_horsetrack(question['subpopulations'][subpop]['responses'], normal_poll_question)
					#add populated poll question to main list
					normal_poll_list.append(normal_poll_question)
				
	return normal_poll_list
Exemple #2
0
from models import db, Region, PoliticalPollQuestion, Pollster, PollItem, Politician, ElectionSummary, PoliticalPoll
from models import  get_or_create
from normalizer import get_polls
from sqlalchemy.exc import IntegrityError
from politics import states, senate_class_3_states
from datetime import datetime, timedelta
from models import app, generate_presidential_snapshot, PollUpdateReport
from sqlalchemy import desc
import sys
import pprint 
from operator import itemgetter
from subprocess import call

for abv, name in states.iteritems():
	try:
		reg = Region(national = False, name = name, abv = abv, iso = "US-" + abv)
		db.session.add(reg)
		db.session.commit()
		#print reg
	except IntegrityError:
		db.session.rollback()
		
try:	
	reg = Region(national = True, name = "United States", abv = "US", iso = "US")
	db.session.add(reg)
	db.session.commit()
except IntegrityError:
		db.session.rollback()
	
#app.debug = False