def test_get_polling_data_by_name(self):
polls = get_polls(candidate="trump")
first = polls[0]
for p in polls:
self.assertIn("trump", p["title"].lower())
td = get_poll_data(first["url"])
self.assertIsNotNone(td)
def test_to_csv(self):
csv_file = "output.csv"
success = False
polls = get_polls(candidate="Biden")[0]
data = get_poll_data(polls["url"], csv_output=True)
to_csv("output.csv", data)
cur_dir = os.getcwd()
file_list = os.listdir(cur_dir)
if csv_file in file_list:
success = True
os.remove(csv_file)
self.assertTrue(success)
from rcp import get_poll_data
from prettytable import PrettyTable
x = PrettyTable()
td = get_poll_data(
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_biden-6247.html"
)
x.field_names = list(td[0]["data"][0].keys())
x.align = "l"
for row in td[0]["data"]:
x.add_row(row.values())
print(x.field_names)
from django.shortcuts import render, get_object_or_404
from .models import Post, Comment
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.views.generic import ListView
from django.contrib.postgres.search import SearchVector, SearchQuery, SearchRank
from .forms import CommentForm, SearchForm
from rcp import get_polls
from rcp import get_poll_data
from prettytable import PrettyTable
td = get_poll_data(
'https://www.realclearpolitics.com/epolls/2020/president/nc/north_carolina_trump_vs_biden-6744.html'
)
field_names = list(td[0]["data"][0].keys())
field_values = list(td[0]["data"][0].values())
#for row in td[0]["data"]:
# x.add_row(row.values())
#print(x)
def post_share(request, post_id):
post = get_object_or_404(Post, id=post_id, status='published')
if request.method == 'POST':
form = EmailPostForm(request.POST)
if form.is_valid():
cd = form.cleaned_data
else:
from rcp import get_polls, get_poll_data
from pprint import pprint
polls = get_polls(q="Trump", p="Fox")
for poll in polls:
td = get_poll_data(poll['url'])
pprint(td)
def get_rcp_averages(filename):
"""
Loads all election data from a JSON file, collects the
RCP averages, then constructs an HTML file for displaying
the results on a static website.
"""
# Load the election data from the JSON file specified
with open(filename, "r") as handle:
elections = json.load(handle)
# Define the base RCP URL to simplify the elections parameter
base_url = "https://www.realclearpolitics.com/epolls/"
# Unpack the election data to check each race
for election in elections.values():
for place, race in election.items():
# If the poll is not enabled, do nothing and loop again
if not race["enable"]:
continue
# Get the poll data by combining the base URL with a the race URL
polls = get_poll_data(poll=base_url + race["url"])
# Sometimes RCP removes polls after they've been published.
# If no poll data is returned, so display a sensible message
if len(polls[0]["data"]) == 0:
race["text"] = f"{place}: No polls conducted"
race["color"] = "black"
continue
# There are polls; extract the first element
rcp_avg = polls[0]["data"][0]
# The first element should always be the RCP average. If it isn't,
# then the RCP average doesn't exist, so don't display anything
if not rcp_avg["Poll"].startswith("RCP"):
race["text"] = f"{place}: Did not find RCP average"
race["color"] = "black"
continue
# The RCP average does exist; find out the candidate's party
party = get_party(rcp_avg)
# Update the race dictionary with the text to display
spread, date = rcp_avg["Spread"], rcp_avg["Date"].replace(" ", "")
race["text"] = f"{place}: {party} {spread} ({date})"
race["color"] = COLOR_MAP[party]
# Print the final elections data for a quick visual check (optional)
# print(json.dumps(elections, indent=2))
# Setup the jinja2 templating environment and instantiate the template
j2_env = Environment(loader=FileSystemLoader("."), trim_blocks=True)
template = j2_env.get_template("index.j2")
# Build a pretty timestamp and render the template
updated = datetime.utcnow().strftime("%H:%M UTC on %d %B %Y")
j2_data = {
"elections": elections,
"updated": updated,
"base_url": base_url
}
html_text = template.render(data=j2_data)
# Return the HTML text
return html_text
from rcp import get_polls, get_poll_data
from pprint import pprint
polls = get_polls(candidate="Trump", pollster="Fox")
for poll in polls:
td = get_poll_data(poll["url"])
pprint(td)
from rcp import get_polls, get_poll_data, to_csv
polls = get_polls(q="Biden")[0]
data = get_poll_data(polls['url'])
to_csv('output.csv', data)
def test_get_polling_data(self):
p = get_polls()
polling_data = get_poll_data(p[0]['url'], d=True)
self.assertIsNotNone(polling_data)
self.assertIn('poll', polling_data[0])
self.assertIn('data', polling_data[0])
def get_poll_result(team_name, national=True):
team = s.query(Team).filter_by(name=team_name).first()
states = []
try:
for st in team.get_states():
states.append(us.states.lookup(st))
except Exception as e:
pass
poll_table = PrettyTable()
poll_table.border = False
poll_table.field_names = ["Poll", "Biden", "Trump", "Sp"]
message = ""
if national:
td = get_poll_data(
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_biden-6247.html"
)
# pa = get_poll_data("https://www.realclearpolitics.com/epolls/2020/president/pa/pennsylvania_trump_vs_biden-6861.html")
# ky = get_poll_data("https://www.realclearpolitics.com/epolls/2020/president/ky/kentucky_trump_vs_biden-6915.html")
# battle_grounds = "https://www.realclearpolitics.com/json/battleground_script/key_battleground_states_2020_spread_average_oct_23.json"
# print(pa)
for row in td[0]["data"]:
if row['Poll'] == 'RCP Average':
message += f'\n\nNational Real Clear Politics Average:\n```' \
f'Date: {row["Date"]}\n' \
f'Biden: {row["Biden (D)"]} ' \
f'Trump: {row["Trump (R)"]}\n' \
f'Spread: {row["Spread"]}'
message += "```\n\n"
try:
for state in states:
spread_total = 0
row_count = 0
biden_total = 0
trump_total = 0
spread = 0
message += f"{state} Polls:\n```"
for row in get_polls(state=str(state)):
poll_name = (row["poll"][:13] +
'..') if len(row['poll']) > 13 else row['poll']
if row['result'].split(",")[0].strip().split(
" ")[0] == 'Biden':
biden = int(
row['result'].split(",")[0].strip().split(" ")[1])
trump = int(
row['result'].split(",")[1].strip().split(" ")[1])
elif row['result'].split(",")[0].strip().split(
" ")[0] == 'Trump':
biden = int(
row['result'].split(",")[1].strip().split(" ")[1])
trump = int(
row['result'].split(",")[0].strip().split(" ")[1])
else:
continue
biden_total = biden + biden_total
trump_total = trump + trump_total
if biden > trump:
spread = biden - trump
biden = f"{biden}+"
else:
spread = trump - biden
trump = f"{trump}+"
poll_table.add_row([poll_name, biden, trump, spread])
row_count += 1
if row_count > 1:
if biden_total > trump_total:
spread_total = round(biden_total / row_count, 3) - round(
trump_total / row_count, 3)
biden_total = f"{round(biden_total/row_count, 1)}+"
trump_total = round(trump_total / row_count, 1)
else:
spread_total = round(trump_total / row_count, 3) - round(
biden_total / row_count, 3)
biden_total = round(biden_total / row_count, 1)
trump_total = f"{round(trump_total / row_count, 1)}+"
poll_table.add_row([
"Total", biden_total, trump_total,
round(spread_total, 2)
])
poll_table.align = "l"
# poll_table.sortby = '#'
# poll_table.reversesort = True
message += poll_table.get_string()
message += "```\n\n"
poll_table.clear_rows()
except UnboundLocalError:
message += "Set Team Local to get State Polling Data"
except ZeroDivisionError:
pass
# for row in pa[0]["data"]:
# if row['Poll'] == 'RCP Average':
# message += f'Pennsylvania Real Clear Politics Average:\n' \
# f'Date: {row["Date"]}\n' \
# f'Biden: {row["Biden (D)"]} ' \
# f'Trump: {row["Trump (R)"]} ' \
# f'Spread: {row["Spread"]}'
s.close()
return message
def test_get_polling_data(self):
p = get_polls()
polling_data = get_poll_data(p[0]["url"], csv_output=True)
self.assertIsNotNone(polling_data)
self.assertIn("Poll", polling_data[0])
self.assertIn("Date", polling_data[0])
def test_create_table(self):
td = get_poll_data(
"https://www.realclearpolitics.com/epolls/2020/president/me/maine_trump_vs_biden-6922.html"
)
self.assertIsNotNone(create_table(td, html_format=True))
def test_get_polling_data_invalid_url(self):
polling_data = get_poll_data("https://www.rcp.com")
self.assertFalse(polling_data)
def get_real_clear_politics(candidates, cur, conn):
cur.execute(
"SELECT count(name) FROM sqlite_master WHERE type=? AND name=?",
('table', 'DemPrimary'))
if cur.fetchone()[0] != 1:
cur.execute(
"CREATE TABLE DemPrimary (id INT PRIMARY KEY, name TEXT, percent REAL)"
)
data = get_poll_data(
'https://www.realclearpolitics.com/epolls/2020/president/us/2020_democratic_presidential_nomination-6730.html',
csv_output=False)
test_data = get_poll_data(
'https://www.realclearpolitics.com/epolls/2020/president/mi/michigan_trump_vs_sanders-6768.html'
)
trump_links = {
"Joe Biden": [
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_biden-6247.html",
"https://www.realclearpolitics.com/epolls/2020/president/nh/new_hampshire_trump_vs_biden-6779.html",
"https://www.realclearpolitics.com/epolls/2020/president/ca/california_trump_vs_biden-6755.html",
"https://www.realclearpolitics.com/epolls/2020/president/wi/wisconsin_trump_vs_biden-6849.html",
"https://www.realclearpolitics.com/epolls/2020/president/nv/nevada_trump_vs_biden-6867.html",
"https://www.realclearpolitics.com/epolls/2020/president/nc/north_carolina_trump_vs_biden-6744.html",
"https://www.realclearpolitics.com/epolls/2020/president/pa/pennsylvania_trump_vs_biden-6861.html"
],
"Bernie Sanders": [
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_sanders-6250.html",
"https://www.realclearpolitics.com/epolls/2020/president/nh/new_hampshire_trump_vs_sanders-6780.html",
"https://www.realclearpolitics.com/epolls/2020/president/ca/california_trump_vs_sanders-6880.html",
"https://www.realclearpolitics.com/epolls/2020/president/wi/wisconsin_trump_vs_sanders-6850.html",
"https://www.realclearpolitics.com/epolls/2020/president/nv/nevada_trump_vs_sanders-6868.html",
"https://www.realclearpolitics.com/epolls/2020/president/nc/north_carolina_trump_vs_sanders-6745.html"
],
"Elizabeth Warren": [
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_warren-6251.html",
"https://www.realclearpolitics.com/epolls/2020/president/nh/new_hampshire_trump_vs_warren-6781.html",
"https://www.realclearpolitics.com/epolls/2020/president/ca/california_trump_vs_warren-6756.html",
"https://www.realclearpolitics.com/epolls/2020/president/wi/wisconsin_trump_vs_warren-6852.html",
"https://www.realclearpolitics.com/epolls/2020/president/nv/nevada_trump_vs_warren-6870.html",
"https://www.realclearpolitics.com/epolls/2020/president/nc/north_carolina_trump_vs_warren-6746.html"
],
"Pete Buttigieg": [
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_buttigieg-6872.html",
"https://www.realclearpolitics.com/epolls/2020/president/nh/new_hampshire_trump_vs_buttigieg-6981.html",
"https://www.realclearpolitics.com/epolls/2020/president/ca/california_trump_vs_buttigieg-6937.html",
"https://www.realclearpolitics.com/epolls/2020/president/wi/wisconsin_trump_vs_buttigieg-6970.html",
"https://www.realclearpolitics.com/epolls/2020/president/nv/nevada_trump_vs_buttigieg-6871.html",
"https://www.realclearpolitics.com/epolls/2020/president/nc/north_carolina_trump_vs_buttigieg-6907.html"
],
"Kamala Harris": [
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_harris-6252.html",
"https://www.realclearpolitics.com/epolls/2020/president/ca/california_trump_vs_harris-6759.html",
"https://www.realclearpolitics.com/epolls/2020/president/ga/georgia_trump_vs_harris-6978.html"
],
"Andrew Yang": [
"https://www.realclearpolitics.com/epolls/2020/president/nh/new_hampshire_trump_vs_yang-6947.html"
],
"Amy Klobuchar": [
"https://www.realclearpolitics.com/epolls/2020/president/wi/wisconsin_trump_vs_klobuchar-6854.html"
],
"Mike Bloomberg": [
"https://www.realclearpolitics.com/epolls/2020/president/ca/california_trump_vs_bloomberg-6985.html",
"https://www.realclearpolitics.com/epolls/2020/president/us/general_election_trump_vs_bloomberg-6797.html"
],
"Cory Booker": [
"https://www.realclearpolitics.com/epolls/2020/president/wi/wisconsin_trump_vs_booker-6982.html"
]
}
candidate_count = 0
poll_count = 0
cur.execute("SELECT * from DemPrimary WHERE name =?", ('Joe Biden', ))
current_prim_source = len(cur.fetchall())
cur.execute("SELECT * from DemPrimary")
prim_count = len(cur.fetchall())
if (current_prim_source < len(data[0]['data'])):
for candidate in candidates:
name = candidate[0]
if (name != "Donald Trump"):
surname = re.findall(r"\w+\s(\w+)", name)[0]
prim_percent = data[0]['data'][current_prim_source].get(
surname)
if ((prim_percent == None) or (prim_percent == '--')):
continue
cur.execute(
"INSERT INTO DemPrimary (id, name, percent) VALUES (?,?,?)",
(prim_count, name, prim_percent))
prim_count += 1
else:
starting_link = ""
found_start = True
cur.execute(
"SELECT count(name) FROM sqlite_master WHERE type=? AND name=?",
('table', 'DemGeneral'))
if cur.fetchone()[0] != 1:
cur.execute(
"CREATE TABLE DemGeneral (id INT PRIMARY KEY, name TEXT, dem_percent REAL, trump_percent REAL, url TEXT)"
)
else:
cur.execute("SELECT url FROM DemGeneral ORDER BY id DESC")
starting_link = cur.fetchone()[0]
found_start = False
cur.execute("SELECT * from DemGeneral")
past_total = len(cur.fetchall())
poll_count = past_total
for candidate in candidates:
starting_index = 0
if ((poll_count - past_total) >= 20):
continue
name = candidate[0]
print(name + ": " + str(poll_count - past_total))
if (name != "Donald Trump"):
surname = re.findall(r"\w+\s(\w+)", name)[0]
if (trump_links.get(name) == None):
continue
if ((found_start == False)
and (starting_link in trump_links[name])):
found_start = True
for i in range(len(trump_links[name])):
if (trump_links[name][i] == starting_link):
starting_index = i + 1
elif ((found_start == False)
and (starting_link not in trump_links[name])):
continue
if (starting_index >= len(trump_links[name])):
continue
remaining_polls = trump_links[name][starting_index:]
# print(remaining_polls)
for poll in remaining_polls:
trump_data = get_poll_data(poll)
for poll_source in trump_data[0]['data']:
if ((poll_count - past_total) >= 20):
continue
dem_percent = poll_source.get(surname + ' (D)', 0.0)
trump_percent = poll_source['Trump (R)']
cur.execute(
"INSERT INTO DemGeneral (id, name, dem_percent, trump_percent, url) VALUES (?,?,?,?,?)",
(poll_count, name, dem_percent, trump_percent,
poll))
poll_count += 1
conn.commit()
from rcp import get_poll_data, create_table
td = get_poll_data(
"https://www.realclearpolitics.com/epolls/2020/president/me/maine_trump_vs_biden-6922.html"
)
print(create_table(td, html_format=True))
from rcp import get_polls, get_poll_data, to_csv
polls = get_polls(candidate="Biden")[0]
data = get_poll_data(polls["url"], csv_output=True)
to_csv("output.csv", data)
def test_get_polling_data_invalid_url(self):
polling_data = get_poll_data('https://www.rcp.com')
self.assertIsNone(polling_data)
