"SGDQ '11": {"start": datetime(2011, 8, 4), "end": datetime(2011, 8, 7)},
"AGDQ '12": {"start": datetime(2012, 1, 4), "end": datetime(2012, 1, 10)},
"SGDQ '12": {"start": datetime(2012, 5, 24), "end": datetime(2012, 5, 29)},
"AGDQ '13": {"start": datetime(2013, 1, 6), "end": datetime(2013, 1, 13)},
"SGDQ '13": {"start": datetime(2013, 7, 25), "end": datetime(2013, 7, 31)},
"AGDQ '14": {"start": datetime(2014, 1, 5), "end": datetime(2014, 1, 12)},
"SGDQ '14": {"start": datetime(2014, 6, 22), "end": datetime(2014, 6, 29)},
"AGDQ '15": {"start": datetime(2015, 1, 4), "end": datetime(2015, 1, 11)},
"SGDQ '15": {"start": datetime(2015, 7, 26), "end": datetime(2015, 8, 3)},
"AGDQ '16": {"start": datetime(2016, 1, 3), "end": datetime(2016, 1, 11)},
}
for name, dates in gdq_list.items():
section_data = []
data = Donation.select().where(Donation.donation_date >= dates["start"], Donation.donation_date <= dates["end"]).order_by(Donation.donation_date).asc()
event_length = (dates["end"] - dates["start"]).total_seconds()
g = groupby(data, key=get_key)
iter_count = 0
for key, items in g:
total = sum([d.donation_amount for d in list(items)])
total = float(total)
if iter_count > 0:
total += section_data[-1][1]
# Get percentage complete
section = (key - dates["start"]).total_seconds()
progress = (section / event_length) * 100
args = {"page": 1}
while True:
print("Parsing page: %s" % args["page"])
req = requests.get("https://gamesdonequick.com/tracker/donations/", params=args)
if req.status_code != 200:
print("Not found. Exiting...")
break
soup = BeautifulSoup(req.content, 'html.parser')
table = soup.findAll("table", class_="table")
rows = table[0].findAll("tr")[1:]
for row in reversed(rows):
td = row.findAll("td")
donation_date = td[1].text.strip()
donation_date = datetime.datetime.strptime(donation_date, "%m/%d/%Y %H:%M:%S +0000")
donation_amount = float(td[2].text.strip()[1:].replace(",", ""))
donation_url = td[2].find('a')["href"].rsplit("/", 1)[1]
print("%s - %s - %s" % (donation_date, donation_amount, donation_url))
try:
Donation.create(donation_date=donation_date, donation_amount=donation_amount, donation_id=donation_url)
except peewee.IntegrityError:
exit("Duplicate donation found")
args["page"] += 1