"SGDQ '11": {"start": datetime(2011, 8, 4), "end": datetime(2011, 8, 7)}, "AGDQ '12": {"start": datetime(2012, 1, 4), "end": datetime(2012, 1, 10)}, "SGDQ '12": {"start": datetime(2012, 5, 24), "end": datetime(2012, 5, 29)}, "AGDQ '13": {"start": datetime(2013, 1, 6), "end": datetime(2013, 1, 13)}, "SGDQ '13": {"start": datetime(2013, 7, 25), "end": datetime(2013, 7, 31)}, "AGDQ '14": {"start": datetime(2014, 1, 5), "end": datetime(2014, 1, 12)}, "SGDQ '14": {"start": datetime(2014, 6, 22), "end": datetime(2014, 6, 29)}, "AGDQ '15": {"start": datetime(2015, 1, 4), "end": datetime(2015, 1, 11)}, "SGDQ '15": {"start": datetime(2015, 7, 26), "end": datetime(2015, 8, 3)}, "AGDQ '16": {"start": datetime(2016, 1, 3), "end": datetime(2016, 1, 11)}, } for name, dates in gdq_list.items(): section_data = [] data = Donation.select().where(Donation.donation_date >= dates["start"], Donation.donation_date <= dates["end"]).order_by(Donation.donation_date).asc() event_length = (dates["end"] - dates["start"]).total_seconds() g = groupby(data, key=get_key) iter_count = 0 for key, items in g: total = sum([d.donation_amount for d in list(items)]) total = float(total) if iter_count > 0: total += section_data[-1][1] # Get percentage complete section = (key - dates["start"]).total_seconds() progress = (section / event_length) * 100
args = {"page": 1} while True: print("Parsing page: %s" % args["page"]) req = requests.get("https://gamesdonequick.com/tracker/donations/", params=args) if req.status_code != 200: print("Not found. Exiting...") break soup = BeautifulSoup(req.content, 'html.parser') table = soup.findAll("table", class_="table") rows = table[0].findAll("tr")[1:] for row in reversed(rows): td = row.findAll("td") donation_date = td[1].text.strip() donation_date = datetime.datetime.strptime(donation_date, "%m/%d/%Y %H:%M:%S +0000") donation_amount = float(td[2].text.strip()[1:].replace(",", "")) donation_url = td[2].find('a')["href"].rsplit("/", 1)[1] print("%s - %s - %s" % (donation_date, donation_amount, donation_url)) try: Donation.create(donation_date=donation_date, donation_amount=donation_amount, donation_id=donation_url) except peewee.IntegrityError: exit("Duplicate donation found") args["page"] += 1