def parse_petition(petitions): for petition in petitions: # get_petition_status(petition) data = [] petition_url = 'https://petition.president.gov.ua/petition/' + str(petition) pages = [petition_url + '/votes/' + str(i) for i in range(1, max_page(petition_url) + 1)] for page in pages: html = requests.get(page).text soup = BeautifulSoup(html, 'lxml') rows = soup.find_all('div', class_=re.compile(r'^table_row$')) for r in rows: position_number = r.find('div', class_='table_cell number').string.replace('.', '') username = r.find('div', class_='table_cell name').string day, month, year = r.find('div', class_='table_cell date').string.split(' ') new_month = m.parse(month)[0].inflect({'nomn'}).word.title() sign_date = datetime.strptime(' '.join([day, new_month, year]), '%d %B %Y') data.append((petition, position_number, username, sign_date)) if Vote.select().where(Vote.petition == petition): print(f'petition {petition} was in db with', Vote.delete().where(Vote.petition == petition).execute(), 'rows') with db.atomic(): # by default SQLite limits the number of bound variables in a SQL query to 999 for batch in chunked(data, 249): Vote.insert_many(batch, fields=['petition', 'position_number', 'username', 'sign_date']).execute() status = get_petition_status(petition) Petition.update(status=status).where(Petition.petition_id == petition).execute() if petitions: set_gender()
def setup(): # clear tables first... Vote.delete().execute() Proposal.delete().execute() Superblock.delete().execute() GovernanceObject.delete().execute()
def setup(): # clear tables first Vote.delete().execute() Proposal.delete().execute() GovernanceObject.delete().execute()
def track_playback_ended(self, tl_track, time_position): # Remove old votes Vote.delete().where(Vote.track_uri == tl_track.track.uri).execute()