def parse_petition(petitions):
for petition in petitions:
# get_petition_status(petition)
data = []
petition_url = 'https://petition.president.gov.ua/petition/' + str(petition)
pages = [petition_url + '/votes/' + str(i) for i in range(1, max_page(petition_url) + 1)]
for page in pages:
html = requests.get(page).text
soup = BeautifulSoup(html, 'lxml')
rows = soup.find_all('div', class_=re.compile(r'^table_row$'))
for r in rows:
position_number = r.find('div', class_='table_cell number').string.replace('.', '')
username = r.find('div', class_='table_cell name').string
day, month, year = r.find('div', class_='table_cell date').string.split(' ')
new_month = m.parse(month)[0].inflect({'nomn'}).word.title()
sign_date = datetime.strptime(' '.join([day, new_month, year]), '%d %B %Y')
data.append((petition, position_number, username, sign_date))
if Vote.select().where(Vote.petition == petition):
print(f'petition {petition} was in db with',
Vote.delete().where(Vote.petition == petition).execute(), 'rows')
with db.atomic():
# by default SQLite limits the number of bound variables in a SQL query to 999
for batch in chunked(data, 249):
Vote.insert_many(batch, fields=['petition', 'position_number', 'username', 'sign_date']).execute()
status = get_petition_status(petition)
Petition.update(status=status).where(Petition.petition_id == petition).execute()
if petitions:
set_gender()
def copy_db():
# Name_2, Petition_2, User
# db.drop_tables([Peticia], safe=True)
# db.create_tables([Peticia], safe=True)
# db.create_tables([Name], safe=True)
# query = Name_2.select()
# data = []
# for i in query:
# data.append((i.username, i.gender))
#
# with db.atomic():
# # by default SQLite limits the number of bound variables in a SQL query to 999
# for batch in chunked(data, 450):
# Name.insert_many(batch, fields=[Name.username, Name.gender]).execute()
#
#
#
# db.create_tables([Petition], safe=True)
# query = Petition_2.select()
# for i in query:
# Petition.create(
# petition_id=i.petition_id,
# status=i.status,
# title=i.title,
# article=i.article,
# answer=i.answer
# )
#
#
#
db.create_tables([Vote], safe=True)
for i in User.select(User.petition_id).distinct():
data = []
# p = Petition.get(petition_id=i.petition_id).petition_id
p = i.petition_id
query = User.select().where(User.petition == i.petition_id)
for user in query:
data.append((p, user.position_number, user.username,
user.sign_date, user.gender))
with db.atomic():
# by default SQLite limits the number of bound variables in a SQL query to 999
for batch in chunked(data, 198):
# User.insert_many(batch, fields=[User.petition, User.position_number,
# User.username, User.sign_date, User.gender]).execute()
Vote.insert_many(batch).execute()
