def update_petitions(url):
petitions_to_parse = []
page_links = [f'{url}{i}' for i in range(1, 11)]
petition_links = get_petition_links(page_links)
first, second, third, fourth = 0, 0, 0, 0
parsed_petitions = [int(i.split('/')[-1]) for i in petition_links]
query_closed = Petition.select(
Petition.petition_id).where(~Petition.status)
closed = [i.petition_id for i in query_closed]
petitions_to_check = set(parsed_petitions) - set(closed) - {45758}
query_open = Petition.select(Petition.petition_id).where(Petition.status)
open_ = [i.petition_id for i in query_open]
for i in petitions_to_check:
first += 1
if i not in open_:
second += 1
print(i, 'not in db')
petitions_to_parse.append(i)
title, article, answer = get_petition_info(i)
Petition.create(petition_id=i,
title=title,
article=article,
answer=answer)
else:
third += 1
count_in_db = Vote.select().where(Vote.petition_id == i).count()
count = get_votes_count(
f'https://petition.president.gov.ua/petition/{i}')
if count - count_in_db > 1000:
fourth += 1
print(i, count_in_db, count)
petitions_to_parse.append(i)
print(first, second, third, fourth, sum((first, second, third, fourth)))
return petitions_to_parse
def post_vote(room_code):
sessid = request.cookies.get("sessid")
you = Player.select(Player, Room).join(Room).where(Player.sessid == sessid, Room.code == room_code).get()
room = you.room
vote = request.params["vote"]
mission_id = request.params["missionId"]
mission = Mission.get(Mission.id == mission_id, Mission.room == room)
Vote.insert(player=you, mission=mission, vote=vote).execute()
votes = Vote.select(Vote, Player).join(Player).where(Vote.mission == mission).execute()
if votes.count == room.size:
upvotes = sum(1 for v in votes if v.vote == "yes")
agreed = True if (upvotes * 2 > votes.count) else False
votes_message_data = {
"votes": [
{"player": {"id": vote.player.id, "name": vote.player.name}, "vote": vote.vote} for vote in votes
],
"agreed": agreed,
}
messages = [Msg("votes", votes_message_data)]
if agreed:
room.phase = Room.PHASE_MISSION
room.save()
else:
room.phase = Room.PHASE_TEAM_BUILD
leader = next_leader(room)
leader_message_data = {"leader": {"id": leader.id, "name": leader.name}}
messages.append(Msg("leader", leader_message_data))
post_message(messages, room=room)
def parse_petition(petitions):
for petition in petitions:
# get_petition_status(petition)
data = []
petition_url = 'https://petition.president.gov.ua/petition/' + str(petition)
pages = [petition_url + '/votes/' + str(i) for i in range(1, max_page(petition_url) + 1)]
for page in pages:
html = requests.get(page).text
soup = BeautifulSoup(html, 'lxml')
rows = soup.find_all('div', class_=re.compile(r'^table_row$'))
for r in rows:
position_number = r.find('div', class_='table_cell number').string.replace('.', '')
username = r.find('div', class_='table_cell name').string
day, month, year = r.find('div', class_='table_cell date').string.split(' ')
new_month = m.parse(month)[0].inflect({'nomn'}).word.title()
sign_date = datetime.strptime(' '.join([day, new_month, year]), '%d %B %Y')
data.append((petition, position_number, username, sign_date))
if Vote.select().where(Vote.petition == petition):
print(f'petition {petition} was in db with',
Vote.delete().where(Vote.petition == petition).execute(), 'rows')
with db.atomic():
# by default SQLite limits the number of bound variables in a SQL query to 999
for batch in chunked(data, 249):
Vote.insert_many(batch, fields=['petition', 'position_number', 'username', 'sign_date']).execute()
status = get_petition_status(petition)
Petition.update(status=status).where(Petition.petition_id == petition).execute()
if petitions:
set_gender()
def get_names(petition_n):
names = []
query = Vote.select(Vote.username).where(Vote.petition == petition_n)
for i in query:
try:
names.append(" ".join(i.username.split()).split(' ')[1])
except IndexError:
names.append(" ".join(i.username.split()).split(' ')[0])
return Counter(names)
def votes_json(track):
votes = Vote.select().where(Vote.track_uri == track.uri)
for vote in votes:
yield {
'user': vote.user.name,
'uid': vote.user.uid,
'picture': vote.user.picture,
'time': vote.timestamp.isoformat()
}
def set_gender():
query = Vote.select().where(Vote.gender.is_null())
query_genders = get_genders()
data = []
for i in query:
try:
name = " ".join(i.username.split()).split(' ')[1]
except IndexError:
name = " ".join(i.username.split()).split(' ')[0]
gender = query_genders.get(name, None)
if gender is not None:
if gender:
i.gender = True
data.append(i)
elif not gender:
i.gender = False
data.append(i)
with db.atomic():
Vote.bulk_update(data, fields=['gender'], batch_size=165)
def product(productid):
from models import Review, Vote, User
form = forms.ReviewForm()
product = int(productid)
reviews = (Review.select(Review.content, Review.product_id, User.id,
User.username, Review.id, Review.buy_again,
Review.helpful_votes,
Review.not_helpful_votes).join(User).where(
User.id == Review.user
and Review.product_id == productid)).where(
fn.length(Review.content) > 0)
buy_again_votes_total = reviews.count()
buy_again_votes_true = 0
for review in reviews:
if review.buy_again == 1:
buy_again_votes_true += 1
if buy_again_votes_total > 0:
buy_again_votes_percent = (buy_again_votes_true /
buy_again_votes_total) * 100
elif buy_again_votes_total == 0:
buy_again_votes_percent = 0
votes = (Vote.select(Vote.user, Vote.helpful,
Review.id).join(Review).where(
Vote.review == Review.id
and Review.product_id == productid))
voter_exists = False
if votes:
for vote in votes:
if vote.user.id == current_user.id:
voter_exists = True
if form.validate_on_submit() and 'POST':
if form.buy_again.data == True:
models.Review.create(user=g.user._get_current_object(),
buy_again=1,
content=form.content.data.strip(),
product_id=product,
helpful_votes=0,
not_helpful_votes=0)
flash("Review posted! Thanks!", "success")
return redirect(url_for('product', productid=productid))
else:
models.Review.create(user=g.user._get_current_object(),
buy_again=0,
content=form.content.data.strip(),
product_id=product,
helpful_votes=0,
not_helpful_votes=0)
flash("Review posted! Thanks!", "success")
return redirect(url_for('product', productid=productid))
elif request.method == 'POST':
models.List.create_list_item(current_user.id, productid)
return 'success'
return render_template('product.html',
form=form,
product=product,
reviews=reviews,
currentuser=g.user.id,
votes=votes,
buy_again_votes_percent=buy_again_votes_percent,
voter_exists=voter_exists)
def get_dates(petition_n):
query = Vote.select(Vote.sign_date).where(Vote.petition == petition_n)
dates = Counter([i.sign_date for i in query])
dates = OrderedDict(sorted(dates.items(), reverse=False))
return dates
def get_genders(petition_n):
query = Vote.select(Vote.gender).where(Vote.petition == petition_n)
genders = [i.gender for i in query]
return Counter(genders)