def even_if_emailed_not_invited_candidacies():
log("Getting all candidacies")
fs = Candidacy.all().filter("deleted = ", False).filter("survey_invite_posted = ", False).filter("survey_filled_in =", False).fetch(100)
while fs:
for f in fs:
yield f
fs = Candidacy.all().filter("deleted = ", False).filter("survey_invite_posted = ", False).filter("survey_filled_in =", False).filter('__key__ >', fs[-1].key()).fetch(100)
def emailed_candidacies():
log("Getting all candidacies")
fs = Candidacy.all().filter("survey_invite_emailed =", True).fetch(100)
while fs:
for f in fs:
yield f
fs = Candidacy.all().filter("survey_invite_emailed =", True).filter('__key__ >', fs[-1].key()).fetch(100)
def all_candidacies():
log("Getting all candidacies")
fs = Candidacy.all().filter("deleted = ", False).fetch(100)
all = []
while fs:
for f in fs:
all.append(f)
fs = Candidacy.all().filter("deleted = ", False).filter("__key__ >", fs[-1].key()).fetch(100)
return all
def emailed_candidacies():
log("Getting all candidacies")
fs = Candidacy.all().filter("survey_invite_emailed =", True).fetch(100)
while fs:
for f in fs:
yield f
fs = Candidacy.all().filter("survey_invite_emailed =",
True).filter('__key__ >',
fs[-1].key()).fetch(100)
def all_candidacies():
log("Getting all candidacies")
fs = Candidacy.all().filter("deleted = ", False).fetch(100)
all = []
while fs:
for f in fs:
all.append(f)
fs = Candidacy.all().filter("deleted = ", False).filter('__key__ >', fs[-1].key()).fetch(100)
return all
def lookup_candidacies_by_id():
log("Getting all candidacies")
fs = Candidacy.all().fetch(100)
candidacies_by_id = {}
c = 0
while fs:
log(" getting batch from " + str(c))
for f in fs:
c = c + 1
candidacies_by_id[str(f.key())] = f
fs = Candidacy.all().filter('__key__ >', fs[-1].key()).fetch(100)
return candidacies_by_id
def even_if_emailed_not_invited_candidacies():
log("Getting all candidacies")
fs = Candidacy.all().filter("deleted = ", False).filter(
"survey_invite_posted = ", False).filter("survey_filled_in =",
False).fetch(100)
while fs:
for f in fs:
yield f
fs = Candidacy.all().filter("deleted = ", False).filter(
"survey_invite_posted = ",
False).filter("survey_filled_in =",
False).filter('__key__ >', fs[-1].key()).fetch(100)
def task_invite_candidacy_survey(request, candidacy_key_name):
candidacy = Candidacy.get_by_key_name(candidacy_key_name)
if candidacy.survey_invite_emailed:
return HttpResponse("Given up, already emailed")
if candidacy.survey_filled_in:
return HttpResponse("Given up, already filled in")
# Get email and name
to_email = candidacy.candidate.validated_email()
if not to_email:
candidacy.log("Abandoned sending survey invite email as email address invalid")
return HttpResponse("Given up, email address invalid")
to_name = candidacy.candidate.name
# Generate candidate auth login URL
if not candidacy.survey_token:
candidacy.generate_survey_token()
url = settings.EMAIL_URL_PREFIX + "/survey/" + candidacy.survey_token
message = mail.EmailMessage()
message.sender = settings.TEAM_FROM_EMAIL
message.subject = "Help your future constituents know your views"
message.to = email.utils.formataddr((to_name, to_email))
message.body = """Hi %s,
TheyWorkForYou is inviting all MP candidates to share their positions
on a range of major national and local issues.
Click this link, it should only take you a few minutes.
%s
Millions of people use TheyWorkForYou to find out about their MPs
every year. Your answers will be used in a quiz to help voters in
your constituency decide who to vote for. If you are elected,
your page on TheyWorkForYou will include your answers, and note
which new MPs declined to participate.
What is unique about this survey is that the local issues, and
indeed much of the entire project, have been provided by a
network of over 5000 new volunteers, under the banner of
Democracy Club. These individuals are looking to you, as a
candidate, to embody the accountability that everyone wants to
see from the new Parliament.
TheyWorkForYou team
on behalf of the voters of %s constituency
""" % (candidacy.candidate.name, url, candidacy.seat.name)
candidate_name = "candidate <strong>%s</strong> in seat <strong>%s</strong>.</p>" % (candidacy.candidate.name, candidacy.seat.name)
message.send()
candidacy.survey_invite_emailed = True
candidacy.survey_invite_sent_to_emails.append(to_email)
candidacy.log("Sent survey invite email to " + to_email)
text = "</p>Survey invitation sent to %s</p>" % candidate_name
text += "<pre>%s</pre>" % str(message.body)
return HttpResponse(text)
def survey_autosave(request, token):
candidacy = Candidacy.find_by_token(token)
if not candidacy:
raise Exception("Invalid token " + token)
candidacy.survey_autosave = request.POST['ser']
candidacy.survey_autosave_when = datetime.datetime.now()
candidacy.put()
return render_to_response('survey_autosave_ok.html')
def survey_autosave(request, token):
if settings.SURVEY_RESPONSES_CLOSED:
raise Exception("Survey closed to new responses")
candidacy = Candidacy.find_by_token(token)
if not candidacy:
raise Exception("Invalid token " + token)
candidacy.survey_autosave = request.POST['ser']
candidacy.survey_autosave_when = datetime.datetime.now()
candidacy.put()
return render_to_response('survey_autosave_ok.html')
def _check_auth(post, ip_address, first_auth):
form = forms.AuthCandidacyForm(post or None)
if not post:
return render_to_response('survey_candidacy_auth.html', { 'form': form })
token = post['token']
candidacy = Candidacy.find_by_token(token)
if not candidacy:
return render_to_response('survey_candidacy_auth.html', { 'form': form, 'error': True })
if first_auth:
if not candidacy.survey_token_use_count:
candidacy.survey_token_use_count = 0
candidacy.survey_token_use_count += 1
candidacy.log('Survey token authenticated from IP %s' % ip_address)
return candidacy
def load_from_ynmp(ynmp, frozen_seats):
# Put parties in datastore - don't worry about deleted ones, they just
# won't be referenced by other tables.
parties_by_key = {}
for party_id, party_data in ynmp["Party"].iteritems():
key_name = party_id
party = Party(
ynmp_id = int(party_id),
name = party_data["name"],
code = party_data["code"],
image_id = int_or_null(party_data["image_id"]),
created = convdate(party_data["created"]),
updated = convdate(party_data["updated"]),
key_name = key_name
)
log(" Storing party " + party.name)
parties_by_key[key_name] = party
log("Putting all parties")
put_in_batches(parties_by_key.values())
# Put candidates in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
candidates_by_key = {}
for candidate_id, candidate_data in ynmp["Candidate"].iteritems():
if "status" not in candidate_data:
raise Exception("No status entry for " + str(candidate_data))
key_name = candidate_id
candidate = Candidate(
ynmp_id = int(candidate_id),
name = candidate_data["name"],
code = candidate_data["code"],
status = candidate_data["status"],
email = candidate_data["email"],
address = candidate_data["address"],
party = parties_by_key[candidate_data["party_id"]],
image_id = int_or_null(candidate_data["image_id"]),
created = convdate(candidate_data["created"]),
updated = convdate(candidate_data["updated"]),
key_name = key_name
)
log(" Storing candidate " + candidate.name)
candidates_by_key[key_name] = candidate
log("Putting all candidates")
put_in_batches(candidates_by_key.values())
# Put seats in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
seats_by_key = {}
for seat_id, seat_data in ynmp["Seat"].iteritems():
key_name = seat_id
seat = Seat(
ynmp_id = int(seat_id),
name = seat_data["name"],
code = seat_data["code"],
created = convdate(seat_data["created"]),
updated = convdate(seat_data["updated"]),
key_name = key_name
)
if key_name in frozen_seats:
seat.frozen_local_issues = True
log(" Storing seat " + seat.name)
seats_by_key[key_name] = seat
seats_by_name[seat.name] = seat
log("Putting all seats")
put_in_batches(seats_by_key.values())
# Get list of existing candiacies in remote datastore
# in batches due to 100 entity at a time limit, as per http://code.google.com/appengine/articles/remote_api.html
log("Getting list of Candidacies")
candidacies = Candidacy.all().filter("deleted =", False).fetch(100)
to_be_marked_deleted = {}
while candidacies:
for candidacy in candidacies:
key_name = candidacy.key().name()
log("Marking before have candidacy key " + key_name)
to_be_marked_deleted[key_name] = candidacy
candidacies = Candidacy.all().filter("deleted =", False).filter('__key__ >', candidacies[-1].key()).fetch(100)
# Loop through new dump of candidacies from YourNextMP, adding new ones
candidacies_by_key = {}
for candidacy_id, candidacy_data in ynmp["Candidacy"].iteritems():
candidate = candidates_by_key[candidacy_data["candidate_id"]]
assert candidate.status in ['standing', 'standing_down', 'not-standing']
if candidate.status == 'standing_down' or candidate.status == 'not-standing':
continue
key_name = candidacy_data["seat_id"] + "-" + candidacy_data["candidate_id"]
# find existing entry if there is one, or else make new one
if key_name in to_be_marked_deleted:
candidacy = to_be_marked_deleted[key_name]
else:
candidacy = Candidacy(key_name = key_name)
# fill in values
candidacy.ynmp_id = int(candidacy_id)
candidacy.seat = seats_by_key[candidacy_data["seat_id"]]
candidacy.candidate = candidate
candidacy.created = convdate(candidacy_data["created"])
candidacy.updated = convdate(candidacy_data["updated"])
candidacy.deleted = False
# make sure it has a survey token
if not candidacy.survey_token:
log("Generating survey token for " + candidacy.seat.name + " " + candidacy.candidate.name)
candidacy.generate_survey_token() # this does save too, since it logs
if candidacy.survey_invite_posted == None:
candidacy.survey_invite_posted = False
log("Storing candidacy " + candidacy.seat.name + " " + candidacy.candidate.name)
candidacies_by_key[key_name] = candidacy
# record we still have this candidacy
if key_name in to_be_marked_deleted:
del to_be_marked_deleted[key_name]
log("Putting all candidacies")
put_in_batches(candidacies_by_key.values())
# See which candidacies are left, i.e. are deleted
for key_name, candidacy in to_be_marked_deleted.iteritems():
log("Marking deleted " + candidacy.seat.name + " " + candidacy.candidate.name)
candidacy.deleted = True
log("Putting marked deleted candidacies")
put_in_batches(to_be_marked_deleted.values())
######################################################################
# Load from DemocracyClub
fs = Seat.all().filter("frozen_local_issues =", True).fetch(100)
while fs:
for f in fs:
log(" Seat is frozen to local issues changes: " + f.name)
frozen_seats[f.key().name()] = f
fs = Seat.all().filter("frozen_local_issues =",True).filter('__key__ >', fs[-1].key()).fetch(100)
def load_from_ynmp(ynmp, frozen_seats):
# Put parties in datastore - don't worry about deleted ones, they just
# won't be referenced by other tables.
parties_by_key = {}
for party_id, party_data in ynmp["Party"].iteritems():
key_name = party_id
party = Party(ynmp_id=int(party_id),
name=party_data["name"],
code=party_data["code"],
image_id=int_or_null(party_data["image_id"]),
created=convdate(party_data["created"]),
updated=convdate(party_data["updated"]),
key_name=key_name)
log(" Storing party " + party.name)
parties_by_key[key_name] = party
log("Putting all parties")
put_in_batches(parties_by_key.values())
# Put candidates in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
candidates_by_key = {}
for candidate_id, candidate_data in ynmp["Candidate"].iteritems():
if "status" not in candidate_data:
raise Exception("No status entry for " + str(candidate_data))
key_name = candidate_id
candidate = Candidate(ynmp_id=int(candidate_id),
name=candidate_data["name"],
code=candidate_data["code"],
status=candidate_data["status"],
email=candidate_data["email"],
address=candidate_data["address"],
party=parties_by_key[candidate_data["party_id"]],
image_id=int_or_null(candidate_data["image_id"]),
created=convdate(candidate_data["created"]),
updated=convdate(candidate_data["updated"]),
key_name=key_name)
log(" Storing candidate " + candidate.name)
candidates_by_key[key_name] = candidate
log("Putting all candidates")
put_in_batches(candidates_by_key.values())
# Put seats in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
seats_by_key = {}
for seat_id, seat_data in ynmp["Seat"].iteritems():
key_name = seat_id
seat = Seat(ynmp_id=int(seat_id),
name=seat_data["name"],
code=seat_data["code"],
created=convdate(seat_data["created"]),
updated=convdate(seat_data["updated"]),
key_name=key_name)
if key_name in frozen_seats:
seat.frozen_local_issues = True
log(" Storing seat " + seat.name)
seats_by_key[key_name] = seat
seats_by_name[seat.name] = seat
log("Putting all seats")
put_in_batches(seats_by_key.values())
# Get list of existing candiacies in remote datastore
# in batches due to 100 entity at a time limit, as per http://code.google.com/appengine/articles/remote_api.html
log("Getting list of Candidacies")
candidacies = Candidacy.all().filter("deleted =", False).fetch(100)
to_be_marked_deleted = {}
while candidacies:
for candidacy in candidacies:
key_name = candidacy.key().name()
log("Marking before have candidacy key " + key_name)
to_be_marked_deleted[key_name] = candidacy
candidacies = Candidacy.all().filter("deleted =", False).filter(
'__key__ >', candidacies[-1].key()).fetch(100)
# Loop through new dump of candidacies from YourNextMP, adding new ones
candidacies_by_key = {}
for candidacy_id, candidacy_data in ynmp["Candidacy"].iteritems():
candidate = candidates_by_key[candidacy_data["candidate_id"]]
assert candidate.status in [
'standing', 'standing_down', 'not-standing'
]
if candidate.status == 'standing_down' or candidate.status == 'not-standing':
continue
key_name = candidacy_data["seat_id"] + "-" + candidacy_data[
"candidate_id"]
# find existing entry if there is one, or else make new one
if key_name in to_be_marked_deleted:
candidacy = to_be_marked_deleted[key_name]
else:
candidacy = Candidacy(key_name=key_name)
# fill in values
candidacy.ynmp_id = int(candidacy_id)
candidacy.seat = seats_by_key[candidacy_data["seat_id"]]
candidacy.candidate = candidate
candidacy.created = convdate(candidacy_data["created"])
candidacy.updated = convdate(candidacy_data["updated"])
candidacy.deleted = False
# make sure it has a survey token
if not candidacy.survey_token:
log("Generating survey token for " + candidacy.seat.name + " " +
candidacy.candidate.name)
candidacy.generate_survey_token(
) # this does save too, since it logs
if candidacy.survey_invite_posted == None:
candidacy.survey_invite_posted = False
log("Storing candidacy " + candidacy.seat.name + " " +
candidacy.candidate.name)
candidacies_by_key[key_name] = candidacy
# record we still have this candidacy
if key_name in to_be_marked_deleted:
del to_be_marked_deleted[key_name]
log("Putting all candidacies")
put_in_batches(candidacies_by_key.values())
# See which candidacies are left, i.e. are deleted
for key_name, candidacy in to_be_marked_deleted.iteritems():
log("Marking deleted " + candidacy.seat.name + " " +
candidacy.candidate.name)
candidacy.deleted = True
log("Putting marked deleted candidacies")
put_in_batches(to_be_marked_deleted.values())
######################################################################
# Load from DemocracyClub
fs = Seat.all().filter("frozen_local_issues =", True).fetch(100)
while fs:
for f in fs:
log(" Seat is frozen to local issues changes: " + f.name)
frozen_seats[f.key().name()] = f
fs = Seat.all().filter("frozen_local_issues =",
True).filter('__key__ >',
fs[-1].key()).fetch(100)
