def load_from_ynmp(ynmp, frozen_seats):
# Put parties in datastore - don't worry about deleted ones, they just
# won't be referenced by other tables.
parties_by_key = {}
for party_id, party_data in ynmp["Party"].iteritems():
key_name = party_id
party = Party(
ynmp_id = int(party_id),
name = party_data["name"],
code = party_data["code"],
image_id = int_or_null(party_data["image_id"]),
created = convdate(party_data["created"]),
updated = convdate(party_data["updated"]),
key_name = key_name
)
log(" Storing party " + party.name)
parties_by_key[key_name] = party
log("Putting all parties")
put_in_batches(parties_by_key.values())
# Put candidates in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
candidates_by_key = {}
for candidate_id, candidate_data in ynmp["Candidate"].iteritems():
if "status" not in candidate_data:
raise Exception("No status entry for " + str(candidate_data))
key_name = candidate_id
candidate = Candidate(
ynmp_id = int(candidate_id),
name = candidate_data["name"],
code = candidate_data["code"],
status = candidate_data["status"],
email = candidate_data["email"],
address = candidate_data["address"],
party = parties_by_key[candidate_data["party_id"]],
image_id = int_or_null(candidate_data["image_id"]),
created = convdate(candidate_data["created"]),
updated = convdate(candidate_data["updated"]),
key_name = key_name
)
log(" Storing candidate " + candidate.name)
candidates_by_key[key_name] = candidate
log("Putting all candidates")
put_in_batches(candidates_by_key.values())
# Put seats in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
seats_by_key = {}
for seat_id, seat_data in ynmp["Seat"].iteritems():
key_name = seat_id
seat = Seat(
ynmp_id = int(seat_id),
name = seat_data["name"],
code = seat_data["code"],
created = convdate(seat_data["created"]),
updated = convdate(seat_data["updated"]),
key_name = key_name
)
if key_name in frozen_seats:
seat.frozen_local_issues = True
log(" Storing seat " + seat.name)
seats_by_key[key_name] = seat
seats_by_name[seat.name] = seat
log("Putting all seats")
put_in_batches(seats_by_key.values())
# Get list of existing candiacies in remote datastore
# in batches due to 100 entity at a time limit, as per http://code.google.com/appengine/articles/remote_api.html
log("Getting list of Candidacies")
candidacies = Candidacy.all().filter("deleted =", False).fetch(100)
to_be_marked_deleted = {}
while candidacies:
for candidacy in candidacies:
key_name = candidacy.key().name()
log("Marking before have candidacy key " + key_name)
to_be_marked_deleted[key_name] = candidacy
candidacies = Candidacy.all().filter("deleted =", False).filter('__key__ >', candidacies[-1].key()).fetch(100)
# Loop through new dump of candidacies from YourNextMP, adding new ones
candidacies_by_key = {}
for candidacy_id, candidacy_data in ynmp["Candidacy"].iteritems():
candidate = candidates_by_key[candidacy_data["candidate_id"]]
assert candidate.status in ['standing', 'standing_down', 'not-standing']
if candidate.status == 'standing_down' or candidate.status == 'not-standing':
continue
key_name = candidacy_data["seat_id"] + "-" + candidacy_data["candidate_id"]
# find existing entry if there is one, or else make new one
if key_name in to_be_marked_deleted:
candidacy = to_be_marked_deleted[key_name]
else:
candidacy = Candidacy(key_name = key_name)
# fill in values
candidacy.ynmp_id = int(candidacy_id)
candidacy.seat = seats_by_key[candidacy_data["seat_id"]]
candidacy.candidate = candidate
candidacy.created = convdate(candidacy_data["created"])
candidacy.updated = convdate(candidacy_data["updated"])
candidacy.deleted = False
# make sure it has a survey token
if not candidacy.survey_token:
log("Generating survey token for " + candidacy.seat.name + " " + candidacy.candidate.name)
candidacy.generate_survey_token() # this does save too, since it logs
if candidacy.survey_invite_posted == None:
candidacy.survey_invite_posted = False
log("Storing candidacy " + candidacy.seat.name + " " + candidacy.candidate.name)
candidacies_by_key[key_name] = candidacy
# record we still have this candidacy
if key_name in to_be_marked_deleted:
del to_be_marked_deleted[key_name]
log("Putting all candidacies")
put_in_batches(candidacies_by_key.values())
# See which candidacies are left, i.e. are deleted
for key_name, candidacy in to_be_marked_deleted.iteritems():
log("Marking deleted " + candidacy.seat.name + " " + candidacy.candidate.name)
candidacy.deleted = True
log("Putting marked deleted candidacies")
put_in_batches(to_be_marked_deleted.values())
######################################################################
# Load from DemocracyClub
fs = Seat.all().filter("frozen_local_issues =", True).fetch(100)
while fs:
for f in fs:
log(" Seat is frozen to local issues changes: " + f.name)
frozen_seats[f.key().name()] = f
fs = Seat.all().filter("frozen_local_issues =",True).filter('__key__ >', fs[-1].key()).fetch(100)
def load_from_ynmp(ynmp, frozen_seats):
# Put parties in datastore - don't worry about deleted ones, they just
# won't be referenced by other tables.
parties_by_key = {}
for party_id, party_data in ynmp["Party"].iteritems():
key_name = party_id
party = Party(ynmp_id=int(party_id),
name=party_data["name"],
code=party_data["code"],
image_id=int_or_null(party_data["image_id"]),
created=convdate(party_data["created"]),
updated=convdate(party_data["updated"]),
key_name=key_name)
log(" Storing party " + party.name)
parties_by_key[key_name] = party
log("Putting all parties")
put_in_batches(parties_by_key.values())
# Put candidates in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
candidates_by_key = {}
for candidate_id, candidate_data in ynmp["Candidate"].iteritems():
if "status" not in candidate_data:
raise Exception("No status entry for " + str(candidate_data))
key_name = candidate_id
candidate = Candidate(ynmp_id=int(candidate_id),
name=candidate_data["name"],
code=candidate_data["code"],
status=candidate_data["status"],
email=candidate_data["email"],
address=candidate_data["address"],
party=parties_by_key[candidate_data["party_id"]],
image_id=int_or_null(candidate_data["image_id"]),
created=convdate(candidate_data["created"]),
updated=convdate(candidate_data["updated"]),
key_name=key_name)
log(" Storing candidate " + candidate.name)
candidates_by_key[key_name] = candidate
log("Putting all candidates")
put_in_batches(candidates_by_key.values())
# Put seats in datastore - don't worry about deleted ones, they
# just won't be referenced by a candidacy
seats_by_key = {}
for seat_id, seat_data in ynmp["Seat"].iteritems():
key_name = seat_id
seat = Seat(ynmp_id=int(seat_id),
name=seat_data["name"],
code=seat_data["code"],
created=convdate(seat_data["created"]),
updated=convdate(seat_data["updated"]),
key_name=key_name)
if key_name in frozen_seats:
seat.frozen_local_issues = True
log(" Storing seat " + seat.name)
seats_by_key[key_name] = seat
seats_by_name[seat.name] = seat
log("Putting all seats")
put_in_batches(seats_by_key.values())
# Get list of existing candiacies in remote datastore
# in batches due to 100 entity at a time limit, as per http://code.google.com/appengine/articles/remote_api.html
log("Getting list of Candidacies")
candidacies = Candidacy.all().filter("deleted =", False).fetch(100)
to_be_marked_deleted = {}
while candidacies:
for candidacy in candidacies:
key_name = candidacy.key().name()
log("Marking before have candidacy key " + key_name)
to_be_marked_deleted[key_name] = candidacy
candidacies = Candidacy.all().filter("deleted =", False).filter(
'__key__ >', candidacies[-1].key()).fetch(100)
# Loop through new dump of candidacies from YourNextMP, adding new ones
candidacies_by_key = {}
for candidacy_id, candidacy_data in ynmp["Candidacy"].iteritems():
candidate = candidates_by_key[candidacy_data["candidate_id"]]
assert candidate.status in [
'standing', 'standing_down', 'not-standing'
]
if candidate.status == 'standing_down' or candidate.status == 'not-standing':
continue
key_name = candidacy_data["seat_id"] + "-" + candidacy_data[
"candidate_id"]
# find existing entry if there is one, or else make new one
if key_name in to_be_marked_deleted:
candidacy = to_be_marked_deleted[key_name]
else:
candidacy = Candidacy(key_name=key_name)
# fill in values
candidacy.ynmp_id = int(candidacy_id)
candidacy.seat = seats_by_key[candidacy_data["seat_id"]]
candidacy.candidate = candidate
candidacy.created = convdate(candidacy_data["created"])
candidacy.updated = convdate(candidacy_data["updated"])
candidacy.deleted = False
# make sure it has a survey token
if not candidacy.survey_token:
log("Generating survey token for " + candidacy.seat.name + " " +
candidacy.candidate.name)
candidacy.generate_survey_token(
) # this does save too, since it logs
if candidacy.survey_invite_posted == None:
candidacy.survey_invite_posted = False
log("Storing candidacy " + candidacy.seat.name + " " +
candidacy.candidate.name)
candidacies_by_key[key_name] = candidacy
# record we still have this candidacy
if key_name in to_be_marked_deleted:
del to_be_marked_deleted[key_name]
log("Putting all candidacies")
put_in_batches(candidacies_by_key.values())
# See which candidacies are left, i.e. are deleted
for key_name, candidacy in to_be_marked_deleted.iteritems():
log("Marking deleted " + candidacy.seat.name + " " +
candidacy.candidate.name)
candidacy.deleted = True
log("Putting marked deleted candidacies")
put_in_batches(to_be_marked_deleted.values())
######################################################################
# Load from DemocracyClub
fs = Seat.all().filter("frozen_local_issues =", True).fetch(100)
while fs:
for f in fs:
log(" Seat is frozen to local issues changes: " + f.name)
frozen_seats[f.key().name()] = f
fs = Seat.all().filter("frozen_local_issues =",
True).filter('__key__ >',
fs[-1].key()).fetch(100)
