def personsToCsv(persons):
"""
Converts a list of Person instances to a CSV, returning the CSV.
Input: persons - a list of Person instances.
Output: a CSV.
"""
csv_buffer = StringIO()
dict_writer = UnicodeDictWriter(csv_buffer,
settings['final_csv_column_order'],
extrasaction='ignore')
dict_writer.writeheader()
for person in persons:
person = person.asDict()
forums = person['forums']
del person['forums']
for forum in forums:
if forum in settings['forum_mappings']:
forums.extend(settings['forum_mappings'][forum])
continue
person['group_id'] = forum
dict_writer.writerow(person)
csv_string = csv_buffer.getvalue()
csv_buffer.close()
return csv_string
def _write_csv(cls, params, filename):
" Note: run deferred only. "
org = pickle.loads(params['org_pickle'])
event = pickle.loads(params['event_pickle'])
post_data = pickle.loads(params['post_pickle'])
_, query = form_and_query_from_params(org, event, None, post_data)
# get unique zip codes without using distinct projections (for simpler indexes)
zip_codes = set(site.zip_code.strip() for site in query if site.zip_code)
zip_data = {zip_code: {} for zip_code in zip_codes}
# gather statistics on site statuses
for zip_code in zip_codes:
status_counts = {}
site_statuses = Query(Site, projection=('status',)) \
.filter('zip_code', zip_code)
for site in site_statuses:
status_counts[site.status] = status_counts.get(site.status, 0) + 1
zip_data[zip_code]['stats'] = status_counts
# lookup primary city from zip code
for zip_code in zip_codes:
zip_code_obj = ZipCode.get_by_key_name(zip_code)
zip_data[zip_code]['primary_city'] = \
zip_code_obj.primary_city if zip_code_obj else u"Unknown"
# call votesmart for data on officials
candidate_ids = set()
for zip_code in zip_codes:
officials = votesmart.officials_by_zip(zip_code)
zip_data[zip_code][u'officials'] = officials
candidate_ids.update(official[u'candidateId'] for official in officials)
# lookup addresses of officials
official_addresses = {
candidate_id: votesmart.candidate_addresses(candidate_id)
for candidate_id in candidate_ids
}
# create CSV sio of officials by zip code
candidate_field_names = officials[0].keys()
official_field_names = (
[u'zip_code', u'primary_city'] +
STATUSES_UNICODE +
[u'candidateId'] + candidate_field_names
)
officials_csv_sio = StringIO()
csv_writer = UnicodeDictWriter(officials_csv_sio, official_field_names)
csv_writer.writeheader()
for zip_code in zip_data:
for official in zip_data[zip_code][u'officials']:
row_d = {
u'zip_code': zip_code,
u'primary_city': zip_data[zip_code][u'primary_city']
}
row_d.update(zip_data[zip_code][u'stats'])
row_d.update(official)
csv_writer.writerow(row_d)
# create CSV sio of addresses by candidate
def flatten_office_dict(d):
return dict([
(u'address.' + k, v) for (k,v) in d.get(u'address', {}).items()
] + [
(u'phone.' + k, v) for (k,v) in d.get(u'phone', {}).items()
])
addresses_field_names = (
[u'candidateId'] +
sorted(
flatten_office_dict(
next(official_addresses.itervalues())[u'offices'][0]
).keys()
)
)
addresses_csv_sio = StringIO()
csv_writer = UnicodeDictWriter(addresses_csv_sio, addresses_field_names)
csv_writer.writeheader()
for candidate_id, addresses_sub_dict in official_addresses.items():
for office in addresses_sub_dict[u'offices']:
row_d = flatten_office_dict(office)
row_d[u'candidateId'] = candidate_id
csv_writer.writerow(row_d)
# create XML sio of addresses
rewritten_addresses_for_xml = {
u'root': {
u'addresses': [
dict(
[(u'@candidateID', candidate_id)] +
addresses_sub_dict.items()
) for candidate_id, addresses_sub_dict in official_addresses.items()
]
}
}
xml = xmltodict.unparse(
rewritten_addresses_for_xml,
pretty=True
)
xml_sio = StringIO()
xml_sio.write(xml.encode('utf-8'))
# create zip archive of both
zip_sio = StringIO()
zf = zipfile.ZipFile(zip_sio, 'w')
zf.writestr('zips.csv', officials_csv_sio.getvalue())
zf.writestr('addresses.xml', xml_sio.getvalue())
zf.writestr('addresses.csv', addresses_csv_sio.getvalue())
zf.close()
# create CSV file from data
bucket_path = BUCKET_NAME + '/' + filename
zip_gcs_fd = cloudstorage.open(
bucket_path,
'w',
content_type='application/zip'
)
zip_gcs_fd.write(zip_sio.getvalue())
zip_gcs_fd.close()
