def personsToCsv(persons): """ Converts a list of Person instances to a CSV, returning the CSV. Input: persons - a list of Person instances. Output: a CSV. """ csv_buffer = StringIO() dict_writer = UnicodeDictWriter(csv_buffer, settings['final_csv_column_order'], extrasaction='ignore') dict_writer.writeheader() for person in persons: person = person.asDict() forums = person['forums'] del person['forums'] for forum in forums: if forum in settings['forum_mappings']: forums.extend(settings['forum_mappings'][forum]) continue person['group_id'] = forum dict_writer.writerow(person) csv_string = csv_buffer.getvalue() csv_buffer.close() return csv_string
def _write_csv(cls, params, filename): " Note: run deferred only. " org = pickle.loads(params['org_pickle']) event = pickle.loads(params['event_pickle']) post_data = pickle.loads(params['post_pickle']) _, query = form_and_query_from_params(org, event, None, post_data) # get unique zip codes without using distinct projections (for simpler indexes) zip_codes = set(site.zip_code.strip() for site in query if site.zip_code) zip_data = {zip_code: {} for zip_code in zip_codes} # gather statistics on site statuses for zip_code in zip_codes: status_counts = {} site_statuses = Query(Site, projection=('status',)) \ .filter('zip_code', zip_code) for site in site_statuses: status_counts[site.status] = status_counts.get(site.status, 0) + 1 zip_data[zip_code]['stats'] = status_counts # lookup primary city from zip code for zip_code in zip_codes: zip_code_obj = ZipCode.get_by_key_name(zip_code) zip_data[zip_code]['primary_city'] = \ zip_code_obj.primary_city if zip_code_obj else u"Unknown" # call votesmart for data on officials candidate_ids = set() for zip_code in zip_codes: officials = votesmart.officials_by_zip(zip_code) zip_data[zip_code][u'officials'] = officials candidate_ids.update(official[u'candidateId'] for official in officials) # lookup addresses of officials official_addresses = { candidate_id: votesmart.candidate_addresses(candidate_id) for candidate_id in candidate_ids } # create CSV sio of officials by zip code candidate_field_names = officials[0].keys() official_field_names = ( [u'zip_code', u'primary_city'] + STATUSES_UNICODE + [u'candidateId'] + candidate_field_names ) officials_csv_sio = StringIO() csv_writer = UnicodeDictWriter(officials_csv_sio, official_field_names) csv_writer.writeheader() for zip_code in zip_data: for official in zip_data[zip_code][u'officials']: row_d = { u'zip_code': zip_code, u'primary_city': zip_data[zip_code][u'primary_city'] } row_d.update(zip_data[zip_code][u'stats']) row_d.update(official) csv_writer.writerow(row_d) # create CSV sio of addresses by candidate def flatten_office_dict(d): return dict([ (u'address.' + k, v) for (k,v) in d.get(u'address', {}).items() ] + [ (u'phone.' + k, v) for (k,v) in d.get(u'phone', {}).items() ]) addresses_field_names = ( [u'candidateId'] + sorted( flatten_office_dict( next(official_addresses.itervalues())[u'offices'][0] ).keys() ) ) addresses_csv_sio = StringIO() csv_writer = UnicodeDictWriter(addresses_csv_sio, addresses_field_names) csv_writer.writeheader() for candidate_id, addresses_sub_dict in official_addresses.items(): for office in addresses_sub_dict[u'offices']: row_d = flatten_office_dict(office) row_d[u'candidateId'] = candidate_id csv_writer.writerow(row_d) # create XML sio of addresses rewritten_addresses_for_xml = { u'root': { u'addresses': [ dict( [(u'@candidateID', candidate_id)] + addresses_sub_dict.items() ) for candidate_id, addresses_sub_dict in official_addresses.items() ] } } xml = xmltodict.unparse( rewritten_addresses_for_xml, pretty=True ) xml_sio = StringIO() xml_sio.write(xml.encode('utf-8')) # create zip archive of both zip_sio = StringIO() zf = zipfile.ZipFile(zip_sio, 'w') zf.writestr('zips.csv', officials_csv_sio.getvalue()) zf.writestr('addresses.xml', xml_sio.getvalue()) zf.writestr('addresses.csv', addresses_csv_sio.getvalue()) zf.close() # create CSV file from data bucket_path = BUCKET_NAME + '/' + filename zip_gcs_fd = cloudstorage.open( bucket_path, 'w', content_type='application/zip' ) zip_gcs_fd.write(zip_sio.getvalue()) zip_gcs_fd.close()