def get_people(self): page = lxmlize(COUNCIL_PAGE) councillors = page.xpath('//div[@id="c2087"]//a') for councillor in councillors: name = councillor.text_content() url = councillor.attrib['href'] page = lxmlize(url) if 'Maire' in page.xpath('//h2/text()')[0]: district = 'Sherbrooke' role = 'Maire' else: district = page.xpath('//div[@class="csc-default"]//a[@target="_blank"]/text()')[0].replace('district', '').replace('Domaine Howard', 'Domaine-Howard').strip() role = 'Conseiller' if district in ('de Brompton', 'de Lennoxville'): district = district.replace('de ', '') p = Legislator(name=name, post_id=district, role=role) p.add_source(COUNCIL_PAGE) p.add_source(url) p.image = page.xpath('//div[@class="csc-textpic-image csc-textpic-last"]//img/@src')[0] parts = page.xpath('//li[contains(text(), "phone")]/text()')[0].split(':') note = parts[0] phone = parts[1] p.add_contact(note, phone, note) email = page.xpath('//a[contains(@href, "mailto:")]/@href') if email: email = email[0].split(':')[1] p.add_contact('email', email, None) if district == 'Brompton': p.add_extra('boundary_url', '/boundaries/sherbrooke-boroughs/brompton/') elif district == 'Lennoxville': p.add_extra('boundary_url', '/boundaries/sherbrooke-boroughs/lennoxville/') yield p
def get_people(self): yield chair_info(CHAIR_URL) for row in csv_reader(COUNCIL_PAGE, header=True, headers={'Cookie': 'incap_ses_168_68279=7jCHCh608QQSFVti3dtUAviu/1IAAAAAIRf6OsZL0NttnlzANkVb6w=='}): p = Legislator( name='%(FirstName0)s %(LastName0)s' % row, post_id='%(MUNIC)s Ward %(WARDNUM)s' % row, role='Councillor', ) p.add_contact('email', row['email0'], None) p.add_contact('voice', row['Phone0'], 'legislature') p.add_extra('boundary_url', '/boundaries/%s-wards/ward-%s/' % (row['MUNIC'].lower(), row['WARDNUM'])) p.add_source(COUNCIL_PAGE) yield p if row['FirstName1'].strip(): p = Legislator( name='%s %s' % (row['FirstName1'], row['LastName1']), post_id='%(MUNIC)s Ward %(WARDNUM)s' % row, role='Councillor', ) p.add_contact('email', row['email1'], None) p.add_contact('voice', row['Phone1'], 'legislature') p.add_extra('boundary_url', '/boundaries/%s-wards/ward-%s/' % (row['MUNIC'].lower(), row['WARDNUM'])) p.add_source(COUNCIL_PAGE) yield p
def get_people(self): reader = csv_reader(COUNCIL_PAGE, header=True) for row in reader: kwargs = {'role': 'candidate'} email = None links = [] extra = {} offices = [] for k, v in row.items(): v = v.strip() if not v: continue k = k.strip() match = re.search(r'\AOffice (\d): ', k) if match: index = int(match.group(1)) while index > len(offices): offices.append({}) if k[10:] == 'Type': offices[index - 1]['note'] = v elif k[10:] in CONTACT_TYPE_KEYS: offices[index - 1][CONTACT_TYPE_KEYS[k[10:]]] = v else: raise Exception(k) elif k == 'Party Name': kwargs['party'] = PARTY_MAP[v] elif k in KEYS: kwargs[KEYS[k]] = v elif k == 'Email': email = v elif k in LINKS_KEYS: links.append({'url': v, 'note': k}) elif k in IGNORE_KEYS: continue elif k in EXTRA_KEYS: extra[re.sub(r'[^a-z0-9_]', '', k.lower().replace(' ', '_'))] = v else: raise Exception(k) contacts = [] for office in offices: for _, type in CONTACT_TYPE_KEYS.items(): if office.get(type): contacts.push({'note': office['note'], type: type, 'value': office[type]}) if 'name' in kwargs: p = Legislator(**kwargs) p.add_source(COUNCIL_PAGE) if email: p.add_contact('email', email, None) for link in links: p.add_link(**links) for contact in contacts: p.add_contact(**contact) for k, v in extra.items(): p.add_extra(k, v) yield p
def get_people(self): yield chair_info(CHAIR_URL) for row in csv_reader( COUNCIL_PAGE, header=True, headers= { 'Cookie': 'incap_ses_168_68279=7jCHCh608QQSFVti3dtUAviu/1IAAAAAIRf6OsZL0NttnlzANkVb6w==' }): p = Legislator( name='%(FirstName0)s %(LastName0)s' % row, post_id='%(MUNIC)s Ward %(WARDNUM)s' % row, role='Councillor', ) p.add_contact('email', row['email0'], None) p.add_contact('voice', row['Phone0'], 'legislature') p.add_extra( 'boundary_url', '/boundaries/%s-wards/ward-%s/' % (row['MUNIC'].lower(), row['WARDNUM'])) p.add_source(COUNCIL_PAGE) yield p if row['FirstName1'].strip(): p = Legislator( name='%s %s' % (row['FirstName1'], row['LastName1']), post_id='%(MUNIC)s Ward %(WARDNUM)s' % row, role='Councillor', ) p.add_contact('email', row['email1'], None) p.add_contact('voice', row['Phone1'], 'legislature') p.add_extra( 'boundary_url', '/boundaries/%s-wards/ward-%s/' % (row['MUNIC'].lower(), row['WARDNUM'])) p.add_source(COUNCIL_PAGE) yield p
def get_people(self): reader = csv_reader(COUNCIL_PAGE, header=True) for row in reader: kwargs = {'role': 'candidate'} email = None links = [] extra = {} offices = [] for k, v in row.items(): v = v.strip() if not v: continue k = k.strip() match = re.search(r'\AOffice (\d): ', k) if match: index = int(match.group(1)) while index > len(offices): offices.append({}) if k[10:] == 'Type': offices[index - 1]['note'] = v elif k[10:] in CONTACT_TYPE_KEYS: offices[index - 1][CONTACT_TYPE_KEYS[k[10:]]] = v else: raise Exception(k) elif k == 'Party Name': kwargs['party'] = PARTY_MAP[v] elif k in KEYS: kwargs[KEYS[k]] = v elif k == 'Email': email = v elif k in LINKS_KEYS: links.append({'url': v, 'note': k}) elif k in IGNORE_KEYS: continue elif k in EXTRA_KEYS: extra[re.sub(r'[^a-z0-9_]', '', k.lower().replace(' ', '_'))] = v else: raise Exception(k) contacts = [] for office in offices: for _, type in CONTACT_TYPE_KEYS.items(): if office.get(type): contacts.push({ 'note': office['note'], type: type, 'value': office[type] }) if 'name' in kwargs: p = Legislator(**kwargs) p.add_source(COUNCIL_PAGE) if email: p.add_contact('email', email, None) for link in links: p.add_link(**links) for contact in contacts: p.add_contact(**contact) for k, v in extra.items(): p.add_extra(k, v) yield p