class TestCountryMethods(TestCase): def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country_aland = self.ep.country('Aland') self.country_argentina = self.ep.country('Argentina') self.country_bv = self.ep.country('British-Virgin-Islands') def test_country_repr(self): if six.PY2: assert repr(self.country_aland) == b'<Country: \xc3\x85land>' else: assert repr(self.country_aland) == '<Country: \xc5land>' def test_get_legislatures(self): ls = self.country_aland.legislatures() assert len(ls) == 1 def test_most_recent_house_no_house_matches(self): with pytest.raises(NotFound): print(self.country_argentina.house_most_recent('random_house')) def test_most_recent_house_where_multiple_match_lower_house(self): assert self.country_bv.house_most_recent( 'lower_house').name == 'House of Assembly' def test_lower_house_shortcut(self): assert self.country_argentina.lower_house( ).name == 'Cámara de Diputados' def test_upper_house_shortcut(self): assert self.country_argentina.upper_house( ).name == 'Cámara de Senadores'
def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country_aland = self.ep.country('Aland') self.country_argentina = self.ep.country('Argentina') self.country_bv = self.ep.country('British-Virgin-Islands')
def test_no_matches_for_unknown_house_type(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): ep = EveryPolitician() country = ep.country('Argentina') houses = country.houses('quiet area') assert len(houses) == 0
def test_countries(self, patched_requests_get): ep = EveryPolitician() countries = ep.countries() assert len(countries) == 3 assert text_type(countries[0]) == '<Country: Åland>' assert text_type(countries[1]) == '<Country: Argentina>' assert text_type(countries[2]) == '<Country: British Virgin Islands>'
def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country = self.ep.country('Argentina') self.legislature = self.country.legislature('Diputados') self.period = self.legislature.legislative_periods()[0]
def output_file(country, legislature, filename): data = EveryPolitician().country(country).legislature(legislature) output_filename = "../members/{0}.xml".format(filename) root = etree.Element('publicwhip') sorted_people = sorted( data.popolo().persons, key=operator.attrgetter('name') ) for person in sorted_people: parlparse_id = person.identifier_value('parlparse') if parlparse_id is not None: props = {} if person.twitter: props['twitter_username'] = person.twitter if person.facebook: props['facebook_page'] = person.facebook if props: props['id'] = parlparse_id info = etree.Element('personinfo', props) root.append(info) et = etree.ElementTree(root) et.write(output_filename, pretty_print=True)
def getlegislators(): housecounts = defaultdict(lambda: 0) senatecounts = defaultdict(lambda: 0) nonvoting = [ 'American Samoa', 'District of Columbia', 'Guam', 'Northern Mariana Islands', 'Puerto Rico', 'US Virgin Islands' ] ep = EveryPolitician() epusa = ep.country('United-States-of-America') epussenate = epusa.legislature('Senate') epushouse = epusa.legislature('House') housedata = epushouse.latest_legislative_period().csv() senatedata = epussenate.latest_legislative_period().csv() housedata[:] = [ entry for entry in housedata if entry.get('end_date') == '' and entry.get('area') not in nonvoting ] senatedata[:] = [ entry for entry in senatedata if entry.get('end_date') == '' ] for entry in housedata: housecounts[entry['group']] = housecounts[entry['group']] + 1 for entry in senatedata: senatecounts[entry['group']] = senatecounts[entry['group']] + 1 housecounts = dict(housecounts) senatecounts = dict(senatecounts) return housedata, senatedata, housecounts, senatecounts
def test_finds_upper_house_if_present(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): ep = EveryPolitician() country = ep.country('Argentina') houses = country.houses('upper house') assert len(houses) == 1 assert houses[0].name == 'Cámara de Senadores'
def test_finds_unicameral_legislature_for_lower_house(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): ep = EveryPolitician() country = ep.country('Aland') houses = country.houses('lower house') assert len(houses) == 1 assert houses[0].name == 'Lagting'
def build_rep_lookup(): result = defaultdict(set) ep = EveryPolitician() nigeria = ep.country('Nigeria') for lname in ('Senate', 'Representatives'): popolo = nigeria.legislature(lname).popolo() term = popolo.latest_term for membership in term.memberships: result[membership.area_id] = membership return result
def download(country, legislature, destination): """ download current info from ep to file """ ep = EveryPolitician() country, leg = ep.country_legislature(country, legislature) url = leg.popolo_url js = requests.get(url).content with open(destination, 'w') as f: f.write(js)
def main(): ep = EveryPolitician() legislatures = get_legislatures(ep.countries()) politicians = get_people(legislatures) memberships = get_memberships(legislatures) organizations = get_organizations(legislatures) legislative_periods = get_legislative_periods(legislatures) politicians = create_additional_columns_every_politician( politicians, memberships, organizations, legislative_periods) filtered_politicians = filter_politicians_for_sufficient_date(politicians) write_csv_s3(filtered_politicians, 'politicians', fs)
def test_ep_from_local_file(self, patched_requests_get): filename = join(dirname(__file__), 'test-data', 'example-countries.json') ep = EveryPolitician(countries_json_filename=filename) assert re.search( r'EveryPolitician\(countries_json_filename=".*example-countries.json"\)', repr(ep))
class TestLeglislatureMethods(TestCase): def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country = self.ep.country('Argentina') self.legislatures = self.country.legislatures() def test_legislature_repr(self): if six.PY2: assert repr( self.legislatures[0] ) == b'<Legislature: C\xc3\xa1mara de Diputados in Argentina>' else: assert repr(self.legislatures[1] ) == '<Legislature: Cámara de Senadores in Argentina>' def test_legislature_str(self): assert text_type(self.legislatures[1] ) == '<Legislature: Cámara de Senadores in Argentina>' def test_legislature_popolo_url(self): l = self.legislatures[1] assert l.popolo_url == 'https://cdn.rawgit.com/everypolitician/' \ 'everypolitician-data/c323b935f2dce83fcdfcbb5c2f94614a25207d98/' \ 'data/Argentina/Senado/ep-popolo-v1.0.json' def test_directory(self): l = self.legislatures[0] assert l.directory() == 'Argentina/Diputados' @patch('everypolitician.lib.Popolo') def test_popolo_call(self, mocked_popolo_class): mocked_popolo_class.from_url.return_value = Popolo( {'persons': [{ 'name': 'Joe Bloggs' }]}) l = self.legislatures[0] popolo = l.popolo() mocked_popolo_class.from_url.assert_called_with( u'https://cdn.rawgit.com/everypolitician/everypolitician-data/' u'd3afadff7d5a08e1745b7e48782a869ec4979e78/data/Argentina/' u'Diputados/ep-popolo-v1.0.json') assert len(popolo.persons) == 1 assert popolo.persons.first.name == 'Joe Bloggs' @patch('everypolitician.lib.Popolo') def test_popolo_data_only_fetched_once(self, mocked_popolo_class): mocked_popolo_class.from_url.return_value = Popolo( {'persons': [{ 'name': 'Joe Bloggs' }]}) l = self.legislatures[0] l.popolo() l.popolo() mocked_popolo_class.from_url.assert_called_once_with( u'https://cdn.rawgit.com/everypolitician/everypolitician-data/' u'd3afadff7d5a08e1745b7e48782a869ec4979e78/data/Argentina/' u'Diputados/ep-popolo-v1.0.json')
def handle(self, **options): verbose_level = options['verbosity'] url_template = ( 'https://cdn.rawgit.com/everypolitician/everypolitician-data/' '{git_ref}/countries.json') url = url_template.format( git_ref=options['everypolitician_countries_json_git_ref']) ep = EveryPolitician(countries_json_url=url) south_africa_assembly = ep.country('South-Africa').legislature( 'Assembly').popolo() id_lookup = {} for popolo_person in south_africa_assembly.persons: id_lookup[popolo_person.identifier_value( 'peoples_assembly')] = popolo_person.identifier_value( 'wikidata') error_msg = u"No EveryPolitician UUID found for {0.id} {0.name} https://www.pa.org.za/person/{0.slug}/\n" for person in Person.objects.filter(hidden=False): wikidata_id = id_lookup.get(str(person.id)) if wikidata_id is None: verbose_level > 1 and self.stderr.write( error_msg.format(person)) continue identifier, created = person.identifiers.get_or_create( scheme='wikidata', identifier=wikidata_id, ) if verbose_level > 0: if created: msg = u"Created new identifier for {name}: {identifier}" else: msg = u"Existing identifier found for {name}: {identifier}" self.stdout.write( msg.format(name=person.name, identifier=identifier.identifier))
def output_file(country, legislature, filename): data = EveryPolitician().country(country).legislature(legislature) output_filename = "../members/{0}.xml".format(filename) root = etree.Element('publicwhip') sorted_people = sorted(data.popolo().persons, key=operator.attrgetter('name')) for person in sorted_people: parlparse_id = person.identifier_value('parlparse') if parlparse_id is not None: props = {} if person.twitter: props['twitter_username'] = person.twitter if person.facebook: props['facebook_page'] = person.facebook if props: props['id'] = parlparse_id info = etree.Element('personinfo', props) root.append(info) et = etree.ElementTree(root) et.write(output_filename, pretty_print=True)
class TestLegislativePeriod(TestCase): def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country = self.ep.country('Argentina') self.legislature = self.country.legislature('Diputados') self.period = self.legislature.legislative_periods()[0] def test_start_date(self, patched_requests_get): assert self.period.start_date == "2015" def test_end_date(self, patched_requests_get): assert self.period.end_date is None def test_csv(self, patched_requests_get): assert self.period.csv() == \ [{u'area': u'BUENOS AIRES', u'area_id': u'area/buenos_aires', u'chamber': u'C\xe1mara de Diputados', u'email': u'*****@*****.**', u'end_date': u'2015-12-09', u'facebook': u'', u'gender': u'female', u'group': u'FRENTE PARA LA VICTORIA - PJ', u'group_id': u'frente_para_la_victoria_-_pj', u'id': u'b882751f-4014-4f6f-b3cf-e0a5d6d3c605', u'image': u'http://www4.hcdn.gob.ar/fotos/asegarra.jpg', u'name': u'ADELA ROSA SEGARRA', u'sort_name': u'SEGARRA, ADELA ROSA', u'start_date': u'', u'term': u'133', u'twitter': u''}, {u'area': u'BUENOS AIRES', u'area_id': u'area/buenos_aires', u'chamber': u'C\xe1mara de Diputados', u'email': u'*****@*****.**', u'end_date': u'', u'facebook': u'', u'gender': u'male', u'group': u'FRENTE RENOVADOR', u'group_id': u'frente_renovador', u'id': u'8efb1e0e-8454-4c6b-9f87-0d4fef875fd2', u'image': u'http://www4.hcdn.gob.ar/fotos/aperez.jpg', u'name': u'ADRIAN PEREZ', u'sort_name': u'PEREZ, ADRIAN', u'start_date': u'', u'term': u'133', u'twitter': u'adrianperezARG'}]
class TestCountryMethods(TestCase): def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country = self.ep.country('Aland') def test_country_repr(self): if six.PY2: assert repr(self.country) == b'<Country: \xc3\x85land>' else: assert repr(self.country) == '<Country: \xc5land>' def test_get_legislatures(self): ls = self.country.legislatures() assert len(ls) == 1
#!/usr/bin/env python3 import itertools import os from everypolitician import EveryPolitician import requests import scraperwiki ep = EveryPolitician() consumer_key = os.environ.get('MORPH_TWITTER_CONSUMER_KEY') consumer_secret = os.environ.get('MORPH_TWITTER_CONSUMER_SECRET') # get a bearer token from twitter def _get_token(consumer_key, consumer_secret): auth = requests.auth.HTTPBasicAuth(consumer_key, consumer_secret) auth_url = 'https://api.twitter.com/oauth2/token' auth_data = {'grant_type': 'client_credentials'} r = requests.post(url=auth_url, data=auth_data, auth=auth) j = r.json() return j['access_token'] # Run twitter API query def _run_query(payload): r = requests.post( 'https://api.twitter.com/1.1/users/lookup.json', data=payload, headers=auth_header, ) data = r.json()
def test_get_a_single_country_bad_case(self, patched_requests_get): ep = EveryPolitician() with pytest.raises(NotFound): ep.country('argentina')
def test_json_only_fetched_once(self, patched_requests_get): ep = EveryPolitician() ep.countries() ep.countries() assert patched_requests_get.call_count == 1
#!/usr/bin/env python3 import itertools import os from everypolitician import EveryPolitician import requests import scraperwiki ep = EveryPolitician() consumer_key = os.environ.get('MORPH_TWITTER_CONSUMER_KEY') consumer_secret = os.environ.get('MORPH_TWITTER_CONSUMER_SECRET') # get a bearer token from twitter def _get_token(consumer_key, consumer_secret): auth = requests.auth.HTTPBasicAuth(consumer_key, consumer_secret) auth_url = 'https://api.twitter.com/oauth2/token' auth_data = {'grant_type': 'client_credentials'} r = requests.post(url=auth_url, data=auth_data, auth=auth) j = r.json() return j['access_token'] # Run twitter API query def _run_query(payload): r = requests.post( 'https://api.twitter.com/1.1/users/lookup.json', data=payload, headers=auth_header, )
def test_countries_from_local_file(self, patched_requests_get): filename = join(dirname(__file__), 'test-data', 'example-countries.json') ep = EveryPolitician(countries_json_filename=filename) countries = ep.countries() assert len(countries) == 3
def test_get_a_country_and_legislature(self, patched_requests_get): ep = EveryPolitician() country, legislature = ep.country_legislature('Argentina', 'Diputados') assert country.name == 'Argentina' assert legislature.name == 'Cámara de Diputados'
def test_ep_repr_custom_url(self, patched_requests_get): ep = EveryPolitician(countries_json_url='foobar') assert repr(ep) == 'EveryPolitician(countries_json_url="foobar")'
def test_ep_repr(self, patched_requests_get): ep = EveryPolitician() assert repr(ep) == 'EveryPolitician()'
def test_create_ep(self, patched_requests_get): ep = EveryPolitician() assert str(ep) == \ '<EveryPolitician: https://raw.githubusercontent.com/everypolitician/everypolitician-data/master/countries.json>'
def test_get_a_single_country(self, patched_requests_get): ep = EveryPolitician() country = ep.country('Argentina') assert country.name == 'Argentina'
def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country = self.ep.country('Argentina') self.legislatures = self.country.legislatures()
def setUp(self): with patch('everypolitician.lib.requests.get', side_effect=fake_requests_get): self.ep = EveryPolitician() self.country = self.ep.country('Aland')
}, { 'ep_country_slug': 'Scotland', 'ep_legislature_slug': 'Parliament' }, { 'ep_country_slug': 'Northern-Ireland', 'ep_legislature_slug': 'Assembly' }, ] pp_data = Popolo() for legislature in legislatures_to_fetch: ep_country = EveryPolitician().country(legislature['ep_country_slug']) ep_legislature = ep_country.legislature(legislature['ep_legislature_slug']) ep_people = ep_legislature.popolo().persons logger.info('{}/{}: Found {} people.'.format(legislature['ep_country_slug'], legislature['ep_legislature_slug'], len(ep_people) )) for ep_person in ep_people: if ep_person.identifier_value('parlparse') and ep_person.wikidata: pp_person = pp_data.get_person(id=ep_person.identifier_value('parlparse')) pp_wikidata_identifier = getPersonIdentifierBySchema(pp_person, 'wikidata')
def test_get_a_country_legislature_neither_found(self, patched_requests_get): ep = EveryPolitician() with pytest.raises(NotFound): ep.country_legislature('FOO', 'FOO')