class TestCountryMethods(TestCase):
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country_aland = self.ep.country('Aland')
self.country_argentina = self.ep.country('Argentina')
self.country_bv = self.ep.country('British-Virgin-Islands')
def test_country_repr(self):
if six.PY2:
assert repr(self.country_aland) == b'<Country: \xc3\x85land>'
else:
assert repr(self.country_aland) == '<Country: \xc5land>'
def test_get_legislatures(self):
ls = self.country_aland.legislatures()
assert len(ls) == 1
def test_most_recent_house_no_house_matches(self):
with pytest.raises(NotFound):
print(self.country_argentina.house_most_recent('random_house'))
def test_most_recent_house_where_multiple_match_lower_house(self):
assert self.country_bv.house_most_recent(
'lower_house').name == 'House of Assembly'
def test_lower_house_shortcut(self):
assert self.country_argentina.lower_house(
).name == 'Cámara de Diputados'
def test_upper_house_shortcut(self):
assert self.country_argentina.upper_house(
).name == 'Cámara de Senadores'
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country_aland = self.ep.country('Aland')
self.country_argentina = self.ep.country('Argentina')
self.country_bv = self.ep.country('British-Virgin-Islands')
def test_no_matches_for_unknown_house_type(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
ep = EveryPolitician()
country = ep.country('Argentina')
houses = country.houses('quiet area')
assert len(houses) == 0
def test_countries(self, patched_requests_get):
ep = EveryPolitician()
countries = ep.countries()
assert len(countries) == 3
assert text_type(countries[0]) == '<Country: Åland>'
assert text_type(countries[1]) == '<Country: Argentina>'
assert text_type(countries[2]) == '<Country: British Virgin Islands>'
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country = self.ep.country('Argentina')
self.legislature = self.country.legislature('Diputados')
self.period = self.legislature.legislative_periods()[0]
def output_file(country, legislature, filename):
data = EveryPolitician().country(country).legislature(legislature)
output_filename = "../members/{0}.xml".format(filename)
root = etree.Element('publicwhip')
sorted_people = sorted(
data.popolo().persons,
key=operator.attrgetter('name')
)
for person in sorted_people:
parlparse_id = person.identifier_value('parlparse')
if parlparse_id is not None:
props = {}
if person.twitter:
props['twitter_username'] = person.twitter
if person.facebook:
props['facebook_page'] = person.facebook
if props:
props['id'] = parlparse_id
info = etree.Element('personinfo', props)
root.append(info)
et = etree.ElementTree(root)
et.write(output_filename, pretty_print=True)
def getlegislators():
housecounts = defaultdict(lambda: 0)
senatecounts = defaultdict(lambda: 0)
nonvoting = [
'American Samoa', 'District of Columbia', 'Guam',
'Northern Mariana Islands', 'Puerto Rico', 'US Virgin Islands'
]
ep = EveryPolitician()
epusa = ep.country('United-States-of-America')
epussenate = epusa.legislature('Senate')
epushouse = epusa.legislature('House')
housedata = epushouse.latest_legislative_period().csv()
senatedata = epussenate.latest_legislative_period().csv()
housedata[:] = [
entry for entry in housedata
if entry.get('end_date') == '' and entry.get('area') not in nonvoting
]
senatedata[:] = [
entry for entry in senatedata if entry.get('end_date') == ''
]
for entry in housedata:
housecounts[entry['group']] = housecounts[entry['group']] + 1
for entry in senatedata:
senatecounts[entry['group']] = senatecounts[entry['group']] + 1
housecounts = dict(housecounts)
senatecounts = dict(senatecounts)
return housedata, senatedata, housecounts, senatecounts
def test_finds_upper_house_if_present(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
ep = EveryPolitician()
country = ep.country('Argentina')
houses = country.houses('upper house')
assert len(houses) == 1
assert houses[0].name == 'Cámara de Senadores'
def test_finds_unicameral_legislature_for_lower_house(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
ep = EveryPolitician()
country = ep.country('Aland')
houses = country.houses('lower house')
assert len(houses) == 1
assert houses[0].name == 'Lagting'
def build_rep_lookup():
result = defaultdict(set)
ep = EveryPolitician()
nigeria = ep.country('Nigeria')
for lname in ('Senate', 'Representatives'):
popolo = nigeria.legislature(lname).popolo()
term = popolo.latest_term
for membership in term.memberships:
result[membership.area_id] = membership
return result
def download(country, legislature, destination):
"""
download current info from ep to file
"""
ep = EveryPolitician()
country, leg = ep.country_legislature(country, legislature)
url = leg.popolo_url
js = requests.get(url).content
with open(destination, 'w') as f:
f.write(js)
def main():
ep = EveryPolitician()
legislatures = get_legislatures(ep.countries())
politicians = get_people(legislatures)
memberships = get_memberships(legislatures)
organizations = get_organizations(legislatures)
legislative_periods = get_legislative_periods(legislatures)
politicians = create_additional_columns_every_politician(
politicians, memberships, organizations, legislative_periods)
filtered_politicians = filter_politicians_for_sufficient_date(politicians)
write_csv_s3(filtered_politicians, 'politicians', fs)
def test_ep_from_local_file(self, patched_requests_get):
filename = join(dirname(__file__), 'test-data',
'example-countries.json')
ep = EveryPolitician(countries_json_filename=filename)
assert re.search(
r'EveryPolitician\(countries_json_filename=".*example-countries.json"\)',
repr(ep))
class TestLeglislatureMethods(TestCase):
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country = self.ep.country('Argentina')
self.legislatures = self.country.legislatures()
def test_legislature_repr(self):
if six.PY2:
assert repr(
self.legislatures[0]
) == b'<Legislature: C\xc3\xa1mara de Diputados in Argentina>'
else:
assert repr(self.legislatures[1]
) == '<Legislature: Cámara de Senadores in Argentina>'
def test_legislature_str(self):
assert text_type(self.legislatures[1]
) == '<Legislature: Cámara de Senadores in Argentina>'
def test_legislature_popolo_url(self):
l = self.legislatures[1]
assert l.popolo_url == 'https://cdn.rawgit.com/everypolitician/' \
'everypolitician-data/c323b935f2dce83fcdfcbb5c2f94614a25207d98/' \
'data/Argentina/Senado/ep-popolo-v1.0.json'
def test_directory(self):
l = self.legislatures[0]
assert l.directory() == 'Argentina/Diputados'
@patch('everypolitician.lib.Popolo')
def test_popolo_call(self, mocked_popolo_class):
mocked_popolo_class.from_url.return_value = Popolo(
{'persons': [{
'name': 'Joe Bloggs'
}]})
l = self.legislatures[0]
popolo = l.popolo()
mocked_popolo_class.from_url.assert_called_with(
u'https://cdn.rawgit.com/everypolitician/everypolitician-data/'
u'd3afadff7d5a08e1745b7e48782a869ec4979e78/data/Argentina/'
u'Diputados/ep-popolo-v1.0.json')
assert len(popolo.persons) == 1
assert popolo.persons.first.name == 'Joe Bloggs'
@patch('everypolitician.lib.Popolo')
def test_popolo_data_only_fetched_once(self, mocked_popolo_class):
mocked_popolo_class.from_url.return_value = Popolo(
{'persons': [{
'name': 'Joe Bloggs'
}]})
l = self.legislatures[0]
l.popolo()
l.popolo()
mocked_popolo_class.from_url.assert_called_once_with(
u'https://cdn.rawgit.com/everypolitician/everypolitician-data/'
u'd3afadff7d5a08e1745b7e48782a869ec4979e78/data/Argentina/'
u'Diputados/ep-popolo-v1.0.json')
def handle(self, **options):
verbose_level = options['verbosity']
url_template = (
'https://cdn.rawgit.com/everypolitician/everypolitician-data/'
'{git_ref}/countries.json')
url = url_template.format(
git_ref=options['everypolitician_countries_json_git_ref'])
ep = EveryPolitician(countries_json_url=url)
south_africa_assembly = ep.country('South-Africa').legislature(
'Assembly').popolo()
id_lookup = {}
for popolo_person in south_africa_assembly.persons:
id_lookup[popolo_person.identifier_value(
'peoples_assembly')] = popolo_person.identifier_value(
'wikidata')
error_msg = u"No EveryPolitician UUID found for {0.id} {0.name} https://www.pa.org.za/person/{0.slug}/\n"
for person in Person.objects.filter(hidden=False):
wikidata_id = id_lookup.get(str(person.id))
if wikidata_id is None:
verbose_level > 1 and self.stderr.write(
error_msg.format(person))
continue
identifier, created = person.identifiers.get_or_create(
scheme='wikidata',
identifier=wikidata_id,
)
if verbose_level > 0:
if created:
msg = u"Created new identifier for {name}: {identifier}"
else:
msg = u"Existing identifier found for {name}: {identifier}"
self.stdout.write(
msg.format(name=person.name,
identifier=identifier.identifier))
def output_file(country, legislature, filename):
data = EveryPolitician().country(country).legislature(legislature)
output_filename = "../members/{0}.xml".format(filename)
root = etree.Element('publicwhip')
sorted_people = sorted(data.popolo().persons,
key=operator.attrgetter('name'))
for person in sorted_people:
parlparse_id = person.identifier_value('parlparse')
if parlparse_id is not None:
props = {}
if person.twitter:
props['twitter_username'] = person.twitter
if person.facebook:
props['facebook_page'] = person.facebook
if props:
props['id'] = parlparse_id
info = etree.Element('personinfo', props)
root.append(info)
et = etree.ElementTree(root)
et.write(output_filename, pretty_print=True)
class TestLegislativePeriod(TestCase):
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country = self.ep.country('Argentina')
self.legislature = self.country.legislature('Diputados')
self.period = self.legislature.legislative_periods()[0]
def test_start_date(self, patched_requests_get):
assert self.period.start_date == "2015"
def test_end_date(self, patched_requests_get):
assert self.period.end_date is None
def test_csv(self, patched_requests_get):
assert self.period.csv() == \
[{u'area': u'BUENOS AIRES',
u'area_id': u'area/buenos_aires',
u'chamber': u'C\xe1mara de Diputados',
u'email': u'*****@*****.**',
u'end_date': u'2015-12-09',
u'facebook': u'',
u'gender': u'female',
u'group': u'FRENTE PARA LA VICTORIA - PJ',
u'group_id': u'frente_para_la_victoria_-_pj',
u'id': u'b882751f-4014-4f6f-b3cf-e0a5d6d3c605',
u'image': u'http://www4.hcdn.gob.ar/fotos/asegarra.jpg',
u'name': u'ADELA ROSA SEGARRA',
u'sort_name': u'SEGARRA, ADELA ROSA',
u'start_date': u'',
u'term': u'133',
u'twitter': u''},
{u'area': u'BUENOS AIRES',
u'area_id': u'area/buenos_aires',
u'chamber': u'C\xe1mara de Diputados',
u'email': u'*****@*****.**',
u'end_date': u'',
u'facebook': u'',
u'gender': u'male',
u'group': u'FRENTE RENOVADOR',
u'group_id': u'frente_renovador',
u'id': u'8efb1e0e-8454-4c6b-9f87-0d4fef875fd2',
u'image': u'http://www4.hcdn.gob.ar/fotos/aperez.jpg',
u'name': u'ADRIAN PEREZ',
u'sort_name': u'PEREZ, ADRIAN',
u'start_date': u'',
u'term': u'133',
u'twitter': u'adrianperezARG'}]
class TestCountryMethods(TestCase):
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country = self.ep.country('Aland')
def test_country_repr(self):
if six.PY2:
assert repr(self.country) == b'<Country: \xc3\x85land>'
else:
assert repr(self.country) == '<Country: \xc5land>'
def test_get_legislatures(self):
ls = self.country.legislatures()
assert len(ls) == 1
#!/usr/bin/env python3
import itertools
import os
from everypolitician import EveryPolitician
import requests
import scraperwiki
ep = EveryPolitician()
consumer_key = os.environ.get('MORPH_TWITTER_CONSUMER_KEY')
consumer_secret = os.environ.get('MORPH_TWITTER_CONSUMER_SECRET')
# get a bearer token from twitter
def _get_token(consumer_key, consumer_secret):
auth = requests.auth.HTTPBasicAuth(consumer_key, consumer_secret)
auth_url = 'https://api.twitter.com/oauth2/token'
auth_data = {'grant_type': 'client_credentials'}
r = requests.post(url=auth_url, data=auth_data, auth=auth)
j = r.json()
return j['access_token']
# Run twitter API query
def _run_query(payload):
r = requests.post(
'https://api.twitter.com/1.1/users/lookup.json',
data=payload,
headers=auth_header,
)
data = r.json()
def test_get_a_single_country_bad_case(self, patched_requests_get):
ep = EveryPolitician()
with pytest.raises(NotFound):
ep.country('argentina')
def test_json_only_fetched_once(self, patched_requests_get):
ep = EveryPolitician()
ep.countries()
ep.countries()
assert patched_requests_get.call_count == 1
#!/usr/bin/env python3
import itertools
import os
from everypolitician import EveryPolitician
import requests
import scraperwiki
ep = EveryPolitician()
consumer_key = os.environ.get('MORPH_TWITTER_CONSUMER_KEY')
consumer_secret = os.environ.get('MORPH_TWITTER_CONSUMER_SECRET')
# get a bearer token from twitter
def _get_token(consumer_key, consumer_secret):
auth = requests.auth.HTTPBasicAuth(consumer_key, consumer_secret)
auth_url = 'https://api.twitter.com/oauth2/token'
auth_data = {'grant_type': 'client_credentials'}
r = requests.post(url=auth_url, data=auth_data, auth=auth)
j = r.json()
return j['access_token']
# Run twitter API query
def _run_query(payload):
r = requests.post(
'https://api.twitter.com/1.1/users/lookup.json',
data=payload,
headers=auth_header,
)
def test_countries_from_local_file(self, patched_requests_get):
filename = join(dirname(__file__), 'test-data',
'example-countries.json')
ep = EveryPolitician(countries_json_filename=filename)
countries = ep.countries()
assert len(countries) == 3
def test_get_a_country_and_legislature(self, patched_requests_get):
ep = EveryPolitician()
country, legislature = ep.country_legislature('Argentina', 'Diputados')
assert country.name == 'Argentina'
assert legislature.name == 'Cámara de Diputados'
def test_ep_repr_custom_url(self, patched_requests_get):
ep = EveryPolitician(countries_json_url='foobar')
assert repr(ep) == 'EveryPolitician(countries_json_url="foobar")'
def test_ep_repr(self, patched_requests_get):
ep = EveryPolitician()
assert repr(ep) == 'EveryPolitician()'
def test_create_ep(self, patched_requests_get):
ep = EveryPolitician()
assert str(ep) == \
'<EveryPolitician: https://raw.githubusercontent.com/everypolitician/everypolitician-data/master/countries.json>'
def test_get_a_single_country(self, patched_requests_get):
ep = EveryPolitician()
country = ep.country('Argentina')
assert country.name == 'Argentina'
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country = self.ep.country('Argentina')
self.legislatures = self.country.legislatures()
def setUp(self):
with patch('everypolitician.lib.requests.get',
side_effect=fake_requests_get):
self.ep = EveryPolitician()
self.country = self.ep.country('Aland')
},
{
'ep_country_slug': 'Scotland',
'ep_legislature_slug': 'Parliament'
},
{
'ep_country_slug': 'Northern-Ireland',
'ep_legislature_slug': 'Assembly'
},
]
pp_data = Popolo()
for legislature in legislatures_to_fetch:
ep_country = EveryPolitician().country(legislature['ep_country_slug'])
ep_legislature = ep_country.legislature(legislature['ep_legislature_slug'])
ep_people = ep_legislature.popolo().persons
logger.info('{}/{}: Found {} people.'.format(legislature['ep_country_slug'],
legislature['ep_legislature_slug'],
len(ep_people)
))
for ep_person in ep_people:
if ep_person.identifier_value('parlparse') and ep_person.wikidata:
pp_person = pp_data.get_person(id=ep_person.identifier_value('parlparse'))
pp_wikidata_identifier = getPersonIdentifierBySchema(pp_person, 'wikidata')
def test_get_a_country_legislature_neither_found(self,
patched_requests_get):
ep = EveryPolitician()
with pytest.raises(NotFound):
ep.country_legislature('FOO', 'FOO')
