def setUp(self):
self.client = EDemocracyClient()
# Set some fixtures
self.group_a_members_json = json.dumps(['member0', 'member1'])
self.member_profile = {
"id": "member0",
"givenName": "First",
"familyName": "Last",
"fn": "First Last",
"biography": "This is my life",
"neighbourhood": "Neighborhood",
"region": "Region",
"locality": "Locality",
"countryName": "Country",
"adr_postal_code": "12345",
"url": "http://example.com",
"org": "Organization",
"org_url": "http://organization.example.com",
"tz": None,
"email": ["*****@*****.**", "*****@*****.**"]
}
self.member_profile_json = json.dumps(self.member_profile)
self.group_a_messages = ['message1', 'message2']
self.group_a_messages_list_json = json.dumps(self.group_a_messages)
self.group_a_message0 = """
This is a message
It Has Lines
"""
with open('tests/fixtures/pages/groups.html') as f:
self.group_page_html = f.read()
with open('tests/fixtures/pages/messages_export.html') as f:
self.group_message_export_html = f.read()
def test_init_with_existing_client(self, mr):
self.client.session.cookies['__ac'] = 'foo'
self.client.session.cookies['SERVERID'] = 'bar'
new_client = EDemocracyClient(self.client)
self.assertEqual('foo', new_client.session.cookies['__ac'])
self.assertEqual('bar', new_client.session.cookies['SERVERID'])
def sync_group_members_and_profiles():
(username, password) = get_username_password()
with EDemocracyClient() as master_client, \
Store(DB_PATH) as master_store:
try:
# Login to the site; worker clients will use the same server
# session
master_client.login(username, password)
# Fetch list of groups
logger.info("Fetching list of groups")
groups = master_client.get_groups()
logger.info("Fetching list of groups complete")
# Multithreaded sync of group membership
logger.info("%i groups to sync membership of" % len(groups))
Threaded(Sync.group_members, groups, master_client, DB_PATH)()
logger.info("Group membership syncing complete")
# List of all members to get profiles for
members = master_store.fetch_all_unique_group_members()
logger.info("%i member profiles to sync" % len(members))
# Multithreaded sync of member profiles
Threaded(Sync.member_profile, members, master_client, DB_PATH)()
logger.info("Member Profile syncing complete")
finally:
master_client.logout()
def sync_empty_messages():
(username, password) = get_username_password()
with EDemocracyClient() as master_client, \
Store(DB_PATH) as master_store:
try:
# Login to the site; worker clients will use the same server
# session
master_client.login(username, password)
# Get empty messages in the store
messages = master_store.fetch_empty_group_messages()
# Multithreaded sync of message IDs
logger.info("%i message bodies to sync" % len(messages))
Threaded(Sync.message, messages, master_client, DB_PATH)()
logger.info("Message syncing complete")
finally:
master_client.logout()
def sync_message_ids_for_all_months():
def get_months(group, client):
logger.info("Fetching months for group %s" % group)
try:
time.sleep(0.5)
return client.get_message_months_of_group(group)
except Exception as e:
logger.exception(e)
return []
(username, password) = get_username_password()
with EDemocracyClient() as master_client, \
Store(DB_PATH) as master_store:
try:
# Login to the site; worker clients will use the same server
# session
master_client.login(username, password)
# Get groups already in the store
groups = master_store.fetch_all_groups()
# Get months to fetch for each group
# This does a serial fetch of months from each group.
logger.info(
"Gathering all months which have posts from all groups")
args = [
{'group': group, 'month': month}
for group in groups
for month in get_months(group, master_client)
]
logger.info("Finished gathering all months from all groups")
# Multithreaded sync of message IDs
logger.info("%i groups X months to get message IDs for" %
len(args))
Threaded(Sync.message_ids_of_group_and_month, args,
master_client, DB_PATH)()
logger.info("Group message ID syncing complete")
finally:
master_client.logout()
def sync_message_ids_for_month(month):
(username, password) = get_username_password()
logger.info("Getting ids for month %s" % month)
with EDemocracyClient() as master_client, \
Store(DB_PATH) as master_store:
try:
# Login to the site; worker clients will use the same server
# session
master_client.login(username, password)
# Get groups already in the store
groups = master_store.fetch_all_groups()
args = [{'group': group, 'month': month} for group in groups]
# Multithreaded sync of message IDs
logger.info("%i groups to message IDs for" % len(groups))
Threaded(Sync.message_ids_of_group_and_month, args,
master_client, DB_PATH)()
logger.info("Group message ID syncing complete")
finally:
master_client.logout()
def worker(func, queue, master_client, db_path, current_count,
total_count):
client = EDemocracyClient(master_client)
with Store(db_path) as store:
while True:
try:
item = queue.get(False)
except Empty:
break
with current_count.get_lock():
current_count.value += 1
logger.info("Syncing item %s of %s" %
(current_count.value, total_count.value))
try:
func(item, client, store)
except Exception as e:
logger.exception(e)
finally:
queue.task_done()
time.sleep(random.uniform(0.5, 1.5))
def setUp(self):
self.client = EDemocracyClient()
self.store = NonCallableMock()
# Set some fixtures
self.member_profile = {
"id": "member0",
"givenName": "First",
"familyName": "Last",
"fn": "First Last",
"biography": "This is my life",
"neighbourhood": "Neighborhood",
"region": "Region",
"locality": "Locality",
"countryName": "Country",
"adr_postal_code": "12345",
"url": "http://example.com",
"org": "Organization",
"org_url": "http://organization.example.com",
"tz": None,
"email": ["*****@*****.**", "*****@*****.**"]
}
self.group_messages = {'message0': 'Hello', 'message1': 'There'}
class EDemocracyClientTestCase(unittest.TestCase):
def setUp(self):
self.client = EDemocracyClient()
# Set some fixtures
self.group_a_members_json = json.dumps(['member0', 'member1'])
self.member_profile = {
"id": "member0",
"givenName": "First",
"familyName": "Last",
"fn": "First Last",
"biography": "This is my life",
"neighbourhood": "Neighborhood",
"region": "Region",
"locality": "Locality",
"countryName": "Country",
"adr_postal_code": "12345",
"url": "http://example.com",
"org": "Organization",
"org_url": "http://organization.example.com",
"tz": None,
"email": ["*****@*****.**", "*****@*****.**"]
}
self.member_profile_json = json.dumps(self.member_profile)
self.group_a_messages = ['message1', 'message2']
self.group_a_messages_list_json = json.dumps(self.group_a_messages)
self.group_a_message0 = """
This is a message
It Has Lines
"""
with open('tests/fixtures/pages/groups.html') as f:
self.group_page_html = f.read()
with open('tests/fixtures/pages/messages_export.html') as f:
self.group_message_export_html = f.read()
def test_init_with_existing_client(self, mr):
self.client.session.cookies['__ac'] = 'foo'
self.client.session.cookies['SERVERID'] = 'bar'
new_client = EDemocracyClient(self.client)
self.assertEqual('foo', new_client.session.cookies['__ac'])
self.assertEqual('bar', new_client.session.cookies['SERVERID'])
########################################
# Group Membership
########################################
def test_get_groups(self, mr):
mr.get('http://forums.e-democracy.org/groups/',
text=self.group_page_html)
expectedGroups = [
'another_group', 'city-central', 'city-issues', 'design', 'hub',
'other_secret', 'secret'
]
self.assertEqual(self.client.get_groups(), expectedGroups)
def test_get_groups_retrieval_error(self, mr):
mr.get('http://forums.e-democracy.org/groups/', status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.get_groups()
def test_get_group_members(self, mr):
mr.get('http://forums.e-democracy.org/groups/a_group/members.json',
text=self.group_a_members_json)
expectedMembers = ['member0', 'member1']
self.assertEqual(self.client.get_group_members('a_group'),
expectedMembers)
def test_group_members_retrieval_error(self, mr):
mr.get('http://forums.e-democracy.org/groups/a_group/members.json',
status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.get_group_members('a_group')
########################################
# Member Profiles
########################################
def test_get_profile_of_member(self, mr):
mr.get('http://forums.e-democracy.org/p/member0/profile.json',
text=self.member_profile_json)
self.assertEqual(self.client.get_profile_of_member('member0'),
self.member_profile)
def test_get_profile_of_member_retrieval_error(self, mr):
mr.get('http://forums.e-democracy.org/p/member0/profile.json',
status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.get_profile_of_member('member0')
########################################
# Group Messages
########################################
def test_get_messages_of_group_and_month(self, mr):
mr.get(
'http://forums.e-democracy.org/groups/a_group/messages/'
'gs-group-messages-export-posts.json?month=201801',
text=self.group_a_messages_list_json)
self.assertCountEqual(
self.client.get_messages_of_group_and_month('a_group', '201801'),
self.group_a_messages)
def test_get_messages_of_group_and_month_retrieval_error(self, mr):
mr.get(
'http://forums.e-democracy.org/groups/a_group/messages/'
'gs-group-messages-export-posts.json?month=201801',
status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.get_messages_of_group_and_month('a_group', '201801')
def test_get_message_of_group(self, mr):
mr.get(
'http://forums.e-democracy.org/groups/a_group/messages/'
'gs-group-messages-export-mbox/message0',
text=self.group_a_message0)
self.assertCountEqual(
self.client.get_message_of_group('a_group', 'message0'),
self.group_a_message0)
def test_get_message_of_group_retrieval_error(self, mr):
mr.get(
'http://forums.e-democracy.org/groups/a_group/messages/'
'gs-group-messages-export-mbox/message0',
status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.get_message_of_group('a_group', 'message0')
def test_get_message_months_of_group(self, mr):
mr.get('http://forums.e-democracy.org/groups/hub/messages/export.html',
text=self.group_message_export_html)
expected_months = ['201510', '201403']
self.assertEqual(self.client.get_message_months_of_group('hub'),
expected_months)
def test_get_message_months_of_group_retrieval_error(self, mr):
mr.get('http://forums.e-democracy.org/groups/hub/messages/export.html',
status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.get_message_months_of_group('hub')
########################################
# Log In
########################################
def test_login_success(self, mr):
mr.post('http://forums.e-democracy.org/login.html',
status_code=302,
cookies={'__ac': 'session'},
headers={'location': '/'})
try:
self.client.login('foo', 'bar')
except EDemocracyLoginException:
self.fail('Test Login Failed')
def test_login_failure(self, mr):
mr.post('http://forums.e-democracy.org/login.html', status_code=200)
with self.assertRaises(EDemocracyLoginException):
self.client.login('foo', 'bar')
def test_login_error(self, mr):
mr.post('http://forums.e-democracy.org/login.html', status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.login('foo', 'bar')
def test_whoami_logged_in(self, mr):
mr.head('http://forums.e-democracy.org/p/',
status_code=302,
headers={'location': '/p/username'})
self.assertEqual(self.client.whoami(), 'username')
def test_whoami_logged_out(self, mr):
mr.head('http://forums.e-democracy.org/p/',
status_code=302,
headers={'location': '/login.html?came_from=/p'})
self.assertEqual(self.client.whoami(), None)
def test_whoami_error(self, mr):
mr.head('http://forums.e-democracy.org/p/', status_code=500)
with self.assertRaises(EDemocracyClientException):
self.client.whoami()