def test_use_build_script_instead_of_docker_file_if_available(self, docker_mock, res_mock, tempDir_mock):
# given
provider = DockerProvider(self._adapter, 'leap_provider', self._leap_provider_x509)
tempBuildDir = TempDir()
try:
tempDir_mock.return_value = tempBuildDir
tempBuildDir_name = tempBuildDir.name
with NamedTemporaryFile() as file:
res_mock.resource_exists.return_value = True
res_mock.resource_string.return_value = '#!/bin/bash\necho %s $PWD > %s' % (file.name, file.name)
# when
provider.initialize()
# then
res_mock.resource_exists.assert_called_with('pixelated.resources', 'init-pixelated-docker-context.sh')
res_mock.resource_string.assert_called_with('pixelated.resources', 'init-pixelated-docker-context.sh')
with open(file.name, "r") as input:
data = input.read().replace('\n', '')
self.assertEqual('%s %s' % (file.name, os.path.realpath(tempBuildDir_name)), data)
docker_mock.return_value.build.assert_called_once_with(path=tempBuildDir_name, tag='pixelated:latest', fileobj=None)
finally:
tempBuildDir.dissolve()
class SearchEngineTest(unittest.TestCase):
def setUp(self):
self.tempdir = TempDir()
self.agent_home = self.tempdir.name
def tearDown(self):
self.tempdir.dissolve()
def test_encoding(self):
# given
se = SearchEngine(INDEX_KEY, self.agent_home)
headers = {
"From": "*****@*****.**",
"To": "=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=",
"Cc": "=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=",
"Subject": "Some test mail",
}
# when
se.index_mail(test_helper.pixelated_mail(extra_headers=headers, chash="mailid"))
result = se.search("folker")
self.assertEqual((["mailid"], 1), result)
class SearchEngineTest(unittest.TestCase):
def setUp(self):
self.tempdir = TempDir()
self.agent_home = self.tempdir.name
def tearDown(self):
self.tempdir.dissolve()
def test_encoding(self):
# given
se = SearchEngine(INDEX_KEY, self.agent_home)
headers = {
'From': '*****@*****.**',
'To':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
}
# when
se.index_mail(
LeapMail('mailid', 'INBOX', headers=headers)
) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search('folker')
self.assertEqual((['mailid'], 1), result)
class SearchEngineTest(unittest.TestCase):
def setUp(self):
self.tempdir = TempDir()
self.agent_home = self.tempdir.name
def tearDown(self):
self.tempdir.dissolve()
def test_encoding(self):
# given
se = SearchEngine(INDEX_KEY, self.agent_home)
headers = {
'From': '*****@*****.**',
'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
}
# when
se.index_mail(LeapMail('mailid', 'INBOX', headers=headers)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search('folker')
self.assertEqual((['mailid'], 1), result)
def test_use_build_script_instead_of_docker_file_if_available(
self, docker_mock, res_mock, tempDir_mock):
# given
provider = DockerProvider(self._adapter, 'leap_provider',
self._leap_provider_x509)
tempBuildDir = TempDir()
try:
tempDir_mock.return_value = tempBuildDir
tempBuildDir_name = tempBuildDir.name
with NamedTemporaryFile() as file:
res_mock.resource_exists.return_value = True
res_mock.resource_string.return_value = '#!/bin/bash\necho %s $PWD > %s' % (
file.name, file.name)
# when
provider.initialize()
# then
res_mock.resource_exists.assert_called_with(
'pixelated.resources', 'init-pixelated-docker-context.sh')
res_mock.resource_string.assert_called_with(
'pixelated.resources', 'init-pixelated-docker-context.sh')
with open(file.name, "r") as input:
data = input.read().replace('\n', '')
self.assertEqual(
'%s %s' %
(file.name, os.path.realpath(tempBuildDir_name)), data)
docker_mock.return_value.build.assert_called_once_with(
path=tempBuildDir_name,
tag='pixelated:latest',
fileobj=None)
finally:
tempBuildDir.dissolve()
class SearchEngineTest(unittest.TestCase):
def setUp(self):
self.tempdir = TempDir()
self.agent_home = self.tempdir.name
def tearDown(self):
self.tempdir.dissolve()
def test_index_mail_secured_by_lock(self):
# given
soledad_querier = mock()
lock_stub = LockStub()
when(soledad_querier).get_index_masterkey().thenReturn(INDEX_KEY)
self.assertEqual(INDEX_KEY, soledad_querier.get_index_masterkey())
se = SearchEngine(soledad_querier, self.agent_home)
se._write_lock = lock_stub
headers = {
'From': '*****@*****.**',
'To': '*****@*****.**',
'Subject': 'Some test mail',
}
# when
se.index_mail(test_helper.pixelated_mail(extra_headers=headers))
# then
self.assertTrue(lock_stub.called)
def test_encoding(self):
# given
soledad_querier = mock()
when(soledad_querier).get_index_masterkey().thenReturn(INDEX_KEY)
se = SearchEngine(soledad_querier, self.agent_home)
headers = {
'From': '*****@*****.**',
'To':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
}
# when
se.index_mail(
test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search('folker')
self.assertEqual((['mailid'], 1), result)
class SearchEngineTest(unittest.TestCase):
def setUp(self):
self.tempdir = TempDir()
self.agent_home = self.tempdir.name
def tearDown(self):
self.tempdir.dissolve()
def test_index_mail_secured_by_lock(self):
# given
soledad_querier = mock()
lock_stub = LockStub()
when(soledad_querier).get_index_masterkey().thenReturn(INDEX_KEY)
self.assertEqual(INDEX_KEY, soledad_querier.get_index_masterkey())
se = SearchEngine(soledad_querier, self.agent_home)
se._write_lock = lock_stub
headers = {
'From': '*****@*****.**',
'To': '*****@*****.**',
'Subject': 'Some test mail',
}
# when
se.index_mail(test_helper.pixelated_mail(extra_headers=headers))
# then
self.assertTrue(lock_stub.called)
def test_encoding(self):
# given
soledad_querier = mock()
when(soledad_querier).get_index_masterkey().thenReturn(INDEX_KEY)
se = SearchEngine(soledad_querier, self.agent_home)
headers = {
'From': '*****@*****.**',
'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
}
# when
se.index_mail(test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search('folker')
self.assertEqual((['mailid'], 1), result)
class SearchEngineTest(unittest.TestCase):
def setUp(self):
self.tempdir = TempDir()
self.user_home = self.tempdir.name
def tearDown(self):
self.tempdir.dissolve()
def test_headers_encoding(self):
# given
se = SearchEngine(INDEX_KEY, self.user_home)
headers = {
'From': '*****@*****.**',
'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
}
# when
se.index_mail(LeapMail('mailid', 'INBOX', headers=headers)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search('folker')
self.assertEqual((['mailid'], 1), result)
def test_contents_encoding_accents(self):
# given
se = SearchEngine(INDEX_KEY, self.user_home)
headers = {
'From': '*****@*****.**',
'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
}
body = "When doing the search, it's not possible to find words with graphical accents, e.g.: 'coração', 'é', 'Fièvre', La Pluie d'été, 'não'."
# when
se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search(u"'coração', 'é',")
self.assertEqual((['mailid'], 1), result)
result = se.search(u"Fièvre")
self.assertEqual((['mailid'], 1), result)
result = se.search(u"été")
self.assertEqual((['mailid'], 1), result)
def test_contents_encoding_special_characters(self):
# given
se = SearchEngine(INDEX_KEY, self.user_home)
headers = {
'From': '*****@*****.**',
'To': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc': '=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
}
body = "When doing the search, 您好 أهلا"
# when
se.index_mail(LeapMail('mailid', 'INBOX', headers=headers, body=body)) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search(u"您好")
self.assertEqual((['mailid'], 1), result)
result = se.search(u"أهلا")
self.assertEqual((['mailid'], 1), result)
class SmokeTest(unittest.TestCase):
__slots__ = ('_run_method', '_shutdown_method', '_thread_name', '_thread')
class Server(object):
def __init__(self, run_method, shutdown_method, thread_name=None):
self._run_method = run_method
self._shutdown_method = shutdown_method
self._thread_name = thread_name
self._thread = None
def _start_server(self):
self._thread = threading.Thread(target=self._run_method)
self._thread.setDaemon(True)
if self._thread_name:
self._thread.setName(self._thread_name)
self._thread.start()
def __enter__(self):
self._start_server()
time.sleep(0.3) # let server start
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._shutdown_method()
self._thread.join()
self._kill_subprocesses()
def _kill_subprocesses(self):
for child in psutil.Process(os.getpid()).children():
try:
p = psutil.Process(child.pid)
p.kill()
except psutil.Error:
pass
def setUp(self):
self._tmpdir = TempDir()
self.ssl_request = requests.Session()
self.ssl_request.mount('https://', EnforceTLSv1Adapter())
def tearDown(self):
self._tmpdir.dissolve()
def _dispatcher_manager(self):
fake_mailpile = os.path.join(os.path.dirname(__file__), 'fake_mailpile.py')
ssl_config = SSLConfig(certfile(), keyfile())
provider_ca = None
server = DispatcherManager(self._tmpdir.name, fake_mailpile, ssl_config, 'leap provider hostname', provider_ca, mailpile_virtualenv=INHERIT)
return SmokeTest.Server(server.serve_forever, server.shutdown, thread_name='PixelatedServerManager')
def _dispatcher_proxy(self):
dispatcher = DispatcherProxy(PixelatedDispatcherClient('localhost', DEFAULT_PORT, cacert=cafile(), assert_hostname=False), port=12345, certfile=certfile(),
keyfile=keyfile())
return SmokeTest.Server(dispatcher.serve_forever, dispatcher.shutdown, thread_name='PixelatedDispatcherProxy')
def _method(self, method, url, form_data=None, json_data=None, timeout=2.0):
if json_data:
headers = {'content-type': 'application/json'}
data = json.dumps(json_data)
cookies = None
else:
cookies = {'_xsrf': '2|7586b241|47c876d965112a2f547c63c95cbc44b1|1402910163'}
headers = None
data = form_data.copy()
data['_xsrf'] = '2|7586b241|47c876d965112a2f547c63c95cbc44b1|1402910163'
return method(url, data=data, headers=headers, cookies=cookies, timeout=timeout, verify=cafile())
def get(self, url):
return self.ssl_request.get(url, verify=cafile())
def put(self, url, form_data=None, json_data=None):
return self._method(self.ssl_request.put, url, form_data=form_data, json_data=json_data)
def post(self, url, form_data=None, json_data=None):
return self._method(self.ssl_request.post, url, form_data=form_data, json_data=json_data)
def test_dispatcher_run(self):
with self._dispatcher_manager():
self.assertSuccess(
self.post('https://localhost:4443/agents', json_data={'name': 'test', 'password': '******'}))
self.assertSuccess(self.get('https://localhost:4443/agents'), json_body={
'agents': [{'name': 'test', 'state': 'stopped', 'uri': 'http://localhost:4443/agents/test'}]})
self.assertSuccess(
self.put('https://localhost:4443/agents/test/state', json_data={'state': 'running'}))
self.assertSuccess(self.get('https://localhost:4443/agents/test/runtime'),
json_body={'state': 'running', 'port': 5000})
time.sleep(2) # let mailpile start
self.assertSuccess(self.get('http://localhost:5000/'))
self.assertSuccess(
self.put('https://localhost:4443/agents/test/state', json_data={'state': 'stopped'}))
def test_dispatcher_starts(self):
with self._dispatcher_proxy():
self.assertSuccess(self.get('https://localhost:12345/auth/login'))
def test_server_dispatcher_combination(self):
with self._dispatcher_manager():
with self._dispatcher_proxy():
# add user
self.assertSuccess(
self.post('https://localhost:4443/agents', json_data={'name': 'test', 'password': '******'}))
# try to login with agent down
# self.assertError(302, self.post('https://localhost:12345/auth/login',
# form_data={'username': '******', 'password': '******'}))
# start agent
self.assertSuccess(
self.put('https://localhost:4443/agents/test/state', json_data={'state': 'running'}))
# let mailpile start
time.sleep(1)
self.assertMemoryUsage(
self.get('https://localhost:4443/stats/memory_usage'))
try:
# try to login with agent up
self.assertSuccess(self.post('https://localhost:12345/auth/login',
form_data={'username': '******', 'password': '******'}),
body='Hello World!')
finally:
# shutdown mailple
self.put('https://localhost:4443/agents/test/state', json_data={'state': 'stopped'})
def assertSuccess(self, response, body=None, json_body=None):
status = response.status_code
self.assertTrue(200 <= status < 300, msg='%d: %s' % (response.status_code, response.reason))
if body:
self.assertEqual(body, response.content)
if json_body:
self.assertEqual(json_body, response.json())
def assertError(self, error_code, response):
self.assertEqual(error_code, response.status_code,
'Expected status code %d but got %d' % (error_code, response.status_code))
def assertMemoryUsage(self, response):
self.assertSuccess(response)
usage = response.json()
self.assertEqual(1, len(usage['agents']))
class DockerProviderTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self._adapter = PixelatedDockerAdapter()
def tearDown(self):
self._tmpdir.dissolve()
def test_constructor_expects_docker_url(self):
DockerProvider(self.root_path, self._adapter, 'leap_provider', 'some docker url')
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_builds_docker_image(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
dockerfile = pkg_resources.resource_string('pixelated.resources', 'Dockerfile.pixelated')
# when
DockerProvider(self.root_path, self._adapter, 'leap_provider', 'some leap ca', 'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url="some docker url")
client.build.assert_called_once_with(path=None, fileobj=StringIOMatcher(dockerfile), tag='pixelated:latest')
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_skips_image_build_if_available(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = [{'Created': 1404833111, 'VirtualSize': 297017244, 'ParentId': '57885511c8444c2b89743bef8b89eccb65f302b2a95daa95dfcc9b972807b6db', 'RepoTags': ['pixelated:latest'], 'Id': 'b4f10a2395ab8dfc5e1c0fae26fa56c7f5d2541debe54263105fe5af1d263189', 'Size': 181956643}]
# when
DockerProvider(self.root_path, self._adapter, 'leap_provider', 'some docker url').initialize()
# then
self.assertFalse(client.build.called)
@patch('pixelated.provider.docker.docker.Client')
def test_reports_initializing_while_initialize_is_running(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
def build(path, fileobj, tag):
sleep(0.2)
return []
client.build.side_effect = build
provider = DockerProvider(self.root_path, self._adapter, 'some provider', 'some provider ca', 'some docker url')
self.assertTrue(provider.initializing)
# when
t = Thread(target=provider.initialize) # move to thread so that initializing behaviour is observable
t.start()
# then
sleep(0.1)
self.assertTrue(provider.initializing)
t.join()
self.assertFalse(provider.initializing)
@patch('pixelated.provider.docker.LeapProvider')
@patch('pixelated.provider.docker.LeapSecureRemotePassword')
def test_throws_initializing_exception_while_initializing(self, leap_provider_mock, leap_srp_mock):
# given
provider = DockerProvider(self.root_path, self._adapter, 'provider url', 'provider ca', 'some docker url')
# when/then
self.assertRaises(ProviderInitializingException, provider.start, 'test')
self.assertRaises(ProviderInitializingException, provider.add, 'test', 'password')
self.assertRaises(ProviderInitializingException, provider.remove, 'test')
self.assertRaises(ProviderInitializingException, provider.list)
self.assertRaises(ProviderInitializingException, provider.list_running)
self.assertRaises(ProviderInitializingException, provider.stop, 'test')
self.assertRaises(ProviderInitializingException, provider.status, 'test')
self.assertRaises(ProviderInitializingException, provider.authenticate, 'test', 'password')
self.assertRaises(ProviderInitializingException, provider.memory_usage)
def test_add(self):
self._create_initialized_provider(self.root_path, self._adapter, 'some docker url').add('test', 'password')
instance_path = join(self.root_path, 'test')
data_dir = join(instance_path, 'data')
cfg_file = join(instance_path, BaseProvider.CFG_FILE_NAME)
self.assertTrue(isdir(instance_path), 'No folder for user has been created')
self.assertTrue(isdir(data_dir), 'No folder for pixelated has been created')
self.assertTrue(isfile(cfg_file), 'No config file had been created')
@patch('pixelated.provider.docker.docker.Client')
def test_that_non_existing_instance_cannot_be_started(self, docker_mock):
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
self.assertRaises(InstanceNotFoundError, provider.start, 'test')
@patch('pixelated.provider.docker.docker.Client')
def test_that_instance_can_be_started(self, docker_mock):
client = docker_mock.return_value
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
prepare_pixelated_container = MagicMock()
container = MagicMock()
client.create_container.side_effect = [prepare_pixelated_container, container]
client.wait.return_value = 0
provider.add('test', 'password')
provider.start('test')
client.create_container.assert_any_call('pixelated', '/bin/bash -l -c "/usr/bin/pixelated-user-agent --host 0.0.0.0 --port 4567 --dispatcher /mnt/user/credentials-fifo"', name='test', volumes=['/mnt/user'], ports=[4567], environment={'DISPATCHER_LOGOUT_URL': '/auth/logout'})
client.create_container.assert_any_call('pixelated', '/bin/true', name='pixelated_prepare', volumes=['/mnt/user'], environment={'DISPATCHER_LOGOUT_URL': '/auth/logout'})
data_path = join(self.root_path, 'test', 'data')
client.start.assert_any_call(container, binds={data_path: {'bind': '/mnt/user', 'ro': False}}, port_bindings={4567: 5000})
client.start.assert_any_call(prepare_pixelated_container, binds={data_path: {'bind': '/mnt/user', 'ro': False}})
@patch('pixelated.provider.docker.docker.Client')
def test_that_existing_container_gets_reused(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [[], [{u'Status': u'Exited (-1) About an hour ago', u'Created': 1405332375, u'Image': u'pixelated:latest', u'Ports': [], u'Command': u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"', u'Names': [u'/test'], u'Id': u'adfd4633fc42734665d7d98076b19b5f439648678b3b76db891f9d5072af50b6'}]]
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
container = MagicMock()
client.create_container.return_value = container
provider.add('test', 'password')
provider.start('test')
client.containers.assert_called_with(all=True)
self.assertFalse(client.build.called)
@patch('pixelated.provider.docker.docker.Client')
def test_running_containers_empty_if_none_started(self, docker_mock):
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
running = provider.list_running()
self.assertEqual([], running)
@patch('pixelated.provider.docker.docker.Client')
def test_running_returns_running_container(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [[], [], [{u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
provider.start('test')
running = provider.list_running()
self.assertEqual(['test'], running)
@patch('pixelated.provider.docker.docker.Client')
def test_a_container_cannot_be_started_twice(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [[], [], [{u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
provider.start('test')
self.assertRaises(InstanceAlreadyRunningError, provider.start, 'test')
@patch('pixelated.provider.docker.docker.Client')
def test_stopping_not_running_container_raises_value_error(self, docker_mock):
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
self.assertRaises(InstanceNotRunningError, provider.stop, 'test')
@patch('pixelated.provider.docker.docker.Client')
def test_stop_running_container(self, docker_mock):
# given
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 4567}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container], [container], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
provider.start('test')
# when
provider.stop('test')
# then
client.stop.assert_called_once_with(container, timeout=10)
self.assertFalse(5000 in provider._used_ports())
@patch('pixelated.provider.docker.docker.Client')
def test_stop_running_container_calls_kill_if_stop_times_out(self, docker_mock):
# given
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 4567}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container], [container], [container]]
client.wait.return_value = 0
client.stop.side_effect = requests.exceptions.Timeout
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
provider.start('test')
# when
provider.stop('test')
# then
client.stop.assert_called_once_with(container, timeout=10)
client.kill.assert_called_once_with(container)
@patch('pixelated.provider.docker.docker.Client')
def test_status_stopped(self, docker_mock):
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
self.assertEqual({'state': 'stopped'}, provider.status('test'))
@patch('pixelated.provider.docker.docker.Client')
def test_status_running(self, docker_mock):
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 33144}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
provider.start('test')
self.assertEqual({'state': 'running', 'port': 5000}, provider.status('test'))
def test_empty_list(self):
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
self.assertEqual([], provider.list())
def test_list(self):
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
self.assertEqual(['test'], provider.list())
@patch('pixelated.provider.docker.Process')
@patch('pixelated.provider.docker.docker.Client')
def test_memory_usage(self, docker_mock, process_mock):
# given
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 33144}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
info = {u'HostsPath': u'/var/lib/docker/containers/f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89/hosts', u'Created': u'2014-07-14T13:17:46.17558664Z', u'Image': u'f63df19194389be6481a174b36d291c483c8982d5c07485baa71a46b7f6582c8', u'Args': [], u'Driver': u'aufs', u'HostConfig': {u'PortBindings': {u'4567/tcp': [{u'HostPort': u'5000', u'HostIp': u'0.0.0.0'}]}, u'NetworkMode': u'', u'Links': None, u'LxcConf': None, u'ContainerIDFile': u'', u'Binds': [u'/tmp/multipile/folker:/mnt/user:rw'], u'PublishAllPorts': False, u'Dns': None, u'DnsSearch': None, u'Privileged': False, u'VolumesFrom': None}, u'MountLabel': u'', u'VolumesRW': {u'/mnt/user': True}, u'State': {u'Pid': 3250, u'Paused': False, u'Running': True, u'FinishedAt': u'0001-01-01T00:00:00Z', u'StartedAt': u'2014-07-14T13:17:46.601922899Z', u'ExitCode': 0}, u'ExecDriver': u'native-0.2', u'ResolvConfPath': u'/etc/resolv.conf', u'Volumes': {u'/mnt/user': u'/tmp/multipile/folker'}, u'Path': u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"', u'HostnamePath': u'/var/lib/docker/containers/f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89/hostname', u'ProcessLabel': u'', u'Config': {u'MemorySwap': 0, u'Hostname': u'f2cdb04277e9', u'Entrypoint': None, u'PortSpecs': None, u'Memory': 0, u'OnBuild': None, u'OpenStdin': False, u'Cpuset': u'', u'Env': [u'HOME=/', u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], u'User': u'', u'CpuShares': 0, u'AttachStdout': True, u'NetworkDisabled': False, u'WorkingDir': u'', u'Cmd': [u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"'], u'StdinOnce': False, u'AttachStdin': False, u'Volumes': {u'/mnt/user': {}}, u'Tty': False, u'AttachStderr': True, u'Domainname': u'', u'Image': u'pixelated', u'ExposedPorts': {u'4567/tcp': {}}}, u'Id': u'f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89', u'NetworkSettings': {u'Bridge': u'docker0', u'PortMapping': None, u'Gateway': u'172.17.42.1', u'IPPrefixLen': 16, u'IPAddress': u'172.17.0.14', u'Ports': {u'4567/tcp': [{u'HostPort': u'5000', u'HostIp': u'0.0.0.0'}]}}, u'Name': u'/folker'}
client = docker_mock.return_value
client.containers.return_value = [container]
client.inspect_container.return_value = info
psutil_mock = process_mock.return_value
psutil_mock.memory_info.return_value = pmem(1024, 2048)
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
# when
usage = provider.memory_usage()
# then
self.assertEqual({'total_usage': 1024,
'average_usage': 1024,
'agents': [
{'name': 'test', 'memory_usage': 1024}
]}, usage)
def test_that_existing_agents_are_autodiscovered(self):
agent = os.path.join(self.root_path, 'test')
os.mkdir(agent)
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
self.assertEqual(['test'], provider.list())
def test_authenticate(self):
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
self.assertTrue(provider.authenticate('test', 'password'))
self.assertFalse(provider.authenticate('test', 'something else'))
def test_remove_error_if_not_exist(self):
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
self.assertRaises(ValueError, provider.remove, 'does_not_exist')
@patch('pixelated.provider.docker.docker.Client')
def test_remove(self, docker_mock):
# given
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
# when
provider.remove('test')
# then
self.assertFalse(exists(join(self.root_path, 'test')))
self.assertFalse('test' in provider.list())
@patch('pixelated.provider.docker.docker.Client')
def test_cannot_remove_while_running(self, docker_mock):
# given
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 4567}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self.root_path, self._adapter, 'some docker url')
provider.add('test', 'password')
provider.start('test')
# when/then
self.assertRaises(ValueError, provider.remove, 'test')
@patch('pixelated.provider.docker.TempDir')
@patch('pixelated.provider.docker.pkg_resources')
@patch('pixelated.provider.docker.docker.Client')
def test_use_build_script_instead_of_docker_file_if_available(self, docker_mock, res_mock, tempDir_mock):
# given
provider = DockerProvider(self.root_path, self._adapter, 'leap_provider', 'some docker url')
tempBuildDir = TempDir()
try:
tempDir_mock.return_value = tempBuildDir
tempBuildDir_name = tempBuildDir.name
with NamedTemporaryFile() as file:
res_mock.resource_exists.return_value = True
res_mock.resource_string.return_value = '#!/bin/bash\necho %s $PWD > %s' % (file.name, file.name)
# when
provider.initialize()
# then
res_mock.resource_exists.assert_called_with('pixelated.resources', 'init-pixelated-docker-context.sh')
res_mock.resource_string.assert_called_with('pixelated.resources', 'init-pixelated-docker-context.sh')
with open(file.name, "r") as input:
data = input.read().replace('\n', '')
self.assertEqual('%s %s' % (file.name, os.path.realpath(tempBuildDir_name)), data)
docker_mock.return_value.build.assert_called_once_with(path=tempBuildDir_name, tag='pixelated:latest', fileobj=None)
finally:
tempBuildDir.dissolve()
@patch('pixelated.provider.docker.docker.Client')
def test_that_authenticate_writes_password_to_fifo(self, docker_mock):
provider = DockerProvider(self.root_path, self._adapter, 'leap_provider_hostname', 'some docker url')
provider.initialize()
provider.add('test', 'password')
provider.authenticate('test', 'password')
fifo_file = join(self.root_path, 'test', 'data', 'credentials-fifo')
self.assertTrue(stat.S_ISFIFO(os.stat(fifo_file).st_mode))
with open(fifo_file, 'r') as fifo:
config = json.loads(fifo.read())
self.assertEqual('leap_provider_hostname', config['leap_provider_hostname'])
self.assertEqual('test', config['user'])
self.assertEqual('password', config['password'])
self._assert_file_gets_deleted(fifo_file)
def _assert_file_gets_deleted(self, filename):
start = clock()
timeout = 5
while (clock() - start) < timeout and exists(filename):
sleep(0.1)
self.assertFalse(exists(filename))
@patch('pixelated.provider.docker.docker.Client')
def footest_that_authenticate_deletes_fifo_after_timeout(self, docker_mock):
provider = DockerProvider(self.root_path, self._adapter, 'some docker url')
provider.initialize()
provider.add('test', 'password')
fifo_file = join(self.root_path, 'test', 'data', 'credentials-fifo')
provider.authenticate('test', 'password')
sleep(3)
self.assertFalse(stat.S_ISFIFO(os.stat(fifo_file).st_mode))
def _create_initialized_provider(self, root_path, adapter, docker_url=DockerProvider.DEFAULT_DOCKER_URL):
provider = DockerProvider(root_path, adapter, 'leap_provider_hostname', 'leap provider ca', docker_url)
provider._initializing = False
return provider
class SmokeTest(unittest.TestCase):
__slots__ = ('_run_method', '_shutdown_method', '_thread_name', '_thread')
class Server(object):
def __init__(self, run_method, shutdown_method, thread_name=None):
self._run_method = run_method
self._shutdown_method = shutdown_method
self._thread_name = thread_name
self._thread = None
def _start_server(self):
self._thread = threading.Thread(target=self._run_method)
self._thread.setDaemon(True)
if self._thread_name:
self._thread.setName(self._thread_name)
self._thread.start()
def __enter__(self):
self._start_server()
time.sleep(0.3) # let server start
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._shutdown_method()
self._thread.join()
self._kill_subprocesses()
def _kill_subprocesses(self):
for child in psutil.Process(os.getpid()).children():
try:
p = psutil.Process(child.pid)
p.kill()
except psutil.Error:
pass
def setUp(self):
self._tmpdir = TempDir()
self.ssl_request = requests.Session()
self.ssl_request.mount('https://', EnforceTLSv1Adapter())
def tearDown(self):
self._tmpdir.dissolve()
def _dispatcher_manager(self):
fake_mailpile = os.path.join(os.path.dirname(__file__),
'fake_mailpile.py')
ssl_config = SSLConfig(certfile(), keyfile())
provider_ca = None
server = DispatcherManager(self._tmpdir.name,
fake_mailpile,
ssl_config,
'leap provider hostname',
provider_ca,
mailpile_virtualenv=INHERIT)
return SmokeTest.Server(server.serve_forever,
server.shutdown,
thread_name='PixelatedServerManager')
def _dispatcher_proxy(self):
dispatcher = DispatcherProxy(PixelatedDispatcherClient(
'localhost', DEFAULT_PORT, cacert=cafile(), assert_hostname=False),
port=12345,
certfile=certfile(),
keyfile=keyfile())
return SmokeTest.Server(dispatcher.serve_forever,
dispatcher.shutdown,
thread_name='PixelatedDispatcherProxy')
def _method(self,
method,
url,
form_data=None,
json_data=None,
timeout=5.0):
if json_data:
headers = {'content-type': 'application/json'}
data = json.dumps(json_data)
cookies = None
else:
cookies = {
'_xsrf':
'2|7586b241|47c876d965112a2f547c63c95cbc44b1|1402910163'
}
headers = None
data = form_data.copy()
data[
'_xsrf'] = '2|7586b241|47c876d965112a2f547c63c95cbc44b1|1402910163'
return method(url,
data=data,
headers=headers,
cookies=cookies,
timeout=timeout,
verify=cafile())
def get(self, url):
return self.ssl_request.get(url, verify=cafile())
def put(self, url, form_data=None, json_data=None):
return self._method(self.ssl_request.put,
url,
form_data=form_data,
json_data=json_data)
def post(self, url, form_data=None, json_data=None):
return self._method(self.ssl_request.post,
url,
form_data=form_data,
json_data=json_data)
@patch('pixelated.manager.LeapCertificate')
@patch('pixelated.manager.LeapProvider')
def test_dispatcher_run(self, leap_provider_mock, leap_certificate_mock):
with self._dispatcher_manager():
self.assertSuccess(
self.post('https://*****:*****@patch('pixelated.manager.LeapCertificate')
@patch('pixelated.manager.LeapProvider')
def test_server_dispatcher_combination(self, leap_provider_mock,
leap_certificate_mock):
with self._dispatcher_manager():
with self._dispatcher_proxy():
# add user
self.assertSuccess(
self.post('https://localhost:4443/agents',
json_data={
'name': 'test',
'password': '******'
}))
# try to login with agent down
# self.assertError(302, self.post('https://localhost:12345/auth/login',
# form_data={'username': '******', 'password': '******'}))
# start agent
self.assertSuccess(
self.put('https://localhost:4443/agents/test/state',
json_data={'state': 'running'}))
# let mailpile start
time.sleep(1)
self.assertMemoryUsage(
self.get('https://localhost:4443/stats/memory_usage'))
try:
# try to login with agent up
self.assertSuccess(self.post(
'https://localhost:12345/auth/login',
form_data={
'username': '******',
'password': '******'
}),
body='Hello World!')
finally:
# shutdown mailple
self.put('https://localhost:4443/agents/test/state',
json_data={'state': 'stopped'})
def assertSuccess(self, response, body=None, json_body=None):
status = response.status_code
self.assertTrue(200 <= status < 300,
msg='%d: %s' % (response.status_code, response.reason))
if body:
self.assertEqual(body, response.content)
if json_body:
self.assertEqual(json_body, response.json())
def assertError(self, error_code, response):
self.assertEqual(
error_code, response.status_code,
'Expected status code %d but got %d' %
(error_code, response.status_code))
def assertMemoryUsage(self, response):
self.assertSuccess(response)
usage = response.json()
self.assertEqual(1, len(usage['agents']))
class UsersTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self.users = Users(self.root_path)
def test_that_constructor_throws_exception_if_root_path_does_not_exist(
self):
self.assertRaises(ValueError, Users, '/does/not/exist')
def tearDown(self):
self._tmpdir.dissolve()
def test_add_user(self):
self.users.add('name')
def test_user_cannot_be_added_twice(self):
self.users.add('name')
self.assertRaises(UserAlreadyExistsError, self.users.add, 'name')
def test_list_empty(self):
self.assertEqual([], self.users.list())
def test_list_single_user(self):
self.users.add('name')
self.assertEqual(['name'], self.users.list())
def test_get_user_config_throws_exception_if_user_not_exists(self):
self.assertRaises(UserNotExistError, self.users.config, 'name')
def test_add_user_creates_data_folder_and_config(self):
self.users.add('name')
data_path = self._data_path('name')
config_file = self._config_file('name')
self.assertTrue(exists(data_path))
self.assertTrue(exists(config_file))
def test_add_user_does_not_override_existing_config(self):
data_path = self._data_path('name')
config_file = self._config_file('name')
mkdir(data_path)
with open(config_file, 'w') as fd:
fd.write('[section]\none = first value\ntwo = second value\n\n')
self.users.add('name')
with open(self._config_file('name'), 'r') as fd:
self.assertEqual(
'[section]\none = first value\ntwo = second value\n\n',
fd.read())
def test_add_validates_username(self):
self.users.add('a_name.with-valid-chars')
with self.assertRaises(ValueError):
self.users.add('CapitalLetters')
with self.assertRaises(ValueError):
self.users.add('some/name/with/slashes')
with self.assertRaises(ValueError):
self.users.add('name with spaces')
with self.assertRaises(ValueError):
self.users.add('name=with%&')
def test_loads_all_existing_users_on_startup(self):
user1_data_path = self._data_path('user1')
mkdir(user1_data_path)
user1_config_path = self._config_file('user1')
with open(user1_config_path, 'w') as fd:
fd.write('[section]\none = first value\ntwo = second value\n\n')
users = Users(self.root_path)
self.assertEqual(['user1'], users.list())
self.assertEqual('first value', users.config('user1')['section.one'])
def test_update_user_config(self):
self.users.add('name')
config = self.users.config('name')
config['foo.bar'] = 'some value'
self.users.update_config(config)
config = self.users.config('name')
self.assertEqual('some value', config['foo.bar'])
def test_has_user(self):
self.users.add('name')
self.assertTrue(self.users.has_user('name'))
self.assertFalse(self.users.has_user('other'))
def test_has_user_config(self):
self.users.add('name')
self.assertTrue(self.users.has_user('name'))
self.assertTrue(self.users.has_user_config('name'))
shutil.rmtree(self._data_path('name'))
self.assertTrue(self.users.has_user('name'))
self.assertFalse(self.users.has_user_config('name'))
def _data_path(self, username):
return join(self.root_path, username)
def _config_file(self, username):
return join(self._data_path(username), 'user.cfg')
class RESTfulServerTest(unittest.TestCase):
mock_provider = None
ssl_config = None
server = None
@classmethod
def setUpClass(cls):
RESTfulServerTest.mock_provider = MagicMock(spec=Provider)
RESTfulServerTest.mock_users = MagicMock(spec=Users)
RESTfulServerTest.mock_authenticator = MagicMock(spec=Authenticator)
RESTfulServerTest.ssl_config = SSLConfig(certfile(),
keyfile())
RESTfulServerTest.server = RESTfulServer(RESTfulServerTest.ssl_config, RESTfulServerTest.mock_users, RESTfulServerTest.mock_authenticator, RESTfulServerTest.mock_provider)
RESTfulServerTest.server.server_forever_in_backgroud()
time.sleep(1) # let it get up to speed
@classmethod
def tearDownClass(cls):
RESTfulServerTest.server.shutdown()
print 'Stopped test server'
def setUp(self):
self.mock_provider = RESTfulServerTest.mock_provider
self.mock_provider.reset_mock()
self.mock_users.reset_mock()
self.mock_authenticator.reset_mock()
self.ssl_request = requests.Session()
self.ssl_request.mount('https://', EnforceTLSv1Adapter())
self._tmpdir = TempDir()
self._root_path = self._tmpdir.name
def tearDown(self):
self._tmpdir.dissolve()
def get(self, url):
return self.ssl_request.get(url, verify=cafile())
def put(self, url, data=None):
if data:
data = json.dumps(data)
return self.ssl_request.put(url, data=data, headers={'content-type': 'application/json'}, verify=cafile())
def post(self, url, data=None):
if data:
data = json.dumps(data)
return self.ssl_request.post(url, data=data, headers={'content-type': 'application/json'}, verify=cafile())
def delete(self, url):
return self.ssl_request.delete(url, verify=cafile())
def assertSuccessJson(self, dict, response):
self.assertEqual(200, response.status_code)
self.assertEquals('application/json', response.headers['content-type'])
self.assertEqual(dict, response.json())
def assertInternalError(self, response):
self.assertEqual(500, response.status_code)
def test_list_empty_agents(self):
# given
self.mock_provider.status.return_value = {'state': 'stopped'}
self.mock_users.list.return_value = []
# when
r = self.get('https://*****:*****@patch('pixelated.manager.SSLWSGIRefServerAdapter')
@patch('pixelated.manager.run') # mock run call to avoid actually startng the server
def test_that_ssl_server_adapter_gets_used_when_ssl_config_is_provided(self, run_mock, ssl_adapter_mock):
server = RESTfulServer(RESTfulServerTest.ssl_config, RESTfulServerTest.mock_users, RESTfulServerTest.mock_authenticator, RESTfulServerTest.mock_provider)
# when
server.serve_forever()
expected_ca_certs = None # which means system ciphers
expected_ciphers = DEFAULT_CIPHERS
expected_ssl_version = latest_available_ssl_version()
expected_host = '127.0.0.1'
expected_port = 4443
expected_certfile = certfile()
expected_keyfile = keyfile()
ssl_adapter_mock.assert_called_once_with(ssl_ca_certs=expected_ca_certs, ssl_ciphers=expected_ciphers, ssl_version=expected_ssl_version, host=expected_host, port=expected_port, ssl_cert_file=expected_certfile, ssl_key_file=expected_keyfile)
@patch('pixelated.manager.WSGIRefServer')
@patch('pixelated.manager.run') # mock run call to avoid actually startng the server
def test_that_serve_forever_runs_without_ssl_context(self, run_mock, wsgiRefServer_mock):
# given
server = RESTfulServer(None, RESTfulServerTest.mock_users, RESTfulServerTest.mock_authenticator, RESTfulServerTest.mock_provider)
# when
server.serve_forever()
# then
wsgiRefServer_mock.assert_called_once_with(host='localhost', port=4443)
def test_handles_provider_initializing(self):
self.mock_users.list.return_value = ['test']
self.mock_provider.status.side_effect = ProviderInitializingException
r = self.get('https://*****:*****@patch('pixelated.manager.DockerProvider')
@patch('pixelated.manager.RESTfulServer')
@patch('pixelated.manager.Thread')
@patch('pixelated.manager.Users')
@patch('pixelated.manager.LeapProvider')
@patch('pixelated.manager.LeapCertificate')
def test_that_initialize_happens_in_background_thread(self, leap_certificate_mock, leap_provider_mock, users_mock, thread_mock, server_mock, docker_provider_mock):
# given
docker_provider_mock.return_value = self.mock_provider
manager = DispatcherManager(self._root_path, None, None, None, None, provider='docker')
# when
manager.serve_forever()
# then
thread_mock.assert_called_with(target=self.mock_provider.initialize)
self.assertFalse(self.mock_provider.initialize.called)
@patch('pixelated.manager.LeapCertificate')
@patch('pixelated.manager.Authenticator')
@patch('pixelated.manager.DockerProvider')
@patch('pixelated.manager.RESTfulServer')
@patch('pixelated.manager.Thread')
@patch('pixelated.manager.Users')
@patch('pixelated.manager.LeapProvider')
def test_that_tls_config_gets_passed_to_authenticator(self, leap_provider_mock, users_mock, thread_mock, server_mock, docker_provider_mock, authenticator_mock, leap_certificate_mock):
# given
manager = DispatcherManager(self._root_path, None, None, None, 'some ca bundle', leap_provider_fingerprint='some fingerprint', provider='docker')
# when
manager.serve_forever()
# then
authenticator_mock.assert_called_once_with(users_mock.return_value, leap_provider_mock.return_value)
@patch('pixelated.manager.LeapCertificate')
@patch('pixelated.manager.Authenticator')
@patch('pixelated.manager.DockerProvider')
@patch('pixelated.manager.RESTfulServer')
@patch('pixelated.manager.Thread')
@patch('pixelated.manager.Users')
@patch('pixelated.manager.LeapProvider')
def test_that_leap_certificate_gets_downloaded_on_serve_forever(self, leap_provider_mock, users_mock, thread_mock, server_mock, docker_provider_mock, authenticator_mock, leap_certificate_mock):
# given
cert_file = join(self._root_path, 'ca.crt')
manager = DispatcherManager(self._root_path, None, None, None, 'some ca bundle', leap_provider_fingerprint='some fingerprint', provider='docker')
# when
manager.serve_forever()
# then
leap_certificate_mock.return_value.refresh_api_ca_bundle.assert_called_once_with()
class ForkProviderTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self.runner = MagicMock(spec=Adapter)
self.provider = ForkProvider(self.runner)
self.provider.initialize()
def tearDown(self):
self._tmpdir.dissolve()
def test_instances_can_be_removed(self):
user_config = UserConfig('test', join(self.root_path, 'test'))
os.makedirs(join(user_config.path, 'data'))
self.provider.remove(user_config)
def test_that_instance_can_be_started_and_gets_initialized(self):
self.provider.start(self._user_config('test'))
self.runner.initialize.assert_called_with('test')
self.runner.start.assert_called_with('test')
def test_that_instance_cannot_be_started_twice(self):
user_config = self._user_config('test')
self.provider.start(user_config)
self.assertRaises(InstanceAlreadyRunningError, self.provider.start, user_config)
def test_that_running_instances_are_in_runnig_list(self):
self._init_runner_memory_usage()
for name in ['one', 'two', 'three']:
self.provider.start(self._user_config(name))
self.assertEqual({'one', 'two', 'three'}, set(self.provider.list_running()))
def test_that_non_existing_instance_cannot_be_stopped(self):
self.assertRaises(InstanceNotRunningError, self.provider.stop, 'test')
def test_that_non_started_instance_cannot_be_stopped(self):
self.assertRaises(InstanceNotRunningError, self.provider.stop, UserConfig('not-started', None))
def test_that_running_instance_can_be_stopped(self):
process = MagicMock(spec=ForkedProcess)
self.runner.start.return_value = process
user_config = self._user_config('test')
self.provider.start(user_config)
self.provider.stop(user_config.username)
process.terminate.assert_called_once_with()
def test_that_instance_cannot_be_stopped_twice(self):
self.provider.start(self._user_config('test'))
self.provider.stop('test')
self.assertRaises(InstanceNotRunningError, self.provider.stop, 'test')
def test_that_status_returns_current_port(self):
process = MagicMock(spec=ForkedProcess(None, 1234))
process.port = 1234
self.runner.start.return_value = process
self.provider.start(self._user_config('test'))
status = self.provider.status('test')
self.assertEqual({'port': 1234, 'state': 'running'}, status)
def assert_config_file(self, filename, name, hashed_password, salt):
with open(filename, 'r') as file:
content = file.read()
self.assertEqual('[agent]\nname = %s\nhashed_password = %s\nsalt = %s\n\n' % (name, hashed_password, salt),
content)
def test_memory_usage_zero_if_no_processes(self):
self.assertEqual({'total_usage': 0, 'average_usage': 0, 'agents': []}, self.provider.memory_usage())
def test_memory_usage_with_process(self):
# given
process = MagicMock(spec=ForkedProcess(None, 1234))
process.port = 1234
process.memory_usage.return_value = 1024
self.runner.start.return_value = process
self.provider.start(self._user_config('test'))
# when
usage = self.provider.memory_usage()
# then
self.assertEqual({'total_usage': 1024,
'average_usage': 1024,
'agents': [
{'name': 'test', 'memory_usage': 1024}
]}, usage)
@patch('pixelated.provider.fork.psutil.virtual_memory')
def test_that_instance_cannot_be_started_with_too_little_memory_left(self, vm_mock):
# given
svmem = namedtuple('svmem', ['free'])
free_memory = 1024 * 1024
vm_mock.return_value = svmem(free_memory)
process = MagicMock(spec=ForkedProcess)
process.memory_usage.return_value = free_memory + 1
self.runner.start.return_value = process
self.provider.start(self._user_config('memory monster'))
second_config = self._user_config('second')
# when/then
self.assertRaises(NotEnoughFreeMemory, self.provider.start, second_config)
process.memory_usage.return_value = free_memory - 1
self.provider.start(second_config)
def _init_runner_memory_usage(self):
def new_process(*args):
process = MagicMock(spec=ForkedProcess)
process.memory_usage.return_value = 1024
return process
self.runner.start.side_effect = new_process
def _user_config(self, name):
path = join(self.root_path, name)
os.makedirs(path)
return UserConfig(name, path)
class ForkProviderTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self.runner = MagicMock(spec=Adapter)
self.provider = ForkProvider(self.runner)
self.provider.initialize()
def tearDown(self):
self._tmpdir.dissolve()
def test_instances_can_be_removed(self):
user_config = UserConfig('test', join(self.root_path, 'test'))
os.makedirs(join(user_config.path, 'data'))
self.provider.remove(user_config)
def test_that_instance_can_be_started_and_gets_initialized(self):
self.provider.start(self._user_config('test'))
self.runner.initialize.assert_called_with('test')
self.runner.start.assert_called_with('test')
def test_that_instance_cannot_be_started_twice(self):
user_config = self._user_config('test')
self.provider.start(user_config)
self.assertRaises(InstanceAlreadyRunningError, self.provider.start,
user_config)
def test_that_running_instances_are_in_runnig_list(self):
self._init_runner_memory_usage()
for name in ['one', 'two', 'three']:
self.provider.start(self._user_config(name))
self.assertEqual({'one', 'two', 'three'},
set(self.provider.list_running()))
def test_that_non_existing_instance_cannot_be_stopped(self):
self.assertRaises(InstanceNotRunningError, self.provider.stop, 'test')
def test_that_non_started_instance_cannot_be_stopped(self):
self.assertRaises(InstanceNotRunningError, self.provider.stop,
UserConfig('not-started', None))
def test_that_running_instance_can_be_stopped(self):
process = MagicMock(spec=ForkedProcess)
self.runner.start.return_value = process
user_config = self._user_config('test')
self.provider.start(user_config)
self.provider.stop(user_config.username)
process.terminate.assert_called_once_with()
def test_that_instance_cannot_be_stopped_twice(self):
self.provider.start(self._user_config('test'))
self.provider.stop('test')
self.assertRaises(InstanceNotRunningError, self.provider.stop, 'test')
def test_that_status_returns_current_port(self):
process = MagicMock(spec=ForkedProcess(None, 1234))
process.port = 1234
self.runner.start.return_value = process
self.provider.start(self._user_config('test'))
status = self.provider.status('test')
self.assertEqual({'port': 1234, 'state': 'running'}, status)
def assert_config_file(self, filename, name, hashed_password, salt):
with open(filename, 'r') as file:
content = file.read()
self.assertEqual(
'[agent]\nname = %s\nhashed_password = %s\nsalt = %s\n\n' %
(name, hashed_password, salt), content)
def test_memory_usage_zero_if_no_processes(self):
self.assertEqual({
'total_usage': 0,
'average_usage': 0,
'agents': []
}, self.provider.memory_usage())
def test_memory_usage_with_process(self):
# given
process = MagicMock(spec=ForkedProcess(None, 1234))
process.port = 1234
process.memory_usage.return_value = 1024
self.runner.start.return_value = process
self.provider.start(self._user_config('test'))
# when
usage = self.provider.memory_usage()
# then
self.assertEqual(
{
'total_usage': 1024,
'average_usage': 1024,
'agents': [{
'name': 'test',
'memory_usage': 1024
}]
}, usage)
@patch('pixelated.provider.fork.psutil.virtual_memory')
def test_that_instance_cannot_be_started_with_too_little_memory_left(
self, vm_mock):
# given
svmem = namedtuple('svmem', ['free'])
free_memory = 1024 * 1024
vm_mock.return_value = svmem(free_memory)
process = MagicMock(spec=ForkedProcess)
process.memory_usage.return_value = free_memory + 1
self.runner.start.return_value = process
self.provider.start(self._user_config('memory monster'))
second_config = self._user_config('second')
# when/then
self.assertRaises(NotEnoughFreeMemory, self.provider.start,
second_config)
process.memory_usage.return_value = free_memory - 1
self.provider.start(second_config)
def _init_runner_memory_usage(self):
def new_process(*args):
process = MagicMock(spec=ForkedProcess)
process.memory_usage.return_value = 1024
return process
self.runner.start.side_effect = new_process
def _user_config(self, name):
path = join(self.root_path, name)
os.makedirs(path)
return UserConfig(name, path)
class SearchEngineTest(unittest.TestCase):
def setUp(self):
self.tempdir = TempDir()
self.user_home = self.tempdir.name
def tearDown(self):
self.tempdir.dissolve()
def test_headers_encoding(self):
# given
se = SearchEngine(INDEX_KEY, self.user_home)
headers = {
'From': '*****@*****.**',
'To':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
'Date': str(datetime.now())
}
# when
se.index_mail(
LeapMail('mailid', 'INBOX', headers=headers)
) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search('folker')
self.assertEqual((['mailid'], 1), result)
def test_contents_encoding_accents(self):
# given
se = SearchEngine(INDEX_KEY, self.user_home)
headers = {
'From': '*****@*****.**',
'To':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
'Date': str(datetime.now())
}
body = "When doing the search, it's not possible to find words with graphical accents, e.g.: 'coração', 'é', 'Fièvre', La Pluie d'été, 'não'."
# when
se.index_mail(
LeapMail('mailid', 'INBOX', headers=headers, body=body)
) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search(u"'coração', 'é',")
self.assertEqual((['mailid'], 1), result)
result = se.search(u"Fièvre")
self.assertEqual((['mailid'], 1), result)
result = se.search(u"été")
self.assertEqual((['mailid'], 1), result)
def test_contents_encoding_special_characters(self):
# given
se = SearchEngine(INDEX_KEY, self.user_home)
headers = {
'From': '*****@*****.**',
'To':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Cc':
'=?utf-8?b?IsOEw7zDtiDDlsO8w6QiIDxmb2xrZXJAcGl4ZWxhdGVkLXByb2plY3Qub3Jn?=\n =?utf-8?b?PiwgRsO2bGtlciA8Zm9sa2VyQHBpeGVsYXRlZC1wcm9qZWN0Lm9yZz4=?=',
'Subject': 'Some test mail',
'Date': str(datetime.now())
}
body = "When doing the search, 您好 أهلا"
# when
se.index_mail(
LeapMail('mailid', 'INBOX', headers=headers, body=body)
) # test_helper.pixelated_mail(extra_headers=headers, chash='mailid'))
result = se.search(u"您好")
self.assertEqual((['mailid'], 1), result)
result = se.search(u"أهلا")
self.assertEqual((['mailid'], 1), result)
class TestDatabase(unittest.TestCase):
domain = 'domain'
username = '******'
plainpassword = '******'
cipherpassword = '******'.decode('hex')
pin = '1234'
key = hashlib.sha256(domain + username + pin).digest()
iv = 'a3206f4194d1d7a252a9fe24b7b063b9'.decode('hex')
def setUp(self):
self.d = TempDir()
self.keyfile_path = os.path.join(self.d.name, 'keyfile')
self.database_path = os.path.join(self.d.name, 'database')
self.keyfile = K.create(
self.keyfile_path,
self.database_path)
D.create(self.keyfile)
def tearDown(self):
self.d.dissolve()
@raises(IOError)
def test_create_database_exists(self):
D.create(self.keyfile)
@raises(AttributeError)
def test_enter_missing_key(self):
del self.keyfile.key
with D(self.keyfile):
pass
@raises(ValueError)
def test_enter_invalid_key(self):
self.keyfile.key = 'not a key'
with D(self.keyfile):
pass
@raises(ValueError)
def test_enter_incorrect_key(self):
self.keyfile.key = '1b660492d98c54412d3d91818de5a2ae0b3110850a12010768b80fb277f55aa5'.decode('hex')
with D(self.keyfile):
pass
@raises(AttributeError)
def test_enter_missing_iv(self):
del self.keyfile.iv
with D(self.keyfile):
pass
@raises(ValueError)
def test_enter_invalid_iv(self):
self.keyfile.iv = 'not an iv'
with D(self.keyfile):
pass
@raises(ValueError)
def test_enter_incorrect_iv(self):
self.keyfile.iv = 'a059d464b93397a2a98e8e1f00b596c7'.decode('hex')
with D(self.keyfile):
pass
@raises(AttributeError)
def test_enter_missing_path(self):
del self.keyfile.database_path
with D(self.keyfile):
pass
@raises(IOError)
def test_enter_unreadable_path(self):
self.keyfile.database_path = '/nopermission'
with D(self.keyfile):
pass
@raises(ValueError)
def test_enter_not_a_database_path(self):
self.keyfile.database_path = os.path.join(self.d.name, 'database')
with open(self.keyfile.database_path, 'wb') as f:
f.write(Random.new().read(10))
with D(self.keyfile):
pass
def test_exit_saves_changes(self):
c = C(self.domain, self.username, cipherpassword=self.cipherpassword, iv=self.iv)
with D(self.keyfile) as d:
d.add(c, self.pin)
with D(self.keyfile) as d:
self.assertEqual(
[c],
d.find(self.domain, self.username))
def test_add(self):
c1 = C(self.domain, self.username, cipherpassword=self.cipherpassword, iv=self.iv)
with D(self.keyfile) as d:
d.add(c1, self.pin)
c2 = d.find(self.domain, self.username)[0]
print c1
print c2
self.assertEqual(self.plainpassword, c2.unlock(self.pin))
@raises(sqlite3.IntegrityError)
def test_add_domain_username_exists(self):
c = C(self.domain, self.username, cipherpassword=self.cipherpassword, iv=self.iv)
with D(self.keyfile) as d:
d.add(c, self.pin)
d.add(c, self.pin)
def test_find_none(self):
with D(self.keyfile) as d:
self.assertEqual([], d.find())
def test_find_domain(self):
c1 = C('d', 'u', plainpassword='******')
c2 = C('d', 'v', plainpassword='******')
c3 = C('e', 'u', plainpassword='******')
with D(self.keyfile) as d:
d.add(c1, self.pin)
d.add(c2, self.pin)
d.add(c3, self.pin)
results = d.find(domain='d')
self.assertIn(c1, results)
self.assertIn(c2, results)
self.assertNotIn(c3, results)
def test_find_username(self):
c1 = C('d', 'u', plainpassword='******')
c2 = C('d', 'v', plainpassword='******')
c3 = C('e', 'u', plainpassword='******')
with D(self.keyfile) as d:
d.add(c1, self.pin)
d.add(c2, self.pin)
d.add(c3, self.pin)
results = d.find(username='******')
self.assertIn(c1, results)
self.assertNotIn(c2, results)
self.assertIn(c3, results)
def test_find_fuzzy_start(self):
c1 = C('domain', 'u', plainpassword='******')
c2 = C('e', 'v', plainpassword='******')
with D(self.keyfile) as d:
d.add(c1, self.pin)
d.add(c2, self.pin)
results = d.find(domain='dom')
self.assertIn(c1, results)
self.assertNotIn(c2, results)
def test_find_fuzzy_middle(self):
c1 = C('domain', 'u', plainpassword='******')
c2 = C('e', 'v', plainpassword='******')
with D(self.keyfile) as d:
d.add(c1, self.pin)
d.add(c2, self.pin)
results = d.find(domain='mai')
self.assertIn(c1, results)
self.assertNotIn(c2, results)
def test_find_fuzzy_end(self):
c1 = C('domain', 'u', plainpassword='******')
c2 = C('e', 'v', plainpassword='******')
with D(self.keyfile) as d:
d.add(c1, self.pin)
d.add(c2, self.pin)
results = d.find(domain='ain')
self.assertIn(c1, results)
self.assertNotIn(c2, results)
def test_update(self):
c1 = C('domain', 'u', plainpassword='******')
c2 = C('domain', 'u', plainpassword='******')
with D(self.keyfile) as d:
d.add(c1, self.pin)
d.update(c2, self.pin)
c3 = d.find(domain='domain')[0]
self.assertEqual(c2.plainpassword, c3.unlock(self.pin))
self.assertNotEqual(c1.plainpassword, c3.unlock(self.pin))
def test_delete(self):
c = C('domain', 'u', plainpassword='******')
with D(self.keyfile) as d:
d.add(c, self.pin)
d.delete(c)
self.assertEqual([], d.find(domain='domain'))
class ForkProviderTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self.runner = MagicMock(spec=Adapter)
self.provider = ForkProvider(self.root_path, self.runner)
self.provider.initialize()
def tearDown(self):
self._tmpdir.dissolve()
@patch('random.getrandbits')
def test_new_instance_can_be_added(self, randmock):
hashed_password = '******'
hex_salt = 31
randmock.return_value = 1
self.provider.add('test', 'password')
instance_path = join(self.root_path, 'test')
cfg_path = join(instance_path, 'agent.cfg')
data_path = join(instance_path, 'data')
gnupg_path = join(instance_path, 'gnupg')
self.assertTrue(isfile(cfg_path), 'No config file created')
self.assertTrue(isdir(instance_path), 'No folder for user has been created')
self.assertTrue(isdir(data_path), 'No maipile folder for user has been created')
self.assertTrue(isdir(gnupg_path), 'No gnupg folder for user has been created')
self.assert_config_file(cfg_path, 'test', hashed_password, hex_salt)
self.assertEqual(stat.S_IFDIR | stat.S_IRWXU, os.stat(instance_path).st_mode)
self.assertEqual(stat.S_IFDIR | stat.S_IRWXU, os.stat(data_path).st_mode)
self.assertEqual(stat.S_IFDIR | stat.S_IRWXU, os.stat(gnupg_path).st_mode)
@patch('random.getrandbits')
def test_random_salt_is_used(self, randmock):
hashed_password = '******'
salt = 2
hex_salt = 32
randmock.return_value = salt
instance_path = join(self.root_path, 'test')
cfg_path = join(instance_path, 'agent.cfg')
self.provider.add('test', 'password')
self.assert_config_file(cfg_path, 'test', hashed_password, hex_salt)
def test_multiple_instances_can_be_added(self):
self.provider.add('first', 'password')
self.provider.add('second', 'password')
self.assertEqual(['first', 'second'], self.provider.list())
def test_instances_can_not_be_added_twice(self):
self.provider.add('test', 'password')
self.assertRaises(InstanceAlreadyExistsError, self.provider.add, 'test', 'password')
def test_remove_raises_exception_if_instance_does_not_exist(self):
self.assertRaises(ValueError, self.provider.remove, 'test')
def test_instances_can_be_removed(self):
self.provider.add('test', 'password')
self.provider.remove('test')
def test_that_non_existing_instance_cannot_be_started(self):
self.assertRaises(InstanceNotFoundError, self.provider.start, 'test')
def test_that_instance_can_be_started_and_gets_initialized(self):
self.provider.add('test', 'password')
self.provider.start('test')
self.runner.initialize.assert_called_with('test')
self.runner.start.assert_called_with('test')
def test_that_instance_cannot_be_started_twice(self):
self.provider.add('test', 'password')
self.provider.start('test')
self.assertRaises(InstanceAlreadyRunningError, self.provider.start, 'test')
def test_that_running_instances_are_in_runnig_list(self):
self._init_runner_memory_usage()
for name in ['one', 'two', 'three']:
self.provider.add(name, 'password')
self.provider.start(name)
self.provider.add('not-started', 'password')
self.assertEqual({'one', 'two', 'three'}, set(self.provider.list_running()))
def test_that_non_existing_instance_cannot_be_stopped(self):
self.assertRaises(InstanceNotRunningError, self.provider.stop, 'test')
def test_that_non_started_instance_cannot_be_stopped(self):
self.provider.add('test', 'password')
self.assertRaises(InstanceNotRunningError, self.provider.stop, 'test')
def test_that_running_instance_can_be_stopped(self):
process = MagicMock(spec=ForkedProcess)
self.runner.start.return_value = process
self.provider.add('test', 'password')
self.provider.start('test')
self.provider.stop('test')
process.terminate.assert_called_once_with()
def test_that_instance_cannot_be_stopped_twice(self):
self.provider.add('test', 'password')
self.provider.start('test')
self.provider.stop('test')
self.assertRaises(InstanceNotRunningError, self.provider.stop, 'test')
def test_that_existing_agents_are_autodiscovered(self):
agent = os.path.join(self.root_path, 'test')
os.mkdir(agent)
self.provider = ForkProvider(self.root_path, self.runner)
self.provider.initialize()
self.assertEqual(['test'], self.provider.list())
def test_that_status_returns_current_port(self):
process = MagicMock(spec=ForkedProcess(None, 1234))
process.port = 1234
self.runner.start.return_value = process
self.provider.add('test', 'password')
self.provider.start('test')
status = self.provider.status('test')
self.assertEqual({'port': 1234, 'state': 'running'}, status)
def test_authenticate(self):
self.provider.add('test', 'password')
self.assertTrue(self.provider.authenticate('test', 'password'))
self.assertFalse(self.provider.authenticate('test', 'something else'))
def test_unicode_passwords_dont_cause_type_error(self):
self.provider.add('test', u'password')
self.assertTrue(self.provider.authenticate('test', u'password'))
def assert_config_file(self, filename, name, hashed_password, salt):
with open(filename, 'r') as file:
content = file.read()
self.assertEqual('[agent]\nname = %s\nhashed_password = %s\nsalt = %s\n\n' % (name, hashed_password, salt),
content)
def test_memory_usage_zero_if_no_processes(self):
self.assertEqual({'total_usage': 0, 'average_usage': 0, 'agents': []}, self.provider.memory_usage())
def test_memory_usage_with_process(self):
# given
process = MagicMock(spec=ForkedProcess(None, 1234))
process.port = 1234
process.memory_usage.return_value = 1024
self.runner.start.return_value = process
self.provider.add('test', 'password')
self.provider.start('test')
# when
usage = self.provider.memory_usage()
# then
self.assertEqual({'total_usage': 1024,
'average_usage': 1024,
'agents': [
{'name': 'test', 'memory_usage': 1024}
]}, usage)
@patch('pixelated.provider.fork.psutil.virtual_memory')
def test_that_instance_cannot_be_started_with_too_little_memory_left(self, vm_mock):
# given
svmem = namedtuple('svmem', ['free'])
free_memory = 1024 * 1024
vm_mock.return_value = svmem(free_memory)
process = MagicMock(spec=ForkedProcess)
process.memory_usage.return_value = free_memory + 1
self.runner.start.return_value = process
self.provider.add('memory monster', 'password')
self.provider.start('memory monster')
self.provider.add('second', 'password')
# when/then
self.assertRaises(NotEnoughFreeMemory, self.provider.start, 'second')
process.memory_usage.return_value = free_memory - 1
self.provider.start('second')
def _init_runner_memory_usage(self):
def new_process(*args):
process = MagicMock(spec=ForkedProcess)
process.memory_usage.return_value = 1024
return process
self.runner.start.side_effect = new_process
class TestCli(unittest.TestCase):
key = '0b660492d98c54412d3d91818de5a2ae0b3110850a12010768b80fb277f55aa6'.decode('hex')
iv = '9059d464b93397a2a98e8e1f00b596c6'.decode('hex')
length = 5
characters = 'abc'
domain = 'domain'
username = '******'
password = '******'
pin = '1234'
def setUp(self):
self.d = TempDir()
self.k = os.path.join(self.d.name, 'keyfile')
d = os.path.join(self.d.name, 'database')
cli.main(('init %s %s' % (self.k, d)).split(' '))
self.keyfile = Keyfile.load(self.k)
def tearDown(self):
self.d.dissolve()
@not_raises(AttributeError)
def test_init_defaults(self):
with TempDir() as d:
k = os.path.join(d.name, 'keyfile')
d = os.path.join(d.name, 'database')
cli.main(('init %s %s' % (k, d)).split(' '))
keyfile = Keyfile.load(k)
self.assertEqual(k, keyfile.path)
self.assertEqual(d, keyfile.database_path)
self.assertEqual(Keyfile.LENGTH, keyfile.length)
self.assertEqual(Keyfile.CHARACTERS, keyfile.characters)
keyfile.key
keyfile.iv
def test_init_override(self):
with TempDir() as d:
k = os.path.join(d.name, 'keyfile')
d = os.path.join(d.name, 'database')
cli.main(('init %s %s --key %s --iv %s --length %d --characters %s' % (k, d, self.key, self.iv, self.length, self.characters)).split(' '))
keyfile = Keyfile.load(k)
self.assertEqual(k, keyfile.path)
self.assertEqual(d, keyfile.database_path)
self.assertEqual(self.length, keyfile.length)
self.assertEqual(self.characters, keyfile.characters)
self.assertEqual(self.key, keyfile.key)
self.assertEqual(self.iv, keyfile.iv)
def test_add_defaults(self):
results = cli.main(
('add %s --domain %s --username %s --pin %s'
% (self.k, self.domain, self.username, self.pin)).split(' '))
with Database(self.keyfile) as d:
c = d.find(domain=self.domain, username=self.username)[0]
c.unlock(self.pin)
self.assertEqual(results, '\n'.join([c.domain, c.username, c.plainpassword]))
def test_add_override(self):
results = cli.main(
('add %s --domain %s --username %s --pin %s --length %d --characters %s'
% (self.k, self.domain, self.username, self.pin, self.length, self.characters)).split(' '))
parts = results.split('\n')
self.assertEqual(self.length, len(parts[2]))
for c in parts[2]:
self.assertIn(c, self.characters)
def test_add_provide_password(self):
results = cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
parts = results.split('\n')
self.assertEqual(self.password, parts[2])
def test_find_single_keyfile_single_result(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s --domain %s --username %s --pin %s'
% (self.k, self.domain, self.username, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, self.password]))
def test_find_single_keyfile_multiple_results(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain + '2', self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s --domain %s --username %s --pin %s'
% (self.k, self.domain, self.username, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, '', self.domain + '2', self.username]))
def test_find_multiple_keyfiles_single_result(self):
k2 = os.path.join(self.d.name, 'keyfile2')
d2 = os.path.join(self.d.name, 'database2')
cli.main(('init %s %s' % (k2, d2)).split(' '))
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s %s --domain %s --username %s --pin %s'
% (self.k, k2, self.domain, self.username, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, self.password]))
def test_find_multiple_keyfiles_multiple_results(self):
k2 = os.path.join(self.d.name, 'keyfile2')
d2 = os.path.join(self.d.name, 'database2')
cli.main(('init %s %s' % (k2, d2)).split(' '))
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (k2, self.domain + '2', self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s %s --domain %s --username %s --pin %s'
% (self.k, k2, self.domain, self.username, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, '', self.domain + '2', self.username]))
def test_find_single_result_no_pin(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s --domain %s --username %s'
% (self.k, self.domain, self.username)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, '********']))
def test_find_fuzzy_match(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s --domain %s --username %s --pin %s'
% (self.k, self.domain[0:2], self.username, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, self.password]))
def test_find_only_domain(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s --domain %s --pin %s'
% (self.k, self.domain, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, self.password]))
def test_find_only_username(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
results = cli.main(
('find %s --username %s --pin %s'
% (self.k, self.username, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, self.password]))
def test_update_defaults(self):
results1 = cli.main(
('add %s --domain %s --username %s --pin %s'
% (self.k, self.domain, self.username, self.pin)).split(' '))
results2 = cli.main(
('update %s --domain %s --username %s --pin %s'
% (self.k, self.domain, self.username, self.pin)).split(' '))
with Database(self.keyfile) as d:
c = d.find(domain=self.domain, username=self.username)[0]
c.unlock(self.pin)
print c
self.assertEqual(results2, '\n'.join([c.domain, c.username, c.plainpassword]))
self.assertNotEqual(results1, results2)
def test_update_override(self):
results1 = cli.main(
('add %s --domain %s --username %s --pin %s --length %d --characters %s'
% (self.k, self.domain, self.username, self.pin, self.length, self.characters)).split(' '))
results2 = cli.main(
('update %s --domain %s --username %s --pin %s --length %d --characters %s'
% (self.k, self.domain, self.username, self.pin, self.length, self.characters)).split(' '))
parts = results2.split('\n')
self.assertEqual(self.length, len(parts[2]))
for c in parts[2]:
self.assertIn(c, self.characters)
self.assertNotEqual(results1, results2)
def test_update_provide_password(self):
results1 = cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
results2 = cli.main(
('update %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password + '2')).split(' '))
parts = results2.split('\n')
self.assertEqual(self.password + '2', parts[2])
self.assertNotEqual(results1, results2)
def test_delete(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
cli.main(
('delete %s --domain %s --username %s'
% (self.k, self.domain, self.username)).split(' '))
results = cli.main(
('find %s --domain %s --username %s --pin %s'
% (self.k, self.domain, self.username, self.pin)).split(' '))
self.assertEqual(results, '')
def test_delete_not_fuzzy(self):
cli.main(
('add %s --domain %s --username %s --pin %s --password %s'
% (self.k, self.domain, self.username, self.pin, self.password)).split(' '))
cli.main(
('delete %s --domain %s --username %s'
% (self.k, self.domain[0:2], self.username)).split(' '))
results = cli.main(
('find %s --domain %s --username %s --pin %s'
% (self.k, self.domain, self.username, self.pin)).split(' '))
self.assertEqual(results, '\n'.join([self.domain, self.username, self.password]))
class UsersTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self.users = Users(self.root_path)
def test_that_constructor_throws_exception_if_root_path_does_not_exist(self):
self.assertRaises(ValueError, Users, '/does/not/exist')
def tearDown(self):
self._tmpdir.dissolve()
def test_add_user(self):
self.users.add('name')
def test_user_cannot_be_added_twice(self):
self.users.add('name')
self.assertRaises(UserAlreadyExistsError, self.users.add, 'name')
def test_list_empty(self):
self.assertEqual([], self.users.list())
def test_list_single_user(self):
self.users.add('name')
self.assertEqual(['name'], self.users.list())
def test_get_user_config_throws_exception_if_user_not_exists(self):
self.assertRaises(UserNotExistError, self.users.config, 'name')
def test_add_user_creates_data_folder_and_config(self):
self.users.add('name')
data_path = join(self.root_path, 'name')
config_file = join(data_path, 'user.cfg')
self.assertTrue(exists(data_path))
self.assertTrue(exists(config_file))
def test_add_user_does_not_override_existing_config(self):
data_path = join(self.root_path, 'name')
config_file = join(data_path, 'user.cfg')
mkdir(data_path)
with open(config_file, 'w') as fd:
fd.write('[section]\none = first value\ntwo = second value\n\n')
self.users.add('name')
data_path = join(self.root_path, 'name')
config_file = join(data_path, 'user.cfg')
with open(config_file, 'r') as fd:
self.assertEqual('[section]\none = first value\ntwo = second value\n\n', fd.read())
def test_add_validates_username(self):
self.users.add('a_name.with-valid-chars')
with self.assertRaises(ValueError):
self.users.add('CapitalLetters')
with self.assertRaises(ValueError):
self.users.add('some/name/with/slashes')
with self.assertRaises(ValueError):
self.users.add('name with spaces')
with self.assertRaises(ValueError):
self.users.add('name=with%&')
def test_loads_all_existing_users_on_startup(self):
user1_data_path = join(self.root_path, 'user1')
mkdir(user1_data_path)
user1_config_path = join(self.root_path, 'user1', 'user.cfg')
with open(user1_config_path, 'w') as fd:
fd.write('[section]\none = first value\ntwo = second value\n\n')
users = Users(self.root_path)
self.assertEqual(['user1'], users.list())
self.assertEqual('first value', users.config('user1')['section.one'])
def test_update_user_config(self):
self.users.add('name')
config = self.users.config('name')
config['foo.bar'] = 'some value'
self.users.update_config(config)
config = self.users.config('name')
self.assertEqual('some value', config['foo.bar'])
def test_has_user(self):
self.users.add('name')
self.assertTrue(self.users.has_user('name'))
self.assertFalse(self.users.has_user('other'))
class AppTestClient(object):
INDEX_KEY = '\xde3?\x87\xff\xd9\xd3\x14\xf0\xa7>\x1f%C{\x16.\\\xae\x8c\x13\xa7\xfb\x04\xd4]+\x8d_\xed\xd1\x8d\x0bI' \
'\x8a\x0e\xa4tm\xab\xbf\xb4\xa5\x99\x00d\xd5w\x9f\x18\xbc\x1d\xd4_W\xd2\xb6\xe8H\x83\x1b\xd8\x9d\xad'
ACCOUNT = 'test'
MAIL_ADDRESS = '*****@*****.**'
def _initialize(self):
self._tmp_dir = TempDir()
self.accounts = {}
@defer.inlineCallbacks
def start_client(self, mode=UserAgentMode(is_single_user=True)):
self._initialize()
self._mode = mode
self._test_account = AppTestAccount(self.ACCOUNT, self._tmp_dir.name)
yield self._test_account.start()
self.cleanup = lambda: self._tmp_dir.dissolve()
# copy fields for single user tests
self.soledad = self._test_account.soledad
self.search_engine = self._test_account.search_engine
self.keymanager = self._test_account.keymanager
self.mail_sender = self._test_account.mail_sender
self.mail_store = self._test_account.mail_store
self.attachment_store = self._test_account.attachment_store
self.draft_service = self._test_account.draft_service
self.leap_session = self._test_account.leap_session
self.feedback_service = self._test_account.feedback_service
self.mail_service = self._test_account.mail_service
self.account = self._test_account.account
if mode.is_single_user:
self.service_factory = SingleUserServicesFactory(mode)
services = self._test_account.services
self.service_factory.add_session('someuserid', services)
self.resource = RootResource(self.service_factory)
self.resource.initialize()
else:
self.service_factory = StubServicesFactory(self.accounts, mode)
provider = mock()
provider.config = LeapConfig(self._tmp_dir.name)
self.resource = set_up_protected_resources(RootResource(self.service_factory), provider, self.service_factory, checker=StubSRPChecker(provider))
@defer.inlineCallbacks
def create_user(self, account_name):
if self._mode.is_single_user:
raise Exception('Not supported in single user mode')
account = AppTestAccount(account_name, self._tmp_dir.name)
yield account.start()
self.accounts[account_name] = account
def _render(self, request, as_json=True):
def get_str(_str):
return json.loads(_str) if as_json else _str
def get_request_written_data(_=None):
written_data = request.getWrittenData()
if written_data:
return get_str(written_data)
resource = getChildForRequest(self.resource, request)
result = resource.render(request)
if isinstance(result, basestring):
return get_str(result), request
# result is NOT_DONE_YET
d = succeed(request) if request.finished else request.notifyFinish()
d.addCallback(get_request_written_data)
return d, request
def listenTCP(self, port=4567, host='127.0.0.1'):
reactor.listenTCP(port, PixelatedSite(self.resource), interface=host)
def run_on_a_thread(self, logfile='/tmp/app_test_client.log', port=4567, host='127.0.0.1'):
def _start():
self.listenTCP(port, host)
reactor.run()
process = multiprocessing.Process(target=_start)
process.start()
time.sleep(1)
return lambda: process.terminate()
def stop(self):
reactor.stop()
def get(self, path, get_args='', as_json=True, ajax=True, csrf='token'):
request = request_mock(path, ajax=ajax, csrf=csrf)
request.args = get_args
return self._render(request, as_json)
def post(self, path, body='', headers=None, ajax=True, csrf='token'):
headers = headers or {'Content-Type': 'application/json'}
request = request_mock(path=path, method="POST", body=body, headers=headers, ajax=ajax, csrf=csrf)
return self._render(request)
def put(self, path, body, ajax=True, csrf='token'):
request = request_mock(path=path, method="PUT", body=body, headers={'Content-Type': ['application/json']}, ajax=ajax, csrf=csrf)
return self._render(request)
def delete(self, path, body="", ajax=True, csrf='token'):
request = request_mock(path=path, body=body, headers={'Content-Type': ['application/json']}, method="DELETE", ajax=ajax, csrf=csrf)
return self._render(request)
@defer.inlineCallbacks
def add_mail_to_inbox(self, input_mail):
mail = yield self.mail_store.add_mail('INBOX', input_mail.raw)
defer.returnValue(mail)
def account_for(self, username):
return self.accounts[username]
def add_mail_to_user_inbox(self, input_mail, username):
return self.account_for(username).mail_store.add_mail('INBOX', input_mail.raw)
@defer.inlineCallbacks
def add_multiple_to_mailbox(self, num, mailbox='', flags=[], tags=[], to='*****@*****.**', cc='*****@*****.**', bcc='*****@*****.**'):
mails = []
yield self.mail_store.add_mailbox(mailbox)
for _ in range(num):
builder = MailBuilder().with_status(flags).with_tags(tags).with_to(to).with_cc(cc).with_bcc(bcc)
builder.with_body(str(random.random()))
input_mail = builder.build_input_mail()
mail = yield self.mail_store.add_mail(mailbox, input_mail.raw)
if tags:
mail.tags |= set(tags)
if flags:
for flag in flags:
mail.flags.add(flag)
if tags or flags:
yield self.mail_store.update_mail(mail)
mails.append(mail)
defer.returnValue(mails)
def _create_mail_sender(self):
mail_sender = Mock()
mail_sender.sendmail.side_effect = lambda mail: succeed(mail)
return mail_sender
def _generate_soledad_test_folder_name(self, soledad_test_folder='/tmp/soledad-test/test'):
return os.path.join(soledad_test_folder, str(uuid.uuid4()))
def get_mails_by_tag(self, tag, page=1, window=100):
tags = 'tag:%s' % tag
return self.search(tags, page, window)
@defer.inlineCallbacks
def search(self, query, page=1, window=100):
res, _ = self.get("/mails", {
'q': [query],
'w': [str(window)],
'p': [str(page)]
})
res = yield res
defer.returnValue([ResponseMail(m) for m in res['mails']])
@defer.inlineCallbacks
def get_mails_by_mailbox_name(self, mbox_name):
mail_ids = yield self.mail_store.get_mailbox_mail_ids(mbox_name)
mails = yield self.mail_store.get_mails(mail_ids)
defer.returnValue(mails)
@defer.inlineCallbacks
def get_attachment(self, ident, encoding, filename=None, content_type=None, ajax=True, csrf='token'):
params = {'encoding': [encoding]}
if filename:
params['filename'] = [filename]
if content_type:
params['content_type'] = [content_type]
deferred_result, req = self.get("/attachment/%s" % ident, params, as_json=False, ajax=ajax, csrf=csrf)
res = yield deferred_result
defer.returnValue((res, req))
@defer.inlineCallbacks
def post_attachment(self, data, headers):
deferred_result, req = self.post('/attachment', body=data, headers=headers)
res = yield deferred_result
defer.returnValue((res, req))
def put_mail(self, data):
res, req = self.put('/mails', data)
return res, req
def post_tags(self, mail_ident, tags_json):
res, req = self.post("/mail/%s/tags" % mail_ident, tags_json)
return res
def get_tags(self, **kwargs):
res, req = self.get('/tags', kwargs)
return res
def get_mail(self, mail_ident):
res, req = self.get('/mail/%s' % mail_ident)
return res
def delete_mail(self, mail_ident):
res, req = self.delete("/mail/%s" % mail_ident)
return res
def delete_mails(self, idents):
res, req = self.post("/mails/delete", json.dumps({'idents': idents}))
return res
def mark_many_as_unread(self, idents):
res, req = self.post('/mails/unread', json.dumps({'idents': idents}))
return res
def mark_many_as_read(self, idents):
res, req = self.post('/mails/read', json.dumps({'idents': idents}))
return res
def get_contacts(self, query):
res, req = self.get('/contacts', get_args={'q': query})
return res
class DockerProviderTest(unittest.TestCase):
def setUp(self):
self._provider_hostname = 'example.org'
self.users = MagicMock(spec=Users)
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self._adapter = MagicMock(wraps=PixelatedDockerAdapter(self._provider_hostname))
self._adapter.docker_image_name.return_value = 'pixelated'
self._leap_provider_x509 = LeapProviderX509Info()
def tearDown(self):
self._tmpdir.dissolve()
def test_that_docker_api_version_is_pinned_to_v1_14(self):
self.assertEqual('1.14', DOCKER_API_VERSION)
@patch('pixelated.provider.docker.docker.Client')
def test_constructor_expects_docker_url(self, docker_mock):
DockerProvider(self.root_path, self._adapter, self._leap_provider_x509, 'some docker url')
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_builds_docker_image(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
dockerfile = pkg_resources.resource_string('pixelated.resources', 'Dockerfile.pixelated')
# when
DockerProvider(self._adapter, 'leap_provider', self._leap_provider_x509, 'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url="some docker url", version=DOCKER_API_VERSION)
client.build.assert_called_once_with(path=None, fileobj=StringIOMatcher(dockerfile), tag='pixelated:latest')
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_downloads_docker_image_if_image_name_contains_slash(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
self._adapter.docker_image_name.return_value = 'pixelated/pixelated-user-agent'
# when
DockerProvider(self._adapter, 'leap_provider', self._leap_provider_x509, 'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url='some docker url', version=DOCKER_API_VERSION)
client.pull.assert_called_with(tag='latest', repository='pixelated/pixelated-user-agent', stream=True)
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_downloads_and_starts_logger_docker_image_if_not_yet_available(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
container = {'Id': 'some id'}
client.create_container.return_value = container
expected_syslog_tag = '.user_agent'
# when
DockerProvider(self._adapter, 'leap_provider', self._leap_provider_x509, 'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url='some docker url', version=DOCKER_API_VERSION)
client.pull.assert_called_with(tag='latest', repository='pixelated/logspout', stream=True)
client.create_container.assert_called_once_with(
image='pixelated/logspout:latest',
command='syslog://*****:*****@patch('pixelated.provider.docker.docker.Client')
def test_initialize_skips_image_build_or_download_if_already_available(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = [
{'Created': 1404833111,
'VirtualSize': 297017244,
'ParentId': '57885511c8444c2b89743bef8b89eccb65f302b2a95daa95dfcc9b972807b6db',
'RepoTags': ['pixelated:latest'],
'Id': 'b4f10a2395ab8dfc5e1c0fae26fa56c7f5d2541debe54263105fe5af1d263189',
'Size': 181956643}]
provider = DockerProvider(self._adapter, 'leap_provider', self._leap_provider_x509)
# when
provider.initialize()
# then
self.assertFalse(client.build.called)
self.assertFalse(provider.initializing)
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_doesnt_download_logger_image_if_already_available(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = [
{'Created': 1404833111,
'VirtualSize': 297017244,
'ParentId': '57885511c8444c2b89743bef8b89eccb65f302b2a95daa95dfcc9b972807b6db',
'RepoTags': ['pixelated/logspout:latest'],
'Id': 'b4f10a2395ab8dfc5e1c0fae26fa56c7f5d2541debe54263105fe5af1d263189', 'Size': 181956643}]
# when
DockerProvider(self._adapter, 'leap_provider', self._leap_provider_x509, 'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url='some docker url', version=DOCKER_API_VERSION)
client.pull.assert_never_called_with(tag='latest', repository='pixelated/logspout', stream=True)
@patch('pixelated.provider.docker.docker.Client')
def test_reports_initializing_while_initialize_is_running(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
def build(path, fileobj, tag):
sleep(0.2)
return []
client.build.side_effect = build
provider = DockerProvider(self._adapter, 'some provider', self._leap_provider_x509, 'some docker url')
self.assertTrue(provider.initializing)
# when
t = Thread(target=provider.initialize) # move to thread so that initializing behaviour is observable
t.start()
# then
sleep(0.1)
self.assertTrue(provider.initializing)
t.join()
self.assertFalse(provider.initializing)
@patch('pixelated.provider.docker.docker.Client')
def test_reports_initializing_while_initialize_is_running_and_image_downloaded(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
self._adapter.docker_image_name.return_value = 'pixelated/pixelated-user-agent'
def download(repository, tag, stream):
sleep(0.2)
return []
client.pull.side_effect = download
provider = DockerProvider(self._adapter, 'some provider', self._leap_provider_x509, 'some docker url')
self.assertTrue(provider.initializing)
# when
t = Thread(target=provider.initialize) # move to thread so that initializing behaviour is observable
t.start()
# then
sleep(0.1)
self.assertTrue(provider.initializing)
t.join()
self.assertFalse(provider.initializing)
@patch('pixelated.provider.docker.docker.Client')
def test_throws_initializing_exception_while_initializing(self, docker_mock):
# given
provider = DockerProvider(self._adapter, 'provider url', self._leap_provider_x509, 'some docker url')
# when/then
self.assertRaises(ProviderInitializingException, provider.start, 'test')
self.assertRaises(ProviderInitializingException, provider.remove, 'test')
self.assertRaises(ProviderInitializingException, provider.list_running)
self.assertRaises(ProviderInitializingException, provider.stop, 'test')
self.assertRaises(ProviderInitializingException, provider.status, 'test')
self.assertRaises(ProviderInitializingException, provider.memory_usage)
@patch('pixelated.provider.docker.docker.Client')
def test_that_instance_can_be_started(self, docker_mock):
expected_extra_hosts = {'nicknym.example.tld': '172.17.42.1', 'pixelated.example.tld': '172.17.42.1', 'api.example.tld': '172.17.42.1', 'example.tld': '172.17.42.1'}
uid = os.getuid()
self._adapter.docker_image_name.return_value = 'pixelated/pixelated-user-agent'
client = docker_mock.return_value
provider = self._create_initialized_provider(self._adapter, 'some docker url')
prepare_pixelated_container = MagicMock()
container = MagicMock()
client.create_container.side_effect = [prepare_pixelated_container, container]
client.wait.return_value = 0
self._leap_provider_x509.ca_bundle = 'some ca bundle'
with patch('pixelated.provider.docker.socket.getfqdn') as mock:
mock.return_value = 'pixelated.example.tld'
provider.start(self._user_config('test'))
client.create_container.assert_any_call('pixelated/pixelated-user-agent', '/bin/bash -l -c "/usr/bin/pixelated-user-agent --leap-home /mnt/user --host 0.0.0.0 --port 4567 --organization-mode --leap-provider-cert /mnt/user/dispatcher-leap-provider-ca.crt"', mem_limit='300m', user=uid, name='test', volumes=['/mnt/user'], ports=[4567], environment={'DISPATCHER_LOGOUT_URL': '/auth/logout', 'FEEDBACK_URL': 'https://example.org/tickets'}, stdin_open=True)
client.create_container.assert_any_call('pixelated/pixelated-user-agent', '/bin/true', name='pixelated_prepare', volumes=['/mnt/user'], environment={'DISPATCHER_LOGOUT_URL': '/auth/logout', 'FEEDBACK_URL': 'https://example.org/tickets'})
data_path = join(self.root_path, 'test', 'data')
client.start.assert_any_call(container, binds={data_path: {'bind': '/mnt/user', 'ro': False}}, port_bindings={4567: ('127.0.0.1', 5000)}, extra_hosts=expected_extra_hosts)
client.start.assert_any_call(prepare_pixelated_container, binds={data_path: {'bind': '/mnt/user', 'ro': False}})
@patch('pixelated.provider.docker.docker.Client')
def test_that_existing_container_gets_reused(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [[], [{u'Status': u'Exited (-1) About an hour ago', u'Created': 1405332375, u'Image': u'pixelated:latest', u'Ports': [], u'Command': u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"', u'Names': [u'/test'], u'Id': u'adfd4633fc42734665d7d98076b19b5f439648678b3b76db891f9d5072af50b6'}]]
provider = self._create_initialized_provider(self._adapter, 'some docker url')
container = MagicMock()
client.create_container.return_value = container
provider.start(self._user_config('test'))
client.containers.assert_called_with(all=True)
self.assertFalse(client.build.called)
@patch('pixelated.provider.docker.docker.Client')
def test_running_containers_empty_if_none_started(self, docker_mock):
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter, 'some docker url')
running = provider.list_running()
self.assertEqual([], running)
@patch('pixelated.provider.docker.docker.Client')
def test_running_returns_running_container(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [[], [], [{u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter, 'some docker url')
provider.start(self._user_config('test'))
running = provider.list_running()
self.assertEqual(['test'], running)
@patch('pixelated.provider.docker.docker.Client')
def test_a_container_cannot_be_started_twice(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [[], [], [{u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter, 'some docker url')
user_config = self._user_config('test')
provider.start(user_config)
self.assertRaises(InstanceAlreadyRunningError, provider.start, user_config)
@patch('pixelated.provider.docker.docker.Client')
def test_stopping_not_running_container_raises_value_error(self, docker_mock):
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter, 'some docker url')
self.assertRaises(InstanceNotRunningError, provider.stop, 'test')
@patch('pixelated.provider.docker.docker.Client')
def test_stop_running_container(self, docker_mock):
# given
user_config = self._user_config('test')
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 4567}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container], [container], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter, 'some docker url')
provider.pass_credentials_to_agent(user_config, 'test')
provider.start(user_config)
# when
provider.stop('test')
# then
client.stop.assert_called_once_with(container, timeout=10)
self.assertFalse(5000 in provider._used_ports())
self.assertTrue('test' not in provider._credentials)
@patch('pixelated.provider.docker.docker.Client')
def test_stop_running_container_calls_kill_if_stop_times_out(self, docker_mock):
# given
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 4567}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container], [container], [container]]
client.wait.return_value = 0
client.stop.side_effect = requests.exceptions.Timeout
provider = self._create_initialized_provider(self._adapter, 'some docker url')
provider.start(self._user_config('test'))
# when
provider.stop('test')
# then
client.stop.assert_called_once_with(container, timeout=10)
client.kill.assert_called_once_with(container)
@patch('pixelated.provider.docker.docker.Client')
def test_status_stopped(self, docker_mock):
provider = self._create_initialized_provider(self._adapter, 'some docker url')
self.assertEqual({'state': 'stopped'}, provider.status('test'))
@patch('pixelated.provider.docker.docker.Client')
def test_status_running(self, docker_mock):
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 33144}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter, 'some docker url')
provider.start(self._user_config('test'))
self.assertEqual({'state': 'running', 'port': 5000}, provider.status('test'))
@patch('pixelated.provider.docker.Process')
@patch('pixelated.provider.docker.docker.Client')
def test_memory_usage(self, docker_mock, process_mock):
# given
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 33144}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
info = {u'HostsPath': u'/var/lib/docker/containers/f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89/hosts', u'Created': u'2014-07-14T13:17:46.17558664Z', u'Image': u'f63df19194389be6481a174b36d291c483c8982d5c07485baa71a46b7f6582c8', u'Args': [], u'Driver': u'aufs', u'HostConfig': {u'PortBindings': {u'4567/tcp': [{u'HostPort': u'5000', u'HostIp': u'0.0.0.0'}]}, u'NetworkMode': u'', u'Links': None, u'LxcConf': None, u'ContainerIDFile': u'', u'Binds': [u'/tmp/multipile/folker:/mnt/user:rw'], u'PublishAllPorts': False, u'Dns': None, u'DnsSearch': None, u'Privileged': False, u'VolumesFrom': None}, u'MountLabel': u'', u'VolumesRW': {u'/mnt/user': True}, u'State': {u'Pid': 3250, u'Paused': False, u'Running': True, u'FinishedAt': u'0001-01-01T00:00:00Z', u'StartedAt': u'2014-07-14T13:17:46.601922899Z', u'ExitCode': 0}, u'ExecDriver': u'native-0.2', u'ResolvConfPath': u'/etc/resolv.conf', u'Volumes': {u'/mnt/user': u'/tmp/multipile/folker'}, u'Path': u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"', u'HostnamePath': u'/var/lib/docker/containers/f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89/hostname', u'ProcessLabel': u'', u'Config': {u'MemorySwap': 0, u'Hostname': u'f2cdb04277e9', u'Entrypoint': None, u'PortSpecs': None, u'Memory': 0, u'OnBuild': None, u'OpenStdin': False, u'Cpuset': u'', u'Env': [u'HOME=/', u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], u'User': u'', u'CpuShares': 0, u'AttachStdout': True, u'NetworkDisabled': False, u'WorkingDir': u'', u'Cmd': [u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"'], u'StdinOnce': False, u'AttachStdin': False, u'Volumes': {u'/mnt/user': {}}, u'Tty': False, u'AttachStderr': True, u'Domainname': u'', u'Image': u'pixelated', u'ExposedPorts': {u'4567/tcp': {}}}, u'Id': u'f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89', u'NetworkSettings': {u'Bridge': u'docker0', u'PortMapping': None, u'Gateway': u'172.17.42.1', u'IPPrefixLen': 16, u'IPAddress': u'172.17.0.14', u'Ports': {u'4567/tcp': [{u'HostPort': u'5000', u'HostIp': u'0.0.0.0'}]}}, u'Name': u'/folker'}
client = docker_mock.return_value
client.containers.return_value = [container]
client.inspect_container.return_value = info
psutil_mock = process_mock.return_value
psutil_mock.memory_info.return_value = pmem(1024, 2048)
provider = self._create_initialized_provider(self._adapter, 'some docker url')
# when
usage = provider.memory_usage()
# then
self.assertEqual({'total_usage': 1024,
'average_usage': 1024,
'agents': [
{'name': 'test', 'memory_usage': 1024}
]}, usage)
@patch('pixelated.provider.docker.docker.Client')
def test_remove_error_if_not_exist(self, docker_mock):
provider = self._create_initialized_provider(self._adapter, 'some docker url')
self.assertRaises(ValueError, provider.remove, self._user_config('does_not_exist'))
@patch('pixelated.provider.docker.docker.Client')
def test_remove(self, docker_mock):
# given
user_config = self._user_config('test')
os.makedirs(join(user_config.path, 'data'))
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter, 'some docker url')
# when
provider.remove(user_config)
# then
self.assertTrue(exists(user_config.path))
self.assertFalse(exists(join(user_config.path, 'data')))
@patch('pixelated.provider.docker.docker.Client')
def test_cannot_remove_while_running(self, docker_mock):
# given
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 4567}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter, 'some docker url')
user_config = self._user_config('test')
provider.start(user_config)
# when/then
self.assertRaises(ValueError, provider.remove, user_config)
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data(self, docker_mock):
# given
user_config = self._user_config('test')
os.makedirs(join(user_config.path, 'data'))
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter, 'some docker url')
# when
provider.reset_data(user_config)
# then
self.assertTrue(exists(user_config.path))
self.assertFalse(exists(join(user_config.path, 'data')))
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data_does_not_complain_if_there_is_no_data(self, docker_mock):
# given
user_config = self._user_config('test')
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter, 'some docker url')
# when
provider.reset_data(user_config)
# then
self.assertTrue(exists(user_config.path))
self.assertFalse(exists(join(user_config.path, 'data')))
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data_fails_if_user_does_not_exist(self, docker_mock):
# given
user_config = self._user_config('test')
shutil.rmtree(user_config.path)
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter, 'some docker url')
# when/then
self.assertRaises(ValueError, provider.reset_data, user_config)
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data_fails_if_agent_is_running(self, docker_mock):
# given
client = docker_mock.return_value
container = {u'Status': u'Up 20 seconds', u'Created': 1404904929, u'Image': u'pixelated:latest', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 5000, u'PrivatePort': 4567}], u'Command': u'sleep 100', u'Names': [u'/test'], u'Id': u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'}
client.containers.side_effect = [[], [], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter, 'some docker url')
user_config = self._user_config('test')
provider.start(user_config)
# when/then
self.assertRaises(InstanceAlreadyRunningError, provider.reset_data, user_config)
@patch('pixelated.provider.docker.TempDir')
@patch('pixelated.provider.docker.pkg_resources')
@patch('pixelated.provider.docker.docker.Client')
def test_use_build_script_instead_of_docker_file_if_available(self, docker_mock, res_mock, tempDir_mock):
# given
provider = DockerProvider(self._adapter, 'leap_provider', self._leap_provider_x509)
tempBuildDir = TempDir()
try:
tempDir_mock.return_value = tempBuildDir
tempBuildDir_name = tempBuildDir.name
with NamedTemporaryFile() as file:
res_mock.resource_exists.return_value = True
res_mock.resource_string.return_value = '#!/bin/bash\necho %s $PWD > %s' % (file.name, file.name)
# when
provider.initialize()
# then
res_mock.resource_exists.assert_called_with('pixelated.resources', 'init-pixelated-docker-context.sh')
res_mock.resource_string.assert_called_with('pixelated.resources', 'init-pixelated-docker-context.sh')
with open(file.name, "r") as input:
data = input.read().replace('\n', '')
self.assertEqual('%s %s' % (file.name, os.path.realpath(tempBuildDir_name)), data)
docker_mock.return_value.build.assert_called_once_with(path=tempBuildDir_name, tag='pixelated:latest', fileobj=None)
finally:
tempBuildDir.dissolve()
@patch('pixelated.provider.docker.docker.Client')
@patch('pixelated.provider.docker.CredentialsToDockerStdinWriter')
def test_that_credentials_are_passed_to_agent_by_stdin(self, credentials_mock, docker_mock):
# given
user_config = self._user_config('test')
provider = self._create_initialized_provider(self._adapter, 'some docker url')
prepare_pixelated_container = MagicMock()
container = MagicMock()
class ProcessStub(object):
def start(self):
self._target()
def __init__(self, target):
self._target = target
client = docker_mock.return_value
client.create_container.side_effect = [prepare_pixelated_container, container]
client.wait.return_value = 0
# when
provider.pass_credentials_to_agent(user_config, 'password')
provider.start(user_config)
# then
credentials_mock.return_value.start.assert_called_once_with()
@patch('pixelated.provider.docker.docker.Client')
def test_provider_checks_working_connection_to_docker(self, docker_mock):
client = docker_mock.return_value
client.info.side_effect = Exception
self.assertRaises(Exception, DockerProvider, self._adapter, 'leap_provider', self._leap_provider_x509)
@patch('pixelated.provider.docker.docker.Client')
def test_that_provider_x509_ca_bundle_is_copied_to_agent(self, docker_mock):
user_config = self._user_config('test')
provider = self._create_initialized_provider(self._adapter, 'some docker url')
client = docker_mock.return_value
client.wait.return_value = 0
with NamedTemporaryFile() as ca_file:
with open(ca_file.name, 'w') as fd:
fd.write('some certificate')
self._leap_provider_x509.ca_bundle = ca_file.name
provider.start(user_config)
self.assertTrue(exists(join(self.root_path, 'test', 'data', 'dispatcher-leap-provider-ca.crt')))
def _create_initialized_provider(self, adapter, docker_url=DockerProvider.DEFAULT_DOCKER_URL):
provider = DockerProvider(adapter, 'leap_provider_hostname', self._leap_provider_x509, docker_url)
provider._initializing = False
return provider
def _user_config(self, name):
path = join(self.root_path, name)
os.makedirs(path)
return UserConfig(name, path)
class DockerProviderTest(unittest.TestCase):
def setUp(self):
self._provider_hostname = 'example.org'
self.users = MagicMock(spec=Users)
self._tmpdir = TempDir()
self.root_path = self._tmpdir.name
self._adapter = MagicMock(
wraps=PixelatedDockerAdapter(self._provider_hostname))
self._adapter.docker_image_name.return_value = 'pixelated'
self._leap_provider_x509 = LeapProviderX509Info()
def tearDown(self):
self._tmpdir.dissolve()
def test_that_docker_api_version_is_pinned_to_v1_14(self):
self.assertEqual('1.14', DOCKER_API_VERSION)
@patch('pixelated.provider.docker.docker.Client')
def test_constructor_expects_docker_url(self, docker_mock):
DockerProvider(self.root_path, self._adapter, self._leap_provider_x509,
'some docker url')
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_builds_docker_image(self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
dockerfile = pkg_resources.resource_string('pixelated.resources',
'Dockerfile.pixelated')
# when
DockerProvider(self._adapter, 'leap_provider',
self._leap_provider_x509,
'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url="some docker url",
version=DOCKER_API_VERSION)
client.build.assert_called_once_with(
path=None,
fileobj=StringIOMatcher(dockerfile),
tag='pixelated:latest')
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_downloads_docker_image_if_image_name_contains_slash(
self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
self._adapter.docker_image_name.return_value = 'pixelated/pixelated-user-agent'
# when
DockerProvider(self._adapter, 'leap_provider',
self._leap_provider_x509,
'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url='some docker url',
version=DOCKER_API_VERSION)
client.pull.assert_called_with(
tag='latest',
repository='pixelated/pixelated-user-agent',
stream=True)
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_downloads_and_starts_logger_docker_image_if_not_yet_available(
self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
container = {'Id': 'some id'}
client.create_container.return_value = container
expected_syslog_tag = '.user_agent'
# when
DockerProvider(self._adapter, 'leap_provider',
self._leap_provider_x509,
'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url='some docker url',
version=DOCKER_API_VERSION)
client.pull.assert_called_with(tag='latest',
repository='pixelated/logspout',
stream=True)
client.create_container.assert_called_once_with(
image='pixelated/logspout:latest',
command='syslog://*****:*****@patch('pixelated.provider.docker.docker.Client')
def test_initialize_skips_image_build_or_download_if_already_available(
self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = [{
'Created': 1404833111,
'VirtualSize': 297017244,
'ParentId':
'57885511c8444c2b89743bef8b89eccb65f302b2a95daa95dfcc9b972807b6db',
'RepoTags': ['pixelated:latest'],
'Id':
'b4f10a2395ab8dfc5e1c0fae26fa56c7f5d2541debe54263105fe5af1d263189',
'Size': 181956643
}]
provider = DockerProvider(self._adapter, 'leap_provider',
self._leap_provider_x509)
# when
provider.initialize()
# then
self.assertFalse(client.build.called)
self.assertFalse(provider.initializing)
@patch('pixelated.provider.docker.docker.Client')
def test_initialize_doesnt_download_logger_image_if_already_available(
self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = [{
'Created': 1404833111,
'VirtualSize': 297017244,
'ParentId':
'57885511c8444c2b89743bef8b89eccb65f302b2a95daa95dfcc9b972807b6db',
'RepoTags': ['pixelated/logspout:latest'],
'Id':
'b4f10a2395ab8dfc5e1c0fae26fa56c7f5d2541debe54263105fe5af1d263189',
'Size': 181956643
}]
# when
DockerProvider(self._adapter, 'leap_provider',
self._leap_provider_x509,
'some docker url').initialize()
# then
docker_mock.assert_called_once_with(base_url='some docker url',
version=DOCKER_API_VERSION)
client.pull.assert_never_called_with(tag='latest',
repository='pixelated/logspout',
stream=True)
@patch('pixelated.provider.docker.docker.Client')
def test_reports_initializing_while_initialize_is_running(
self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
def build(path, fileobj, tag):
sleep(0.2)
return []
client.build.side_effect = build
provider = DockerProvider(self._adapter, 'some provider',
self._leap_provider_x509, 'some docker url')
self.assertTrue(provider.initializing)
# when
t = Thread(
target=provider.initialize
) # move to thread so that initializing behaviour is observable
t.start()
# then
sleep(0.1)
self.assertTrue(provider.initializing)
t.join()
self.assertFalse(provider.initializing)
@patch('pixelated.provider.docker.docker.Client')
def test_reports_initializing_while_initialize_is_running_and_image_downloaded(
self, docker_mock):
# given
client = docker_mock.return_value
client.images.return_value = []
self._adapter.docker_image_name.return_value = 'pixelated/pixelated-user-agent'
def download(repository, tag, stream):
sleep(0.2)
return []
client.pull.side_effect = download
provider = DockerProvider(self._adapter, 'some provider',
self._leap_provider_x509, 'some docker url')
self.assertTrue(provider.initializing)
# when
t = Thread(
target=provider.initialize
) # move to thread so that initializing behaviour is observable
t.start()
# then
sleep(0.1)
self.assertTrue(provider.initializing)
t.join()
self.assertFalse(provider.initializing)
@patch('pixelated.provider.docker.docker.Client')
def test_throws_initializing_exception_while_initializing(
self, docker_mock):
# given
provider = DockerProvider(self._adapter, 'provider url',
self._leap_provider_x509, 'some docker url')
# when/then
self.assertRaises(ProviderInitializingException, provider.start,
'test')
self.assertRaises(ProviderInitializingException, provider.remove,
'test')
self.assertRaises(ProviderInitializingException, provider.list_running)
self.assertRaises(ProviderInitializingException, provider.stop, 'test')
self.assertRaises(ProviderInitializingException, provider.status,
'test')
self.assertRaises(ProviderInitializingException, provider.memory_usage)
@patch('pixelated.provider.docker.docker.Client')
def test_that_instance_can_be_started(self, docker_mock):
expected_extra_hosts = {
'nicknym.example.tld': '172.17.42.1',
'pixelated.example.tld': '172.17.42.1',
'api.example.tld': '172.17.42.1',
'example.tld': '172.17.42.1'
}
uid = os.getuid()
self._adapter.docker_image_name.return_value = 'pixelated/pixelated-user-agent'
client = docker_mock.return_value
provider = self._create_initialized_provider(self._adapter,
'some docker url')
prepare_pixelated_container = MagicMock()
container = MagicMock()
client.create_container.side_effect = [
prepare_pixelated_container, container
]
client.wait.return_value = 0
self._leap_provider_x509.ca_bundle = 'some ca bundle'
with patch('pixelated.provider.docker.socket.getfqdn') as mock:
mock.return_value = 'pixelated.example.tld'
provider.start(self._user_config('test'))
client.create_container.assert_any_call(
'pixelated/pixelated-user-agent',
'/bin/bash -l -c "/usr/bin/pixelated-user-agent --leap-home /mnt/user --host 0.0.0.0 --port 4567 --organization-mode --leap-provider-cert /mnt/user/dispatcher-leap-provider-ca.crt"',
mem_limit='300m',
user=uid,
name='test',
volumes=['/mnt/user'],
ports=[4567],
environment={
'DISPATCHER_LOGOUT_URL': '/auth/logout',
'FEEDBACK_URL': 'https://example.org/tickets'
},
stdin_open=True)
client.create_container.assert_any_call(
'pixelated/pixelated-user-agent',
'/bin/true',
name='pixelated_prepare',
volumes=['/mnt/user'],
environment={
'DISPATCHER_LOGOUT_URL': '/auth/logout',
'FEEDBACK_URL': 'https://example.org/tickets'
})
data_path = join(self.root_path, 'test', 'data')
client.start.assert_any_call(
container,
binds={data_path: {
'bind': '/mnt/user',
'ro': False
}},
port_bindings={4567: ('127.0.0.1', 5000)},
extra_hosts=expected_extra_hosts)
client.start.assert_any_call(
prepare_pixelated_container,
binds={data_path: {
'bind': '/mnt/user',
'ro': False
}})
@patch('pixelated.provider.docker.docker.Client')
def test_that_existing_container_gets_reused(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [
[],
[{
u'Status':
u'Exited (-1) About an hour ago',
u'Created':
1405332375,
u'Image':
u'pixelated:latest',
u'Ports': [],
u'Command':
u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"',
u'Names': [u'/test'],
u'Id':
u'adfd4633fc42734665d7d98076b19b5f439648678b3b76db891f9d5072af50b6'
}]
]
provider = self._create_initialized_provider(self._adapter,
'some docker url')
container = MagicMock()
client.create_container.return_value = container
provider.start(self._user_config('test'))
client.containers.assert_called_with(all=True)
self.assertFalse(client.build.called)
@patch('pixelated.provider.docker.docker.Client')
def test_running_containers_empty_if_none_started(self, docker_mock):
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter,
'some docker url')
running = provider.list_running()
self.assertEqual([], running)
@patch('pixelated.provider.docker.docker.Client')
def test_running_returns_running_container(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [
[], [],
[{
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}]
]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter,
'some docker url')
provider.start(self._user_config('test'))
running = provider.list_running()
self.assertEqual(['test'], running)
@patch('pixelated.provider.docker.docker.Client')
def test_a_container_cannot_be_started_twice(self, docker_mock):
client = docker_mock.return_value
client.containers.side_effect = [
[], [],
[{
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}]
]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter,
'some docker url')
user_config = self._user_config('test')
provider.start(user_config)
self.assertRaises(InstanceAlreadyRunningError, provider.start,
user_config)
@patch('pixelated.provider.docker.docker.Client')
def test_stopping_not_running_container_raises_value_error(
self, docker_mock):
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter,
'some docker url')
self.assertRaises(InstanceNotRunningError, provider.stop, 'test')
@patch('pixelated.provider.docker.docker.Client')
def test_stop_running_container(self, docker_mock):
# given
user_config = self._user_config('test')
client = docker_mock.return_value
container = {
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [{
u'IP': u'0.0.0.0',
u'Type': u'tcp',
u'PublicPort': 5000,
u'PrivatePort': 4567
}],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}
client.containers.side_effect = [[], [], [container], [container],
[container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter,
'some docker url')
provider.pass_credentials_to_agent(user_config, 'test')
provider.start(user_config)
# when
provider.stop('test')
# then
client.stop.assert_called_once_with(container, timeout=10)
self.assertFalse(5000 in provider._used_ports())
self.assertTrue('test' not in provider._credentials)
@patch('pixelated.provider.docker.docker.Client')
def test_stop_running_container_calls_kill_if_stop_times_out(
self, docker_mock):
# given
client = docker_mock.return_value
container = {
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [{
u'IP': u'0.0.0.0',
u'Type': u'tcp',
u'PublicPort': 5000,
u'PrivatePort': 4567
}],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}
client.containers.side_effect = [[], [], [container], [container],
[container]]
client.wait.return_value = 0
client.stop.side_effect = requests.exceptions.Timeout
provider = self._create_initialized_provider(self._adapter,
'some docker url')
provider.start(self._user_config('test'))
# when
provider.stop('test')
# then
client.stop.assert_called_once_with(container, timeout=10)
client.kill.assert_called_once_with(container)
@patch('pixelated.provider.docker.docker.Client')
def test_status_stopped(self, docker_mock):
provider = self._create_initialized_provider(self._adapter,
'some docker url')
self.assertEqual({'state': 'stopped'}, provider.status('test'))
@patch('pixelated.provider.docker.docker.Client')
def test_status_running(self, docker_mock):
client = docker_mock.return_value
container = {
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [{
u'IP': u'0.0.0.0',
u'Type': u'tcp',
u'PublicPort': 5000,
u'PrivatePort': 33144
}],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}
client.containers.side_effect = [[], [], [container], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter,
'some docker url')
provider.start(self._user_config('test'))
self.assertEqual({
'state': 'running',
'port': 5000
}, provider.status('test'))
@patch('pixelated.provider.docker.Process')
@patch('pixelated.provider.docker.docker.Client')
def test_memory_usage(self, docker_mock, process_mock):
# given
container = {
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [{
u'IP': u'0.0.0.0',
u'Type': u'tcp',
u'PublicPort': 5000,
u'PrivatePort': 33144
}],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}
info = {
u'HostsPath':
u'/var/lib/docker/containers/f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89/hosts',
u'Created': u'2014-07-14T13:17:46.17558664Z',
u'Image':
u'f63df19194389be6481a174b36d291c483c8982d5c07485baa71a46b7f6582c8',
u'Args': [],
u'Driver': u'aufs',
u'HostConfig': {
u'PortBindings': {
u'4567/tcp': [{
u'HostPort': u'5000',
u'HostIp': u'0.0.0.0'
}]
},
u'NetworkMode': u'',
u'Links': None,
u'LxcConf': None,
u'ContainerIDFile': u'',
u'Binds': [u'/tmp/multipile/folker:/mnt/user:rw'],
u'PublishAllPorts': False,
u'Dns': None,
u'DnsSearch': None,
u'Privileged': False,
u'VolumesFrom': None
},
u'MountLabel': u'',
u'VolumesRW': {
u'/mnt/user': True
},
u'State': {
u'Pid': 3250,
u'Paused': False,
u'Running': True,
u'FinishedAt': u'0001-01-01T00:00:00Z',
u'StartedAt': u'2014-07-14T13:17:46.601922899Z',
u'ExitCode': 0
},
u'ExecDriver': u'native-0.2',
u'ResolvConfPath': u'/etc/resolv.conf',
u'Volumes': {
u'/mnt/user': u'/tmp/multipile/folker'
},
u'Path':
u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"',
u'HostnamePath':
u'/var/lib/docker/containers/f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89/hostname',
u'ProcessLabel': u'',
u'Config': {
u'MemorySwap':
0,
u'Hostname':
u'f2cdb04277e9',
u'Entrypoint':
None,
u'PortSpecs':
None,
u'Memory':
0,
u'OnBuild':
None,
u'OpenStdin':
False,
u'Cpuset':
u'',
u'Env': [
u'HOME=/',
u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
],
u'User':
u'',
u'CpuShares':
0,
u'AttachStdout':
True,
u'NetworkDisabled':
False,
u'WorkingDir':
u'',
u'Cmd': [
u'/bin/bash -l -c "/usr/bin/pixelated-user-agent --dispatcher"'
],
u'StdinOnce':
False,
u'AttachStdin':
False,
u'Volumes': {
u'/mnt/user': {}
},
u'Tty':
False,
u'AttachStderr':
True,
u'Domainname':
u'',
u'Image':
u'pixelated',
u'ExposedPorts': {
u'4567/tcp': {}
}
},
u'Id':
u'f2cdb04277e9e056c610240edffe8ff94ae272e462312c270e5300975d60af89',
u'NetworkSettings': {
u'Bridge': u'docker0',
u'PortMapping': None,
u'Gateway': u'172.17.42.1',
u'IPPrefixLen': 16,
u'IPAddress': u'172.17.0.14',
u'Ports': {
u'4567/tcp': [{
u'HostPort': u'5000',
u'HostIp': u'0.0.0.0'
}]
}
},
u'Name': u'/folker'
}
client = docker_mock.return_value
client.containers.return_value = [container]
client.inspect_container.return_value = info
psutil_mock = process_mock.return_value
psutil_mock.memory_info.return_value = pmem(1024, 2048)
provider = self._create_initialized_provider(self._adapter,
'some docker url')
# when
usage = provider.memory_usage()
# then
self.assertEqual(
{
'total_usage': 1024,
'average_usage': 1024,
'agents': [{
'name': 'test',
'memory_usage': 1024
}]
}, usage)
@patch('pixelated.provider.docker.docker.Client')
def test_remove_error_if_not_exist(self, docker_mock):
provider = self._create_initialized_provider(self._adapter,
'some docker url')
self.assertRaises(ValueError, provider.remove,
self._user_config('does_not_exist'))
@patch('pixelated.provider.docker.docker.Client')
def test_remove(self, docker_mock):
# given
user_config = self._user_config('test')
os.makedirs(join(user_config.path, 'data'))
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter,
'some docker url')
# when
provider.remove(user_config)
# then
self.assertTrue(exists(user_config.path))
self.assertFalse(exists(join(user_config.path, 'data')))
@patch('pixelated.provider.docker.docker.Client')
def test_cannot_remove_while_running(self, docker_mock):
# given
client = docker_mock.return_value
container = {
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [{
u'IP': u'0.0.0.0',
u'Type': u'tcp',
u'PublicPort': 5000,
u'PrivatePort': 4567
}],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}
client.containers.side_effect = [[], [], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter,
'some docker url')
user_config = self._user_config('test')
provider.start(user_config)
# when/then
self.assertRaises(ValueError, provider.remove, user_config)
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data(self, docker_mock):
# given
user_config = self._user_config('test')
os.makedirs(join(user_config.path, 'data'))
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter,
'some docker url')
# when
provider.reset_data(user_config)
# then
self.assertTrue(exists(user_config.path))
self.assertFalse(exists(join(user_config.path, 'data')))
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data_does_not_complain_if_there_is_no_data(
self, docker_mock):
# given
user_config = self._user_config('test')
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter,
'some docker url')
# when
provider.reset_data(user_config)
# then
self.assertTrue(exists(user_config.path))
self.assertFalse(exists(join(user_config.path, 'data')))
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data_fails_if_user_does_not_exist(self, docker_mock):
# given
user_config = self._user_config('test')
shutil.rmtree(user_config.path)
client = docker_mock.return_value
client.containers.return_value = []
provider = self._create_initialized_provider(self._adapter,
'some docker url')
# when/then
self.assertRaises(ValueError, provider.reset_data, user_config)
@patch('pixelated.provider.docker.docker.Client')
def test_reset_data_fails_if_agent_is_running(self, docker_mock):
# given
client = docker_mock.return_value
container = {
u'Status':
u'Up 20 seconds',
u'Created':
1404904929,
u'Image':
u'pixelated:latest',
u'Ports': [{
u'IP': u'0.0.0.0',
u'Type': u'tcp',
u'PublicPort': 5000,
u'PrivatePort': 4567
}],
u'Command':
u'sleep 100',
u'Names': [u'/test'],
u'Id':
u'f59ee32d2022b1ab17eef608d2cd617b7c086492164b8c411f1cbcf9bfef0d87'
}
client.containers.side_effect = [[], [], [container]]
client.wait.return_value = 0
provider = self._create_initialized_provider(self._adapter,
'some docker url')
user_config = self._user_config('test')
provider.start(user_config)
# when/then
self.assertRaises(InstanceAlreadyRunningError, provider.reset_data,
user_config)
@patch('pixelated.provider.docker.TempDir')
@patch('pixelated.provider.docker.pkg_resources')
@patch('pixelated.provider.docker.docker.Client')
def test_use_build_script_instead_of_docker_file_if_available(
self, docker_mock, res_mock, tempDir_mock):
# given
provider = DockerProvider(self._adapter, 'leap_provider',
self._leap_provider_x509)
tempBuildDir = TempDir()
try:
tempDir_mock.return_value = tempBuildDir
tempBuildDir_name = tempBuildDir.name
with NamedTemporaryFile() as file:
res_mock.resource_exists.return_value = True
res_mock.resource_string.return_value = '#!/bin/bash\necho %s $PWD > %s' % (
file.name, file.name)
# when
provider.initialize()
# then
res_mock.resource_exists.assert_called_with(
'pixelated.resources', 'init-pixelated-docker-context.sh')
res_mock.resource_string.assert_called_with(
'pixelated.resources', 'init-pixelated-docker-context.sh')
with open(file.name, "r") as input:
data = input.read().replace('\n', '')
self.assertEqual(
'%s %s' %
(file.name, os.path.realpath(tempBuildDir_name)), data)
docker_mock.return_value.build.assert_called_once_with(
path=tempBuildDir_name,
tag='pixelated:latest',
fileobj=None)
finally:
tempBuildDir.dissolve()
@patch('pixelated.provider.docker.docker.Client')
@patch('pixelated.provider.docker.CredentialsToDockerStdinWriter')
def test_that_credentials_are_passed_to_agent_by_stdin(
self, credentials_mock, docker_mock):
# given
user_config = self._user_config('test')
provider = self._create_initialized_provider(self._adapter,
'some docker url')
prepare_pixelated_container = MagicMock()
container = MagicMock()
class ProcessStub(object):
def start(self):
self._target()
def __init__(self, target):
self._target = target
client = docker_mock.return_value
client.create_container.side_effect = [
prepare_pixelated_container, container
]
client.wait.return_value = 0
# when
provider.pass_credentials_to_agent(user_config, 'password')
provider.start(user_config)
# then
credentials_mock.return_value.start.assert_called_once_with()
@patch('pixelated.provider.docker.docker.Client')
def test_provider_checks_working_connection_to_docker(self, docker_mock):
client = docker_mock.return_value
client.info.side_effect = Exception
self.assertRaises(Exception, DockerProvider, self._adapter,
'leap_provider', self._leap_provider_x509)
@patch('pixelated.provider.docker.docker.Client')
def test_that_provider_x509_ca_bundle_is_copied_to_agent(
self, docker_mock):
user_config = self._user_config('test')
provider = self._create_initialized_provider(self._adapter,
'some docker url')
client = docker_mock.return_value
client.wait.return_value = 0
with NamedTemporaryFile() as ca_file:
with open(ca_file.name, 'w') as fd:
fd.write('some certificate')
self._leap_provider_x509.ca_bundle = ca_file.name
provider.start(user_config)
self.assertTrue(
exists(
join(self.root_path, 'test', 'data',
'dispatcher-leap-provider-ca.crt')))
def _create_initialized_provider(self,
adapter,
docker_url=DockerProvider.
DEFAULT_DOCKER_URL):
provider = DockerProvider(adapter, 'leap_provider_hostname',
self._leap_provider_x509, docker_url)
provider._initializing = False
return provider
def _user_config(self, name):
path = join(self.root_path, name)
os.makedirs(path)
return UserConfig(name, path)
class ForkRunnerTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TempDir()
self._tmpbin = NamedTemporaryFile()
self.mailpile_bin = self._tmpbin.name
self.root_path = self._tmpdir.name
self.gpg_initializer = MagicMock()
self._adapter = MailpileAdapter(self.mailpile_bin,
None,
gpg_initializer=self.gpg_initializer)
self.runner = ForkRunner(self.root_path, self._adapter)
def tearDown(self):
self._tmpbin.close()
self._tmpdir.dissolve()
def _create_expected_env_check(self, agent_name='test'):
env_check = MailpileEnvCheck({
'MAILPILE_HOME':
os.path.join(self.root_path, agent_name, 'data'),
'GNUPGHOME':
os.path.join(self.root_path, agent_name, 'data', 'gnupg')
})
return env_check
@patch('subprocess.call')
@patch('os.environ', new={})
def test_initialize_calls_mailpile_setup_with_proper_environment(
self, call_mock):
self.runner.initialize('test')
env_check = self._create_expected_env_check()
call_mock.assert_called_once_with([self.mailpile_bin, '--setup'],
close_fds=True,
env=env_check)
@patch('subprocess.call')
@patch('subprocess.Popen')
def test_start_calls_mailpile_www_with_proper_environment(
self, popen_mock, call_mock):
# given
popen_mock.return_value = popen_mock
# when
p = self.runner.start('test')
# then
self.assertEqual(ForkedProcess(popen_mock, 5000), p)
env_check = self._create_expected_env_check()
popen_mock.assert_called_once_with([self.mailpile_bin, '--www'],
stdin=subprocess.PIPE,
close_fds=True,
env=env_check)
call_mock.assert_called_once_with(
[self.mailpile_bin, '--set', 'sys.http_port=5000'],
close_fds=True,
env=env_check)
@patch('subprocess.call')
@patch('subprocess.Popen')
def test_that_different_instances_get_different_ports(
self, popen_mock, call_mock):
# given
popen_mock.return_value = popen_mock
# when
self.runner.start('first')
self.runner.start('second')
# then
env_check_first = self._create_expected_env_check('first')
env_check_second = self._create_expected_env_check('second')
call_mock.assert_any_call(
[self.mailpile_bin, '--set', 'sys.http_port=5000'],
close_fds=True,
env=env_check_first)
call_mock.assert_any_call(
[self.mailpile_bin, '--set', 'sys.http_port=5001'],
close_fds=True,
env=env_check_second)
def _create_expected_env_check_with_virtualenv(self, virtualenv,
absent_keys):
env_check = MailpileEnvCheck(
{
'MAILPILE_HOME': os.path.join(self.root_path, 'test', 'data'),
'GNUPGHOME': os.path.join(self.root_path, 'test', 'data',
'gnupg'),
'VIRTUAL_ENV': virtualenv,
'PATH': '%s/bin:/bin:/usr/bin' % virtualenv
},
absent_keys=absent_keys)
return env_check
@patch('subprocess.call')
@patch('os.environ',
new={
'PATH': '/bin:/usr/bin',
'PYTHONHOME': '/some/python/home'
})
def test_that_virtualenv_is_honored(self, call_mock):
# given
virtualenv_path = '/some/virtual/env'
adapter = MailpileAdapter(self.mailpile_bin, virtualenv_path,
self.gpg_initializer)
self.runner = ForkRunner(self.root_path, adapter)
# when
self.runner.initialize('test')
# then
keys_to_remove = ['PYTHONHOME'] # must not be set for virtualenvs
env_check = self._create_expected_env_check_with_virtualenv(
virtualenv_path, keys_to_remove)
call_mock.assert_called_once_with([self.mailpile_bin, '--setup'],
close_fds=True,
env=env_check)
class AppTestClient(object):
INDEX_KEY = (
"\xde3?\x87\xff\xd9\xd3\x14\xf0\xa7>\x1f%C{\x16.\\\xae\x8c\x13\xa7\xfb\x04\xd4]+\x8d_\xed\xd1\x8d\x0bI"
"\x8a\x0e\xa4tm\xab\xbf\xb4\xa5\x99\x00d\xd5w\x9f\x18\xbc\x1d\xd4_W\xd2\xb6\xe8H\x83\x1b\xd8\x9d\xad"
)
ACCOUNT = "test"
MAIL_ADDRESS = "*****@*****.**"
def _initialize(self):
self._tmp_dir = TempDir()
self.accounts = {}
@defer.inlineCallbacks
def start_client(self, mode=UserAgentMode(is_single_user=True)):
self._initialize()
self._mode = mode
self._test_account = AppTestAccount(self.ACCOUNT, self._tmp_dir.name)
yield self._test_account.start()
self.cleanup = lambda: self._tmp_dir.dissolve()
# copy fields for single user tests
self.soledad = self._test_account.soledad
self.search_engine = self._test_account.search_engine
self.keymanager = self._test_account.keymanager
self.mail_sender = self._test_account.mail_sender
self.mail_store = self._test_account.mail_store
self.attachment_store = self._test_account.attachment_store
self.draft_service = self._test_account.draft_service
self.leap_session = self._test_account.leap_session
self.feedback_service = self._test_account.feedback_service
self.mail_service = self._test_account.mail_service
self.account = self._test_account.account
if mode.is_single_user:
self.service_factory = SingleUserServicesFactory(mode)
services = self._test_account.services
self.service_factory.add_session("someuserid", services)
self.resource = RootResource(self.service_factory)
self.resource.initialize()
else:
self.service_factory = StubServicesFactory(self.accounts, mode)
provider = mock()
provider.config = LeapConfig(self._tmp_dir.name)
self.resource = set_up_protected_resources(
RootResource(self.service_factory), provider, self.service_factory, checker=StubSRPChecker(provider)
)
@defer.inlineCallbacks
def create_user(self, account_name):
if self._mode.is_single_user:
raise Exception("Not supported in single user mode")
account = AppTestAccount(account_name, self._tmp_dir.name)
yield account.start()
self.accounts[account_name] = account
def _render(self, request, as_json=True):
def get_str(_str):
return json.loads(_str) if as_json else _str
def get_request_written_data(_=None):
written_data = request.getWrittenData()
if written_data:
return get_str(written_data)
resource = getChildForRequest(self.resource, request)
result = resource.render(request)
if isinstance(result, basestring):
return get_str(result), request
# result is NOT_DONE_YET
d = succeed(request) if request.finished else request.notifyFinish()
d.addCallback(get_request_written_data)
return d, request
def listenTCP(self, port=4567, host="127.0.0.1"):
reactor.listenTCP(port, PixelatedSite(self.resource), interface=host)
def run_on_a_thread(self, logfile="/tmp/app_test_client.log", port=4567, host="127.0.0.1"):
def _start():
self.listenTCP(port, host)
reactor.run()
process = multiprocessing.Process(target=_start)
process.start()
time.sleep(1)
return lambda: process.terminate()
def stop(self):
reactor.stop()
def get(self, path, get_args="", as_json=True, ajax=True, csrf="token"):
request = request_mock(path, ajax=ajax, csrf=csrf)
request.args = get_args
return self._render(request, as_json)
def post(self, path, body="", headers=None, ajax=True, csrf="token"):
headers = headers or {"Content-Type": "application/json"}
request = request_mock(path=path, method="POST", body=body, headers=headers, ajax=ajax, csrf=csrf)
return self._render(request)
def put(self, path, body, ajax=True, csrf="token"):
request = request_mock(
path=path, method="PUT", body=body, headers={"Content-Type": ["application/json"]}, ajax=ajax, csrf=csrf
)
return self._render(request)
def delete(self, path, body="", ajax=True, csrf="token"):
request = request_mock(
path=path, body=body, headers={"Content-Type": ["application/json"]}, method="DELETE", ajax=ajax, csrf=csrf
)
return self._render(request)
@defer.inlineCallbacks
def add_mail_to_inbox(self, input_mail):
mail = yield self.mail_store.add_mail("INBOX", input_mail.raw)
defer.returnValue(mail)
def account_for(self, username):
return self.accounts[username]
def add_mail_to_user_inbox(self, input_mail, username):
return self.account_for(username).mail_store.add_mail("INBOX", input_mail.raw)
@defer.inlineCallbacks
def add_multiple_to_mailbox(
self, num, mailbox="", flags=[], tags=[], to="*****@*****.**", cc="*****@*****.**", bcc="*****@*****.**"
):
mails = []
yield self.mail_store.add_mailbox(mailbox)
for _ in range(num):
builder = MailBuilder().with_status(flags).with_tags(tags).with_to(to).with_cc(cc).with_bcc(bcc)
builder.with_body(str(random.random()))
input_mail = builder.build_input_mail()
mail = yield self.mail_store.add_mail(mailbox, input_mail.raw)
if tags:
mail.tags |= set(tags)
if flags:
for flag in flags:
mail.flags.add(flag)
if tags or flags:
yield self.mail_store.update_mail(mail)
mails.append(mail)
defer.returnValue(mails)
def _create_mail_sender(self):
mail_sender = Mock()
mail_sender.sendmail.side_effect = lambda mail: succeed(mail)
return mail_sender
def _generate_soledad_test_folder_name(self, soledad_test_folder="/tmp/soledad-test/test"):
return os.path.join(soledad_test_folder, str(uuid.uuid4()))
def get_mails_by_tag(self, tag, page=1, window=100):
tags = "tag:%s" % tag
return self.search(tags, page, window)
@defer.inlineCallbacks
def search(self, query, page=1, window=100):
res, _ = self.get("/mails", {"q": [query], "w": [str(window)], "p": [str(page)]})
res = yield res
defer.returnValue([ResponseMail(m) for m in res["mails"]])
@defer.inlineCallbacks
def get_mails_by_mailbox_name(self, mbox_name):
mail_ids = yield self.mail_store.get_mailbox_mail_ids(mbox_name)
mails = yield self.mail_store.get_mails(mail_ids)
defer.returnValue(mails)
@defer.inlineCallbacks
def get_attachment(self, ident, encoding, filename=None, content_type=None, ajax=True, csrf="token"):
params = {"encoding": [encoding]}
if filename:
params["filename"] = [filename]
if content_type:
params["content_type"] = [content_type]
deferred_result, req = self.get("/attachment/%s" % ident, params, as_json=False, ajax=ajax, csrf=csrf)
res = yield deferred_result
defer.returnValue((res, req))
@defer.inlineCallbacks
def post_attachment(self, data, headers):
deferred_result, req = self.post("/attachment", body=data, headers=headers)
res = yield deferred_result
defer.returnValue((res, req))
def put_mail(self, data):
res, req = self.put("/mails", data)
return res, req
def post_tags(self, mail_ident, tags_json):
res, req = self.post("/mail/%s/tags" % mail_ident, tags_json)
return res
def get_tags(self, **kwargs):
res, req = self.get("/tags", kwargs)
return res
def get_mail(self, mail_ident):
res, req = self.get("/mail/%s" % mail_ident)
return res
def delete_mail(self, mail_ident):
res, req = self.delete("/mail/%s" % mail_ident)
return res
def delete_mails(self, idents):
res, req = self.post("/mails/delete", json.dumps({"idents": idents}))
return res
def mark_many_as_unread(self, idents):
res, req = self.post("/mails/unread", json.dumps({"idents": idents}))
return res
def mark_many_as_read(self, idents):
res, req = self.post("/mails/read", json.dumps({"idents": idents}))
return res
def get_contacts(self, query):
res, req = self.get("/contacts", get_args={"q": query})
return res
class RESTfulServerTest(unittest.TestCase):
mock_provider = None
ssl_config = None
server = None
@classmethod
def setUpClass(cls):
RESTfulServerTest.mock_provider = MagicMock(spec=Provider)
RESTfulServerTest.mock_users = MagicMock(spec=Users)
RESTfulServerTest.mock_authenticator = MagicMock(spec=Authenticator)
RESTfulServerTest.ssl_config = SSLConfig(certfile(), keyfile())
RESTfulServerTest.server = RESTfulServer(
RESTfulServerTest.ssl_config, RESTfulServerTest.mock_users,
RESTfulServerTest.mock_authenticator,
RESTfulServerTest.mock_provider)
RESTfulServerTest.server.server_forever_in_backgroud()
time.sleep(1) # let it get up to speed
@classmethod
def tearDownClass(cls):
RESTfulServerTest.server.shutdown()
print 'Stopped test server'
def setUp(self):
self.mock_provider = RESTfulServerTest.mock_provider
self.mock_provider.reset_mock()
self.mock_users.reset_mock()
self.mock_authenticator.reset_mock()
self.ssl_request = requests.Session()
self.ssl_request.mount('https://', EnforceTLSv1Adapter())
self._tmpdir = TempDir()
self._root_path = self._tmpdir.name
def tearDown(self):
self._tmpdir.dissolve()
def get(self, url):
return self.ssl_request.get(url, verify=cafile())
def put(self, url, data=None):
if data:
data = json.dumps(data)
return self.ssl_request.put(
url,
data=data,
headers={'content-type': 'application/json'},
verify=cafile())
def post(self, url, data=None):
if data:
data = json.dumps(data)
return self.ssl_request.post(
url,
data=data,
headers={'content-type': 'application/json'},
verify=cafile())
def delete(self, url):
return self.ssl_request.delete(url, verify=cafile())
def assertSuccessJson(self, dict, response):
self.assertEqual(200, response.status_code)
self.assertEquals('application/json', response.headers['content-type'])
self.assertEqual(dict, response.json())
def assertInternalError(self, response):
self.assertEqual(500, response.status_code)
def test_list_empty_agents(self):
# given
self.mock_provider.status.return_value = {'state': 'stopped'}
self.mock_users.list.return_value = []
# when
r = self.get('https://*****:*****@patch('pixelated.manager.SSLWSGIRefServerAdapter')
@patch('pixelated.manager.run'
) # mock run call to avoid actually startng the server
def test_that_ssl_server_adapter_gets_used_when_ssl_config_is_provided(
self, run_mock, ssl_adapter_mock):
server = RESTfulServer(RESTfulServerTest.ssl_config,
RESTfulServerTest.mock_users,
RESTfulServerTest.mock_authenticator,
RESTfulServerTest.mock_provider)
# when
server.serve_forever()
expected_ca_certs = None # which means system ciphers
expected_ciphers = DEFAULT_CIPHERS
expected_ssl_version = latest_available_ssl_version()
expected_host = '127.0.0.1'
expected_port = 4443
expected_certfile = certfile()
expected_keyfile = keyfile()
ssl_adapter_mock.assert_called_once_with(
ssl_ca_certs=expected_ca_certs,
ssl_ciphers=expected_ciphers,
ssl_version=expected_ssl_version,
host=expected_host,
port=expected_port,
ssl_cert_file=expected_certfile,
ssl_key_file=expected_keyfile)
@patch('pixelated.manager.WSGIRefServer')
@patch('pixelated.manager.run'
) # mock run call to avoid actually startng the server
def test_that_serve_forever_runs_without_ssl_context(
self, run_mock, wsgiRefServer_mock):
# given
server = RESTfulServer(None, RESTfulServerTest.mock_users,
RESTfulServerTest.mock_authenticator,
RESTfulServerTest.mock_provider)
# when
server.serve_forever()
# then
wsgiRefServer_mock.assert_called_once_with(host='localhost', port=4443)
def test_handles_provider_initializing(self):
self.mock_users.list.return_value = ['test']
self.mock_provider.status.side_effect = ProviderInitializingException
r = self.get('https://*****:*****@patch('pixelated.manager.DockerProvider')
@patch('pixelated.manager.RESTfulServer')
@patch('pixelated.manager.Thread')
@patch('pixelated.manager.Users')
@patch('pixelated.manager.LeapProvider')
@patch('pixelated.manager.LeapCertificate')
def test_that_initialize_happens_in_background_thread(
self, leap_certificate_mock, leap_provider_mock, users_mock,
thread_mock, server_mock, docker_provider_mock):
# given
docker_provider_mock.return_value = self.mock_provider
manager = DispatcherManager(self._root_path,
None,
None,
None,
None,
provider='docker')
# when
manager.serve_forever()
# then
thread_mock.assert_called_with(target=self.mock_provider.initialize)
self.assertFalse(self.mock_provider.initialize.called)
@patch('pixelated.manager.LeapCertificate')
@patch('pixelated.manager.Authenticator')
@patch('pixelated.manager.DockerProvider')
@patch('pixelated.manager.RESTfulServer')
@patch('pixelated.manager.Thread')
@patch('pixelated.manager.Users')
@patch('pixelated.manager.LeapProvider')
def test_that_tls_config_gets_passed_to_authenticator(
self, leap_provider_mock, users_mock, thread_mock, server_mock,
docker_provider_mock, authenticator_mock, leap_certificate_mock):
# given
manager = DispatcherManager(
self._root_path,
None,
None,
None,
'some ca bundle',
leap_provider_fingerprint='some fingerprint',
provider='docker')
# when
manager.serve_forever()
# then
authenticator_mock.assert_called_once_with(
users_mock.return_value, leap_provider_mock.return_value)
@patch('pixelated.manager.LeapCertificate')
@patch('pixelated.manager.Authenticator')
@patch('pixelated.manager.DockerProvider')
@patch('pixelated.manager.RESTfulServer')
@patch('pixelated.manager.Thread')
@patch('pixelated.manager.Users')
@patch('pixelated.manager.LeapProvider')
def test_that_leap_certificate_gets_downloaded_on_serve_forever(
self, leap_provider_mock, users_mock, thread_mock, server_mock,
docker_provider_mock, authenticator_mock, leap_certificate_mock):
# given
cert_file = join(self._root_path, 'ca.crt')
manager = DispatcherManager(
self._root_path,
None,
None,
None,
'some ca bundle',
leap_provider_fingerprint='some fingerprint',
provider='docker')
# when
manager.serve_forever()
# then
leap_certificate_mock.return_value.refresh_api_ca_bundle.assert_called_once_with(
)
