def setUp(self):
self.request_mock = MagicMock(name='Request Mock')
self.request_mock.only.return_value = self.request_mock
self.request_mock.search_text.return_value = self.request_mock
self.request_mock.order_by.return_value = self.request_mock
self.request_mock.id = 'id'
self.request_mock.instance_name = 'default'
self.request_mock.__getitem__.return_value = self.request_mock
self.request_mock.__len__.return_value = 1
mongo_patcher = patch(
'brew_view.controllers.request_list_api.Request.objects')
self.addCleanup(mongo_patcher.stop)
self.mongo_mock = mongo_patcher.start()
self.mongo_mock.count.return_value = 1
serialize_patcher = patch(
'brew_view.controllers.request_list_api.BeerGardenSchemaParser.serialize_request'
)
self.addCleanup(serialize_patcher.stop)
self.serialize_mock = serialize_patcher.start()
self.serialize_mock.return_value = 'serialized_request'
self.client_mock = Mock(name='client_mock')
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False))
self.future_mock = Future()
super(RequestListAPITest, self).setUp()
def test_start_app_failed_copy_java(self):
configuration = {
'app_port': 2000,
'login_server': 'public1',
'service_id': 'default',
'version_id': 'v1',
'env_vars': {}
}
version_details = {'runtime': 'java',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details)
app_manager_server.projects_manager = {
'test': {'default': {'v1': version_manager}}}
app_manager_server.deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'java').\
and_return(response)
app_manager_server.source_manager = source_manager
flexmock(app_manager_server).should_receive('find_web_inf').\
and_return('/path/to/dir/WEB-INF')
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_raise(IOError)
with self.assertRaises(IOError):
yield app_manager_server.start_app('test_default_v1', configuration)
def release_stream(self, stream):
release_future = Future()
release_future.set_result(None)
if stream.closed():
return release_future
if not self._wait_streams:
if self._closed:
stream.close()
else:
try:
del self._used_streams[id(stream)]
stream.idle_time = time.time()
self._streams.append(stream)
except KeyError:
logging.error("release unknown stream %s", stream)
else:
future = self._wait_streams.popleft()
stream.used_time = time.time()
future.set_result(stream)
while self._wait_streams and self._streams:
stream = self._streams.pop()
self._used_streams[id(stream)] = stream
stream.used_time = time.time()
if not stream.closed():
future = self._wait_streams.popleft()
future.set_result(stream)
return release_future
def test_start_app_failed_copy_java(self):
configuration = {
'app_port': 2000,
'login_server': 'public1',
'service_id': 'default',
'version_id': 'v1',
'env_vars': {}
}
version_details = {'runtime': 'java',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details)
app_manager_server.projects_manager = {
'test': {'default': {'v1': version_manager}}}
app_manager_server.deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'java').\
and_return(response)
app_manager_server.source_manager = source_manager
flexmock(app_manager_server).should_receive('find_web_inf').\
and_return('/path/to/dir/WEB-INF')
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_raise(IOError)
with self.assertRaises(IOError):
yield app_manager_server.start_app('test_default_v1', configuration)
def test_start_app_failed_copy_java(self):
version_details = {'runtime': 'java',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details,
project_id='test',
revision_key='test_default_v1_1',
version_key='test_default_v1')
instance_manager = InstanceManager(
None, None, None, None, None, None, None, None, None)
instance_manager._projects_manager = {
'test': {'default': {'v1': version_manager}}}
instance_manager._deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'java').\
and_return(response)
instance_manager._source_manager = source_manager
flexmock(utils).should_receive('find_web_inf').\
and_return('/path/to/dir/WEB-INF')
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_raise(IOError)
with self.assertRaises(IOError):
yield instance_manager._start_instance(version_manager, 20000)
def setUpModule(): patcher = patch.object(gen, 'sleep') patchers.append(patcher) sleep_response = Future() sleep_response.set_result(None) sleep_mock = patcher.start() sleep_mock.return_value = sleep_response
def get_user_stats_by_id(self, userid):
future = Future()
stats = self.controller.get_user_stats_by_id(userid)
if stats:
stats = self.format_stats(stats)
future.set_result(stats)
return future
def test_wait_for_app(self):
port = 20000
ip = '127.0.0.1'
testing.disable_logging()
fake_opener = flexmock(open=lambda url, timeout: flexmock(
code=200, headers=flexmock(headers=[])))
flexmock(urllib2).should_receive('build_opener').and_return(
fake_opener)
flexmock(appscale_info).should_receive('get_private_ip').and_return(ip)
instance_manager = InstanceManager(None, None, None, None, None, None,
None, None, None)
instance_manager._private_ip = ip
instance_started = yield instance_manager._wait_for_app(port)
self.assertEqual(True, instance_started)
current_time = time.time()
flexmock(monotonic).should_receive('monotonic').and_return(current_time).\
and_return(current_time + START_APP_TIMEOUT + 1)
response = Future()
response.set_result(None)
flexmock(gen).should_receive('sleep').and_return(response)
fake_opener.should_receive('open').and_raise(IOError)
instance_started = yield instance_manager._wait_for_app(port)
self.assertEqual(False, instance_started)
def setUp(self):
self.request_mock = MagicMock(name="Request Mock")
self.request_mock.only.return_value = self.request_mock
self.request_mock.search_text.return_value = self.request_mock
self.request_mock.order_by.return_value = self.request_mock
self.request_mock.id = "id"
self.request_mock.instance_name = "default"
self.request_mock.__getitem__.return_value = self.request_mock
self.request_mock.__len__.return_value = 1
mongo_patcher = patch(
"beer_garden.api.http.handlers.v1.request.Request.objects")
self.addCleanup(mongo_patcher.stop)
self.mongo_mock = mongo_patcher.start()
self.mongo_mock.count.return_value = 1
serialize_patcher = patch(
"beer_garden.api.http.handlers.v1.request.MongoParser.serialize_request"
)
self.addCleanup(serialize_patcher.stop)
self.serialize_mock = serialize_patcher.start()
self.serialize_mock.return_value = "serialized_request"
self.client_mock = Mock(name="client_mock")
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False),
)
self.future_mock = Future()
super(RequestListAPITest, self).setUp()
def setUpModule():
patcher = patch.object(gen, 'sleep')
patchers.append(patcher)
sleep_response = Future()
sleep_response.set_result(None)
sleep_mock = patcher.start()
sleep_mock.return_value = sleep_response
def test_start_app_failed_copy_java(self):
version_details = {'runtime': 'java',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details,
project_id='test',
revision_key='test_default_v1_1',
version_key='test_default_v1')
instance_manager = InstanceManager(
None, None, None, None, None, None, None, None, None)
instance_manager._projects_manager = {
'test': {'default': {'v1': version_manager}}}
instance_manager._deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'java').\
and_return(response)
instance_manager._source_manager = source_manager
flexmock(utils).should_receive('find_web_inf').\
and_return('/path/to/dir/WEB-INF')
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_raise(IOError)
with self.assertRaises(IOError):
yield instance_manager._start_instance(version_manager, 20000)
def test_wait_for_app(self):
port = 20000
ip = '127.0.0.1'
testing.disable_logging()
fake_opener = flexmock(
open=lambda url, timeout: flexmock(code=200,
headers=flexmock(headers=[])))
flexmock(urllib2).should_receive('build_opener').and_return(fake_opener)
flexmock(appscale_info).should_receive('get_private_ip').and_return(ip)
instance_manager = InstanceManager(
None, None, None, None, None, None, None, None, None)
instance_manager._private_ip = ip
instance_started = yield instance_manager._wait_for_app(port)
self.assertEqual(True, instance_started)
current_time = time.time()
flexmock(time).should_receive('time').and_return(current_time).\
and_return(current_time + START_APP_TIMEOUT + 1)
response = Future()
response.set_result(None)
flexmock(gen).should_receive('sleep').and_return(response)
fake_opener.should_receive('open').and_raise(IOError)
instance_started = yield instance_manager._wait_for_app(port)
self.assertEqual(False, instance_started)
def release_stream(self, stream):
release_future = Future()
release_future.set_result(None)
if stream.closed():
return release_future
if not self._wait_streams:
if self._closed:
stream.close()
else:
try:
del self._used_streams[id(stream)]
stream.idle_time = time.time()
self._streams.append(stream)
except KeyError:
logging.error("release unknown stream %s", stream)
else:
future = self._wait_streams.popleft()
stream.used_time = time.time()
future.set_result(stream)
while self._wait_streams and self._streams:
stream = self._streams.pop()
self._used_streams[id(stream)] = stream
stream.used_time = time.time()
if not stream.closed():
future = self._wait_streams.popleft()
future.set_result(stream)
return release_future
def setUp(self):
self.client_mock = MagicMock(name='client_mock')
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False))
self.future_mock = Future()
super(AdminAPITest, self).setUp()
def test_default_sleep_manual_mock(self, sleep_mock):
# We need to wrap the mock in a Future
mock_future = Future()
mock_future.set_result(None)
sleep_mock.return_value = mock_future
response = self.fetch('/timer?seconds=10')
self.assertEqual(response.code, 200)
self.assertEqual(response.body, b'Slept for 10 seconds')
sleep_mock.assert_called_once_with(10)
def setUp(self):
self.client_mock = Mock(name="client_mock")
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False),
)
self.future_mock = Future()
super(VersionHandlerTest, self).setUp()
def test_start_app_goodconfig_python(self):
configuration = {
'app_port': 2000,
'login_server': 'public1',
'service_id': 'default',
'version_id': 'v1',
'env_vars': {}
}
version_details = {
'runtime': 'python27',
'revision': 1,
'deployment': {
'zip': {
'sourceUrl': 'source.tar.gz'
}
}
}
version_manager = flexmock(version_details=version_details)
app_manager_server.projects_manager = {
'test': {
'default': {
'v1': version_manager
}
}
}
app_manager_server.deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source'). \
with_args('test_default_v1_1', 'source.tar.gz', 'python27'). \
and_return(response)
app_manager_server.source_manager = source_manager
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_return('fakeconfig')
flexmock(monit_interface).should_receive('start').\
and_return(True)
flexmock(app_manager_server).should_receive('wait_on_app').\
and_return(True)
flexmock(os).should_receive('popen').\
and_return(flexmock(read=lambda: '12345\n'))
flexmock(file_io).should_receive('write').\
and_return()
flexmock(threading).should_receive('Thread').\
and_return(flexmock(start=lambda: None))
flexmock(app_manager_server).should_receive(
"setup_logrotate").and_return()
app_manager_server.zk_client = flexmock()
app_manager_server.zk_client.should_receive('ensure_path')
yield app_manager_server.start_app('test_default_v1', configuration)
def test_start_app_goodconfig_java(self):
configuration = {
'app_port': 20000,
'login_server': 'public1',
'service_id': 'default',
'version_id': 'v1',
'env_vars': {}
}
version_details = {'runtime': 'java',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details)
app_manager_server.projects_manager = {
'test': {'default': {'v1': version_manager}}}
app_manager_server.deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'java').\
and_return(response)
app_manager_server.source_manager = source_manager
start_cmd = ('/root/appscale/AppServer_Java/appengine-java-repacked/bin/'
'dev_appserver.sh --port 20000')
flexmock(app_manager_server).should_receive('create_java_start_cmd').\
and_return(start_cmd)
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_return('fakeconfig')
flexmock(monit_interface).should_receive('start').\
and_return(True)
flexmock(app_manager_server).should_receive('create_java_app_env').\
and_return({})
flexmock(app_manager_server).should_receive('wait_on_app').\
and_return(True)
flexmock(app_manager_server).should_receive('locate_dir').\
and_return('/path/to/dir/')
flexmock(os).should_receive('popen').\
and_return(flexmock(read=lambda: '0\n'))
flexmock(file_io).should_receive('write').and_return()
flexmock(subprocess).should_receive('call').and_return(0)
flexmock(threading).should_receive('Thread').\
and_return(flexmock(start=lambda: None))
flexmock(app_manager_server).should_receive("setup_logrotate").and_return()
flexmock(os).should_receive('listdir').and_return([])
app_manager_server.zk_client = flexmock()
app_manager_server.zk_client.should_receive('ensure_path')
yield app_manager_server.start_app('test_default_v1', configuration)
def __init__(self, returnValueDict={}):
# make returnvalue yieldable
NotDict.__init__(self, **returnValueDict)
Future.__init__(self)
self.set_result(self)
# init
self._wrap_result = None
if TRANSLATE:
self['rawmsg'] = self.get('errmsg', '')
self['errmsg'] = \
TRANSLATION[TRANSLATE].get(self.get('errcode', '')) \
or self.get('errmsg', u'没有errmsg')
if not self['rawmsg']: self['rawmsg'] = self['errmsg']
def __init__(self, returnValueDict={}):
# make returnvalue yieldable
NotDict.__init__(self, **returnValueDict)
Future.__init__(self)
self.set_result(self)
# init
self._wrap_result = None
if TRANSLATE:
self['rawmsg'] = self.get('errmsg', '')
self['errmsg'] = \
TRANSLATION[TRANSLATE].get(self.get('errcode', '')) \
or self.get('errmsg', u'没有errmsg')
if not self['rawmsg']: self['rawmsg'] = self['errmsg']
def start_tensorflow(model_name,
client=None,
options=None,
port=TF_PORT,
**kwargs):
""" Start Tensorflow on Dask Cluster
This launches Tensorflow Servers alongside Dask workers in-process
Examples
--------
>>> client = Client('dask-scheduler-address:8786')
>>> tf_spec, dask_spec = start_tensorflow(client)
>>> tf_spec.as_dict()
{'worker': ['192.168.1.100:2222', '192.168.1.101:2222']}
Specify desired number of jobs types as keyword args
>>> tf_config = tf.compat.v1.OptimizerOptions()
>>> tf_config.GlobalJitLevel = tf_config.OFF
>>> tf_config.do_function_inlining = True
>> tf_config.opt_level
>>> tf_config.gpu_options.force_gpu_compatible = True
>>> tf_spec, dask_spec = start_tensorflow(client, tf_config=tf_config, chief=1, master=1, ps=2, worker=30)
>>> tf_spec.as_dict()
{
'chief': ['192.168.1.1:2222'],
'master': ['192.168.1.1:2223'],
'ps': ['192.168.1.104:2222', '192.168.1.105:2222'],
'worker': ['192.168.1.100:2222', '192.168.1.101:2222',
'192.168.1.102:2222', '192.168.1.103:2222']
}
"""
client = client if client is not None else global_cluster(
asynchronous=True)
global_future = Future()
tensorflow_scheduler_wrap = partial(tensorflow_scheduler,
global_future,
model_name,
client=client,
tf_port=port,
**kwargs)
if client.asynchronous:
global_future.add_done_callback(lambda fu: client.loop.stop())
client.loop.add_callback(tensorflow_scheduler_wrap)
client.loop.start()
result = global_future.result()
else:
result = client.sync(client.loop, tensorflow_scheduler_wrap)
return result
class AdminAPITest(TestHandlerBase):
def setUp(self):
self.client_mock = MagicMock(name="client_mock")
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False),
)
self.future_mock = Future()
super(AdminAPITest, self).setUp()
@patch("brew_view.controllers.admin_api.thrift_context")
def test_patch(self, context_mock):
context_mock.return_value = self.fake_context
self.client_mock.rescanSystemDirectory.return_value = self.future_mock
self.future_mock.set_result(None)
response = self.fetch(
"/api/v1/admin/",
method="PATCH",
body='{"operations": [{"operation": "rescan"}]}',
headers={"content-type": "application/json"},
)
self.assertEqual(204, response.code)
self.client_mock.rescanSystemDirectory.assert_called_once_with()
@patch("brew_view.controllers.admin_api.thrift_context")
def test_patch_exception(self, context_mock):
context_mock.return_value = self.fake_context
self.client_mock.rescanSystemDirectory.return_value = self.future_mock
self.future_mock.set_exception(ValueError())
response = self.fetch(
"/api/v1/admin/",
method="PATCH",
body='{"operations": [{"operation": "rescan"}]}',
headers={"content-type": "application/json"},
)
self.assertGreaterEqual(response.code, 500)
self.client_mock.rescanSystemDirectory.assert_called_once_with()
def test_patch_bad_operation(self):
response = self.fetch(
"/api/v1/admin/",
method="PATCH",
body='{"operations": [{"operation": "fake"}]}',
headers={"content-type": "application/json"},
)
self.assertGreaterEqual(response.code, 400)
self.assertLess(response.code, 500)
def test_stop_app_instance(self):
version_key = 'test_default_v1'
port = 20000
flexmock(misc).should_receive('is_app_name_valid').and_return(False)
instance_manager = InstanceManager(None, None, None, None, None, None,
None, None, None)
flexmock(misc).should_receive('is_app_name_valid').and_return(True)
response = Future()
response.set_result(None)
instance_manager._routing_client = flexmock(
unregister_instance=lambda instance: response)
flexmock(ServiceOperator).should_receive('stop_async').\
with_args('appscale-instance-run@test_default_v1-20000').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(instance_manager).should_receive('_clean_old_sources').\
and_return(response)
instance_manager._service_operator = flexmock(
stop_async=lambda service: response)
yield instance_manager._stop_app_instance(
instance.Instance('_'.join([version_key, 'revid']), port))
def setUp(self):
self.instance_mock = Mock(status='RUNNING')
type(self.instance_mock).name = PropertyMock(return_value='default')
self.system_mock = Mock(id='1234', display_name='display', version='1.0.0',
instances=[self.instance_mock])
type(self.system_mock).name = PropertyMock(return_value='system_name')
self.queue_name = "%s[%s]-%s" % (self.system_mock.name, self.instance_mock.name,
self.system_mock.version)
self.client_mock = Mock(name='client_mock')
self.fake_context = MagicMock(__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False))
self.future_mock = Future()
super(QueueListAPITest, self).setUp()
def _main():
f1 = Future()
f2 = Future()
with MyLocal() as ctx:
ctx.data['foo'] = 1
io_loop.add_callback(_cb, f1)
io_loop.spawn_callback(_spawned_cb, f2)
with MyLocal() as ctx1:
with MyLocal() as ctx2:
assert ctx2.prev_ctx() == ctx1
return multi([f1, f2])
def setUp(self):
self.system_mock = Mock(version='1.0', commands=[])
mongo_patcher = patch(
'mongoengine.queryset.manager.QuerySetManager.__get__')
self.addCleanup(mongo_patcher.stop)
self.get_mock = mongo_patcher.start()
self.get_mock.return_value.get.return_value = self.system_mock
self.client_mock = Mock(name='client_mock')
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False))
self.future_mock = Future()
super(SystemAPITest, self).setUp()
def send_message(self, message, async_reply_code=None):
"""Sends a message. If async is not None, this returns a Future that can
be yielded. The Future will resolve when the supplied reply code is next
received. The Future will error-out if no reply is received in the delay
(seconds) given in the constructor.
"""
try:
logger.debug("Sending \"{}\" over SPC.".format(message))
data = message + "\x0a"
self._stream.write(data.encode("ascii"))
except Exception as e:
logger.exception("Error sending \"{}\" over SPC.".format(message))
raise ConnectionError(
"Error sending \"{}\" over SPC.".format(message))
# async tracking?
if async_reply_code:
queue = self._reply_q[async_reply_code]
fut = Future()
handle = self._io_loop.call_later(delay=self._async_reply_delay,
callback=self._async_timeout,
code=async_reply_code)
queue.append({"future": fut, "handle": handle})
return fut
return None
def __wait(self):
log.debug("Waiting for events")
while not (yield sleep(0.001)):
try:
state = self.__connection.poll()
except QueryCanceledError:
yield sleep(0.1)
continue
f = Future()
def resolve(fileno, io_op):
if f.running():
f.set_result(True)
self.__io_loop.remove_handler(fileno)
if state == psycopg2.extensions.POLL_OK:
raise Return(True)
elif state == psycopg2.extensions.POLL_READ:
self.__io_loop.add_handler(self.__connection.fileno(), resolve,
IOLoop.READ)
yield f
elif state == psycopg2.extensions.POLL_WRITE:
self.__io_loop.add_handler(self.__connection.fileno(), resolve,
IOLoop.WRITE)
yield f
def test_wait_on_app(self):
port = 20000
ip = '127.0.0.1'
testing.disable_logging()
fake_opener = flexmock(
open=lambda opener: flexmock(code=200, headers=flexmock(headers=[])))
flexmock(urllib2).should_receive('build_opener').and_return(fake_opener)
flexmock(appscale_info).should_receive('get_private_ip').and_return(ip)
instance_started = yield app_manager_server.wait_on_app(port)
self.assertEqual(True, instance_started)
response = Future()
response.set_result(None)
flexmock(gen).should_receive('sleep').and_return(response)
fake_opener.should_receive('open').and_raise(IOError)
instance_started = yield app_manager_server.wait_on_app(port)
self.assertEqual(False, instance_started)
def get_stream(self):
if self._closed:
raise TStreamPoolClosedError()
future = Future()
while self._streams:
stream = self._streams.pop()
self._used_streams[id(stream)] = stream
stream.used_time = time.time()
if not stream.closed():
future.set_result(stream)
return future
if self._stream_count < self._max_stream:
self.init_stream(future)
else:
self._wait_streams.append(future)
return future
def get_stream(self):
if self._closed:
raise TStreamPoolClosedError()
future = Future()
while self._streams:
stream = self._streams.pop()
self._used_streams[id(stream)] = stream
stream.used_time = time.time()
if not stream.closed():
future.set_result(stream)
return future
if self._stream_count < self._max_stream:
self.init_stream(future)
else:
self._wait_streams.append(future)
return future
class VersionHandlerTest(TestHandlerBase):
def setUp(self):
self.client_mock = Mock(name='client_mock')
self.fake_context = MagicMock(
__enter__=Mock(return_value=self.client_mock),
__exit__=Mock(return_value=False))
self.future_mock = Future()
super(VersionHandlerTest, self).setUp()
@patch('brew_view.controllers.misc_controllers.thrift_context')
def test_version_everything_works(self, context_mock):
context_mock.return_value = self.fake_context
self.client_mock.getVersion.return_value = self.future_mock
self.future_mock.set_result("bartender_version")
response = self.fetch('/version')
output = json.loads(response.body.decode("utf-8"))
self.client_mock.getVersion.assert_called_once_with()
self.assertEqual(
output, {
'brew_view_version': __version__,
'bartender_version': 'bartender_version',
'current_api_version': 'v1',
'supported_api_versions': ['v1']
})
@patch('brew_view.controllers.misc_controllers.thrift_context')
def test_version_fail_to_get_backend_version(self, context_mock):
context_mock.return_value = self.fake_context
self.client_mock.getVersion.return_value = self.future_mock
self.future_mock.set_exception(ValueError('ERROR'))
response = self.fetch('/version')
output = json.loads(response.body.decode("utf-8"))
self.client_mock.getVersion.assert_called_once_with()
self.assertEqual(
output, {
'brew_view_version': __version__,
'bartender_version': 'unknown',
'current_api_version': 'v1',
'supported_api_versions': ['v1']
})
def test_start_app_goodconfig_python(self):
configuration = {
'app_port': 2000,
'login_server': 'public1',
'service_id': 'default',
'version_id': 'v1',
'env_vars': {}
}
version_details = {'runtime': 'python27',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details)
app_manager_server.projects_manager = {
'test': {'default': {'v1': version_manager}}}
app_manager_server.deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source'). \
with_args('test_default_v1_1', 'source.tar.gz', 'python27'). \
and_return(response)
app_manager_server.source_manager = source_manager
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_return('fakeconfig')
flexmock(monit_interface).should_receive('start').\
and_return(True)
flexmock(app_manager_server).should_receive('wait_on_app').\
and_return(True)
flexmock(os).should_receive('popen').\
and_return(flexmock(read=lambda: '12345\n'))
flexmock(file_io).should_receive('write').\
and_return()
flexmock(threading).should_receive('Thread').\
and_return(flexmock(start=lambda: None))
flexmock(app_manager_server).should_receive("setup_logrotate").and_return()
app_manager_server.zk_client = flexmock()
app_manager_server.zk_client.should_receive('ensure_path')
yield app_manager_server.start_app('test_default_v1', configuration)
def test_start(self):
client = AsyncHTTPClient()
response = Future()
response.set_result(FakeHTTPResponse(200))
client.fetch = MagicMock(return_value=response)
fake_process = FakeProcess()
fake_process.is_running = MagicMock(return_value=True)
server = DatastoreServer(4000, client, False)
# Test that a Datastore server process is started.
with patch.object(psutil, 'Popen', return_value=fake_process) as mock_popen:
yield server.start()
cmd = ['cgexec', '-g', 'memory:appscale-datastore',
'appscale-datastore', '--type', 'cassandra', '--port', '4000']
self.assertEqual(mock_popen.call_count, 1)
self.assertEqual(mock_popen.call_args[0][0], cmd)
def sync_to_async(sync_f, *args):
f = Future()
# use closure here:
def wrapper():
res = sync_f(*args)
f.set_result(res) # keypoint !!
threading.Thread(
target=wrapper).start() #使用多线程去跑同步阻塞函数,跑完后把值设置到future里面即可.
return f
def test_start_app_failed_copy_java(self):
version_details = {
'runtime': 'java',
'revision': 1,
'deployment': {
'zip': {
'sourceUrl': 'source.tar.gz'
}
},
'appscaleExtensions': {
'httpPort': '8080'
}
}
version_manager = flexmock(version_details=version_details,
project_id='test',
revision_key='test_default_v1_1',
version_key='test_default_v1')
instance_manager = InstanceManager(None, None, None, None, None, None,
None, None, None)
instance_manager._login_server = '192.168.33.10'
instance_manager._projects_manager = {
'test': {
'default': {
'v1': version_manager
}
}
}
instance_manager._deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'java').\
and_return(response)
instance_manager._source_manager = source_manager
flexmock(instance).should_receive('find_web_inf'). \
and_return('/path/to/dir/WEB-INF')
response = Future()
response.set_result((19999, []))
flexmock(instance_manager).should_receive('_ensure_api_server'). \
and_return(response)
flexmock(file_io).should_receive('write').and_raise(IOError)
with self.assertRaises(IOError):
yield instance_manager._start_instance(version_manager, 20000)
def connect(self):
future = Future()
def on_connected(connection_future):
if connection_future._exc_info is None:
future.set_result(self)
else:
future.set_exc_info(connection_future.exc_info())
self._connection = Connection(defer_connect = True, *self._args, **self._kwargs)
self._connection.set_close_callback(self.connection_close_callback)
connection_future = async_call_method(self._connection.connect)
IOLoop.current().add_future(connection_future, on_connected)
return future
def cursor(self, **kwargs):
f = Future()
self.__io_loop.add_callback(
self.__queue.put,
(functools.partial(AsyncCursor,
self.__connection,
self.__thread_pool,
self.__wait,
on_open=self.__on_cursor_open,
on_close=self.__on_cursor_close,
**kwargs), f))
return f
async def get(self, url, headers=None, *args, **kwargs):
request = HTTPRequest(url=url, use_gzip=True, headers=headers)
existing_futures = self.rc_cache.get(url, None)
if existing_futures is not None:
future = Future()
existing_futures.append(future)
result = await future
return result
new_futures = []
self.rc_cache[url] = new_futures
try:
response = await self.http_client.fetch(request)
except HTTPError as e:
e = APIError(e.code, e.message)
for future in new_futures:
future.set_exception(e)
del self.rc_cache[url]
raise e
else:
body = response.body
for future in new_futures:
future.set_result(body)
del self.rc_cache[url]
return body
def test_start(self):
client = AsyncHTTPClient()
response = Future()
response.set_result(FakeHTTPResponse(200))
client.fetch = MagicMock(return_value=response)
fake_process = FakeProcess()
fake_process.is_running = MagicMock(return_value=True)
server = DatastoreServer(4000, client, False)
# Test that a Datastore server process is started.
with patch.object(psutil, 'Popen',
return_value=fake_process) as mock_popen:
yield server.start()
cmd = [
'cgexec', '-g', 'memory:appscale-datastore', 'appscale-datastore',
'--type', 'cassandra', '--port', '4000'
]
self.assertEqual(mock_popen.call_count, 1)
self.assertEqual(mock_popen.call_args[0][0], cmd)
def wrapper(*args, **kwargs):
future = Future()
# try:
# result = func(*args, **kwargs)
# except (Return, StopIteration) as e:
# result = getattr(e, "value", None)
# except Exception:
# future.set_exc_info(sys.exc_info())
# return future
# else:
# if isinstance(result, types.GeneratorType):
# try except else
# yielded = next(result)
# Runner(result, future, yielded)
# return future
result = func(*args, **kwargs)
yielded = next(result)
Runner(result, future, yielded)
future.set_result(result)
return future
def test_stop_app_instance(self):
version_key = 'test_default_v1'
port = 20000
flexmock(misc).should_receive('is_app_name_valid').and_return(False)
with self.assertRaises(BadConfigurationException):
yield app_manager_server.stop_app_instance(version_key, port)
flexmock(misc).should_receive('is_app_name_valid').and_return(True)
flexmock(app_manager_server).should_receive('unmonitor').\
and_raise(HTTPError)
unmonitor_future = Future()
unmonitor_future.set_exception(HTTPError(500))
flexmock(app_manager_server).should_receive('unmonitor_and_terminate').\
and_return(unmonitor_future)
entries_response = Future()
entries_response.set_result(['app___test_default_v1_revid-20000'])
flexmock(MonitOperator).should_receive('get_entries').\
and_return(entries_response)
with self.assertRaises(HTTPError):
yield app_manager_server.stop_app_instance(version_key, port)
flexmock(app_manager_server).should_receive('unmonitor')
flexmock(os).should_receive('remove')
flexmock(monit_interface).should_receive('safe_monit_run')
response = Future()
response.set_result(None)
flexmock(app_manager_server).should_receive('clean_old_sources').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(app_manager_server).should_receive('unmonitor_and_terminate').\
and_return(response)
yield app_manager_server.stop_app_instance(version_key, port)
def do_test(self):
future = Future()
future.set_result("test")
return future
def test_start_app_goodconfig_python(self):
testing.disable_logging()
version_details = {'runtime': 'python27',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details,
project_id='test',
revision_key='test_default_v1_1',
version_key='test_default_v1')
projects_manager = {
'test': {'default': {'v1': version_manager}}}
deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'python27').\
and_return(response)
instance_manager = InstanceManager(
None, None, None, projects_manager, deployment_config,
source_manager, None, None, None)
instance_manager._login_server = '192.168.33.10'
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_return('fakeconfig')
response = Future()
response.set_result(None)
flexmock(instance_manager).should_receive('_ensure_api_server').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(MonitOperator).should_receive('reload').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(MonitOperator).should_receive('send_command_retry_process').\
with_args('app___test_default_v1_1-20000', 'start').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(instance_manager).should_receive('_add_routing').\
and_return(response)
flexmock(instance_manager).should_receive('_wait_for_app').\
and_return(True)
flexmock(os).should_receive('popen').\
and_return(flexmock(read=lambda: '12345\n'))
flexmock(file_io).should_receive('write').\
and_return()
flexmock(utils).should_receive("setup_logrotate").and_return()
instance_manager._zk_client = flexmock()
instance_manager._zk_client.should_receive('ensure_path')
instance_manager._monit_operator = flexmock(
reload=lambda x: response,
send_command_retry_process=lambda watch, cmd: response)
yield instance_manager._start_instance(version_manager, 20000)
def test_stop_app_instance(self):
version_key = 'test_default_v1'
port = 20000
flexmock(misc).should_receive('is_app_name_valid').and_return(False)
instance_manager = InstanceManager(
None, None, None, None, None, None, None, None, None)
flexmock(misc).should_receive('is_app_name_valid').and_return(True)
response = Future()
response.set_result(None)
instance_manager._routing_client = flexmock(
unregister_instance=lambda instance: response)
flexmock(MonitOperator).should_receive('send_command_sync').\
with_args('app___test_default_v1-20000', 'unmonitor').\
and_raise(HTTPError)
unmonitor_future = Future()
unmonitor_future.set_exception(HTTPError(500))
flexmock(instance_manager).should_receive('_unmonitor_and_terminate').\
and_return(unmonitor_future)
entries_response = Future()
entries_response.set_result(['app___test_default_v1_revid-20000'])
instance_manager._monit_operator = flexmock(
get_entries=lambda: entries_response,
reload=lambda x: response)
with self.assertRaises(HTTPError):
yield instance_manager._stop_app_instance(
instance.Instance('_'.join([version_key, 'revid']), port))
flexmock(MonitOperator).should_receive('send_command_sync').\
with_args('app___test_default_v1-20000', 'unmonitor')
flexmock(os).should_receive('remove')
flexmock(monit_interface).should_receive('safe_monit_run')
response = Future()
response.set_result(None)
flexmock(MonitOperator).should_receive('reload').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(instance_manager).should_receive('_clean_old_sources').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(instance_manager).should_receive('_unmonitor_and_terminate').\
and_return(response)
yield instance_manager._stop_app_instance(
instance.Instance('_'.join([version_key, 'revid']), port))
def test_start_app_goodconfig_java(self):
testing.disable_logging()
version_details = {'runtime': 'java',
'revision': 1,
'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}}
version_manager = flexmock(version_details=version_details,
project_id='test',
revision_key='test_default_v1_1',
version_key='test_default_v1')
instance_manager = InstanceManager(
None, None, None, None, None, None, None, None, None)
instance_manager._projects_manager = {
'test': {'default': {'v1': version_manager}}}
instance_manager._deployment_config = flexmock(
get_config=lambda x: {'default_max_appserver_memory': 400})
source_manager = flexmock()
response = Future()
response.set_result(None)
source_manager.should_receive('ensure_source').\
with_args('test_default_v1_1', 'source.tar.gz', 'java').\
and_return(response)
instance_manager._source_manager = source_manager
start_cmd = ('/root/appscale/AppServer_Java/appengine-java-repacked/bin/'
'dev_appserver.sh --port 20000')
flexmock(instance).should_receive('create_java_start_cmd').\
and_return(start_cmd)
flexmock(monit_app_configuration).should_receive('create_config_file').\
and_return('fakeconfig')
response = Future()
response.set_result(None)
flexmock(instance_manager).should_receive('_ensure_api_server').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(MonitOperator).should_receive('reload').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(MonitOperator).should_receive('send_command_retry_process').\
with_args('app___test_default_v1_1-20000', 'start').\
and_return(response)
response = Future()
response.set_result(None)
flexmock(instance_manager).should_receive('_add_routing').\
and_return(response)
flexmock(instance).should_receive('create_java_app_env').\
and_return({})
flexmock(instance_manager_module).should_receive('create_java_start_cmd').\
and_return('/root/appscale/AppServer_Java/appengine-java-repacked/bin/'
'dev_appserver.sh --port 20000')
flexmock(instance_manager).should_receive('_wait_for_app').\
and_return(True)
flexmock(os).should_receive('popen').\
and_return(flexmock(read=lambda: '0\n'))
flexmock(file_io).should_receive('write').and_return()
flexmock(subprocess).should_receive('call').and_return(0)
flexmock(utils).should_receive("setup_logrotate").and_return()
flexmock(os).should_receive('listdir').and_return([])
instance_manager._zk_client = flexmock()
instance_manager._zk_client.should_receive('ensure_path')
response = Future()
response.set_result(None)
instance_manager._monit_operator = flexmock(
reload=lambda x: response,
send_command_retry_process=lambda watch, cmd: response)
yield instance_manager._start_instance(version_manager, 20000)
def compose_message(to_user, from_user, create_time, args):
future = Future()
future.set_result(text_message(to_user, from_user, create_time, wx_dispatch(args)))
return future
def get_url_by_id(self, urlid, callback=None):
url = self.controller.get_url_by_urlid(urlid)
LOG.debug("Received url (%s) from id: %s", url, urlid)
future = Future()
future.set_result(url)
return future
def add_user_url(self, userid, url_dict):
future = Future()
result = self.controller.add_user_url(userid, url_dict)
future.set_result(self.format_result(result))
return future
def get_total_stats(self, callback=None):
future = Future()
stats = self.controller.get_total_stats()
future.set_result(self.format_stats(stats))
return future
