def test_stop_app_instance(self): version_key = 'test_default_v1' port = 20000 flexmock(misc).should_receive('is_app_name_valid').and_return(False) instance_manager = InstanceManager(None, None, None, None, None, None, None, None, None) flexmock(misc).should_receive('is_app_name_valid').and_return(True) response = Future() response.set_result(None) instance_manager._routing_client = flexmock( unregister_instance=lambda instance: response) flexmock(ServiceOperator).should_receive('stop_async').\ with_args('appscale-instance-run@test_default_v1-20000').\ and_return(response) response = Future() response.set_result(None) flexmock(instance_manager).should_receive('_clean_old_sources').\ and_return(response) instance_manager._service_operator = flexmock( stop_async=lambda service: response) yield instance_manager._stop_app_instance( instance.Instance('_'.join([version_key, 'revid']), port))
def test_start_app_failed_copy_java(self): version_details = { 'runtime': 'java', 'revision': 1, 'deployment': { 'zip': { 'sourceUrl': 'source.tar.gz' } }, 'appscaleExtensions': { 'httpPort': '8080' } } version_manager = flexmock(version_details=version_details, project_id='test', revision_key='test_default_v1_1', version_key='test_default_v1') instance_manager = InstanceManager(None, None, None, None, None, None, None, None, None) instance_manager._login_server = '192.168.33.10' instance_manager._projects_manager = { 'test': { 'default': { 'v1': version_manager } } } instance_manager._deployment_config = flexmock( get_config=lambda x: {'default_max_appserver_memory': 400}) source_manager = flexmock() response = Future() response.set_result(None) source_manager.should_receive('ensure_source').\ with_args('test_default_v1_1', 'source.tar.gz', 'java').\ and_return(response) instance_manager._source_manager = source_manager flexmock(instance).should_receive('find_web_inf'). \ and_return('/path/to/dir/WEB-INF') response = Future() response.set_result((19999, [])) flexmock(instance_manager).should_receive('_ensure_api_server'). \ and_return(response) flexmock(file_io).should_receive('write').and_raise(IOError) with self.assertRaises(IOError): yield instance_manager._start_instance(version_manager, 20000)
def test_stop_app_instance(self): version_key = 'test_default_v1' port = 20000 flexmock(misc).should_receive('is_app_name_valid').and_return(False) with self.assertRaises(BadConfigurationException): yield app_manager_server.stop_app_instance(version_key, port) flexmock(misc).should_receive('is_app_name_valid').and_return(True) flexmock(app_manager_server).should_receive('unregister_instance') flexmock(MonitOperator).should_receive('send_command_sync').\ with_args('app___test_default_v1-20000', 'unmonitor').\ and_raise(HTTPError) unmonitor_future = Future() unmonitor_future.set_exception(HTTPError(500)) flexmock(app_manager_server).should_receive('unmonitor_and_terminate').\ and_return(unmonitor_future) entries_response = Future() entries_response.set_result(['app___test_default_v1_revid-20000']) flexmock(MonitOperator).should_receive('get_entries').\ and_return(entries_response) with self.assertRaises(HTTPError): yield app_manager_server.stop_app_instance(version_key, port) flexmock(MonitOperator).should_receive('send_command_sync').\ with_args('app___test_default_v1-20000', 'unmonitor') flexmock(os).should_receive('remove') flexmock(monit_interface).should_receive('safe_monit_run') response = Future() response.set_result(None) flexmock(MonitOperator).should_receive('reload').\ and_return(response) response = Future() response.set_result(None) flexmock(app_manager_server).should_receive('clean_old_sources').\ and_return(response) response = Future() response.set_result(None) flexmock(app_manager_server).should_receive('unmonitor_and_terminate').\ and_return(response) yield app_manager_server.stop_app_instance(version_key, port)
async def get(self, url, headers=None, *args, **kwargs): request = HTTPRequest(url=url, use_gzip=True, headers=headers) existing_futures = self.rc_cache.get(url, None) if existing_futures is not None: future = Future() existing_futures.append(future) result = await future return result new_futures = [] self.rc_cache[url] = new_futures try: response = await self.http_client.fetch(request) except HTTPError as e: e = APIError(e.code, e.message) for future in new_futures: future.set_exception(e) del self.rc_cache[url] raise e else: body = response.body for future in new_futures: future.set_result(body) del self.rc_cache[url] return body
def send_message(self, message, async_reply_code=None): """Sends a message. If async is not None, this returns a Future that can be yielded. The Future will resolve when the supplied reply code is next received. The Future will error-out if no reply is received in the delay (seconds) given in the constructor. """ try: logger.debug("Sending \"{}\" over SPC.".format(message)) data = message + "\x0a" self._stream.write(data.encode("ascii")) except Exception as e: logger.exception("Error sending \"{}\" over SPC.".format(message)) raise ConnectionError( "Error sending \"{}\" over SPC.".format(message)) # async tracking? if async_reply_code: queue = self._reply_q[async_reply_code] fut = Future() handle = self._io_loop.call_later(delay=self._async_reply_delay, callback=self._async_timeout, code=async_reply_code) queue.append({"future": fut, "handle": handle}) return fut return None
def setUp(self): self.request_mock = MagicMock(name='Request Mock') self.request_mock.only.return_value = self.request_mock self.request_mock.search_text.return_value = self.request_mock self.request_mock.order_by.return_value = self.request_mock self.request_mock.id = 'id' self.request_mock.instance_name = 'default' self.request_mock.__getitem__.return_value = self.request_mock self.request_mock.__len__.return_value = 1 mongo_patcher = patch( 'brew_view.controllers.request_list_api.Request.objects') self.addCleanup(mongo_patcher.stop) self.mongo_mock = mongo_patcher.start() self.mongo_mock.count.return_value = 1 serialize_patcher = patch( 'brew_view.controllers.request_list_api.BeerGardenSchemaParser.serialize_request' ) self.addCleanup(serialize_patcher.stop) self.serialize_mock = serialize_patcher.start() self.serialize_mock.return_value = 'serialized_request' self.client_mock = Mock(name='client_mock') self.fake_context = MagicMock( __enter__=Mock(return_value=self.client_mock), __exit__=Mock(return_value=False)) self.future_mock = Future() super(RequestListAPITest, self).setUp()
def test_start_app_failed_copy_java(self): configuration = { 'app_port': 2000, 'login_server': 'public1', 'service_id': 'default', 'version_id': 'v1', 'env_vars': {} } version_details = {'runtime': 'java', 'revision': 1, 'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}} version_manager = flexmock(version_details=version_details) app_manager_server.projects_manager = { 'test': {'default': {'v1': version_manager}}} app_manager_server.deployment_config = flexmock( get_config=lambda x: {'default_max_appserver_memory': 400}) source_manager = flexmock() response = Future() response.set_result(None) source_manager.should_receive('ensure_source').\ with_args('test_default_v1_1', 'source.tar.gz', 'java').\ and_return(response) app_manager_server.source_manager = source_manager flexmock(app_manager_server).should_receive('find_web_inf').\ and_return('/path/to/dir/WEB-INF') flexmock(monit_app_configuration).should_receive('create_config_file').\ and_raise(IOError) with self.assertRaises(IOError): yield app_manager_server.start_app('test_default_v1', configuration)
def setUp(self): self.request_mock = MagicMock(name="Request Mock") self.request_mock.only.return_value = self.request_mock self.request_mock.search_text.return_value = self.request_mock self.request_mock.order_by.return_value = self.request_mock self.request_mock.id = "id" self.request_mock.instance_name = "default" self.request_mock.__getitem__.return_value = self.request_mock self.request_mock.__len__.return_value = 1 mongo_patcher = patch( "beer_garden.api.http.handlers.v1.request.Request.objects") self.addCleanup(mongo_patcher.stop) self.mongo_mock = mongo_patcher.start() self.mongo_mock.count.return_value = 1 serialize_patcher = patch( "beer_garden.api.http.handlers.v1.request.MongoParser.serialize_request" ) self.addCleanup(serialize_patcher.stop) self.serialize_mock = serialize_patcher.start() self.serialize_mock.return_value = "serialized_request" self.client_mock = Mock(name="client_mock") self.fake_context = MagicMock( __enter__=Mock(return_value=self.client_mock), __exit__=Mock(return_value=False), ) self.future_mock = Future() super(RequestListAPITest, self).setUp()
def test_start_app_failed_copy_java(self): version_details = {'runtime': 'java', 'revision': 1, 'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}} version_manager = flexmock(version_details=version_details, project_id='test', revision_key='test_default_v1_1', version_key='test_default_v1') instance_manager = InstanceManager( None, None, None, None, None, None, None, None, None) instance_manager._projects_manager = { 'test': {'default': {'v1': version_manager}}} instance_manager._deployment_config = flexmock( get_config=lambda x: {'default_max_appserver_memory': 400}) source_manager = flexmock() response = Future() response.set_result(None) source_manager.should_receive('ensure_source').\ with_args('test_default_v1_1', 'source.tar.gz', 'java').\ and_return(response) instance_manager._source_manager = source_manager flexmock(utils).should_receive('find_web_inf').\ and_return('/path/to/dir/WEB-INF') flexmock(monit_app_configuration).should_receive('create_config_file').\ and_raise(IOError) with self.assertRaises(IOError): yield instance_manager._start_instance(version_manager, 20000)
def test_wait_for_app(self): port = 20000 ip = '127.0.0.1' testing.disable_logging() fake_opener = flexmock(open=lambda url, timeout: flexmock( code=200, headers=flexmock(headers=[]))) flexmock(urllib2).should_receive('build_opener').and_return( fake_opener) flexmock(appscale_info).should_receive('get_private_ip').and_return(ip) instance_manager = InstanceManager(None, None, None, None, None, None, None, None, None) instance_manager._private_ip = ip instance_started = yield instance_manager._wait_for_app(port) self.assertEqual(True, instance_started) current_time = time.time() flexmock(monotonic).should_receive('monotonic').and_return(current_time).\ and_return(current_time + START_APP_TIMEOUT + 1) response = Future() response.set_result(None) flexmock(gen).should_receive('sleep').and_return(response) fake_opener.should_receive('open').and_raise(IOError) instance_started = yield instance_manager._wait_for_app(port) self.assertEqual(False, instance_started)
def setUpModule(): patcher = patch.object(gen, 'sleep') patchers.append(patcher) sleep_response = Future() sleep_response.set_result(None) sleep_mock = patcher.start() sleep_mock.return_value = sleep_response
def __wait(self): log.debug("Waiting for events") while not (yield sleep(0.001)): try: state = self.__connection.poll() except QueryCanceledError: yield sleep(0.1) continue f = Future() def resolve(fileno, io_op): if f.running(): f.set_result(True) self.__io_loop.remove_handler(fileno) if state == psycopg2.extensions.POLL_OK: raise Return(True) elif state == psycopg2.extensions.POLL_READ: self.__io_loop.add_handler(self.__connection.fileno(), resolve, IOLoop.READ) yield f elif state == psycopg2.extensions.POLL_WRITE: self.__io_loop.add_handler(self.__connection.fileno(), resolve, IOLoop.WRITE) yield f
def _main(): f1 = Future() f2 = Future() with MyLocal() as ctx: ctx.data['foo'] = 1 io_loop.add_callback(_cb, f1) io_loop.spawn_callback(_spawned_cb, f2) with MyLocal() as ctx1: with MyLocal() as ctx2: assert ctx2.prev_ctx() == ctx1 return multi([f1, f2])
def release_stream(self, stream): release_future = Future() release_future.set_result(None) if stream.closed(): return release_future if not self._wait_streams: if self._closed: stream.close() else: try: del self._used_streams[id(stream)] stream.idle_time = time.time() self._streams.append(stream) except KeyError: logging.error("release unknown stream %s", stream) else: future = self._wait_streams.popleft() stream.used_time = time.time() future.set_result(stream) while self._wait_streams and self._streams: stream = self._streams.pop() self._used_streams[id(stream)] = stream stream.used_time = time.time() if not stream.closed(): future = self._wait_streams.popleft() future.set_result(stream) return release_future
def setUp(self): self.client_mock = MagicMock(name='client_mock') self.fake_context = MagicMock( __enter__=Mock(return_value=self.client_mock), __exit__=Mock(return_value=False)) self.future_mock = Future() super(AdminAPITest, self).setUp()
def test_default_sleep_manual_mock(self, sleep_mock): # We need to wrap the mock in a Future mock_future = Future() mock_future.set_result(None) sleep_mock.return_value = mock_future response = self.fetch('/timer?seconds=10') self.assertEqual(response.code, 200) self.assertEqual(response.body, b'Slept for 10 seconds') sleep_mock.assert_called_once_with(10)
def setUp(self): self.client_mock = Mock(name="client_mock") self.fake_context = MagicMock( __enter__=Mock(return_value=self.client_mock), __exit__=Mock(return_value=False), ) self.future_mock = Future() super(VersionHandlerTest, self).setUp()
def test_start_app_goodconfig_python(self): configuration = { 'app_port': 2000, 'login_server': 'public1', 'service_id': 'default', 'version_id': 'v1', 'env_vars': {} } version_details = { 'runtime': 'python27', 'revision': 1, 'deployment': { 'zip': { 'sourceUrl': 'source.tar.gz' } } } version_manager = flexmock(version_details=version_details) app_manager_server.projects_manager = { 'test': { 'default': { 'v1': version_manager } } } app_manager_server.deployment_config = flexmock( get_config=lambda x: {'default_max_appserver_memory': 400}) source_manager = flexmock() response = Future() response.set_result(None) source_manager.should_receive('ensure_source'). \ with_args('test_default_v1_1', 'source.tar.gz', 'python27'). \ and_return(response) app_manager_server.source_manager = source_manager flexmock(monit_app_configuration).should_receive('create_config_file').\ and_return('fakeconfig') flexmock(monit_interface).should_receive('start').\ and_return(True) flexmock(app_manager_server).should_receive('wait_on_app').\ and_return(True) flexmock(os).should_receive('popen').\ and_return(flexmock(read=lambda: '12345\n')) flexmock(file_io).should_receive('write').\ and_return() flexmock(threading).should_receive('Thread').\ and_return(flexmock(start=lambda: None)) flexmock(app_manager_server).should_receive( "setup_logrotate").and_return() app_manager_server.zk_client = flexmock() app_manager_server.zk_client.should_receive('ensure_path') yield app_manager_server.start_app('test_default_v1', configuration)
def sync_to_async(sync_f, *args): f = Future() # use closure here: def wrapper(): res = sync_f(*args) f.set_result(res) # keypoint !! threading.Thread( target=wrapper).start() #使用多线程去跑同步阻塞函数,跑完后把值设置到future里面即可. return f
def cursor(self, **kwargs): f = Future() self.__io_loop.add_callback( self.__queue.put, (functools.partial(AsyncCursor, self.__connection, self.__thread_pool, self.__wait, on_open=self.__on_cursor_open, on_close=self.__on_cursor_close, **kwargs), f)) return f
def test_start_app_goodconfig_java(self): configuration = { 'app_port': 20000, 'login_server': 'public1', 'service_id': 'default', 'version_id': 'v1', 'env_vars': {} } version_details = {'runtime': 'java', 'revision': 1, 'deployment': {'zip': {'sourceUrl': 'source.tar.gz'}}} version_manager = flexmock(version_details=version_details) app_manager_server.projects_manager = { 'test': {'default': {'v1': version_manager}}} app_manager_server.deployment_config = flexmock( get_config=lambda x: {'default_max_appserver_memory': 400}) source_manager = flexmock() response = Future() response.set_result(None) source_manager.should_receive('ensure_source').\ with_args('test_default_v1_1', 'source.tar.gz', 'java').\ and_return(response) app_manager_server.source_manager = source_manager start_cmd = ('/root/appscale/AppServer_Java/appengine-java-repacked/bin/' 'dev_appserver.sh --port 20000') flexmock(app_manager_server).should_receive('create_java_start_cmd').\ and_return(start_cmd) flexmock(monit_app_configuration).should_receive('create_config_file').\ and_return('fakeconfig') flexmock(monit_interface).should_receive('start').\ and_return(True) flexmock(app_manager_server).should_receive('create_java_app_env').\ and_return({}) flexmock(app_manager_server).should_receive('wait_on_app').\ and_return(True) flexmock(app_manager_server).should_receive('locate_dir').\ and_return('/path/to/dir/') flexmock(os).should_receive('popen').\ and_return(flexmock(read=lambda: '0\n')) flexmock(file_io).should_receive('write').and_return() flexmock(subprocess).should_receive('call').and_return(0) flexmock(threading).should_receive('Thread').\ and_return(flexmock(start=lambda: None)) flexmock(app_manager_server).should_receive("setup_logrotate").and_return() flexmock(os).should_receive('listdir').and_return([]) app_manager_server.zk_client = flexmock() app_manager_server.zk_client.should_receive('ensure_path') yield app_manager_server.start_app('test_default_v1', configuration)
def connect(self): future = Future() def on_connected(connection_future): if connection_future._exc_info is None: future.set_result(self) else: future.set_exc_info(connection_future.exc_info()) self._connection = Connection(defer_connect = True, *self._args, **self._kwargs) self._connection.set_close_callback(self.connection_close_callback) connection_future = async_call_method(self._connection.connect) IOLoop.current().add_future(connection_future, on_connected) return future
def close(self): self._close_future = Future() self._closed = True while self._wait_streams: future = self._wait_streams.popleft() future.set_exception(TStreamPoolClosedError()) while self._streams: stream = self._streams.popleft() self._used_streams[id(stream)] = stream stream.close() return self._close_future
def start_tensorflow(model_name, client=None, options=None, port=TF_PORT, **kwargs): """ Start Tensorflow on Dask Cluster This launches Tensorflow Servers alongside Dask workers in-process Examples -------- >>> client = Client('dask-scheduler-address:8786') >>> tf_spec, dask_spec = start_tensorflow(client) >>> tf_spec.as_dict() {'worker': ['192.168.1.100:2222', '192.168.1.101:2222']} Specify desired number of jobs types as keyword args >>> tf_config = tf.compat.v1.OptimizerOptions() >>> tf_config.GlobalJitLevel = tf_config.OFF >>> tf_config.do_function_inlining = True >> tf_config.opt_level >>> tf_config.gpu_options.force_gpu_compatible = True >>> tf_spec, dask_spec = start_tensorflow(client, tf_config=tf_config, chief=1, master=1, ps=2, worker=30) >>> tf_spec.as_dict() { 'chief': ['192.168.1.1:2222'], 'master': ['192.168.1.1:2223'], 'ps': ['192.168.1.104:2222', '192.168.1.105:2222'], 'worker': ['192.168.1.100:2222', '192.168.1.101:2222', '192.168.1.102:2222', '192.168.1.103:2222'] } """ client = client if client is not None else global_cluster( asynchronous=True) global_future = Future() tensorflow_scheduler_wrap = partial(tensorflow_scheduler, global_future, model_name, client=client, tf_port=port, **kwargs) if client.asynchronous: global_future.add_done_callback(lambda fu: client.loop.stop()) client.loop.add_callback(tensorflow_scheduler_wrap) client.loop.start() result = global_future.result() else: result = client.sync(client.loop, tensorflow_scheduler_wrap) return result
def connect(self): f = self._waiting_connect = Future() err = self.socket.connect_ex((self.host, self.port)) if err != 0 and err not in _ERRNO_INPROGRESS\ and err not in _ERRNO_WOULDBLOCK: error = _set_stack(socket.error(err, os.strerror(err))) f.set_exception(error) self.close() return f self._add_io_state(self.io_loop.WRITE) return f
def fwrap(gf, ioloop=None): ''' Wraps a GRPC result in a future that can be yielded by tornado Usage:: @coroutine def my_fn(param): result = yield fwrap(stub.function_name.future(param, timeout)) ''' f = Future() if ioloop is None: ioloop = IOLoop.current() gf.add_done_callback(lambda _: ioloop.add_callback(_fwrap, f, gf)) return f
def setUp(self): self.instance_mock = Mock(status='RUNNING') type(self.instance_mock).name = PropertyMock(return_value='default') self.system_mock = Mock(id='1234', display_name='display', version='1.0.0', instances=[self.instance_mock]) type(self.system_mock).name = PropertyMock(return_value='system_name') self.queue_name = "%s[%s]-%s" % (self.system_mock.name, self.instance_mock.name, self.system_mock.version) self.client_mock = Mock(name='client_mock') self.fake_context = MagicMock(__enter__=Mock(return_value=self.client_mock), __exit__=Mock(return_value=False)) self.future_mock = Future() super(QueueListAPITest, self).setUp()
def call(self, func, **kwargs): assert isinstance(func, str) serial = self._get_serial() self.io_loop.add_callback( self._call_coro, json.dumps({ 'call': func, 'serial': serial, 'arguments': kwargs })) f = Future() self._futures[serial] = f return f
def setUp(self): self.system_mock = Mock(version='1.0', commands=[]) mongo_patcher = patch( 'mongoengine.queryset.manager.QuerySetManager.__get__') self.addCleanup(mongo_patcher.stop) self.get_mock = mongo_patcher.start() self.get_mock.return_value.get.return_value = self.system_mock self.client_mock = Mock(name='client_mock') self.fake_context = MagicMock( __enter__=Mock(return_value=self.client_mock), __exit__=Mock(return_value=False)) self.future_mock = Future() super(SystemAPITest, self).setUp()
def test_stop_app_instance(self): version_key = 'test_default_v1' port = 20000 flexmock(misc).should_receive('is_app_name_valid').and_return(False) instance_manager = InstanceManager( None, None, None, None, None, None, None, None, None) flexmock(misc).should_receive('is_app_name_valid').and_return(True) response = Future() response.set_result(None) instance_manager._routing_client = flexmock( unregister_instance=lambda instance: response) flexmock(MonitOperator).should_receive('send_command_sync').\ with_args('app___test_default_v1-20000', 'unmonitor').\ and_raise(HTTPError) unmonitor_future = Future() unmonitor_future.set_exception(HTTPError(500)) flexmock(instance_manager).should_receive('_unmonitor_and_terminate').\ and_return(unmonitor_future) entries_response = Future() entries_response.set_result(['app___test_default_v1_revid-20000']) instance_manager._monit_operator = flexmock( get_entries=lambda: entries_response, reload=lambda x: response) with self.assertRaises(HTTPError): yield instance_manager._stop_app_instance( instance.Instance('_'.join([version_key, 'revid']), port)) flexmock(MonitOperator).should_receive('send_command_sync').\ with_args('app___test_default_v1-20000', 'unmonitor') flexmock(os).should_receive('remove') flexmock(monit_interface).should_receive('safe_monit_run') response = Future() response.set_result(None) flexmock(MonitOperator).should_receive('reload').\ and_return(response) response = Future() response.set_result(None) flexmock(instance_manager).should_receive('_clean_old_sources').\ and_return(response) response = Future() response.set_result(None) flexmock(instance_manager).should_receive('_unmonitor_and_terminate').\ and_return(response) yield instance_manager._stop_app_instance( instance.Instance('_'.join([version_key, 'revid']), port))