def install_plugins(plugins, **_): installer = PluginInstaller(logger=ctx.logger) for plugin in plugins: ctx.logger.info('Installing plugin: {0}'.format(plugin['name'])) try: installer.install(plugin=plugin, deployment_id=ctx.deployment.id, blueprint_id=ctx.blueprint.id) except exceptions.PluginInstallationError as e: # preserve traceback tpe, value, tb = sys.exc_info() raise NonRecoverableError, NonRecoverableError(str(e)), tb
def _install_plugins(plugins): installer = PluginInstaller(logger=ctx.logger) for plugin in plugins: ctx.logger.info('Installing plugin: {0}'.format(plugin['name'])) try: package_name = installer.install(plugin, blueprint_id=ctx.blueprint.id) except exceptions.PluginInstallationError as e: # preserve traceback tpe, value, tb = sys.exc_info() raise NonRecoverableError, NonRecoverableError(str(e)), tb daemon = _load_daemon(logger=ctx.logger) daemon.register(package_name) _save_daemon(daemon)
def install(source, args): """ Install a cloudify plugin into the current virtualenv. This will also register the plugin to all daemons created from this virtualenv. """ from cloudify_agent.shell.main import get_logger click.echo('Installing plugin from {0}'.format(source)) installer = PluginInstaller(logger=get_logger()) name = installer.install(source, args) daemons = DaemonFactory().load_all(logger=get_logger()) for daemon in daemons: click.echo('Registering plugin {0} to {1}'.format(name, daemon.name)) if daemon.virtualenv == VIRTUALENV: daemon.register(name) _save_daemon(daemon) click.echo('Successfully installed plugin: {0}'.format(name))
def install(source, args): """ Install a cloudify plugin into the current virtualenv. This will also register the plugin to all daemons created from this virtualenv. """ from cloudify_agent.shell.main import get_logger click.echo('Installing plugin from {0}'.format(source)) installer = PluginInstaller(logger=get_logger()) name = installer.install(source, args) daemons = DaemonFactory().load_all(logger=get_logger()) for daemon in daemons: click.echo('Registering plugin {0} to {1}' .format(name, daemon.name)) if daemon.virtualenv == VIRTUALENV: daemon.register(name) _save_daemon(daemon) click.echo('Successfully installed plugin: {0}'.format(name))
class BaseDaemonProcessManagementTest(BaseDaemonLiveTestCase): def setUp(self): super(BaseDaemonProcessManagementTest, self).setUp() self.installer = PluginInstaller(logger=self.logger) def tearDown(self): super(BaseDaemonProcessManagementTest, self).tearDown() self.installer.uninstall('mock-plugin') self.installer.uninstall('mock-plugin-error') @property def daemon_cls(self): raise NotImplementedError('Must be implemented by sub-class') def create_daemon(self, **attributes): name = utils.internal.generate_agent_name() params = { 'manager_ip': '127.0.0.1', 'user': self.username, 'workdir': self.temp_folder, 'logger': self.logger, 'name': name, 'queue': '{0}-queue'.format(name) } params.update(attributes) daemon = self.daemon_cls(**params) self.daemons.append(daemon) return daemon def test_create(self): daemon = self.create_daemon() daemon.create() def test_configure(self): raise NotImplementedError('Must be implemented by sub-class') def test_start(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assert_daemon_alive(daemon.name) self.assert_registered_tasks(daemon.name) def test_start_delete_amqp_queue(self): daemon = self.create_daemon() daemon.create() daemon.configure() # this creates the queue daemon.start() daemon.stop() daemon.start(delete_amqp_queue=True) def test_start_with_error(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install( os.path.join(resources.get_resource('plugins'), 'mock-plugin-error')) daemon.register('mock-plugin-error') try: daemon.start() self.fail('Expected start operation to fail ' 'due to bad import') except exceptions.DaemonError as e: self.assertIn('cannot import name non_existent', str(e)) def test_start_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() try: daemon.start(timeout=-1) except exceptions.DaemonStartupTimeout as e: self.assertTrue('failed to start in -1 seconds' in str(e)) def test_status(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.assertFalse(daemon.status()) daemon.start() self.assertTrue(daemon.status()) def test_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.stop() self.assert_daemon_dead(daemon.name) def test_stop_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() try: daemon.stop(timeout=-1) except exceptions.DaemonShutdownTimeout as e: self.assertTrue('failed to stop in -1 seconds' in str(e)) def test_register(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install( os.path.join(resources.get_resource('plugins'), 'mock-plugin')) daemon.register('mock-plugin') daemon.start() self.assert_registered_tasks( daemon.name, additional_tasks=set(['mock_plugin.tasks.run', 'mock_plugin.tasks.get_env_variable']) ) def test_unregister(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install( os.path.join(resources.get_resource('plugins'), 'mock-plugin')) daemon.register('mock-plugin') daemon.start() self.assert_registered_tasks( daemon.name, additional_tasks=set(['mock_plugin.tasks.run', 'mock_plugin.tasks.get_env_variable']) ) daemon.unregister('mock-plugin') daemon.restart() self.assert_registered_tasks(daemon.name) def test_restart(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install( os.path.join(resources.get_resource('plugins'), 'mock-plugin')) daemon.start() daemon.register('mock-plugin') daemon.restart() self.assert_registered_tasks( daemon.name, additional_tasks=set(['mock_plugin.tasks.run', 'mock_plugin.tasks.get_env_variable']) ) def test_two_daemons(self): daemon1 = self.create_daemon() daemon1.create() daemon1.configure() daemon1.start() self.assert_daemon_alive(daemon1.name) self.assert_registered_tasks(daemon1.name) daemon2 = self.create_daemon() daemon2.create() daemon2.configure() daemon2.start() self.assert_daemon_alive(daemon2.name) self.assert_registered_tasks(daemon2.name) def test_conf_env_variables(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install( os.path.join(resources.get_resource('plugins'), 'mock-plugin')) daemon.register('mock-plugin') daemon.start() expected = { constants.MANAGER_IP_KEY: str(daemon.manager_ip), constants.MANAGER_REST_PORT_KEY: str(daemon.manager_port), constants.MANAGER_FILE_SERVER_URL_KEY: 'http://{0}:53229'.format(daemon.manager_ip), constants.MANAGER_FILE_SERVER_BLUEPRINTS_ROOT_URL_KEY: 'http://{0}:53229/blueprints'.format(daemon.manager_ip), constants.CELERY_BROKER_URL_KEY: daemon.broker_url, utils.internal.CLOUDIFY_DAEMON_STORAGE_DIRECTORY_KEY: utils.internal.get_storage_directory(), utils.internal.CLOUDIFY_DAEMON_NAME_KEY: daemon.name, utils.internal.CLOUDIFY_DAEMON_USER_KEY: daemon.user } def _check_env_var(var, expected_value): _value = self.celery.send_task( name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, args=[var]).get(timeout=5) self.assertEqual(_value, expected_value) for key, value in expected.iteritems(): _check_env_var(key, value) def test_extra_env_path(self): daemon = self.create_daemon() daemon.extra_env_path = utils.env_to_file( {'TEST_ENV_KEY': 'TEST_ENV_VALUE'}, posix=os.name == 'posix' ) daemon.create() daemon.configure() self.installer.install( os.path.join(resources.get_resource('plugins'), 'mock-plugin')) daemon.register('mock-plugin') daemon.start() # check the env file was properly sourced by querying the env # variable from the daemon process. this is done by a task value = self.celery.send_task( name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, args=['TEST_ENV_KEY']).get(timeout=10) self.assertEqual(value, 'TEST_ENV_VALUE') def test_delete(self): raise NotImplementedError('Must be implemented by sub-class') def test_delete_before_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assertRaises(exceptions.DaemonStillRunningException, daemon.delete) def test_delete_before_stop_with_force(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.delete(force=True) self.assert_daemon_dead(daemon.name)
class BaseDaemonProcessManagementTest(BaseDaemonLiveTestCase): def setUp(self): super(BaseDaemonProcessManagementTest, self).setUp() self.installer = PluginInstaller(logger=self.logger) def tearDown(self): super(BaseDaemonProcessManagementTest, self).tearDown() self.installer.uninstall(plugin=self.plugin_struct()) self.installer.uninstall(plugin=self.plugin_struct(), deployment_id=DEPLOYMENT_ID) @property def daemon_cls(self): raise NotImplementedError('Must be implemented by sub-class') def create_daemon(self, **attributes): name = utils.internal.generate_agent_name() params = { 'rest_host': '127.0.0.1', 'broker_ip': '127.0.0.1', 'user': self.username, 'workdir': self.temp_folder, 'logger': self.logger, 'name': name, 'queue': '{0}-queue'.format(name), 'local_rest_cert_file': self._rest_cert_path, 'broker_ssl_enabled': False } params.update(attributes) daemon = self.daemon_cls(**params) self.daemons.append(daemon) return daemon def test_create(self): daemon = self.create_daemon() daemon.create() def test_configure(self): raise NotImplementedError('Must be implemented by sub-class') def test_start(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assert_daemon_alive(daemon.name) self.assert_registered_tasks(daemon.name) def test_start_delete_amqp_queue(self): daemon = self.create_daemon() daemon.create() daemon.configure() # this creates the queue daemon.start() daemon.stop() daemon.start(delete_amqp_queue=True) @patch_get_source def test_start_with_error(self): log_file = 'H:\\WATT\\lo' if os.name == 'nt' else '/root/no_permission' daemon = self.create_daemon(log_file=log_file) daemon.create() daemon.configure() try: daemon.start() self.fail('Expected start operation to fail due to bad logfile') except exceptions.DaemonError as e: if os.name == 'nt': expected_error = "No such file or directory: '" else: expected_error = "Permission denied: '{0}" self.assertIn(expected_error.format(log_file), str(e)) def test_start_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() try: daemon.start(timeout=-1) except exceptions.DaemonStartupTimeout as e: self.assertTrue('failed to start in -1 seconds' in str(e)) def test_status(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.assertFalse(daemon.status()) daemon.start() self.assertTrue(daemon.status()) def test_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.stop() self.assert_daemon_dead(daemon.name) def test_stop_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() try: daemon.stop(timeout=-1) except exceptions.DaemonShutdownTimeout as e: self.assertTrue('failed to stop in -1 seconds' in str(e)) @patch_get_source def test_restart(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() daemon.restart() self.assert_daemon_alive(daemon.name) self.assert_registered_tasks(daemon.name) def test_two_daemons(self): daemon1 = self.create_daemon() daemon1.create() daemon1.configure() daemon1.start() self.assert_daemon_alive(daemon1.name) self.assert_registered_tasks(daemon1.name) daemon2 = self.create_daemon() daemon2.create() daemon2.configure() daemon2.start() self.assert_daemon_alive(daemon2.name) self.assert_registered_tasks(daemon2.name) @patch_get_source def test_conf_env_variables(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() expected = { constants.REST_HOST_KEY: str(daemon.rest_host), constants.REST_PORT_KEY: str(daemon.rest_port), constants.MANAGER_FILE_SERVER_URL_KEY: 'https://{0}:{1}/resources'.format(daemon.rest_host, daemon.rest_port), constants.CELERY_WORK_DIR_KEY: daemon.workdir, utils.internal.CLOUDIFY_DAEMON_STORAGE_DIRECTORY_KEY: utils.internal.get_storage_directory(), utils.internal.CLOUDIFY_DAEMON_NAME_KEY: daemon.name, utils.internal.CLOUDIFY_DAEMON_USER_KEY: daemon.user } def _get_env_var(var): return self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': var}) def _check_env_var(var, expected_value): _value = _get_env_var(var) self.assertEqual(_value, expected_value) for key, value in expected.iteritems(): _check_env_var(key, value) @patch_get_source def test_extra_env(self): daemon = self.create_daemon() daemon.extra_env_path = utils.env_to_file( {'TEST_ENV_KEY': 'TEST_ENV_VALUE'}, posix=os.name == 'posix') daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check the env file was properly sourced by querying the env # variable from the daemon process. this is done by a task value = self.send_task(task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY'}) self.assertEqual(value, 'TEST_ENV_VALUE') @patch_get_source def test_execution_env(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check that cloudify.dispatch.dispatch 'execution_env' processing # works. # not the most ideal place for this test. but on the other hand # all the boilerplate is already here, so this is too tempting. value = self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY2'}, execution_env={'TEST_ENV_KEY2': 'TEST_ENV_VALUE2'}) self.assertEqual(value, 'TEST_ENV_VALUE2') def test_delete(self): raise NotImplementedError('Must be implemented by sub-class') def test_delete_before_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assertRaises(exceptions.DaemonStillRunningException, daemon.delete) def test_delete_before_stop_with_force(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.delete(force=True) self.assert_daemon_dead(daemon.name) @patch_get_source def test_logging(self): message = 'THIS IS THE TEST MESSAGE LOG CONTENT' daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) self.installer.install(self.plugin_struct(), deployment_id=DEPLOYMENT_ID) daemon.start() def log_and_assert(_message, _deployment_id=None): self.send_task(task_name='mock_plugin.tasks.do_logging', queue=daemon.queue, kwargs={'message': _message}, deployment_id=_deployment_id) name = _deployment_id if _deployment_id else '__system__' logdir = os.path.join(daemon.workdir, 'logs') logfile = os.path.join(logdir, '{0}.log'.format(name)) try: with open(logfile) as f: self.assertIn(_message, f.read()) except IOError: self.logger.warning('{0} content: {1}'.format( logdir, os.listdir(logdir))) raise # Test __system__ logs log_and_assert(message) # Test deployment logs log_and_assert(message, DEPLOYMENT_ID) @staticmethod def plugin_struct(plugin_name='mock-plugin'): return { 'source': os.path.join(resources.get_resource('plugins'), plugin_name), 'name': PLUGIN_NAME } def send_task(self, task_name, queue, deployment_id=None, args=None, kwargs=None, timeout=10, execution_env=None): cloudify_context = test_utils.op_context(task_name, task_target=queue, plugin_name=PLUGIN_NAME, execution_env=execution_env, deployment_id=deployment_id) kwargs = kwargs or {} kwargs['__cloudify_context'] = cloudify_context return self.celery.send_task(name='cloudify.dispatch.dispatch', queue=queue, args=args, kwargs=kwargs).get(timeout=timeout)
class BaseDaemonProcessManagementTest(BaseDaemonLiveTestCase): def setUp(self): super(BaseDaemonProcessManagementTest, self).setUp() self.installer = PluginInstaller(logger=self.logger) def tearDown(self): super(BaseDaemonProcessManagementTest, self).tearDown() self.installer.uninstall(plugin=self.plugin_struct()) self.installer.uninstall(plugin=self.plugin_struct(), deployment_id=DEPLOYMENT_ID) @property def daemon_cls(self): raise NotImplementedError('Must be implemented by sub-class') def create_daemon(self, **attributes): name = utils.internal.generate_agent_name() params = { 'rest_host': '127.0.0.1', 'broker_ip': '127.0.0.1', 'file_server_host': '127.0.0.1', 'user': self.username, 'workdir': self.temp_folder, 'logger': self.logger, 'name': name, 'queue': '{0}-queue'.format(name) } params.update(attributes) daemon = self.daemon_cls(**params) self.daemons.append(daemon) return daemon def test_create(self): daemon = self.create_daemon() daemon.create() def test_configure(self): raise NotImplementedError('Must be implemented by sub-class') def test_start(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assert_daemon_alive(daemon.name) self.assert_registered_tasks(daemon.name) def test_start_delete_amqp_queue(self): daemon = self.create_daemon() daemon.create() daemon.configure() # this creates the queue daemon.start() daemon.stop() daemon.start(delete_amqp_queue=True) @patch_get_source def test_start_with_error(self): log_file = 'H:\\WATT\\lo' if os.name == 'nt' else '/root/no_permission' daemon = self.create_daemon(log_file=log_file) daemon.create() daemon.configure() try: daemon.start() self.fail('Expected start operation to fail due to bad logfile') except exceptions.DaemonError as e: if os.name == 'nt': expected_error = "No such file or directory: '" else: expected_error = "Permission denied: '{0}" self.assertIn(expected_error.format(log_file), str(e)) def test_start_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() try: daemon.start(timeout=-1) except exceptions.DaemonStartupTimeout as e: self.assertTrue('failed to start in -1 seconds' in str(e)) def test_status(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.assertFalse(daemon.status()) daemon.start() self.assertTrue(daemon.status()) def test_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.stop() self.assert_daemon_dead(daemon.name) def test_stop_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() try: daemon.stop(timeout=-1) except exceptions.DaemonShutdownTimeout as e: self.assertTrue('failed to stop in -1 seconds' in str(e)) @patch_get_source def test_restart(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() daemon.restart() self.assert_daemon_alive(daemon.name) self.assert_registered_tasks(daemon.name) def test_two_daemons(self): daemon1 = self.create_daemon() daemon1.create() daemon1.configure() daemon1.start() self.assert_daemon_alive(daemon1.name) self.assert_registered_tasks(daemon1.name) daemon2 = self.create_daemon() daemon2.create() daemon2.configure() daemon2.start() self.assert_daemon_alive(daemon2.name) self.assert_registered_tasks(daemon2.name) @patch_get_source def test_conf_env_variables(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() expected = { constants.REST_HOST_KEY: str(daemon.rest_host), constants.REST_PORT_KEY: str(daemon.rest_port), constants.FILE_SERVER_HOST_KEY: str(daemon.file_server_host), constants.FILE_SERVER_PORT_KEY: str(daemon.file_server_port), constants.FILE_SERVER_PROTOCOL_KEY: str(daemon.file_server_protocol), constants.MANAGER_FILE_SERVER_URL_KEY: '{0}://{1}:{2}'.format(daemon.file_server_protocol, daemon.file_server_host, daemon.file_server_port), constants.MANAGER_FILE_SERVER_BLUEPRINTS_ROOT_URL_KEY: '{0}://{1}:{2}/blueprints'.format(daemon.file_server_protocol, daemon.file_server_host, daemon.file_server_port), constants.MANAGER_FILE_SERVER_DEPLOYMENTS_ROOT_URL_KEY: '{0}://{1}:{2}/deployments'.format(daemon.file_server_protocol, daemon.file_server_host, daemon.file_server_port), constants.CELERY_WORK_DIR_KEY: daemon.workdir, utils.internal.CLOUDIFY_DAEMON_STORAGE_DIRECTORY_KEY: utils.internal.get_storage_directory(), utils.internal.CLOUDIFY_DAEMON_NAME_KEY: daemon.name, utils.internal.CLOUDIFY_DAEMON_USER_KEY: daemon.user } def _get_env_var(var): return self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': var}) def _check_env_var(var, expected_value): _value = _get_env_var(var) self.assertEqual(_value, expected_value) for key, value in expected.iteritems(): _check_env_var(key, value) @patch_get_source def test_extra_env(self): daemon = self.create_daemon() daemon.extra_env_path = utils.env_to_file( {'TEST_ENV_KEY': 'TEST_ENV_VALUE'}, posix=os.name == 'posix' ) daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check the env file was properly sourced by querying the env # variable from the daemon process. this is done by a task value = self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY'}) self.assertEqual(value, 'TEST_ENV_VALUE') @patch_get_source def test_execution_env(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check that cloudify.dispatch.dispatch 'execution_env' processing # works. # not the most ideal place for this test. but on the other hand # all the boilerplate is already here, so this is too tempting. value = self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY2'}, execution_env={'TEST_ENV_KEY2': 'TEST_ENV_VALUE2'}) self.assertEqual(value, 'TEST_ENV_VALUE2') def test_delete(self): raise NotImplementedError('Must be implemented by sub-class') def test_delete_before_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assertRaises(exceptions.DaemonStillRunningException, daemon.delete) def test_delete_before_stop_with_force(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.delete(force=True) self.assert_daemon_dead(daemon.name) @patch_get_source def test_logging(self): message = 'THIS IS THE TEST MESSAGE LOG CONTENT' daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) self.installer.install(self.plugin_struct(), deployment_id=DEPLOYMENT_ID) daemon.start() def log_and_assert(_message, _deployment_id=None): self.send_task( task_name='mock_plugin.tasks.do_logging', queue=daemon.queue, kwargs={'message': _message}, deployment_id=_deployment_id) name = _deployment_id if _deployment_id else '__system__' logdir = os.path.join(daemon.workdir, 'logs') logfile = os.path.join(logdir, '{0}.log'.format(name)) try: with open(logfile) as f: self.assertIn(_message, f.read()) except IOError: self.logger.warning('{0} content: {1}' .format(logdir, os.listdir(logdir))) raise # Test __system__ logs log_and_assert(message) # Test deployment logs log_and_assert(message, DEPLOYMENT_ID) @staticmethod def plugin_struct(plugin_name='mock-plugin'): return { 'source': os.path.join(resources.get_resource('plugins'), plugin_name), 'name': PLUGIN_NAME } def send_task(self, task_name, queue, deployment_id=None, args=None, kwargs=None, timeout=10, execution_env=None): cloudify_context = test_utils.op_context(task_name, task_target=queue, plugin_name=PLUGIN_NAME, execution_env=execution_env, deployment_id=deployment_id) kwargs = kwargs or {} kwargs['__cloudify_context'] = cloudify_context return self.celery.send_task( name='cloudify.dispatch.dispatch', queue=queue, args=args, kwargs=kwargs).get(timeout=timeout)
class BaseDaemonProcessManagementTest(BaseDaemonLiveTestCase): def setUp(self): super(BaseDaemonProcessManagementTest, self).setUp() self.installer = PluginInstaller(logger=self.logger) def tearDown(self): super(BaseDaemonProcessManagementTest, self).tearDown() self.installer.uninstall_source(plugin=self.plugin_struct()) self.installer.uninstall_source(plugin=self.plugin_struct(), deployment_id=DEPLOYMENT_ID) @property def daemon_cls(self): raise NotImplementedError('Must be implemented by sub-class') def create_daemon(self, **attributes): name = utils.internal.generate_agent_name() params = { 'rest_host': '127.0.0.1', 'broker_ip': '127.0.0.1', 'user': self.username, 'workdir': self.temp_folder, 'logger': self.logger, 'name': name, 'queue': '{0}-queue'.format(name), 'local_rest_cert_file': self._rest_cert_path, 'broker_ssl_enabled': False, # No SSL on the CI machines } params.update(attributes) factory = DaemonFactory() daemon = self.daemon_cls(**params) factory.save(daemon) self.addCleanup(factory.delete, daemon.name) self.daemons.append(daemon) return daemon def test_create(self): daemon = self.create_daemon() daemon.create() def test_create_overwrite(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.wait_for_daemon_alive(daemon.queue) daemon.create() daemon.configure() daemon.start() self.wait_for_daemon_alive(daemon.queue) daemon.stop() self.wait_for_daemon_dead(daemon.queue) def test_configure(self): raise NotImplementedError('Must be implemented by sub-class') def test_start(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() def test_start_delete_amqp_queue(self): daemon = self.create_daemon() daemon.create() daemon.configure() # this creates the queue daemon.start() daemon.stop() daemon.start(delete_amqp_queue=True) @patch_get_source def test_start_with_error(self): if os.name == 'nt': log_dir = 'H:\\WATT_NONEXISTENT_DIR\\lo' else: log_dir = '/root/no_permission' daemon = self.create_daemon(log_dir=log_dir) daemon.create() daemon.configure() try: daemon.start(timeout=5) self.fail('Expected start operation to fail due to bad log_dir') except exceptions.DaemonError as e: if os.name == 'nt': # windows messages vary, and will be escaped, so let's just # check that the dir name is there expected_error = 'WATT_NONEXISTENT_DIR' else: expected_error = "Permission denied: '{0}" self.assertIn(expected_error.format(log_dir), str(e)) def test_start_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() try: daemon.start(timeout=-1) except exceptions.DaemonStartupTimeout as e: self.assertTrue('failed to start in -1 seconds' in str(e)) def test_status(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.assertFalse(daemon.status()) daemon.start() self.assertTrue(daemon.status()) def test_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.stop() self.wait_for_daemon_dead(daemon.queue) def test_stop_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() try: daemon.stop(timeout=-1) except exceptions.DaemonShutdownTimeout as e: self.assertTrue('failed to stop in -1 seconds' in str(e)) @patch_get_source def test_restart(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() daemon.restart() def test_two_daemons(self): daemon1 = self.create_daemon() daemon1.create() daemon1.configure() daemon1.start() self.assert_daemon_alive(daemon1.queue) daemon2 = self.create_daemon() daemon2.create() daemon2.configure() daemon2.start() self.assert_daemon_alive(daemon2.queue) @patch_get_source def test_conf_env_variables(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() expected = { constants.REST_HOST_KEY: str(daemon.rest_host), constants.REST_PORT_KEY: str(daemon.rest_port), constants.MANAGER_FILE_SERVER_URL_KEY: 'https://{0}:{1}/resources'.format( daemon.rest_host, daemon.rest_port ), constants.AGENT_WORK_DIR_KEY: daemon.workdir, } def _get_env_var(var): return self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': var}) def _check_env_var(var, expected_value): _value = _get_env_var(var) self.assertEqual(_value, expected_value) for key, value in expected.iteritems(): _check_env_var(key, value) @patch_get_source def test_extra_env(self): daemon = self.create_daemon() daemon.extra_env_path = utils.env_to_file( {'TEST_ENV_KEY': 'TEST_ENV_VALUE'}, posix=os.name == 'posix' ) daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check the env file was properly sourced by querying the env # variable from the daemon process. this is done by a task value = self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY'}) self.assertEqual(value, 'TEST_ENV_VALUE') @patch_get_source def test_execution_env(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check that cloudify.dispatch.dispatch 'execution_env' processing # works. # not the most ideal place for this test. but on the other hand # all the boilerplate is already here, so this is too tempting. value = self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY2'}, execution_env={'TEST_ENV_KEY2': 'TEST_ENV_VALUE2'}) self.assertEqual(value, 'TEST_ENV_VALUE2') def test_delete(self): raise NotImplementedError('Must be implemented by sub-class') def test_delete_before_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assertRaises(exceptions.DaemonStillRunningException, daemon.delete) def test_delete_before_stop_with_force(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.delete(force=True) self.wait_for_daemon_dead(daemon.queue) @patch_get_source def test_logging(self): message = 'THIS IS THE TEST MESSAGE LOG CONTENT' daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) self.installer.install(self.plugin_struct(), deployment_id=DEPLOYMENT_ID) daemon.start() def log_and_assert(_message, _deployment_id=None): self.send_task( task_name='mock_plugin.tasks.do_logging', queue=daemon.queue, kwargs={'message': _message}, deployment_id=_deployment_id) name = _deployment_id if _deployment_id else '__system__' logdir = os.path.join(daemon.workdir, 'logs') logfile = os.path.join(logdir, '{0}.log'.format(name)) try: with open(logfile) as f: self.assertIn(_message, f.read()) except IOError: self.logger.warning('{0} content: {1}' .format(logdir, os.listdir(logdir))) raise # Test __system__ logs log_and_assert(message) # Test deployment logs log_and_assert(message, DEPLOYMENT_ID) @staticmethod def plugin_struct(plugin_name='mock-plugin'): return { 'source': os.path.join(resources.get_resource('plugins'), plugin_name), 'name': PLUGIN_NAME } def send_task(self, task_name, queue, deployment_id=None, args=None, kwargs=None, timeout=10, execution_env=None): cloudify_context = test_utils.op_context(task_name, task_target=queue, plugin_name=PLUGIN_NAME, execution_env=execution_env, deployment_id=deployment_id) kwargs = kwargs or {} kwargs['__cloudify_context'] = cloudify_context handler = amqp_client.BlockingRequestResponseHandler(exchange=queue) client = amqp_client.get_client() client.add_handler(handler) with client: task = {'cloudify_task': {'kwargs': kwargs}} result = handler.publish(task, routing_key='operation', timeout=timeout) error = result.get('error') if error: raise deserialize_known_exception(error) else: return result.get('result')
class BaseDaemonProcessManagementTest(BaseDaemonLiveTestCase): def setUp(self): super(BaseDaemonProcessManagementTest, self).setUp() self.installer = PluginInstaller(logger=self.logger) def tearDown(self): super(BaseDaemonProcessManagementTest, self).tearDown() self.installer.uninstall_source(plugin=self.plugin_struct()) self.installer.uninstall_source(plugin=self.plugin_struct(), deployment_id=DEPLOYMENT_ID) @property def daemon_cls(self): raise NotImplementedError('Must be implemented by sub-class') def create_daemon(self, **attributes): name = utils.internal.generate_agent_name() params = { 'rest_host': ['127.0.0.1'], 'broker_ip': ['127.0.0.1'], 'user': self.username, 'workdir': self.temp_folder, 'logger': self.logger, 'name': name, 'queue': '{0}-queue'.format(name), 'local_rest_cert_file': self._rest_cert_path, 'broker_ssl_enabled': False, # No SSL on the CI machines } params.update(attributes) factory = DaemonFactory() daemon = self.daemon_cls(**params) factory.save(daemon) self.addCleanup(factory.delete, daemon.name) self.daemons.append(daemon) return daemon def test_create(self): daemon = self.create_daemon() daemon.create() def test_create_overwrite(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.wait_for_daemon_alive(daemon.queue) daemon.create() daemon.configure() daemon.start() self.wait_for_daemon_alive(daemon.queue) daemon.stop() self.wait_for_daemon_dead(daemon.queue) def test_configure(self): raise NotImplementedError('Must be implemented by sub-class') def test_start(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() def test_start_delete_amqp_queue(self): daemon = self.create_daemon() daemon.create() daemon.configure() # this creates the queue daemon.start() daemon.stop() daemon.start(delete_amqp_queue=True) @patch_get_source def test_start_with_error(self): if os.name == 'nt': log_dir = 'H:\\WATT_NONEXISTENT_DIR\\lo' else: log_dir = '/root/no_permission' daemon = self.create_daemon(log_dir=log_dir) daemon.create() daemon.configure() try: daemon.start(timeout=5) self.fail('Expected start operation to fail due to bad log_dir') except exceptions.DaemonError as e: if os.name == 'nt': # windows messages vary, and will be escaped, so let's just # check that the dir name is there expected_error = 'WATT_NONEXISTENT_DIR' else: expected_error = "Permission denied: '{0}" self.assertIn(expected_error.format(log_dir), str(e)) def test_start_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() try: daemon.start(timeout=-1) except exceptions.DaemonStartupTimeout as e: self.assertTrue('failed to start in -1 seconds' in str(e)) def test_status(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.assertFalse(daemon.status()) daemon.start() self.assertTrue(daemon.status()) def test_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.stop() self.wait_for_daemon_dead(daemon.queue) def test_stop_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() try: daemon.stop(timeout=-1) except exceptions.DaemonShutdownTimeout as e: self.assertTrue('failed to stop in -1 seconds' in str(e)) @patch_get_source def test_restart(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() daemon.restart() def test_two_daemons(self): daemon1 = self.create_daemon() daemon1.create() daemon1.configure() daemon1.start() self.assert_daemon_alive(daemon1.queue) daemon2 = self.create_daemon() daemon2.create() daemon2.configure() daemon2.start() self.assert_daemon_alive(daemon2.queue) @patch_get_source def test_conf_env_variables(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() expected = { constants.REST_HOST_KEY: ','.join(daemon.rest_host), constants.REST_PORT_KEY: str(daemon.rest_port), constants.MANAGER_FILE_SERVER_URL_KEY: ','.join( 'https://{0}:{1}/resources'.format(host, daemon.rest_port) for host in daemon.rest_host), constants.AGENT_WORK_DIR_KEY: daemon.workdir, } def _get_env_var(var): return self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': var}) def _check_env_var(var, expected_value): _value = _get_env_var(var) self.assertEqual(_value, expected_value) for key, value in expected.items(): _check_env_var(key, value) @patch_get_source def test_extra_env(self): daemon = self.create_daemon() daemon.extra_env_path = utils.env_to_file( {'TEST_ENV_KEY': 'TEST_ENV_VALUE'}, posix=os.name == 'posix') daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check the env file was properly sourced by querying the env # variable from the daemon process. this is done by a task value = self.send_task(task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY'}) self.assertEqual(value, 'TEST_ENV_VALUE') @patch_get_source def test_execution_env(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() # check that cloudify.dispatch.dispatch 'execution_env' processing # works. # not the most ideal place for this test. but on the other hand # all the boilerplate is already here, so this is too tempting. value = self.send_task( task_name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, kwargs={'env_variable': 'TEST_ENV_KEY2'}, execution_env={'TEST_ENV_KEY2': 'TEST_ENV_VALUE2'}) self.assertEqual(value, 'TEST_ENV_VALUE2') def test_delete(self): raise NotImplementedError('Must be implemented by sub-class') def test_delete_before_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assertRaises(exceptions.DaemonStillRunningException, daemon.delete) def test_delete_before_stop_with_force(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.delete(force=True) self.wait_for_daemon_dead(daemon.queue) @patch_get_source def test_logging(self): message = 'THIS IS THE TEST MESSAGE LOG CONTENT' daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) self.installer.install(self.plugin_struct(), deployment_id=DEPLOYMENT_ID) daemon.start() def log_and_assert(_message, _deployment_id=None): self.send_task(task_name='mock_plugin.tasks.do_logging', queue=daemon.queue, kwargs={'message': _message}, deployment_id=_deployment_id) name = _deployment_id if _deployment_id else '__system__' logdir = os.path.join(daemon.workdir, 'logs') logfile = os.path.join(logdir, '{0}.log'.format(name)) try: with open(logfile) as f: self.assertIn(_message, f.read()) except IOError: self.logger.warning('{0} content: {1}'.format( logdir, os.listdir(logdir))) raise # Test __system__ logs log_and_assert(message) # Test deployment logs log_and_assert(message, DEPLOYMENT_ID) @staticmethod def plugin_struct(plugin_name='mock-plugin'): return { 'source': os.path.join(resources.get_resource('plugins'), plugin_name), 'name': PLUGIN_NAME } def send_task(self, task_name, queue, deployment_id=None, args=None, kwargs=None, timeout=10, execution_env=None): cloudify_context = test_utils.op_context(task_name, task_target=queue, plugin_name=PLUGIN_NAME, execution_env=execution_env, deployment_id=deployment_id) kwargs = kwargs or {} kwargs['__cloudify_context'] = cloudify_context handler = amqp_client.BlockingRequestResponseHandler(queue) client = amqp_client.get_client() client.add_handler(handler) with client: task = {'cloudify_task': {'kwargs': kwargs}} result = handler.publish(task, routing_key='operation', timeout=timeout) error = result.get('error') if error: raise deserialize_known_exception(error) else: return result.get('result')
class BaseDaemonProcessManagementTest(BaseDaemonLiveTestCase): def setUp(self): super(BaseDaemonProcessManagementTest, self).setUp() self.installer = PluginInstaller(logger=self.logger) def tearDown(self): super(BaseDaemonProcessManagementTest, self).tearDown() self.installer.uninstall('mock-plugin') self.installer.uninstall('mock-plugin-error') @property def daemon_cls(self): raise NotImplementedError('Must be implemented by sub-class') def create_daemon(self, **attributes): name = utils.internal.generate_agent_name() params = { 'manager_ip': '127.0.0.1', 'user': self.username, 'workdir': self.temp_folder, 'logger': self.logger, 'name': name, 'queue': '{0}-queue'.format(name) } params.update(attributes) daemon = self.daemon_cls(**params) self.daemons.append(daemon) return daemon def test_create(self): daemon = self.create_daemon() daemon.create() def test_configure(self): raise NotImplementedError('Must be implemented by sub-class') def test_start(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assert_daemon_alive(daemon.name) self.assert_registered_tasks(daemon.name) def test_start_delete_amqp_queue(self): daemon = self.create_daemon() daemon.create() daemon.configure() # this creates the queue daemon.start() daemon.stop() daemon.start(delete_amqp_queue=True) @patch_get_source def test_start_with_error(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct('mock-plugin-error')) daemon.register('mock-plugin-error') try: daemon.start() self.fail('Expected start operation to fail ' 'due to bad import') except exceptions.DaemonError as e: self.assertIn('cannot import name non_existent', str(e)) def test_start_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() try: daemon.start(timeout=-1) except exceptions.DaemonStartupTimeout as e: self.assertTrue('failed to start in -1 seconds' in str(e)) def test_status(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.assertFalse(daemon.status()) daemon.start() self.assertTrue(daemon.status()) def test_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.stop() self.assert_daemon_dead(daemon.name) def test_stop_short_timeout(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() try: daemon.stop(timeout=-1) except exceptions.DaemonShutdownTimeout as e: self.assertTrue('failed to stop in -1 seconds' in str(e)) @patch_get_source def test_register(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.register('mock-plugin') daemon.start() self.assert_registered_tasks( daemon.name, additional_tasks=set(['mock_plugin.tasks.run', 'mock_plugin.tasks.get_env_variable']) ) @patch_get_source def test_unregister(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.register('mock-plugin') daemon.start() self.assert_registered_tasks( daemon.name, additional_tasks=set(['mock_plugin.tasks.run', 'mock_plugin.tasks.get_env_variable']) ) daemon.unregister('mock-plugin') daemon.restart() self.assert_registered_tasks(daemon.name) @patch_get_source def test_restart(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.start() daemon.register('mock-plugin') daemon.restart() self.assert_registered_tasks( daemon.name, additional_tasks=set(['mock_plugin.tasks.run', 'mock_plugin.tasks.get_env_variable']) ) def test_two_daemons(self): daemon1 = self.create_daemon() daemon1.create() daemon1.configure() daemon1.start() self.assert_daemon_alive(daemon1.name) self.assert_registered_tasks(daemon1.name) daemon2 = self.create_daemon() daemon2.create() daemon2.configure() daemon2.start() self.assert_daemon_alive(daemon2.name) self.assert_registered_tasks(daemon2.name) @patch_get_source def test_conf_env_variables(self): daemon = self.create_daemon() daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.register('mock-plugin') daemon.start() expected = { constants.MANAGER_IP_KEY: str(daemon.manager_ip), constants.MANAGER_REST_PORT_KEY: str(daemon.manager_port), constants.MANAGER_FILE_SERVER_URL_KEY: 'http://{0}:53229'.format(daemon.manager_ip), constants.MANAGER_FILE_SERVER_BLUEPRINTS_ROOT_URL_KEY: 'http://{0}:53229/blueprints'.format(daemon.manager_ip), utils.internal.CLOUDIFY_DAEMON_STORAGE_DIRECTORY_KEY: utils.internal.get_storage_directory(), utils.internal.CLOUDIFY_DAEMON_NAME_KEY: daemon.name, utils.internal.CLOUDIFY_DAEMON_USER_KEY: daemon.user } def _get_env_var(var): return self.celery.send_task( name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, args=[var]).get(timeout=5) def _check_env_var(var, expected_value): _value = _get_env_var(var) self.assertEqual(_value, expected_value) for key, value in expected.iteritems(): _check_env_var(key, value) # def _check_env_path(): # _path = _get_env_var('PATH') # self.assertIn(VIRTUALENV, _path) # _check_env_path() @patch_get_source def test_extra_env_path(self): daemon = self.create_daemon() daemon.extra_env_path = utils.env_to_file( {'TEST_ENV_KEY': 'TEST_ENV_VALUE'}, posix=os.name == 'posix' ) daemon.create() daemon.configure() self.installer.install(self.plugin_struct()) daemon.register('mock-plugin') daemon.start() # check the env file was properly sourced by querying the env # variable from the daemon process. this is done by a task value = self.celery.send_task( name='mock_plugin.tasks.get_env_variable', queue=daemon.queue, args=['TEST_ENV_KEY']).get(timeout=10) self.assertEqual(value, 'TEST_ENV_VALUE') def test_delete(self): raise NotImplementedError('Must be implemented by sub-class') def test_delete_before_stop(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() self.assertRaises(exceptions.DaemonStillRunningException, daemon.delete) def test_delete_before_stop_with_force(self): daemon = self.create_daemon() daemon.create() daemon.configure() daemon.start() daemon.delete(force=True) self.assert_daemon_dead(daemon.name) @staticmethod def plugin_struct(plugin_name='mock-plugin'): return { 'source': os.path.join(resources.get_resource('plugins'), plugin_name) }