def cluster_update_task(self, brokers, broker_ca, managers, manager_ca):
"""Update the running agent with the new cluster.
When a node is added or removed from the cluster, the agent will
receive the current cluster nodes in this task. We need to update
both the current process envvars, the cert files, and all the
daemon config files.
"""
if not self.name:
raise RuntimeError('cluster-update sent to agent with no name set')
factory = DaemonFactory()
daemon = factory.load(self.name)
os.environ[constants.REST_HOST_KEY] = ','.join(managers)
with open(daemon.local_rest_cert_file, 'w') as f:
f.write(manager_ca)
with open(daemon.broker_ssl_cert_path, 'w') as f:
f.write(broker_ca)
daemon.rest_host = managers
daemon.broker_ip = brokers
daemon.create_broker_conf()
daemon.create_config()
factory.save(daemon)
def cluster_update(nodes):
cluster.set_cluster_nodes(nodes)
factory = DaemonFactory()
daemon = factory.load(daemon_name)
network_name = daemon.network
daemon.cluster = [n['networks'][network_name] for n in nodes]
factory.save(daemon)
def cluster_update_task(self, nodes):
if not self.name:
raise RuntimeError('cluster-update sent to agent with no name set')
factory = DaemonFactory()
daemon = factory.load(self.name)
network_name = daemon.network
nodes = [n['networks'][network_name] for n in nodes]
daemon.cluster = nodes
factory.save(daemon)
def cluster_update_task(self, nodes):
if not self.name:
raise RuntimeError('cluster-update sent to agent with no name set')
factory = DaemonFactory()
daemon = factory.load(self.name)
network_name = daemon.network
nodes = [n['networks'][network_name] for n in nodes]
daemon.cluster = nodes
factory.save(daemon)
def _set_master(daemon_name, node_ip):
factory = DaemonFactory()
try:
daemon = factory.load(daemon_name)
except exceptions.DaemonNotFoundError:
return
daemon.broker_ip = node_ip
factory.save(daemon)
cluster.set_cluster_active(node_ip)
def _set_master(daemon_name, node):
factory = DaemonFactory()
try:
daemon = factory.load(daemon_name)
except exceptions.DaemonNotFoundError:
return
daemon.broker_ip = node['broker_ip']
daemon.broker_ssl_cert_path = node.get('internal_cert_path')
with _cluster_settings_lock(daemon_name):
factory.save(daemon)
cluster.set_cluster_active(node)
def replace_ca_certs_task(self, new_manager_ca, new_broker_ca):
"""Update the running agent with new CAs."""
self._assert_name('replace-ca-certs')
factory = DaemonFactory()
daemon = factory.load(self.name)
if new_broker_ca:
with open(daemon.broker_ssl_cert_path, 'w') as f:
f.write(new_broker_ca)
daemon.create_broker_conf()
if new_manager_ca:
with open(daemon.local_rest_cert_file, 'w') as f:
f.write(new_manager_ca)
daemon.create_config()
factory.save(daemon)
def create_daemon(self, **attributes):
name = utils.internal.generate_agent_name()
params = {
'rest_host': '127.0.0.1',
'broker_ip': '127.0.0.1',
'user': self.username,
'workdir': self.temp_folder,
'logger': self.logger,
'name': name,
'queue': '{0}-queue'.format(name),
'local_rest_cert_file': self._rest_cert_path,
'broker_ssl_enabled': False, # No SSL on the CI machines
}
params.update(attributes)
factory = DaemonFactory()
daemon = self.daemon_cls(**params)
factory.save(daemon)
self.addCleanup(factory.delete, daemon.name)
self.daemons.append(daemon)
return daemon
def create_daemon(self, **attributes):
name = utils.internal.generate_agent_name()
params = {
'rest_host': ['127.0.0.1'],
'broker_ip': ['127.0.0.1'],
'user': self.username,
'workdir': self.temp_folder,
'logger': self.logger,
'name': name,
'queue': '{0}-queue'.format(name),
'local_rest_cert_file': self._rest_cert_path,
'broker_ssl_enabled': False, # No SSL on the CI machines
}
params.update(attributes)
factory = DaemonFactory()
daemon = self.daemon_cls(**params)
factory.save(daemon)
self.addCleanup(factory.delete, daemon.name)
self.daemons.append(daemon)
return daemon
class BaseDaemon:
def __init__(self, tmp_path, logger, ssl_cert):
self.daemons = []
self.temp_folder = str(tmp_path)
self.username = getpass.getuser()
self.logger = logger
self.rest_cert_path = ssl_cert.local_cert_path()
self.factory = DaemonFactory()
self.runner = LocalCommandRunner(logger=logger)
@property
def daemon_cls(self):
raise NotImplementedError('Must be implemented by sub-class')
def create_daemon(self, **attributes):
name = utils.internal.generate_agent_name()
params = {
'rest_host': ['127.0.0.1'],
'broker_ip': ['127.0.0.1'],
'user': self.username,
'workdir': self.temp_folder,
'logger': self.logger,
'name': name,
'queue': '{0}-queue'.format(name),
'local_rest_cert_file': self.rest_cert_path,
'broker_ssl_enabled': False, # No SSL on the CI machines
}
params.update(attributes)
daemon = self.daemon_cls(**params)
self.factory.save(daemon)
self.daemons.append(daemon)
return daemon
@staticmethod
def plugin_struct(plugin_name='mock-plugin'):
return {
'source': os.path.join(resources.get_resource('plugins'),
plugin_name),
'package_name': PLUGIN_NAME
}
def send_task(self,
task_name,
queue,
deployment_id=None,
args=None,
kwargs=None,
timeout=10,
execution_env=None):
cloudify_context = test_utils.op_context(task_name,
task_target=queue,
plugin_name=PLUGIN_NAME,
execution_env=execution_env,
deployment_id=deployment_id)
kwargs = kwargs or {}
kwargs['__cloudify_context'] = cloudify_context
handler = amqp_client.BlockingRequestResponseHandler(queue)
client = amqp_client.get_client()
client.add_handler(handler)
with client:
task = {'cloudify_task': {'kwargs': kwargs}}
result = handler.publish(task,
routing_key='operation',
timeout=timeout)
error = result.get('error')
if error:
raise deserialize_known_exception(error)
else:
return result.get('result')
class TestDaemonFactory(BaseTest, unittest.TestCase):
def setUp(self):
super(TestDaemonFactory, self).setUp()
self.daemon_name = 'test-daemon-{0}'.format(uuid.uuid4())
self.factory = DaemonFactory(storage=get_storage_directory())
self.daemon_params = {
'process_management': 'init.d',
'name': self.daemon_name,
'queue': 'queue',
'rest_host': '127.0.0.1',
'broker_ip': '127.0.0.1',
'user': '******',
'broker_url': '127.0.0.1',
'broker_ssl_enabled': True,
'local_rest_cert_file': self._rest_cert_path
}
def test_new_initd(self):
daemon = self.factory.new(**self.daemon_params)
self.assertEqual(self.daemon_name, daemon.name)
self.assertEqual('queue', daemon.queue)
self.assertEqual('127.0.0.1', daemon.rest_host)
self.assertEqual('user', daemon.user)
self.assertEqual(self._rest_cert_path, daemon.local_rest_cert_file)
def test_new_no_implementation(self):
self.assertRaises(exceptions.DaemonNotImplementedError,
self.factory.new,
process_management='no-impl')
def test_save_load_delete(self):
daemon = self.factory.new(**self.daemon_params)
self.factory.save(daemon)
loaded = self.factory.load(self.daemon_name)
self.assertEqual('init.d', loaded.PROCESS_MANAGEMENT)
self.assertEqual(self.daemon_name, loaded.name)
self.assertEqual('queue', loaded.queue)
self.assertEqual('127.0.0.1', loaded.rest_host)
self.assertEqual('user', loaded.user)
self.factory.delete(daemon.name)
self.assertRaises(exceptions.DaemonNotFoundError, self.factory.load,
daemon.name)
def test_load_non_existing(self):
self.assertRaises(exceptions.DaemonNotFoundError, self.factory.load,
'non_existing_name')
def test_load_all(self):
def _save_daemon(name):
params = self.daemon_params.copy()
params['name'] = name
daemon = self.factory.new(**params)
self.factory.save(daemon)
if os.path.exists(get_storage_directory()):
shutil.rmtree(get_storage_directory())
daemons = self.factory.load_all()
self.assertEquals(0, len(daemons))
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
daemons = self.factory.load_all()
self.assertEquals(3, len(daemons))
def test_new_existing_agent(self):
daemon = self.factory.new(**self.daemon_params)
self.factory.save(daemon)
# without no_overwrite, this will overwrite the existing daemon
daemon = self.factory.new(**self.daemon_params)
self.assertRaises(exceptions.DaemonAlreadyExistsError,
self.factory.new,
no_overwrite=True,
**self.daemon_params)
def _save_daemon(daemon):
factory = DaemonFactory(username=utils.internal.get_daemon_user(),
storage=utils.internal.get_daemon_storage_dir())
factory.save(daemon)
def _save_daemon(daemon):
factory = DaemonFactory(
username=utils.internal.get_daemon_user(),
storage=utils.internal.get_daemon_storage_dir())
factory.save(daemon)
def cluster_update(nodes):
cluster.set_cluster_nodes(nodes)
factory = DaemonFactory()
daemon = factory.load(daemon_name)
daemon.cluster = nodes
factory.save(daemon)
class TestDaemonFactory(BaseShellTest):
def setUp(self):
super(TestDaemonFactory, self).setUp()
self.daemon_name = 'test-daemon-{0}'.format(uuid.uuid4())
self.factory = DaemonFactory(storage=get_storage_directory())
def test_new_initd(self):
daemon = self.factory.new(process_management='init.d',
name=self.daemon_name,
queue='queue',
manager_ip='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.assertEqual(self.daemon_name, daemon.name)
self.assertEqual('queue', daemon.queue)
self.assertEqual('127.0.0.1', daemon.manager_ip)
self.assertEqual('amqp://*****:*****@127.0.0.1:5672//',
daemon.broker_url)
self.assertEqual('user', daemon.user)
def test_new_wrong_includes_attribute(self):
self.assertRaises(ValueError,
self.factory.new,
process_management='init.d',
queue='{0}-queue'.format(self.daemon_name),
name=self.daemon_name,
manager_ip='127.0.0.1',
includes=set(['plugin']))
def test_new_no_implementation(self):
self.assertRaises(exceptions.DaemonNotImplementedError,
self.factory.new,
process_management='no-impl')
def test_save_load_delete(self):
daemon = self.factory.new(process_management='init.d',
name=self.daemon_name,
queue='queue',
manager_ip='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.factory.save(daemon)
loaded = self.factory.load(self.daemon_name)
self.assertEqual('init.d', loaded.PROCESS_MANAGEMENT)
self.assertEqual(self.daemon_name, loaded.name)
self.assertEqual('queue', loaded.queue)
self.assertEqual('127.0.0.1', loaded.manager_ip)
self.assertEqual('user', loaded.user)
self.assertEqual('amqp://*****:*****@127.0.0.1:5672//',
daemon.broker_url)
self.factory.delete(daemon.name)
self.assertRaises(exceptions.DaemonNotFoundError, self.factory.load,
daemon.name)
def test_load_non_existing(self):
self.assertRaises(exceptions.DaemonNotFoundError, self.factory.load,
'non_existing_name')
def test_load_all(self):
def _save_daemon(name):
daemon = self.factory.new(process_management='init.d',
name=name,
queue='queue',
manager_ip='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.factory.save(daemon)
if os.path.exists(get_storage_directory()):
shutil.rmtree(get_storage_directory())
daemons = self.factory.load_all()
self.assertEquals(0, len(daemons))
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
daemons = self.factory.load_all()
self.assertEquals(3, len(daemons))
def test_new_existing_agent(self):
daemon = self.factory.new(process_management='init.d',
name=self.daemon_name,
queue='queue',
manager_ip='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.factory.save(daemon)
self.assertRaises(exceptions.DaemonAlreadyExistsError,
self.factory.new,
process_management='init.d',
name=self.daemon_name,
queue='queue',
manager_ip='127.0.0.1',
user='******',
broker_url='127.0.0.1')
class TestDaemonFactory(BaseTest, unittest.TestCase):
def setUp(self):
super(TestDaemonFactory, self).setUp()
self.daemon_name = 'test-daemon-{0}'.format(uuid.uuid4())
self.factory = DaemonFactory(storage=get_storage_directory())
self.daemon_params = {
'process_management': 'init.d',
'name': self.daemon_name,
'queue': 'queue',
'rest_host': '127.0.0.1',
'broker_ip': '127.0.0.1',
'user': '******',
'broker_url': '127.0.0.1',
'broker_ssl_enabled': True,
'local_rest_cert_file': self._rest_cert_path
}
def test_new_initd(self):
daemon = self.factory.new(**self.daemon_params)
self.assertEqual(self.daemon_name, daemon.name)
self.assertEqual('queue', daemon.queue)
self.assertEqual('127.0.0.1', daemon.rest_host)
self.assertEqual('user', daemon.user)
self.assertEqual(self._rest_cert_path,
daemon.local_rest_cert_file)
def test_new_no_implementation(self):
self.assertRaises(exceptions.DaemonNotImplementedError,
self.factory.new,
process_management='no-impl')
def test_save_load_delete(self):
daemon = self.factory.new(**self.daemon_params)
self.factory.save(daemon)
loaded = self.factory.load(self.daemon_name)
self.assertEqual('init.d', loaded.PROCESS_MANAGEMENT)
self.assertEqual(self.daemon_name, loaded.name)
self.assertEqual('queue', loaded.queue)
self.assertEqual('127.0.0.1', loaded.rest_host)
self.assertEqual('user', loaded.user)
self.factory.delete(daemon.name)
self.assertRaises(exceptions.DaemonNotFoundError,
self.factory.load, daemon.name)
def test_load_non_existing(self):
self.assertRaises(exceptions.DaemonNotFoundError,
self.factory.load,
'non_existing_name')
def test_load_all(self):
def _save_daemon(name):
params = self.daemon_params.copy()
params['name'] = name
daemon = self.factory.new(**params)
self.factory.save(daemon)
if os.path.exists(get_storage_directory()):
shutil.rmtree(get_storage_directory())
daemons = self.factory.load_all()
self.assertEquals(0, len(daemons))
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
daemons = self.factory.load_all()
self.assertEquals(3, len(daemons))
def test_new_existing_agent(self):
daemon = self.factory.new(**self.daemon_params)
self.factory.save(daemon)
# without no_overwrite, this will overwrite the existing daemon
daemon = self.factory.new(**self.daemon_params)
self.assertRaises(exceptions.DaemonAlreadyExistsError,
self.factory.new,
no_overwrite=True,
**self.daemon_params)
class TestDaemonFactory(BaseShellTest):
def setUp(self):
super(TestDaemonFactory, self).setUp()
self.daemon_name = 'test-daemon-{0}'.format(uuid.uuid4())
self.factory = DaemonFactory(storage=get_storage_directory())
def test_new_initd(self):
daemon = self.factory.new(
process_management='init.d',
name=self.daemon_name,
queue='queue',
rest_host='127.0.0.1',
broker_ip='127.0.0.1',
file_server_host='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.assertEqual(self.daemon_name, daemon.name)
self.assertEqual('queue', daemon.queue)
self.assertEqual('127.0.0.1', daemon.rest_host)
self.assertEqual('amqp://*****:*****@127.0.0.1:5672//',
daemon.broker_url)
self.assertEqual('user', daemon.user)
def test_new_no_implementation(self):
self.assertRaises(exceptions.DaemonNotImplementedError,
self.factory.new,
process_management='no-impl')
def test_save_load_delete(self):
daemon = self.factory.new(
process_management='init.d',
name=self.daemon_name,
queue='queue',
rest_host='127.0.0.1',
broker_ip='127.0.0.1',
file_server_host='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.factory.save(daemon)
loaded = self.factory.load(self.daemon_name)
self.assertEqual('init.d', loaded.PROCESS_MANAGEMENT)
self.assertEqual(self.daemon_name, loaded.name)
self.assertEqual('queue', loaded.queue)
self.assertEqual('127.0.0.1', loaded.rest_host)
self.assertEqual('user', loaded.user)
self.assertEqual('amqp://*****:*****@127.0.0.1:5672//',
daemon.broker_url)
self.factory.delete(daemon.name)
self.assertRaises(exceptions.DaemonNotFoundError,
self.factory.load, daemon.name)
def test_load_non_existing(self):
self.assertRaises(exceptions.DaemonNotFoundError,
self.factory.load,
'non_existing_name')
def test_load_all(self):
def _save_daemon(name):
daemon = self.factory.new(
process_management='init.d',
name=name,
queue='queue',
rest_host='127.0.0.1',
broker_ip='127.0.0.1',
file_server_host='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.factory.save(daemon)
if os.path.exists(get_storage_directory()):
shutil.rmtree(get_storage_directory())
daemons = self.factory.load_all()
self.assertEquals(0, len(daemons))
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
_save_daemon(utils.internal.generate_agent_name())
daemons = self.factory.load_all()
self.assertEquals(3, len(daemons))
def test_new_existing_agent(self):
daemon = self.factory.new(
process_management='init.d',
name=self.daemon_name,
queue='queue',
rest_host='127.0.0.1',
broker_ip='127.0.0.1',
file_server_host='127.0.0.1',
user='******',
broker_url='127.0.0.1')
self.factory.save(daemon)
self.assertRaises(exceptions.DaemonAlreadyExistsError,
self.factory.new,
process_management='init.d',
name=self.daemon_name,
queue='queue',
rest_host='127.0.0.1',
broker_ip='127.0.0.1',
file_server_host='127.0.0.1',
user='******',
broker_url='127.0.0.1')
