def __init__(self,
cluster,
nodes,
affected_nodes=None,
task_ids=None,
events=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param affected_nodes: the list of nodes, that affected by deployment
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:param events: the events (see TaskEvents)
"""
if affected_nodes:
self.affected_node_ids = frozenset(n.uid for n in affected_nodes)
self.deployment_nodes = copy.copy(nodes)
self.deployment_nodes.extend(affected_nodes)
else:
self.deployment_nodes = nodes
self.affected_node_ids = frozenset()
self.cluster = cluster
self.role_resolver = RoleResolver(self.deployment_nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_connections = collections.defaultdict(dict)
self.tasks_dictionary = dict()
self.task_filter = self.make_task_filter(task_ids)
self.events = events
def __init__(self, cluster, nodes):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
def __init__(self, cluster, nodes, task_ids=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
self.task_filter = self.make_task_filter(task_ids)
def __init__(self, cluster, nodes,
affected_nodes=None, task_ids=None, events=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param affected_nodes: the list of nodes, that affected by deployment
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:param events: the events (see TaskEvents)
"""
if affected_nodes:
self.affected_node_ids = frozenset(n.uid for n in affected_nodes)
self.deployment_nodes = copy.copy(nodes)
self.deployment_nodes.extend(affected_nodes)
else:
self.deployment_nodes = nodes
self.affected_node_ids = frozenset()
self.cluster = cluster
self.role_resolver = RoleResolver(self.deployment_nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_connections = collections.defaultdict(dict)
self.tasks_dictionary = dict()
self.task_filter = self.make_task_filter(task_ids)
self.events = events
def __init__(self, cluster, nodes, role_resolver=None):
"""Initialises.
:param cluster: the cluster object instance
:param nodes: the list of nodes for deployment
:param role_resolver: the instance of BaseRoleResolver
"""
self.cluster = cluster
self.nodes = nodes
self.role_resolver = role_resolver or RoleResolver(nodes)
def __init__(self, cluster, nodes):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
def __init__(self, cluster, nodes, task_ids=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
self.task_filter = self.make_task_filter(task_ids)
def stage_tasks_serialize(self, tasks, nodes):
"""Serialize tasks for certain stage
:param stage: oneof consts.STAGES
:param nodes: list of node db objects
"""
serialized = []
role_resolver = RoleResolver(nodes)
for task in tasks:
if self.graph.should_exclude_task(task['id']):
continue
serializer = self.serializers.get_stage_serializer(task)(
task, self.cluster, nodes, role_resolver=role_resolver)
if not serializer.should_execute():
continue
serialized.extend(serializer.serialize())
return serialized
def initialize(self, cluster):
self.all_nodes = objects.Cluster.get_nodes_not_for_deletion(cluster)
self.role_resolver = RoleResolver(self.all_nodes)
self.initialized = cluster.id
class TasksSerializer(object):
"""The deploy tasks serializer."""
def __init__(self, cluster, nodes, task_ids=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
self.task_filter = self.make_task_filter(task_ids)
@classmethod
def serialize(cls, cluster, nodes, tasks, task_ids=None):
"""Resolves roles and dependencies for tasks.
:param cluster: the cluster instance
:param nodes: the list of nodes
:param tasks: the list of tasks
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:return: the list of serialized task per node
"""
serializer = cls(cluster, nodes, task_ids)
serializer.resolve_nodes(add_plugin_deployment_hooks(tasks), nodes)
serializer.resolve_dependencies()
return dict(
(k, list(six.itervalues(v)))
for k, v in six.iteritems(serializer.tasks_per_node)
)
def resolve_nodes(self, tasks, nodes):
"""Resolves node roles in tasks.
:param tasks: the deployment tasks
:param nodes: the list of nodes to deploy
:return the mapping tasks per node
"""
tasks_mapping = dict()
groups = list()
for task in tasks:
if task.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group:
groups.append(task)
else:
tasks_mapping[task['id']] = task
self.process_task(
task, nodes, lambda _: self.role_resolver,
skip=not self.task_filter(task['id'])
)
self.expand_task_groups(groups, tasks_mapping)
# make sure that null node is present
self.tasks_per_node.setdefault(None, dict())
def process_task(self, task, nodes, resolver_factory, skip=False):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param nodes: the list of nodes
:param resolver_factory: the factory creates role-resolver
:param skip: make the task as skipped
"""
serializer_factory = self.task_serializer.get_stage_serializer(
task
)
task_serializer = serializer_factory(
task, self.cluster, nodes, role_resolver=resolver_factory(nodes)
)
# do not pass skipped attribute to astute
skipped = skip or task.pop('skipped', False) or \
not task_serializer.should_execute()
for astute_task in self.task_processor.process_tasks(
task, task_serializer.serialize()):
# all skipped task shall have type skipped
# do not exclude them from graph to keep connections between nodes
if skipped:
astute_task['type'] = \
consts.ORCHESTRATOR_TASK_TYPES.skipped
for node_id in astute_task.pop('uids', ()):
node_tasks = self.tasks_per_node[node_id]
# de-duplication the tasks on node
# since task can be added after expand group need to
# overwrite if existed task is skipped and new is not skipped.
if self.need_update_task(node_tasks, astute_task):
node_tasks[astute_task['id']] = copy.deepcopy(astute_task)
def resolve_dependencies(self):
"""Resolves tasks dependencies."""
for node_id, tasks in six.iteritems(self.tasks_per_node):
for task in six.itervalues(tasks):
task['requires'] = list(
self.expand_dependencies(
node_id, task.get('requires'), False
)
)
task['required_for'] = list(
self.expand_dependencies(
node_id, task.get('required_for'), True
)
)
task['requires'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depends', None), False
)
)
task['required_for'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depended-by', None), True
)
)
task['requires'].extend(task.pop('requires_ex', ()))
task['required_for'].extend(task.pop('required_for_ex', ()))
def expand_task_groups(self, groups, task_mapping):
"""Expand group of tasks.
:param groups: the all tasks with type 'group'
:param task_mapping: the mapping task id to task object
"""
for task in groups:
skipped = not self.task_filter(task['id'])
node_ids = self.role_resolver.resolve(task.get('role', ()))
for sub_task_id in task.get('tasks', ()):
try:
sub_task = task_mapping[sub_task_id]
except KeyError:
raise errors.InvalidData(
'Task %s cannot be resolved', sub_task_id
)
# if group is not excluded, all task should be run as well
# otherwise check each task individually
self.process_task(
sub_task, node_ids, NullResolver,
skip=skipped and not self.task_filter(sub_task_id)
)
def expand_dependencies(self, node_id, dependencies, is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of dependencies on same node
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
# need to search dependencies on node and in sync points
node_ids = [node_id, None]
for name in dependencies:
for rel in self.resolve_relation(name, node_ids, is_required_for):
yield rel
def expand_cross_dependencies(
self, node_id, dependencies, is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of cross-node dependencies
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
for dep in dependencies:
roles = dep.get('role', consts.TASK_ROLES.all)
if roles == consts.TASK_ROLES.self:
node_ids = [node_id]
else:
node_ids = self.role_resolver.resolve(
roles, dep.get('policy', consts.NODE_RESOLVE_POLICY.all)
)
relations = self.resolve_relation(
dep['name'], node_ids, is_required_for
)
for rel in relations:
yield rel
def resolve_relation(self, name, node_ids, is_required_for):
"""Resolves the task relation.
:param name: the name of task
:param node_ids: the ID of nodes where need to search
:param is_required_for: means task from required_for section
"""
match_policy = NameMatchingPolicy.create(name)
for node_id in node_ids:
applied_tasks = set()
for task_name in self.tasks_per_node[node_id]:
if task_name == name:
# the simple case when name of current task
# is exact math to name of task that is search
yield {"name": task_name, "node_id": node_id}
continue
# at first get the original task name, actual
# when current task is part of chain
original_task = self.task_processor.get_origin(task_name)
if original_task in applied_tasks or \
not match_policy.match(original_task):
continue
applied_tasks.add(original_task)
if original_task is not task_name:
if is_required_for:
task_name_gen = self.task_processor.get_first_task_id
else:
task_name_gen = self.task_processor.get_last_task_id
task_name = task_name_gen(original_task)
yield {"name": task_name, "node_id": node_id}
@classmethod
def need_update_task(cls, tasks, task):
"""Checks that task shall overwrite existed one or should be added.
:param tasks: the current node tasks
:param task: the astute task object
:return True if task is not present or must be overwritten
otherwise False
"""
existed_task = tasks.get(task['id'])
if existed_task is None:
return True
if existed_task['type'] == task['type']:
return False
return task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped
@classmethod
def make_task_filter(cls, task_ids):
"""Makes task filter according to specified ids.
:param task_ids: the selected ids of tasks
:return: function that check task
"""
if not task_ids:
return lambda _: True
if not isinstance(task_ids, set):
task_ids = set(task_ids)
return lambda task_id: task_id in task_ids
def setUpClass(cls):
cls.tasks = [{
'id': 'task1',
'roles': ['controller'],
'type': 'puppet',
'version': '2.0.0',
'condition': {
'yaql_exp': '$.public_ssl.hostname = localhost'
},
'parameters': {},
'requires': ['task3'],
'required_for': ['task2'],
'cross_depends': [{
'name': 'task2',
'role': 'compute'
}],
'cross_depended_by': [{
'name': 'task3',
'role': 'cinder'
}]
}, {
'id': 'task2',
'roles': ['compute', 'controller'],
'type': 'puppet',
'version': '2.0.0',
'condition': {
'yaql_exp': '$.public_ssl.hostname != localhost'
},
'parameters': {},
'cross_depends': [{
'name': 'task3',
'role': 'cinder'
}]
}, {
'id': 'task3',
'roles': ['cinder', 'controller'],
'type': 'puppet',
'version': '2.0.0',
'condition': 'settings:public_ssl.hostname != "localhost"',
'parameters': {},
'cross_depends': [{
'name': 'task3',
'role': '/.*/'
}],
'cross_depended_by': [{
'name': 'task2',
'role': 'self'
}]
}, {
'id': 'task4',
'roles': ['controller'],
'type': 'puppet',
'version': '2.0.0',
'parameters': {},
'cross_depended_by': [{
'name': 'task3'
}]
}]
cls.nodes = [
mock.MagicMock(uid='1', roles=['controller']),
mock.MagicMock(uid='2', roles=['compute']),
mock.MagicMock(uid='3', roles=['cinder']),
mock.MagicMock(uid='4', roles=['custom']),
]
cls.context = lcm.TransactionContext({
'1': {
'cluster': {
'id': 1
},
'release': {
'version': 'liberty-9.0'
},
'openstack_version': 'liberty-9.0',
'public_ssl': {
'hostname': 'localhost'
},
'attributes': {
'a_str': 'text1',
'a_int': 1
}
},
'2': {
'cluster': {
'id': 1
},
'release': {
'version': 'liberty-9.0'
},
'openstack_version': 'liberty-9.0',
'public_ssl': {
'hostname': 'localhost'
},
'attributes': {
'a_str': 'text2',
'a_int': 2
}
},
'3': {
'cluster': {
'id': 1
},
'release': {
'version': 'liberty-9.0'
},
'openstack_version': 'liberty-9.0',
'public_ssl': {
'hostname': 'localhost'
},
'attributes': {
'a_str': 'text3',
'a_int': 3
}
},
'4': {
'cluster': {
'id': 1
},
'release': {
'version': 'liberty-9.0'
},
'openstack_version': 'liberty-9.0',
'public_ssl': {
'hostname': 'localhost'
},
'attributes': {
'a_str': 'text3',
'a_int': 3
}
}
})
with mock.patch('nailgun.utils.role_resolver.objects') as m_objects:
m_objects.Node.all_roles = lambda x: x.roles
cls.role_resolver = RoleResolver(cls.nodes)
class DeploymentMultinodeSerializer(object):
nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
neutron_network_serializer = \
neutron_serializers.NeutronNetworkDeploymentSerializer
critical_roles = frozenset(('controller', 'ceph-osd', 'primary-mongo'))
def __init__(self, tasks_graph=None):
self.task_graph = tasks_graph
self.all_nodes = None
self.role_resolver = None
self.initialized = None
def initialize(self, cluster):
self.all_nodes = objects.Cluster.get_nodes_not_for_deletion(cluster)
self.role_resolver = RoleResolver(self.all_nodes)
self.initialized = cluster.id
def finalize(self):
self.all_nodes = None
self.role_resolver = None
self.initialized = None
def _ensure_initialized_for(self, cluster):
# TODO(bgaifullin) need to move initialize into __init__
if self.initialized != cluster.id:
self.initialize(cluster)
def serialize(self,
cluster,
nodes,
ignore_customized=False,
skip_extensions=False):
"""Method generates facts which are passed to puppet."""
try:
self.initialize(cluster)
common_attrs = self.get_common_attrs(cluster)
if not ignore_customized and cluster.replaced_deployment_info:
# patch common attributes with custom deployment info
utils.dict_update(common_attrs,
cluster.replaced_deployment_info)
if not skip_extensions:
extensions.\
fire_callback_on_cluster_serialization_for_deployment(
cluster, common_attrs
)
serialized_nodes = []
origin_nodes = []
customized_nodes = []
if ignore_customized:
origin_nodes = nodes
else:
for node in nodes:
if node.replaced_deployment_info:
customized_nodes.append(node)
else:
origin_nodes.append(node)
serialized_nodes.extend(
self.serialize_generated(origin_nodes, skip_extensions))
serialized_nodes.extend(
self.serialize_customized(customized_nodes))
# NOTE(dshulyak) tasks should not be preserved from replaced
# deployment info, there is different mechanism to control
# changes in tasks introduced during granular deployment,
# and that mech should be used
self.set_tasks(serialized_nodes)
deployment_info = {
'common': common_attrs,
'nodes': serialized_nodes
}
finally:
self.finalize()
return deployment_info
def serialize_generated(self, nodes, skip_extensions):
serialized_nodes = self.serialize_nodes(nodes)
nodes_map = {n.uid: n for n in nodes}
self.set_deployment_priorities(serialized_nodes)
for node_data in serialized_nodes:
# the serialized nodes may contain fake nodes like master node
# which does not have related db object. it shall be excluded.
if not skip_extensions and node_data['uid'] in nodes_map:
extensions.fire_callback_on_node_serialization_for_deployment(
nodes_map[node_data['uid']], node_data)
yield node_data
def serialize_customized(self, nodes):
for node in nodes:
for role_data in node.replaced_deployment_info:
yield role_data
def get_common_attrs(self, cluster):
"""Cluster attributes."""
# tests call this method directly.
# and we need this workaround to avoid refactoring a lot of tests.
self._ensure_initialized_for(cluster)
attrs = objects.Cluster.get_attributes(cluster)
attrs = objects.Attributes.merged_attrs_values(attrs)
attrs['deployment_mode'] = cluster.mode
attrs['deployment_id'] = cluster.id
attrs['openstack_version'] = cluster.release.version
attrs['fuel_version'] = cluster.fuel_version
attrs['nodes'] = self.node_list(self.all_nodes)
# Adding params to workloads_collector
if 'workloads_collector' not in attrs:
attrs['workloads_collector'] = {}
attrs['workloads_collector']['create_user'] = \
objects.MasterNodeSettings.must_send_stats()
username = attrs['workloads_collector'].pop('user', None)
attrs['workloads_collector']['username'] = username
if self.role_resolver.resolve(['cinder']):
attrs['use_cinder'] = True
net_serializer = self.get_net_provider_serializer(cluster)
net_common_attrs = net_serializer.get_common_attrs(cluster, attrs)
utils.dict_update(attrs, net_common_attrs)
self.inject_list_of_plugins(attrs, cluster)
return attrs
@classmethod
def node_list(cls, nodes):
"""Generate nodes list. Represents as "nodes" parameter in facts."""
node_list = []
for node in nodes:
for role in objects.Node.all_roles(node):
node_list.append(cls.serialize_node_for_node_list(node, role))
return node_list
@classmethod
def serialize_node_for_node_list(cls, node, role):
return {
'uid': node.uid,
'fqdn': objects.Node.get_node_fqdn(node),
'name': objects.Node.get_slave_name(node),
'role': role
}
# TODO(apopovych): we have more generical method 'filter_by_roles'
def by_role(self, nodes, role):
return filter(lambda node: node['role'] == role, nodes)
def not_roles(self, nodes, roles):
return filter(lambda node: node['role'] not in roles, nodes)
def serialize_nodes(self, nodes):
"""Serialize node for each role.
For example if node has two roles then
in orchestrator will be passed two serialized
nodes.
"""
serialized_nodes = []
for node in nodes:
for role in objects.Node.all_roles(node):
serialized_nodes.append(self.serialize_node(node, role))
return serialized_nodes
def serialize_node(self, node, role):
"""Serialize node, then it will be merged with common attributes."""
node_attrs = {
# Yes, uid is really should be a string
'uid': node.uid,
'fqdn': objects.Node.get_node_fqdn(node),
'status': node.status,
'role': role,
'vms_conf': node.vms_conf,
'fail_if_error': role in self.critical_roles,
# TODO(eli): need to remove, requried for the fake thread only
'online': node.online,
}
net_serializer = self.get_net_provider_serializer(node.cluster)
node_attrs.update(net_serializer.get_node_attrs(node))
node_attrs.update(net_serializer.network_ranges(node.group_id))
node_attrs.update(self.generate_test_vm_image_data(node))
return node_attrs
def generate_properties_arguments(self, properties_data):
"""build a string of properties from a key value hash"""
properties = []
for key, value in six.iteritems(properties_data):
properties.append('--property {key}={value}'.format(key=key,
value=value))
return ' '.join(properties)
def generate_test_vm_image_data(self, node):
# Instantiate all default values in dict.
image_data = {
'container_format': 'bare',
'public': 'true',
'disk_format': 'qcow2',
'img_name': 'TestVM',
'img_path': '',
'os_name': 'cirros',
'min_ram': 64,
'glance_properties': '',
'properties': {},
}
# Generate a right path to image.
c_attrs = node.cluster.attributes
if 'ubuntu' in c_attrs['generated']['cobbler']['profile']:
img_dir = '/usr/share/cirros-testvm/'
else:
img_dir = '/opt/vm/'
image_data['img_path'] = '{0}cirros-x86_64-disk.img'.format(img_dir)
properties_data = {}
# Alternate VMWare specific values.
if c_attrs['editable']['common']['libvirt_type']['value'] == 'vcenter':
image_data.update({
'disk_format':
'vmdk',
'img_path':
'{0}cirros-i386-disk.vmdk'.format(img_dir),
})
properties_data = {
'vmware_disktype': 'sparse',
'vmware_adaptertype': 'lsiLogic',
'hypervisor_type': 'vmware'
}
# NOTE(aschultz): properties was added as part of N and should be
# used infavor of glance_properties
image_data['glance_properties'] = self.generate_properties_arguments(
properties_data)
image_data['properties'] = properties_data
return {'test_vm_image': image_data}
@classmethod
def get_net_provider_serializer(cls, cluster):
if cluster.net_provider == 'nova_network':
return cls.nova_network_serializer
else:
return cls.neutron_network_serializer
def filter_by_roles(self, nodes, roles):
return filter(lambda node: node['role'] in roles, nodes)
def set_deployment_priorities(self, nodes):
if self.task_graph is not None:
self.task_graph.add_priorities(nodes)
def set_tasks(self, serialized_nodes):
if self.task_graph is not None:
for node in serialized_nodes:
node['tasks'] = self.task_graph.deploy_task_serialize(node)
def inject_list_of_plugins(self, attributes, cluster):
"""Added information about plugins to serialized attributes.
:param attributes: the serialized attributes
:param cluster: the cluster object
"""
plugins = objects.ClusterPlugin.get_enabled(cluster.id)
attributes['plugins'] = [
self.serialize_plugin(cluster, p) for p in plugins
]
@classmethod
def serialize_plugin(cls, cluster, plugin):
"""Gets plugin information to include into serialized attributes.
:param cluster: the cluster object
:param plugin: the plugin object
"""
return plugin['name']
def initialize(self, cluster):
self.all_nodes = objects.Cluster.get_nodes_not_for_deletion(cluster)
self.role_resolver = RoleResolver(self.all_nodes)
self.initialized = cluster.id
def __init__(self, task, cluster, nodes, role_resolver=None):
super(StandardConfigRolesHook, self).__init__(task, cluster)
self.nodes = nodes
self.role_resolver = role_resolver or RoleResolver(nodes)
class TasksSerializer(object):
"""The deploy tasks serializer."""
def __init__(self, cluster, nodes, task_ids=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
self.task_filter = self.make_task_filter(task_ids)
@classmethod
def serialize(cls, cluster, nodes, tasks, task_ids=None):
"""Resolves roles and dependencies for tasks.
:param cluster: the cluster instance
:param nodes: the list of nodes
:param tasks: the list of tasks
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:return: the list of serialized task per node
"""
serializer = cls(cluster, nodes, task_ids)
serializer.resolve_nodes(add_plugin_deployment_hooks(tasks), nodes)
serializer.resolve_dependencies()
return dict((k, list(six.itervalues(v)))
for k, v in six.iteritems(serializer.tasks_per_node))
def resolve_nodes(self, tasks, nodes):
"""Resolves node roles in tasks.
:param tasks: the deployment tasks
:param nodes: the list of nodes to deploy
:return the mapping tasks per node
"""
tasks_mapping = dict()
groups = list()
for task in tasks:
if task.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group:
groups.append(task)
else:
tasks_mapping[task['id']] = task
self.process_task(task,
nodes,
lambda _: self.role_resolver,
skip=not self.task_filter(task['id']))
self.expand_task_groups(groups, tasks_mapping)
# make sure that null node is present
self.tasks_per_node.setdefault(None, dict())
def process_task(self, task, nodes, resolver_factory, skip=False):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param nodes: the list of nodes
:param resolver_factory: the factory creates role-resolver
:param skip: make the task as skipped
"""
serializer_factory = self.task_serializer.get_stage_serializer(task)
task_serializer = serializer_factory(
task, self.cluster, nodes, role_resolver=resolver_factory(nodes))
# do not pass skipped attribute to astute
skipped = skip or task.pop('skipped', False) or \
not task_serializer.should_execute()
for astute_task in self.task_processor.process_tasks(
task, task_serializer.serialize()):
# all skipped task shall have type skipped
# do not exclude them from graph to keep connections between nodes
if skipped:
astute_task['type'] = \
consts.ORCHESTRATOR_TASK_TYPES.skipped
for node_id in astute_task.pop('uids', ()):
node_tasks = self.tasks_per_node[node_id]
# de-duplication the tasks on node
# since task can be added after expand group need to
# overwrite if existed task is skipped and new is not skipped.
if self.need_update_task(node_tasks, astute_task):
node_tasks[astute_task['id']] = copy.deepcopy(astute_task)
def resolve_dependencies(self):
"""Resolves tasks dependencies."""
for node_id, tasks in six.iteritems(self.tasks_per_node):
for task in six.itervalues(tasks):
task['requires'] = list(
self.expand_dependencies(node_id, task.get('requires'),
False))
task['required_for'] = list(
self.expand_dependencies(node_id, task.get('required_for'),
True))
task['requires'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depends', None), False))
task['required_for'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depended-by', None), True))
task['requires'].extend(task.pop('requires_ex', ()))
task['required_for'].extend(task.pop('required_for_ex', ()))
def expand_task_groups(self, groups, task_mapping):
"""Expand group of tasks.
:param groups: the all tasks with type 'group'
:param task_mapping: the mapping task id to task object
"""
for task in groups:
skipped = not self.task_filter(task['id'])
node_ids = self.role_resolver.resolve(task.get('role', ()))
for sub_task_id in task.get('tasks', ()):
try:
sub_task = task_mapping[sub_task_id]
except KeyError:
raise errors.InvalidData('Task %s cannot be resolved',
sub_task_id)
# if group is not excluded, all task should be run as well
# otherwise check each task individually
self.process_task(sub_task,
node_ids,
NullResolver,
skip=skipped
and not self.task_filter(sub_task_id))
def expand_dependencies(self, node_id, dependencies, is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of dependencies on same node
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
# need to search dependencies on node and in sync points
node_ids = [node_id, None]
for name in dependencies:
for rel in self.resolve_relation(name, node_ids, is_required_for):
yield rel
def expand_cross_dependencies(self, node_id, dependencies,
is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of cross-node dependencies
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
for dep in dependencies:
roles = dep.get('role', consts.TASK_ROLES.all)
if roles == consts.TASK_ROLES.self:
node_ids = [node_id]
else:
node_ids = self.role_resolver.resolve(
roles, dep.get('policy', consts.NODE_RESOLVE_POLICY.all))
relations = self.resolve_relation(dep['name'], node_ids,
is_required_for)
for rel in relations:
yield rel
def resolve_relation(self, name, node_ids, is_required_for):
"""Resolves the task relation.
:param name: the name of task
:param node_ids: the ID of nodes where need to search
:param is_required_for: means task from required_for section
"""
found = False
match_policy = NameMatchingPolicy.create(name)
for node_id in node_ids:
applied_tasks = set()
for task_name in self.tasks_per_node[node_id]:
if task_name == name:
# the simple case when name of current task
# is exact math to name of task that is search
found = True
yield {"name": task_name, "node_id": node_id}
continue
# at first get the original task name, actual
# when current task is part of chain
original_task = self.task_processor.get_origin(task_name)
if original_task in applied_tasks or \
not match_policy.match(original_task):
continue
found = True
applied_tasks.add(original_task)
if original_task is not task_name:
if is_required_for:
task_name_gen = self.task_processor.get_first_task_id
else:
task_name_gen = self.task_processor.get_last_task_id
task_name = task_name_gen(original_task)
yield {"name": task_name, "node_id": node_id}
if not found:
logger.warning(
"Dependency '%s' cannot be resolved: "
"no candidates in nodes '%s'.", name,
", ".join(six.moves.map(str, node_ids)))
@classmethod
def need_update_task(cls, tasks, task):
"""Checks that task shall overwrite existed one or should be added.
:param tasks: the current node tasks
:param task: the astute task object
:return True if task is not present or must be overwritten
otherwise False
"""
existed_task = tasks.get(task['id'])
if existed_task is None:
return True
if existed_task['type'] == task['type']:
return False
return task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped
@classmethod
def make_task_filter(cls, task_ids):
"""Makes task filter according to specified ids.
:param task_ids: the selected ids of tasks
:return: function that check task
"""
if task_ids is None:
return lambda _: True
if not isinstance(task_ids, set):
task_ids = set(task_ids)
return lambda task_id: task_id in task_ids
class TasksSerializer(object):
"""The deploy tasks serializer."""
def __init__(self, cluster, nodes):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
@classmethod
def serialize(cls, cluster, nodes, tasks):
"""Resolves roles and dependencies for tasks.
:param cluster: the cluster instance
:param nodes: the list of nodes
:param tasks: the list of tasks
:return: the list of serialized task per node
"""
serializer = cls(cluster, nodes)
serializer.resolve_nodes(add_plugin_deployment_hooks(tasks), nodes)
serializer.resolve_dependencies()
return dict((k, list(six.itervalues(v)))
for k, v in six.iteritems(serializer.tasks_per_node))
def resolve_nodes(self, tasks, nodes):
"""Resolves node roles in tasks.
:param tasks: the deployment tasks
:param nodes: the list of nodes to deploy
:return the mapping tasks per node
"""
tasks_mapping = dict()
tasks_groups = collections.defaultdict(set)
for task in tasks:
if task.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group:
tasks_for_role = task.get('tasks')
if tasks_for_role:
tasks_groups[tuple(task.get('role',
()))].update(tasks_for_role)
continue
tasks_mapping[task['id']] = task
self.process_task(task, nodes, lambda _: self.role_resolver)
self.expand_task_groups(tasks_groups, tasks_mapping)
# make sure that null node is present
self.tasks_per_node.setdefault(None, dict())
def process_task(self, task, nodes, resolver_factory):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param nodes: the list of nodes
:param resolver_factory: the factory creates role-resolver
"""
serializer_factory = self.task_serializer.get_stage_serializer(task)
task_serializer = serializer_factory(
task, self.cluster, nodes, role_resolver=resolver_factory(nodes))
# do not pass skipped attribute to astute
skipped = task.pop('skipped', False) or \
not task_serializer.should_execute()
for astute_task in self.task_processor.process_tasks(
task, task_serializer.serialize()):
# all skipped task shall have type skipped
if skipped:
astute_task['type'] = \
consts.ORCHESTRATOR_TASK_TYPES.skipped
for node_id in astute_task.pop('uids', ()):
node_tasks = self.tasks_per_node[node_id]
# de-duplication the tasks on node
if astute_task['id'] in node_tasks:
continue
node_tasks[astute_task['id']] = copy.deepcopy(astute_task)
def resolve_dependencies(self):
"""Resolves tasks dependencies."""
for node_id, tasks in six.iteritems(self.tasks_per_node):
for task in six.itervalues(tasks):
task['requires'] = list(
self.expand_dependencies(node_id, task.get('requires'),
False))
task['required_for'] = list(
self.expand_dependencies(node_id, task.get('required_for'),
True))
task['requires'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depends', None), False))
task['required_for'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depended-by', None), True))
task['requires'].extend(task.pop('requires_ex', ()))
task['required_for'].extend(task.pop('required_for_ex', ()))
def expand_task_groups(self, tasks_per_role, task_mapping):
"""Expand group of tasks.
:param tasks_per_role: the set of tasks per role
:param task_mapping: the mapping task id to task object
"""
for roles, task_ids in six.iteritems(tasks_per_role):
for task_id in task_ids:
try:
task = task_mapping[task_id]
except KeyError:
raise errors.InvalidData('Task %s cannot be resolved',
task_id)
for node_id in self.role_resolver.resolve(roles):
self.process_task(task, [node_id], NullResolver)
def expand_dependencies(self, node_id, dependencies, is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of dependencies on same node
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
# need to search dependencies on node and in sync points
node_ids = [node_id, None]
for name in dependencies:
for rel in self.resolve_relation(name, node_ids, is_required_for):
yield rel
def expand_cross_dependencies(self, node_id, dependencies,
is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of cross-node dependencies
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
for dep in dependencies:
roles = dep.get('role', consts.TASK_ROLES.all)
if roles == consts.TASK_ROLES.self:
node_ids = [node_id]
else:
node_ids = self.role_resolver.resolve(
roles, dep.get('policy', consts.NODE_RESOLVE_POLICY.all))
relations = self.resolve_relation(dep['name'], node_ids,
is_required_for)
for rel in relations:
yield rel
def resolve_relation(self, name, node_ids, is_required_for):
"""Resolves the task relation.
:param name: the name of task
:param node_ids: the ID of nodes where need to search
:param is_required_for: means task from required_for section
"""
found = False
match_policy = NameMatchingPolicy.create(name)
for node_id in node_ids:
applied_tasks = set()
for task_name in self.tasks_per_node[node_id]:
if task_name == name:
# the simple case when name of current task
# is exact math to name of task that is search
found = True
yield {"name": task_name, "node_id": node_id}
continue
# at first get the original task name, actual
# when current task is part of chain
original_task = self.task_processor.get_origin(task_name)
if original_task in applied_tasks or \
not match_policy.match(original_task):
continue
found = True
applied_tasks.add(original_task)
if original_task is not task_name:
if is_required_for:
task_name_gen = self.task_processor.get_first_task_id
else:
task_name_gen = self.task_processor.get_last_task_id
task_name = task_name_gen(original_task)
yield {"name": task_name, "node_id": node_id}
if not found:
logger.warning(
"Dependency '%s' cannot be resolved: "
"no candidates in nodes '%s'.", name,
", ".join(six.moves.map(str, node_ids)))
class DeploymentMultinodeSerializer(object):
nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
neutron_network_serializer = \
neutron_serializers.NeutronNetworkDeploymentSerializer
critical_roles = frozenset(('controller', 'ceph-osd', 'primary-mongo'))
def __init__(self, tasks_graph=None):
self.task_graph = tasks_graph
self.all_nodes = None
self.role_resolver = None
self.initialized = None
def initialize(self, cluster):
self.all_nodes = objects.Cluster.get_nodes_not_for_deletion(cluster)
self.role_resolver = RoleResolver(self.all_nodes)
self.initialized = cluster.id
def finalize(self):
self.all_nodes = None
self.role_resolver = None
self.initialized = None
def _ensure_initialized_for(self, cluster):
# TODO(bgaifullin) need to move initialize into __init__
if self.initialized != cluster.id:
self.initialize(cluster)
def serialize(self, cluster, nodes, ignore_customized=False):
"""Method generates facts which are passed to puppet."""
try:
self.initialize(cluster)
serialized_nodes = []
origin_nodes = []
customized_nodes = []
if ignore_customized:
origin_nodes = nodes
else:
for node in nodes:
if node.replaced_deployment_info:
customized_nodes.append(node)
else:
origin_nodes.append(node)
serialized_nodes.extend(
self.serialize_generated(cluster, origin_nodes)
)
serialized_nodes.extend(
self.serialize_customized(cluster, customized_nodes)
)
# NOTE(dshulyak) tasks should not be preserved from replaced
# deployment info, there is different mechanism to control
# changes in tasks introduced during granular deployment,
# and that mech should be used
self.set_tasks(serialized_nodes)
finally:
self.finalize()
return serialized_nodes
def serialize_generated(self, cluster, nodes):
nodes = self.serialize_nodes(nodes)
common_attrs = self.get_common_attrs(cluster)
self.set_deployment_priorities(nodes)
for node in nodes:
yield utils.dict_merge(node, common_attrs)
def serialize_customized(self, cluster, nodes):
for node in nodes:
for role_data in node.replaced_deployment_info:
yield role_data
def get_common_attrs(self, cluster):
"""Cluster attributes."""
# tests call this method directly.
# and we need this workaround to avoid refactoring a lot of tests.
self._ensure_initialized_for(cluster)
attrs = objects.Cluster.get_attributes(cluster)
attrs = objects.Attributes.merged_attrs_values(attrs)
attrs['deployment_mode'] = cluster.mode
attrs['deployment_id'] = cluster.id
attrs['openstack_version'] = cluster.release.version
attrs['fuel_version'] = cluster.fuel_version
attrs['nodes'] = self.node_list(self.all_nodes)
# Adding params to workloads_collector
if 'workloads_collector' not in attrs:
attrs['workloads_collector'] = {}
attrs['workloads_collector']['create_user'] = \
objects.MasterNodeSettings.must_send_stats()
username = attrs['workloads_collector'].pop('user', None)
attrs['workloads_collector']['username'] = username
if self.role_resolver.resolve(['cinder']):
attrs['use_cinder'] = True
net_serializer = self.get_net_provider_serializer(cluster)
net_common_attrs = net_serializer.get_common_attrs(cluster, attrs)
attrs = utils.dict_merge(attrs, net_common_attrs)
self.inject_list_of_plugins(attrs, cluster)
return attrs
@classmethod
def node_list(cls, nodes):
"""Generate nodes list. Represents as "nodes" parameter in facts."""
node_list = []
for node in nodes:
for role in objects.Node.all_roles(node):
node_list.append(cls.serialize_node_for_node_list(node, role))
return node_list
@classmethod
def serialize_node_for_node_list(cls, node, role):
return {
'uid': node.uid,
'fqdn': objects.Node.get_node_fqdn(node),
'name': objects.Node.get_slave_name(node),
'role': role}
# TODO(apopovych): we have more generical method 'filter_by_roles'
def by_role(self, nodes, role):
return filter(lambda node: node['role'] == role, nodes)
def not_roles(self, nodes, roles):
return filter(lambda node: node['role'] not in roles, nodes)
def serialize_nodes(self, nodes):
"""Serialize node for each role.
For example if node has two roles then
in orchestrator will be passed two serialized
nodes.
"""
serialized_nodes = []
for node in nodes:
for role in objects.Node.all_roles(node):
serialized_nodes.append(self.serialize_node(node, role))
return serialized_nodes
def serialize_node(self, node, role):
"""Serialize node, then it will be merged with common attributes."""
node_attrs = {
# Yes, uid is really should be a string
'uid': node.uid,
'fqdn': objects.Node.get_node_fqdn(node),
'status': node.status,
'role': role,
'vms_conf': node.vms_conf,
'fail_if_error': role in self.critical_roles,
# TODO(eli): need to remove, requried for the fake thread only
'online': node.online,
}
net_serializer = self.get_net_provider_serializer(node.cluster)
node_attrs.update(net_serializer.get_node_attrs(node))
node_attrs.update(net_serializer.network_ranges(node.group_id))
node_attrs.update(self.generate_test_vm_image_data(node))
return node_attrs
def generate_properties_arguments(self, properties_data):
"""build a string of properties from a key value hash"""
properties = []
for key, value in six.iteritems(properties_data):
properties.append('--property {key}={value}'.format(
key=key, value=value))
return ' '.join(properties)
def generate_test_vm_image_data(self, node):
# Instantiate all default values in dict.
image_data = {
'container_format': 'bare',
'public': 'true',
'disk_format': 'qcow2',
'img_name': 'TestVM',
'img_path': '',
'os_name': 'cirros',
'min_ram': 64,
'glance_properties': '',
'properties': {},
}
# Generate a right path to image.
c_attrs = node.cluster.attributes
if 'ubuntu' in c_attrs['generated']['cobbler']['profile']:
img_dir = '/usr/share/cirros-testvm/'
else:
img_dir = '/opt/vm/'
image_data['img_path'] = '{0}cirros-x86_64-disk.img'.format(img_dir)
properties_data = {}
# Alternate VMWare specific values.
if c_attrs['editable']['common']['libvirt_type']['value'] == 'vcenter':
image_data.update({
'disk_format': 'vmdk',
'img_path': '{0}cirros-i386-disk.vmdk'.format(img_dir),
})
properties_data = {
'vmware_disktype': 'sparse',
'vmware_adaptertype': 'lsiLogic',
'hypervisor_type': 'vmware'
}
# NOTE(aschultz): properties was added as part of N and should be
# used infavor of glance_properties
image_data['glance_properties'] = self.generate_properties_arguments(
properties_data)
image_data['properties'] = properties_data
return {'test_vm_image': image_data}
@classmethod
def get_net_provider_serializer(cls, cluster):
if cluster.net_provider == 'nova_network':
return cls.nova_network_serializer
else:
return cls.neutron_network_serializer
def filter_by_roles(self, nodes, roles):
return filter(
lambda node: node['role'] in roles, nodes)
def set_deployment_priorities(self, nodes):
if self.task_graph is not None:
self.task_graph.add_priorities(nodes)
def set_tasks(self, serialized_nodes):
if self.task_graph is not None:
for node in serialized_nodes:
node['tasks'] = self.task_graph.deploy_task_serialize(node)
def inject_list_of_plugins(self, attributes, cluster):
"""Added information about plugins to serialized attributes.
:param attributes: the serialized attributes
:param cluster: the cluster object
"""
plugins = objects.ClusterPlugin.get_enabled(cluster.id)
attributes['plugins'] = [
self.serialize_plugin(cluster, p) for p in plugins
]
@classmethod
def serialize_plugin(cls, cluster, plugin):
"""Gets plugin information to include into serialized attributes.
:param cluster: the cluster object
:param plugin: the plugin object
"""
return plugin['name']
class TasksSerializer(object):
"""The deploy tasks serializer."""
def __init__(self,
cluster,
nodes,
affected_nodes=None,
task_ids=None,
events=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param affected_nodes: the list of nodes, that affected by deployment
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:param events: the events (see TaskEvents)
"""
if affected_nodes:
self.affected_node_ids = frozenset(n.uid for n in affected_nodes)
self.deployment_nodes = copy.copy(nodes)
self.deployment_nodes.extend(affected_nodes)
else:
self.deployment_nodes = nodes
self.affected_node_ids = frozenset()
self.cluster = cluster
self.role_resolver = RoleResolver(self.deployment_nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_connections = collections.defaultdict(dict)
self.tasks_dictionary = dict()
self.task_filter = self.make_task_filter(task_ids)
self.events = events
@classmethod
def serialize(cls,
cluster,
nodes,
tasks,
affected_nodes=None,
task_ids=None,
events=None):
"""Resolves roles and dependencies for tasks.
:param cluster: the cluster instance
:param nodes: the list of nodes
:param affected_nodes: the list of nodes, that affected by deployment
:param tasks: the list of tasks
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:param events: the events (see TaskEvents)
:return: the list of serialized task per node
"""
serializer = cls(cluster, nodes, affected_nodes, task_ids, events)
serializer.resolve_nodes(add_plugin_deployment_hooks(tasks))
serializer.resolve_dependencies()
tasks_dictionary = serializer.tasks_dictionary
tasks_connections = serializer.tasks_connections
for node_id in tasks_connections:
tasks_connections[node_id] = list(
six.itervalues(tasks_connections[node_id]))
return tasks_dictionary, tasks_connections
def resolve_nodes(self, tasks):
"""Resolves node roles in tasks.
:param tasks: the deployment tasks
:return the mapping tasks per node
"""
tasks_mapping = dict()
groups = list()
for task in tasks:
if task.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group:
groups.append(task)
else:
tasks_mapping[task['id']] = task
skip = not self.task_filter(task['id'])
self.process_task(task, self.role_resolver, skip)
self.expand_task_groups(groups, tasks_mapping)
# make sure that null node is present
self.tasks_connections.setdefault(None, dict())
def process_task(self, task, role_resolver, skip=False):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param role_resolver: the role resolver
:param skip: make the task as skipped
"""
serializer_factory = self.task_serializer.get_stage_serializer(task)
task_serializer = serializer_factory(task,
self.cluster,
self.deployment_nodes,
role_resolver=role_resolver)
skipped = skip or not task_serializer.should_execute()
force = self.events and self.events.check_subscription(task)
if skipped and not force:
# Do not call real serializer if it should be skipped
task_serializer = NoopSerializer(task,
self.cluster,
self.deployment_nodes,
role_resolver=role_resolver)
serialised_tasks = self.task_processor.process_tasks(
task, task_serializer.serialize())
for serialized in serialised_tasks:
# all skipped task shall have type skipped
# do not exclude them from graph to keep connections between nodes
if skipped:
task_type = consts.ORCHESTRATOR_TASK_TYPES.skipped
else:
task_type = serialized['type']
task_relations = {
'id': serialized['id'],
'type': task_type,
'requires': serialized.pop('requires', []),
'required_for': serialized.pop('required_for', []),
'cross_depends': serialized.pop('cross_depends', []),
'cross_depended_by': serialized.pop('cross_depended_by', []),
'requires_ex': serialized.pop('requires_ex', []),
'required_for_ex': serialized.pop('required_for_ex', [])
}
node_ids = serialized.pop('uids', ())
self.tasks_dictionary[serialized['id']] = serialized
for node_id in node_ids:
node_task = task_relations.copy()
if not force and node_id in self.affected_node_ids:
node_task['type'] = consts.ORCHESTRATOR_TASK_TYPES.skipped
node_tasks = self.tasks_connections[node_id]
# de-duplication the tasks on node
# since task can be added after expand group need to
# overwrite if existed task is skipped and new is not skipped.
if self.need_update_task(node_tasks, node_task):
node_tasks[serialized['id']] = node_task
def resolve_dependencies(self):
"""Resolves tasks dependencies."""
for node_id, tasks in six.iteritems(self.tasks_connections):
for task in six.itervalues(tasks):
requires = set(
self.expand_dependencies(
node_id, task.pop('requires', None),
self.task_processor.get_last_task_id))
requires.update(
self.expand_cross_dependencies(
task['id'],
node_id,
task.pop('cross_depends', None),
self.task_processor.get_last_task_id,
))
requires.update(task.pop('requires_ex', ()))
required_for = set(
self.expand_dependencies(
node_id, task.pop('required_for', None),
self.task_processor.get_first_task_id))
required_for.update(
self.expand_cross_dependencies(
task['id'], node_id, task.pop('cross_depended_by',
None),
self.task_processor.get_first_task_id))
required_for.update(task.pop('required_for_ex', ()))
# render
if requires:
task['requires'] = [
dict(six.moves.zip(('name', 'node_id'), r))
for r in requires
]
if required_for:
task['required_for'] = [
dict(six.moves.zip(('name', 'node_id'), r))
for r in required_for
]
def expand_task_groups(self, groups, task_mapping):
"""Expand group of tasks.
:param groups: the all tasks with type 'group'
:param task_mapping: the mapping task id to task object
"""
for task in groups:
skipped = not self.task_filter(task['id'])
node_ids = self.role_resolver.resolve(task.get('role', ()))
for sub_task_id in task.get('tasks', ()):
try:
sub_task = task_mapping[sub_task_id]
except KeyError:
raise errors.InvalidData('Task %s cannot be resolved',
sub_task_id)
# if group is not excluded, all task should be run as well
# otherwise check each task individually
self.process_task(sub_task,
NullResolver(node_ids),
skip=skipped
and not self.task_filter(sub_task_id))
def expand_dependencies(self, node_id, dependencies, task_resolver):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of dependencies on same node
:param task_resolver: the task name resolver
"""
if not dependencies:
return
# need to search dependencies on node and in sync points
node_ids = [node_id, None]
for name in dependencies:
relations = self.resolve_relation(name, node_ids, task_resolver,
[])
for rel in relations:
yield rel
def expand_cross_dependencies(self, task_id, node_id, dependencies,
task_resolver):
"""Expands task dependencies on same node.
:param task_id: the ID of the task, for which we resolve the dependency
:param node_id: the ID of target node
:param dependencies: the list of cross-node dependencies
:param task_resolver: the task name resolver
"""
if not dependencies:
return
for dep in dependencies:
roles = dep.get('role', consts.TASK_ROLES.all)
if roles == consts.TASK_ROLES.self:
node_ids = [node_id]
excludes = []
else:
node_ids = self.role_resolver.resolve(
roles, dep.get('policy', consts.NODE_RESOLVE_POLICY.all))
excludes = [(node_id, task_id)]
relations = self.resolve_relation(dep['name'], node_ids,
task_resolver, excludes)
for rel in relations:
yield rel
def resolve_relation(self, name, node_ids, task_resolver, excludes):
"""Resolves the task relation.
:param name: the name of task
:param node_ids: the ID of nodes where need to search
:param task_resolver: the task name resolver
:param excludes: the nodes to exclude
"""
match_policy = NameMatchingPolicy.create(name)
for node_id in node_ids:
applied_tasks = set()
for task_name in self.tasks_connections[node_id]:
if (node_id, task_name) in excludes:
continue
if task_name == name:
# the simple case when name of current task
# is exact math to name of task that is search
yield task_name, node_id
continue
# at first get the original task name, actual
# when current task is part of chain
original_task = self.task_processor.get_origin(task_name)
if original_task in applied_tasks or \
not match_policy.match(original_task):
continue
applied_tasks.add(original_task)
if original_task is not task_name:
task_name = task_resolver(original_task)
yield task_name, node_id
@classmethod
def need_update_task(cls, tasks, task):
"""Checks that task shall overwrite existed one or should be added.
:param tasks: the current node tasks
:param task: the astute task object
:return True if task is not present or must be overwritten
otherwise False
"""
existed_task = tasks.get(task['id'])
if existed_task is None:
return True
if existed_task['type'] == task['type']:
return False
return task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped
@classmethod
def make_task_filter(cls, task_ids):
"""Makes task filter according to specified ids.
:param task_ids: the selected ids of tasks
:return: function that check task
"""
if not task_ids:
return lambda _: True
if not isinstance(task_ids, set):
task_ids = set(task_ids)
return lambda task_id: task_id in task_ids
class TasksSerializer(object):
"""The deploy tasks serializer."""
def __init__(self, cluster, nodes):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
"""
self.cluster = cluster
self.role_resolver = RoleResolver(nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_per_node = collections.defaultdict(dict)
@classmethod
def serialize(cls, cluster, nodes, tasks):
"""Resolves roles and dependencies for tasks.
:param cluster: the cluster instance
:param nodes: the list of nodes
:param tasks: the list of tasks
:return: the list of serialized task per node
"""
serializer = cls(cluster, nodes)
serializer.resolve_nodes(add_plugin_deployment_hooks(tasks), nodes)
serializer.resolve_dependencies()
return dict(
(k, list(six.itervalues(v)))
for k, v in six.iteritems(serializer.tasks_per_node)
)
def resolve_nodes(self, tasks, nodes):
"""Resolves node roles in tasks.
:param tasks: the deployment tasks
:param nodes: the list of nodes to deploy
:return the mapping tasks per node
"""
tasks_mapping = dict()
tasks_groups = collections.defaultdict(set)
for task in tasks:
if task.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group:
tasks_for_role = task.get('tasks')
if tasks_for_role:
tasks_groups[tuple(task.get('role', ()))].update(
tasks_for_role
)
continue
tasks_mapping[task['id']] = task
self.process_task(task, nodes, lambda _: self.role_resolver)
self.expand_task_groups(tasks_groups, tasks_mapping)
# make sure that null node is present
self.tasks_per_node.setdefault(None, dict())
def process_task(self, task, nodes, resolver_factory):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param nodes: the list of nodes
:param resolver_factory: the factory creates role-resolver
"""
serializer_factory = self.task_serializer.get_stage_serializer(
task
)
task_serializer = serializer_factory(
task, self.cluster, nodes, role_resolver=resolver_factory(nodes)
)
# do not pass skipped attribute to astute
skipped = task.pop('skipped', False) or \
not task_serializer.should_execute()
for astute_task in self.task_processor.process_tasks(
task, task_serializer.serialize()):
# all skipped task shall have type skipped
if skipped:
astute_task['type'] = \
consts.ORCHESTRATOR_TASK_TYPES.skipped
for node_id in astute_task.pop('uids', ()):
node_tasks = self.tasks_per_node[node_id]
# de-duplication the tasks on node
if astute_task['id'] in node_tasks:
continue
node_tasks[astute_task['id']] = copy.deepcopy(astute_task)
def resolve_dependencies(self):
"""Resolves tasks dependencies."""
for node_id, tasks in six.iteritems(self.tasks_per_node):
for task in six.itervalues(tasks):
task['requires'] = list(
self.expand_dependencies(
node_id, task.get('requires'), False
)
)
task['required_for'] = list(
self.expand_dependencies(
node_id, task.get('required_for'), True
)
)
task['requires'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depends', None), False
)
)
task['required_for'].extend(
self.expand_cross_dependencies(
node_id, task.pop('cross-depended-by', None), True
)
)
task['requires'].extend(task.pop('requires_ex', ()))
task['required_for'].extend(task.pop('required_for_ex', ()))
def expand_task_groups(self, tasks_per_role, task_mapping):
"""Expand group of tasks.
:param tasks_per_role: the set of tasks per role
:param task_mapping: the mapping task id to task object
"""
for roles, task_ids in six.iteritems(tasks_per_role):
for task_id in task_ids:
try:
task = task_mapping[task_id]
except KeyError:
raise errors.InvalidData(
'Task %s cannot be resolved', task_id
)
for node_id in self.role_resolver.resolve(roles):
self.process_task(task, [node_id], NullResolver)
def expand_dependencies(self, node_id, dependencies, is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of dependencies on same node
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
# need to search dependencies on node and in sync points
node_ids = [node_id, None]
for name in dependencies:
for rel in self.resolve_relation(name, node_ids, is_required_for):
yield rel
def expand_cross_dependencies(
self, node_id, dependencies, is_required_for):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of cross-node dependencies
:param is_required_for: means task from required_for section
"""
if not dependencies:
return
for dep in dependencies:
roles = dep.get('role', consts.TASK_ROLES.all)
if roles == consts.TASK_ROLES.self:
node_ids = [node_id]
else:
node_ids = self.role_resolver.resolve(
roles, dep.get('policy', consts.NODE_RESOLVE_POLICY.all)
)
relations = self.resolve_relation(
dep['name'], node_ids, is_required_for
)
for rel in relations:
yield rel
def resolve_relation(self, name, node_ids, is_required_for):
"""Resolves the task relation.
:param name: the name of task
:param node_ids: the ID of nodes where need to search
:param is_required_for: means task from required_for section
"""
found = False
match_policy = NameMatchingPolicy.create(name)
for node_id in node_ids:
applied_tasks = set()
for task_name in self.tasks_per_node[node_id]:
if task_name == name:
# the simple case when name of current task
# is exact math to name of task that is search
found = True
yield {"name": task_name, "node_id": node_id}
continue
# at first get the original task name, actual
# when current task is part of chain
original_task = self.task_processor.get_origin(task_name)
if original_task in applied_tasks or \
not match_policy.match(original_task):
continue
found = True
applied_tasks.add(original_task)
if original_task is not task_name:
if is_required_for:
task_name_gen = self.task_processor.get_first_task_id
else:
task_name_gen = self.task_processor.get_last_task_id
task_name = task_name_gen(original_task)
yield {"name": task_name, "node_id": node_id}
if not found:
logger.warning(
"Dependency '%s' cannot be resolved: "
"no candidates in nodes '%s'.",
name, ", ".join(six.moves.map(str, node_ids))
)
class TasksSerializer(object):
"""The deploy tasks serializer."""
def __init__(self, cluster, nodes,
affected_nodes=None, task_ids=None, events=None):
"""Initializes.
:param cluster: Cluster instance
:param nodes: the sequence of nodes for deploy
:param affected_nodes: the list of nodes, that affected by deployment
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:param events: the events (see TaskEvents)
"""
if affected_nodes:
self.affected_node_ids = frozenset(n.uid for n in affected_nodes)
self.deployment_nodes = copy.copy(nodes)
self.deployment_nodes.extend(affected_nodes)
else:
self.deployment_nodes = nodes
self.affected_node_ids = frozenset()
self.cluster = cluster
self.role_resolver = RoleResolver(self.deployment_nodes)
self.task_serializer = DeployTaskSerializer()
self.task_processor = TaskProcessor()
self.tasks_connections = collections.defaultdict(dict)
self.tasks_dictionary = dict()
self.task_filter = self.make_task_filter(task_ids)
self.events = events
@classmethod
def serialize(cls, cluster, nodes, tasks,
affected_nodes=None, task_ids=None, events=None):
"""Resolves roles and dependencies for tasks.
:param cluster: the cluster instance
:param nodes: the list of nodes
:param affected_nodes: the list of nodes, that affected by deployment
:param tasks: the list of tasks
:param task_ids: Only specified tasks will be executed,
If None, all tasks will be executed
:param events: the events (see TaskEvents)
:return: the list of serialized task per node
"""
serializer = cls(cluster, nodes, affected_nodes, task_ids, events)
serializer.resolve_nodes(add_plugin_deployment_hooks(tasks))
serializer.resolve_dependencies()
tasks_dictionary = serializer.tasks_dictionary
tasks_connections = serializer.tasks_connections
for node_id in tasks_connections:
tasks_connections[node_id] = list(
six.itervalues(tasks_connections[node_id])
)
return tasks_dictionary, tasks_connections
def resolve_nodes(self, tasks):
"""Resolves node roles in tasks.
:param tasks: the deployment tasks
:return the mapping tasks per node
"""
tasks_mapping = dict()
groups = list()
for task in tasks:
if task.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group:
groups.append(task)
else:
tasks_mapping[task['id']] = task
skip = not self.task_filter(task['id'])
self.process_task(task, self.role_resolver, skip)
self.expand_task_groups(groups, tasks_mapping)
# make sure that null node is present
self.tasks_connections.setdefault(None, dict())
def process_task(self, task, role_resolver, skip=False):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param role_resolver: the role resolver
:param skip: make the task as skipped
"""
serializer_factory = self.task_serializer.get_stage_serializer(
task
)
task_serializer = serializer_factory(
task, self.cluster, self.deployment_nodes,
role_resolver=role_resolver
)
skipped = skip or not task_serializer.should_execute()
force = self.events and self.events.check_subscription(task)
if skipped and not force:
# Do not call real serializer if it should be skipped
task_serializer = NoopSerializer(
task, self.cluster, self.deployment_nodes,
role_resolver=role_resolver
)
serialised_tasks = self.task_processor.process_tasks(
task, task_serializer.serialize()
)
for serialized in serialised_tasks:
# all skipped task shall have type skipped
# do not exclude them from graph to keep connections between nodes
if skipped:
task_type = consts.ORCHESTRATOR_TASK_TYPES.skipped
else:
task_type = serialized['type']
task_relations = {
'id': serialized['id'],
'type': task_type,
'requires': serialized.pop('requires', []),
'required_for': serialized.pop('required_for', []),
'cross_depends': serialized.pop('cross_depends', []),
'cross_depended_by': serialized.pop('cross_depended_by', []),
'requires_ex': serialized.pop('requires_ex', []),
'required_for_ex': serialized.pop('required_for_ex', [])
}
node_ids = serialized.pop('uids', ())
self.tasks_dictionary[serialized['id']] = serialized
for node_id in node_ids:
node_task = task_relations.copy()
if not force and node_id in self.affected_node_ids:
node_task['type'] = consts.ORCHESTRATOR_TASK_TYPES.skipped
node_tasks = self.tasks_connections[node_id]
# de-duplication the tasks on node
# since task can be added after expand group need to
# overwrite if existed task is skipped and new is not skipped.
if self.need_update_task(node_tasks, node_task):
node_tasks[serialized['id']] = node_task
def resolve_dependencies(self):
"""Resolves tasks dependencies."""
for node_id, tasks in six.iteritems(self.tasks_connections):
for task in six.itervalues(tasks):
requires = set(self.expand_dependencies(
node_id, task.pop('requires', None),
self.task_processor.get_last_task_id
))
requires.update(self.expand_cross_dependencies(
task['id'], node_id, task.pop('cross_depends', None),
self.task_processor.get_last_task_id,
))
requires.update(task.pop('requires_ex', ()))
required_for = set(self.expand_dependencies(
node_id, task.pop('required_for', None),
self.task_processor.get_first_task_id
))
required_for.update(self.expand_cross_dependencies(
task['id'], node_id, task.pop('cross_depended_by', None),
self.task_processor.get_first_task_id
))
required_for.update(task.pop('required_for_ex', ()))
# render
if requires:
task['requires'] = [
dict(six.moves.zip(('name', 'node_id'), r))
for r in requires
]
if required_for:
task['required_for'] = [
dict(six.moves.zip(('name', 'node_id'), r))
for r in required_for
]
def expand_task_groups(self, groups, task_mapping):
"""Expand group of tasks.
:param groups: the all tasks with type 'group'
:param task_mapping: the mapping task id to task object
"""
for task in groups:
skipped = not self.task_filter(task['id'])
node_ids = self.role_resolver.resolve(task.get('role', ()))
for sub_task_id in task.get('tasks', ()):
try:
sub_task = task_mapping[sub_task_id]
except KeyError:
raise errors.InvalidData(
'Task %s cannot be resolved', sub_task_id
)
# if group is not excluded, all task should be run as well
# otherwise check each task individually
self.process_task(
sub_task, NullResolver(node_ids),
skip=skipped and not self.task_filter(sub_task_id)
)
def expand_dependencies(self, node_id, dependencies, task_resolver):
"""Expands task dependencies on same node.
:param node_id: the ID of target node
:param dependencies: the list of dependencies on same node
:param task_resolver: the task name resolver
"""
if not dependencies:
return
# need to search dependencies on node and in sync points
node_ids = [node_id, None]
for name in dependencies:
relations = self.resolve_relation(
name, node_ids, task_resolver, []
)
for rel in relations:
yield rel
def expand_cross_dependencies(
self, task_id, node_id, dependencies, task_resolver):
"""Expands task dependencies on same node.
:param task_id: the ID of the task, for which we resolve the dependency
:param node_id: the ID of target node
:param dependencies: the list of cross-node dependencies
:param task_resolver: the task name resolver
"""
if not dependencies:
return
for dep in dependencies:
roles = dep.get('role', consts.TASK_ROLES.all)
if roles == consts.TASK_ROLES.self:
node_ids = [node_id]
excludes = []
else:
node_ids = self.role_resolver.resolve(
roles, dep.get('policy', consts.NODE_RESOLVE_POLICY.all)
)
excludes = [(node_id, task_id)]
relations = self.resolve_relation(
dep['name'], node_ids, task_resolver, excludes
)
for rel in relations:
yield rel
def resolve_relation(self, name, node_ids, task_resolver, excludes):
"""Resolves the task relation.
:param name: the name of task
:param node_ids: the ID of nodes where need to search
:param task_resolver: the task name resolver
:param excludes: the nodes to exclude
"""
match_policy = NameMatchingPolicy.create(name)
for node_id in node_ids:
applied_tasks = set()
for task_name in self.tasks_connections[node_id]:
if (node_id, task_name) in excludes:
continue
if task_name == name:
# the simple case when name of current task
# is exact math to name of task that is search
yield task_name, node_id
continue
# at first get the original task name, actual
# when current task is part of chain
original_task = self.task_processor.get_origin(task_name)
if original_task in applied_tasks or \
not match_policy.match(original_task):
continue
applied_tasks.add(original_task)
if original_task is not task_name:
task_name = task_resolver(original_task)
yield task_name, node_id
@classmethod
def need_update_task(cls, tasks, task):
"""Checks that task shall overwrite existed one or should be added.
:param tasks: the current node tasks
:param task: the astute task object
:return True if task is not present or must be overwritten
otherwise False
"""
existed_task = tasks.get(task['id'])
if existed_task is None:
return True
if existed_task['type'] == task['type']:
return False
return task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped
@classmethod
def make_task_filter(cls, task_ids):
"""Makes task filter according to specified ids.
:param task_ids: the selected ids of tasks
:return: function that check task
"""
if not task_ids:
return lambda _: True
if not isinstance(task_ids, set):
task_ids = set(task_ids)
return lambda task_id: task_id in task_ids
