def update_discovered_state(new_discovered_state, temp_dir, discovery_id,
changed_subtree_path, subtree):
discovered_state_file = os.path.join(
temp_dir, 'project', f'discovered_state_{discovery_id}.yml')
if os.path.exists(discovered_state_file):
with open(discovered_state_file) as f:
discovered_subtree_state = yaml.safe_load(f.read())
print(changed_subtree_path)
print(
yaml.safe_dump(discovered_subtree_state,
default_flow_style=False))
print(yaml.safe_dump(subtree, default_flow_style=False))
# List case
match_list = re.match(r"(.*)\[(\d+)\]$", changed_subtree_path)
if match_list:
parent_path = match_list.groups()[0]
index = int(match_list.groups()[1])
extract(new_discovered_state,
parent_path)[index] = discovered_subtree_state
# Dict case
match_dict = re.match(r"(.*)\['(\S+)'\]$", changed_subtree_path)
if match_dict and not match_list:
parent_path = match_dict.groups()[0]
index = match_dict.groups()[1]
extract(new_discovered_state,
parent_path)[index] = discovered_subtree_state
if not match_dict and not match_list:
assert False, f"type of changed_subtree_path not supported {changed_subtree_path}"
print(yaml.safe_dump(new_discovered_state, default_flow_style=False))
def destructure_vars(rule, subtree):
destructured_vars = {}
for name, extract_path in rule.get('vars', {}).items():
destructured_vars[name] = extract(subtree, extract_path)
return destructured_vars
def _search(
self,
fields: Dict[str, EntityID],
models: OptionalModelOrModels[Entity] = None,
) -> List[Entity]:
"""Get the entities whose attributes match one or several conditions.
Args:
models: Entity class or classes to obtain.
fields: Dictionary with the {key}:{value} to search.
Returns:
entities: List of Entity object that matches the search criteria.
Raises:
EntityNotFoundError: If the entities are not found.
"""
models = self._build_models(models)
if len(models) == 1:
models = models[0]
all_entities: List[Entity] = self.all(models)
entities_dict = {entity.id_: entity for entity in all_entities}
entity_attributes = {
entity.id_: entity.dict()
for entity in all_entities
}
for key, value in fields.items():
# Get entities that have the value `value`
entities_with_value = entity_attributes | grep(
value, use_regexp=True, strict_checking=False)
matching_entity_attributes = {}
try:
entities_with_value["matched_values"]
except KeyError as error:
raise self._model_not_found(
models,
f" that match the search filter {fields}") from error
for path in entities_with_value["matched_values"]:
entity_id = re.sub(r"root\['?(.*?)'?\]\[.*", r"\1", path)
# Convert int ids from str to int
try:
entity_id = int(entity_id)
except ValueError:
entity_id = re.sub(r"'(.*)'", r"\1", entity_id)
# Add the entity to the matching ones only if the value is of the
# attribute `key`.
if re.match(rf"root\['?{entity_id}'?\]\['{key}'\]", path):
matching_entity_attributes[entity_id] = extract(
entity_attributes, f"root[{entity_id}]")
entity_attributes = matching_entity_attributes
entities = [entities_dict[key] for key in entity_attributes.keys()]
return entities
def test_extract_and_modify():
t1 = yaml.safe_load('''
routers:
- name: R1
- name: R2
''')
assert extract(t1, "root['routers'][0]") == {'name': 'R1'}
extract(t1, "root['routers']")[0] = {'name': 'R3'}
assert extract(t1, "root['routers'][0]") == {'name': 'R3'}
# find parent and index of node
# List case
match = re.match(r"(.*)\[(\d+)\]$", "root['routers'][0]")
assert match.groups()[0] == "root['routers']"
assert match.groups()[1] == "0"
parent = match.groups()[0]
index = int(match.groups()[1])
extract(t1, parent)[index] = {'name': 'R4'}
assert extract(t1, "root['routers']") == [{'name': 'R4'}, {'name': 'R2'}]
# Dictionary case
match = re.match(r"(.*)\['(\S+)'\]$", "root['routers']")
assert match.groups()[0] == "root"
assert match.groups()[1] == "routers"
parent = match.groups()[0]
index = match.groups()[1]
extract(t1, parent)[index] = [{'name': 'R5'}, {'name': 'R2'}]
assert extract(t1, "root") == {'routers': [{'name': 'R5'}, {'name': 'R2'}]}
def get_rule_action_subtree(matching_rule, current_desired_state,
new_desired_state):
change_type, rule, match, value = matching_rule
print('change_type', change_type)
print('rule', rule)
print('match', match)
print('value', value)
changed_subtree_path = match.groups()[0]
print('changed_subtree_path', changed_subtree_path)
try:
new_subtree = extract(new_desired_state, changed_subtree_path)
new_subtree_missing = False
except (KeyError, IndexError, TypeError):
new_subtree_missing = True
try:
old_subtree = extract(current_desired_state, changed_subtree_path)
old_subtree_missing = False
except (KeyError, IndexError, TypeError):
old_subtree_missing = True
print('new_subtree_missing', new_subtree_missing)
print('old_subtree_missing', old_subtree_missing)
if change_type == 'iterable_item_added':
action = Action.CREATE
subtree = new_subtree
elif change_type == 'iterable_item_removed':
action = Action.DELETE
subtree = old_subtree
elif new_subtree_missing is False and old_subtree_missing is False:
action = Action.UPDATE
subtree = new_subtree
elif new_subtree_missing and old_subtree_missing is False:
action = Action.DELETE
subtree = old_subtree
elif old_subtree_missing and new_subtree_missing is False:
action = Action.CREATE
subtree = new_subtree
else:
assert False, "Logic bug"
print('action', action)
return action, subtree
def _search(
self,
fields: Dict[str, EntityID],
model: Type[EntityT],
) -> List[EntityT]:
"""Get the entities whose attributes match one or several conditions.
Particular implementation of the database adapter.
Args:
model: Entity class to obtain.
fields: Dictionary with the {key}:{value} to search.
Returns:
entities: List of Entity object that matches the search criteria.
"""
all_entities = self.all(model)
entities_dict = {entity.id_: entity for entity in all_entities}
entity_attributes = {
entity.id_: entity.dict()
for entity in all_entities
}
for key, value in fields.items():
# Get entities that have the value `value`
entities_with_value = entity_attributes | grep(
value, use_regexp=True, strict_checking=False)
matching_entity_attributes = {}
try:
entities_with_value["matched_values"]
except KeyError:
return []
for path in entities_with_value["matched_values"]:
entity_id = re.sub(r"root\['?(.*?)'?\]\[.*", r"\1", path)
# Convert int ids from str to int
try:
# ignore: waiting for ADR-006 to be resolved
entity_id = int(entity_id) # type: ignore
except ValueError:
entity_id = re.sub(r"'(.*)'", r"\1", entity_id)
# Add the entity to the matching ones only if the value is of the
# attribute `key`.
if re.match(rf"root\['?{entity_id}'?\]\['{key}'\]", path):
matching_entity_attributes[entity_id] = extract(
entity_attributes, f"root[{entity_id}]")
# ignore: waiting for ADR-006 to be resolved
entity_attributes = matching_entity_attributes # type: ignore
entities = [entities_dict[key] for key in entity_attributes.keys()]
return entities
def select_rules_recursive(diff, rules, current_desired_state,
new_desired_state):
matching_rules = []
matchers = [(make_matcher(build_rule_selector(rule['rule_selector'])),
rule) for rule in rules]
for key, value in diff.get('values_changed', {}).items():
for (matcher, rule) in matchers:
match = re.match(matcher, key)
if match:
matching_rules.append(('values_changed', rule, match, value))
for item in diff.get('dictionary_item_added', []):
for (matcher, rule) in matchers:
match = re.match(matcher, item)
if match:
matching_rules.append(
('dictionary_item_added', rule, match, None))
new_subtree = extract(new_desired_state, item)
select_rules_recursive_helper(diff, matchers, matching_rules, item,
new_subtree)
for item in diff.get('dictionary_item_removed', []):
for (matcher, rule) in matchers:
match = re.match(matcher, item)
if match:
matching_rules.append(
('dictionary_item_removed', rule, match, None))
for item in diff.get('iterable_item_added', []):
print(item)
for (matcher, rule) in matchers:
match = re.match(matcher, item)
if match:
matching_rules.append(
('iterable_item_added', rule, match, None))
for item in diff.get('iterable_item_removed', []):
print(item)
for (matcher, rule) in matchers:
match = re.match(matcher, item)
if match:
matching_rules.append(
('iterable_item_removed', rule, match, None))
for key, value in diff.get('type_changes', {}).items():
# Handles case in YAML where an empty list defaults to None type
if value.get('old_type') == type(None) and value.get(
'new_type') == list:
# Add a single new element to the key
# TODO: this should probably loop over all the new elements in the list not just one
key += '[0]'
elif value.get('old_type') == list and value.get('new_type') == type(
None):
# Add a single new element to the key
key += '[0]'
for (matcher, rule) in matchers:
match = re.match(matcher, key)
if match:
matching_rules.append(('type_changes', rule, match, None))
# Handles case in YAML where an empty dict defaults to None type on the old state
if value.get('old_type') == type(None) and value.get(
'new_type') == dict:
# Try the matcher against all the keys in the dict
for dict_key in value.get('new_value').keys():
new_key = f"{key}['{dict_key}']"
for (matcher, rule) in matchers:
match = re.match(matcher, new_key)
if match:
matching_rules.append(
('type_changes', rule, match, None))
select_rules_recursive_helper(diff, matchers, matching_rules, key,
value.get('new_value'))
# Handles case in YAML where an empty dict defaults to None type on the new state
if value.get('old_type') == dict and value.get('new_type') == type(
None):
# Try the matcher against all the keys in the dict
for dict_key in value.get('old_value').keys():
old_key = f"{key}['{dict_key}']"
for (matcher, rule) in matchers:
match = re.match(matcher, old_key)
if match:
matching_rules.append(
('type_changes', rule, match, None))
select_rules_recursive_helper(diff, matchers, matching_rules, key,
value.get('old_value'))
return matching_rules
def desired_state_discovery(monitor, secrets, project_src,
current_desired_state, new_desired_state,
ran_rules, inventory, explain):
# Discovers the state of a subset of a system
diff = DeepDiff(current_desired_state,
new_desired_state,
ignore_order=True)
# deep copy
new_discovered_state = yaml.safe_load(yaml.safe_dump(new_desired_state))
plays = []
destructured_vars_list = []
discovered_rules = []
for discovery_id, (rule, changed_subtree_path, subtree,
inventory_name) in enumerate(ran_rules):
# Experiment: Build the vars using destructuring
destructured_vars = {}
for name, extract_path in rule.get('vars', {}).items():
destructured_vars[name] = extract(subtree, extract_path)
# Experiment: Make the subtree available as node
destructured_vars['node'] = subtree
destructured_vars['discovery_id'] = discovery_id
print('destructured_vars', destructured_vars)
# Build a play using tasks or role from rule
play = {
'name':
f'discovery for {inventory_name} discovery_id {discovery_id}',
'hosts': inventory_name,
'gather_facts': False,
'tasks': []
}
if 'tasks' in rule.get(ACTION_RULES[Action.RETRIEVE], {}):
play['tasks'].append({
'include_tasks': {
'file':
find_tasks(
rule.get(ACTION_RULES[Action.RETRIEVE]).get('tasks'))
},
'name': 'include retrieve'
})
print(play)
plays.append(play)
destructured_vars_list.append(destructured_vars)
discovered_rules.append(
[discovery_id, changed_subtree_path, subtree])
if not plays:
return new_discovered_state
def runner_process_message(data):
monitor.stream.put_message(Stdout(0, now(), data.get('stdout', '')))
runner = PlaybookRunner(runner_process_message, new_desired_state, diff,
destructured_vars_list, plays, secrets,
project_src, inventory)
result = runner.run()
if result:
for discovery_id, changed_subtree_path, subtree in discovered_rules:
update_discovered_state(new_discovered_state, runner.temp_dir,
discovery_id, changed_subtree_path,
subtree)
return new_discovered_state
def desired_state_diff(monitor, secrets, project_src, current_desired_state,
new_desired_state, rules, inventory, explain):
'''
desired_state_diff creates playbooks and runs them with ansible-runner to implement the differences
between two version of state: current_desired_state and new_desired_state.
'''
# Find the difference between states
diff = DeepDiff(current_desired_state,
new_desired_state,
ignore_order=True)
print(diff)
# Find matching rules
matching_rules = select_rules_recursive(diff, rules['rules'],
current_desired_state,
new_desired_state)
if explain:
print('matching_rules')
pprint(matching_rules)
dedup_matching_rules = deduplicate_rules(matching_rules)
if explain:
print('dedup_matching_rules:')
pprint(dedup_matching_rules)
# Build up the set of ansible-runner executions to implement the changes using the rules
ran_rules = []
plays = []
destructured_vars_list = []
for matching_rule in dedup_matching_rules:
change_type, rule, match, value = matching_rule
changed_subtree_path = match.groups()[0]
action, subtree = get_rule_action_subtree(matching_rule,
current_desired_state,
new_desired_state)
print('action', action)
print('rule action', rule.get(ACTION_RULES[action]))
# Experiment: Build the vars using destructuring
destructured_vars = {}
for name, extract_path in rule.get('vars', {}).items():
destructured_vars[name] = extract(subtree, extract_path)
# Experiment: Make the subtree available as node
destructured_vars['node'] = subtree
print('destructured_vars', destructured_vars)
# Determine the inventory to run on
inventory_selector = build_inventory_selector(
rule.get('inventory_selector'))
if inventory_selector:
try:
inventory_name = extract(subtree, inventory_selector)
except KeyError:
raise Exception(
f'Invalid inventory_selector {inventory_selector}')
print('inventory_name', inventory_name)
# Build a play using tasks or role from rule
play = {
'name':
"{0} {1} {2}".format(ACTION_RULES[action], changed_subtree_path,
inventory_name),
'hosts':
inventory_name,
'gather_facts':
False,
'tasks': []
}
if 'tasks' in rule.get(ACTION_RULES[action], {}):
play['tasks'].append({
'include_tasks': {
'file':
find_tasks(rule.get(ACTION_RULES[action]).get('tasks'))
},
'name':
"{0} {1}".format(ACTION_RULES[action], changed_subtree_path)
})
if 'become' in rule:
play['become'] = rule['become']
if explain:
print(yaml.dump(play))
else:
# Run the action play
plays.append(play)
destructured_vars_list.append(destructured_vars)
ran_rules.append(
(rule, changed_subtree_path, subtree, inventory_name))
def runner_process_message(data):
monitor.stream.put_message(Stdout(0, now(), data.get('stdout', '')))
PlaybookRunner(runner_process_message, new_desired_state, diff,
destructured_vars_list, plays, secrets, project_src,
inventory).run()
return ran_rules