def test_delete(self):
bd1 = self._get_example_aim_bd(tenant_name='t1', name='bd1')
bd2 = self._get_example_aim_bd(tenant_name='t1', name='bd2')
htree = tree.StructuredHashTree()
self.maker.update(htree, [bd1, bd2])
self.maker.delete(htree, [bd1])
fvBD_attr = {
'arpFlood': 'no',
'epMoveDetectMode': '',
'limitIpLearnToSubnets': 'no',
'ipLearning': 'yes',
'unicastRoute': 'yes',
'unkMacUcastAct': 'proxy',
'nameAlias': ''
}
fvRsCtx_attr = {'tnFvCtxName': 'default'}
exp_tree = tree.StructuredHashTree()
exp_tree = exp_tree.add(('fvTenant|t1', 'fvBD|bd2'), **fvBD_attr)
exp_tree = exp_tree.add(('fvTenant|t1', 'fvBD|bd2', 'fvRsCtx|rsctx'),
**fvRsCtx_attr)
self.assertEqual(exp_tree, htree)
self.maker.delete(htree, [bd2])
self.assertFalse(htree.has_subtree())
def test_deleted_bulk(self):
data1 = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
data2 = tree.StructuredHashTree().include([{
'key': ('keyA1', 'keyB')
}, {
'key': ('keyA1', 'keyC')
}, {
'key': ('keyA1', 'keyC', 'keyD')
}])
data3 = tree.StructuredHashTree().include([{
'key': ('keyA2', 'keyB')
}, {
'key': ('keyA2', 'keyC')
}, {
'key': ('keyA2', 'keyC', 'keyD')
}])
self.mgr.update_bulk(self.ctx, [data1, data2, data3])
self.mgr.delete_bulk(self.ctx, [data1, data2])
self.assertEqual([], self.mgr.find(self.ctx, root_rn=['keyA',
'keyA1']))
# data3 still persists
self.assertEqual([data3], self.mgr.find(self.ctx, root_rn=['keyA2']))
def test_get_tenants(self):
data1 = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
data2 = tree.StructuredHashTree().include([{
'key': ('keyA1', 'keyB')
}, {
'key': ('keyA1', 'keyC')
}, {
'key': ('keyA1', 'keyC', 'keyD')
}])
data3 = tree.StructuredHashTree().include([{
'key': ('keyA2', 'keyB')
}, {
'key': ('keyA2', 'keyC')
}, {
'key': ('keyA2', 'keyC', 'keyD')
}])
self.mgr.update_bulk(self.ctx, [data1, data2, data3])
tenants = self.mgr.get_roots(self.ctx)
self.assertEqual(set(['keyA', 'keyA1', 'keyA2']), set(tenants))
def test_state(self, tree_type=tree_manager.CONFIG_TREE):
# Create some trees in the AIM DB
data1 = tree.StructuredHashTree().include(
[{'key': ('fvTenant|tnA', 'keyB')},
{'key': ('fvTenant|tnA', 'keyC')},
{'key': ('fvTenant|tnA', 'keyC', 'keyD')}])
data2 = tree.StructuredHashTree().include(
[{'key': ('fvTenant|tnA1', 'keyB')},
{'key': ('fvTenant|tnA1', 'keyC')},
{'key': ('fvTenant|tnA1', 'keyC', 'keyD')}])
data3 = tree.StructuredHashTree().include(
[{'key': ('fvTenant|tnA2', 'keyB')},
{'key': ('fvTenant|tnA2', 'keyC')},
{'key': ('fvTenant|tnA2', 'keyC', 'keyD')}])
self.tree_mgr.update_bulk(self.ctx, [data1, data2, data3],
tree=tree_type)
# Serve tnA, tnA2 and tnExtra
self.universe.serve(self.ctx, ['tn-tnA', 'tn-tnA2', 'tn-tnExtra'])
# Now observe
self.universe.observe(self.ctx)
state = self.universe.state
# tnA and tnA2 have updated values, tnExtra is still empty
self.assertEqual(data1, state['tn-tnA'])
self.assertEqual(data3, state['tn-tnA2'])
self.assertIsNone(state.get('tn-tnExtra'))
# Change tree in the DB
data1.add(('fvTenant|tnA', 'keyB'), attribute='something')
self.tree_mgr.update_bulk(self.ctx, [data1], tree=tree_type)
# Observe and verify that trees are back in sync
self.assertNotEqual(data1, state['tn-tnA'])
self.universe.observe(self.ctx)
state = self.universe.state
self.assertEqual(data1, state['tn-tnA'])
def test_find_changed(self):
data1 = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
data2 = tree.StructuredHashTree().include([{
'key': ('keyA1', 'keyB')
}, {
'key': ('keyA1', 'keyC')
}, {
'key': ('keyA1', 'keyC', 'keyD')
}])
data3 = tree.StructuredHashTree().include([{
'key': ('keyA2', 'keyB')
}, {
'key': ('keyA2', 'keyC')
}, {
'key': ('keyA2', 'keyC', 'keyD')
}])
self.mgr.update_bulk(self.ctx, [data1, data2, data3])
data1.add(('keyA', ), test='test')
changed = self.mgr.find_changed(
self.ctx, {
data1.root.key[0]: data1.root.full_hash,
data2.root.key[0]: data2.root.full_hash,
data3.root.key[0]: data3.root.full_hash
})
self.assertEqual(1, len(changed))
self.assertEqual(data1.root.key, changed.values()[0].root.key)
def test_agents_to_trees_association(self):
# N, M association
with self.ctx.store.begin(subtransactions=True):
data = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
data2 = tree.StructuredHashTree().include([{
'key': ('keyA1', 'keyB')
}, {
'key': ('keyA1', 'keyC')
}, {
'key': ('keyA1', 'keyC', 'keyD')
}])
data3 = tree.StructuredHashTree().include([{
'key': ('keyA2', 'keyB')
}, {
'key': ('keyA2', 'keyC')
}, {
'key': ('keyA2', 'keyC', 'keyD')
}])
self.mgr.update_bulk(self.ctx, [data, data2, data3])
agent1 = resource.Agent(agent_type='aid',
host='host',
binary_file='binary',
hash_trees=['keyA', 'keyA1', 'keyA2'],
version='1.0')
agent2 = resource.Agent(agent_type='aid',
host='host2',
binary_file='binary',
hash_trees=['keyA', 'keyA2'],
version='1.0')
agent1 = aim_manager.AimManager().create(self.ctx, agent1)
agent2 = aim_manager.AimManager().create(self.ctx, agent2)
self.assertEqual(set(['keyA', 'keyA1', 'keyA2']),
set(agent1.hash_trees))
self.assertEqual(set(['keyA', 'keyA2']), set(agent2.hash_trees))
# Empty agent2
agent2 = aim_manager.AimManager().update(self.ctx,
agent2,
hash_trees=[])
# Delete a tree
self.mgr.delete(self.ctx, data)
if self.ctx.store.supports_foreign_keys:
agent1 = aim_manager.AimManager().get(self.ctx, agent1)
self.assertEqual(set(['keyA1', 'keyA2']), set(agent1.hash_trees))
self.assertEqual(set(), set(agent2.hash_trees))
# Add rogue key
self.assertRaises(exc.HashTreeNotFound,
aim_manager.AimManager().update,
self.ctx,
agent1,
hash_trees=['notakey'])
# Verify agent1 was rolled back properly
agent1 = aim_manager.AimManager().get(self.get_new_context(),
agent1)
self.assertEqual(set(['keyA1', 'keyA2']), set(agent1.hash_trees))
def test_missing(self):
depl = resource.VmmInjectedDeployment(
**{
'display_name': '',
'name': 'kubedns',
'replicas': 1,
'domain_name': 'kube',
'controller_name': 'kube',
'domain_type': 'Kubernetes',
'guid': 'a',
'namespace_name': 'k'
})
ns = resource.VmmInjectedNamespace(
**{
'display_name': '',
'name': 'k',
'domain_name': 'kube',
'controller_name': 'kube',
'domain_type': 'Kubernetes'
})
updates = [depl, ns, ns]
mgr = aim_manager.AimManager()
tt_maker = tree_manager.AimHashTreeMaker()
tt_builder = tree_manager.HashTreeBuilder(mgr)
trees = {}
exp_key = tt_maker._build_hash_tree_key(depl)
for aim_res in updates:
key = tt_maker.get_root_key(aim_res)
if key and trees is not None:
cfg = trees.setdefault(tt_builder.CONFIG, {}).setdefault(
key, tree.StructuredHashTree())
mo = trees.setdefault(tt_builder.MONITOR,
{}).setdefault(key,
tree.StructuredHashTree())
oper = trees.setdefault(tt_builder.OPER, {}).setdefault(
key, tree.StructuredHashTree())
tt_builder.build(
[aim_res], [], [], {
tt_builder.CONFIG: {
key: cfg
},
tt_builder.MONITOR: {
key: mo
},
tt_builder.OPER: {
key: oper
}
},
aim_ctx=self.ctx)
if not isinstance(aim_res, resource.VmmInjectedDeployment):
self.assertIsNotNone(cfg.find(exp_key),
'Resource %s' % aim_res)
self.assertIsNotNone(trees['config']['comp'].find(exp_key),
'Resource %s' % aim_res)
def test_update_1(self):
htree = tree.StructuredHashTree()
exp_tree = tree.StructuredHashTree()
subj = resource.ContractSubject(tenant_name='t1',
contract_name='c1',
name='s1',
in_filters=['i1'],
out_filters=['o1'],
bi_filters=['f1'])
self.maker.update(htree, [subj])
exp_tree = exp_tree.add(('fvTenant|t1', 'vzBrCP|c1', 'vzSubj|s1'),
nameAlias='')
exp_tree = exp_tree.add(('fvTenant|t1', 'vzBrCP|c1', 'vzSubj|s1',
'vzInTerm|intmnl', 'vzRsFiltAtt|i1'),
tnVzFilterName='i1')
exp_tree = exp_tree.add(('fvTenant|t1', 'vzBrCP|c1', 'vzSubj|s1',
'vzOutTerm|outtmnl', 'vzRsFiltAtt|o1'),
tnVzFilterName='o1')
exp_tree = exp_tree.add(
('fvTenant|t1', 'vzBrCP|c1', 'vzSubj|s1', 'vzRsSubjFiltAtt|f1'),
tnVzFilterName='f1')
exp_tree = exp_tree.add(
('fvTenant|t1', 'vzBrCP|c1', 'vzSubj|s1', 'vzInTerm|intmnl'))
exp_tree = exp_tree.add(
('fvTenant|t1', 'vzBrCP|c1', 'vzSubj|s1', 'vzOutTerm|outtmnl'))
self.assertEqual(exp_tree, htree,
'differences: %s' % exp_tree.diff(htree))
def _test_resource_ops(self, resource, tenant, tree_objects,
tree_objects_update,
tree_type=tree_manager.CONFIG_TREE, **updates):
# add
tenant = 'tn-' + tenant
self.db_l.on_commit(self.ctx.store, [resource], [], [])
self.db_l.catch_up_with_action_log(self.ctx.store)
db_tree = self.tt_mgr.get(self.ctx, tenant, tree=tree_type)
exp_tree = tree.StructuredHashTree().include(tree_objects)
self.assertEqual(exp_tree, db_tree)
# update
resource.__dict__.update(**updates)
self.db_l.on_commit(self.ctx.store, [], [resource], [])
self.db_l.catch_up_with_action_log(self.ctx.store)
db_tree = self.tt_mgr.get(self.ctx, tenant, tree=tree_type)
exp_tree = tree.StructuredHashTree().include(tree_objects_update)
self.assertEqual(exp_tree, db_tree)
# delete
self.db_l.on_commit(self.ctx.store, [], [], [resource])
self.db_l.catch_up_with_action_log(self.ctx.store)
db_tree = self.tt_mgr.get(self.ctx, tenant, tree=tree_type)
exp_tree = tree.StructuredHashTree()
self.assertEqual(exp_tree, db_tree)
def _event_loop(self):
start_time = time.time()
# Push the backlog at right before the event loop, so that
# all the events we generate here are likely caught in this
# iteration.
self._push_aim_resources()
if self.ws_context.has_event(self.tenant.urls):
# Continuously check for events
events = self.ws_context.get_event_data(self.tenant.urls)
for event in events:
if (event.keys()[0] == TENANT_KEY and
not event[TENANT_KEY]['attributes'].get(STATUS_FIELD)):
LOG.info("Resetting Tree %s" % self.tenant_name)
# This is a full resync, trees need to be reset
self._state = structured_tree.StructuredHashTree()
self._operational_state = (
structured_tree.StructuredHashTree())
self._monitored_state = (
structured_tree.StructuredHashTree())
self.tag_set = set()
break
LOG.debug("received events for root %s: %s" %
(self.tenant_name, events))
# Make events list flat
self.flat_events(events)
# Pull incomplete objects
events = self._fill_events(events)
# Manage Tags
events = self._filter_ownership(events)
self._event_to_tree(events)
time.sleep(max(0, self.polling_yield - (time.time() - start_time)))
def _process_event(self, event):
event = self._parse_event(event)
affected_tenants = set()
if not event:
return affected_tenants
aim_res = event['resource']
if isinstance(aim_res, resource.AciResourceBase):
is_oper = False
elif isinstance(aim_res, status.OperationalResource):
is_oper = True
else:
return affected_tenants
# special handling for some objects
self._process_pod_status_event(event)
# push event into tree
action = event['event_type']
changes = {'added': [], 'deleted': []}
if action.lower() in [ACTION_CREATED, ACTION_MODIFIED]:
changes['added'].append(aim_res)
elif action.lower() in [ACTION_DELETED]:
self._cleanup_status(aim_res)
changes['deleted'].append(aim_res)
key = self.tt_maker.get_root_key(aim_res)
LOG.info('K8s event: %s %s', action, aim_res)
# Initialize tree if needed
if key and self.trees is not None:
cfg = self.trees.setdefault(self.tt_builder.CONFIG, {}).setdefault(
key, structured_tree.StructuredHashTree())
mo = self.trees.setdefault(self.tt_builder.MONITOR, {}).setdefault(
key, structured_tree.StructuredHashTree())
oper = self.trees.setdefault(self.tt_builder.OPER, {}).setdefault(
key, structured_tree.StructuredHashTree())
old_hash = (cfg.root_full_hash, mo.root_full_hash,
oper.root_full_hash)
self.tt_builder.build(changes['added'], [],
changes['deleted'], {
self.tt_builder.CONFIG: {
key: cfg
},
self.tt_builder.MONITOR: {
key: mo
},
self.tt_builder.OPER: {
key: oper
}
},
aim_ctx=self.ctx)
new_hash = (cfg.root_full_hash, mo.root_full_hash,
oper.root_full_hash)
# Operational state changes can modify trees without changing
# their hash
if old_hash != new_hash or is_oper:
affected_tenants.add(key)
return affected_tenants
def test_monitored_state_change(self):
tn_name = 'test_monitored_state_change'
tn_rn = 'tn-' + tn_name
tn = aim_res.Tenant(name=tn_name, monitored=True)
ap = aim_res.ApplicationProfile(tenant_name=tn_name, name='ap',
monitored=True)
epg = aim_res.EndpointGroup(
tenant_name=tn_name, app_profile_name='ap', name='epg',
bd_name='some', monitored=True)
self.mgr.create(self.ctx, tn)
self.mgr.create(self.ctx, ap)
self.mgr.create(self.ctx, epg)
cfg_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.CONFIG_TREE)
mon_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.MONITORED_TREE)
# Create my own tree representation
my_cfg_tree = tree.StructuredHashTree()
my_mon_tree = tree.StructuredHashTree()
self.db_l.tt_maker.update(my_mon_tree, [tn])
# Succeed their creation
self.mgr.set_resource_sync_synced(self.ctx, ap)
self.mgr.set_resource_sync_synced(self.ctx, epg)
self.db_l.tt_maker.update(my_mon_tree, [ap, epg])
cfg_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.CONFIG_TREE)
mon_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.MONITORED_TREE)
self.assertEqual(my_mon_tree, mon_tree)
self.assertEqual(my_cfg_tree, cfg_tree)
# Change ownership of the AP
self.mgr.update(self.ctx, ap, monitored=False)
my_mon_tree = tree.StructuredHashTree()
# This is equivalent of adding only tenant and epg to the conf tree
self.db_l.tt_maker.update(my_mon_tree, [tn, epg])
self.db_l.tt_maker.update(my_cfg_tree, [ap])
# Refresh trees
cfg_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.CONFIG_TREE)
mon_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.MONITORED_TREE)
self.assertEqual(my_mon_tree, mon_tree,
'differences: %s' % my_mon_tree.diff(mon_tree))
self.assertEqual(my_cfg_tree, cfg_tree)
# Unset monitored to EPG as well
self.mgr.update(self.ctx, epg, monitored=False)
my_mon_tree = tree.StructuredHashTree()
self.db_l.tt_maker.update(my_mon_tree, [tn])
self.db_l.tt_maker.update(my_cfg_tree, [epg])
# Refresh trees
cfg_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.CONFIG_TREE)
mon_tree = self.tt_mgr.get(self.ctx, tn_rn,
tree=tree_manager.MONITORED_TREE)
self.assertEqual(my_mon_tree, mon_tree)
self.assertEqual(my_cfg_tree, cfg_tree)
def test_initialize(self):
data = tree.StructuredHashTree()
self.assertIsNone(data.root)
data = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}])
self.assertIsNotNone(data.root)
self.assertEqual(('keyA', ), data.root.key)
def __init__(self,
tenant_name,
apic_config,
apic_session,
ws_context,
creation_succeeded=None,
creation_failed=None,
aim_system_id=None,
get_resources=None,
*args,
**kwargs):
super(AciTenantManager, self).__init__(*args, **kwargs)
LOG.info("Init manager for tenant %s" % tenant_name)
self.get_resources = get_resources
self.apic_config = apic_config
# Each tenant has its own sessions
self.aci_session = apic_session
self.dn_manager = apic_client.DNManager()
self.tenant_name = tenant_name
children_mos = get_children_mos(self.aci_session, self.tenant_name)
ws_subscription_to = self.apic_config.get_option(
'websocket_subscription_timeout', 'aim') or DEFAULT_WS_TO
self.tenant = Root(self.tenant_name,
filtered_children=children_mos,
rn=self.tenant_name,
ws_subscription_to=ws_subscription_to)
self._state = structured_tree.StructuredHashTree()
self._operational_state = structured_tree.StructuredHashTree()
self._monitored_state = structured_tree.StructuredHashTree()
self.polling_yield = self.apic_config.get_option(
'aci_tenant_polling_yield', 'aim')
self.to_aim_converter = converter.AciToAimModelConverter()
self.to_aci_converter = converter.AimToAciModelConverter()
self._reset_object_backlog()
self.tree_builder = tree_manager.HashTreeBuilder(None)
self.tag_name = aim_system_id or self.apic_config.get_option(
'aim_system_id', 'aim')
self.tag_set = set()
self.failure_log = {}
def noop(*args, **kwargs):
pass
self.creation_succeeded = creation_succeeded or noop
self.creation_failed = creation_failed or noop
# Warm bit to avoid rushed synchronization before receiving the first
# batch of APIC events
self._warm = False
self.ws_context = ws_context
self.recovery_retries = None
self.max_retries = 5
self.error_handler = error.APICAPIErrorHandler()
# For testing purposes
self.num_loop_runs = float('inf')
def _push_changes_to_trees(self, ctx, log_by_root, delete_logs=True,
check_reset=True):
conf = tree_manager.CONFIG_TREE
monitor = tree_manager.MONITORED_TREE
oper = tree_manager.OPERATIONAL_TREE
for root_rn in log_by_root:
try:
tree_map = {}
with ctx.store.begin(subtransactions=True):
try:
ttree = self.tt_mgr.get_base_tree(ctx, root_rn,
lock_update=True)
if check_reset and ttree and ttree.needs_reset:
LOG.warn('RESET action received for root %s, '
'resetting trees' % root_rn)
self.reset(ctx.store, root_rn)
continue
ttree_conf = self.tt_mgr.get(
ctx, root_rn, lock_update=True, tree=conf)
ttree_operational = self.tt_mgr.get(
ctx, root_rn, lock_update=True, tree=oper)
ttree_monitor = self.tt_mgr.get(
ctx, root_rn, lock_update=True, tree=monitor)
except hexc.HashTreeNotFound:
ttree_conf = htree.StructuredHashTree()
ttree_operational = htree.StructuredHashTree()
ttree_monitor = htree.StructuredHashTree()
tree_map.setdefault(
self.tt_builder.CONFIG, {})[root_rn] = ttree_conf
tree_map.setdefault(
self.tt_builder.OPER, {})[root_rn] = ttree_operational
tree_map.setdefault(
self.tt_builder.MONITOR, {})[root_rn] = ttree_monitor
for action, aim_res, _ in log_by_root[root_rn]:
added = deleted = []
if action == aim_tree.ActionLog.CREATE:
added = [aim_res]
else:
deleted = [aim_res]
self.tt_builder.build(added, [], deleted, tree_map,
aim_ctx=ctx)
if ttree_conf.root_key:
self.tt_mgr.update(ctx, ttree_conf)
if ttree_operational.root_key:
self.tt_mgr.update(ctx, ttree_operational, tree=oper)
if ttree_monitor.root_key:
self.tt_mgr.update(ctx, ttree_monitor, tree=monitor)
if delete_logs:
self._delete_logs(ctx, log_by_root[root_rn])
except Exception as e:
LOG.error('Failed to update root %s '
'tree for: %s' % (root_rn, e.message))
LOG.debug(traceback.format_exc())
def _event_loop(self):
start_time = time.time()
# Push the backlog at right before the event loop, so that
# all the events we generate here are likely caught in this
# iteration.
self._push_aim_resources()
if self.ws_context.has_event(self.tenant.urls):
with utils.get_rlock(lcon.ACI_TREE_LOCK_NAME_PREFIX +
self.tenant_name):
events = self.ws_context.get_event_data(self.tenant.urls)
for event in events:
# REVISIT(ivar): remove vmmDomP once websocket ACI bug is
# fixed
if (list(event.keys())[0] in [
self.tenant.type, 'vmmDomP'
] and not event[list(
event.keys())[0]]['attributes'].get(STATUS_FIELD)):
LOG.info("Resetting Tree %s" % self.tenant_name)
# REVISIT(ivar): on subscription to VMMPolicy objects,
# aci doesn't return the root object itself because of
# a bug. Let's craft a fake root to work around this
# problem
if self.tenant_name.startswith('vmmp-'):
LOG.debug('Faking vmmProvP %s' % self.tenant_name)
events.append({
'vmmProvP': {
'attributes': {
'dn': self.tenant.dn
}
}
})
# This is a full resync, trees need to be reset
self._state = structured_tree.StructuredHashTree()
self._operational_state = (
structured_tree.StructuredHashTree())
self._monitored_state = (
structured_tree.StructuredHashTree())
self.tag_set = set()
break
# REVISIT(ivar): there's already a debug log in acitoolkit
# listing all the events received one by one. The following
# would be more compact, we need to choose which to keep.
# LOG.debug("received events for root %s: %s" %
# (self.tenant_name, events))
# Make events list flat
self.flat_events(events)
# Pull incomplete objects
events = self._fill_events(events)
# Manage Tags
events = self.ownership_mgr.filter_ownership(events)
self._event_to_tree(events)
time.sleep(max(0, self.polling_yield - (time.time() - start_time)))
def test_dummy_diff(self):
data = tree.StructuredHashTree().include([{
'key': ('keyA', )
}, {
'key': ('keyA', 'keyB')
}])
# Same as data, with dummy root
data2 = tree.StructuredHashTree().include([{'key': ('keyA', 'keyB')}])
self.assertEqual({"add": [], "remove": [('keyA', )]}, data2.diff(data))
self.assertEqual({"add": [('keyA', )], "remove": []}, data.diff(data2))
data2.add(('keyA', ))
self.assertEqual({"add": [], "remove": []}, data2.diff(data))
self.assertEqual({"add": [], "remove": []}, data.diff(data2))
def test_pop_dummies(self):
data = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
# Removing keyD, keyC will also be removed
data.pop(('keyA', 'keyC', 'keyD'))
expected = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}])
self.assertEqual(expected, data)
# Popping keyB will now also pop the root
data.pop(('keyA', 'keyB'))
self.assertIsNone(data.root)
def test_update(self):
data = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
self.mgr.update(self.ctx, data)
data2 = self.mgr.find(self.ctx, root_rn=['keyA'])[0]
self.assertEqual(data, data2)
# Change an existing tree
data.add(('keyA', 'keyF'), test='test')
self.mgr.update(self.ctx, data)
data3 = self.mgr.find(self.ctx, root_rn=['keyA'])[0]
self.assertEqual(data, data3)
self.assertNotEqual(data, data2)
# Empty the tree completely
data3.remove(('keyA', ))
self.assertEqual(('keyA', ), data3.root_key)
self.mgr.update(self.ctx, data3)
data4 = self.mgr.find(self.ctx, root_rn=['keyA'])[0]
# Verify this is an empty tree
self.assertIsNone(data4.root)
self.assertEqual(data3.root_key, data4.root_key)
def _remove_data_from_tree(self, data, state):
aim_res = converter.AciToAimModelConverter().convert(data)
by_root = {}
for res in aim_res:
by_root.setdefault(res.root, []).append(res)
for root, updates in by_root.iteritems():
tree_manager.AimHashTreeMaker().delete(
state.setdefault(root, structured_tree.StructuredHashTree()),
updates)
def test_get_optimized_state(self, tree_type=tree_manager.CONFIG_TREE):
data1 = tree.StructuredHashTree().include([{
'key': ('fvTenant|tnA', 'keyB')
}, {
'key': ('fvTenant|tnA', 'keyC')
}, {
'key': ('fvTenant|tnA', 'keyC', 'keyD')
}])
data2 = tree.StructuredHashTree().include([{
'key': ('fvTenant|tnA1', 'keyB')
}, {
'key': ('fvTenant|tnA1', 'keyC')
}, {
'key': ('fvTenant|tnA1', 'keyC', 'keyD')
}])
data3 = tree.StructuredHashTree().include([{
'key': ('fvTenant|tnA2', 'keyB')
}, {
'key': ('fvTenant|tnA2', 'keyC')
}, {
'key': ('fvTenant|tnA2', 'keyC', 'keyD')
}])
self.tree_mgr.update_bulk(self.ctx, [data1, data2, data3],
tree=tree_type)
self.universe.serve(self.ctx,
['tn-tnA', 'tn-tnA1', 'tn-tnA2', 'tn-tnA3'])
# Other state is in sync
other_state = {
'tn-tnA': tree.StructuredHashTree().from_string(str(data1)),
'tn-tnA1': tree.StructuredHashTree().from_string(str(data2)),
'tn-tnA2': tree.StructuredHashTree().from_string(str(data3))
}
# Optimized state is empty
self.assertEqual({},
self.universe.get_optimized_state(
self.ctx, other_state))
# Add a new tenant
data4 = tree.StructuredHashTree().include([{
'key': ('fvTenant|tnA3', 'keyB')
}, {
'key': ('fvTenant|tnA3', 'keyC')
}, {
'key': ('fvTenant|tnA3', 'keyC', 'keyD')
}])
self.tree_mgr.update_bulk(self.ctx, [data4], tree=tree_type)
self.assertEqual({'tn-tnA3': data4},
self.universe.get_optimized_state(
self.ctx, other_state))
# Modify data1
data1.add(('fvTenant|tnA', 'keyZ'), attribute='something')
self.tree_mgr.update_bulk(self.ctx, [data1], tree=tree_type)
# Now Data1 is included too
self.assertEqual({
'tn-tnA3': data4,
'tn-tnA': data1
}, self.universe.get_optimized_state(self.ctx, other_state))
def test_clear(self):
data = tree.StructuredHashTree().include([{
'key': ('keyA', )
}, {
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyB', 'keyC')
}])
data_copy = tree.StructuredHashTree().from_string(str(data))
data.clear(('keyA', 'keyB', 'keyNO'))
self.assertEqual(data_copy, data)
self.assertEqual(1, len(data.root.get_children()))
self.assertFalse(data.root.get_children()[0].dummy)
data.clear(('keyA', 'keyB'))
self.assertTrue(data.root.get_children()[0].dummy)
self.assertNotEqual(data_copy, data)
data.clear(('keyA', 'keyB', 'keyC'))
# Dummies removed
self.assertEqual(0, len(data.root.get_children()))
def test_deleted(self):
data = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
self.mgr.update(self.ctx, data)
self.mgr.delete(self.ctx, data)
self.assertEqual([], self.mgr.find(self.ctx, root_rn=['keyA']))
def test_find(self):
data = tree.StructuredHashTree()
# root None
self.assertIsNone(data.find(('KeyA', )))
data.add(('KeyA', ))
# root is Key
self.assertEqual(data.root, data.find(('KeyA', )))
# Not found
self.assertIsNone(data.find(('KeyA', 'KeyB')))
# Multi Head search
self.assertIsNone(data.find(('KeyZ', 'KeyB')))
def test_pop(self):
data = tree.StructuredHashTree()
self.assertIsNone(data.pop(('keyA', )))
data.add(('keyA', 'keyB'))
data_copy = copy.deepcopy(data)
data2 = data.pop(('keyA', ))
self.assertIsNone(data.root)
self.assertEqual(data_copy, data2)
data = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
# Key not found
self.assertIsNone(data.pop(('keyA', 'keyF', 'keyE')))
def test_remove(self):
data = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
data2 = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB')
}, {
'key': ('keyA', 'keyC')
}])
self.assertNotEqual(data.root.full_hash, data2.root.full_hash)
data.remove(('keyA', 'keyC', 'keyD'))
self.assertEqual(data2, data)
# Raises on NotFound
self.assertRaises(KeyError, data.remove, ('keyA', 'keyC', 'keyZ'))
# Nothing happened
self.assertEqual(data2, data)
def test_include_with_metadata(self):
t = tree.StructuredHashTree().include([{
'key': ('keyA', 'keyB'),
'_metadata': {
"foo": 1
}
}, {
'key': ('keyA', 'keyC'),
'_metadata': {
"bar": 2
}
}, {
'key': ('keyA', 'keyC', 'keyD')
}])
node = t.find(('keyA', 'keyB'))
self.assertIsNotNone(node)
self.assertEqual({'foo': 1}, node.metadata)
node = t.find(('keyA', 'keyC'))
self.assertIsNotNone(node)
self.assertEqual({'bar': 2}, node.metadata)
node = t.find(('keyA', 'keyC', 'keyD'))
self.assertIsNotNone(node)
self.assertEqual({}, node.metadata)
# change the metadata values
t.include([{
'key': ('keyA', 'keyB'),
'_metadata': None
}, {
'key': ('keyA', 'keyC'),
'_metadata': {
"bar": 2
}
}, {
'key': ('keyA', 'keyC', 'keyD'),
'_metadata': {
"baz": 3
}
}])
node = t.find(('keyA', 'keyB'))
self.assertIsNotNone(node)
self.assertEqual({}, node.metadata)
node = t.find(('keyA', 'keyC'))
self.assertIsNotNone(node)
self.assertEqual({'bar': 2}, node.metadata)
node = t.find(('keyA', 'keyC', 'keyD'))
self.assertIsNotNone(node)
self.assertEqual({"baz": 3}, node.metadata)
def clean_all(self, context):
empty_tree = structured_tree.StructuredHashTree()
with context.store.begin(subtransactions=True):
for tree_type in SUPPORTED_TREES:
db_objs = self._find_query(context,
tree_type,
lock_update=True)
for db_obj in db_objs:
db_obj.tree = str(empty_tree).encode('utf-8')
context.store.add(db_obj)
db_objs = self._find_query(context, ROOT_TREE, lock_update=True)
for db_obj in db_objs:
db_obj.needs_reset = False
context.store.add(db_obj)
def _build(key, added, deleted):
if key and trees is not None:
cfg = trees.setdefault(tt_builder.CONFIG, {}).setdefault(
key, tree.StructuredHashTree())
mo = trees.setdefault(tt_builder.MONITOR,
{}).setdefault(key,
tree.StructuredHashTree())
oper = trees.setdefault(tt_builder.OPER, {}).setdefault(
key, tree.StructuredHashTree())
tt_builder.build(added, [],
deleted, {
tt_builder.CONFIG: {
key: cfg
},
tt_builder.MONITOR: {
key: mo
},
tt_builder.OPER: {
key: oper
}
},
aim_ctx=self.ctx)
def test_sync_object_status(self):
mgr = aim_manager.AimManager()
epg = mgr.create(
self.ctx,
resource.EndpointGroup(tenant_name='test',
app_profile_name='test',
name='test',
sync=False))
status = mgr.get_status(self.ctx, epg)
mgr.update(self.ctx, status, sync_status=status.SYNCED)
tt_builder = tree_manager.HashTreeBuilder(mgr)
trees = {}
tt_maker = tree_manager.AimHashTreeMaker()
key = tt_maker.get_root_key(epg)
cfg = trees.setdefault(tt_builder.CONFIG,
{}).setdefault(key, tree.StructuredHashTree())
mo = trees.setdefault(tt_builder.MONITOR,
{}).setdefault(key, tree.StructuredHashTree())
oper = trees.setdefault(tt_builder.OPER,
{}).setdefault(key, tree.StructuredHashTree())
tt_builder.build(
[status], [], [], {
tt_builder.CONFIG: {
key: cfg
},
tt_builder.MONITOR: {
key: mo
},
tt_builder.OPER: {
key: oper
}
},
aim_ctx=self.ctx)
# Should not add parent back
exp_key = tt_maker._build_hash_tree_key(epg)
self.assertIsNone(cfg.find(exp_key))