def test_diff_values_nested(self):
from pillar.api.utils import doc_diff
diff = doc_diff({'a': 'b', 'props': {'status': 'todo', 'notes': 'jemoeder'}},
{'a': 'c', 'props': {'status': 'done', 'notes': 'jemoeder'}})
self.assertEqual({('a', 'b', 'c'), ('props.status', 'todo', 'done')},
set(diff))
def replace_schemas(project):
project_url = project.get('url', '-no-url-')
log_proj = _single_logger('Upgrading schema project %s (%s)', project_url, project['_id'])
orig_proj = copy.deepcopy(project)
for proj_nt in project['node_types']:
nt_name = proj_nt['name']
if nt_name not in nts_by_name:
continue
pillar_nt = nts_by_name[nt_name]
pillar_dyn_schema = pillar_nt['dyn_schema']
if proj_nt['dyn_schema'] == pillar_dyn_schema:
# Schema already up to date.
continue
log_proj()
log.info(' - replacing dyn_schema on node type "%s"', nt_name)
proj_nt['dyn_schema'] = copy.deepcopy(pillar_dyn_schema)
seen_changes = False
for key, val1, val2 in doc_diff(orig_proj, project):
if not seen_changes:
log.info('Schema changes to project %s (%s):', project_url, project['_id'])
seen_changes = True
log.info(' - %30s: %s → %s', key, val1, val2)
if go:
# Use Eve to PUT, so we have schema checking.
db_proj = remove_private_keys(project)
r, _, _, status = current_app.put_internal('projects', db_proj, _id=project['_id'])
if status != 200:
log.error('Error %i storing altered project %s %s', status, project['_id'], r)
raise SystemExit('Error storing project, see log.')
log.debug('Project saved succesfully.')
def test_diff_keys_nested(self):
from pillar.api.utils import doc_diff, DoesNotExist
diff = doc_diff({'a': 'b', 'props': {'status1': 'todo', 'notes': 'jemoeder'}},
{'a': 'b', 'props': {'status2': 'todo', 'notes': 'jemoeder'}})
self.assertEqual({('props.status1', 'todo', DoesNotExist),
('props.status2', DoesNotExist, 'todo')},
set(diff))
def test_diff_list_values(self):
from pillar.api.utils import doc_diff
diff = doc_diff({'a': 'b', 'props': ['status', 'todo', 'notes', 'jemoeder']},
{'a': 'b', 'props': ['todo', 'others', 'notes', 'jemoeder']})
self.assertEqual({
('props[0]', 'status', 'todo'),
('props[1]', 'todo', 'others'),
}, set(diff))
def test_diff_list_unequal_lengths(self):
from pillar.api.utils import doc_diff, DoesNotExist
diff = doc_diff({'a': 'b', 'props': ['status', 'todo', 'notes']},
{'a': 'b', 'props': ['todo', 'others', 'notes', 'jemoeder']})
self.assertEqual({
('props[0]', 'status', 'todo'),
('props[1]', 'todo', 'others'),
('props[3]', DoesNotExist, 'jemoeder'),
}, set(diff))
def test_diff_values_falsey(self):
from pillar.api.utils import doc_diff, DoesNotExist
# DoesNotExist vs. empty string
diff = doc_diff({'a': 'b', 3: ''},
{'a': 'b'})
self.assertEqual([], list(diff))
diff = doc_diff({'a': 'b', 3: ''},
{'a': 'b'}, falsey_is_equal=False)
self.assertEqual([(3, '', DoesNotExist)], list(diff))
# Empty string vs. None
diff = doc_diff({'a': 'b', 3: ''},
{'a': 'b', 3: None})
self.assertEqual([], list(diff))
diff = doc_diff({'a': 'b', 3: ''},
{'a': 'b', 3: None}, falsey_is_equal=False)
self.assertEqual([(3, '', None)], list(diff))
def test_no_diff_privates(self):
from pillar.api.utils import doc_diff
diff = doc_diff({
'a': 'b',
3: 42,
'_updated': 5133
}, {
'a': 'b',
3: 42,
'_updated': 42
})
self.assertEqual([], list(diff))
def test_no_diff_nested(self):
from pillar.api.utils import doc_diff
diff = doc_diff(
{
'a': 'b',
'props': {
'status': 'todo',
'notes': 'jemoeder'
}
}, {
'a': 'b',
'props': {
'status': 'todo',
'notes': 'jemoeder'
}
})
self.assertEqual([], list(diff))
def test_diff_keys_simple(self):
from pillar.api.utils import doc_diff, DoesNotExist
diff = doc_diff({'a': 'b', 3: 42}, {'a': 'b', 2: 42})
self.assertEqual({(3, 42, DoesNotExist), (2, DoesNotExist, 42)},
set(diff))
def test_diff_values_simple(self):
from pillar.api.utils import doc_diff
diff = doc_diff({'a': 'b', 3: 42}, {'a': 'b', 3: 513})
self.assertEqual([(3, 42, 513)], list(diff))
def test_no_diff_simple(self):
from pillar.api.utils import doc_diff
diff = doc_diff({'a': 'b', 3: 42}, {'a': 'b', 3: 42})
self.assertEqual([], list(diff))
def replace_pillar_node_type_schemas(project_url=None, all_projects=False, missing=False, go=False,
project_id=None):
"""Replaces the project's node type schemas with the standard Pillar ones.
Non-standard node types are left alone.
"""
from pillar.api.utils.authentication import force_cli_user
force_cli_user()
from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES
from pillar.api.utils import remove_private_keys, doc_diff
will_would = 'Will' if go else 'Would'
projects_changed = projects_seen = 0
for proj in _db_projects(project_url, all_projects, project_id, go=go):
projects_seen += 1
orig_proj = copy.deepcopy(proj)
proj_id = proj['_id']
if 'url' not in proj:
log.warning('Project %s has no URL!', proj_id)
proj_url = proj.get('url', f'-no URL id {proj_id}')
log.debug('Handling project %s', proj_url)
for proj_nt in proj['node_types']:
nt_name = proj_nt['name']
try:
pillar_nt = PILLAR_NAMED_NODE_TYPES[nt_name]
except KeyError:
log.debug(' - skipping non-standard node type "%s"', nt_name)
continue
log.debug(' - replacing schema on node type "%s"', nt_name)
# This leaves node type keys intact that aren't in Pillar's node_type_xxx definitions,
# such as permissions. It also keeps form schemas as-is.
pillar_nt.pop('form_schema', None)
proj_nt.update(copy.deepcopy(pillar_nt))
# Find new node types that aren't in the project yet.
if missing:
project_ntnames = set(nt['name'] for nt in proj['node_types'])
for nt_name in set(PILLAR_NAMED_NODE_TYPES.keys()) - project_ntnames:
log.info(' - Adding node type "%s"', nt_name)
pillar_nt = PILLAR_NAMED_NODE_TYPES[nt_name]
proj['node_types'].append(copy.deepcopy(pillar_nt))
proj_has_difference = False
for key, val1, val2 in doc_diff(orig_proj, proj, falsey_is_equal=False):
if not proj_has_difference:
if proj.get('_deleted', False):
deleted = ' (deleted)'
else:
deleted = ''
log.info('%s change project %s%s', will_would, proj_url, deleted)
proj_has_difference = True
log.info(' %30r: %r → %r', key, val1, val2)
projects_changed += proj_has_difference
if go and proj_has_difference:
# Use Eve to PUT, so we have schema checking.
db_proj = remove_private_keys(proj)
try:
r, _, _, status = current_app.put_internal('projects', db_proj, _id=proj_id)
except Exception:
log.exception('Error saving project %s (url=%s)', proj_id, proj_url)
raise SystemExit(5)
if status != 200:
log.error('Error %i storing altered project %s %s', status, proj['_id'], r)
raise SystemExit('Error storing project, see log.')
log.debug('Project saved succesfully.')
log.info('%s %d of %d projects',
'Changed' if go else 'Would change',
projects_changed, projects_seen)
