def test_diff_values_nested(self): from pillar.api.utils import doc_diff diff = doc_diff({'a': 'b', 'props': {'status': 'todo', 'notes': 'jemoeder'}}, {'a': 'c', 'props': {'status': 'done', 'notes': 'jemoeder'}}) self.assertEqual({('a', 'b', 'c'), ('props.status', 'todo', 'done')}, set(diff))
def replace_schemas(project): project_url = project.get('url', '-no-url-') log_proj = _single_logger('Upgrading schema project %s (%s)', project_url, project['_id']) orig_proj = copy.deepcopy(project) for proj_nt in project['node_types']: nt_name = proj_nt['name'] if nt_name not in nts_by_name: continue pillar_nt = nts_by_name[nt_name] pillar_dyn_schema = pillar_nt['dyn_schema'] if proj_nt['dyn_schema'] == pillar_dyn_schema: # Schema already up to date. continue log_proj() log.info(' - replacing dyn_schema on node type "%s"', nt_name) proj_nt['dyn_schema'] = copy.deepcopy(pillar_dyn_schema) seen_changes = False for key, val1, val2 in doc_diff(orig_proj, project): if not seen_changes: log.info('Schema changes to project %s (%s):', project_url, project['_id']) seen_changes = True log.info(' - %30s: %s → %s', key, val1, val2) if go: # Use Eve to PUT, so we have schema checking. db_proj = remove_private_keys(project) r, _, _, status = current_app.put_internal('projects', db_proj, _id=project['_id']) if status != 200: log.error('Error %i storing altered project %s %s', status, project['_id'], r) raise SystemExit('Error storing project, see log.') log.debug('Project saved succesfully.')
def test_diff_keys_nested(self): from pillar.api.utils import doc_diff, DoesNotExist diff = doc_diff({'a': 'b', 'props': {'status1': 'todo', 'notes': 'jemoeder'}}, {'a': 'b', 'props': {'status2': 'todo', 'notes': 'jemoeder'}}) self.assertEqual({('props.status1', 'todo', DoesNotExist), ('props.status2', DoesNotExist, 'todo')}, set(diff))
def test_diff_list_values(self): from pillar.api.utils import doc_diff diff = doc_diff({'a': 'b', 'props': ['status', 'todo', 'notes', 'jemoeder']}, {'a': 'b', 'props': ['todo', 'others', 'notes', 'jemoeder']}) self.assertEqual({ ('props[0]', 'status', 'todo'), ('props[1]', 'todo', 'others'), }, set(diff))
def test_diff_list_unequal_lengths(self): from pillar.api.utils import doc_diff, DoesNotExist diff = doc_diff({'a': 'b', 'props': ['status', 'todo', 'notes']}, {'a': 'b', 'props': ['todo', 'others', 'notes', 'jemoeder']}) self.assertEqual({ ('props[0]', 'status', 'todo'), ('props[1]', 'todo', 'others'), ('props[3]', DoesNotExist, 'jemoeder'), }, set(diff))
def test_diff_values_falsey(self): from pillar.api.utils import doc_diff, DoesNotExist # DoesNotExist vs. empty string diff = doc_diff({'a': 'b', 3: ''}, {'a': 'b'}) self.assertEqual([], list(diff)) diff = doc_diff({'a': 'b', 3: ''}, {'a': 'b'}, falsey_is_equal=False) self.assertEqual([(3, '', DoesNotExist)], list(diff)) # Empty string vs. None diff = doc_diff({'a': 'b', 3: ''}, {'a': 'b', 3: None}) self.assertEqual([], list(diff)) diff = doc_diff({'a': 'b', 3: ''}, {'a': 'b', 3: None}, falsey_is_equal=False) self.assertEqual([(3, '', None)], list(diff))
def test_no_diff_privates(self): from pillar.api.utils import doc_diff diff = doc_diff({ 'a': 'b', 3: 42, '_updated': 5133 }, { 'a': 'b', 3: 42, '_updated': 42 }) self.assertEqual([], list(diff))
def test_no_diff_nested(self): from pillar.api.utils import doc_diff diff = doc_diff( { 'a': 'b', 'props': { 'status': 'todo', 'notes': 'jemoeder' } }, { 'a': 'b', 'props': { 'status': 'todo', 'notes': 'jemoeder' } }) self.assertEqual([], list(diff))
def test_diff_keys_simple(self): from pillar.api.utils import doc_diff, DoesNotExist diff = doc_diff({'a': 'b', 3: 42}, {'a': 'b', 2: 42}) self.assertEqual({(3, 42, DoesNotExist), (2, DoesNotExist, 42)}, set(diff))
def test_diff_values_simple(self): from pillar.api.utils import doc_diff diff = doc_diff({'a': 'b', 3: 42}, {'a': 'b', 3: 513}) self.assertEqual([(3, 42, 513)], list(diff))
def test_no_diff_simple(self): from pillar.api.utils import doc_diff diff = doc_diff({'a': 'b', 3: 42}, {'a': 'b', 3: 42}) self.assertEqual([], list(diff))
def replace_pillar_node_type_schemas(project_url=None, all_projects=False, missing=False, go=False, project_id=None): """Replaces the project's node type schemas with the standard Pillar ones. Non-standard node types are left alone. """ from pillar.api.utils.authentication import force_cli_user force_cli_user() from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES from pillar.api.utils import remove_private_keys, doc_diff will_would = 'Will' if go else 'Would' projects_changed = projects_seen = 0 for proj in _db_projects(project_url, all_projects, project_id, go=go): projects_seen += 1 orig_proj = copy.deepcopy(proj) proj_id = proj['_id'] if 'url' not in proj: log.warning('Project %s has no URL!', proj_id) proj_url = proj.get('url', f'-no URL id {proj_id}') log.debug('Handling project %s', proj_url) for proj_nt in proj['node_types']: nt_name = proj_nt['name'] try: pillar_nt = PILLAR_NAMED_NODE_TYPES[nt_name] except KeyError: log.debug(' - skipping non-standard node type "%s"', nt_name) continue log.debug(' - replacing schema on node type "%s"', nt_name) # This leaves node type keys intact that aren't in Pillar's node_type_xxx definitions, # such as permissions. It also keeps form schemas as-is. pillar_nt.pop('form_schema', None) proj_nt.update(copy.deepcopy(pillar_nt)) # Find new node types that aren't in the project yet. if missing: project_ntnames = set(nt['name'] for nt in proj['node_types']) for nt_name in set(PILLAR_NAMED_NODE_TYPES.keys()) - project_ntnames: log.info(' - Adding node type "%s"', nt_name) pillar_nt = PILLAR_NAMED_NODE_TYPES[nt_name] proj['node_types'].append(copy.deepcopy(pillar_nt)) proj_has_difference = False for key, val1, val2 in doc_diff(orig_proj, proj, falsey_is_equal=False): if not proj_has_difference: if proj.get('_deleted', False): deleted = ' (deleted)' else: deleted = '' log.info('%s change project %s%s', will_would, proj_url, deleted) proj_has_difference = True log.info(' %30r: %r → %r', key, val1, val2) projects_changed += proj_has_difference if go and proj_has_difference: # Use Eve to PUT, so we have schema checking. db_proj = remove_private_keys(proj) try: r, _, _, status = current_app.put_internal('projects', db_proj, _id=proj_id) except Exception: log.exception('Error saving project %s (url=%s)', proj_id, proj_url) raise SystemExit(5) if status != 200: log.error('Error %i storing altered project %s %s', status, proj['_id'], r) raise SystemExit('Error storing project, see log.') log.debug('Project saved succesfully.') log.info('%s %d of %d projects', 'Changed' if go else 'Would change', projects_changed, projects_seen)