def replace_schemas(project):
project_url = project.get('url', '-no-url-')
log_proj = _single_logger('Upgrading schema project %s (%s)', project_url, project['_id'])
orig_proj = copy.deepcopy(project)
for proj_nt in project['node_types']:
nt_name = proj_nt['name']
if nt_name not in nts_by_name:
continue
pillar_nt = nts_by_name[nt_name]
pillar_dyn_schema = pillar_nt['dyn_schema']
if proj_nt['dyn_schema'] == pillar_dyn_schema:
# Schema already up to date.
continue
log_proj()
log.info(' - replacing dyn_schema on node type "%s"', nt_name)
proj_nt['dyn_schema'] = copy.deepcopy(pillar_dyn_schema)
seen_changes = False
for key, val1, val2 in doc_diff(orig_proj, project):
if not seen_changes:
log.info('Schema changes to project %s (%s):', project_url, project['_id'])
seen_changes = True
log.info(' - %30s: %s → %s', key, val1, val2)
if go:
# Use Eve to PUT, so we have schema checking.
db_proj = remove_private_keys(project)
r, _, _, status = current_app.put_internal('projects', db_proj, _id=project['_id'])
if status != 200:
log.error('Error %i storing altered project %s %s', status, project['_id'], r)
raise SystemExit('Error storing project, see log.')
log.debug('Project saved succesfully.')
def remove_user(self,
org_id: bson.ObjectId,
*,
user_id: bson.ObjectId = None,
email: str = None) -> dict:
"""Removes a user from the organization.
The user can be identified by either user ID or email.
Returns the new organization document.
"""
users_coll = current_app.db('users')
assert user_id or email
# Collect the email address if not given. This ensures the removal
# if the email was accidentally in the unknown_members list.
if email is None:
user_doc = users_coll.find_one(user_id, projection={'email': 1})
if user_doc is not None:
email = user_doc['email']
# See if we know this user.
if user_id is None:
user_doc = users_coll.find_one({'email': email}, projection={'_id': 1})
if user_doc is not None:
user_id = user_doc['_id']
if user_id and not users_coll.count({'_id': user_id}):
raise wz_exceptions.UnprocessableEntity('User does not exist')
self._log.info('Removing user %s / %s from organization %s', user_id, email, org_id)
org_doc = self._get_org(org_id)
# Compute the new members.
if user_id:
members = set(org_doc.get('members') or []) - {user_id}
org_doc['members'] = list(members)
if email:
unknown_members = set(org_doc.get('unknown_members')) - {email}
org_doc['unknown_members'] = list(unknown_members)
r, _, _, status = current_app.put_internal('organizations',
remove_private_keys(org_doc),
_id=org_id)
if status != 200:
self._log.error('Error updating organization; status should be 200, not %i: %s',
status, r)
raise ValueError(f'Unable to update organization, status code {status}')
org_doc.update(r)
# Update the roles for the affected member.
if user_id:
self.refresh_roles(user_id)
return org_doc
def create_blog(proj_url):
"""Adds a blog to the project."""
from pillar.api.utils.authentication import force_cli_user
from pillar.api.utils import node_type_utils
from pillar.api.node_types.blog import node_type_blog
from pillar.api.node_types.post import node_type_post
from pillar.api.utils import remove_private_keys
force_cli_user()
db = current_app.db()
# Add the blog & post node types to the project.
projects_coll = db['projects']
proj = projects_coll.find_one({'url': proj_url})
if not proj:
log.error('Project url=%s not found', proj_url)
return 3
node_type_utils.add_to_project(proj, (node_type_blog, node_type_post),
replace_existing=False)
proj_id = proj['_id']
r, _, _, status = current_app.put_internal('projects',
remove_private_keys(proj),
_id=proj_id)
if status != 200:
log.error('Error %i storing altered project %s %s', status, proj_id, r)
return 4
log.info('Project saved succesfully.')
# Create a blog node.
nodes_coll = db['nodes']
blog = nodes_coll.find_one({'node_type': 'blog', 'project': proj_id})
if not blog:
blog = {
'node_type': node_type_blog['name'],
'name': 'Blog',
'description': '',
'properties': {},
'project': proj_id,
}
r, _, _, status = current_app.post_internal('nodes', blog)
if status != 201:
log.error('Error %i storing blog node: %s', status, r)
return 4
log.info('Blog node saved succesfully: %s', r)
else:
log.info('Blog node already exists: %s', blog)
return 0
def _assign_users(self, org_id: bson.ObjectId,
unknown_users: typing.Set[str],
existing_users: typing.Set[bson.ObjectId]) -> dict:
if self._log.isEnabledFor(logging.INFO):
self._log.info(' - found users: %s',
', '.join(str(uid) for uid in existing_users))
self._log.info(' - unknown users: %s', ', '.join(unknown_users))
org_doc = self._get_org(org_id)
# Compute the new members.
members = set(org_doc.get('members') or []) | existing_users
unknown_members = set(org_doc.get('unknown_members')
or []) | unknown_users
# Make sure we don't exceed the current seat count.
new_seat_count = len(members) + len(unknown_members)
if new_seat_count > org_doc['seat_count']:
self._log.warning(
'assign_users(%s, ...): Trying to increase seats to %i, '
'but org only has %i seats.', org_id, new_seat_count,
org_doc['seat_count'])
raise NotEnoughSeats(org_id, org_doc['seat_count'], new_seat_count)
# Update the organization.
org_doc['members'] = list(members)
org_doc['unknown_members'] = list(unknown_members)
r, _, _, status = current_app.put_internal(
'organizations', remove_private_keys(org_doc), _id=org_id)
if status != 200:
self._log.error(
'Error updating organization; status should be 200, not %i: %s',
status, r)
raise ValueError(
f'Unable to update organization, status code {status}')
org_doc.update(r)
# Update the roles for the affected members
for uid in existing_users:
self.refresh_roles(uid)
return org_doc
def put_project(project: dict):
"""Puts a project into the database via Eve.
:param project: the project data, should be the entire project document
:raises ValueError: if the project cannot be saved.
"""
from pillar.api.utils import remove_private_keys
from pillarsdk.utils import remove_none_attributes
pid = ObjectId(project['_id'])
proj_no_priv = remove_private_keys(project)
proj_no_none = remove_none_attributes(proj_no_priv)
result, _, _, status_code = current_app.put_internal('projects',
proj_no_none,
_id=pid)
if status_code != 200:
raise ValueError(f"Can't update project {pid}, "
f"status {status_code} with issues: {result}")
def patch_set_username(self, user_id: bson.ObjectId, patch: dict):
"""Updates a user's username."""
if user_id != current_user.user_id:
log.info('User %s tried to change username of user %s',
current_user.user_id, user_id)
raise wz_exceptions.Forbidden(
'You may only change your own username')
new_username = patch['username']
log.info('User %s uses PATCH to set username to %r',
current_user.user_id, new_username)
users_coll = current_app.db('users')
db_user = users_coll.find_one({'_id': user_id})
db_user['username'] = new_username
# Save via Eve to check the schema and trigger update hooks.
response, _, _, status = current_app.put_internal(
'users', remove_private_keys(db_user), _id=user_id)
return jsonify(response), status
def upgrade_attachment_usage(proj_url=None, all_projects=False, go=False):
"""Replaces '@[slug]' with '{attachment slug}'.
Also moves links from the attachment dict to the attachment shortcode.
"""
if bool(proj_url) == all_projects:
log.error('Use either --project or --all.')
return 1
import html
from pillar.api.projects.utils import node_type_dict
from pillar.api.utils import remove_private_keys
from pillar.api.utils.authentication import force_cli_user
force_cli_user()
nodes_coll = current_app.db('nodes')
total_nodes = 0
failed_node_ids = set()
# Use a mixture of the old slug RE that still allowes spaces in the slug
# name and the new RE that allows dashes.
old_slug_re = re.compile(r'@\[([a-zA-Z0-9_\- ]+)\]')
for proj in _db_projects(proj_url, all_projects, go=go):
proj_id = proj['_id']
proj_url = proj.get('url', '-no-url-')
nodes = nodes_coll.find({
'_deleted': {'$ne': True},
'project': proj_id,
'properties.attachments': {'$exists': True},
})
node_count = nodes.count()
if node_count == 0:
log.debug('Skipping project %s (%s)', proj_url, proj_id)
continue
proj_node_types = node_type_dict(proj)
for node in nodes:
attachments = node['properties']['attachments']
replaced = False
# Inner functions because of access to the node's attachments.
def replace(match):
nonlocal replaced
slug = match.group(1)
log.debug(' - OLD STYLE attachment slug %r', slug)
try:
att = attachments[slug]
except KeyError:
log.info("Attachment %r not found for node %s", slug, node['_id'])
link = ''
else:
link = att.get('link', '')
if link == 'self':
link = " link='self'"
elif link == 'custom':
url = att.get('link_custom')
if url:
link = " link='%s'" % html.escape(url)
replaced = True
return '{attachment %r%s}' % (slug.replace(' ', '-'), link)
def update_markdown(value: str) -> str:
return old_slug_re.sub(replace, value)
iter_markdown(proj_node_types, node, update_markdown)
# Remove no longer used properties from attachments
new_attachments = {}
for slug, attachment in attachments.items():
replaced |= 'link' in attachment # link_custom implies link
attachment.pop('link', None)
attachment.pop('link_custom', None)
new_attachments[slug.replace(' ', '-')] = attachment
node['properties']['attachments'] = new_attachments
if replaced:
total_nodes += 1
else:
# Nothing got replaced,
continue
if go:
# Use Eve to PUT, so we have schema checking.
db_node = remove_private_keys(node)
r, _, _, status = current_app.put_internal('nodes', db_node, _id=node['_id'])
if status != 200:
log.error('Error %i storing altered node %s %s', status, node['_id'], r)
failed_node_ids.add(node['_id'])
# raise SystemExit('Error storing node; see log.')
log.debug('Updated node %s: %s', node['_id'], r)
log.info('Project %s (%s) has %d nodes with attachments',
proj_url, proj_id, node_count)
log.info('%s %d nodes', 'Updated' if go else 'Would update', total_nodes)
if failed_node_ids:
log.warning('Failed to update %d of %d nodes: %s', len(failed_node_ids), total_nodes,
', '.join(str(nid) for nid in failed_node_ids))
def replace_attachments(project):
project_url = project.get('url', '-no-url-')
log_proj = _single_logger('Upgrading nodes for project %s (%s)',
project_url, project['_id'])
# Remove empty attachments
if go:
res = nodes_coll.update_many(
{'properties.attachments': {},
'project': project['_id']},
{'$unset': {'properties.attachments': 1}},
)
if res.matched_count > 0:
log_proj()
log.info('Removed %d empty attachment dicts', res.modified_count)
else:
to_remove = nodes_coll.count({'properties.attachments': {},
'project': project['_id']})
if to_remove:
log_proj()
log.info('Would remove %d empty attachment dicts', to_remove)
# Convert attachments.
nodes = nodes_coll.find({
'_deleted': False,
'project': project['_id'],
'node_type': {'$in': list(nts_by_name)},
'properties.attachments': {'$exists': True},
})
for node in nodes:
attachments = node['properties']['attachments']
if not attachments:
# If we're not modifying the database (e.g. go=False),
# any attachments={} will not be filtered out earlier.
if go or attachments != {}:
log_proj()
log.info(' - Node %s (%s) still has empty attachments %r',
node['_id'], node.get('name'), attachments)
continue
if isinstance(attachments, dict):
# This node has already been upgraded.
continue
# Upgrade from list [{'slug': 'xxx', 'oid': 'yyy'}, ...]
# to dict {'xxx': {'oid': 'yyy'}, ...}
log_proj()
log.info(' - Updating schema on node %s (%s)', node['_id'], node.get('name'))
new_atts = {}
for field_info in attachments:
for attachment in field_info.get('files', []):
new_atts[attachment['slug']] = {'oid': attachment['file']}
node['properties']['attachments'] = new_atts
log.info(' from %s to %s', attachments, new_atts)
if go:
# Use Eve to PUT, so we have schema checking.
db_node = remove_private_keys(node)
r, _, _, status = current_app.put_internal('nodes', db_node, _id=node['_id'])
if status != 200:
log.error('Error %i storing altered node %s %s', status, node['_id'], r)
raise SystemExit('Error storing node; see log.')
def replace_pillar_node_type_schemas(project_url=None, all_projects=False, missing=False, go=False,
project_id=None):
"""Replaces the project's node type schemas with the standard Pillar ones.
Non-standard node types are left alone.
"""
from pillar.api.utils.authentication import force_cli_user
force_cli_user()
from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES
from pillar.api.utils import remove_private_keys, doc_diff
will_would = 'Will' if go else 'Would'
projects_changed = projects_seen = 0
for proj in _db_projects(project_url, all_projects, project_id, go=go):
projects_seen += 1
orig_proj = copy.deepcopy(proj)
proj_id = proj['_id']
if 'url' not in proj:
log.warning('Project %s has no URL!', proj_id)
proj_url = proj.get('url', f'-no URL id {proj_id}')
log.debug('Handling project %s', proj_url)
for proj_nt in proj['node_types']:
nt_name = proj_nt['name']
try:
pillar_nt = PILLAR_NAMED_NODE_TYPES[nt_name]
except KeyError:
log.debug(' - skipping non-standard node type "%s"', nt_name)
continue
log.debug(' - replacing schema on node type "%s"', nt_name)
# This leaves node type keys intact that aren't in Pillar's node_type_xxx definitions,
# such as permissions. It also keeps form schemas as-is.
pillar_nt.pop('form_schema', None)
proj_nt.update(copy.deepcopy(pillar_nt))
# Find new node types that aren't in the project yet.
if missing:
project_ntnames = set(nt['name'] for nt in proj['node_types'])
for nt_name in set(PILLAR_NAMED_NODE_TYPES.keys()) - project_ntnames:
log.info(' - Adding node type "%s"', nt_name)
pillar_nt = PILLAR_NAMED_NODE_TYPES[nt_name]
proj['node_types'].append(copy.deepcopy(pillar_nt))
proj_has_difference = False
for key, val1, val2 in doc_diff(orig_proj, proj, falsey_is_equal=False):
if not proj_has_difference:
if proj.get('_deleted', False):
deleted = ' (deleted)'
else:
deleted = ''
log.info('%s change project %s%s', will_would, proj_url, deleted)
proj_has_difference = True
log.info(' %30r: %r → %r', key, val1, val2)
projects_changed += proj_has_difference
if go and proj_has_difference:
# Use Eve to PUT, so we have schema checking.
db_proj = remove_private_keys(proj)
try:
r, _, _, status = current_app.put_internal('projects', db_proj, _id=proj_id)
except Exception:
log.exception('Error saving project %s (url=%s)', proj_id, proj_url)
raise SystemExit(5)
if status != 200:
log.error('Error %i storing altered project %s %s', status, proj['_id'], r)
raise SystemExit('Error storing project, see log.')
log.debug('Project saved succesfully.')
log.info('%s %d of %d projects',
'Changed' if go else 'Would change',
projects_changed, projects_seen)
