def activity_object_add(actor_user_id, verb, object_type, object_id,
context_object_type, context_object_id):
"""Add a notification object and creates a notification for each user that
- is not the original author of the post
- is actively subscribed to the object
This works using the following pattern:
ACTOR -> VERB -> OBJECT -> CONTEXT
:param actor_user_id: id of the user who is changing the object
:param verb: the action on the object ('commented', 'replied')
:param object_type: hardcoded name
:param object_id: object id, to be traced with object_type_id
"""
subscriptions = notification_get_subscriptions(
context_object_type, context_object_id, actor_user_id)
if subscriptions.count() == 0:
return
info, status = register_activity(actor_user_id, verb, object_type, object_id,
context_object_type, context_object_id)
if status != 201:
# If creation failed for any reason, do not create a any notifcation
return
for subscription in subscriptions:
notification = dict(
user=subscription['user'],
activity=info['_id'])
current_app.post_internal('notifications', notification)
def setup_db(admin_email):
"""Extends Pillar setup_db."""
from pillar.cli.setup import setup_db as pillar_setup_db
# Define the dillo_user_main group, which is automatically assigned
# after every user creation.
g = {'name': 'dillo_user_main'}
current_app.post_internal('groups', g)
# Execute the default user creation
pillar_setup_db(admin_email)
def store_token(user_id, token: str, token_expiry, oauth_subclient_id=False):
"""Stores an authentication token.
:returns: the token document from MongoDB
"""
assert isinstance(token,
str), 'token must be string type, not %r' % type(token)
token_data = {
'user': user_id,
'token': token,
'expire_time': token_expiry,
}
if oauth_subclient_id:
token_data['is_subclient_token'] = True
r, _, _, status = current_app.post_internal('tokens', token_data)
if status not in {200, 201}:
log.error('Unable to store authentication token: %s', r)
raise RuntimeError('Unable to store authentication token.')
token_data.update(r)
return token_data
def api_create_job(self, job_name, job_desc, job_type, job_settings,
project_id, user_id, manager_id, priority=50,
*, start_paused=False):
"""Creates a job, returning a dict with its generated fields."""
job = {
'name': job_name,
'description': job_desc,
'job_type': job_type,
'project': project_id,
'user': user_id,
'manager': manager_id,
'status': 'under-construction',
'priority': int(priority),
'settings': copy.deepcopy(job_settings),
}
if start_paused:
job['start_paused'] = True
self._log.info('Creating job %r for user %s and manager %s',
job_name, user_id, manager_id)
r, _, _, status = current_app.post_internal('flamenco_jobs', job)
if status != 201:
self._log.error('Status should be 201, not %i: %s' % (status, r))
raise ValueError('Unable to create Flamenco job, status code %i' % status)
job.update(r)
return job
def api_create_task(self, job, commands, name, parents=None, priority=50,
status='queued', *, task_type: str) -> bson.ObjectId:
"""Creates a task in MongoDB for the given job, executing commands.
Returns the ObjectId of the created task.
"""
task = {
'job': job['_id'],
'manager': job['manager'],
'user': job['user'],
'name': name,
'status': status,
'job_type': job['job_type'],
'task_type': task_type,
'commands': [cmd.to_dict() for cmd in commands],
'job_priority': job['priority'],
'priority': priority,
'project': job['project'],
}
# Insertion of None parents is not supported
if parents:
task['parents'] = parents
self._log.info('Creating task %s for manager %s, user %s',
name, job['manager'], job['user'])
r, _, _, status = current_app.post_internal('flamenco_tasks', task)
if status != 201:
self._log.error('Error %i creating task %s: %s',
status, task, r)
raise wz_exceptions.InternalServerError('Unable to create task')
return r['_id']
def api_create_task(self, job, commands, name, parents=None, priority=50,
status='queued', *, task_type: str) -> bson.ObjectId:
"""Creates a task in MongoDB for the given job, executing commands.
Returns the ObjectId of the created task.
"""
task = {
'job': job['_id'],
'manager': job['manager'],
'user': job['user'],
'name': name,
'status': status,
'job_type': job['job_type'],
'task_type': task_type,
'commands': [cmd.to_dict() for cmd in commands],
'job_priority': job['priority'],
'priority': priority,
'project': job['project'],
}
# Insertion of None parents is not supported
if parents:
task['parents'] = parents
self._log.info('Creating task %s for manager %s, user %s',
name, job['manager'], job['user'])
r, _, _, status = current_app.post_internal('flamenco_tasks', task)
if status != 201:
self._log.error('Error %i creating task %s: %s',
status, task, r)
raise wz_exceptions.InternalServerError('Unable to create task')
return r['_id']
def api_create_job(self, job_name, job_desc, job_type, job_settings,
project_id, user_id, manager_id, priority=50):
"""Creates a job, returning a dict with its generated fields."""
job = {
'name': job_name,
'description': job_desc,
'job_type': job_type,
'project': project_id,
'user': user_id,
'manager': manager_id,
'status': 'under-construction',
'priority': int(priority),
'settings': copy.deepcopy(job_settings),
}
self._log.info('Creating job %r for user %s and manager %s',
job_name, user_id, manager_id)
r, _, _, status = current_app.post_internal('flamenco_jobs', job)
if status != 201:
self._log.error('Status should be 201, not %i: %s' % (status, r))
raise ValueError('Unable to create Flamenco job, status code %i' % status)
job.update(r)
return job
def create_file_doc_for_upload(project_id, uploaded_file):
"""Creates a secure filename and a document in MongoDB for the file.
The (project_id, filename) tuple should be unique. If such a document already
exists, it is updated with the new file.
:param uploaded_file: file from request.files['form-key']
:type uploaded_file: werkzeug.datastructures.FileStorage
:returns: a tuple (file_id, filename, status), where 'filename' is the internal
filename used on GCS.
"""
project_id = ObjectId(project_id)
# Hash the filename with path info to get the internal name. This should
# be unique for the project.
# internal_filename = uploaded_file.filename
_, ext = os.path.splitext(uploaded_file.filename)
internal_filename = uuid.uuid4().hex + ext
# For now, we don't support overwriting files, and create a new one every time.
# # See if we can find a pre-existing file doc.
# files = current_app.data.driver.db['files']
# file_doc = files.find_one({'project': project_id,
# 'name': internal_filename})
file_doc = None
# TODO: at some point do name-based and content-based content-type sniffing.
new_props = {
'filename': uploaded_file.filename,
'content_type': uploaded_file.mimetype,
'length': uploaded_file.content_length,
'project': project_id,
'status': 'uploading'
}
if file_doc is None:
# Create a file document on MongoDB for this file.
file_doc = create_file_doc(name=internal_filename, **new_props)
file_fields, _, _, status = current_app.post_internal(
'files', file_doc)
else:
file_doc.update(new_props)
file_fields, _, _, status = current_app.put_internal(
'files', remove_private_keys(file_doc))
if status not in (200, 201):
log.error(
'Unable to create new file document in MongoDB, status=%i: %s',
status, file_fields)
raise wz_exceptions.InternalServerError()
log.debug(
'Created file document %s for uploaded file %s; internal name %s',
file_fields['_id'], uploaded_file.filename, internal_filename)
return file_fields['_id'], internal_filename, status
def activity_subscribe(user_id, context_object_type, context_object_id):
"""Subscribe a user to changes for a specific context. We create a subscription
if none is found.
:param user_id: id of the user we are going to subscribe
:param context_object_type: hardcoded index, check the notifications/model.py
:param context_object_id: object id, to be traced with context_object_type_id
"""
subscriptions_collection = current_app.data.driver.db['activities-subscriptions']
lookup = {
'user': user_id,
'context_object_type': context_object_type,
'context_object': context_object_id
}
subscription = subscriptions_collection.find_one(lookup)
# If no subscription exists, we create one
if not subscription:
current_app.post_internal('activities-subscriptions', lookup)
def register_activity(actor_user_id,
verb,
object_type,
object_id,
context_object_type,
context_object_id,
project_id=None,
node_type=None):
"""Registers an activity.
This works using the following pattern:
ACTOR -> VERB -> OBJECT -> CONTEXT
:param actor_user_id: id of the user who is changing the object
:param verb: the action on the object ('commented', 'replied')
:param object_type: hardcoded name, see database schema
:param object_id: object id, to be traced with object_type
:param context_object_type: the type of the context object, like 'project' or 'node',
see database schema
:param context_object_id:
:param project_id: optional project ID to make the activity easily queryable
per project.
:param node_type: optional, node type of the node receiving the activity.
:returns: tuple (info, status_code), where a successful operation should have
status_code=201. If it is not 201, a warning is logged.
"""
activity = {
'actor_user': actor_user_id,
'verb': verb,
'object_type': object_type,
'object': object_id,
'context_object_type': context_object_type,
'context_object': context_object_id
}
if project_id:
activity['project'] = project_id
if node_type:
activity['node_type'] = node_type
info, _, _, status_code = current_app.post_internal('activities', activity)
if status_code != 201:
log.error('register_activity: code %i creating activity %s: %s',
status_code, activity, info)
else:
log.info('register_activity: user %s "%s" on %s %s, context %s %s',
actor_user_id, verb, object_type, object_id,
context_object_type, context_object_id)
return info, status_code
def create_new_user(email, username, user_id):
"""Creates a new user in our local database.
@param email: the user's email
@param username: the username, which is also used as full name.
@param user_id: the user ID from the Blender ID server.
@returns: the user ID from our local database.
"""
user_data = create_new_user_document(email, user_id, username)
r = current_app.post_internal('users', user_data)
user_id = r[0]['_id']
return user_id
def create_blender_sync_node(project_id, admin_group_id, user_id):
"""Creates a node for Blender Sync, with explicit write access for the admin group.
Writes the node to the database.
:param project_id: ID of the home project
:type project_id: ObjectId
:param admin_group_id: ID of the admin group of the project. This group will
receive write access to the node.
:type admin_group_id: ObjectId
:param user_id: ID of the owner of the node.
:type user_id: ObjectId
:returns: The created node.
:rtype: dict
"""
log.debug('Creating sync node for project %s, user %s', project_id,
user_id)
node = {
'project': ObjectId(project_id),
'node_type': 'group',
'name': SYNC_GROUP_NODE_NAME,
'user': ObjectId(user_id),
'description': SYNC_GROUP_NODE_DESC,
'properties': {
'status': 'published'
},
'permissions': {
'users': [],
'groups': [{
'group': ObjectId(admin_group_id),
'methods': ['GET', 'PUT', 'POST', 'DELETE']
}],
'world': [],
}
}
r, _, _, status = current_app.post_internal('nodes', node)
if status != 201:
log.warning(
'Unable to create Blender Sync node for home project %s: %s',
project_id, r)
raise wz_exceptions.InternalServerError(
'Unable to create Blender Sync node')
node.update(r)
return node
def create_local_user(email, password):
"""For internal user only. Given username and password, create a user."""
# Hash the password
hashed_password = hash_password(password, bcrypt.gensalt())
db_user = create_new_user_document(email, '', email, provider='local',
token=hashed_password)
# Make username unique
db_user['username'] = make_unique_username(email)
# Create the user
r, _, _, status = current_app.post_internal('users', db_user)
if status != 201:
log.error('internal response: %r %r', status, r)
return abort(500)
# Return user ID
return r['_id']
def create_service_account(email: str,
roles: typing.Iterable,
service: dict,
*,
full_name: str = None):
"""Creates a service account with the given roles + the role 'service'.
:param email: optional email address associated with the account.
:param roles: iterable of role names
:param service: dict of the 'service' key in the user.
:param full_name: Full name of the service account. If None, will be set to
something reasonable.
:return: tuple (user doc, token doc)
"""
# Create a user with the correct roles.
roles = sorted(set(roles).union({'service'}))
user_id = bson.ObjectId()
log.info('Creating service account %s with roles %s', user_id, roles)
user = {
'_id': user_id,
'username': f'SRV-{user_id}',
'groups': [],
'roles': roles,
'settings': {
'email_communications': 0
},
'auth': [],
'full_name': full_name or f'SRV-{user_id}',
'service': service
}
if email:
user['email'] = email
result, _, _, status = current_app.post_internal('users', user)
if status != 201:
raise ServiceAccountCreationError('Error creating user {}: {}'.format(
user_id, result))
user.update(result)
# Create an authentication token that won't expire for a long time.
token = generate_auth_token(user['_id'])
return user, token
def _ensure_user_main_group() -> ObjectId:
"""Retrieve the dillo_user_main group.
If the group does not exist, create it.
Returns the group ObjectId.
"""
grp_collection = current_app.data.driver.db['groups']
dillo_user_main_group = grp_collection.find_one(
{'name': 'dillo_user_main'})
if not dillo_user_main_group:
dillo_user_main_group, _, _, status = current_app.post_internal(
'groups', {'name': 'dillo_user_main'})
if status != 201:
log.error('Unable to create dillo_user_main group')
return abort_with_error(status)
return dillo_user_main_group['_id']
def store_token(
user_id,
token: str,
token_expiry,
oauth_subclient_id=False,
*,
org_roles: typing.Set[str] = frozenset(),
oauth_scopes: typing.Optional[typing.List[str]] = None,
):
"""Stores an authentication token.
:returns: the token document from MongoDB
"""
assert isinstance(token,
str), 'token must be string type, not %r' % type(token)
token_data = {
'user': user_id,
'token': token,
'expire_time': token_expiry,
}
if oauth_subclient_id:
token_data['is_subclient_token'] = True
if org_roles:
token_data['org_roles'] = sorted(org_roles)
if oauth_scopes:
token_data['oauth_scopes'] = oauth_scopes
r, _, _, status = current_app.post_internal('tokens', token_data)
if status not in {200, 201}:
log.error('Unable to store authentication token: %s', r)
raise RuntimeError('Unable to store authentication token.')
token_data.update(r)
return token_data
def create_new_project(project_name, user_id, overrides):
"""Creates a new project owned by the given user."""
log.info('Creating new project "%s" for user %s', project_name, user_id)
# Create the project itself, the rest will be done by the after-insert hook.
project = {
'description': '',
'name': project_name,
'node_types': [],
'status': 'published',
'user': user_id,
'is_private': True,
'permissions': {},
'url': '',
'summary': '',
'category': 'assets', # TODO: allow the user to choose this.
}
if overrides is not None:
project.update(overrides)
result, _, _, status = current_app.post_internal('projects', project)
if status != 201:
log.error('Unable to create project "%s": %s', project_name, result)
return abort_with_error(status)
project.update(result)
# Now re-fetch the project, as both the initial document and the returned
# result do not contain the same etag as the database. This also updates
# other fields set by hooks.
document = current_app.data.driver.db['projects'].find_one(project['_id'])
project.update(document)
log.info('Created project %s for user %s', project['_id'], user_id)
return project
def after_inserting_project(project, db_user):
from pillar.auth import UserClass
project_id = project['_id']
user_id = db_user['_id']
# Create a project-specific admin group (with name matching the project id)
result, _, _, status = current_app.post_internal('groups',
{'name': str(project_id)})
if status != 201:
log.error('Unable to create admin group for new project %s: %s',
project_id, result)
return abort_with_error(status)
admin_group_id = result['_id']
log.debug('Created admin group %s for project %s', admin_group_id,
project_id)
# Assign the current user to the group
db_user.setdefault('groups', []).append(admin_group_id)
result, _, _, status = current_app.patch_internal(
'users', {'groups': db_user['groups']}, _id=user_id)
if status != 200:
log.error(
'Unable to add user %s as member of admin group %s for new project %s: %s',
user_id, admin_group_id, project_id, result)
return abort_with_error(status)
log.debug('Made user %s member of group %s', user_id, admin_group_id)
# Assign the group to the project with admin rights
owner_user = UserClass.construct('', db_user)
is_admin = authorization.is_admin(owner_user)
world_permissions = ['GET'] if is_admin else []
permissions = {
'world':
world_permissions,
'users': [],
'groups': [
{
'group': admin_group_id,
'methods': DEFAULT_ADMIN_GROUP_PERMISSIONS[:]
},
]
}
def with_permissions(node_type):
copied = copy.deepcopy(node_type)
copied['permissions'] = permissions
return copied
# Assign permissions to the project itself, as well as to the node_types
project['permissions'] = permissions
project['node_types'] = [
with_permissions(node_type_group),
with_permissions(node_type_asset),
with_permissions(node_type_comment),
with_permissions(node_type_texture),
with_permissions(node_type_group_texture),
]
# Allow admin users to use whatever url they want.
if not is_admin or not project.get('url'):
if project.get('category', '') == 'home':
project['url'] = 'home'
else:
project['url'] = "p-{!s}".format(project_id)
# Initialize storage using the default specified in STORAGE_BACKEND
default_storage_backend(str(project_id))
# Commit the changes directly to the MongoDB; a PUT is not allowed yet,
# as the project doesn't have a valid permission structure.
projects_collection = current_app.data.driver.db['projects']
result = projects_collection.update_one(
{'_id': project_id}, {'$set': remove_private_keys(project)})
if result.matched_count != 1:
log.error('Unable to update project %s: %s', project_id,
result.raw_result)
abort_with_error(500)
def upsert_user(db_user):
"""Inserts/updates the user in MongoDB.
Retries a few times when there are uniqueness issues in the username.
:returns: the user's database ID and the status of the PUT/POST.
The status is 201 on insert, and 200 on update.
:type: (ObjectId, int)
"""
if 'subscriber' in db_user.get('groups', []):
log.error('Non-ObjectID string found in user.groups: %s', db_user)
raise wz_exceptions.InternalServerError(
'Non-ObjectID string found in user.groups: %s' % db_user)
if not db_user['full_name']:
# Blender ID doesn't need a full name, but we do.
db_user['full_name'] = db_user['username']
r = {}
for retry in range(5):
if '_id' in db_user:
# Update the existing user
attempted_eve_method = 'PUT'
db_id = db_user['_id']
r, _, _, status = current_app.put_internal(
'users', remove_private_keys(db_user), _id=db_id)
if status == 422:
log.error(
'Status %i trying to PUT user %s with values %s, should not happen! %s',
status, db_id, remove_private_keys(db_user), r)
else:
# Create a new user, retry for non-unique usernames.
attempted_eve_method = 'POST'
r, _, _, status = current_app.post_internal('users', db_user)
if status not in {200, 201}:
log.error('Status %i trying to create user with values %s: %s',
status, db_user, r)
raise wz_exceptions.InternalServerError()
db_id = r['_id']
db_user.update(r) # update with database/eve-generated fields.
if status == 422:
# Probably non-unique username, so retry a few times with different usernames.
log.info('Error creating new user: %s', r)
username_issue = r.get('_issues', {}).get('username', '')
if 'not unique' in username_issue:
# Retry
db_user['username'] = make_unique_username(db_user['email'])
continue
# Saving was successful, or at least didn't break on a non-unique username.
break
else:
log.error('Unable to create new user %s: %s', db_user, r)
raise wz_exceptions.InternalServerError()
if status not in (200, 201):
log.error('internal response from %s to Eve: %r %r',
attempted_eve_method, status, r)
raise wz_exceptions.InternalServerError()
return db_id, status
