def test_other_user_subscriber(self):
from pillar.api.utils import remove_private_keys
# Requesting another user should be limited to full name and email.
user_info = self.get('/api/users/223456789abc123456789abc',
auth_token='token').json()
self.assertNotIn('auth', user_info)
regular_info = remove_private_keys(user_info)
self.assertEqual(PUBLIC_USER_FIELDS, set(regular_info.keys()))
def test_user_adheres_to_schema(self):
from pillar.api.utils import remove_private_keys
# This check is necessary because the API code uses direct MongoDB manipulation,
# which means that the user can end up not matching the Cerberus schema.
self.create_video_and_set_progress()
db_user = self.fetch_user_from_db(self.uid)
r, _, _, status = self.app.put_internal(
'users', payload=remove_private_keys(db_user), _id=db_user['_id'])
self.assertEqual(200, status, r)
def test_put_other_user(self):
from pillar.api.utils import remove_private_keys
# PUTting the user as another user should fail.
user_info = self.get('/api/users/123456789abc123456789abc', auth_token='token').json()
put_user = remove_private_keys(user_info)
self.put('/api/users/123456789abc123456789abc', auth_token='other-token',
json=put_user, etag=user_info['_etag'],
expected_status=403)
def test_edit_with_other_user(self, algolia_index_post_save):
"""Test editing a node as another user than the owner. We ignore the
algolia_index_post_save call in one of the hooks.
"""
from pillar.api.utils import remove_private_keys
from pillarsdk.utils import remove_none_attributes
other_user_id = self.create_user(user_id='cafef005972666988bef6500',
groups=[self.dillo_user_main_grp])
self.create_valid_auth_token(other_user_id, 'other_token')
test_node = copy.deepcopy(self.test_node)
node_doc = self._test_user(test_node, auth_token='other_token')
# Some basic properties need to be set in order to publish correctly
node_doc['properties']['content'] = 'Some great content'
node_doc['properties']['status'] = 'published'
node_doc_no_priv = remove_private_keys(node_doc)
node_doc_no_none = remove_none_attributes(node_doc_no_priv)
# The owner of the post can edit
self.put(f"/api/nodes/{node_doc['_id']}",
json=node_doc_no_none,
headers={'If-Match': node_doc['_etag']},
auth_token='other_token',
expected_status=200)
# Some other user tries to get the node and edit it
resp = self.get(f"/api/nodes/{node_doc['_id']}", auth_token='token')
node_doc = resp.get_json()
node_doc['properties']['content'] = 'Some illegally edited content'
node_doc_no_priv = remove_private_keys(node_doc)
node_doc_no_none = remove_none_attributes(node_doc_no_priv)
# Such user is not allowed
self.put(f"/api/nodes/{node_doc['_id']}",
json=node_doc_no_none,
headers={'If-Match': node_doc['_etag']},
auth_token='token',
expected_status=403)
def generate_all_links(response, now):
"""Generate a new link for the file and all its variations.
:param response: the file document that should be updated.
:param now: datetime that reflects 'now', for consistent expiry generation.
"""
project_id = str(response['project']) if 'project' in response else None
# TODO: add project id to all files
backend = response['backend']
if 'file_path' in response:
response['link'] = generate_link(backend, response['file_path'],
project_id)
else:
import pprint
log.error(
'File without file_path properly, unable to generate links: %s',
pprint.pformat(response))
return
variations = response.get('variations')
if variations:
for variation in variations:
variation['link'] = generate_link(backend, variation['file_path'],
project_id)
# Construct the new expiry datetime.
validity_secs = current_app.config['FILE_LINK_VALIDITY'][backend]
response['link_expires'] = now + datetime.timedelta(seconds=validity_secs)
patch_info = remove_private_keys(response)
# The project could have been soft-deleted, in which case it's fine to
# update the links to the file. However, Eve/Cerberus doesn't allow this;
# removing the 'project' key from the PATCH works around this.
patch_info.pop('project', None)
file_id = ObjectId(response['_id'])
(patch_resp, _, _, _) = current_app.patch_internal('files',
patch_info,
_id=file_id)
if patch_resp.get('_status') == 'ERR':
log.warning('Unable to save new links for file %s: %r',
response['_id'], patch_resp)
# TODO: raise a snag.
response['_updated'] = now
else:
response['_updated'] = patch_resp['_updated']
# Be silly and re-fetch the etag ourselves. TODO: handle this better.
etag_doc = current_app.data.driver.db['files'].find_one({'_id': file_id},
{'_etag': 1})
response['_etag'] = etag_doc['_etag']
def test_editing_as_admin(self):
"""Test that we can set all fields as admin."""
from pillar.api.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
project_info = self._create_user_and_project(['subscriber', 'admin'])
project_url = '/api/projects/%(_id)s' % project_info
resp = self.client.get(project_url)
project = json.loads(resp.data.decode('utf-8'))
# Create another user we can try and assign the project to.
other_user_id = 'f00dd00df00dd00df00dd00d'
self._create_user_with_token(['subscriber'],
'other-token',
user_id=other_user_id)
# Admin user should be able to PUT everything.
put_project = remove_private_keys(project)
put_project['url'] = 'very-offensive-url'
put_project['description'] = 'Blender je besplatan set alata za izradu interaktivnog 3D ' \
'sadržaja pod različitim operativnim sustavima.'
put_project['name'] = 'โครงการปั่นเมฆ'
put_project['summary'] = 'Это переведена на Google'
put_project['is_private'] = False
put_project['status'] = 'pending'
put_project['category'] = 'software'
put_project['user'] = other_user_id
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization': self.make_header('token'),
'Content-Type': 'application/json',
'If-Match': project['_etag']
})
self.assertEqual(200, resp.status_code, resp.data)
# Re-fetch from database to see which fields actually made it there.
# equal to put_project -> changed in DB
# equal to project -> not changed in DB
resp = self.client.get('/api/projects/%s' % project['_id'])
db_proj = json.loads(resp.data)
self.assertEqual(put_project['url'], db_proj['url'])
self.assertEqual(put_project['description'], db_proj['description'])
self.assertEqual(put_project['name'], db_proj['name'])
self.assertEqual(put_project['summary'], db_proj['summary'])
self.assertEqual(put_project['is_private'], db_proj['is_private'])
self.assertEqual(put_project['status'], db_proj['status'])
self.assertEqual(put_project['category'], db_proj['category'])
self.assertEqual(put_project['user'], db_proj['user'])
def create_blog(proj_url):
"""Adds a blog to the project."""
from pillar.api.utils.authentication import force_cli_user
from pillar.api.utils import node_type_utils
from pillar.api.node_types.blog import node_type_blog
from pillar.api.node_types.post import node_type_post
from pillar.api.utils import remove_private_keys
force_cli_user()
db = current_app.db()
# Add the blog & post node types to the project.
projects_coll = db['projects']
proj = projects_coll.find_one({'url': proj_url})
if not proj:
log.error('Project url=%s not found', proj_url)
return 3
node_type_utils.add_to_project(proj, (node_type_blog, node_type_post),
replace_existing=False)
proj_id = proj['_id']
r, _, _, status = current_app.put_internal('projects',
remove_private_keys(proj),
_id=proj_id)
if status != 200:
log.error('Error %i storing altered project %s %s', status, proj_id, r)
return 4
log.info('Project saved succesfully.')
# Create a blog node.
nodes_coll = db['nodes']
blog = nodes_coll.find_one({'node_type': 'blog', 'project': proj_id})
if not blog:
blog = {
'node_type': node_type_blog['name'],
'name': 'Blog',
'description': '',
'properties': {},
'project': proj_id,
}
r, _, _, status = current_app.post_internal('nodes', blog)
if status != 201:
log.error('Error %i storing blog node: %s', status, r)
return 4
log.info('Blog node saved succesfully: %s', r)
else:
log.info('Blog node already exists: %s', blog)
return 0
def test_is_private_updated_by_world_permissions(self):
"""For backward compatibility, is_private should reflect absence of world-GET"""
from pillar.api.utils import remove_private_keys, dumps
project_url = '/api/projects/%s' % self.project_id
put_project = remove_private_keys(self.project)
# Create admin user.
self._create_user_with_token(['admin'],
'admin-token',
user_id='cafef00dbeefcafef00dbeef')
# Make the project public
put_project['permissions']['world'] = ['GET'] # make public
put_project['is_private'] = True # This should be overridden.
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization':
self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': self.project['_etag']
})
self.assertEqual(200, resp.status_code, resp.data)
with self.app.test_request_context():
projects = self.app.data.driver.db['projects']
db_proj = projects.find_one(self.project_id)
self.assertEqual(['GET'], db_proj['permissions']['world'])
self.assertFalse(db_proj['is_private'])
# Make the project private
put_project['permissions']['world'] = []
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization':
self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': db_proj['_etag']
})
self.assertEqual(200, resp.status_code, resp.data)
with self.app.test_request_context():
projects = self.app.data.driver.db['projects']
db_proj = projects.find_one(self.project_id)
self.assertEqual([], db_proj['permissions']['world'])
self.assertTrue(db_proj['is_private'])
def test_requeue_clears_failed_by(self):
"""The 'failed by workers' list should be emptied when a task is re-queued."""
self.assert_job_status('queued')
# The test job consists of 4 tasks; get their IDs through the scheduler.
# This should set the job status to active.
tasks = self.get('/api/flamenco/managers/%s/depsgraph' % self.mngr_id,
auth_token=self.mngr_token).json['depsgraph']
self.assertEqual(4, len(tasks))
# Mark a task as failed by a few workers.
task = {
**tasks[0], "failed_by_workers": [
{
"id":
"5cc05ee49fbac13c12dee430",
"identifier":
"40.68.245.202 (82f813d38f594bcc86913bfb15db1f07000001)"
},
{
"id":
"5cc06df19fbac16ca117c40d",
"identifier":
"40.68.245.202 (82f813d38f594bcc86913bfb15db1f07000003)"
},
]
}
task_url = f'/api/flamenco/tasks/{task["_id"]}'
self.put(
task_url,
etag=task['_etag'],
json=remove_private_keys(task),
auth_token='fladmin-token',
expected_status=200,
)
# Re-queueing the task should clear the `failed_by_workers` list.
self.patch(
task_url,
json={
'op': 'set-task-status',
'status': 'queued'
},
auth_token='fladmin-token',
expected_status=204,
)
requeued_task = self.get(task_url, auth_token='fladmin-token')
self.assertNotIn('failed_by_workers', requeued_task.json)
def test_put_admin(self):
from pillar.api.utils import remove_private_keys
# PUTting a user should work, and not mess up the auth field.
user_info = self.get('/api/users/123456789abc123456789abc', auth_token='token').json()
put_user = remove_private_keys(user_info)
self.put('/api/users/123456789abc123456789abc', auth_token='admin-token',
json=put_user, etag=user_info['_etag'])
# Get directly from MongoDB, Eve blocks access to the auth field.
with self.app.test_request_context():
users = self.app.data.driver.db['users']
db_user = users.find_one(ObjectId('123456789abc123456789abc'))
self.assertIn('auth', db_user)
def _put_test(self, auth_token: typing.Optional[str]):
"""Generic PUT test, should be same result for all cases."""
put_doc = remove_private_keys(self.org_doc)
put_doc['name'] = 'new name'
self.put(f'/api/organizations/{self.org_id}',
json=put_doc,
etag=self.org_doc['_etag'],
auth_token=auth_token,
expected_status=405)
# The name shouldn't have changed in the database.
db_org = self._from_db()
self.assertEqual(self.org_doc['name'], db_org['name'])
def test_replace_user_without_roles(self):
from pillar.api.utils import remove_private_keys
self.enter_app_context()
user_id = bson.ObjectId(24 * '1')
self.create_user(user_id, roles=(), token='token')
user_doc = self.get(f'/api/users/{user_id}',
auth_token='token').get_json()
self.assertNotIn('roles', user_doc)
self.put(f'/api/users/{user_id}',
auth_token='token',
json=remove_private_keys(user_doc),
etag=user_doc['_etag'])
def _assign_users(self, org_id: bson.ObjectId,
unknown_users: typing.Set[str],
existing_users: typing.Set[bson.ObjectId]) -> dict:
if self._log.isEnabledFor(logging.INFO):
self._log.info(' - found users: %s',
', '.join(str(uid) for uid in existing_users))
self._log.info(' - unknown users: %s', ', '.join(unknown_users))
org_doc = self._get_org(org_id)
# Compute the new members.
members = set(org_doc.get('members') or []) | existing_users
unknown_members = set(org_doc.get('unknown_members')
or []) | unknown_users
# Make sure we don't exceed the current seat count.
new_seat_count = len(members) + len(unknown_members)
if new_seat_count > org_doc['seat_count']:
self._log.warning(
'assign_users(%s, ...): Trying to increase seats to %i, '
'but org only has %i seats.', org_id, new_seat_count,
org_doc['seat_count'])
raise NotEnoughSeats(org_id, org_doc['seat_count'], new_seat_count)
# Update the organization.
org_doc['members'] = list(members)
org_doc['unknown_members'] = list(unknown_members)
r, _, _, status = current_app.put_internal(
'organizations', remove_private_keys(org_doc), _id=org_id)
if status != 200:
self._log.error(
'Error updating organization; status should be 200, not %i: %s',
status, r)
raise ValueError(
f'Unable to update organization, status code {status}')
org_doc.update(r)
# Update the roles for the affected members
for uid in existing_users:
self.refresh_roles(uid)
return org_doc
def edit_comment(user_id, node_id, patch):
"""Edits a single comment.
Doesn't do permission checking; users are allowed to edit their own
comment, and this is not something you want to revoke anyway. Admins
can edit all comments.
"""
# Find the node. We need to fetch some more info than we use here, so that
# we can pass this stuff to Eve's patch_internal; that way the validation &
# authorisation system has enough info to work.
nodes_coll = current_app.data.driver.db['nodes']
node = nodes_coll.find_one(node_id)
if node is None:
log.warning('User %s wanted to patch non-existing node %s' %
(user_id, node_id))
raise wz_exceptions.NotFound('Node %s not found' % node_id)
if node['user'] != user_id and not authorization.user_has_role('admin'):
raise wz_exceptions.Forbidden('You can only edit your own comments.')
node = remove_private_keys(node)
node['properties']['content'] = patch['content']
node['properties']['attachments'] = patch.get('attachments', {})
# Use Eve to PUT this node, as that also updates the etag and we want to replace attachments.
r, _, _, status = current_app.put_internal('nodes',
node,
concurrency_check=False,
_id=node_id)
if status != 200:
log.error('Error %i editing comment %s for user %s: %s', status,
node_id, user_id, r)
raise wz_exceptions.InternalServerError('Internal error %i from Eve' %
status)
else:
log.info('User %s edited comment %s', user_id, node_id)
# Fetch the new content, so the client can show these without querying again.
node = nodes_coll.find_one(node_id,
projection={
'properties.content': 1,
'properties._content_html': 1,
})
return status, node
def setup_for_film(project_url):
"""Add Blender Cloud extension_props specific for film projects.
Returns the updated project.
"""
projects_collection = current_app.data.driver.db['projects']
# Find the project in the database.
project = projects_collection.find_one({'url': project_url})
if not project:
raise RuntimeError('Project %s does not exist.' % project_url)
# Set default extension properties. Be careful not to overwrite any properties that
# are already there.
all_extension_props = project.setdefault('extension_props', {})
cloud_extension_props = {
'category': 'film',
'theme_css': '',
# The accent color (can be 'blue' or '#FFBBAA' or 'rgba(1, 1, 1, 1)
'theme_color': '',
'is_in_production': False,
'video_url': '', # Oembeddable url
'poster': None, # File ObjectId
'logo': None, # File ObjectId
# TODO(fsiddi) when we introduce other setup_for_* in Blender Cloud, make available
# at a higher scope
'is_featured': False,
}
all_extension_props.setdefault(EXTENSION_NAME, cloud_extension_props)
project_id = ObjectId(project['_id'])
project = remove_private_keys(project)
result, _, _, status_code = put_internal('projects',
project,
_id=project_id)
if status_code != 200:
raise RuntimeError("Can't update project %s, issues: %s", project_id,
result)
log.info('Project %s was updated for Blender Cloud.', project_url)
def test_user_without_email_address(self):
"""Regular users should always have an email address.
Regular users are created by authentication with Blender ID, so we do not
have to test that (Blender ID ensures there is an email address). We do need
to test PUT access to erase the email address, though.
"""
from pillar.api.utils import remove_private_keys
user_id = self.create_user(24 * 'd', token='user-token')
with self.app.test_request_context():
users_coll = self.app.db().users
db_user = users_coll.find_one(user_id)
puttable = remove_private_keys(db_user)
empty_email = copy.deepcopy(puttable)
empty_email['email'] = ''
without_email = copy.deepcopy(puttable)
del without_email['email']
etag = db_user['_etag']
resp = self.put(f'/api/users/{user_id}', json=puttable, etag=etag,
auth_token='user-token', expected_status=200).json()
etag = resp['_etag']
self.put(f'/api/users/{user_id}', json=empty_email, etag=etag,
auth_token='user-token', expected_status=422)
self.put(f'/api/users/{user_id}', json=without_email, etag=etag,
auth_token='user-token', expected_status=422)
# An admin should be able to edit this user, but also not clear the email address.
self.create_user(24 * 'a', roles={'admin'}, token='admin-token')
resp = self.put(f'/api/users/{user_id}', json=puttable, etag=etag,
auth_token='admin-token', expected_status=200).json()
etag = resp['_etag']
self.put(f'/api/users/{user_id}', json=empty_email, etag=etag,
auth_token='admin-token', expected_status=422)
self.put(f'/api/users/{user_id}', json=without_email, etag=etag,
auth_token='admin-token', expected_status=422)
def put_project(project: dict):
"""Puts a project into the database via Eve.
:param project: the project data, should be the entire project document
:raises ValueError: if the project cannot be saved.
"""
from pillar.api.utils import remove_private_keys
from pillarsdk.utils import remove_none_attributes
pid = ObjectId(project['_id'])
proj_no_priv = remove_private_keys(project)
proj_no_none = remove_none_attributes(proj_no_priv)
result, _, _, status_code = current_app.put_internal('projects',
proj_no_none,
_id=pid)
if status_code != 200:
raise ValueError(f"Can't update project {pid}, "
f"status {status_code} with issues: {result}")
def test_put_job(self, handle_job_status_change):
"""Test that flamenco.jobs.JobManager.handle_job_status_change is called when we PUT."""
from pillar.api.utils import remove_private_keys
self.create_user(24 * 'a',
roles={'admin', 'flamenco-admin'},
token='fladmin-token')
json_job = self.get('/api/flamenco/jobs/%s' % self.job_id,
auth_token='fladmin-token').json
json_job['status'] = 'canceled'
self.put('/api/flamenco/jobs/%s' % self.job_id,
json=remove_private_keys(json_job),
headers={'If-Match': json_job['_etag']},
auth_token='fladmin-token')
handle_job_status_change.assert_called_once_with(self.job_id, 'queued', 'canceled')
def _update_project(project):
"""Updates a project in the database, or SystemExit()s.
:param project: the project data, should be the entire project document
:type: dict
:return: the project
:rtype: dict
"""
from pillar.api.utils import remove_private_keys
project_id = ObjectId(project['_id'])
project = remove_private_keys(project)
result, _, _, status_code = put_internal('projects',
project,
_id=project_id)
if status_code != 200:
raise RuntimeError("Can't update project %s, issues: %s", project_id,
result)
def test_save_own_user(self):
"""Tests that a user can't change their own fields."""
from pillar.api.utils import authentication as auth
from pillar.api.utils import PillarJSONEncoder, remove_private_keys
user_id = self.create_user(roles=['subscriber'], token='token')
def fetch_user():
with self.app.test_request_context():
users_coll = self.app.db('users')
return users_coll.find_one(user_id)
db_user = fetch_user()
updated_fields = remove_private_keys(db_user)
updated_fields['roles'] = ['admin', 'subscriber',
'demo'] # Try to elevate our roles.
# POSTing updated info to a specific user URL is not allowed by Eve.
self.post('/api/users/%s' % user_id,
json=updated_fields,
auth_token='token',
expected_status=405)
# PUT is allowed, but shouldn't change roles.
self.put('/api/users/%s' % user_id,
json=updated_fields,
auth_token='token',
etag=db_user['_etag'])
db_user = fetch_user()
self.assertEqual(['subscriber'], db_user['roles'])
# PATCH should not be allowed.
updated_fields = {'roles': ['admin', 'subscriber', 'demo']}
self.patch('/api/users/%s' % user_id,
json=updated_fields,
auth_token='token',
etag=db_user['_etag'],
expected_status=405)
db_user = fetch_user()
self.assertEqual(['subscriber'], db_user['roles'])
def test_put_without_email_address(self):
from pillar.api.utils import remove_private_keys
from pillar.api.utils.authentication import force_cli_user
from pillar.api.service import create_service_account as create_sa
with self.app.test_request_context():
force_cli_user()
account, token = create_sa('', ['flamenco_manager'],
{'flamenco_manager': {}})
puttable = remove_private_keys(account)
user_id = account['_id']
# The user should be able to edit themselves, even without email address.
etag = account['_etag']
puttable['full_name'] = 'þor'
resp = self.put(f'/api/users/{user_id}',
json=puttable,
auth_token=token['token'],
etag=etag).json()
etag = resp['_etag']
with self.app.test_request_context():
users_coll = self.app.db().users
db_user = users_coll.find_one(user_id)
self.assertNotIn('email', db_user)
self.assertEqual('þor', db_user['full_name'])
# An admin should be able to edit this email-less user.
self.create_user(24 * 'a', roles={'admin'}, token='admin-token')
puttable['username'] = '******'
self.put(f'/api/users/{user_id}',
json=puttable,
auth_token='admin-token',
etag=etag)
with self.app.test_request_context():
users_coll = self.app.db().users
db_user = users_coll.find_one(user_id)
self.assertNotIn('email', db_user)
self.assertEqual('bigdüde', db_user['username'])
def test_put_job(self, handle_job_status_change):
"""Test that flamenco.jobs.JobManager.handle_job_status_change is called when we PUT."""
from pillar.api.utils import remove_private_keys
self.create_user(24 * 'a',
roles={'admin', 'flamenco-admin'},
token='fladmin-token')
json_job = self.get('/api/flamenco/jobs/%s' % self.job_id,
auth_token='fladmin-token').json()
json_job['status'] = 'canceled'
self.put('/api/flamenco/jobs/%s' % self.job_id,
json=remove_private_keys(json_job),
headers={'If-Match': json_job['_etag']},
auth_token='fladmin-token')
handle_job_status_change.assert_called_once_with(
self.job_id, 'queued', 'canceled')
def patch_set_username(self, user_id: bson.ObjectId, patch: dict):
"""Updates a user's username."""
if user_id != current_user.user_id:
log.info('User %s tried to change username of user %s',
current_user.user_id, user_id)
raise wz_exceptions.Forbidden(
'You may only change your own username')
new_username = patch['username']
log.info('User %s uses PATCH to set username to %r',
current_user.user_id, new_username)
users_coll = current_app.db('users')
db_user = users_coll.find_one({'_id': user_id})
db_user['username'] = new_username
# Save via Eve to check the schema and trigger update hooks.
response, _, _, status = current_app.put_internal(
'users', remove_private_keys(db_user), _id=user_id)
return jsonify(response), status
def generate_all_links(response, now):
"""Generate a new link for the file and all its variations.
:param response: the file document that should be updated.
:param now: datetime that reflects 'now', for consistent expiry generation.
"""
project_id = str(response['project']) if 'project' in response else None
# TODO: add project id to all files
backend = response['backend']
response['link'] = generate_link(backend, response['file_path'],
project_id)
variations = response.get('variations')
if variations:
for variation in variations:
variation['link'] = generate_link(backend, variation['file_path'],
project_id)
# Construct the new expiry datetime.
validity_secs = current_app.config['FILE_LINK_VALIDITY'][backend]
response['link_expires'] = now + datetime.timedelta(seconds=validity_secs)
patch_info = remove_private_keys(response)
file_id = ObjectId(response['_id'])
(patch_resp, _, _, _) = current_app.patch_internal('files',
patch_info,
_id=file_id)
if patch_resp.get('_status') == 'ERR':
log.warning('Unable to save new links for file %s: %r',
response['_id'], patch_resp)
# TODO: raise a snag.
response['_updated'] = now
else:
response['_updated'] = patch_resp['_updated']
# Be silly and re-fetch the etag ourselves. TODO: handle this better.
etag_doc = current_app.data.driver.db['files'].find_one({'_id': file_id},
{'_etag': 1})
response['_etag'] = etag_doc['_etag']
def test_delete_node(self):
with self.app.app_context():
self.delete(f'/api/nodes/{self.node_id}',
auth_token='token',
headers={'If-Match': self.node_etag},
expected_status=204)
node_after = self.app.db('nodes').find_one(self.node_id)
self.assertTrue(node_after.get('_deleted'))
project_after = self.app.db('projects').find_one(self.pid)
self.assertIsNone(project_after.get('header_node'))
self.assertNotEqual(self.project['_etag'], project_after['_etag'])
self.assertNotIn(self.node_id, project_after['nodes_blog'])
self.assertNotIn(self.node_id, project_after['nodes_featured'])
self.assertNotIn(self.node_id, project_after['nodes_latest'])
# Verifying that the project is still valid
from pillar.api.utils import remove_private_keys
self.put(f'/api/projects/{self.pid}',
json=remove_private_keys(project_after),
etag=project_after['_etag'],
auth_token='token')
def test_edits_by_nonowner_admin(self):
"""Any admin should be able to edit any project."""
from pillar.api.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
# Create test project.
project = self._create_user_and_project(['subscriber'])
project_id = project['_id']
project_url = '/api/projects/%s' % project_id
# Create test user.
self._create_user_with_token(['admin'], 'admin-token', user_id='cafef00dbeefcafef00dbeef')
# Admin user should be able to PUT.
put_project = remove_private_keys(project)
put_project['name'] = 'โครงการปั่นเมฆ'
resp = self.client.put(project_url,
data=dumps(put_project),
headers={'Authorization': self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': project['_etag']})
self.assertEqual(200, resp.status_code, resp.data)
def setUp(self, **kwargs):
super().setUp(**kwargs)
# Create multiple projects:
# 1) user is member, owned manager assigned to it.
# 2) user is member, non-owned manager assigned to it.
# 3) user is member, no manager assigned to it.
# 4) user is not member, both non-owned and owned manager assigned to it.
# 5) user is not member, only non-owned manager assigned to it.
# 6) user is GET-only member, owned manager assigned to it.
# 7) user is GET-only member, non-owned manager assigned to it.
from pillar.api.projects.utils import get_admin_group_id
from pillar.api.utils import remove_private_keys
# Create the projects
self.project: typing.MutableMapping[int, dict] = {}
self.prid: typing.MutableMapping[int, bson.ObjectId] = {}
for idx in range(1, 8):
admin_id = 24 * str(idx)
proj = self._create_user_and_project(user_id=admin_id,
roles={'subscriber'},
project_name=f'Prøject {idx}',
token=f'token-proj{idx}-admin')
self.project[idx] = proj
self.prid[idx] = bson.ObjectId(proj['_id'])
# For projects 6 and 7, add GET access group
self.group_map = self.create_standard_groups(additional_groups=['get-only-6', 'get-only-7'])
for idx in (6, 7):
self.project[idx]['permissions']['groups'].append({
'group': self.group_map[f'get-only-{idx}'],
'methods': ['GET'],
})
self.put(f'/api/projects/{self.prid[idx]}',
json=remove_private_keys(self.project[idx]),
etag=self.project[idx]['_etag'],
auth_token=f'token-proj{idx}-admin')
# Create the managers
self.owned_mngr, _, self.owned_mngr_token = self.create_manager_service_account()
self.owned_mngr_id = bson.ObjectId(self.owned_mngr['_id'])
self.nonowned_mngr, _, self.nonowned_mngr_token = self.create_manager_service_account()
self.nonowned_mngr_id = bson.ObjectId(self.nonowned_mngr['_id'])
self.assign_manager_to_project(self.owned_mngr_id, self.prid[1])
self.assign_manager_to_project(self.nonowned_mngr_id, self.prid[2])
self.assign_manager_to_project(self.owned_mngr_id, self.prid[4])
self.assign_manager_to_project(self.nonowned_mngr_id, self.prid[5])
self.assign_manager_to_project(self.owned_mngr_id, self.prid[6])
self.assign_manager_to_project(self.nonowned_mngr_id, self.prid[7])
# Create the test user.
self.admin_gid: typing.MutableMapping[int, bson.ObjectId] = {}
with self.app.test_request_context():
for idx, prid in self.prid.items():
self.admin_gid[idx] = get_admin_group_id(prid)
self.create_user(
roles={'subscriber'},
groups=[
self.admin_gid[1],
self.admin_gid[2],
self.admin_gid[3],
self.owned_mngr['owner'],
self.group_map['get-only-6'],
self.group_map['get-only-7'],
], token='user-token')
# Make some assertions about the access rights on the projects.
for idx in (1, 2, 3):
p = self.get(f'/api/projects/{self.prid[idx]}', auth_token='user-token').json
self.assertEqual({'GET', 'PUT', 'DELETE', 'POST'}, set(p['allowed_methods']),
f'Unexpected methods {p["allowed_methods"]} in project nr {idx}')
for idx in (4, 5):
self.get(f'/api/projects/{self.prid[idx]}', auth_token='user-token',
expected_status=403)
for idx in (6, 7):
p = self.get(f'/api/projects/{self.prid[idx]}', auth_token='user-token').json
self.assertEqual({'GET'}, set(p['allowed_methods']),
f'Unexpected methods {p["allowed_methods"]} in project nr {idx}')
def create_home_project(user_id, write_access):
"""Creates a home project for the given user.
:param user_id: the user ID of the owner
:param write_access: whether the user has full write access to the home project.
:type write_access: bool
:returns: the project
:rtype: dict
"""
log.info('Creating home project for user %s', user_id)
overrides = {
'category': 'home',
'url': 'home',
'summary': HOME_PROJECT_SUMMARY,
'description': HOME_PROJECT_DESCRIPTION
}
# Maybe the user has a deleted home project.
proj_coll = current_app.data.driver.db['projects']
deleted_proj = proj_coll.find_one({
'user': user_id,
'category': 'home',
'_deleted': True
})
if deleted_proj:
log.info('User %s has a deleted project %s, restoring', user_id,
deleted_proj['_id'])
project = deleted_proj
else:
log.debug('User %s does not have a deleted project', user_id)
project = proj_utils.create_new_project(project_name='Home',
user_id=ObjectId(user_id),
overrides=overrides)
# Re-validate the authentication token, so that the put_internal call sees the
# new group created for the project.
authentication.validate_token(force=True)
# There are a few things in the on_insert_projects hook we need to adjust.
# Ensure that the project is private, even for admins.
project['permissions']['world'] = []
# Set up the correct node types. No need to set permissions for them,
# as the inherited project permissions are fine.
from pillar.api.node_types.group import node_type_group
from pillar.api.node_types.asset import node_type_asset
# from pillar.api.node_types.text import node_type_text
from pillar.api.node_types.comment import node_type_comment
# For non-subscribers: take away write access from the admin group,
# and grant it to certain node types.
project['permissions']['groups'][0]['methods'] = home_project_permissions(
write_access)
# Everybody should be able to comment on anything in this project.
# This allows people to comment on shared images and see comments.
node_type_comment = assign_permissions(node_type_comment,
subscriber_methods=['GET', 'POST'],
world_methods=['GET'])
project['node_types'] = [
node_type_group,
node_type_asset,
# node_type_text,
node_type_comment,
]
result, _, _, status = current_app.put_internal(
'projects', utils.remove_private_keys(project), _id=project['_id'])
if status != 200:
log.error('Unable to update home project %s for user %s: %s',
project['_id'], user_id, result)
raise wz_exceptions.InternalServerError(
'Unable to update home project')
project.update(result)
# Create the Blender Sync node, with explicit write permissions on the node itself.
create_blender_sync_node(project['_id'],
project['permissions']['groups'][0]['group'],
user_id)
return project
def test_manager_account_access(self):
"""Should have access to own job and tasks, but not project or other managers."""
from pillar.api.utils import remove_private_keys
# Own manager doc should be gettable, but other one should not.
own_url = '/api/flamenco/managers/%s' % self.mngr_id
own_doc = self.get(own_url,
expected_status=200,
auth_token=self.mngr_token).json
other_url = '/api/flamenco/managers/%s' % self.mngr2_id
self.get(other_url,
expected_status=404,
auth_token=self.mngr_token)
# Managers may not create new managers.
new_doc = remove_private_keys(own_doc)
self.post('/api/flamenco/managers', json=new_doc,
expected_status=403,
auth_token=self.mngr_token)
# Manager docs should not be modified.
self.put(own_url, json=remove_private_keys(own_doc),
etag=own_doc['_etag'],
expected_status=403,
auth_token=self.mngr_token)
self.delete(own_url,
expected_status=405,
etag=own_doc['_etag'],
auth_token=self.mngr_token)
self.put(other_url, json=remove_private_keys(own_doc),
expected_status=403,
etag=self.mngr2_doc['_etag'],
auth_token=self.mngr_token)
self.delete(other_url,
expected_status=405,
etag=self.mngr2_doc['_etag'],
auth_token=self.mngr_token)
# Own job should be GETtable.
own_job_url = '/api/flamenco/jobs/%s' % self.job_id
own_job = self.get(own_job_url,
expected_status=200,
auth_token=self.mngr_token).json
resp = self.get('/api/flamenco/jobs',
expected_status=200,
auth_token=self.mngr_token).json
jobs = resp['_items']
self.assertEqual(1, len(jobs))
self.assertEqual(1, resp['_meta']['total'])
self.assertEqual(str(self.job_id), jobs[0]['_id'])
# Own job should not be modifyable.
self.put(own_job_url, json=remove_private_keys(own_job),
etag=own_job['_etag'],
expected_status=403,
auth_token=self.mngr_token)
self.delete(own_job_url,
etag=own_job['_etag'],
expected_status=403,
auth_token=self.mngr_token)
self.delete('/api/flamenco/jobs',
expected_status=403,
auth_token=self.mngr_token)
# Managers may not create new jobs
new_job = remove_private_keys(own_job)
self.post('/api/flamenco/jobs', json=new_job,
expected_status=403,
auth_token=self.mngr_token)
# Job of other manager should not be GETtable.
self.get('/api/flamenco/jobs/%s' % self.job2_id,
expected_status=403,
auth_token=self.mngr_token)
# Manager should not have direct access to tasks; only via scheduler.
self.get('/api/flamenco/tasks',
expected_status=403,
auth_token=self.mngr_token)
# Manager should be able to fetch their own tasks, once the IDs are known.
self.get('/api/flamenco/tasks/%s' % self.tasks_for_job[0]['_id'],
expected_status=200,
auth_token=self.mngr_token)
def after_inserting_project(project, db_user):
from pillar.auth import UserClass
project_id = project['_id']
user_id = db_user['_id']
# Create a project-specific admin group (with name matching the project id)
result, _, _, status = current_app.post_internal('groups',
{'name': str(project_id)})
if status != 201:
log.error('Unable to create admin group for new project %s: %s',
project_id, result)
return abort_with_error(status)
admin_group_id = result['_id']
log.debug('Created admin group %s for project %s', admin_group_id,
project_id)
# Assign the current user to the group
db_user.setdefault('groups', []).append(admin_group_id)
result, _, _, status = current_app.patch_internal(
'users', {'groups': db_user['groups']}, _id=user_id)
if status != 200:
log.error(
'Unable to add user %s as member of admin group %s for new project %s: %s',
user_id, admin_group_id, project_id, result)
return abort_with_error(status)
log.debug('Made user %s member of group %s', user_id, admin_group_id)
# Assign the group to the project with admin rights
owner_user = UserClass.construct('', db_user)
is_admin = authorization.is_admin(owner_user)
world_permissions = ['GET'] if is_admin else []
permissions = {
'world':
world_permissions,
'users': [],
'groups': [
{
'group': admin_group_id,
'methods': DEFAULT_ADMIN_GROUP_PERMISSIONS[:]
},
]
}
def with_permissions(node_type):
copied = copy.deepcopy(node_type)
copied['permissions'] = permissions
return copied
# Assign permissions to the project itself, as well as to the node_types
project['permissions'] = permissions
project['node_types'] = [
with_permissions(node_type_group),
with_permissions(node_type_asset),
with_permissions(node_type_comment),
with_permissions(node_type_texture),
with_permissions(node_type_group_texture),
]
# Allow admin users to use whatever url they want.
if not is_admin or not project.get('url'):
if project.get('category', '') == 'home':
project['url'] = 'home'
else:
project['url'] = "p-{!s}".format(project_id)
# Initialize storage using the default specified in STORAGE_BACKEND
default_storage_backend(str(project_id))
# Commit the changes directly to the MongoDB; a PUT is not allowed yet,
# as the project doesn't have a valid permission structure.
projects_collection = current_app.data.driver.db['projects']
result = projects_collection.update_one(
{'_id': project_id}, {'$set': remove_private_keys(project)})
if result.matched_count != 1:
log.error('Unable to update project %s: %s', project_id,
result.raw_result)
abort_with_error(500)
def zencoder_notifications():
"""
See: https://app.zencoder.com/docs/guides/getting-started/notifications#api_version_2
"""
if current_app.config['ENCODING_BACKEND'] != 'zencoder':
log.warning('Received notification from Zencoder but app not configured for Zencoder.')
return abort(403)
if not current_app.config['DEBUG']:
# If we are in production, look for the Zencoder header secret
try:
notification_secret_request = request.headers[
'X-Zencoder-Notification-Secret']
except KeyError:
log.warning('Received Zencoder notification without secret.')
return abort(401)
# If the header is found, check it agains the one in the config
notification_secret = current_app.config['ZENCODER_NOTIFICATIONS_SECRET']
if notification_secret_request != notification_secret:
log.warning('Received Zencoder notification with incorrect secret.')
return abort(401)
# Cast request data into a dict
data = request.get_json()
if log.isEnabledFor(logging.DEBUG):
from pprint import pformat
log.debug('Zencoder job JSON: %s', pformat(data))
files_collection = current_app.data.driver.db['files']
# Find the file object based on processing backend and job_id
zencoder_job_id = data['job']['id']
lookup = {'processing.backend': 'zencoder',
'processing.job_id': str(zencoder_job_id)}
file_doc = files_collection.find_one(lookup)
if not file_doc:
log.warning('Unknown Zencoder job id %r', zencoder_job_id)
# Return 200 OK when debugging, or Zencoder will keep trying and trying and trying...
# which is what we want in production.
return "Not found, but that's okay.", 200 if current_app.config['DEBUG'] else 404
file_id = ObjectId(file_doc['_id'])
# Remove internal keys (so that we can run put internal)
file_doc = utils.remove_private_keys(file_doc)
# Update processing status
job_state = data['job']['state']
file_doc['processing']['status'] = job_state
if job_state == 'failed':
log.warning('Zencoder job %i for file %s failed.', zencoder_job_id, file_id)
# Log what Zencoder told us went wrong.
for output in data['outputs']:
if not any('error' in key for key in output):
continue
log.warning('Errors for output %s:', output['url'])
for key in output:
if 'error' in key:
log.info(' %s: %s', key, output[key])
file_doc['status'] = 'failed'
current_app.put_internal('files', file_doc, _id=file_id)
return "You failed, but that's okay.", 200
log.info('Zencoder job %s for file %s completed with status %s.', zencoder_job_id, file_id,
job_state)
# For every variation encoded, try to update the file object
root, _ = os.path.splitext(file_doc['file_path'])
for output in data['outputs']:
video_format = output['format']
# Change the zencoder 'mpeg4' format to 'mp4' used internally
video_format = 'mp4' if video_format == 'mpeg4' else video_format
# Find a variation matching format and resolution
variation = next((v for v in file_doc['variations']
if v['format'] == format and v['width'] == output['width']), None)
# Fall back to a variation matching just the format
if variation is None:
variation = next((v for v in file_doc['variations']
if v['format'] == video_format), None)
if variation is None:
log.warning('Unable to find variation for video format %s for file %s',
video_format, file_id)
continue
# Rename the file to include the now-known size descriptor.
size = size_descriptor(output['width'], output['height'])
new_fname = '{}-{}.{}'.format(root, size, video_format)
# Rename on Google Cloud Storage
try:
rename_on_gcs(file_doc['project'],
'_/' + variation['file_path'],
'_/' + new_fname)
except Exception:
log.warning('Unable to rename GCS blob %r to %r. Keeping old name.',
variation['file_path'], new_fname, exc_info=True)
else:
variation['file_path'] = new_fname
# TODO: calculate md5 on the storage
variation.update({
'height': output['height'],
'width': output['width'],
'length': output['file_size_in_bytes'],
'duration': data['input']['duration_in_ms'] / 1000,
'md5': output['md5_checksum'] or '', # they don't do MD5 for GCS...
'size': size,
})
file_doc['status'] = 'complete'
# Force an update of the links on the next load of the file.
file_doc['link_expires'] = datetime.datetime.now(tz=tz_util.utc) - datetime.timedelta(days=1)
current_app.put_internal('files', file_doc, _id=file_id)
return '', 204
def test_manager_account_access(self):
"""Should have access to own job and tasks, but not project or other managers."""
from pillar.api.utils import remove_private_keys
# Own manager doc should be gettable, but other one should not.
own_url = '/api/flamenco/managers/%s' % self.mngr_id
own_doc = self.get(own_url,
expected_status=200,
auth_token=self.mngr_token).json()
other_url = '/api/flamenco/managers/%s' % self.mngr2_id
self.get(other_url, expected_status=404, auth_token=self.mngr_token)
# Managers may not create new managers.
new_doc = remove_private_keys(own_doc)
self.post('/api/flamenco/managers',
json=new_doc,
expected_status=403,
auth_token=self.mngr_token)
# Manager docs should not be modified.
self.put(own_url,
json=remove_private_keys(own_doc),
etag=own_doc['_etag'],
expected_status=403,
auth_token=self.mngr_token)
self.delete(own_url,
expected_status=405,
etag=own_doc['_etag'],
auth_token=self.mngr_token)
self.put(other_url,
json=remove_private_keys(own_doc),
expected_status=403,
etag=self.mngr2_doc['_etag'],
auth_token=self.mngr_token)
self.delete(other_url,
expected_status=405,
etag=self.mngr2_doc['_etag'],
auth_token=self.mngr_token)
# Own job should be GETtable.
own_job_url = '/api/flamenco/jobs/%s' % self.job_id
own_job = self.get(own_job_url,
expected_status=200,
auth_token=self.mngr_token).json()
resp = self.get('/api/flamenco/jobs',
expected_status=200,
auth_token=self.mngr_token).json()
jobs = resp['_items']
self.assertEqual(1, len(jobs))
self.assertEqual(1, resp['_meta']['total'])
self.assertEqual(str(self.job_id), jobs[0]['_id'])
# Own job should not be modifyable.
self.put(own_job_url,
json=remove_private_keys(own_job),
etag=own_job['_etag'],
expected_status=403,
auth_token=self.mngr_token)
self.delete(own_job_url,
etag=own_job['_etag'],
expected_status=403,
auth_token=self.mngr_token)
self.delete('/api/flamenco/jobs',
expected_status=403,
auth_token=self.mngr_token)
# Managers may not create new jobs
new_job = remove_private_keys(own_job)
self.post('/api/flamenco/jobs',
json=new_job,
expected_status=403,
auth_token=self.mngr_token)
# Job of other manager should not be GETtable.
self.get('/api/flamenco/jobs/%s' % self.job2_id,
expected_status=403,
auth_token=self.mngr_token)
# Manager should not have direct access to tasks; only via scheduler.
self.get('/api/flamenco/tasks',
expected_status=403,
auth_token=self.mngr_token)
# Manager should be able to fetch their own tasks, once the IDs are known.
self.get('/api/flamenco/tasks/%s' % self.tasks_for_job[0]['_id'],
expected_status=200,
auth_token=self.mngr_token)
def setUp(self, **kwargs):
super().setUp(**kwargs)
# Create multiple projects:
# 1) user is member, owned manager assigned to it.
# 2) user is member, non-owned manager assigned to it.
# 3) user is member, no manager assigned to it.
# 4) user is not member, both non-owned and owned manager assigned to it.
# 5) user is not member, only non-owned manager assigned to it.
# 6) user is GET-only member, owned manager assigned to it.
# 7) user is GET-only member, non-owned manager assigned to it.
from pillar.api.projects.utils import get_admin_group_id
from pillar.api.utils import remove_private_keys
# Create the projects
self.project: typing.MutableMapping[int, dict] = {}
self.prid: typing.MutableMapping[int, bson.ObjectId] = {}
for idx in range(1, 8):
admin_id = 24 * str(idx)
proj = self._create_user_and_project(
user_id=admin_id,
roles={'subscriber'},
project_name=f'Prøject {idx}',
token=f'token-proj{idx}-admin')
self.project[idx] = proj
self.prid[idx] = bson.ObjectId(proj['_id'])
# For projects 6 and 7, add GET access group
self.group_map = self.create_standard_groups(
additional_groups=['get-only-6', 'get-only-7'])
for idx in (6, 7):
self.project[idx]['permissions']['groups'].append({
'group':
self.group_map[f'get-only-{idx}'],
'methods': ['GET'],
})
self.put(f'/api/projects/{self.prid[idx]}',
json=remove_private_keys(self.project[idx]),
etag=self.project[idx]['_etag'],
auth_token=f'token-proj{idx}-admin')
# Create the managers
self.owned_mngr, _, self.owned_mngr_token = self.create_manager_service_account(
)
self.owned_mngr_id = bson.ObjectId(self.owned_mngr['_id'])
self.nonowned_mngr, _, self.nonowned_mngr_token = self.create_manager_service_account(
)
self.nonowned_mngr_id = bson.ObjectId(self.nonowned_mngr['_id'])
self.assign_manager_to_project(self.owned_mngr_id, self.prid[1])
self.assign_manager_to_project(self.nonowned_mngr_id, self.prid[2])
self.assign_manager_to_project(self.owned_mngr_id, self.prid[4])
self.assign_manager_to_project(self.nonowned_mngr_id, self.prid[5])
self.assign_manager_to_project(self.owned_mngr_id, self.prid[6])
self.assign_manager_to_project(self.nonowned_mngr_id, self.prid[7])
# Create the test user.
self.admin_gid: typing.MutableMapping[int, bson.ObjectId] = {}
with self.app.test_request_context():
for idx, prid in self.prid.items():
self.admin_gid[idx] = get_admin_group_id(prid)
self.create_user(roles={'subscriber', 'flamenco-user'},
groups=[
self.admin_gid[1],
self.admin_gid[2],
self.admin_gid[3],
self.owned_mngr['owner'],
self.group_map['get-only-6'],
self.group_map['get-only-7'],
],
token='user-token')
# Make some assertions about the access rights on the projects.
for idx in (1, 2, 3):
p = self.get(f'/api/projects/{self.prid[idx]}',
auth_token='user-token').json()
self.assertEqual({
'GET', 'PUT', 'DELETE', 'POST'
}, set(
p['allowed_methods']
), f'Unexpected methods {p["allowed_methods"]} in project nr {idx}'
)
for idx in (4, 5):
self.get(f'/api/projects/{self.prid[idx]}',
auth_token='user-token',
expected_status=403)
for idx in (6, 7):
p = self.get(f'/api/projects/{self.prid[idx]}',
auth_token='user-token').json()
self.assertEqual({'GET'}, set(
p['allowed_methods']
), f'Unexpected methods {p["allowed_methods"]} in project nr {idx}'
)
