def test_editing_as_subscriber(self):
"""Test that we can set certain fields, but not all."""
from application.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
project_info = self._create_user_and_project([u'subscriber'])
project_url = '/projects/%(_id)s' % project_info
resp = self.client.get(project_url,
headers={'Authorization': self.make_header('token')})
project = json.loads(resp.data.decode('utf-8'))
# Create another user we can try and assign the project to.
other_user_id = 'f00dd00df00dd00df00dd00d'
self._create_user_with_token(['subscriber'], 'other-token', user_id=other_user_id)
# Unauthenticated should be forbidden
resp = self.client.put('/projects/%s' % project['_id'],
data=dumps(remove_private_keys(project)),
headers={'Content-Type': 'application/json'})
self.assertEqual(403, resp.status_code)
# Regular user should be able to PUT, but only be able to edit certain fields.
put_project = remove_private_keys(project)
put_project['url'] = u'very-offensive-url'
put_project['description'] = u'Blender je besplatan set alata za izradu interaktivnog 3D ' \
u'sadržaja pod različitim operativnim sustavima.'
put_project['name'] = u'โครงการปั่นเมฆ'
put_project['summary'] = u'Это переведена на Google'
put_project['status'] = 'pending'
put_project['category'] = 'software'
put_project['user'] = other_user_id
# Try making the project public. This should update is_private as well.
put_project['permissions']['world'] = ['GET']
resp = self.client.put(project_url,
data=dumps(put_project),
headers={'Authorization': self.make_header('token'),
'Content-Type': 'application/json',
'If-Match': project['_etag']})
self.assertEqual(200, resp.status_code, resp.data)
# Re-fetch from database to see which fields actually made it there.
# equal to put_project -> changed in DB
# equal to project -> not changed in DB
resp = self.client.get(project_url,
headers={'Authorization': self.make_header('token')})
db_proj = json.loads(resp.data)
self.assertEqual(project['url'], db_proj['url'])
self.assertEqual(put_project['description'], db_proj['description'])
self.assertEqual(put_project['name'], db_proj['name'])
self.assertEqual(put_project['summary'], db_proj['summary'])
self.assertEqual(project['status'], db_proj['status'])
self.assertEqual(project['category'], db_proj['category'])
# Project should be consistent.
self.assertEqual(False, db_proj['is_private'])
self.assertEqual(['GET'], db_proj['permissions']['world'])
def _generate_all_links(response, now):
"""Generate a new link for the file and all its variations.
:param response: the file document that should be updated.
:param now: datetime that reflects 'now', for consistent expiry generation.
"""
project_id = str(
response['project']) if 'project' in response else None # TODO: add project id to all files
backend = response['backend']
response['link'] = generate_link(backend, response['file_path'], project_id)
variations = response.get('variations')
if variations:
for variation in variations:
variation['link'] = generate_link(backend, variation['file_path'], project_id)
# Construct the new expiry datetime.
validity_secs = current_app.config['FILE_LINK_VALIDITY'][backend]
response['link_expires'] = now + datetime.timedelta(seconds=validity_secs)
patch_info = remove_private_keys(response)
file_id = ObjectId(response['_id'])
(patch_resp, _, _, _) = patch_internal('files', patch_info, _id=file_id)
if patch_resp.get('_status') == 'ERR':
log.warning('Unable to save new links for file %s: %r', response['_id'], patch_resp)
# TODO: raise a snag.
response['_updated'] = now
else:
response['_updated'] = patch_resp['_updated']
# Be silly and re-fetch the etag ourselves. TODO: handle this better.
etag_doc = current_app.data.driver.db['files'].find_one({'_id': file_id}, {'_etag': 1})
response['_etag'] = etag_doc['_etag']
def test_edits_by_nonowner_subscriber(self):
"""A subscriber should only be able to edit their own projects."""
from application.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
# Create test project.
project = self._create_user_and_project([u'subscriber'])
project_id = project['_id']
project_url = '/projects/%s' % project_id
# Create test user.
my_user_id = 'cafef00dbeefcafef00dbeef'
self._create_user_with_token(['subscriber'], 'mortal-token', user_id=my_user_id)
# Regular subscriber should not be able to do this.
put_project = remove_private_keys(project)
put_project['name'] = u'Болту́н -- нахо́дка для шпио́на.'
put_project['user'] = my_user_id
resp = self.client.put(project_url,
data=dumps(put_project),
headers={'Authorization': self.make_header('mortal-token'),
'Content-Type': 'application/json',
'If-Match': project['_etag']})
self.assertEqual(403, resp.status_code, resp.data)
def test_edits_by_nonowner_subscriber(self):
"""A subscriber should only be able to edit their own projects."""
from application.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
# Create test project.
project = self._create_user_and_project([u'subscriber'])
project_id = project['_id']
project_url = '/projects/%s' % project_id
# Create test user.
my_user_id = 'cafef00dbeefcafef00dbeef'
self._create_user_with_token(['subscriber'],
'mortal-token',
user_id=my_user_id)
# Regular subscriber should not be able to do this.
put_project = remove_private_keys(project)
put_project['name'] = u'Болту́н -- нахо́дка для шпио́на.'
put_project['user'] = my_user_id
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization':
self.make_header('mortal-token'),
'Content-Type':
'application/json',
'If-Match':
project['_etag']
})
self.assertEqual(403, resp.status_code, resp.data)
def test_edits_by_nonowner_admin(self):
"""Any admin should be able to edit any project."""
from application.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
# Create test project.
project = self._create_user_and_project([u'subscriber'])
project_id = project['_id']
project_url = '/projects/%s' % project_id
# Create test user.
self._create_user_with_token(['admin'],
'admin-token',
user_id='cafef00dbeef')
# Admin user should be able to PUT.
put_project = remove_private_keys(project)
put_project['name'] = u'โครงการปั่นเมฆ'
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization':
self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': project['_etag']
})
self.assertEqual(200, resp.status_code, resp.data)
def test_put_admin(self):
from application.utils import remove_private_keys
# PUTting a user should work, and not mess up the auth field.
resp = self.client.get(
'/users/123456789abc123456789abc',
headers={'Authorization': self.make_header('token')})
user_info = json.loads(resp.data)
put_user = remove_private_keys(user_info)
resp = self.client.put('/users/123456789abc123456789abc',
headers={
'Authorization':
self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': user_info['_etag']
},
data=json.dumps(put_user))
self.assertEqual(200, resp.status_code, resp.data)
# Get directly from MongoDB, Eve blocks access to the auth field.
with self.app.test_request_context():
users = self.app.data.driver.db['users']
db_user = users.find_one(ObjectId('123456789abc123456789abc'))
self.assertIn('auth', db_user)
def test_user_anonymous(self):
from application.utils import remove_private_keys
# Getting a user should be limited to certain fields
resp = self.client.get('/users/123456789abc123456789abc')
self.assertEqual(200, resp.status_code)
user_info = json.loads(resp.data)
regular_info = remove_private_keys(user_info)
self.assertEqual(PUBLIC_USER_FIELDS, set(regular_info.keys()))
def test_user_anonymous(self):
from application.utils import remove_private_keys
# Getting a user should be limited to certain fields
resp = self.client.get('/users/123456789abc123456789abc')
self.assertEqual(200, resp.status_code)
user_info = json.loads(resp.data)
regular_info = remove_private_keys(user_info)
self.assertEqual(PUBLIC_USER_FIELDS, set(regular_info.keys()))
def test_save_own_user(self):
"""Tests that a user can't change their own fields."""
from application.utils import authentication as auth
from application.utils import PillarJSONEncoder, remove_private_keys
user_id = self.create_user(roles=[u'subscriber'])
now = datetime.datetime.now(tz_util.utc)
future = now + datetime.timedelta(days=1)
with self.app.test_request_context():
auth.store_token(user_id, 'nonexpired-main', future, None)
with self.app.test_request_context(
headers={'Authorization': self.make_header('nonexpired-main')}):
self.assertTrue(auth.validate_token())
users = self.app.data.driver.db['users']
db_user = users.find_one(user_id)
updated_fields = remove_private_keys(db_user)
updated_fields['roles'] = ['admin', 'subscriber', 'demo'] # Try to elevate our roles.
# POSTing updated info to a specific user URL is not allowed by Eve.
resp = self.client.post('/users/%s' % user_id,
data=json.dumps(updated_fields, cls=PillarJSONEncoder),
headers={'Authorization': self.make_header('nonexpired-main'),
'Content-Type': 'application/json'})
self.assertEqual(405, resp.status_code)
# PUT and PATCH should not be allowed.
resp = self.client.put('/users/%s' % user_id,
data=json.dumps(updated_fields, cls=PillarJSONEncoder),
headers={'Authorization': self.make_header('nonexpired-main'),
'Content-Type': 'application/json'})
self.assertEqual(403, resp.status_code)
updated_fields = {'roles': ['admin', 'subscriber', 'demo']}
resp = self.client.patch('/users/%s' % user_id,
data=json.dumps(updated_fields, cls=PillarJSONEncoder),
headers={'Authorization': self.make_header('nonexpired-main'),
'Content-Type': 'application/json'})
self.assertEqual(405, resp.status_code)
# After all of this, the roles should be the same.
with self.app.test_request_context(
headers={'Authorization': self.make_header('nonexpired-main')}):
self.assertTrue(auth.validate_token())
users = self.app.data.driver.db['users']
db_user = users.find_one(user_id)
self.assertEqual([u'subscriber'], db_user['roles'])
def test_editing_as_admin(self):
"""Test that we can set all fields as admin."""
from application.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
project_info = self._create_user_and_project([u'subscriber', u'admin'])
project_url = '/projects/%(_id)s' % project_info
resp = self.client.get(project_url)
project = json.loads(resp.data.decode('utf-8'))
# Create another user we can try and assign the project to.
other_user_id = 'f00dd00df00dd00df00dd00d'
self._create_user_with_token(['subscriber'],
'other-token',
user_id=other_user_id)
# Admin user should be able to PUT everything.
put_project = remove_private_keys(project)
put_project['url'] = u'very-offensive-url'
put_project['description'] = u'Blender je besplatan set alata za izradu interaktivnog 3D ' \
u'sadržaja pod različitim operativnim sustavima.'
put_project['name'] = u'โครงการปั่นเมฆ'
put_project['summary'] = u'Это переведена на Google'
put_project['is_private'] = False
put_project['status'] = 'pending'
put_project['category'] = 'software'
put_project['user'] = other_user_id
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization': self.make_header('token'),
'Content-Type': 'application/json',
'If-Match': project['_etag']
})
self.assertEqual(200, resp.status_code, resp.data)
# Re-fetch from database to see which fields actually made it there.
# equal to put_project -> changed in DB
# equal to project -> not changed in DB
resp = self.client.get('/projects/%s' % project['_id'])
db_proj = json.loads(resp.data)
self.assertEqual(put_project['url'], db_proj['url'])
self.assertEqual(put_project['description'], db_proj['description'])
self.assertEqual(put_project['name'], db_proj['name'])
self.assertEqual(put_project['summary'], db_proj['summary'])
self.assertEqual(put_project['is_private'], db_proj['is_private'])
self.assertEqual(put_project['status'], db_proj['status'])
self.assertEqual(put_project['category'], db_proj['category'])
self.assertEqual(put_project['user'], db_proj['user'])
def create_file_doc_for_upload(project_id, uploaded_file):
"""Creates a secure filename and a document in MongoDB for the file.
The (project_id, filename) tuple should be unique. If such a document already
exists, it is updated with the new file.
:param uploaded_file: file from request.files['form-key']
:type uploaded_file: werkzeug.datastructures.FileStorage
:returns: a tuple (file_id, filename, status), where 'filename' is the internal
filename used on GCS.
"""
project_id = ObjectId(project_id)
# Hash the filename with path info to get the internal name. This should
# be unique for the project.
# internal_filename = uploaded_file.filename
_, ext = os.path.splitext(uploaded_file.filename)
internal_filename = uuid.uuid4().hex + ext
# For now, we don't support overwriting files, and create a new one every time.
# # See if we can find a pre-existing file doc.
# files = current_app.data.driver.db['files']
# file_doc = files.find_one({'project': project_id,
# 'name': internal_filename})
file_doc = None
# TODO: at some point do name-based and content-based content-type sniffing.
new_props = {
'filename': uploaded_file.filename,
'content_type': uploaded_file.mimetype,
'length': uploaded_file.content_length,
'project': project_id,
'status': 'uploading'
}
if file_doc is None:
# Create a file document on MongoDB for this file.
file_doc = create_file_doc(name=internal_filename, **new_props)
file_fields, _, _, status = post_internal('files', file_doc)
else:
file_doc.update(new_props)
file_fields, _, _, status = put_internal('files',
remove_private_keys(file_doc))
if status not in (200, 201):
log.error(
'Unable to create new file document in MongoDB, status=%i: %s',
status, file_fields)
raise InternalServerError()
return file_fields['_id'], internal_filename, status
def test_other_user_subscriber(self):
from application.utils import remove_private_keys
# Requesting another user should be limited to full name and email.
resp = self.client.get('/users/%s' % '223456789abc123456789abc',
headers={'Authorization': self.make_header('token')})
user_info = json.loads(resp.data)
self.assertEqual(200, resp.status_code)
self.assertNotIn('auth', user_info)
regular_info = remove_private_keys(user_info)
self.assertEqual(PUBLIC_USER_FIELDS, set(regular_info.keys()))
def test_is_private_updated_by_world_permissions(self):
"""For backward compatibility, is_private should reflect absence of world-GET"""
from application.utils import remove_private_keys, dumps
project_url = '/projects/%s' % self.project_id
put_project = remove_private_keys(self.project)
# Create admin user.
self._create_user_with_token(['admin'],
'admin-token',
user_id='cafef00dbeef')
# Make the project public
put_project['permissions']['world'] = ['GET'] # make public
put_project['is_private'] = True # This should be overridden.
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization':
self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': self.project['_etag']
})
self.assertEqual(200, resp.status_code, resp.data)
with self.app.test_request_context():
projects = self.app.data.driver.db['projects']
db_proj = projects.find_one(self.project_id)
self.assertEqual(['GET'], db_proj['permissions']['world'])
self.assertFalse(db_proj['is_private'])
# Make the project private
put_project['permissions']['world'] = []
resp = self.client.put(project_url,
data=dumps(put_project),
headers={
'Authorization':
self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': db_proj['_etag']
})
self.assertEqual(200, resp.status_code, resp.data)
with self.app.test_request_context():
projects = self.app.data.driver.db['projects']
db_proj = projects.find_one(self.project_id)
self.assertEqual([], db_proj['permissions']['world'])
self.assertTrue(db_proj['is_private'])
def test_other_user_subscriber(self):
from application.utils import remove_private_keys
# Requesting another user should be limited to full name and email.
resp = self.client.get(
'/users/%s' % '223456789abc123456789abc',
headers={'Authorization': self.make_header('token')})
user_info = json.loads(resp.data)
self.assertEqual(200, resp.status_code)
self.assertNotIn('auth', user_info)
regular_info = remove_private_keys(user_info)
self.assertEqual(PUBLIC_USER_FIELDS, set(regular_info.keys()))
def test_put_other_user(self):
from application.utils import remove_private_keys
# PUTting the user as another user should fail.
resp = self.client.get('/users/123456789abc123456789abc',
headers={'Authorization': self.make_header('token')})
user_info = json.loads(resp.data)
put_user = remove_private_keys(user_info)
resp = self.client.put('/users/123456789abc123456789abc',
headers={'Authorization': self.make_header('other-token'),
'Content-Type': 'application/json',
'If-Match': user_info['_etag']},
data=json.dumps(put_user))
self.assertEqual(403, resp.status_code, resp.data)
def create_file_doc_for_upload(project_id, uploaded_file):
"""Creates a secure filename and a document in MongoDB for the file.
The (project_id, filename) tuple should be unique. If such a document already
exists, it is updated with the new file.
:param uploaded_file: file from request.files['form-key']
:type uploaded_file: werkzeug.datastructures.FileStorage
:returns: a tuple (file_id, filename, status), where 'filename' is the internal
filename used on GCS.
"""
project_id = ObjectId(project_id)
# Hash the filename with path info to get the internal name. This should
# be unique for the project.
# internal_filename = uploaded_file.filename
_, ext = os.path.splitext(uploaded_file.filename)
internal_filename = uuid.uuid4().hex + ext
# For now, we don't support overwriting files, and create a new one every time.
# # See if we can find a pre-existing file doc.
# files = current_app.data.driver.db['files']
# file_doc = files.find_one({'project': project_id,
# 'name': internal_filename})
file_doc = None
# TODO: at some point do name-based and content-based content-type sniffing.
new_props = {'filename': uploaded_file.filename,
'content_type': uploaded_file.mimetype,
'length': uploaded_file.content_length,
'project': project_id,
'status': 'uploading'}
if file_doc is None:
# Create a file document on MongoDB for this file.
file_doc = create_file_doc(name=internal_filename, **new_props)
file_fields, _, _, status = post_internal('files', file_doc)
else:
file_doc.update(new_props)
file_fields, _, _, status = put_internal('files', remove_private_keys(file_doc))
if status not in (200, 201):
log.error('Unable to create new file document in MongoDB, status=%i: %s',
status, file_fields)
raise InternalServerError()
return file_fields['_id'], internal_filename, status
def test_is_private_updated_by_world_permissions(self):
"""For backward compatibility, is_private should reflect absence of world-GET"""
from application.utils import remove_private_keys, dumps
project_url = '/projects/%s' % self.project_id
put_project = remove_private_keys(self.project)
# Create admin user.
self._create_user_with_token(['admin'], 'admin-token', user_id='cafef00dbeef')
# Make the project public
put_project['permissions']['world'] = ['GET'] # make public
put_project['is_private'] = True # This should be overridden.
resp = self.client.put(project_url,
data=dumps(put_project),
headers={'Authorization': self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': self.project['_etag']})
self.assertEqual(200, resp.status_code, resp.data)
with self.app.test_request_context():
projects = self.app.data.driver.db['projects']
db_proj = projects.find_one(self.project_id)
self.assertEqual(['GET'], db_proj['permissions']['world'])
self.assertFalse(db_proj['is_private'])
# Make the project private
put_project['permissions']['world'] = []
resp = self.client.put(project_url,
data=dumps(put_project),
headers={'Authorization': self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': db_proj['_etag']})
self.assertEqual(200, resp.status_code, resp.data)
with self.app.test_request_context():
projects = self.app.data.driver.db['projects']
db_proj = projects.find_one(self.project_id)
self.assertEqual([], db_proj['permissions']['world'])
self.assertTrue(db_proj['is_private'])
def test_put_other_user(self):
from application.utils import remove_private_keys
# PUTting the user as another user should fail.
resp = self.client.get(
'/users/123456789abc123456789abc',
headers={'Authorization': self.make_header('token')})
user_info = json.loads(resp.data)
put_user = remove_private_keys(user_info)
resp = self.client.put('/users/123456789abc123456789abc',
headers={
'Authorization':
self.make_header('other-token'),
'Content-Type': 'application/json',
'If-Match': user_info['_etag']
},
data=json.dumps(put_user))
self.assertEqual(403, resp.status_code, resp.data)
def _update_project(project_uuid, project):
"""Updates a project in the database, or SystemExit()s.
:param project_uuid: UUID of the project
:type: str
:param project: the project data, should be the entire project document
:type: dict
:return: the project
:rtype: dict
"""
from application.utils import remove_private_keys
project_id = ObjectId(project_uuid)
project = remove_private_keys(project)
result, _, _, _ = put_internal('projects', project, _id=project_id)
if result['_status'] != 'OK':
log.error("Can't update project %s, issues: %s", project_uuid, result['_issues'])
raise SystemExit()
def test_put_admin(self):
from application.utils import remove_private_keys
# PUTting a user should work, and not mess up the auth field.
resp = self.client.get('/users/123456789abc123456789abc',
headers={'Authorization': self.make_header('token')})
user_info = json.loads(resp.data)
put_user = remove_private_keys(user_info)
resp = self.client.put('/users/123456789abc123456789abc',
headers={'Authorization': self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': user_info['_etag']},
data=json.dumps(put_user))
self.assertEqual(200, resp.status_code, resp.data)
# Get directly from MongoDB, Eve blocks access to the auth field.
with self.app.test_request_context():
users = self.app.data.driver.db['users']
db_user = users.find_one(ObjectId('123456789abc123456789abc'))
self.assertIn('auth', db_user)
def _generate_all_links(response, now):
"""Generate a new link for the file and all its variations.
:param response: the file document that should be updated.
:param now: datetime that reflects 'now', for consistent expiry generation.
"""
project_id = str(
response['project']
) if 'project' in response else None # TODO: add project id to all files
backend = response['backend']
response['link'] = generate_link(backend, response['file_path'],
project_id)
variations = response.get('variations')
if variations:
for variation in variations:
variation['link'] = generate_link(backend, variation['file_path'],
project_id)
# Construct the new expiry datetime.
validity_secs = current_app.config['FILE_LINK_VALIDITY'][backend]
response['link_expires'] = now + datetime.timedelta(seconds=validity_secs)
patch_info = remove_private_keys(response)
file_id = ObjectId(response['_id'])
(patch_resp, _, _, _) = patch_internal('files', patch_info, _id=file_id)
if patch_resp.get('_status') == 'ERR':
log.warning('Unable to save new links for file %s: %r',
response['_id'], patch_resp)
# TODO: raise a snag.
response['_updated'] = now
else:
response['_updated'] = patch_resp['_updated']
# Be silly and re-fetch the etag ourselves. TODO: handle this better.
etag_doc = current_app.data.driver.db['files'].find_one({'_id': file_id},
{'_etag': 1})
response['_etag'] = etag_doc['_etag']
def test_edits_by_nonowner_admin(self):
"""Any admin should be able to edit any project."""
from application.utils import remove_private_keys, PillarJSONEncoder
dumps = functools.partial(json.dumps, cls=PillarJSONEncoder)
# Create test project.
project = self._create_user_and_project([u'subscriber'])
project_id = project['_id']
project_url = '/projects/%s' % project_id
# Create test user.
self._create_user_with_token(['admin'], 'admin-token', user_id='cafef00dbeef')
# Admin user should be able to PUT.
put_project = remove_private_keys(project)
put_project['name'] = u'โครงการปั่นเมฆ'
resp = self.client.put(project_url,
data=dumps(put_project),
headers={'Authorization': self.make_header('admin-token'),
'Content-Type': 'application/json',
'If-Match': project['_etag']})
self.assertEqual(200, resp.status_code, resp.data)
def after_inserting_project(project, db_user):
project_id = project['_id']
user_id = db_user['_id']
# Create a project-specific admin group (with name matching the project id)
result, _, _, status = post_internal('groups', {'name': str(project_id)})
if status != 201:
log.error('Unable to create admin group for new project %s: %s',
project_id, result)
return abort_with_error(status)
admin_group_id = result['_id']
log.debug('Created admin group %s for project %s', admin_group_id,
project_id)
# Assign the current user to the group
db_user.setdefault('groups', []).append(admin_group_id)
result, _, _, status = patch_internal('users',
{'groups': db_user['groups']},
_id=user_id)
if status != 200:
log.error(
'Unable to add user %s as member of admin group %s for new project %s: %s',
user_id, admin_group_id, project_id, result)
return abort_with_error(status)
log.debug('Made user %s member of group %s', user_id, admin_group_id)
# Assign the group to the project with admin rights
is_admin = authorization.is_admin(db_user)
world_permissions = ['GET'] if is_admin else []
permissions = {
'world':
world_permissions,
'users': [],
'groups': [
{
'group': admin_group_id,
'methods': DEFAULT_ADMIN_GROUP_PERMISSIONS[:]
},
]
}
def with_permissions(node_type):
copied = copy.deepcopy(node_type)
copied['permissions'] = permissions
return copied
# Assign permissions to the project itself, as well as to the node_types
project['permissions'] = permissions
project['node_types'] = [
with_permissions(node_type_group),
with_permissions(node_type_asset),
with_permissions(node_type_comment),
with_permissions(node_type_texture),
with_permissions(node_type_group_texture),
]
# Allow admin users to use whatever url they want.
if not is_admin or not project.get('url'):
project['url'] = "p-{!s}".format(project_id)
# Initialize storage page (defaults to GCS)
if current_app.config.get('TESTING'):
log.warning(
'Not creating Google Cloud Storage bucket while running unit tests!'
)
else:
gcs_storage = GoogleCloudStorageBucket(str(project_id))
if gcs_storage.bucket.exists():
log.info('Created CGS instance for project %s', project_id)
else:
log.warning('Unable to create CGS instance for project %s',
project_id)
# Commit the changes directly to the MongoDB; a PUT is not allowed yet,
# as the project doesn't have a valid permission structure.
projects_collection = current_app.data.driver.db['projects']
result = projects_collection.update_one(
{'_id': project_id}, {'$set': remove_private_keys(project)})
if result.matched_count != 1:
log.warning('Unable to update project %s: %s', project_id,
result.raw_result)
abort_with_error(500)
def test_save_own_user(self):
"""Tests that a user can't change their own fields."""
from application.utils import authentication as auth
from application.utils import PillarJSONEncoder, remove_private_keys
user_id = self.create_user(roles=[u'subscriber'])
now = datetime.datetime.now(tz_util.utc)
future = now + datetime.timedelta(days=1)
with self.app.test_request_context():
auth.store_token(user_id, 'nonexpired-main', future, None)
with self.app.test_request_context(
headers={'Authorization': self.make_header('nonexpired-main')
}):
self.assertTrue(auth.validate_token())
users = self.app.data.driver.db['users']
db_user = users.find_one(user_id)
updated_fields = remove_private_keys(db_user)
updated_fields['roles'] = ['admin', 'subscriber',
'demo'] # Try to elevate our roles.
# POSTing updated info to a specific user URL is not allowed by Eve.
resp = self.client.post('/users/%s' % user_id,
data=json.dumps(updated_fields,
cls=PillarJSONEncoder),
headers={
'Authorization':
self.make_header('nonexpired-main'),
'Content-Type':
'application/json'
})
self.assertEqual(405, resp.status_code)
# PUT and PATCH should not be allowed.
resp = self.client.put('/users/%s' % user_id,
data=json.dumps(updated_fields,
cls=PillarJSONEncoder),
headers={
'Authorization':
self.make_header('nonexpired-main'),
'Content-Type':
'application/json'
})
self.assertEqual(403, resp.status_code)
updated_fields = {'roles': ['admin', 'subscriber', 'demo']}
resp = self.client.patch('/users/%s' % user_id,
data=json.dumps(updated_fields,
cls=PillarJSONEncoder),
headers={
'Authorization':
self.make_header('nonexpired-main'),
'Content-Type':
'application/json'
})
self.assertEqual(405, resp.status_code)
# After all of this, the roles should be the same.
with self.app.test_request_context(
headers={'Authorization': self.make_header('nonexpired-main')
}):
self.assertTrue(auth.validate_token())
users = self.app.data.driver.db['users']
db_user = users.find_one(user_id)
self.assertEqual([u'subscriber'], db_user['roles'])
def validate_create_user(blender_id_user_id, token, oauth_subclient_id):
"""Validates a user against Blender ID, creating the user in our database.
:param blender_id_user_id: the user ID at the BlenderID server.
:param token: the OAuth access token.
:param oauth_subclient_id: the subclient ID, or empty string if not a subclient.
:returns: (user in MongoDB, HTTP status 200 or 201)
"""
# Verify with Blender ID
log.debug('Storing token for BlenderID user %s', blender_id_user_id)
user_info, token_expiry = validate_token(blender_id_user_id, token, oauth_subclient_id)
if user_info is None:
log.debug('Unable to verify token with Blender ID.')
return None, None
# Blender ID can be queried without user ID, and will always include the
# correct user ID in its response.
log.debug('Obtained user info from Blender ID: %s', user_info)
blender_id_user_id = user_info['id']
# Store the user info in MongoDB.
db_user = find_user_in_db(blender_id_user_id, user_info)
r = {}
for retry in range(5):
if '_id' in db_user:
# Update the existing user
attempted_eve_method = 'PUT'
db_id = db_user['_id']
r, _, _, status = put_internal('users', remove_private_keys(db_user),
_id=db_id)
if status == 422:
log.error('Status %i trying to PUT user %s with values %s, should not happen! %s',
status, db_id, remove_private_keys(db_user), r)
else:
# Create a new user, retry for non-unique usernames.
attempted_eve_method = 'POST'
r, _, _, status = post_internal('users', db_user)
db_id = r['_id']
db_user.update(r) # update with database/eve-generated fields.
if status == 422:
# Probably non-unique username, so retry a few times with different usernames.
log.info('Error creating new user: %s', r)
username_issue = r.get('_issues', {}).get(u'username', '')
if u'not unique' in username_issue:
# Retry
db_user['username'] = authentication.make_unique_username(db_user['email'])
continue
# Saving was successful, or at least didn't break on a non-unique username.
break
else:
log.error('Unable to create new user %s: %s', db_user, r)
return abort(500)
if status not in (200, 201):
log.error('internal response from %s to Eve: %r %r', attempted_eve_method, status, r)
return abort(500)
# Store the token in MongoDB.
authentication.store_token(db_id, token, token_expiry, oauth_subclient_id)
return db_user, status
def zencoder_notifications():
"""
See: https://app.zencoder.com/docs/guides/getting-started/notifications#api_version_2
"""
if current_app.config['ENCODING_BACKEND'] != 'zencoder':
log.warning(
'Received notification from Zencoder but app not configured for Zencoder.'
)
return abort(403)
if not current_app.config['DEBUG']:
# If we are in production, look for the Zencoder header secret
try:
notification_secret_request = request.headers[
'X-Zencoder-Notification-Secret']
except KeyError:
log.warning('Received Zencoder notification without secret.')
return abort(401)
# If the header is found, check it agains the one in the config
notification_secret = current_app.config[
'ZENCODER_NOTIFICATIONS_SECRET']
if notification_secret_request != notification_secret:
log.warning(
'Received Zencoder notification with incorrect secret.')
return abort(401)
# Cast request data into a dict
data = request.get_json()
if log.isEnabledFor(logging.DEBUG):
from pprint import pformat
log.debug('Zencoder job JSON: %s', pformat(data))
files_collection = current_app.data.driver.db['files']
# Find the file object based on processing backend and job_id
zencoder_job_id = data['job']['id']
lookup = {
'processing.backend': 'zencoder',
'processing.job_id': str(zencoder_job_id)
}
file_doc = files_collection.find_one(lookup)
if not file_doc:
log.warning('Unknown Zencoder job id %r', zencoder_job_id)
# Return 200 OK when debugging, or Zencoder will keep trying and trying and trying...
# which is what we want in production.
return "Not found, but that's okay.", 200 if current_app.config[
'DEBUG'] else 404
file_id = ObjectId(file_doc['_id'])
# Remove internal keys (so that we can run put internal)
file_doc = utils.remove_private_keys(file_doc)
# Update processing status
job_state = data['job']['state']
file_doc['processing']['status'] = job_state
if job_state == 'failed':
log.warning('Zencoder job %i for file %s failed.', zencoder_job_id,
file_id)
# Log what Zencoder told us went wrong.
for output in data['outputs']:
if not any('error' in key for key in output):
continue
log.warning('Errors for output %s:', output['url'])
for key in output:
if 'error' in key:
log.info(' %s: %s', key, output[key])
file_doc['status'] = 'failed'
put_internal('files', file_doc, _id=file_id)
return "You failed, but that's okay.", 200
log.info('Zencoder job %s for file %s completed with status %s.',
zencoder_job_id, file_id, job_state)
# For every variation encoded, try to update the file object
root, _ = os.path.splitext(file_doc['file_path'])
for output in data['outputs']:
video_format = output['format']
# Change the zencoder 'mpeg4' format to 'mp4' used internally
video_format = 'mp4' if video_format == 'mpeg4' else video_format
# Find a variation matching format and resolution
variation = next(
(v for v in file_doc['variations']
if v['format'] == format and v['width'] == output['width']), None)
# Fall back to a variation matching just the format
if variation is None:
variation = next((v for v in file_doc['variations']
if v['format'] == video_format), None)
if variation is None:
log.warning(
'Unable to find variation for video format %s for file %s',
video_format, file_id)
continue
# Rename the file to include the now-known size descriptor.
size = size_descriptor(output['width'], output['height'])
new_fname = '{}-{}.{}'.format(root, size, video_format)
# Rename on Google Cloud Storage
try:
rename_on_gcs(file_doc['project'], '_/' + variation['file_path'],
'_/' + new_fname)
except Exception:
log.warning(
'Unable to rename GCS blob %r to %r. Keeping old name.',
variation['file_path'],
new_fname,
exc_info=True)
else:
variation['file_path'] = new_fname
# TODO: calculate md5 on the storage
variation.update({
'height': output['height'],
'width': output['width'],
'length': output['file_size_in_bytes'],
'duration': data['input']['duration_in_ms'] / 1000,
'md5': output['md5_checksum']
or '', # they don't do MD5 for GCS...
'size': size,
})
file_doc['status'] = 'complete'
# Force an update of the links on the next load of the file.
file_doc['link_expires'] = datetime.datetime.now(
tz=tz_util.utc) - datetime.timedelta(days=1)
put_internal('files', file_doc, _id=file_id)
return '', 204
def validate_create_user(blender_id_user_id, token, oauth_subclient_id):
"""Validates a user against Blender ID, creating the user in our database.
:param blender_id_user_id: the user ID at the BlenderID server.
:param token: the OAuth access token.
:param oauth_subclient_id: the subclient ID, or empty string if not a subclient.
:returns: (user in MongoDB, HTTP status 200 or 201)
"""
# Verify with Blender ID
log.debug('Storing token for BlenderID user %s', blender_id_user_id)
user_info, token_expiry = validate_token(blender_id_user_id, token,
oauth_subclient_id)
if user_info is None:
log.debug('Unable to verify token with Blender ID.')
return None, None
# Blender ID can be queried without user ID, and will always include the
# correct user ID in its response.
log.debug('Obtained user info from Blender ID: %s', user_info)
blender_id_user_id = user_info['id']
# Store the user info in MongoDB.
db_user = find_user_in_db(blender_id_user_id, user_info)
r = {}
for retry in range(5):
if '_id' in db_user:
# Update the existing user
attempted_eve_method = 'PUT'
db_id = db_user['_id']
r, _, _, status = put_internal('users',
remove_private_keys(db_user),
_id=db_id)
if status == 422:
log.error(
'Status %i trying to PUT user %s with values %s, should not happen! %s',
status, db_id, remove_private_keys(db_user), r)
else:
# Create a new user, retry for non-unique usernames.
attempted_eve_method = 'POST'
r, _, _, status = post_internal('users', db_user)
db_id = r['_id']
db_user.update(r) # update with database/eve-generated fields.
if status == 422:
# Probably non-unique username, so retry a few times with different usernames.
log.info('Error creating new user: %s', r)
username_issue = r.get('_issues', {}).get(u'username', '')
if u'not unique' in username_issue:
# Retry
db_user['username'] = authentication.make_unique_username(
db_user['email'])
continue
# Saving was successful, or at least didn't break on a non-unique username.
break
else:
log.error('Unable to create new user %s: %s', db_user, r)
return abort(500)
if status not in (200, 201):
log.error('internal response from %s to Eve: %r %r',
attempted_eve_method, status, r)
return abort(500)
# Store the token in MongoDB.
authentication.store_token(db_id, token, token_expiry, oauth_subclient_id)
return db_user, status
def after_inserting_project(project, db_user):
project_id = project['_id']
user_id = db_user['_id']
# Create a project-specific admin group (with name matching the project id)
result, _, _, status = post_internal('groups', {'name': str(project_id)})
if status != 201:
log.error('Unable to create admin group for new project %s: %s',
project_id, result)
return abort_with_error(status)
admin_group_id = result['_id']
log.debug('Created admin group %s for project %s', admin_group_id, project_id)
# Assign the current user to the group
db_user.setdefault('groups', []).append(admin_group_id)
result, _, _, status = patch_internal('users', {'groups': db_user['groups']}, _id=user_id)
if status != 200:
log.error('Unable to add user %s as member of admin group %s for new project %s: %s',
user_id, admin_group_id, project_id, result)
return abort_with_error(status)
log.debug('Made user %s member of group %s', user_id, admin_group_id)
# Assign the group to the project with admin rights
is_admin = authorization.is_admin(db_user)
world_permissions = ['GET'] if is_admin else []
permissions = {
'world': world_permissions,
'users': [],
'groups': [
{'group': admin_group_id,
'methods': ['GET', 'PUT', 'POST', 'DELETE']},
]
}
def with_permissions(node_type):
copied = copy.deepcopy(node_type)
copied['permissions'] = permissions
return copied
# Assign permissions to the project itself, as well as to the node_types
project['permissions'] = permissions
project['node_types'] = [
with_permissions(node_type_group),
with_permissions(node_type_asset),
with_permissions(node_type_comment),
with_permissions(node_type_texture),
with_permissions(node_type_group_texture),
]
# Allow admin users to use whatever url they want.
if not is_admin or not project.get('url'):
project['url'] = "p-{!s}".format(project_id)
# Initialize storage page (defaults to GCS)
if current_app.config.get('TESTING'):
log.warning('Not creating Google Cloud Storage bucket while running unit tests!')
else:
gcs_storage = GoogleCloudStorageBucket(str(project_id))
if gcs_storage.bucket.exists():
log.info('Created CGS instance for project %s', project_id)
else:
log.warning('Unable to create CGS instance for project %s', project_id)
# Commit the changes directly to the MongoDB; a PUT is not allowed yet,
# as the project doesn't have a valid permission structure.
projects_collection = current_app.data.driver.db['projects']
result = projects_collection.update_one({'_id': project_id},
{'$set': remove_private_keys(project)})
if result.matched_count != 1:
log.warning('Unable to update project %s: %s', project_id, result.raw_result)
abort_with_error(500)