def create_job(self, job_type: str) -> ObjectId:
from pillar.api.utils.authentication import force_cli_user
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
job_type,
{
'frames': '1-5',
'chunk_size': 3,
'render_output': '/render/out/frames-######',
'fps': 5.3,
'format': 'OPEN_EXR',
'filepath': '/agent327/scenes/someshot/somefile.blend',
'blender_cmd': '/path/to/blender --enable-new-depsgraph',
'cycles_sample_count': 30,
# Effectively uncapped so that the number of tasks stays small.
'cycles_sample_cap': 30,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
return job['_id']
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
self.mngr_doc, account, token = self.create_manager_service_account()
self.mngr_id = self.mngr_doc['_id']
self.mngr_token = token['token']
uid = self.create_user(
user_id=24 * 'f',
roles={'flamenco-admin'},
groups=[self.mngr_doc['owner'], ctd.EXAMPLE_ADMIN_GROUP_ID])
self.create_valid_auth_token(uid, 'fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def create_test_job(manager_id, user_email, project_url):
"""Creates a test job for a given manager."""
from pillar.api.utils import dumps, str2id
manager_id = str2id(manager_id)
authentication.force_cli_user()
# Find user
users_coll = current_app.db()['users']
user = users_coll.find_one({'email': user_email}, projection={'_id': 1})
if not user:
raise ValueError('User with email %r not found' % user_email)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url},
projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
# Create the job
job = flamenco.current_flamenco.job_manager.api_create_job(
'CLI test job',
'Test job created from the server CLI',
'sleep',
{
'frames': '1-30, 40-44',
'chunk_size': 13,
'time_in_seconds': 3,
},
proj['_id'], user['_id'], manager_id)
log.info('Job created:\n%s', dumps(job, indent=4))
def test_create_manager(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.setup import create_manager
# Make sure the owner of the manager exists.
owner_id = self.create_user(email='*****@*****.**')
with self.app.test_request_context():
force_cli_user()
mngr_doc, account, token = create_manager('*****@*****.**',
'Unit test mānāgèr',
'¡Awesome in Space!')
self.assertEqual('Unit test mānāgèr', mngr_doc['name'])
self.assertEqual(f'SRV-{account["_id"]}', account['full_name'])
self.assertEqual(f'SRV-{account["_id"]}', account['username'])
self.assertEqual(['flamenco_manager', 'service'], account['roles'])
self.assertEqual([], account['auth'])
self.assertEqual({'flamenco_manager': {}}, account['service'])
self.assertIn('owner', mngr_doc)
self.assertTrue(mngr_doc['owner'])
# Check that the group exists, and that it refers to the manager in its name.
with self.app.test_request_context():
groups_coll = self.app.db().groups
group = groups_coll.find_one(mngr_doc['owner'])
self.assertIn(str(mngr_doc['_id']), group['name'])
self.assertAllowsAccess(token, account['_id'])
# Check that the owner is now member of the owner group.
owner = self.fetch_user_from_db(owner_id)
self.assertIn(mngr_doc['owner'], owner['groups'])
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
self.create_manager()
self.create_user(user_id=24 * 'f',
roles={'flamenco-admin'},
token='fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
from pillar.api.projects.utils import get_admin_group_id
mngr_doc, account, token = self.create_manager_service_account()
self.mngr_id = mngr_doc['_id']
self.mngr_token = token['token']
self.assign_manager_to_project(self.mngr_id, self.proj_id)
with self.app.test_request_context():
group_id = get_admin_group_id(self.proj_id)
self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, groups=[group_id])
self.create_valid_auth_token(24 * 'f', 'fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
owner_email = "jemoeder"
self.mngr_owner = self.create_user(user_id=bson.ObjectId(),
email=owner_email)
mngr_doc, account, token = self.create_manager_service_account(
owner_email)
self.mngr_id = mngr_doc['_id']
self.mngr_doc = mngr_doc
self.mngr_token = token['token']
self.create_user(user_id=24 * 'f',
roles={'flamenco-admin'},
token='fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def test_create_manager(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.setup import create_manager
# Make sure the owner of the manager exists.
owner_id = self.create_user(email='*****@*****.**')
with self.app.test_request_context():
force_cli_user()
mngr_doc, account, token = create_manager(
'*****@*****.**', 'Unit test mānāgèr', '¡Awesome in Space!')
self.assertEqual('Unit test mānāgèr', mngr_doc['name'])
self.assertEqual(f'SRV-{account["_id"]}', account['full_name'])
self.assertEqual(f'SRV-{account["_id"]}', account['username'])
self.assertEqual(['flamenco_manager', 'service'], account['roles'])
self.assertEqual([], account['auth'])
self.assertEqual({'flamenco_manager': {}}, account['service'])
self.assertIn('owner', mngr_doc)
self.assertTrue(mngr_doc['owner'])
# Check that the group exists, and that it refers to the manager in its name.
with self.app.test_request_context():
groups_coll = self.app.db().groups
group = groups_coll.find_one(mngr_doc['owner'])
self.assertIn(str(mngr_doc['_id']), group['name'])
self.assertAllowsAccess(token, account['_id'])
# Check that the owner is now member of the owner group.
owner = self.fetch_user_from_db(owner_id)
self.assertIn(mngr_doc['owner'], owner['groups'])
def create_manager_service_account(self,
owner_email=ctd.EXAMPLE_USER['email'],
name='tēst mānēgūr',
*,
assign_to_project_id: ObjectId = None):
from flamenco.setup import create_manager
from pillar.api.utils.authentication import force_cli_user
# Main project will have a manager, job, and tasks.
with self.app.test_request_context():
force_cli_user()
# Make sure there is an owner for this manager.
users_coll = self.app.db('users')
count = users_coll.find({'email': owner_email}).count()
if count == 0:
self.create_user(user_id=ObjectId(), email=owner_email)
elif count > 1:
self.fail(
f'Found {count} users with email address {owner_email}')
mngr_doc, account, token = create_manager(owner_email, name,
'descr')
if assign_to_project_id:
self.assign_manager_to_project(mngr_doc['_id'],
assign_to_project_id)
return mngr_doc, account, token
def create_job(self, job_type: str) -> ObjectId:
from pillar.api.utils.authentication import force_cli_user
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
job_type,
{
'frames': '1-5',
'chunk_size': 3,
'render_output': '/render/out/frames-######',
'fps': 5.3,
'format': 'OPEN_EXR',
'filepath': '/agent327/scenes/someshot/somefile.blend',
'blender_cmd': '/path/to/blender --enable-new-depsgraph',
'cycles_sample_count': 30,
# Effectively uncapped so that the number of tasks stays small.
'cycles_sample_cap': 30,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
return job['_id']
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
self.mngr_doc, account, token = self.create_manager_service_account()
self.mngr_id = self.mngr_doc['_id']
self.mngr_token = token['token']
uid = self.create_user(user_id=24 * 'f', roles={'flamenco-admin'})
self.create_valid_auth_token(uid, 'fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'blender-video-chunks',
{
'frames': '100-250',
'fps': 24,
'chunk_size': 100,
'render_output': '/tmp/render/spring/export/FILENAME.MKV',
'filepath': '/spring/edit/sprloing.blend',
'output_file_extension': '.mkv',
'images_or_video': 'video',
'extract_audio': True,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def test_create_job(self):
from pillar.api.utils.authentication import force_cli_user
manager, _, _ = self.create_manager_service_account()
with self.app.test_request_context():
force_cli_user()
self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 5,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
manager['_id'],
)
# Test the jobs
with self.app.test_request_context():
jobs_coll = self.flamenco.db('jobs')
jobs = list(jobs_coll.find())
self.assertEqual(1, len(jobs))
job = jobs[0]
self.assertEqual('Wörk wørk w°rk.', job['description'])
self.assertEqual('sleep', job['job_type'])
# Test the tasks
with self.app.test_request_context():
tasks_coll = self.flamenco.db('tasks')
tasks = list(tasks_coll.find())
self.assertEqual(2, len(tasks))
statuses = [task['status'] for task in tasks]
self.assertEqual(['queued', 'queued'], statuses)
self.assertEqual(['sleep', 'sleep'], [task['task_type'] for task in tasks])
task = tasks[0]
self.assertEqual('sleep-12-16', task['name'])
self.assertEqual({
'name': 'echo',
'settings': {
'message': 'Preparing to sleep',
}
}, task['commands'][0])
self.assertEqual({
'name': 'sleep',
'settings': {
'time_in_seconds': 3,
}
}, task['commands'][1])
def setUp(self, **kwargs):
super().setUp(**kwargs)
from pillar.api.utils.authentication import force_cli_user
self.manager, _, _ = self.create_manager_service_account(
assign_to_project_id=self.proj_id)
with self.app.test_request_context():
force_cli_user()
job_doc = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 5,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.manager['_id'],
)
self.job_id = job_doc['_id']
def create_test_job(manager_id, user_email, project_url):
"""Creates a test job for a given manager."""
from pillar.api.utils import dumps, str2id
manager_id = str2id(manager_id)
authentication.force_cli_user()
# Find user
users_coll = current_app.db()['users']
user = users_coll.find_one({'email': user_email}, projection={'_id': 1})
if not user:
raise ValueError('User with email %r not found' % user_email)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
# Create the job
job = flamenco.current_flamenco.job_manager.api_create_job(
'CLI test job', 'Test job created from the server CLI', 'sleep', {
'frames': '1-30, 40-44',
'chunk_size': 13,
'time_in_seconds': 3,
}, proj['_id'], user['_id'], manager_id)
log.info('Job created:\n%s', dumps(job, indent=4))
def test_user_page(self):
# Username 'tester' does not exist
self.get('/u/james', expected_status=404)
# Username will be 'harry'
with self.app.test_request_context():
force_cli_user()
create_local_user('*****@*****.**', 'password')
self.get('/u/james')
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.ensure_group_exists(
'cafef005972666988bef650f', 'dillo_user_main')
self.proj_id, self.project = self.ensure_project_exists()
with self.app.test_request_context():
force_cli_user()
self.local_user_id = create_local_user('*****@*****.**', 'password')
def test_create_user(self):
from pillar.api.utils.authentication import force_cli_user
from pillar.api.local_auth import create_local_user
with self.app.test_request_context():
force_cli_user()
user_id = create_local_user('*****@*****.**', 'password')
db_user = self.fetch_user_from_db(user_id)
print(db_user)
def setUp(self, **kwargs):
super(JobStatusChangeTest, self).setUp(**kwargs)
# Create a job with the same number of tasks as there are task statuses.
from pillar.api.utils.authentication import force_cli_user
from flamenco.eve_settings import tasks_schema
manager, _, token = self.create_manager_service_account()
self.mngr_token = token['token']
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'blender-render',
{
'blender_cmd': '{blender}',
'filepath': '/my/file.blend',
# Frames and chunk size chosen to produce as many tasks
# as there are task statuses - 1 (we don't test 'under-construction')
'frames': '12-18, 20-29',
'chunk_size': 2,
'time_in_seconds': 3,
'render_output': '/not-relevant-now/####',
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
manager['_id'],
)
self.job_id = job['_id']
# Fetch the task IDs and set the task statuses to a fixed list.
tasks_coll = self.flamenco.db('tasks')
tasks = tasks_coll.find(
{
'job': self.job_id,
'name': {
'$regex': '^blender-render-'
}, # don't consider move-out-of-way task.
},
projection={'_id': 1})
self.task_ids = [task['_id'] for task in tasks]
allowed_statuses: list = tasks_schema['status']['allowed']
# this status should never occur after job compilation
allowed_statuses.remove('under-construction')
self.assertEqual(len(allowed_statuses), len(self.task_ids))
self.force_task_status(0, 'queued')
self.force_task_status(1, 'claimed-by-manager')
self.force_task_status(2, 'completed')
self.force_task_status(3, 'active')
self.force_task_status(4, 'canceled')
self.force_task_status(5, 'failed')
self.force_task_status(6, 'cancel-requested')
self.force_task_status(7, 'paused')
self.force_task_status(8, 'soft-failed')
def setup_for_flamenco(project_url, replace=False):
"""Adds Flamenco node types to the project.
Use --replace to replace pre-existing Flamenco node types
(by default already existing Flamenco node types are skipped).
"""
authentication.force_cli_user()
flamenco.setup.setup_for_flamenco(project_url, replace=replace)
def test_manager_without_owner_email_address(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.setup import create_manager
with self.app.test_request_context():
force_cli_user()
with self.assertRaises(ValueError):
create_manager('', 'Unit test mānāgèr', '¡Awesome in Space!')
def test_manager_without_owner_email_address(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.setup import create_manager
with self.app.test_request_context():
force_cli_user()
with self.assertRaises(ValueError):
create_manager('', 'Unit test mānāgèr', '¡Awesome in Space!')
def test_user_page(self):
from pillar.api.utils.authentication import force_cli_user
from pillar.api.local_auth import create_local_user
# Username 'tester' does not exist
self.get('/u/harry', expected_status=404)
# Username will be 'harry'
with self.app.test_request_context():
force_cli_user()
create_local_user('*****@*****.**', 'password')
self.get('/u/harry')
def test_without_email_address(self):
from pillar.api.utils.authentication import force_cli_user
from pillar.api.service import create_service_account as create_sa
with self.app.test_request_context():
force_cli_user()
account, token = create_sa('', ['flamenco_manager'],
{'flamenco_manager': {}})
self.assertNotIn('email', account)
self.assertAllowsAccess(token, account['_id'])
def setUp(self, **kwargs):
super(JobStatusChangeTest, self).setUp(**kwargs)
# Create a job with the same number of tasks as there are task statuses.
from pillar.api.utils.authentication import force_cli_user
from flamenco.eve_settings import tasks_schema
manager, _, token = self.create_manager_service_account()
self.mngr_token = token['token']
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'blender-render',
{
'blender_cmd': '{blender}',
'filepath': '/my/file.blend',
# Frames and chunk size chosen to produce as many tasks
# as there are task statuses - 1 (we don't test 'under-construction')
'frames': '12-18, 20-29',
'chunk_size': 2,
'time_in_seconds': 3,
'render_output': '/not-relevant-now/####',
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
manager['_id'],
)
self.job_id = job['_id']
# Fetch the task IDs and set the task statuses to a fixed list.
tasks_coll = self.flamenco.db('tasks')
tasks = tasks_coll.find({
'job': self.job_id,
'name': {'$regex': '^blender-render-'}, # don't consider move-out-of-way task.
}, projection={'_id': 1})
self.task_ids = [task['_id'] for task in tasks]
allowed_statuses: list = tasks_schema['status']['allowed']
# this status should never occur after job compilation
allowed_statuses.remove('under-construction')
self.assertEqual(len(allowed_statuses), len(self.task_ids))
self.force_task_status(0, 'queued')
self.force_task_status(1, 'claimed-by-manager')
self.force_task_status(2, 'completed')
self.force_task_status(3, 'active')
self.force_task_status(4, 'canceled')
self.force_task_status(5, 'failed')
self.force_task_status(6, 'cancel-requested')
self.force_task_status(7, 'paused')
self.force_task_status(8, 'soft-failed')
def test_recreate_job(self):
from pillar.api.utils.authentication import force_cli_user
old_task_ids = self._pre_test()
with self.app.test_request_context():
# Recreate the job, and check the task statuses.
force_cli_user()
self.flamenco.api_recreate_job(self.job_id)
self._post_test(old_task_ids)
def test_api_find_jobfinal_tasks(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.job_compilers import commands
manager, _, _ = self.create_manager_service_account()
with self.app.test_request_context():
force_cli_user()
job_doc = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep', {
'frames': '12-18, 20-22',
'chunk_size': 7,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
manager['_id'],
)
job_id = job_doc['_id']
# Find the tasks created so far, use them as parents.
tasks = self.flamenco.db('tasks').find({'job': job_id},
projection={'_id': 1})
task_ids = [t['_id'] for t in tasks]
# dependent task that is used as a single parent.
taskid1 = self.tmngr.api_create_task(
job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 1', parents=task_ids,
task_type='sleep',
)
# task dependent on multiple tasks that is not used as a parent.
taskid2 = self.tmngr.api_create_task(
job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 2', parents=task_ids,
task_type='sleep',
)
# task dependent on a single task that is not used as a parent.
taskid3 = self.tmngr.api_create_task(
job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 3', parents=[taskid1],
task_type='sleep',
)
# independent task
taskid4 = self.tmngr.api_create_task(
job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 4',
task_type='sleep',
)
job_enders = self.tmngr.api_find_job_enders(job_id)
self.assertEqual({taskid2, taskid3, taskid4}, set(job_enders))
def create_blog(proj_url):
"""Adds a blog to the project."""
from pillar.api.utils.authentication import force_cli_user
from pillar.api.utils import node_type_utils
from pillar.api.node_types.blog import node_type_blog
from pillar.api.node_types.post import node_type_post
from pillar.api.utils import remove_private_keys
force_cli_user()
db = current_app.db()
# Add the blog & post node types to the project.
projects_coll = db['projects']
proj = projects_coll.find_one({'url': proj_url})
if not proj:
log.error('Project url=%s not found', proj_url)
return 3
node_type_utils.add_to_project(proj, (node_type_blog, node_type_post),
replace_existing=False)
proj_id = proj['_id']
r, _, _, status = current_app.put_internal('projects',
remove_private_keys(proj),
_id=proj_id)
if status != 200:
log.error('Error %i storing altered project %s %s', status, proj_id, r)
return 4
log.info('Project saved succesfully.')
# Create a blog node.
nodes_coll = db['nodes']
blog = nodes_coll.find_one({'node_type': 'blog', 'project': proj_id})
if not blog:
blog = {
'node_type': node_type_blog['name'],
'name': 'Blog',
'description': '',
'properties': {},
'project': proj_id,
}
r, _, _, status = current_app.post_internal('nodes', blog)
if status != 201:
log.error('Error %i storing blog node: %s', status, r)
return 4
log.info('Blog node saved succesfully: %s', r)
else:
log.info('Blog node already exists: %s', blog)
return 0
def test_two_managers_with_same_owner(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.setup import create_manager
self.create_user(email='*****@*****.**')
with self.app.test_request_context():
force_cli_user()
mngr_doc1, account1, token1 = create_manager('*****@*****.**', 'mānāgèr 1', 'Awe!')
mngr_doc2, account2, token2 = create_manager('*****@*****.**', 'mānāgèr 2', 'som!')
self.assertAllowsAccess(token1, account1['_id'])
self.assertAllowsAccess(token2, account2['_id'])
def test_two_managers_with_same_owner(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.setup import create_manager
self.create_user(email='*****@*****.**')
with self.app.test_request_context():
force_cli_user()
mngr_doc1, account1, token1 = create_manager(
'*****@*****.**', 'mānāgèr 1', 'Awe!')
mngr_doc2, account2, token2 = create_manager(
'*****@*****.**', 'mānāgèr 2', 'som!')
self.assertAllowsAccess(token1, account1['_id'])
self.assertAllowsAccess(token2, account2['_id'])
def ensure_project_exists(self, project_overrides=None):
from flamenco.setup import setup_for_flamenco
from pillar.api.utils.authentication import force_cli_user
project_overrides = dict(picture_header=None,
picture_square=None,
**(project_overrides or {}))
proj_id, project = AbstractPillarTest.ensure_project_exists(
self, project_overrides)
with self.app.test_request_context():
force_cli_user()
flamenco_project = setup_for_flamenco(project['url'], replace=True)
return proj_id, flamenco_project
def create_manager(email, name, description):
"""Creates a Flamenco manager."""
from pillar.api.utils import dumps
authentication.force_cli_user()
mngr_doc, account, token = flamenco.setup.create_manager(email, name, description)
print('Service account information:')
print(dumps(account, indent=4, sort_keys=True))
print()
print('Access token: %s' % token['token'])
print(' expires on: %s' % token['expire_time'])
print()
print('Created a new manager:')
print(dumps(mngr_doc, indent=4))
def create_manager(email, name, description):
"""Creates a Flamenco manager."""
from pillar.api.utils import dumps
authentication.force_cli_user()
mngr_doc, account, token = flamenco.setup.create_manager(email, name, description)
print('Service account information:')
print(dumps(account, indent=4, sort_keys=True))
print()
print('Access token: %s' % token['token'])
print(' expires on: %s' % token['expire_time'])
print()
print('Created a new manager:')
print(dumps(mngr_doc, indent=4))
def test_create_service_account(self):
from pillar.api.utils.authentication import force_cli_user
from pillar.api import service
with self.app.test_request_context():
force_cli_user()
account, token = service.create_service_account(
'*****@*****.**', ['flamenco_manager'],
{'flamenco_manager': {}})
self.assertEqual(f'SRV-{account["_id"]}', account['full_name'])
self.assertEqual(f'SRV-{account["_id"]}', account['username'])
self.assertEqual(['flamenco_manager', 'service'], account['roles'])
self.assertEqual([], account['auth'])
self.assertEqual({'flamenco_manager': {}}, account['service'])
self.assertAllowsAccess(token, account['_id'])
def create_svner_account(email, project_url):
"""Creates an account that can push SVN activity to an Flamenco project.
:param email: email address associated with the account
:param project_url:
"""
authentication.force_cli_user()
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url},
projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
account, token = create_service_account(email, [u'svner'], {'svner': {'project': proj['_id']}})
return account, token
def create_job(self):
from pillar.api.utils.authentication import force_cli_user
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
return job['_id']
def _manager_project(manager_id, project_url, action):
from pillar.api.utils import str2id
from flamenco import current_flamenco
authentication.force_cli_user()
manager_id = str2id(manager_id)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
project_id = proj['_id']
ok = current_flamenco.manager_manager.api_assign_to_project(manager_id, project_id, action)
if not ok:
log.error('Unable to assign manager %s to project %s', manager_id, project_id)
return 1
def create_job(self):
from pillar.api.utils.authentication import force_cli_user
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
return job['_id']
def _manager_project(manager_id, project_url, action):
from pillar.api.utils import str2id
from flamenco import current_flamenco
authentication.force_cli_user()
manager_id = str2id(manager_id)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
project_id = proj['_id']
ok = current_flamenco.manager_manager.api_assign_to_project(manager_id, project_id, action)
if not ok:
log.error('Unable to assign manager %s to project %s', manager_id, project_id)
return 1
def setUp(self):
super().setUp()
mngr_doc, account, token = self.create_manager_service_account(
assign_to_project_id=self.proj_id)
self.mngr_id = mngr_doc['_id']
self.mngr_doc = self.fetch_manager_from_db(self.mngr_id)
self.mngr_token = token['token']
assert self.mngr_doc['owner']
self.owner_uid = ObjectId(24 * 'e')
self.create_project_member(user_id=str(self.owner_uid),
roles={'subscriber'},
groups=[self.mngr_doc['owner']],
token='owner-token')
from pillar.api.utils.authentication import force_cli_user
with self.app.app_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
self.tasks = list(self.tmngr.collection().find(
{'job': self.job_id}))
self.task = self.tasks[0]
self.tid = self.task['_id']
# Make sure we can request the task logs.
self.tmngr.api_set_task_status_for_job(self.job_id, 'queued',
'completed')
def test_put_without_email_address(self):
from pillar.api.utils import remove_private_keys
from pillar.api.utils.authentication import force_cli_user
from pillar.api.service import create_service_account as create_sa
with self.app.test_request_context():
force_cli_user()
account, token = create_sa('', ['flamenco_manager'],
{'flamenco_manager': {}})
puttable = remove_private_keys(account)
user_id = account['_id']
# The user should be able to edit themselves, even without email address.
etag = account['_etag']
puttable['full_name'] = 'þor'
resp = self.put(f'/api/users/{user_id}',
json=puttable,
auth_token=token['token'],
etag=etag).json()
etag = resp['_etag']
with self.app.test_request_context():
users_coll = self.app.db().users
db_user = users_coll.find_one(user_id)
self.assertNotIn('email', db_user)
self.assertEqual('þor', db_user['full_name'])
# An admin should be able to edit this email-less user.
self.create_user(24 * 'a', roles={'admin'}, token='admin-token')
puttable['username'] = '******'
self.put(f'/api/users/{user_id}',
json=puttable,
auth_token='admin-token',
etag=etag)
with self.app.test_request_context():
users_coll = self.app.db().users
db_user = users_coll.find_one(user_id)
self.assertNotIn('email', db_user)
self.assertEqual('bigdüde', db_user['username'])
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
from pillar.api.projects.utils import get_admin_group_id
mngr_doc, account, token = self.create_manager_service_account()
self.mngr_id = mngr_doc['_id']
self.mngr_token = token['token']
self.assign_manager_to_project(self.mngr_id, self.proj_id)
with self.app.test_request_context():
group_id = get_admin_group_id(self.proj_id)
self.create_user(user_id=24 * 'f',
roles={'flamenco-admin'},
groups=[group_id])
self.create_valid_auth_token(24 * 'f', 'fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'blender-render-progressive',
{
'frames': '1-5',
'chunk_size': 2,
'render_output': '/render/out/frames-######',
'format': 'EXR',
'filepath': '/agent327/scenes/someshot/somefile.blend',
'blender_cmd': '/path/to/blender --enable-new-depsgraph',
'cycles_sample_count': 30,
'cycles_num_chunks': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def purge_home_projects(go=False):
"""Deletes all home projects that have no owner."""
from pillar.api.utils.authentication import force_cli_user
force_cli_user()
users_coll = current_app.data.driver.db['users']
proj_coll = current_app.data.driver.db['projects']
good = bad = 0
def bad_projects():
nonlocal good, bad
for proj in proj_coll.find({'category': 'home', '_deleted': {'$ne': True}}):
pid = proj['_id']
uid = proj.get('user')
if not uid:
log.info('Project %s has no user assigned', uid)
bad += 1
yield pid
continue
if users_coll.find({'_id': uid, '_deleted': {'$ne': True}}).count() == 0:
log.info('Project %s has non-existing owner %s', pid, uid)
bad += 1
yield pid
continue
good += 1
if not go:
log.info('Dry run, use --go to actually perform the changes.')
for project_id in bad_projects():
log.info('Soft-deleting project %s', project_id)
if go:
r, _, _, status = current_app.delete_internal('projects', _id=project_id)
if status != 204:
raise ValueError(f'Error {status} deleting {project_id}: {r}')
log.info('%i projects OK, %i projects deleted', good, bad)
return bad
def setUp(self, **kwargs):
AbstractTaskBatchUpdateTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
from pillar.api.projects.utils import get_admin_group_id
mngr_doc, account, token = self.create_manager_service_account()
self.mngr_id = mngr_doc['_id']
self.mngr_token = token['token']
self.assign_manager_to_project(self.mngr_id, self.proj_id)
with self.app.test_request_context():
group_id = get_admin_group_id(self.proj_id)
self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, groups=[group_id])
self.create_valid_auth_token(24 * 'f', 'fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'blender-render-progressive',
{
'frames': '1-5',
'chunk_size': 2,
'render_output': '/render/out/frames-######',
'format': 'OPEN_EXR',
'fps': 44,
'filepath': '/agent327/scenes/someshot/somefile.blend',
'blender_cmd': '/path/to/blender --enable-new-depsgraph',
'cycles_sample_count': 30,
'cycles_sample_cap': 30,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
self.create_manager()
self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, token='fladmin-token')
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
def test_create_task(self):
from pillar.api.utils.authentication import force_cli_user
from flamenco.job_compilers import commands
manager, _, _ = self.create_manager_service_account()
with self.app.test_request_context():
force_cli_user()
job_doc = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep', {
'frames': '12-18, 20-22',
'chunk_size': 7,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
manager['_id'],
)
self.tmngr.api_create_task(
job_doc,
[
commands.Echo(message='ẑžƶźz'),
commands.Sleep(time_in_seconds=3),
],
'sleep-1-13',
status='under-construction',
task_type='exr-merge',
)
# Now test the database contents.
with self.app.test_request_context():
tasks_coll = self.flamenco.db('tasks')
dbtasks = list(tasks_coll.find())
self.assertEqual(3, len(dbtasks)) # 2 of compiled job + the one we added after.
statuses = [task['status'] for task in dbtasks]
self.assertEqual(['queued', 'queued', 'under-construction'], statuses)
types = [task['task_type'] for task in dbtasks]
self.assertEqual(['sleep', 'sleep', 'exr-merge'], types)
dbtask = dbtasks[-1]
self.assertEqual({
'name': 'echo',
'settings': {
'message': 'ẑžƶźz',
}
}, dbtask['commands'][0])
self.assertEqual({
'name': 'sleep',
'settings': {
'time_in_seconds': 3,
}
}, dbtask['commands'][1])
return job_doc['_id']
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from flamenco import current_flamenco
from pillar.api.utils.authentication import force_cli_user
# Main project will have a manager, job, and tasks.
mngr_doc, _, token = self.create_manager_service_account()
self.mngr_id = mngr_doc['_id']
self.mngr_doc = mngr_doc
self.mngr_token = token['token']
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.job_id = job['_id']
tasks_coll = current_flamenco.db('tasks')
self.tasks_for_job = list(tasks_coll.find({'job': self.job_id}))
# Another project, also with manager, job, and tasks.
proj2_owner_id = 24 * 'a'
self.proj2 = self._create_user_and_project(user_id=proj2_owner_id,
roles={'subscriber'},
project_name='Prøject 2',
token='token-proj2-owner')
self.proj2_id = self.proj2['_id']
mngr_doc, _, token = self.create_manager_service_account(
owner_email='*****@*****.**', name='manager 2'
)
self.mngr2_id = mngr_doc['_id']
self.mngr2_doc = mngr_doc
self.mngr2_token = token['token']
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job 2',
'Wörk² wørk² w°rk².',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj2_id,
proj2_owner_id,
self.mngr2_id,
)
self.job2_id = job['_id']
def setup_for_flamenco(project_url):
"""Adds Flamenco node types to the project."""
authentication.force_cli_user()
flamenco.setup.setup_for_flamenco(project_url)
def setUp(self, **kwargs):
AbstractFlamencoTest.setUp(self, **kwargs)
from pillar.api.utils.authentication import force_cli_user
mngr_doc, account, token = self.create_manager_service_account(
assign_to_project_id=self.proj_id)
self.mngr_id = mngr_doc['_id']
self.mngr_token = token['token']
with self.app.app_context():
project_gid = get_admin_group_id(self.proj_id)
self.user = self.create_user(roles={'subscriber'},
groups=[project_gid, mngr_doc['owner']],
token='user-token')
# Create three test jobs, one of which is completed and two are queued.
with self.app.test_request_context():
force_cli_user()
job = self.jmngr.api_create_job(
'test job 1',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.jobid1 = job['_id']
job = self.jmngr.api_create_job(
'test job 2',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.jobid2 = job['_id']
job = self.jmngr.api_create_job(
'test job 3',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
)
self.jobid3 = job['_id']
job = self.jmngr.api_create_job(
'test job 4',
'Wörk wørk w°rk.',
'sleep',
{
'frames': '12-18, 20-22',
'chunk_size': 3,
'time_in_seconds': 3,
},
self.proj_id,
ctd.EXAMPLE_PROJECT_OWNER_ID,
self.mngr_id,
start_paused=True,
)
self.jobid4 = job['_id']
assert isinstance(self.jobid1, ObjectId)
assert isinstance(self.jobid2, ObjectId)
assert isinstance(self.jobid3, ObjectId)
assert isinstance(self.jobid4, ObjectId)
self.set_job_status('completed', job_id=self.jobid3)
self.tasks = list(self.flamenco.db('tasks').find({
'job': {'$in': [self.jobid1, self.jobid2]}
}))
self.task_ids = [t['_id'] for t in self.tasks]
