def create_job(self, job_type: str) -> ObjectId: from pillar.api.utils.authentication import force_cli_user with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', job_type, { 'frames': '1-5', 'chunk_size': 3, 'render_output': '/render/out/frames-######', 'fps': 5.3, 'format': 'OPEN_EXR', 'filepath': '/agent327/scenes/someshot/somefile.blend', 'blender_cmd': '/path/to/blender --enable-new-depsgraph', 'cycles_sample_count': 30, # Effectively uncapped so that the number of tasks stays small. 'cycles_sample_cap': 30, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) return job['_id']
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user self.mngr_doc, account, token = self.create_manager_service_account() self.mngr_id = self.mngr_doc['_id'] self.mngr_token = token['token'] uid = self.create_user( user_id=24 * 'f', roles={'flamenco-admin'}, groups=[self.mngr_doc['owner'], ctd.EXAMPLE_ADMIN_GROUP_ID]) self.create_valid_auth_token(uid, 'fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def create_test_job(manager_id, user_email, project_url): """Creates a test job for a given manager.""" from pillar.api.utils import dumps, str2id manager_id = str2id(manager_id) authentication.force_cli_user() # Find user users_coll = current_app.db()['users'] user = users_coll.find_one({'email': user_email}, projection={'_id': 1}) if not user: raise ValueError('User with email %r not found' % user_email) # Find project projs_coll = current_app.db()['projects'] proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1}) if not proj: log.error('Unable to find project url=%s', project_url) return 1 # Create the job job = flamenco.current_flamenco.job_manager.api_create_job( 'CLI test job', 'Test job created from the server CLI', 'sleep', { 'frames': '1-30, 40-44', 'chunk_size': 13, 'time_in_seconds': 3, }, proj['_id'], user['_id'], manager_id) log.info('Job created:\n%s', dumps(job, indent=4))
def test_create_manager(self): from pillar.api.utils.authentication import force_cli_user from flamenco.setup import create_manager # Make sure the owner of the manager exists. owner_id = self.create_user(email='*****@*****.**') with self.app.test_request_context(): force_cli_user() mngr_doc, account, token = create_manager('*****@*****.**', 'Unit test mānāgèr', '¡Awesome in Space!') self.assertEqual('Unit test mānāgèr', mngr_doc['name']) self.assertEqual(f'SRV-{account["_id"]}', account['full_name']) self.assertEqual(f'SRV-{account["_id"]}', account['username']) self.assertEqual(['flamenco_manager', 'service'], account['roles']) self.assertEqual([], account['auth']) self.assertEqual({'flamenco_manager': {}}, account['service']) self.assertIn('owner', mngr_doc) self.assertTrue(mngr_doc['owner']) # Check that the group exists, and that it refers to the manager in its name. with self.app.test_request_context(): groups_coll = self.app.db().groups group = groups_coll.find_one(mngr_doc['owner']) self.assertIn(str(mngr_doc['_id']), group['name']) self.assertAllowsAccess(token, account['_id']) # Check that the owner is now member of the owner group. owner = self.fetch_user_from_db(owner_id) self.assertIn(mngr_doc['owner'], owner['groups'])
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user self.create_manager() self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, token='fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user from pillar.api.projects.utils import get_admin_group_id mngr_doc, account, token = self.create_manager_service_account() self.mngr_id = mngr_doc['_id'] self.mngr_token = token['token'] self.assign_manager_to_project(self.mngr_id, self.proj_id) with self.app.test_request_context(): group_id = get_admin_group_id(self.proj_id) self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, groups=[group_id]) self.create_valid_auth_token(24 * 'f', 'fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user owner_email = "jemoeder" self.mngr_owner = self.create_user(user_id=bson.ObjectId(), email=owner_email) mngr_doc, account, token = self.create_manager_service_account( owner_email) self.mngr_id = mngr_doc['_id'] self.mngr_doc = mngr_doc self.mngr_token = token['token'] self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, token='fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def test_create_manager(self): from pillar.api.utils.authentication import force_cli_user from flamenco.setup import create_manager # Make sure the owner of the manager exists. owner_id = self.create_user(email='*****@*****.**') with self.app.test_request_context(): force_cli_user() mngr_doc, account, token = create_manager( '*****@*****.**', 'Unit test mānāgèr', '¡Awesome in Space!') self.assertEqual('Unit test mānāgèr', mngr_doc['name']) self.assertEqual(f'SRV-{account["_id"]}', account['full_name']) self.assertEqual(f'SRV-{account["_id"]}', account['username']) self.assertEqual(['flamenco_manager', 'service'], account['roles']) self.assertEqual([], account['auth']) self.assertEqual({'flamenco_manager': {}}, account['service']) self.assertIn('owner', mngr_doc) self.assertTrue(mngr_doc['owner']) # Check that the group exists, and that it refers to the manager in its name. with self.app.test_request_context(): groups_coll = self.app.db().groups group = groups_coll.find_one(mngr_doc['owner']) self.assertIn(str(mngr_doc['_id']), group['name']) self.assertAllowsAccess(token, account['_id']) # Check that the owner is now member of the owner group. owner = self.fetch_user_from_db(owner_id) self.assertIn(mngr_doc['owner'], owner['groups'])
def create_manager_service_account(self, owner_email=ctd.EXAMPLE_USER['email'], name='tēst mānēgūr', *, assign_to_project_id: ObjectId = None): from flamenco.setup import create_manager from pillar.api.utils.authentication import force_cli_user # Main project will have a manager, job, and tasks. with self.app.test_request_context(): force_cli_user() # Make sure there is an owner for this manager. users_coll = self.app.db('users') count = users_coll.find({'email': owner_email}).count() if count == 0: self.create_user(user_id=ObjectId(), email=owner_email) elif count > 1: self.fail( f'Found {count} users with email address {owner_email}') mngr_doc, account, token = create_manager(owner_email, name, 'descr') if assign_to_project_id: self.assign_manager_to_project(mngr_doc['_id'], assign_to_project_id) return mngr_doc, account, token
def create_job(self, job_type: str) -> ObjectId: from pillar.api.utils.authentication import force_cli_user with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', job_type, { 'frames': '1-5', 'chunk_size': 3, 'render_output': '/render/out/frames-######', 'fps': 5.3, 'format': 'OPEN_EXR', 'filepath': '/agent327/scenes/someshot/somefile.blend', 'blender_cmd': '/path/to/blender --enable-new-depsgraph', 'cycles_sample_count': 30, # Effectively uncapped so that the number of tasks stays small. 'cycles_sample_cap': 30, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) return job['_id']
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user self.mngr_doc, account, token = self.create_manager_service_account() self.mngr_id = self.mngr_doc['_id'] self.mngr_token = token['token'] uid = self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}) self.create_valid_auth_token(uid, 'fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'blender-video-chunks', { 'frames': '100-250', 'fps': 24, 'chunk_size': 100, 'render_output': '/tmp/render/spring/export/FILENAME.MKV', 'filepath': '/spring/edit/sprloing.blend', 'output_file_extension': '.mkv', 'images_or_video': 'video', 'extract_audio': True, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def test_create_job(self): from pillar.api.utils.authentication import force_cli_user manager, _, _ = self.create_manager_service_account() with self.app.test_request_context(): force_cli_user() self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 5, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, manager['_id'], ) # Test the jobs with self.app.test_request_context(): jobs_coll = self.flamenco.db('jobs') jobs = list(jobs_coll.find()) self.assertEqual(1, len(jobs)) job = jobs[0] self.assertEqual('Wörk wørk w°rk.', job['description']) self.assertEqual('sleep', job['job_type']) # Test the tasks with self.app.test_request_context(): tasks_coll = self.flamenco.db('tasks') tasks = list(tasks_coll.find()) self.assertEqual(2, len(tasks)) statuses = [task['status'] for task in tasks] self.assertEqual(['queued', 'queued'], statuses) self.assertEqual(['sleep', 'sleep'], [task['task_type'] for task in tasks]) task = tasks[0] self.assertEqual('sleep-12-16', task['name']) self.assertEqual({ 'name': 'echo', 'settings': { 'message': 'Preparing to sleep', } }, task['commands'][0]) self.assertEqual({ 'name': 'sleep', 'settings': { 'time_in_seconds': 3, } }, task['commands'][1])
def setUp(self, **kwargs): super().setUp(**kwargs) from pillar.api.utils.authentication import force_cli_user self.manager, _, _ = self.create_manager_service_account( assign_to_project_id=self.proj_id) with self.app.test_request_context(): force_cli_user() job_doc = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 5, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.manager['_id'], ) self.job_id = job_doc['_id']
def create_test_job(manager_id, user_email, project_url): """Creates a test job for a given manager.""" from pillar.api.utils import dumps, str2id manager_id = str2id(manager_id) authentication.force_cli_user() # Find user users_coll = current_app.db()['users'] user = users_coll.find_one({'email': user_email}, projection={'_id': 1}) if not user: raise ValueError('User with email %r not found' % user_email) # Find project projs_coll = current_app.db()['projects'] proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1}) if not proj: log.error('Unable to find project url=%s', project_url) return 1 # Create the job job = flamenco.current_flamenco.job_manager.api_create_job( 'CLI test job', 'Test job created from the server CLI', 'sleep', { 'frames': '1-30, 40-44', 'chunk_size': 13, 'time_in_seconds': 3, }, proj['_id'], user['_id'], manager_id) log.info('Job created:\n%s', dumps(job, indent=4))
def test_user_page(self): # Username 'tester' does not exist self.get('/u/james', expected_status=404) # Username will be 'harry' with self.app.test_request_context(): force_cli_user() create_local_user('*****@*****.**', 'password') self.get('/u/james')
def setUp(self, **kwargs): super().setUp(**kwargs) self.ensure_group_exists( 'cafef005972666988bef650f', 'dillo_user_main') self.proj_id, self.project = self.ensure_project_exists() with self.app.test_request_context(): force_cli_user() self.local_user_id = create_local_user('*****@*****.**', 'password')
def test_create_user(self): from pillar.api.utils.authentication import force_cli_user from pillar.api.local_auth import create_local_user with self.app.test_request_context(): force_cli_user() user_id = create_local_user('*****@*****.**', 'password') db_user = self.fetch_user_from_db(user_id) print(db_user)
def setUp(self, **kwargs): super(JobStatusChangeTest, self).setUp(**kwargs) # Create a job with the same number of tasks as there are task statuses. from pillar.api.utils.authentication import force_cli_user from flamenco.eve_settings import tasks_schema manager, _, token = self.create_manager_service_account() self.mngr_token = token['token'] with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'blender-render', { 'blender_cmd': '{blender}', 'filepath': '/my/file.blend', # Frames and chunk size chosen to produce as many tasks # as there are task statuses - 1 (we don't test 'under-construction') 'frames': '12-18, 20-29', 'chunk_size': 2, 'time_in_seconds': 3, 'render_output': '/not-relevant-now/####', }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, manager['_id'], ) self.job_id = job['_id'] # Fetch the task IDs and set the task statuses to a fixed list. tasks_coll = self.flamenco.db('tasks') tasks = tasks_coll.find( { 'job': self.job_id, 'name': { '$regex': '^blender-render-' }, # don't consider move-out-of-way task. }, projection={'_id': 1}) self.task_ids = [task['_id'] for task in tasks] allowed_statuses: list = tasks_schema['status']['allowed'] # this status should never occur after job compilation allowed_statuses.remove('under-construction') self.assertEqual(len(allowed_statuses), len(self.task_ids)) self.force_task_status(0, 'queued') self.force_task_status(1, 'claimed-by-manager') self.force_task_status(2, 'completed') self.force_task_status(3, 'active') self.force_task_status(4, 'canceled') self.force_task_status(5, 'failed') self.force_task_status(6, 'cancel-requested') self.force_task_status(7, 'paused') self.force_task_status(8, 'soft-failed')
def setup_for_flamenco(project_url, replace=False): """Adds Flamenco node types to the project. Use --replace to replace pre-existing Flamenco node types (by default already existing Flamenco node types are skipped). """ authentication.force_cli_user() flamenco.setup.setup_for_flamenco(project_url, replace=replace)
def test_manager_without_owner_email_address(self): from pillar.api.utils.authentication import force_cli_user from flamenco.setup import create_manager with self.app.test_request_context(): force_cli_user() with self.assertRaises(ValueError): create_manager('', 'Unit test mānāgèr', '¡Awesome in Space!')
def test_manager_without_owner_email_address(self): from pillar.api.utils.authentication import force_cli_user from flamenco.setup import create_manager with self.app.test_request_context(): force_cli_user() with self.assertRaises(ValueError): create_manager('', 'Unit test mānāgèr', '¡Awesome in Space!')
def test_user_page(self): from pillar.api.utils.authentication import force_cli_user from pillar.api.local_auth import create_local_user # Username 'tester' does not exist self.get('/u/harry', expected_status=404) # Username will be 'harry' with self.app.test_request_context(): force_cli_user() create_local_user('*****@*****.**', 'password') self.get('/u/harry')
def test_without_email_address(self): from pillar.api.utils.authentication import force_cli_user from pillar.api.service import create_service_account as create_sa with self.app.test_request_context(): force_cli_user() account, token = create_sa('', ['flamenco_manager'], {'flamenco_manager': {}}) self.assertNotIn('email', account) self.assertAllowsAccess(token, account['_id'])
def setUp(self, **kwargs): super(JobStatusChangeTest, self).setUp(**kwargs) # Create a job with the same number of tasks as there are task statuses. from pillar.api.utils.authentication import force_cli_user from flamenco.eve_settings import tasks_schema manager, _, token = self.create_manager_service_account() self.mngr_token = token['token'] with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'blender-render', { 'blender_cmd': '{blender}', 'filepath': '/my/file.blend', # Frames and chunk size chosen to produce as many tasks # as there are task statuses - 1 (we don't test 'under-construction') 'frames': '12-18, 20-29', 'chunk_size': 2, 'time_in_seconds': 3, 'render_output': '/not-relevant-now/####', }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, manager['_id'], ) self.job_id = job['_id'] # Fetch the task IDs and set the task statuses to a fixed list. tasks_coll = self.flamenco.db('tasks') tasks = tasks_coll.find({ 'job': self.job_id, 'name': {'$regex': '^blender-render-'}, # don't consider move-out-of-way task. }, projection={'_id': 1}) self.task_ids = [task['_id'] for task in tasks] allowed_statuses: list = tasks_schema['status']['allowed'] # this status should never occur after job compilation allowed_statuses.remove('under-construction') self.assertEqual(len(allowed_statuses), len(self.task_ids)) self.force_task_status(0, 'queued') self.force_task_status(1, 'claimed-by-manager') self.force_task_status(2, 'completed') self.force_task_status(3, 'active') self.force_task_status(4, 'canceled') self.force_task_status(5, 'failed') self.force_task_status(6, 'cancel-requested') self.force_task_status(7, 'paused') self.force_task_status(8, 'soft-failed')
def test_recreate_job(self): from pillar.api.utils.authentication import force_cli_user old_task_ids = self._pre_test() with self.app.test_request_context(): # Recreate the job, and check the task statuses. force_cli_user() self.flamenco.api_recreate_job(self.job_id) self._post_test(old_task_ids)
def test_api_find_jobfinal_tasks(self): from pillar.api.utils.authentication import force_cli_user from flamenco.job_compilers import commands manager, _, _ = self.create_manager_service_account() with self.app.test_request_context(): force_cli_user() job_doc = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 7, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, manager['_id'], ) job_id = job_doc['_id'] # Find the tasks created so far, use them as parents. tasks = self.flamenco.db('tasks').find({'job': job_id}, projection={'_id': 1}) task_ids = [t['_id'] for t in tasks] # dependent task that is used as a single parent. taskid1 = self.tmngr.api_create_task( job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 1', parents=task_ids, task_type='sleep', ) # task dependent on multiple tasks that is not used as a parent. taskid2 = self.tmngr.api_create_task( job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 2', parents=task_ids, task_type='sleep', ) # task dependent on a single task that is not used as a parent. taskid3 = self.tmngr.api_create_task( job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 3', parents=[taskid1], task_type='sleep', ) # independent task taskid4 = self.tmngr.api_create_task( job_doc, [commands.Echo(message='ẑžƶźz')], 'zzz 4', task_type='sleep', ) job_enders = self.tmngr.api_find_job_enders(job_id) self.assertEqual({taskid2, taskid3, taskid4}, set(job_enders))
def create_blog(proj_url): """Adds a blog to the project.""" from pillar.api.utils.authentication import force_cli_user from pillar.api.utils import node_type_utils from pillar.api.node_types.blog import node_type_blog from pillar.api.node_types.post import node_type_post from pillar.api.utils import remove_private_keys force_cli_user() db = current_app.db() # Add the blog & post node types to the project. projects_coll = db['projects'] proj = projects_coll.find_one({'url': proj_url}) if not proj: log.error('Project url=%s not found', proj_url) return 3 node_type_utils.add_to_project(proj, (node_type_blog, node_type_post), replace_existing=False) proj_id = proj['_id'] r, _, _, status = current_app.put_internal('projects', remove_private_keys(proj), _id=proj_id) if status != 200: log.error('Error %i storing altered project %s %s', status, proj_id, r) return 4 log.info('Project saved succesfully.') # Create a blog node. nodes_coll = db['nodes'] blog = nodes_coll.find_one({'node_type': 'blog', 'project': proj_id}) if not blog: blog = { 'node_type': node_type_blog['name'], 'name': 'Blog', 'description': '', 'properties': {}, 'project': proj_id, } r, _, _, status = current_app.post_internal('nodes', blog) if status != 201: log.error('Error %i storing blog node: %s', status, r) return 4 log.info('Blog node saved succesfully: %s', r) else: log.info('Blog node already exists: %s', blog) return 0
def test_two_managers_with_same_owner(self): from pillar.api.utils.authentication import force_cli_user from flamenco.setup import create_manager self.create_user(email='*****@*****.**') with self.app.test_request_context(): force_cli_user() mngr_doc1, account1, token1 = create_manager('*****@*****.**', 'mānāgèr 1', 'Awe!') mngr_doc2, account2, token2 = create_manager('*****@*****.**', 'mānāgèr 2', 'som!') self.assertAllowsAccess(token1, account1['_id']) self.assertAllowsAccess(token2, account2['_id'])
def test_two_managers_with_same_owner(self): from pillar.api.utils.authentication import force_cli_user from flamenco.setup import create_manager self.create_user(email='*****@*****.**') with self.app.test_request_context(): force_cli_user() mngr_doc1, account1, token1 = create_manager( '*****@*****.**', 'mānāgèr 1', 'Awe!') mngr_doc2, account2, token2 = create_manager( '*****@*****.**', 'mānāgèr 2', 'som!') self.assertAllowsAccess(token1, account1['_id']) self.assertAllowsAccess(token2, account2['_id'])
def ensure_project_exists(self, project_overrides=None): from flamenco.setup import setup_for_flamenco from pillar.api.utils.authentication import force_cli_user project_overrides = dict(picture_header=None, picture_square=None, **(project_overrides or {})) proj_id, project = AbstractPillarTest.ensure_project_exists( self, project_overrides) with self.app.test_request_context(): force_cli_user() flamenco_project = setup_for_flamenco(project['url'], replace=True) return proj_id, flamenco_project
def create_manager(email, name, description): """Creates a Flamenco manager.""" from pillar.api.utils import dumps authentication.force_cli_user() mngr_doc, account, token = flamenco.setup.create_manager(email, name, description) print('Service account information:') print(dumps(account, indent=4, sort_keys=True)) print() print('Access token: %s' % token['token']) print(' expires on: %s' % token['expire_time']) print() print('Created a new manager:') print(dumps(mngr_doc, indent=4))
def create_manager(email, name, description): """Creates a Flamenco manager.""" from pillar.api.utils import dumps authentication.force_cli_user() mngr_doc, account, token = flamenco.setup.create_manager(email, name, description) print('Service account information:') print(dumps(account, indent=4, sort_keys=True)) print() print('Access token: %s' % token['token']) print(' expires on: %s' % token['expire_time']) print() print('Created a new manager:') print(dumps(mngr_doc, indent=4))
def test_create_service_account(self): from pillar.api.utils.authentication import force_cli_user from pillar.api import service with self.app.test_request_context(): force_cli_user() account, token = service.create_service_account( '*****@*****.**', ['flamenco_manager'], {'flamenco_manager': {}}) self.assertEqual(f'SRV-{account["_id"]}', account['full_name']) self.assertEqual(f'SRV-{account["_id"]}', account['username']) self.assertEqual(['flamenco_manager', 'service'], account['roles']) self.assertEqual([], account['auth']) self.assertEqual({'flamenco_manager': {}}, account['service']) self.assertAllowsAccess(token, account['_id'])
def create_svner_account(email, project_url): """Creates an account that can push SVN activity to an Flamenco project. :param email: email address associated with the account :param project_url: """ authentication.force_cli_user() projs_coll = current_app.db()['projects'] proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1}) if not proj: log.error('Unable to find project url=%s', project_url) return 1 account, token = create_service_account(email, [u'svner'], {'svner': {'project': proj['_id']}}) return account, token
def create_job(self): from pillar.api.utils.authentication import force_cli_user with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) return job['_id']
def _manager_project(manager_id, project_url, action): from pillar.api.utils import str2id from flamenco import current_flamenco authentication.force_cli_user() manager_id = str2id(manager_id) # Find project projs_coll = current_app.db()['projects'] proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1}) if not proj: log.error('Unable to find project url=%s', project_url) return 1 project_id = proj['_id'] ok = current_flamenco.manager_manager.api_assign_to_project(manager_id, project_id, action) if not ok: log.error('Unable to assign manager %s to project %s', manager_id, project_id) return 1
def create_job(self): from pillar.api.utils.authentication import force_cli_user with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) return job['_id']
def _manager_project(manager_id, project_url, action): from pillar.api.utils import str2id from flamenco import current_flamenco authentication.force_cli_user() manager_id = str2id(manager_id) # Find project projs_coll = current_app.db()['projects'] proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1}) if not proj: log.error('Unable to find project url=%s', project_url) return 1 project_id = proj['_id'] ok = current_flamenco.manager_manager.api_assign_to_project(manager_id, project_id, action) if not ok: log.error('Unable to assign manager %s to project %s', manager_id, project_id) return 1
def setUp(self): super().setUp() mngr_doc, account, token = self.create_manager_service_account( assign_to_project_id=self.proj_id) self.mngr_id = mngr_doc['_id'] self.mngr_doc = self.fetch_manager_from_db(self.mngr_id) self.mngr_token = token['token'] assert self.mngr_doc['owner'] self.owner_uid = ObjectId(24 * 'e') self.create_project_member(user_id=str(self.owner_uid), roles={'subscriber'}, groups=[self.mngr_doc['owner']], token='owner-token') from pillar.api.utils.authentication import force_cli_user with self.app.app_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id'] self.tasks = list(self.tmngr.collection().find( {'job': self.job_id})) self.task = self.tasks[0] self.tid = self.task['_id'] # Make sure we can request the task logs. self.tmngr.api_set_task_status_for_job(self.job_id, 'queued', 'completed')
def test_put_without_email_address(self): from pillar.api.utils import remove_private_keys from pillar.api.utils.authentication import force_cli_user from pillar.api.service import create_service_account as create_sa with self.app.test_request_context(): force_cli_user() account, token = create_sa('', ['flamenco_manager'], {'flamenco_manager': {}}) puttable = remove_private_keys(account) user_id = account['_id'] # The user should be able to edit themselves, even without email address. etag = account['_etag'] puttable['full_name'] = 'þor' resp = self.put(f'/api/users/{user_id}', json=puttable, auth_token=token['token'], etag=etag).json() etag = resp['_etag'] with self.app.test_request_context(): users_coll = self.app.db().users db_user = users_coll.find_one(user_id) self.assertNotIn('email', db_user) self.assertEqual('þor', db_user['full_name']) # An admin should be able to edit this email-less user. self.create_user(24 * 'a', roles={'admin'}, token='admin-token') puttable['username'] = '******' self.put(f'/api/users/{user_id}', json=puttable, auth_token='admin-token', etag=etag) with self.app.test_request_context(): users_coll = self.app.db().users db_user = users_coll.find_one(user_id) self.assertNotIn('email', db_user) self.assertEqual('bigdüde', db_user['username'])
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user from pillar.api.projects.utils import get_admin_group_id mngr_doc, account, token = self.create_manager_service_account() self.mngr_id = mngr_doc['_id'] self.mngr_token = token['token'] self.assign_manager_to_project(self.mngr_id, self.proj_id) with self.app.test_request_context(): group_id = get_admin_group_id(self.proj_id) self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, groups=[group_id]) self.create_valid_auth_token(24 * 'f', 'fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'blender-render-progressive', { 'frames': '1-5', 'chunk_size': 2, 'render_output': '/render/out/frames-######', 'format': 'EXR', 'filepath': '/agent327/scenes/someshot/somefile.blend', 'blender_cmd': '/path/to/blender --enable-new-depsgraph', 'cycles_sample_count': 30, 'cycles_num_chunks': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def purge_home_projects(go=False): """Deletes all home projects that have no owner.""" from pillar.api.utils.authentication import force_cli_user force_cli_user() users_coll = current_app.data.driver.db['users'] proj_coll = current_app.data.driver.db['projects'] good = bad = 0 def bad_projects(): nonlocal good, bad for proj in proj_coll.find({'category': 'home', '_deleted': {'$ne': True}}): pid = proj['_id'] uid = proj.get('user') if not uid: log.info('Project %s has no user assigned', uid) bad += 1 yield pid continue if users_coll.find({'_id': uid, '_deleted': {'$ne': True}}).count() == 0: log.info('Project %s has non-existing owner %s', pid, uid) bad += 1 yield pid continue good += 1 if not go: log.info('Dry run, use --go to actually perform the changes.') for project_id in bad_projects(): log.info('Soft-deleting project %s', project_id) if go: r, _, _, status = current_app.delete_internal('projects', _id=project_id) if status != 204: raise ValueError(f'Error {status} deleting {project_id}: {r}') log.info('%i projects OK, %i projects deleted', good, bad) return bad
def setUp(self, **kwargs): AbstractTaskBatchUpdateTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user from pillar.api.projects.utils import get_admin_group_id mngr_doc, account, token = self.create_manager_service_account() self.mngr_id = mngr_doc['_id'] self.mngr_token = token['token'] self.assign_manager_to_project(self.mngr_id, self.proj_id) with self.app.test_request_context(): group_id = get_admin_group_id(self.proj_id) self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, groups=[group_id]) self.create_valid_auth_token(24 * 'f', 'fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'blender-render-progressive', { 'frames': '1-5', 'chunk_size': 2, 'render_output': '/render/out/frames-######', 'format': 'OPEN_EXR', 'fps': 44, 'filepath': '/agent327/scenes/someshot/somefile.blend', 'blender_cmd': '/path/to/blender --enable-new-depsgraph', 'cycles_sample_count': 30, 'cycles_sample_cap': 30, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user self.create_manager() self.create_user(user_id=24 * 'f', roles={'flamenco-admin'}, token='fladmin-token') with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id']
def test_create_task(self): from pillar.api.utils.authentication import force_cli_user from flamenco.job_compilers import commands manager, _, _ = self.create_manager_service_account() with self.app.test_request_context(): force_cli_user() job_doc = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 7, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, manager['_id'], ) self.tmngr.api_create_task( job_doc, [ commands.Echo(message='ẑžƶźz'), commands.Sleep(time_in_seconds=3), ], 'sleep-1-13', status='under-construction', task_type='exr-merge', ) # Now test the database contents. with self.app.test_request_context(): tasks_coll = self.flamenco.db('tasks') dbtasks = list(tasks_coll.find()) self.assertEqual(3, len(dbtasks)) # 2 of compiled job + the one we added after. statuses = [task['status'] for task in dbtasks] self.assertEqual(['queued', 'queued', 'under-construction'], statuses) types = [task['task_type'] for task in dbtasks] self.assertEqual(['sleep', 'sleep', 'exr-merge'], types) dbtask = dbtasks[-1] self.assertEqual({ 'name': 'echo', 'settings': { 'message': 'ẑžƶźz', } }, dbtask['commands'][0]) self.assertEqual({ 'name': 'sleep', 'settings': { 'time_in_seconds': 3, } }, dbtask['commands'][1]) return job_doc['_id']
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from flamenco import current_flamenco from pillar.api.utils.authentication import force_cli_user # Main project will have a manager, job, and tasks. mngr_doc, _, token = self.create_manager_service_account() self.mngr_id = mngr_doc['_id'] self.mngr_doc = mngr_doc self.mngr_token = token['token'] with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.job_id = job['_id'] tasks_coll = current_flamenco.db('tasks') self.tasks_for_job = list(tasks_coll.find({'job': self.job_id})) # Another project, also with manager, job, and tasks. proj2_owner_id = 24 * 'a' self.proj2 = self._create_user_and_project(user_id=proj2_owner_id, roles={'subscriber'}, project_name='Prøject 2', token='token-proj2-owner') self.proj2_id = self.proj2['_id'] mngr_doc, _, token = self.create_manager_service_account( owner_email='*****@*****.**', name='manager 2' ) self.mngr2_id = mngr_doc['_id'] self.mngr2_doc = mngr_doc self.mngr2_token = token['token'] with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job 2', 'Wörk² wørk² w°rk².', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj2_id, proj2_owner_id, self.mngr2_id, ) self.job2_id = job['_id']
def setup_for_flamenco(project_url): """Adds Flamenco node types to the project.""" authentication.force_cli_user() flamenco.setup.setup_for_flamenco(project_url)
def setUp(self, **kwargs): AbstractFlamencoTest.setUp(self, **kwargs) from pillar.api.utils.authentication import force_cli_user mngr_doc, account, token = self.create_manager_service_account( assign_to_project_id=self.proj_id) self.mngr_id = mngr_doc['_id'] self.mngr_token = token['token'] with self.app.app_context(): project_gid = get_admin_group_id(self.proj_id) self.user = self.create_user(roles={'subscriber'}, groups=[project_gid, mngr_doc['owner']], token='user-token') # Create three test jobs, one of which is completed and two are queued. with self.app.test_request_context(): force_cli_user() job = self.jmngr.api_create_job( 'test job 1', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.jobid1 = job['_id'] job = self.jmngr.api_create_job( 'test job 2', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.jobid2 = job['_id'] job = self.jmngr.api_create_job( 'test job 3', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, ) self.jobid3 = job['_id'] job = self.jmngr.api_create_job( 'test job 4', 'Wörk wørk w°rk.', 'sleep', { 'frames': '12-18, 20-22', 'chunk_size': 3, 'time_in_seconds': 3, }, self.proj_id, ctd.EXAMPLE_PROJECT_OWNER_ID, self.mngr_id, start_paused=True, ) self.jobid4 = job['_id'] assert isinstance(self.jobid1, ObjectId) assert isinstance(self.jobid2, ObjectId) assert isinstance(self.jobid3, ObjectId) assert isinstance(self.jobid4, ObjectId) self.set_job_status('completed', job_id=self.jobid3) self.tasks = list(self.flamenco.db('tasks').find({ 'job': {'$in': [self.jobid1, self.jobid2]} })) self.task_ids = [t['_id'] for t in self.tasks]