def execute_job(job_id, data): # Elements common to all job types cluster_id = data['cluster_id'] configs = data.get('job_configs', {}) interface = data.get('interface', {}) # Not in Java job types but present for all others input_id = data.get('input_id', None) output_id = data.get('output_id', None) # Since we will use a unified class in the database, we pass # a superset for all job types job_ex_dict = {'input_id': input_id, 'output_id': output_id, 'job_id': job_id, 'cluster_id': cluster_id, 'info': {'status': edp.JOB_STATUS_PENDING}, 'job_configs': configs, 'extra': {}, 'interface': interface} job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict) context.set_current_job_execution_id(job_execution.id) # check to use proxy user if p.job_execution_requires_proxy_user(job_execution): try: p.create_proxy_user_for_job_execution(job_execution) except ex.SaharaException as e: LOG.error(_LE("Can't run job execution. " "(Reasons: {reason})").format(reason=e)) conductor.job_execution_destroy(context.ctx(), job_execution) raise e OPS.run_edp_job(job_execution.id) return job_execution
def execute_job(job_id, data): # Elements common to all job types cluster_id = data['cluster_id'] configs = data.get('job_configs', {}) # Not in Java job types but present for all others input_id = data.get('input_id', None) output_id = data.get('output_id', None) # Since we will use a unified class in the database, we pass # a superset for all job types job_ex_dict = {'input_id': input_id, 'output_id': output_id, 'job_id': job_id, 'cluster_id': cluster_id, 'info': {'status': edp.JOB_STATUS_PENDING}, 'job_configs': configs, 'extra': {}} job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict) # check to use proxy user if p.job_execution_requires_proxy_user(job_execution): try: p.create_proxy_user_for_job_execution(job_execution) except ex.SaharaException as e: LOG.exception(_LE("Can't run job execution '{0}' " "(reasons: {1})").format(job_execution.id, e)) conductor.job_execution_destroy(context.ctx(), job_execution) raise e OPS.run_edp_job(job_execution.id) return job_execution
def test_job_execution_requires_proxy_user(self, data_source, job): self.override_config('use_domain_for_proxy_users', True) job_execution = mock.Mock(input_id=1, output_id=2, job_id=3, job_configs={}) data_source.return_value = mock.Mock(url='swift://container/object') self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source.return_value = mock.Mock(url='') job.return_value = mock.Mock( mains=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job.return_value = mock.Mock( mains=[], libs=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs['args'] = ['swift://container/object'] job.return_value = mock.Mock(mains=[], libs=[]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
def test_job_execution_requires_proxy_user(self, data_source, job): self.override_config('use_domain_for_proxy_users', True) job_execution = mock.Mock(input_id=1, output_id=2, job_id=3, job_configs={}) data_source.return_value = mock.Mock(url='swift://container/object') self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source.return_value = mock.Mock(url='') job.return_value = mock.Mock( mains=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job.return_value = mock.Mock( mains=[], libs=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs['args'] = ['swift://container/object'] job.return_value = mock.Mock( mains=[], libs=[]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
def execute_job(data): # Elements common to all job types job_templates_id = data['job_templates_id'] cluster_id = data['cluster_id'] configs = data.get('job_configs', {}) interface = data.get('interface', {}) # Not in Java job types but present for all others input_id = data.get('input_id', None) output_id = data.get('output_id', None) # Since we will use a unified class in the database, we pass # a superset for all job types # example configs['start'] = '2015-05-12T08:55Z' frequency = 5 mins # the job will starts from 2015-05-12T08:55Z, runs every 5 mins job_execution_info = data.get('job_execution_info', {}) configs['job_execution_info'] = job_execution_info job_ex_dict = { 'input_id': input_id, 'output_id': output_id, 'job_id': job_templates_id, 'cluster_id': cluster_id, 'info': { 'status': edp.JOB_STATUS_PENDING }, 'job_configs': configs, 'extra': {}, 'interface': interface } job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict) context.set_current_job_execution_id(job_execution.id) # check to use proxy user if p.job_execution_requires_proxy_user(job_execution): try: p.create_proxy_user_for_job_execution(job_execution) except ex.SaharaException as e: LOG.error("Can't run job execution. " "(Reasons: {reason})".format(reason=e)) conductor.job_execution_destroy(context.ctx(), job_execution) raise e api.OPS.run_edp_job(job_execution.id) return job_execution
def test_job_execution_requires_proxy_user(self, ctx, data_source_count, data_source, job): self.override_config('use_domain_for_proxy_users', True) job_execution = mock.Mock(input_id=1, output_id=2, job_id=3, job_configs={}) data_source.return_value = mock.Mock(url='swift://container/object') self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source.return_value = mock.Mock(url='') job.return_value = mock.Mock( mains=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job.return_value = mock.Mock( mains=[], libs=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs = {'args': ['swift://container/object']} job.return_value = mock.Mock(mains=[], libs=[]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs = { 'configs': { 'key': 'swift://container/object' } } self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs = { 'params': { 'key': 'swift://container/object' } } self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source_count.return_value = 0 job_execution.job_configs = { 'configs': { job_utils.DATA_SOURCE_SUBST_NAME: True } } job.return_value = mock.Mock(mains=[], libs=[]) self.assertFalse(p.job_execution_requires_proxy_user(job_execution)) ctx.return_value = 'dummy' data_source_count.return_value = 1 job_execution.job_configs = { 'configs': { job_utils.DATA_SOURCE_SUBST_NAME: True }, 'args': [job_utils.DATA_SOURCE_PREFIX + 'somevalue'] } self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source_count.assert_called_with('dummy', name=('somevalue', ), url='swift://%') data_source_count.reset_mock() data_source_count.return_value = 1 myid = uuidutils.generate_uuid() job_execution.job_configs = { 'configs': { job_utils.DATA_SOURCE_SUBST_UUID: True }, 'args': [myid] } job.return_value = mock.Mock(mains=[], libs=[]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source_count.assert_called_with('dummy', id=(myid, ), url='swift://%')
def test_job_execution_requires_proxy_user(self, ctx, data_source_count, data_source, job): self.override_config('use_domain_for_proxy_users', True) job_execution = mock.Mock(input_id=1, output_id=2, job_id=3, job_configs={}) data_source.return_value = mock.Mock(url='swift://container/object') self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source.return_value = mock.Mock(url='') job.return_value = mock.Mock( mains=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job.return_value = mock.Mock( mains=[], libs=[mock.Mock(url='swift://container/object')]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs = {'args': ['swift://container/object']} job.return_value = mock.Mock( mains=[], libs=[]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs = { 'configs': {'key': 'swift://container/object'}} self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) job_execution.job_configs = { 'params': {'key': 'swift://container/object'}} self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source_count.return_value = 0 job_execution.job_configs = { 'configs': {job_utils.DATA_SOURCE_SUBST_NAME: True}} job.return_value = mock.Mock( mains=[], libs=[]) self.assertFalse(p.job_execution_requires_proxy_user(job_execution)) ctx.return_value = 'dummy' data_source_count.return_value = 1 job_execution.job_configs = { 'configs': {job_utils.DATA_SOURCE_SUBST_NAME: True}, 'args': [job_utils.DATA_SOURCE_PREFIX+'somevalue']} self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source_count.assert_called_with('dummy', name=('somevalue',), url='swift://%') data_source_count.reset_mock() data_source_count.return_value = 1 myid = six.text_type(uuid.uuid4()) job_execution.job_configs = { 'configs': {job_utils.DATA_SOURCE_SUBST_UUID: True}, 'args': [myid]} job.return_value = mock.Mock( mains=[], libs=[]) self.assertTrue(p.job_execution_requires_proxy_user(job_execution)) data_source_count.assert_called_with('dummy', id=(myid,), url='swift://%')