def execute_job(job_id, data):
# Elements common to all job types
cluster_id = data['cluster_id']
configs = data.get('job_configs', {})
interface = data.get('interface', {})
# Not in Java job types but present for all others
input_id = data.get('input_id', None)
output_id = data.get('output_id', None)
# Since we will use a unified class in the database, we pass
# a superset for all job types
job_ex_dict = {'input_id': input_id, 'output_id': output_id,
'job_id': job_id, 'cluster_id': cluster_id,
'info': {'status': edp.JOB_STATUS_PENDING},
'job_configs': configs, 'extra': {},
'interface': interface}
job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict)
context.set_current_job_execution_id(job_execution.id)
# check to use proxy user
if p.job_execution_requires_proxy_user(job_execution):
try:
p.create_proxy_user_for_job_execution(job_execution)
except ex.SaharaException as e:
LOG.error(_LE("Can't run job execution. "
"(Reasons: {reason})").format(reason=e))
conductor.job_execution_destroy(context.ctx(), job_execution)
raise e
OPS.run_edp_job(job_execution.id)
return job_execution
def execute_job(job_id, data):
# Elements common to all job types
cluster_id = data['cluster_id']
configs = data.get('job_configs', {})
# Not in Java job types but present for all others
input_id = data.get('input_id', None)
output_id = data.get('output_id', None)
# Since we will use a unified class in the database, we pass
# a superset for all job types
job_ex_dict = {'input_id': input_id, 'output_id': output_id,
'job_id': job_id, 'cluster_id': cluster_id,
'info': {'status': edp.JOB_STATUS_PENDING},
'job_configs': configs, 'extra': {}}
job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict)
# check to use proxy user
if p.job_execution_requires_proxy_user(job_execution):
try:
p.create_proxy_user_for_job_execution(job_execution)
except ex.SaharaException as e:
LOG.exception(_LE("Can't run job execution '{0}' "
"(reasons: {1})").format(job_execution.id, e))
conductor.job_execution_destroy(context.ctx(), job_execution)
raise e
OPS.run_edp_job(job_execution.id)
return job_execution
def test_job_execution_requires_proxy_user(self, data_source, job):
self.override_config('use_domain_for_proxy_users', True)
job_execution = mock.Mock(input_id=1,
output_id=2,
job_id=3,
job_configs={})
data_source.return_value = mock.Mock(url='swift://container/object')
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source.return_value = mock.Mock(url='')
job.return_value = mock.Mock(
mains=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job.return_value = mock.Mock(
mains=[], libs=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs['args'] = ['swift://container/object']
job.return_value = mock.Mock(mains=[], libs=[])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
def test_job_execution_requires_proxy_user(self, data_source, job):
self.override_config('use_domain_for_proxy_users', True)
job_execution = mock.Mock(input_id=1,
output_id=2,
job_id=3,
job_configs={})
data_source.return_value = mock.Mock(url='swift://container/object')
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source.return_value = mock.Mock(url='')
job.return_value = mock.Mock(
mains=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job.return_value = mock.Mock(
mains=[],
libs=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs['args'] = ['swift://container/object']
job.return_value = mock.Mock(
mains=[],
libs=[])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
def execute_job(data):
# Elements common to all job types
job_templates_id = data['job_templates_id']
cluster_id = data['cluster_id']
configs = data.get('job_configs', {})
interface = data.get('interface', {})
# Not in Java job types but present for all others
input_id = data.get('input_id', None)
output_id = data.get('output_id', None)
# Since we will use a unified class in the database, we pass
# a superset for all job types
# example configs['start'] = '2015-05-12T08:55Z' frequency = 5 mins
# the job will starts from 2015-05-12T08:55Z, runs every 5 mins
job_execution_info = data.get('job_execution_info', {})
configs['job_execution_info'] = job_execution_info
job_ex_dict = {
'input_id': input_id,
'output_id': output_id,
'job_id': job_templates_id,
'cluster_id': cluster_id,
'info': {
'status': edp.JOB_STATUS_PENDING
},
'job_configs': configs,
'extra': {},
'interface': interface
}
job_execution = conductor.job_execution_create(context.ctx(), job_ex_dict)
context.set_current_job_execution_id(job_execution.id)
# check to use proxy user
if p.job_execution_requires_proxy_user(job_execution):
try:
p.create_proxy_user_for_job_execution(job_execution)
except ex.SaharaException as e:
LOG.error("Can't run job execution. "
"(Reasons: {reason})".format(reason=e))
conductor.job_execution_destroy(context.ctx(), job_execution)
raise e
api.OPS.run_edp_job(job_execution.id)
return job_execution
def test_job_execution_requires_proxy_user(self, ctx, data_source_count,
data_source, job):
self.override_config('use_domain_for_proxy_users', True)
job_execution = mock.Mock(input_id=1,
output_id=2,
job_id=3,
job_configs={})
data_source.return_value = mock.Mock(url='swift://container/object')
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source.return_value = mock.Mock(url='')
job.return_value = mock.Mock(
mains=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job.return_value = mock.Mock(
mains=[], libs=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs = {'args': ['swift://container/object']}
job.return_value = mock.Mock(mains=[], libs=[])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs = {
'configs': {
'key': 'swift://container/object'
}
}
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs = {
'params': {
'key': 'swift://container/object'
}
}
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source_count.return_value = 0
job_execution.job_configs = {
'configs': {
job_utils.DATA_SOURCE_SUBST_NAME: True
}
}
job.return_value = mock.Mock(mains=[], libs=[])
self.assertFalse(p.job_execution_requires_proxy_user(job_execution))
ctx.return_value = 'dummy'
data_source_count.return_value = 1
job_execution.job_configs = {
'configs': {
job_utils.DATA_SOURCE_SUBST_NAME: True
},
'args': [job_utils.DATA_SOURCE_PREFIX + 'somevalue']
}
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source_count.assert_called_with('dummy',
name=('somevalue', ),
url='swift://%')
data_source_count.reset_mock()
data_source_count.return_value = 1
myid = uuidutils.generate_uuid()
job_execution.job_configs = {
'configs': {
job_utils.DATA_SOURCE_SUBST_UUID: True
},
'args': [myid]
}
job.return_value = mock.Mock(mains=[], libs=[])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source_count.assert_called_with('dummy',
id=(myid, ),
url='swift://%')
def test_job_execution_requires_proxy_user(self,
ctx,
data_source_count,
data_source,
job):
self.override_config('use_domain_for_proxy_users', True)
job_execution = mock.Mock(input_id=1,
output_id=2,
job_id=3,
job_configs={})
data_source.return_value = mock.Mock(url='swift://container/object')
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source.return_value = mock.Mock(url='')
job.return_value = mock.Mock(
mains=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job.return_value = mock.Mock(
mains=[],
libs=[mock.Mock(url='swift://container/object')])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs = {'args': ['swift://container/object']}
job.return_value = mock.Mock(
mains=[],
libs=[])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs = {
'configs': {'key': 'swift://container/object'}}
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
job_execution.job_configs = {
'params': {'key': 'swift://container/object'}}
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source_count.return_value = 0
job_execution.job_configs = {
'configs': {job_utils.DATA_SOURCE_SUBST_NAME: True}}
job.return_value = mock.Mock(
mains=[],
libs=[])
self.assertFalse(p.job_execution_requires_proxy_user(job_execution))
ctx.return_value = 'dummy'
data_source_count.return_value = 1
job_execution.job_configs = {
'configs': {job_utils.DATA_SOURCE_SUBST_NAME: True},
'args': [job_utils.DATA_SOURCE_PREFIX+'somevalue']}
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source_count.assert_called_with('dummy',
name=('somevalue',),
url='swift://%')
data_source_count.reset_mock()
data_source_count.return_value = 1
myid = six.text_type(uuid.uuid4())
job_execution.job_configs = {
'configs': {job_utils.DATA_SOURCE_SUBST_UUID: True},
'args': [myid]}
job.return_value = mock.Mock(
mains=[],
libs=[])
self.assertTrue(p.job_execution_requires_proxy_user(job_execution))
data_source_count.assert_called_with('dummy',
id=(myid,),
url='swift://%')
