def test_minion_perform_execute_reload_code_success():
workflow_id = '6666'
app_id = '667788'
job_id = '1'
workflow = {
'workflow_id': workflow_id,
'app_id': app_id,
'job_id': job_id,
'type': 'execute',
'workflow': ''
}
function_name = 'juicer.spark.transpiler.SparkTranspiler.transpile'
with mock.patch('redis.StrictRedis',
mock_strict_redis_client) as mocked_redis:
with mock.patch(function_name) as mocked_transpile:
with mock.patch('juicer.workflow.workflow.Workflow'
'._build_initial_workflow_graph') as mocked_fn:
mocked_fn.side_effect = lambda: ""
# Setup for mocked_transpile
mocked_transpile.side_effect = get_side_effect(None, None, 2)
redis_conn = mocked_redis()
minion = SparkMinion(redis_conn=redis_conn,
workflow_id=workflow_id,
app_id=app_id,
config=config)
minion.get_or_create_spark_session = \
dummy_get_or_create_spark_session
minion._emit_event = dummy_emit_event
# Configure mocked redis
state_control = StateControlRedis(redis_conn)
with open(
os.path.join(os.path.dirname(__file__),
'fixtures/simple_workflow.json')) as f:
data = json.loads(f.read())
workflow['workflow'] = data
state_control.push_app_queue(app_id, json.dumps(workflow))
minion._process_message()
assert minion._state == {'res': 'version 1.0'}, 'Invalid state'
# Executes the same workflow, but code should be different
state_control.push_app_queue(app_id, json.dumps(workflow))
assert state_control.get_app_output_queue_size(
app_id) == 1, 'Wrong number of output messages'
state_control.pop_app_queue(app_id, True, 0)
minion.transpiler.transpile = get_side_effect(None, None, 3)
minion._process_message()
assert minion._state == {'res': 'version 2.1'}, 'Invalid state'
assert state_control.get_app_output_queue_size(
app_id) == 2, 'Wrong number of output messages'
def test_runner_read_start_queue_success():
config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}}
app_id = '1'
workflow_id = '1000'
workflow = {
'workflow_id': workflow_id,
'app_id': app_id,
'type': 'execute',
'workflow': {}
}
with mock.patch('redis.StrictRedis',
mock_strict_redis_client) as mocked_redis:
with mock.patch('subprocess.Popen') as mocked_popen:
server = server = JuicerServer(config,
'faked_minions.py',
config_file_path='config.yaml')
mocked_redis_conn = mocked_redis()
state_control = StateControlRedis(mocked_redis_conn)
# Publishes a message to process data
state_control.push_start_queue(json.dumps(workflow))
# Start of testing
server.read_job_start_queue(mocked_redis_conn)
d1 = json.loads(state_control.get_minion_status(app_id))
d2 = {
"port": 36000,
"pid": 1,
}
assert d1 == d2
assert mocked_popen.call_args_list[0][0][0] == [
'nohup', sys.executable, 'faked_minions.py', '-w', workflow_id,
'-a', app_id, '-t', 'spark', '-c', 'config.yaml'
]
assert mocked_popen.called
# Was command removed from the queue?
assert state_control.pop_job_start_queue(False) is None
assert json.loads(state_control.pop_app_queue(app_id)) == workflow
assert state_control.get_workflow_status(
workflow_id) == JuicerServer.STARTED
assert json.loads(state_control.pop_app_output_queue(app_id)) == {
'code': 0,
'message': 'Minion is processing message execute'
}
def test_runner_read_start_queue_minion_already_running_success():
config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}}
app_id = 1
workflow_id = 1000
workflow = {
'workflow_id': workflow_id,
'app_id': app_id,
'type': 'execute',
'workflow': {}
}
with mock.patch('redis.StrictRedis',
mock_strict_redis_client) as mocked_redis:
with mock.patch('subprocess.Popen') as mocked_popen:
server = JuicerServer(config, 'faked_minions.py')
mocked_redis_conn = mocked_redis()
state_control = StateControlRedis(mocked_redis_conn)
# Publishes a message to process data
state_control.push_start_queue(json.dumps(workflow))
state_control.set_minion_status(app_id, JuicerServer.STARTED)
# Start of testing
server.read_job_start_queue(mocked_redis_conn)
assert state_control.get_minion_status(
app_id) == JuicerServer.STARTED
assert not mocked_popen.called
# Was command removed from the queue?
assert mocked_redis_conn.lpop('start') is None
assert json.loads(state_control.pop_app_queue(app_id)) == workflow
assert state_control.get_workflow_status(
workflow_id) == JuicerServer.STARTED
assert json.loads(state_control.pop_app_output_queue(app_id)) == {
'code': 0,
'message': 'Minion is processing message execute'
}
def test_runner_read_start_queue_missing_details_failure():
config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}}
app_id = 1
workflow_id = 1000
# incorrect key, should raise exception
workflow = {
'workflow_id': workflow_id,
'xapp_id': app_id,
'type': 'execute',
'workflow': {}
}
with mock.patch('redis.StrictRedis',
mock_strict_redis_client) as mocked_redis:
with mock.patch('subprocess.Popen') as mocked_popen:
server = JuicerServer(config, 'faked_minions.py')
mocked_redis_conn = mocked_redis()
# Publishes a message to process data
state_control = StateControlRedis(mocked_redis_conn)
# Publishes a message to process data
state_control.push_start_queue(json.dumps(workflow))
state_control.set_minion_status(app_id, JuicerServer.STARTED)
# Start of testing
server.read_job_start_queue(mocked_redis_conn)
assert state_control.get_minion_status(
app_id) == JuicerServer.STARTED
assert not mocked_popen.called
# Was command removed from the queue?
assert state_control.pop_job_start_queue(block=False) is None
assert state_control.pop_app_queue(app_id, block=False) is None
assert state_control.pop_app_output_queue(app_id,
block=False) is None
assert mocked_redis_conn.hget(workflow_id, 'status') is None