def test_minion_perform_execute_reload_code_success(): workflow_id = '6666' app_id = '667788' job_id = '1' workflow = { 'workflow_id': workflow_id, 'app_id': app_id, 'job_id': job_id, 'type': 'execute', 'workflow': '' } function_name = 'juicer.spark.transpiler.SparkTranspiler.transpile' with mock.patch('redis.StrictRedis', mock_strict_redis_client) as mocked_redis: with mock.patch(function_name) as mocked_transpile: with mock.patch('juicer.workflow.workflow.Workflow' '._build_initial_workflow_graph') as mocked_fn: mocked_fn.side_effect = lambda: "" # Setup for mocked_transpile mocked_transpile.side_effect = get_side_effect(None, None, 2) redis_conn = mocked_redis() minion = SparkMinion(redis_conn=redis_conn, workflow_id=workflow_id, app_id=app_id, config=config) minion.get_or_create_spark_session = \ dummy_get_or_create_spark_session minion._emit_event = dummy_emit_event # Configure mocked redis state_control = StateControlRedis(redis_conn) with open( os.path.join(os.path.dirname(__file__), 'fixtures/simple_workflow.json')) as f: data = json.loads(f.read()) workflow['workflow'] = data state_control.push_app_queue(app_id, json.dumps(workflow)) minion._process_message() assert minion._state == {'res': 'version 1.0'}, 'Invalid state' # Executes the same workflow, but code should be different state_control.push_app_queue(app_id, json.dumps(workflow)) assert state_control.get_app_output_queue_size( app_id) == 1, 'Wrong number of output messages' state_control.pop_app_queue(app_id, True, 0) minion.transpiler.transpile = get_side_effect(None, None, 3) minion._process_message() assert minion._state == {'res': 'version 2.1'}, 'Invalid state' assert state_control.get_app_output_queue_size( app_id) == 2, 'Wrong number of output messages'
def test_runner_read_start_queue_success(): config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}} app_id = '1' workflow_id = '1000' workflow = { 'workflow_id': workflow_id, 'app_id': app_id, 'type': 'execute', 'workflow': {} } with mock.patch('redis.StrictRedis', mock_strict_redis_client) as mocked_redis: with mock.patch('subprocess.Popen') as mocked_popen: server = server = JuicerServer(config, 'faked_minions.py', config_file_path='config.yaml') mocked_redis_conn = mocked_redis() state_control = StateControlRedis(mocked_redis_conn) # Publishes a message to process data state_control.push_start_queue(json.dumps(workflow)) # Start of testing server.read_job_start_queue(mocked_redis_conn) d1 = json.loads(state_control.get_minion_status(app_id)) d2 = { "port": 36000, "pid": 1, } assert d1 == d2 assert mocked_popen.call_args_list[0][0][0] == [ 'nohup', sys.executable, 'faked_minions.py', '-w', workflow_id, '-a', app_id, '-t', 'spark', '-c', 'config.yaml' ] assert mocked_popen.called # Was command removed from the queue? assert state_control.pop_job_start_queue(False) is None assert json.loads(state_control.pop_app_queue(app_id)) == workflow assert state_control.get_workflow_status( workflow_id) == JuicerServer.STARTED assert json.loads(state_control.pop_app_output_queue(app_id)) == { 'code': 0, 'message': 'Minion is processing message execute' }
def test_runner_read_start_queue_minion_already_running_success(): config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}} app_id = 1 workflow_id = 1000 workflow = { 'workflow_id': workflow_id, 'app_id': app_id, 'type': 'execute', 'workflow': {} } with mock.patch('redis.StrictRedis', mock_strict_redis_client) as mocked_redis: with mock.patch('subprocess.Popen') as mocked_popen: server = JuicerServer(config, 'faked_minions.py') mocked_redis_conn = mocked_redis() state_control = StateControlRedis(mocked_redis_conn) # Publishes a message to process data state_control.push_start_queue(json.dumps(workflow)) state_control.set_minion_status(app_id, JuicerServer.STARTED) # Start of testing server.read_job_start_queue(mocked_redis_conn) assert state_control.get_minion_status( app_id) == JuicerServer.STARTED assert not mocked_popen.called # Was command removed from the queue? assert mocked_redis_conn.lpop('start') is None assert json.loads(state_control.pop_app_queue(app_id)) == workflow assert state_control.get_workflow_status( workflow_id) == JuicerServer.STARTED assert json.loads(state_control.pop_app_output_queue(app_id)) == { 'code': 0, 'message': 'Minion is processing message execute' }
def test_runner_read_start_queue_missing_details_failure(): config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}} app_id = 1 workflow_id = 1000 # incorrect key, should raise exception workflow = { 'workflow_id': workflow_id, 'xapp_id': app_id, 'type': 'execute', 'workflow': {} } with mock.patch('redis.StrictRedis', mock_strict_redis_client) as mocked_redis: with mock.patch('subprocess.Popen') as mocked_popen: server = JuicerServer(config, 'faked_minions.py') mocked_redis_conn = mocked_redis() # Publishes a message to process data state_control = StateControlRedis(mocked_redis_conn) # Publishes a message to process data state_control.push_start_queue(json.dumps(workflow)) state_control.set_minion_status(app_id, JuicerServer.STARTED) # Start of testing server.read_job_start_queue(mocked_redis_conn) assert state_control.get_minion_status( app_id) == JuicerServer.STARTED assert not mocked_popen.called # Was command removed from the queue? assert state_control.pop_job_start_queue(block=False) is None assert state_control.pop_app_queue(app_id, block=False) is None assert state_control.pop_app_output_queue(app_id, block=False) is None assert mocked_redis_conn.hget(workflow_id, 'status') is None