def test_minion_perform_deliver_missing_state_invalid_port_failure(): workflow_id = '6666' app_id = '5001' job_id = '1' out_queue = 'queue_50001' task_id = 'f033f-284ab-28987e-232add' function_name = 'juicer.spark.transpiler.SparkTranspiler.transpile' with mock.patch('redis.StrictRedis', mock_strict_redis_client) as mocked_redis: with mock.patch(function_name) as mocked_transpile: with mock.patch('juicer.workflow.workflow.Workflow' '._build_initial_workflow_graph') as mocked_fn: mocked_fn.side_effect = lambda: "" # Setup for mocked_transpile mocked_transpile.side_effect = get_side_effect( get_records(), task_id, 0) redis_conn = mocked_redis() state_control = StateControlRedis(redis_conn) data = { 'workflow_id': workflow_id, 'app_id': app_id, 'job_id': job_id, 'type': 'deliver', 'task_id': task_id, 'port': 'port2', # This port is invalid 'output': out_queue, 'workflow': { "tasks": [], "flows": [] } } state_control.push_app_queue(app_id, json.dumps(data)) minion = SparkMinion(redis_conn=redis_conn, workflow_id=workflow_id, app_id=app_id, config=config) minion.get_or_create_spark_session = \ dummy_get_or_create_spark_session minion._emit_event = dummy_emit_event minion._state = {} minion._process_message() # Discard first status message state_control.pop_app_output_queue(app_id, False) msg = json.loads( state_control.pop_app_output_queue(app_id, False)) assert msg['status'] == 'WARNING', 'Invalid status' assert msg['code'] == minion.MNN003[0], 'Invalid code' msg = json.loads( state_control.pop_app_output_queue(app_id, False)) assert msg['status'] == 'ERROR', 'Invalid status' assert msg.get('code') == minion.MNN004[0], 'Invalid code' result = json.loads(state_control.pop_queue(out_queue, False)) assert not result['sample'], 'Wrong CSV generated'
def test_minion_perform_deliver_success(): workflow_id = '6666' app_id = '1000' job_id = '1' out_queue = 'queue_2000' sconf = SparkConf() sc = SparkContext(master='', conf=sconf) rdd = sc.parallelize(get_records()) df0 = DataFrame(rdd=rdd) with mock.patch('redis.StrictRedis', mock_strict_redis_client) as mocked_redis: redis_conn = mocked_redis() state_control = StateControlRedis(redis_conn) data = { 'workflow_id': workflow_id, 'app_id': app_id, 'job_id': job_id, 'type': 'deliver', 'task_id': '033f-284ab-28987e', 'port': 'port0', 'output': out_queue, 'workflow': '' } state_control.push_app_queue(app_id, json.dumps(data)) minion = SparkMinion(redis_conn=redis_conn, workflow_id=workflow_id, app_id=app_id, config=config) minion._emit_event = dummy_emit_event minion._state = { data['task_id']: { 'port0': { 'output': df0, 'sample': [] }, 'time': 35.92 } } minion._process_message() # Discard first status message state_control.pop_app_output_queue(app_id, False) msg = json.loads(state_control.pop_app_output_queue(app_id, False)) assert msg['status'] == 'SUCCESS', 'Invalid status' assert msg['code'] == minion.MNN002[0], 'Invalid code' # CSV data csv_records = '\n'.join( map(dataframe_util.convert_to_csv, get_records())) result = json.loads(state_control.pop_queue(out_queue, False)) assert result['sample'] == csv_records, 'Wrong CSV generated'
def test_minion_perform_deliver_missing_state_process_app_with_failure(): workflow_id = '6666' app_id = '6000' job_id = '1' out_queue = 'queue_2000' task_id = '033f-284ab-28987e' function_name = 'juicer.spark.transpiler.SparkTranspiler.transpile' with mock.patch('redis.StrictRedis', mock_strict_redis_client) as mocked_redis: with mock.patch(function_name) as mocked_transpile: with mock.patch('juicer.workflow.workflow.Workflow' '._build_initial_workflow_graph') as mocked_fn: mocked_fn.side_effect = lambda: "" # Setup for mocked_transpile # Invalid code mocked_transpile.side_effect = get_side_effect( get_records(), task_id, 4) redis_conn = mocked_redis() state_control = StateControlRedis(redis_conn) data = { 'workflow_id': workflow_id, 'app_id': app_id, 'job_id': job_id, 'type': 'deliver', 'task_id': task_id, 'port': 'port0', 'output': out_queue, 'workflow': { "tasks": [], "flows": [] } } state_control.push_app_queue(app_id, json.dumps(data)) minion = SparkMinion(redis_conn=redis_conn, workflow_id=workflow_id, app_id=app_id, config=config) minion._emit_event = dummy_emit_event minion._state = {} minion._process_message() # Discard first status message state_control.pop_app_output_queue(app_id, False) # First message is about missing state msg = json.loads( state_control.pop_app_output_queue(app_id, False)) assert msg['status'] == 'WARNING', 'Invalid status' assert msg['code'] == minion.MNN003[0], 'Invalid code' # Second message is about invalid Python code msg = json.loads( state_control.pop_app_output_queue(app_id, False)) assert msg['status'] == 'ERROR', 'Invalid status' assert msg.get('code') == minion.MNN006[0], 'Invalid code' # Third message is about unable to read data msg = json.loads( state_control.pop_app_output_queue(app_id, False)) assert msg['status'] == 'ERROR', 'Invalid status' assert msg.get('code') == minion.MNN005[0], 'Invalid code' assert state_control.get_app_output_queue_size( app_id) == 0, 'There are messages in app output queue!' result = json.loads(state_control.pop_queue(out_queue, False)) assert not result['sample'], 'Wrong CSV generated'