def test_valid_api_lookups(self, mock_hook, mock_requests, valid_job_params): """test api lookups are correct""" queue_mock = Mock( **{ 'json.return_value': { 'results': [ {'id': 2, 'public_name': 'queue2'}, {'id': 3, 'public_name': 'queue3'}, {'id': 1, 'public_name': 'queue'}, ] } } ) billing_mock = Mock(**{'json.return_value': {'results': [{'id': 1}]}}) app_mock = Mock( **{ 'json.return_value': { 'results': [ {'name': 'pipeline2', 'id': 2, 'is_service': True}, {'name': 'pipeline3', 'id': 3, 'is_service': False}, {'name': 'pipeline', 'id': 1, 'is_service': False}, ] } } ) mock_requests.get.side_effect = [queue_mock, billing_mock, app_mock] plexus_job_operator = PlexusJobOperator(**valid_job_params) assert plexus_job_operator._api_lookup('queue', mock_hook) == 1 assert plexus_job_operator._api_lookup('billing_account_id', mock_hook) == 1 assert plexus_job_operator.is_service is None assert plexus_job_operator._api_lookup('app', mock_hook) == 1 assert plexus_job_operator.is_service is not None
def test_invalid_api_lookups(self, mock_hook, mock_requests, invalid_lookups): """test api lookups that are not found""" queue_mock = Mock(**{ 'json.return_value': { 'results': [{ 'id': 2, 'public_name': 'queue2' }] } }) billing_mock = Mock( **{'json.return_value': { 'results': [{ 'id': None }] }}) app_mock = Mock( **{ 'json.return_value': { 'results': [{ 'name': 'pipeline2', 'id': 2, 'is_service': True }] } }) mock_requests.get.side_effect = [queue_mock, billing_mock, app_mock] plexus_job_operator = PlexusJobOperator(**invalid_lookups) with pytest.raises(AirflowException): plexus_job_operator._api_lookup('queue', mock_hook) plexus_job_operator._api_lookup('app', mock_hook) plexus_job_operator._api_lookup('billing_account_id', mock_hook)
def test_valid_job_params(self, valid_job_params): """test params are initialized correctly""" expected_job_params = { 'name': 'test', 'app': 'pipeline', 'queue': 'queue', 'num_nodes': 1, 'num_cores': 1, 'billing_account_id': None, } plexus_job_operator = PlexusJobOperator(**valid_job_params) assert plexus_job_operator.job_params == expected_job_params assert plexus_job_operator.is_service is None
def test_invalid_job_params(self, mock_hook, invalid_job_params): """test required job params are supplied""" plexus_job_operator = PlexusJobOperator(**invalid_job_params) with pytest.raises(AirflowException): plexus_job_operator.construct_job_params(mock_hook)
def test_create_job(self, mock_hook, mock_construct_params, mock_request, mock_time, valid_job_params): """test submit/get job""" mock_construct_params.return_value = { 'name': 'test', 'app': 1, 'queue': 2, 'num_nodes': 1, 'num_cores': 1, 'billing_account_id': 3, } # assert Error for is_service is None plexus_job_operator = PlexusJobOperator(**valid_job_params) with pytest.raises(AirflowException): plexus_job_operator.execute(None) # assert create_job timeout plexus_job_operator.is_service = False mock_request.post.side_effect = Timeout with pytest.raises(Timeout): plexus_job_operator.execute(None) # assert create_job error mock_request.post.side_effect = None mock_request.post.return_value = Mock(ok=False, status_code=400, reason='Bad', text='Bad') with pytest.raises(AirflowException): plexus_job_operator.execute(None) # assert get_job error mock_request.post.return_value = Mock(**{ 'ok': True, 'json.return_value': { 'id': 1, 'last_state': 'Pending' } }) mock_request.get.return_value = Mock(ok=False) with pytest.raises(AirflowException): plexus_job_operator.execute(None) # assert get_job state error mock_request.get.return_value = Mock(**{ 'ok': True, 'json.return_value': { 'last_state': 'Failed' } }) with pytest.raises(AirflowException): plexus_job_operator.execute(None) # assert get_job state error mock_request.get.return_value = Mock(**{ 'ok': True, 'json.return_value': { 'last_state': 'Cancelled' } }) with pytest.raises(AirflowException): plexus_job_operator.execute(None) # assert get_job timeout mock_request.get.side_effect = Timeout with pytest.raises(Timeout): plexus_job_operator.execute(None)
HOME = '/home/acc' T3_PRERUN_SCRIPT = 'cp {home}/imdb/run_scripts/mlflow.sh {home}/ && chmod +x mlflow.sh'.format( home=HOME) args = {'owner': 'core scientific', 'retries': 1} dag = DAG( 'test', default_args=args, description='testing plexus operator', start_date=days_ago(1), schedule_interval='@once', catchup=False, ) t1 = PlexusJobOperator( task_id='test', job_params={ 'name': 'test', 'app': 'MLFlow Pipeline 01', 'queue': 'DGX-2 (gpu:Tesla V100-SXM3-32GB)', 'num_nodes': 1, 'num_cores': 1, 'prerun_script': T3_PRERUN_SCRIPT, }, dag=dag, ) t1
'depends_on_past': False, 'retries': 1, } dag = DAG('imdb', default_args=args, description='loads imdb dataset and trains a NN model', schedule_interval='@once', catchup=False) t1 = PlexusJobOperator( task_id='process_data', job_params={ 'name': 'process_data', 'app': 'MLFlow Pipeline 01', 'queue': 'DGX-2 (gpu:Tesla V100-SXM3-32GB)', 'num_nodes': 1, 'num_cores': 1, 'run_script': T1_RUN_SCRIPT, }, dag=dag, ) t2 = PlexusJobOperator( task_id='train_model', job_params={ 'name': 'train_model', 'app': 'TensorFlow Pipeline 01', 'queue': 'DGX-2 (gpu:Tesla V100-SXM3-32GB)', 'num_nodes': 1, 'num_cores': 1, 'run_script': T2_RUN_SCRIPT,