def define_context(raise_on_error=True): return DagsterGraphQLContext( RepositoryContainer(repository=define_repository()), PipelineRunStorage(), execution_manager=SynchronousExecutionManager(), raise_on_error=raise_on_error, )
def define_context(raise_on_error=True, log_dir=None): return DagsterGraphQLContext( handle=ExecutionTargetHandle.for_repo_fn(define_repository), pipeline_runs=PipelineRunStorage(log_dir), execution_manager=SynchronousExecutionManager(), raise_on_error=raise_on_error, )
def host_dagit_ui(log, log_dir, repository_container, sync, host, port): if log: def create_pipeline_run(*args, **kwargs): return LogFilePipelineRun(log_dir, *args, **kwargs) else: create_pipeline_run = InMemoryPipelineRun pipeline_run_storage = PipelineRunStorage( create_pipeline_run=create_pipeline_run) app = create_app(repository_container, pipeline_run_storage, use_synchronous_execution_manager=sync) server = pywsgi.WSGIServer((host, port), app, handler_class=WebSocketHandler) print('Serving on http://{host}:{port}'.format(host=host, port=port)) try: server.serve_forever() except OSError as os_error: if 'Address already in use' in str(os_error): six.raise_from( Exception(( 'Another process on your machine is already listening on port {port}. ' 'It is possible that you have another instance of dagit ' 'running somewhere using the same port. Or it could be another ' 'random process. Either kill that process or us the -p option to ' 'select another port.').format(port=port)), os_error, ) else: raise os_error
def host_dagit_ui(log, log_dir, handle, use_sync, host, port): check.inst_param(handle, 'handle', ExecutionTargetHandle) pipeline_run_storage = PipelineRunStorage(log_dir if log else None) app = create_app(handle, pipeline_run_storage, use_synchronous_execution_manager=use_sync) server = pywsgi.WSGIServer((host, port), app, handler_class=WebSocketHandler) print('Serving on http://{host}:{port}'.format(host=host, port=port)) try: server.serve_forever() except OSError as os_error: if 'Address already in use' in str(os_error): six.raise_from( Exception(( 'Another process on your machine is already listening on port {port}. ' 'It is possible that you have another instance of dagit ' 'running somewhere using the same port. Or it could be another ' 'random process. Either kill that process or us the -p option to ' 'select another port.').format(port=port)), os_error, ) else: raise os_error
def test_smoke_app(): repository_container = RepositoryContainer(repository=define_repo()) pipeline_run_storage = PipelineRunStorage() flask_app = app.create_app(repository_container, pipeline_run_storage) client = flask_app.test_client() result = client.post('/graphql', data={'query': 'query { pipelines { nodes { name }}}'}) data = json.loads(result.data.decode('utf-8')) assert len(data['data']['pipelines']['nodes']) == 1 assert {node_data['name'] for node_data in data['data']['pipelines']['nodes']} == set( ['repo_demo_pipeline'] ) result = client.get('/graphql') assert result.status_code == 400 data = json.loads(result.data.decode('utf-8')) assert len(data['errors']) == 1 assert data['errors'][0]['message'] == 'Must provide query string.' result = client.get('/dagit/notebook?path=foo.bar') assert result.status_code == 400 assert result.data.decode('utf-8') == 'Invalid Path' result = client.post('/graphql', data={'query': 'query { version { slkjd } }'}) data = json.loads(result.data.decode('utf-8')) assert 'errors' in data assert len(data['errors']) == 1 assert 'must not have a sub selection' in data['errors'][0]['message'] result = client.get('static/foo/bar') assert result.status_code == 404
def define_examples_context(raise_on_error=True): return DagsterGraphQLContext( handle=ExecutionTargetHandle.for_repo_module('dagster_examples', 'define_demo_repo'), pipeline_runs=PipelineRunStorage(), execution_manager=SynchronousExecutionManager(), raise_on_error=raise_on_error, )
def test_pipelines_python_error(): ctx = DagsterGraphQLContext( handle=ExecutionTargetHandle.for_repo_fn(define_error_pipeline_repo), pipeline_runs=PipelineRunStorage(), execution_manager=SynchronousExecutionManager(), ) result = execute_dagster_graphql(ctx, PIPELINES) assert result.data['pipelinesOrError']['__typename'] == "PythonError"
def test_pipelines_python_error(): ctx = DagsterGraphQLContext( RepositoryContainer(repository=define_error_pipeline_repo()), PipelineRunStorage(), execution_manager=SynchronousExecutionManager(), ) result = execute_dagster_graphql(ctx, PIPELINES) assert result.data['pipelinesOrError']['__typename'] == "PythonError"
def test_index_view(): with create_app( ExecutionTargetHandle.for_repo_yaml(script_relative_path('./repository.yaml')), PipelineRunStorage(), ).test_client() as client: res = client.get('/') assert res.status_code == 200, res.data assert b'You need to enable JavaScript to run this app' in res.data
def test_notebook_view(): notebook_path = script_relative_path('render_uuid_notebook.ipynb') with create_app( ExecutionTargetHandle.for_repo_yaml(script_relative_path('./repository.yaml')), PipelineRunStorage(), ).test_client() as client: res = client.get('/dagit/notebook?path={}'.format(notebook_path)) assert res.status_code == 200 # This magic guid is hardcoded in the notebook assert b'6cac0c38-2c97-49ca-887c-4ac43f141213' in res.data
def test_pipelines_or_error_invalid(): context = DagsterGraphQLContext( handle=ExecutionTargetHandle.for_repo_fn(define_test_repository), pipeline_runs=PipelineRunStorage(), execution_manager=SynchronousExecutionManager(), ) result = execute_dagster_graphql( context, '{ pipelinesOrError { ... on InvalidDefinitionError { message } } }') msg = result.data['pipelinesOrError']['message'] assert 'circular reference detected in solid "csolid"' in msg
def test_pipelines_or_error_invalid(): repository = RepositoryDefinition( name='test', pipeline_dict={'pipeline': define_circular_dependency_pipeline}) context = DagsterGraphQLContext( RepositoryContainer(repository=repository), PipelineRunStorage(), execution_manager=SynchronousExecutionManager(), ) result = execute_dagster_graphql( context, '{ pipelinesOrError { ... on InvalidDefinitionError { message } } }') msg = result.data['pipelinesOrError']['message'] assert "Circular reference detected in solid csolid" in msg
def load_dagit_for_repo_cli_args(n_pipelines=1, **kwargs): handle = handle_for_repo_cli_args(kwargs) pipeline_run_storage = PipelineRunStorage() app = create_app(handle, pipeline_run_storage) client = app.test_client() res = client.get('/graphql?query={query_string}'.format( query_string=PIPELINES_OR_ERROR_QUERY)) json_res = json.loads(res.data.decode('utf-8')) assert 'data' in json_res assert 'pipelinesOrError' in json_res['data'] assert 'nodes' in json_res['data']['pipelinesOrError'] assert len(json_res['data']['pipelinesOrError']['nodes']) == n_pipelines return res
def test_smoke_app(): pipeline_run_storage = PipelineRunStorage() flask_app = app.create_app( ExecutionTargetHandle.for_repo_module( module_name='dagster_examples.intro_tutorial.repos', fn_name='define_repo'), pipeline_run_storage, ) client = flask_app.test_client() result = client.post( '/graphql', data={'query': 'query { pipelines { nodes { name }}}'}) data = json.loads(result.data.decode('utf-8')) assert len(data['data']['pipelines']['nodes']) == 1 assert { node_data['name'] for node_data in data['data']['pipelines']['nodes'] } == set(['repo_demo_pipeline']) result = client.get('/graphql') assert result.status_code == 400 data = json.loads(result.data.decode('utf-8')) assert len(data['errors']) == 1 assert data['errors'][0]['message'] == 'Must provide query string.' result = client.get('/dagit/notebook?path=foo.bar') assert result.status_code == 400 assert result.data.decode('utf-8') == 'Invalid Path' result = client.post('/graphql', data={'query': 'query { version { slkjd } }'}) data = json.loads(result.data.decode('utf-8')) assert 'errors' in data assert len(data['errors']) == 1 assert 'must not have a sub selection' in data['errors'][0]['message'] result = client.get('static/foo/bar') assert result.status_code == 404 result = client.get('vendor/foo/bar') assert result.status_code == 404
def test_create_app(): handle = ExecutionTargetHandle.for_repo_yaml(script_relative_path('./repository.yaml')) pipeline_run_storage = PipelineRunStorage(create_pipeline_run=InMemoryPipelineRun) assert create_app(handle, pipeline_run_storage, use_synchronous_execution_manager=True) assert create_app(handle, pipeline_run_storage, use_synchronous_execution_manager=False)
def test_all_step_events(): # pylint: disable=too-many-locals handle = ExecutionTargetHandle.for_pipeline_fn(define_test_events_pipeline) pipeline = handle.build_pipeline_definition() mode = pipeline.get_default_mode_name() execution_plan = create_execution_plan(pipeline, {}, mode=mode) step_levels = execution_plan.topological_step_levels() run_config = RunConfig( executor_config=InProcessExecutorConfig(raise_on_error=False), storage_mode=RunStorageMode.FILESYSTEM, ) unhandled_events = STEP_EVENTS.copy() # Exclude types that are not step events ignored_events = { 'LogMessageEvent', 'PipelineStartEvent', 'PipelineSuccessEvent', 'PipelineInitFailureEvent', 'PipelineFailureEvent', } step_event_fragment = get_step_event_fragment() log_message_event_fragment = get_log_message_event_fragment() query = '\n'.join( ( PIPELINE_EXECUTION_QUERY_TEMPLATE.format( step_event_fragment=step_event_fragment.include_key, log_message_event_fragment=log_message_event_fragment.include_key, ), step_event_fragment.fragment, log_message_event_fragment.fragment, ) ) event_counts = defaultdict(int) for step_level in step_levels: for step in step_level: variables = { 'executionParams': { 'selector': {'name': pipeline.name}, 'environmentConfigData': {'storage': {'filesystem': {}}}, 'mode': mode, 'executionMetadata': {'runId': run_config.run_id}, 'stepKeys': [step.key], } } pipeline_run_storage = PipelineRunStorage() res = execute_query(handle, query, variables, pipeline_run_storage=pipeline_run_storage) # go through the same dict, decrement all the event records we've seen from the GraphQL # response if not res.get('errors'): run_logs = res['data']['startPipelineExecution']['run']['logs']['nodes'] events = [ dagster_event_from_dict(e, pipeline.name) for e in run_logs if e['__typename'] not in ignored_events ] for event in events: key = event.step_key + '.' + event.event_type_value event_counts[key] -= 1 unhandled_events -= {DagsterEventType(e.event_type_value) for e in events} # build up a dict, incrementing all the event records we've produced in the run storage logs = pipeline_run_storage.get_run_by_id(run_config.run_id).all_logs() for log in logs: if not log.dagster_event or ( DagsterEventType(log.dagster_event.event_type_value) not in STEP_EVENTS ): continue key = log.dagster_event.step_key + '.' + log.dagster_event.event_type_value event_counts[key] += 1 # Ensure we've processed all the events that were generated in the run storage assert sum(event_counts.values()) == 0 # Ensure we've handled the universe of event types assert not unhandled_events