def test_compute_log_subscription(mock_watch_completed):
mock_watch_completed.return_value = False
schema = create_schema()
server = DagsterSubscriptionServer(schema=schema)
with instance_for_test() as instance:
run = execute_pipeline(example_pipeline, instance=instance)
assert run.success
assert run.run_id
with create_subscription_context(instance) as context:
start_subscription(
server,
context,
COMPUTE_LOG_SUBSCRIPTION,
{
"runId": run.run_id,
"stepKey": "example_solid",
"ioType": "STDERR",
},
)
gc.collect()
assert len(objgraph.by_type("SubscriptionObserver")) == 1
assert len(objgraph.by_type("ComputeLogSubscription")) == 1
end_subscription(server, context)
gc.collect()
assert len(objgraph.by_type("SubscriptionObserver")) == 0
assert len(objgraph.by_type("ComputeLogSubscription")) == 0
def create_app_from_workspace(workspace: Workspace,
instance: DagsterInstance,
path_prefix: str = ""):
check.inst_param(workspace, "workspace", Workspace)
check.inst_param(instance, "instance", DagsterInstance)
check.str_param(path_prefix, "path_prefix")
if path_prefix:
if not path_prefix.startswith("/"):
raise Exception(
f'The path prefix should begin with a leading "/": got {path_prefix}'
)
if path_prefix.endswith("/"):
raise Exception(
f'The path prefix should not include a trailing "/": got {path_prefix}'
)
warn_if_compute_logs_disabled()
print("Loading repository...") # pylint: disable=print-call
context = WorkspaceProcessContext(instance=instance,
workspace=workspace,
version=__version__)
log_workspace_stats(instance, context)
schema = create_schema()
return instantiate_app_with_views(context, schema, path_prefix)
def create_app_from_workspace_process_context(
workspace_process_context: WorkspaceProcessContext,
path_prefix: str = "",
) -> Flask:
check.inst_param(workspace_process_context, "workspace_process_context",
WorkspaceProcessContext)
check.str_param(path_prefix, "path_prefix")
instance = workspace_process_context.instance
if path_prefix:
if not path_prefix.startswith("/"):
raise Exception(
f'The path prefix should begin with a leading "/": got {path_prefix}'
)
if path_prefix.endswith("/"):
raise Exception(
f'The path prefix should not include a trailing "/": got {path_prefix}'
)
warn_if_compute_logs_disabled()
log_workspace_stats(instance, workspace_process_context)
schema = create_schema()
return instantiate_app_with_views(workspace_process_context,
schema,
path_prefix,
include_notebook_route=True)
def create_app(handle, instance):
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.inst_param(instance, 'instance', DagsterInstance)
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
execution_manager = MultiprocessingExecutionManager()
print('Loading repository...')
context = DagsterGraphQLContext(handle=handle,
instance=instance,
execution_manager=execution_manager,
version=__version__)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
app.add_url_rule(
# should match the `build_local_download_url`
'/download/<string:run_id>/<string:step_key>/<string:file_type>',
'download_view',
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view',
static_view)
app.add_url_rule('/vendor/<path:path>/<string:file>', 'vendor_view',
vendor_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
def test_event_log_subscription_chunked():
schema = create_schema()
server = DagsterSubscriptionServer(schema=schema)
with instance_for_test() as instance, environ(
{"DAGIT_EVENT_LOAD_CHUNK_SIZE": "2"}):
run = execute_pipeline(example_pipeline, instance=instance)
assert run.success
assert run.run_id
with create_subscription_context(instance) as context:
start_subscription(server, context, EVENT_LOG_SUBSCRIPTION,
{"runId": run.run_id})
gc.collect()
assert len(objgraph.by_type("SubscriptionObserver")) == 1
subs = objgraph.by_type("PipelineRunObservableSubscribe")
assert len(subs) == 1
subscription_obj = subs[0]
end_subscription(server, context)
subscription_obj.stopped.wait(30)
subs = None
subscription_obj = None
gc.collect()
assert len(objgraph.by_type("SubscriptionObserver")) == 0
assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 0
def test_execute_hammer_through_dagit():
handle = ExecutionTargetHandle.for_pipeline_python_file(
file_relative_path(
__file__, '../../../../examples/dagster_examples/toys/hammer.py'),
'hammer_pipeline',
)
instance = DagsterInstance.local_temp()
execution_manager = SubprocessExecutionManager(instance)
context = DagsterGraphQLInProcessRepositoryContext(
handle=handle, execution_manager=execution_manager, instance=instance)
executor = SyncExecutor()
variables = {
'executionParams': {
'environmentConfigData': {
'storage': {
'filesystem': {}
},
'execution': {
'dask': {}
}
},
'selector': {
'name': handle.build_pipeline_definition().name
},
'mode': 'default',
}
}
start_pipeline_result = graphql(
request_string=START_PIPELINE_EXECUTION_MUTATION,
schema=create_schema(),
context=context,
variables=variables,
executor=executor,
)
run_id = start_pipeline_result.data['startPipelineExecution']['run'][
'runId']
context.execution_manager.join()
subscription = execute_dagster_graphql(context,
SUBSCRIPTION_QUERY,
variables={'runId': run_id})
subscribe_results = []
subscription.subscribe(subscribe_results.append)
messages = [
x['__typename']
for x in subscribe_results[0].data['pipelineRunLogs']['messages']
]
assert 'PipelineStartEvent' in messages
assert 'PipelineSuccessEvent' in messages
def create_app(handle,
pipeline_run_storage,
use_synchronous_execution_manager=False):
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.inst_param(pipeline_run_storage, 'pipeline_run_storage',
PipelineRunStorage)
check.bool_param(use_synchronous_execution_manager,
'use_synchronous_execution_manager')
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
if use_synchronous_execution_manager:
execution_manager = SynchronousExecutionManager()
else:
execution_manager = MultiprocessingExecutionManager()
context = DagsterGraphQLContext(
handle=handle,
pipeline_runs=pipeline_run_storage,
execution_manager=execution_manager,
version=__version__,
)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic glabl request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view',
static_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
def test_execute_hammer_through_dagit():
with instance_for_test() as instance:
with get_workspace_process_context_from_kwargs(
instance,
version="",
read_only=False,
kwargs={
"python_file": file_relative_path(
__file__, "../../../dagster-test/dagster_test/toys/hammer.py"
),
"attribute": "hammer_pipeline",
},
) as workspace_process_context:
context = workspace_process_context.create_request_context()
selector = infer_pipeline_selector(context, "hammer_pipeline")
executor = SyncExecutor()
variables = {
"executionParams": {
"runConfigData": {
"execution": {"dask": {"config": {"cluster": {"local": {}}}}},
},
"selector": selector,
"mode": "default",
}
}
start_pipeline_result = graphql(
request_string=LAUNCH_PIPELINE_EXECUTION_MUTATION,
schema=create_schema(),
context=context,
variables=variables,
executor=executor,
)
if start_pipeline_result.errors:
raise Exception("{}".format(start_pipeline_result.errors))
run_id = start_pipeline_result.data["launchPipelineExecution"]["run"]["runId"]
context.instance.run_launcher.join(timeout=60)
subscription = execute_dagster_graphql(
context, SUBSCRIPTION_QUERY, variables={"runId": run_id}
)
subscribe_results = []
subscription.subscribe(subscribe_results.append)
messages = [
x["__typename"] for x in subscribe_results[0].data["pipelineRunLogs"]["messages"]
]
assert "RunStartEvent" in messages
assert "RunSuccessEvent" in messages
def create_app(
process_context: WorkspaceProcessContext,
debug: bool,
app_path_prefix: str,
):
graphql_schema = create_schema()
base_dir = path.dirname(__file__)
bound_index_endpoint = partial(index_endpoint, base_dir, app_path_prefix)
return Starlette(
debug=debug,
routes=[
Route("/dagit_info", dagit_info_endpoint),
Route(
"/graphql",
partial(graphql_http_endpoint, graphql_schema, process_context,
app_path_prefix),
name="graphql-http",
methods=["GET", "POST"],
),
WebSocketRoute(
"/graphql",
partial(graphql_ws_endpoint, graphql_schema, process_context),
name="graphql-ws",
),
# static resources addressed at /static/
Mount(
"/static",
StaticFiles(
directory=path.join(base_dir, "./webapp/build/static")),
name="static",
),
# static resources addressed at /vendor/
Mount(
"/vendor",
StaticFiles(
directory=path.join(base_dir, "./webapp/build/vendor")),
name="vendor",
),
# specific static resources addressed at /
*create_root_static_endpoints(base_dir),
# download file endpoints
Route(
"/download_debug/{run_id:str}",
partial(download_debug_file_endpoint, process_context),
),
Route("/{path:path}", bound_index_endpoint),
Route("/", bound_index_endpoint),
],
)
def test_execute_hammer_through_dagit():
recon_repo = ReconstructableRepository.for_file(
file_relative_path(__file__, '../../../../examples/dagster_examples/toys/hammer.py'),
'hammer_pipeline',
)
instance = DagsterInstance.local_temp()
context = DagsterGraphQLContext(
locations=[InProcessRepositoryLocation(recon_repo)], instance=instance,
)
selector = get_legacy_pipeline_selector(context, 'hammer_pipeline')
executor = SyncExecutor()
variables = {
'executionParams': {
'runConfigData': {
'storage': {'filesystem': {}},
'execution': {'dask': {'config': {'cluster': {'local': {}}}}},
},
'selector': selector,
'mode': 'default',
}
}
start_pipeline_result = graphql(
request_string=START_PIPELINE_EXECUTION_MUTATION,
schema=create_schema(),
context=context,
variables=variables,
executor=executor,
)
if start_pipeline_result.errors:
raise Exception('{}'.format(start_pipeline_result.errors))
run_id = start_pipeline_result.data['startPipelineExecution']['run']['runId']
context.drain_outstanding_executions()
subscription = execute_dagster_graphql(context, SUBSCRIPTION_QUERY, variables={'runId': run_id})
subscribe_results = []
subscription.subscribe(subscribe_results.append)
messages = [x['__typename'] for x in subscribe_results[0].data['pipelineRunLogs']['messages']]
assert 'PipelineStartEvent' in messages
assert 'PipelineSuccessEvent' in messages
def instantiate_app_with_views(context):
app = Flask(
'dagster-ui',
static_url_path='',
static_folder=os.path.join(os.path.dirname(__file__),
'./webapp/build'),
)
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
app.add_url_rule('/graphiql', 'graphiql',
lambda: redirect('/graphql', 301))
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
app.add_url_rule(
# should match the `build_local_download_url`
'/download/<string:run_id>/<string:step_key>/<string:file_type>',
'download_view',
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/dagit_info', 'sanity_view', info_view)
app.register_error_handler(404, index_view)
CORS(app)
return app
def create_app(repository_container, pipeline_runs, use_synchronous_execution_manager=False):
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
if use_synchronous_execution_manager:
execution_manager = SynchronousExecutionManager()
else:
execution_manager = MultiprocessingExecutionManager()
context = DagsterGraphQLContext(
repository_container=repository_container,
pipeline_runs=pipeline_runs,
execution_manager=execution_manager,
version=__version__,
)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context)
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
app.add_url_rule('/dagit/notebook', 'notebook', notebook_view)
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view', static_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
def execute_dagster_graphql(context, query, variables=None):
result = graphql(
create_schema(),
query,
context=context,
variables=variables,
# executor=GeventObservableExecutor(),
allow_subscriptions=True,
return_promise=False,
)
# has to check attr because in subscription case it returns AnonymousObservable
if hasattr(result, 'errors') and result.errors:
if result.errors[0].original_error:
raise result.errors[0].original_error
else:
raise result.errors[0]
return result
def execute_dagster_graphql(context, query, variables=None):
result = graphql(
create_schema(),
query,
context_value=context,
variable_values=variables,
allow_subscriptions=True,
return_promise=False,
)
# has to check attr because in subscription case it returns AnonymousObservable
if hasattr(result, "errors") and result.errors:
first_error = result.errors[0]
if hasattr(first_error, "original_error") and first_error.original_error:
raise result.errors[0].original_error
raise result.errors[0]
return result
def test_event_log_subscription():
schema = create_schema()
server = DagsterSubscriptionServer(schema=schema)
with instance_for_test() as instance:
run = execute_pipeline(example_pipeline, instance=instance)
assert run.success
assert run.run_id
with create_subscription_context(instance) as context:
start_subscription(server, context, EVENT_LOG_SUBSCRIPTION,
{"runId": run.run_id})
gc.collect()
assert len(objgraph.by_type("SubscriptionObserver")) == 1
assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 1
end_subscription(server, context)
gc.collect()
assert len(objgraph.by_type("SubscriptionObserver")) == 0
assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 0
def get_validator_client() -> Client:
return Client(schema=create_schema())
def build_graphql_schema(self) -> Schema:
return create_schema()
def test_execute_hammer_through_dagit():
recon_repo = ReconstructableRepository.for_file(
file_relative_path(
__file__, "../../../dagster-test/dagster_test/toys/hammer.py"),
"hammer_pipeline",
)
instance = DagsterInstance.local_temp()
context = DagsterGraphQLContext(
workspace=Workspace([
RepositoryLocationHandle.create_in_process_location(
recon_repo.pointer)
]),
instance=instance,
)
selector = infer_pipeline_selector(context, "hammer_pipeline")
executor = SyncExecutor()
variables = {
"executionParams": {
"runConfigData": {
"storage": {
"filesystem": {}
},
"execution": {
"dask": {
"config": {
"cluster": {
"local": {}
}
}
}
},
},
"selector": selector,
"mode": "default",
}
}
start_pipeline_result = graphql(
request_string=LAUNCH_PIPELINE_EXECUTION_MUTATION,
schema=create_schema(),
context=context,
variables=variables,
executor=executor,
)
if start_pipeline_result.errors:
raise Exception("{}".format(start_pipeline_result.errors))
run_id = start_pipeline_result.data["launchPipelineExecution"]["run"][
"runId"]
context.drain_outstanding_executions()
subscription = execute_dagster_graphql(context,
SUBSCRIPTION_QUERY,
variables={"runId": run_id})
subscribe_results = []
subscription.subscribe(subscribe_results.append)
messages = [
x["__typename"]
for x in subscribe_results[0].data["pipelineRunLogs"]["messages"]
]
assert "PipelineStartEvent" in messages
assert "PipelineSuccessEvent" in messages
def instantiate_app_with_views(context, app_path_prefix):
app = Flask(
"dagster-ui",
static_url_path=app_path_prefix,
static_folder=os.path.join(os.path.dirname(__file__), "./webapp/build"),
)
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
# Websocket routes
sockets = Sockets(app)
sockets.add_url_rule(
"{}/graphql".format(app_path_prefix),
"graphql",
dagster_graphql_subscription_view(subscription_server, context),
)
# HTTP routes
bp = Blueprint("routes", __name__, url_prefix=app_path_prefix)
bp.add_url_rule(
"/graphiql", "graphiql", lambda: redirect("{}/graphql".format(app_path_prefix), 301)
)
bp.add_url_rule(
"/graphql",
"graphql",
DagsterGraphQLView.as_view(
"graphql",
schema=schema,
graphiql=True,
graphiql_template=PLAYGROUND_TEMPLATE.replace("APP_PATH_PREFIX", app_path_prefix),
executor=Executor(),
context=context,
),
)
bp.add_url_rule(
# should match the `build_local_download_url`
"/download/<string:run_id>/<string:step_key>/<string:file_type>",
"download_view",
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
bp.add_url_rule("/dagit/notebook", "notebook", lambda: notebook_view(request.args))
bp.add_url_rule("/dagit_info", "sanity_view", info_view)
index_path = os.path.join(os.path.dirname(__file__), "./webapp/build/index.html")
def index_view(_path):
try:
with open(index_path) as f:
return (
f.read()
.replace('href="/', 'href="{}/'.format(app_path_prefix))
.replace('src="/', 'src="{}/'.format(app_path_prefix))
.replace(
'<meta name="dagit-path-prefix"',
'<meta name="dagit-path-prefix" content="{}"'.format(app_path_prefix),
)
)
except seven.FileNotFoundError:
raise Exception(
"""Can't find webapp files. Probably webapp isn't built. If you are using
dagit, then probably it's a corrupted installation or a bug. However, if you are
developing dagit locally, your problem can be fixed as follows:
cd ./python_modules/
make rebuild_dagit"""
)
app.app_protocol = lambda environ_path_info: "graphql-ws"
app.register_blueprint(bp)
app.register_error_handler(404, index_view)
# if the user asked for a path prefix, handle the naked domain just in case they are not
# filtering inbound traffic elsewhere and redirect to the path prefix.
if app_path_prefix:
app.add_url_rule("/", "force-path-prefix", lambda: redirect(app_path_prefix, 301))
CORS(app)
return app
def create_app(handle, instance, reloader=None):
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.inst_param(instance, 'instance', DagsterInstance)
check.opt_inst_param(reloader, 'reloader', Reloader)
app = Flask('dagster-ui')
sockets = Sockets(app)
app.app_protocol = lambda environ_path_info: 'graphql-ws'
schema = create_schema()
subscription_server = DagsterSubscriptionServer(schema=schema)
execution_manager_settings = instance.dagit_settings.get(
'execution_manager')
if execution_manager_settings and execution_manager_settings.get(
'max_concurrent_runs'):
execution_manager = QueueingSubprocessExecutionManager(
instance, execution_manager_settings.get('max_concurrent_runs'))
else:
execution_manager = SubprocessExecutionManager(instance)
warn_if_compute_logs_disabled()
print('Loading repository...')
context = DagsterGraphQLContext(
handle=handle,
instance=instance,
execution_manager=execution_manager,
reloader=reloader,
version=__version__,
)
# Automatically initialize scheduler everytime Dagit loads
scheduler_handle = context.scheduler_handle
scheduler = instance.scheduler
if scheduler_handle:
if scheduler:
handle = context.get_handle()
python_path = sys.executable
repository_path = handle.data.repository_yaml
repository = context.get_repository()
scheduler_handle.up(python_path,
repository_path,
repository=repository,
instance=instance)
else:
warnings.warn(MISSING_SCHEDULER_WARNING)
app.add_url_rule(
'/graphql',
'graphql',
DagsterGraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
# XXX(freiksenet): Pass proper ws url
graphiql_template=PLAYGROUND_TEMPLATE,
executor=Executor(),
context=context,
),
)
sockets.add_url_rule(
'/graphql', 'graphql',
dagster_graphql_subscription_view(subscription_server, context))
app.add_url_rule(
# should match the `build_local_download_url`
'/download/<string:run_id>/<string:step_key>/<string:file_type>',
'download_view',
download_view(context),
)
# these routes are specifically for the Dagit UI and are not part of the graphql
# API that we want other people to consume, so they're separate for now.
# Also grabbing the magic global request args dict so that notebook_view is testable
app.add_url_rule('/dagit/notebook', 'notebook',
lambda: notebook_view(request.args))
app.add_url_rule('/static/<path:path>/<string:file>', 'static_view',
static_view)
app.add_url_rule('/vendor/<path:path>/<string:file>', 'vendor_view',
vendor_view)
app.add_url_rule('/<string:worker_name>.worker.js', 'worker_view',
worker_view)
app.add_url_rule('/dagit_info', 'sanity_view', info_view)
app.add_url_rule('/<path:_path>', 'index_catchall', index_view)
app.add_url_rule('/', 'index', index_view, defaults={'_path': ''})
CORS(app)
return app
