def test_dict_roundtrip():
handle = NodeHandle("baz", NodeHandle("bar", NodeHandle("foo", None)))
assert NodeHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
handle = NodeHandle("foo", None)
assert NodeHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
def test_dict_roundtrip():
handle = SolidHandle('baz', SolidHandle('bar', SolidHandle('foo', None)))
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
handle = SolidHandle('foo', None)
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
def get_config_from_args(kwargs):
config_files = kwargs.get("config")
config_json = kwargs.get("config_json")
if not config_files and not config_json:
return {}
if config_files and config_json:
raise click.UsageError("Cannot specify both -c / --config and --config-json")
if config_files:
config_file_list = list(
check.opt_tuple_param(kwargs.get("config"), "config", default=(), of_type=str)
)
return get_run_config_from_file_list(config_file_list)
if config_json:
try:
return json.loads(config_json)
except JSONDecodeError:
raise click.UsageError(
"Invalid JSON-string given for `--config-json`: {}\n\n{}".format(
config_json,
serializable_error_info_from_exc_info(sys.exc_info()).to_string(),
)
)
def get_kernelspec(kernel: str = None):
kernelspecs = loads(
subprocess.check_output(["jupyter", "kernelspec", "list", "--json"]))
check.invariant(
len(kernelspecs["kernelspecs"]) > 0,
"No available Jupyter kernelspecs!")
if kernel is None:
preferred_kernels = (list(
filter(
lambda kernel_name: kernel_name in kernelspecs["kernelspecs"],
["dagster", "python3", "python"],
)) + list(kernelspecs["kernelspecs"].keys()))
kernel = preferred_kernels[0]
print( # pylint: disable=print-call
"No kernel specified, defaulting to '{kernel}'".format(
kernel=kernel))
check.invariant(
kernel in kernelspecs["kernelspecs"],
"Could not find kernel '{kernel}': available kernels are [{kernels}]".
format(
kernel=kernel,
kernels=", ".join(
["'{k}'".format(k=k) for k in kernelspecs["kernelspecs"]]),
),
)
return {
"name": kernel,
"language": kernelspecs["kernelspecs"][kernel]["spec"]["language"],
"display_name":
kernelspecs["kernelspecs"][kernel]["spec"]["display_name"],
}
def get_config_from_args(kwargs: Dict[str, str]) -> Dict[str, object]:
config = kwargs.get("config") # files
config_json = kwargs.get("config_json")
if not config and not config_json:
return {}
elif config and config_json:
raise click.UsageError(
"Cannot specify both -c / --config and --config-json")
elif config:
config_file_list = list(
check.opt_tuple_param(config,
"config",
default=tuple(),
of_type=str))
return get_run_config_from_file_list(config_file_list)
elif config_json:
config_json = cast(str, config_json)
try:
return json.loads(config_json)
except JSONDecodeError:
raise click.UsageError(
"Invalid JSON-string given for `--config-json`: {}\n\n{}".
format(
config_json,
serializable_error_info_from_exc_info(
sys.exc_info()).to_string(),
))
else:
check.failed("Unhandled case getting config from kwargs")
def get_kernelspec(kernel):
kernelspecs = loads(
subprocess.check_output(['jupyter', 'kernelspec', 'list', '--json']))
check.invariant(
len(kernelspecs['kernelspecs']) > 0,
'No available Jupyter kernelspecs!')
if kernel is None:
preferred_kernels = list(
filter(
lambda kernel_name: kernel_name in kernelspecs['kernelspecs'],
['dagster', 'python3', 'python'],
)) + list(kernelspecs['kernelspecs'].keys())
kernel = preferred_kernels[0]
print( # pylint: disable=print-call
'No kernel specified, defaulting to \'{kernel}\''.format(
kernel=kernel))
check.invariant(
kernel in kernelspecs['kernelspecs'],
'Could not find kernel \'{kernel}\': available kernels are [{kernels}]'
.format(
kernel=kernel,
kernels=', '.join(
['\'{k}\''.format(k=k) for k in kernelspecs['kernelspecs']]),
),
)
return {
'name': kernel,
'language': kernelspecs['kernelspecs'][kernel]['spec']['language'],
'display_name':
kernelspecs['kernelspecs'][kernel]['spec']['display_name'],
}
def get_tags_from_args(kwargs):
if kwargs.get("tags") is None:
return {}
try:
return json.loads(kwargs.get("tags"))
except JSONDecodeError:
raise click.UsageError(
"Invalid JSON-string given for `--tags`: {}\n\n{}".format(
kwargs.get("tags"),
serializable_error_info_from_exc_info(sys.exc_info()).to_string(),
)
)
def get_kernelspec(kernel):
kernelspecs = loads(
subprocess.check_output(['jupyter', 'kernelspec', 'list', '--json']))
check.invariant(
len(kernelspecs['kernelspecs']) > 0,
'No available Jupyter kernelspecs!')
if kernel is None:
if 'dagster' in kernelspecs['kernelspecs']:
kernel = 'dagster'
elif 'python3' in kernelspecs['kernelspecs']:
kernel = 'python3'
elif 'python' in kernelspecs['kernelspecs']:
kernel = 'python'
else:
kernel = list(kernelspecs['kernelspecs'].keys())[0]
print('No kernel specified, defaulting to \'{kernel}\''.format(
kernel=kernel))
check.invariant(
kernel in kernelspecs['kernelspecs'],
'Could not find kernel \'{kernel}\': available kernels are [{kernels}]'
.format(
kernel=kernel,
kernels=', '.join(
['\'{k}\''.format(k=k) for k in kernelspecs['kernelspecs']]),
),
)
return {
'name': kernel,
'language': kernelspecs['kernelspecs'][kernel]['spec']['language'],
'display_name':
kernelspecs['kernelspecs'][kernel]['spec']['display_name'],
}
async def graphql_http_endpoint(self, request: Request):
"""
fork of starlette GraphQLApp to allow for
* our context type (crucial)
* our GraphiQL playground (could change)
"""
if request.method == "GET":
# render graphiql
if "text/html" in request.headers.get("Accept", ""):
text = TEMPLATE.replace("{{ app_path_prefix }}",
self._app_path_prefix)
return HTMLResponse(text)
data: Union[Dict[str, str], QueryParams] = request.query_params
elif request.method == "POST":
content_type = request.headers.get("Content-Type", "")
if "application/json" in content_type:
data = await request.json()
elif "application/graphql" in content_type:
body = await request.body()
text = body.decode()
data = {"query": text}
else:
data = request.query_params
else:
return PlainTextResponse(
"Method Not Allowed",
status_code=status.HTTP_405_METHOD_NOT_ALLOWED)
query = data.get("query")
variables = data.get("variables")
operation_name = data.get("operationName")
if query is None:
return PlainTextResponse(
"No GraphQL query found in the request",
status_code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(variables, str):
try:
variables = json.loads(variables)
except json.JSONDecodeError:
return PlainTextResponse(
f"Malformed GraphQL variables. Passed as string but not valid JSON:\n{variables}",
status_code=status.HTTP_400_BAD_REQUEST,
)
result = await self.execute_graphql_request(request, query, variables,
operation_name)
response_data = {"data": result.data}
status_code = status.HTTP_200_OK
if result.errors:
response_data["errors"] = self.handle_graphql_errors(result.errors)
status_code = status.HTTP_400_BAD_REQUEST
return JSONResponse(response_data, status_code=status_code)
'''This file is used by the EphemeralGrpcRunLauncher to execute runs in a subprocess.'''
import sys
from dagster import check
from dagster.api.execute_run import cli_api_execute_run_grpc
from dagster.core.instance import DagsterInstance, InstanceRef
from dagster.core.origin import PipelinePythonOrigin
from dagster.serdes import deserialize_json_to_dagster_namedtuple
from dagster.serdes.ipc import setup_interrupt_support
from dagster.seven import json
if __name__ == '__main__':
setup_interrupt_support()
kwargs = json.loads(sys.argv[1])
instance_ref = check.inst(
deserialize_json_to_dagster_namedtuple(kwargs['instance_ref']),
InstanceRef)
pipeline_origin = check.inst(
deserialize_json_to_dagster_namedtuple(kwargs['pipeline_origin']),
PipelinePythonOrigin)
pipeline_run_id = kwargs['pipeline_run_id']
instance = DagsterInstance.from_ref(instance_ref)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
events = [
evt
for evt in cli_api_execute_run_grpc(instance_ref=instance_ref,
pipeline_origin=pipeline_origin,
pipeline_run=pipeline_run)
]
print(len(events)) # pylint: disable=print-call
