def test_dict_roundtrip(): handle = NodeHandle("baz", NodeHandle("bar", NodeHandle("foo", None))) assert NodeHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle handle = NodeHandle("foo", None) assert NodeHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle
def test_dict_roundtrip(): handle = SolidHandle('baz', SolidHandle('bar', SolidHandle('foo', None))) assert SolidHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle handle = SolidHandle('foo', None) assert SolidHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle
def get_config_from_args(kwargs): config_files = kwargs.get("config") config_json = kwargs.get("config_json") if not config_files and not config_json: return {} if config_files and config_json: raise click.UsageError("Cannot specify both -c / --config and --config-json") if config_files: config_file_list = list( check.opt_tuple_param(kwargs.get("config"), "config", default=(), of_type=str) ) return get_run_config_from_file_list(config_file_list) if config_json: try: return json.loads(config_json) except JSONDecodeError: raise click.UsageError( "Invalid JSON-string given for `--config-json`: {}\n\n{}".format( config_json, serializable_error_info_from_exc_info(sys.exc_info()).to_string(), ) )
def get_kernelspec(kernel: str = None): kernelspecs = loads( subprocess.check_output(["jupyter", "kernelspec", "list", "--json"])) check.invariant( len(kernelspecs["kernelspecs"]) > 0, "No available Jupyter kernelspecs!") if kernel is None: preferred_kernels = (list( filter( lambda kernel_name: kernel_name in kernelspecs["kernelspecs"], ["dagster", "python3", "python"], )) + list(kernelspecs["kernelspecs"].keys())) kernel = preferred_kernels[0] print( # pylint: disable=print-call "No kernel specified, defaulting to '{kernel}'".format( kernel=kernel)) check.invariant( kernel in kernelspecs["kernelspecs"], "Could not find kernel '{kernel}': available kernels are [{kernels}]". format( kernel=kernel, kernels=", ".join( ["'{k}'".format(k=k) for k in kernelspecs["kernelspecs"]]), ), ) return { "name": kernel, "language": kernelspecs["kernelspecs"][kernel]["spec"]["language"], "display_name": kernelspecs["kernelspecs"][kernel]["spec"]["display_name"], }
def get_config_from_args(kwargs: Dict[str, str]) -> Dict[str, object]: config = kwargs.get("config") # files config_json = kwargs.get("config_json") if not config and not config_json: return {} elif config and config_json: raise click.UsageError( "Cannot specify both -c / --config and --config-json") elif config: config_file_list = list( check.opt_tuple_param(config, "config", default=tuple(), of_type=str)) return get_run_config_from_file_list(config_file_list) elif config_json: config_json = cast(str, config_json) try: return json.loads(config_json) except JSONDecodeError: raise click.UsageError( "Invalid JSON-string given for `--config-json`: {}\n\n{}". format( config_json, serializable_error_info_from_exc_info( sys.exc_info()).to_string(), )) else: check.failed("Unhandled case getting config from kwargs")
def get_kernelspec(kernel): kernelspecs = loads( subprocess.check_output(['jupyter', 'kernelspec', 'list', '--json'])) check.invariant( len(kernelspecs['kernelspecs']) > 0, 'No available Jupyter kernelspecs!') if kernel is None: preferred_kernels = list( filter( lambda kernel_name: kernel_name in kernelspecs['kernelspecs'], ['dagster', 'python3', 'python'], )) + list(kernelspecs['kernelspecs'].keys()) kernel = preferred_kernels[0] print( # pylint: disable=print-call 'No kernel specified, defaulting to \'{kernel}\''.format( kernel=kernel)) check.invariant( kernel in kernelspecs['kernelspecs'], 'Could not find kernel \'{kernel}\': available kernels are [{kernels}]' .format( kernel=kernel, kernels=', '.join( ['\'{k}\''.format(k=k) for k in kernelspecs['kernelspecs']]), ), ) return { 'name': kernel, 'language': kernelspecs['kernelspecs'][kernel]['spec']['language'], 'display_name': kernelspecs['kernelspecs'][kernel]['spec']['display_name'], }
def get_tags_from_args(kwargs): if kwargs.get("tags") is None: return {} try: return json.loads(kwargs.get("tags")) except JSONDecodeError: raise click.UsageError( "Invalid JSON-string given for `--tags`: {}\n\n{}".format( kwargs.get("tags"), serializable_error_info_from_exc_info(sys.exc_info()).to_string(), ) )
def get_kernelspec(kernel): kernelspecs = loads( subprocess.check_output(['jupyter', 'kernelspec', 'list', '--json'])) check.invariant( len(kernelspecs['kernelspecs']) > 0, 'No available Jupyter kernelspecs!') if kernel is None: if 'dagster' in kernelspecs['kernelspecs']: kernel = 'dagster' elif 'python3' in kernelspecs['kernelspecs']: kernel = 'python3' elif 'python' in kernelspecs['kernelspecs']: kernel = 'python' else: kernel = list(kernelspecs['kernelspecs'].keys())[0] print('No kernel specified, defaulting to \'{kernel}\''.format( kernel=kernel)) check.invariant( kernel in kernelspecs['kernelspecs'], 'Could not find kernel \'{kernel}\': available kernels are [{kernels}]' .format( kernel=kernel, kernels=', '.join( ['\'{k}\''.format(k=k) for k in kernelspecs['kernelspecs']]), ), ) return { 'name': kernel, 'language': kernelspecs['kernelspecs'][kernel]['spec']['language'], 'display_name': kernelspecs['kernelspecs'][kernel]['spec']['display_name'], }
async def graphql_http_endpoint(self, request: Request): """ fork of starlette GraphQLApp to allow for * our context type (crucial) * our GraphiQL playground (could change) """ if request.method == "GET": # render graphiql if "text/html" in request.headers.get("Accept", ""): text = TEMPLATE.replace("{{ app_path_prefix }}", self._app_path_prefix) return HTMLResponse(text) data: Union[Dict[str, str], QueryParams] = request.query_params elif request.method == "POST": content_type = request.headers.get("Content-Type", "") if "application/json" in content_type: data = await request.json() elif "application/graphql" in content_type: body = await request.body() text = body.decode() data = {"query": text} else: data = request.query_params else: return PlainTextResponse( "Method Not Allowed", status_code=status.HTTP_405_METHOD_NOT_ALLOWED) query = data.get("query") variables = data.get("variables") operation_name = data.get("operationName") if query is None: return PlainTextResponse( "No GraphQL query found in the request", status_code=status.HTTP_400_BAD_REQUEST, ) if isinstance(variables, str): try: variables = json.loads(variables) except json.JSONDecodeError: return PlainTextResponse( f"Malformed GraphQL variables. Passed as string but not valid JSON:\n{variables}", status_code=status.HTTP_400_BAD_REQUEST, ) result = await self.execute_graphql_request(request, query, variables, operation_name) response_data = {"data": result.data} status_code = status.HTTP_200_OK if result.errors: response_data["errors"] = self.handle_graphql_errors(result.errors) status_code = status.HTTP_400_BAD_REQUEST return JSONResponse(response_data, status_code=status_code)
'''This file is used by the EphemeralGrpcRunLauncher to execute runs in a subprocess.''' import sys from dagster import check from dagster.api.execute_run import cli_api_execute_run_grpc from dagster.core.instance import DagsterInstance, InstanceRef from dagster.core.origin import PipelinePythonOrigin from dagster.serdes import deserialize_json_to_dagster_namedtuple from dagster.serdes.ipc import setup_interrupt_support from dagster.seven import json if __name__ == '__main__': setup_interrupt_support() kwargs = json.loads(sys.argv[1]) instance_ref = check.inst( deserialize_json_to_dagster_namedtuple(kwargs['instance_ref']), InstanceRef) pipeline_origin = check.inst( deserialize_json_to_dagster_namedtuple(kwargs['pipeline_origin']), PipelinePythonOrigin) pipeline_run_id = kwargs['pipeline_run_id'] instance = DagsterInstance.from_ref(instance_ref) pipeline_run = instance.get_run_by_id(pipeline_run_id) events = [ evt for evt in cli_api_execute_run_grpc(instance_ref=instance_ref, pipeline_origin=pipeline_origin, pipeline_run=pipeline_run) ] print(len(events)) # pylint: disable=print-call