def print_type(config_type, print_fn=print): check.inst_param(config_type, 'config_type', ConfigType) check.callable_param(print_fn, 'print_fn') printer = IndentingPrinter(printer=print_fn) _do_print(config_type, printer) printer.line('')
def get_config_dir(config_yaml=None): instance = DagsterInstance.get() config_type = celery_executor.config_field.config_type config_value = get_config_value_from_yaml(config_yaml) config_module_name = 'dagster_celery_config' config_dir = os.path.join( instance.root_directory, 'dagster_celery', 'config', str(uuid.uuid4()) ) mkdir_p(config_dir) config_path = os.path.join( config_dir, '{config_module_name}.py'.format(config_module_name=config_module_name) ) validated_config = validate_config(config_type, config_value).value with open(config_path, 'w') as fd: printer = IndentingPrinter(printer=fd.write) if 'broker' in validated_config: printer.line( 'broker_url = \'{broker_url}\''.format(broker_url=str(validated_config['broker'])) ) if 'backend' in validated_config: printer.line( 'result_backend = \'{result_backend}\''.format( result_backend=str(validated_config['backend']) ) ) if 'config_source' in validated_config: for key, value in validated_config['config_source'].items(): printer.line('{key} = {value}'.format(key=key, value=repr(value))) # n.b. right now we don't attempt to clean up this cache, but it might make sense to delete # any files older than some time if there are more than some number of files present, etc. return config_dir
def print_solids(pipeline_snapshot, print_fn): check.inst_param(pipeline_snapshot, "pipeline", PipelineSnapshot) check.callable_param(print_fn, "print_fn") printer = IndentingPrinter(indent_level=2, printer=print_fn) printer.line("Pipeline: {name}".format(name=pipeline_snapshot.name)) printer.line("Solids:") for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps: with printer.with_indent(): printer.line("Solid: {name}".format(name=solid.solid_name))
def print_solids(pipeline, print_fn): check.inst_param(pipeline, 'pipeline', PipelineDefinition) check.callable_param(print_fn, 'print_fn') printer = IndentingPrinter(indent_level=2, printer=print_fn) printer.line('Pipeline: {name}'.format(name=pipeline.name)) printer.line('Solids:') for solid in pipeline.solids: with printer.with_indent(): printer.line('Solid: {name}'.format(name=solid.name))
def print_solids_or_ops(pipeline_snapshot, print_fn, using_job_op_graph_apis=False): check.inst_param(pipeline_snapshot, "pipeline", PipelineSnapshot) check.callable_param(print_fn, "print_fn") printer = IndentingPrinter(indent_level=2, printer=print_fn) printer.line( f"{'Job' if using_job_op_graph_apis else 'Pipeline'}: {pipeline_snapshot.name}" ) printer.line(f"{'Ops' if using_job_op_graph_apis else 'Solids'}") for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps: with printer.with_indent(): printer.line( f"{'Op' if using_job_op_graph_apis else 'Solid'}: {solid.solid_name}" )
def print_pipeline(pipeline_snapshot, print_fn): check.inst_param(pipeline_snapshot, 'pipeline', PipelineSnapshot) check.callable_param(print_fn, 'print_fn') printer = IndentingPrinter(indent_level=2, printer=print_fn) printer.line('Pipeline: {name}'.format(name=pipeline_snapshot.name)) print_description(printer, pipeline_snapshot.description) printer.line('Solids:') for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps: with printer.with_indent(): print_solid(printer, pipeline_snapshot, solid)
def print_type(config_type, print_fn=print, with_lines=True): check.inst_param(config_type, 'config_type', ConfigType) check.callable_param(print_fn, 'print_fn') if with_lines: printer = IndentingPrinter(printer=print_fn) else: printer = IndentingPrinter(printer=print_fn, indent_level=0) _do_print(config_type, printer, with_lines=with_lines) printer.line('')
def _print_type(config_schema_snapshot, config_type_key, print_fn, with_lines): check.inst_param(config_schema_snapshot, "config_schema_snapshot", ConfigSchemaSnapshot) check.str_param(config_type_key, "config_type_key") check.callable_param(print_fn, "print_fn") check.bool_param(with_lines, "with_lines") if with_lines: printer = IndentingPrinter(printer=print_fn) else: printer = IndentingPrinter(printer=print_fn, indent_level=0) _do_print(config_schema_snapshot, config_type_key, printer, with_lines=with_lines) printer.line("")
def print_pipeline(pipeline, full, print_fn): check.inst_param(pipeline, 'pipeline', PipelineDefinition) check.bool_param(full, 'full') check.callable_param(print_fn, 'print_fn') printer = IndentingPrinter(indent_level=2, printer=print_fn) printer.line('Pipeline: {name}'.format(name=pipeline.name)) print_description(printer, pipeline.description) if not full: return printer.line('Solids:') for solid in pipeline.solids: with printer.with_indent(): print_solid(printer, solid)
def celery_worker_config_args(config_yaml, config_module, config_file): check.invariant( not (sum( int(bool(x)) for x in [config_module, config_file, config_yaml]) > 1), 'Can only set one of --config-yaml/-y --config-module/-m or --config-file/-f', ) if config_yaml: instance = DagsterInstance.get() config_type = celery_executor.config_field.config_type config_value = get_config_value_from_yaml(config_yaml) config_hash = hash(yaml.dump(config_value)) config_dir = os.path.join(instance.root_directory, 'dagster_celery', 'config') mkdir_p(config_dir) config_path = os.path.join( config_dir, '{config_hash}.py'.format(config_hash=config_hash)) if not os.path.exists(config_path): validated_config = validate_config(config_type, config_value).value with open(config_path, 'w') as fd: printer = IndentingPrinter(printer=fd.write) if 'broker' in validated_config: printer.line('broker_url = {broker_url}'.format( broker_url=str(validated_config['broker']))) if 'backend' in validated_config: printer.line('result_backend = {result_backend}'.format( result_backend=str(validated_config['backend']))) if 'config_source' in validated_config: for key, value in validated_config['config_source'].items( ): printer.line('{key} = {value}'.format( key=key, value=str(value))) # n.b. right now we don't attempt to clean up this cache, but it might make sense to delete # any files older than some time if there are more than some number of files present, etc. return ['--config', config_path] elif config_module or config_file: return ['--config', config_module or config_file] else: return []
def print_pipeline_or_job( pipeline_snapshot: PipelineSnapshot, print_fn: Callable[..., Any], using_job_op_graph_apis: bool = False, ): check.inst_param(pipeline_snapshot, "pipeline", PipelineSnapshot) check.callable_param(print_fn, "print_fn") printer = IndentingPrinter(indent_level=2, printer=print_fn) printer.line( f"{'Job' if using_job_op_graph_apis else 'Pipeline'}: {pipeline_snapshot.name}" ) print_description(printer, pipeline_snapshot.description) printer.line(f"{'Ops' if using_job_op_graph_apis else 'Solids'}") for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps: with printer.with_indent(): print_solid_or_op(printer, pipeline_snapshot, solid, using_job_op_graph_apis)
def _print_type(config_schema_snapshot, config_type_key, print_fn, with_lines): check.inst_param(config_schema_snapshot, 'config_schema_snapshot', ConfigSchemaSnapshot) check.str_param(config_type_key, 'config_type_key') check.callable_param(print_fn, 'print_fn') check.bool_param(with_lines, 'with_lines') if with_lines: printer = IndentingPrinter(printer=print_fn) else: printer = IndentingPrinter(printer=print_fn, indent_level=0) _do_print(config_schema_snapshot, config_type_key, printer, with_lines=with_lines) printer.line('')
def print_solid_or_op( printer: IndentingPrinter, pipeline_snapshot: PipelineSnapshot, solid_invocation_snap: SolidInvocationSnap, using_job_op_graph_apis: bool, ) -> None: check.inst_param(pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot) check.inst_param(solid_invocation_snap, "solid_invocation_snap", SolidInvocationSnap) printer.line( f"{'Op' if using_job_op_graph_apis else 'Solid'}: {solid_invocation_snap.solid_name}" ) with printer.with_indent(): printer.line("Inputs:") for input_dep_snap in solid_invocation_snap.input_dep_snaps: with printer.with_indent(): printer.line( "Input: {name}".format(name=input_dep_snap.input_name)) printer.line("Outputs:") for output_def_snap in pipeline_snapshot.get_node_def_snap( solid_invocation_snap.solid_def_name).output_def_snaps: printer.line(output_def_snap.name)
def print_graph(graph, printer=print): check.inst_param(graph, 'graph', ComputeNodeGraph) printer = IndentingPrinter(printer=printer) for node in graph.topological_nodes(): with printer.with_indent( 'Node {node.friendly_name} Id: {node.guid}'.format(node=node)): for node_input in node.node_inputs: with printer.with_indent('Input: {node_input.name}'.format( node_input=node_input)): printer.line('Type: {node_input.dagster_type.name}'.format( node_input=node_input)) printer.line( 'From: {node_input.prev_output_handle}'.format( node_input=node_input)) for node_output in node.node_outputs: with printer.with_indent('Output: {node_output.name}'.format( node_output=node_output)): printer.line( 'Type: {node_output.dagster_type.name}'.format( node_output=node_output))