def print_type(config_type, print_fn=print):
check.inst_param(config_type, 'config_type', ConfigType)
check.callable_param(print_fn, 'print_fn')
printer = IndentingPrinter(printer=print_fn)
_do_print(config_type, printer)
printer.line('')
def get_config_dir(config_yaml=None):
instance = DagsterInstance.get()
config_type = celery_executor.config_field.config_type
config_value = get_config_value_from_yaml(config_yaml)
config_module_name = 'dagster_celery_config'
config_dir = os.path.join(
instance.root_directory, 'dagster_celery', 'config', str(uuid.uuid4())
)
mkdir_p(config_dir)
config_path = os.path.join(
config_dir, '{config_module_name}.py'.format(config_module_name=config_module_name)
)
validated_config = validate_config(config_type, config_value).value
with open(config_path, 'w') as fd:
printer = IndentingPrinter(printer=fd.write)
if 'broker' in validated_config:
printer.line(
'broker_url = \'{broker_url}\''.format(broker_url=str(validated_config['broker']))
)
if 'backend' in validated_config:
printer.line(
'result_backend = \'{result_backend}\''.format(
result_backend=str(validated_config['backend'])
)
)
if 'config_source' in validated_config:
for key, value in validated_config['config_source'].items():
printer.line('{key} = {value}'.format(key=key, value=repr(value)))
# n.b. right now we don't attempt to clean up this cache, but it might make sense to delete
# any files older than some time if there are more than some number of files present, etc.
return config_dir
def print_solids(pipeline_snapshot, print_fn):
check.inst_param(pipeline_snapshot, "pipeline", PipelineSnapshot)
check.callable_param(print_fn, "print_fn")
printer = IndentingPrinter(indent_level=2, printer=print_fn)
printer.line("Pipeline: {name}".format(name=pipeline_snapshot.name))
printer.line("Solids:")
for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps:
with printer.with_indent():
printer.line("Solid: {name}".format(name=solid.solid_name))
def print_solids(pipeline, print_fn):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.callable_param(print_fn, 'print_fn')
printer = IndentingPrinter(indent_level=2, printer=print_fn)
printer.line('Pipeline: {name}'.format(name=pipeline.name))
printer.line('Solids:')
for solid in pipeline.solids:
with printer.with_indent():
printer.line('Solid: {name}'.format(name=solid.name))
def print_solids_or_ops(pipeline_snapshot,
print_fn,
using_job_op_graph_apis=False):
check.inst_param(pipeline_snapshot, "pipeline", PipelineSnapshot)
check.callable_param(print_fn, "print_fn")
printer = IndentingPrinter(indent_level=2, printer=print_fn)
printer.line(
f"{'Job' if using_job_op_graph_apis else 'Pipeline'}: {pipeline_snapshot.name}"
)
printer.line(f"{'Ops' if using_job_op_graph_apis else 'Solids'}")
for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps:
with printer.with_indent():
printer.line(
f"{'Op' if using_job_op_graph_apis else 'Solid'}: {solid.solid_name}"
)
def print_pipeline(pipeline_snapshot, print_fn):
check.inst_param(pipeline_snapshot, 'pipeline', PipelineSnapshot)
check.callable_param(print_fn, 'print_fn')
printer = IndentingPrinter(indent_level=2, printer=print_fn)
printer.line('Pipeline: {name}'.format(name=pipeline_snapshot.name))
print_description(printer, pipeline_snapshot.description)
printer.line('Solids:')
for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps:
with printer.with_indent():
print_solid(printer, pipeline_snapshot, solid)
def print_type(config_type, print_fn=print, with_lines=True):
check.inst_param(config_type, 'config_type', ConfigType)
check.callable_param(print_fn, 'print_fn')
if with_lines:
printer = IndentingPrinter(printer=print_fn)
else:
printer = IndentingPrinter(printer=print_fn, indent_level=0)
_do_print(config_type, printer, with_lines=with_lines)
printer.line('')
def _print_type(config_schema_snapshot, config_type_key, print_fn, with_lines):
check.inst_param(config_schema_snapshot, "config_schema_snapshot", ConfigSchemaSnapshot)
check.str_param(config_type_key, "config_type_key")
check.callable_param(print_fn, "print_fn")
check.bool_param(with_lines, "with_lines")
if with_lines:
printer = IndentingPrinter(printer=print_fn)
else:
printer = IndentingPrinter(printer=print_fn, indent_level=0)
_do_print(config_schema_snapshot, config_type_key, printer, with_lines=with_lines)
printer.line("")
def print_pipeline(pipeline, full, print_fn):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.bool_param(full, 'full')
check.callable_param(print_fn, 'print_fn')
printer = IndentingPrinter(indent_level=2, printer=print_fn)
printer.line('Pipeline: {name}'.format(name=pipeline.name))
print_description(printer, pipeline.description)
if not full:
return
printer.line('Solids:')
for solid in pipeline.solids:
with printer.with_indent():
print_solid(printer, solid)
def celery_worker_config_args(config_yaml, config_module, config_file):
check.invariant(
not (sum(
int(bool(x))
for x in [config_module, config_file, config_yaml]) > 1),
'Can only set one of --config-yaml/-y --config-module/-m or --config-file/-f',
)
if config_yaml:
instance = DagsterInstance.get()
config_type = celery_executor.config_field.config_type
config_value = get_config_value_from_yaml(config_yaml)
config_hash = hash(yaml.dump(config_value))
config_dir = os.path.join(instance.root_directory, 'dagster_celery',
'config')
mkdir_p(config_dir)
config_path = os.path.join(
config_dir, '{config_hash}.py'.format(config_hash=config_hash))
if not os.path.exists(config_path):
validated_config = validate_config(config_type, config_value).value
with open(config_path, 'w') as fd:
printer = IndentingPrinter(printer=fd.write)
if 'broker' in validated_config:
printer.line('broker_url = {broker_url}'.format(
broker_url=str(validated_config['broker'])))
if 'backend' in validated_config:
printer.line('result_backend = {result_backend}'.format(
result_backend=str(validated_config['backend'])))
if 'config_source' in validated_config:
for key, value in validated_config['config_source'].items(
):
printer.line('{key} = {value}'.format(
key=key, value=str(value)))
# n.b. right now we don't attempt to clean up this cache, but it might make sense to delete
# any files older than some time if there are more than some number of files present, etc.
return ['--config', config_path]
elif config_module or config_file:
return ['--config', config_module or config_file]
else:
return []
def print_pipeline_or_job(
pipeline_snapshot: PipelineSnapshot,
print_fn: Callable[..., Any],
using_job_op_graph_apis: bool = False,
):
check.inst_param(pipeline_snapshot, "pipeline", PipelineSnapshot)
check.callable_param(print_fn, "print_fn")
printer = IndentingPrinter(indent_level=2, printer=print_fn)
printer.line(
f"{'Job' if using_job_op_graph_apis else 'Pipeline'}: {pipeline_snapshot.name}"
)
print_description(printer, pipeline_snapshot.description)
printer.line(f"{'Ops' if using_job_op_graph_apis else 'Solids'}")
for solid in pipeline_snapshot.dep_structure_snapshot.solid_invocation_snaps:
with printer.with_indent():
print_solid_or_op(printer, pipeline_snapshot, solid,
using_job_op_graph_apis)
def _print_type(config_schema_snapshot, config_type_key, print_fn, with_lines):
check.inst_param(config_schema_snapshot, 'config_schema_snapshot',
ConfigSchemaSnapshot)
check.str_param(config_type_key, 'config_type_key')
check.callable_param(print_fn, 'print_fn')
check.bool_param(with_lines, 'with_lines')
if with_lines:
printer = IndentingPrinter(printer=print_fn)
else:
printer = IndentingPrinter(printer=print_fn, indent_level=0)
_do_print(config_schema_snapshot,
config_type_key,
printer,
with_lines=with_lines)
printer.line('')
def print_solid_or_op(
printer: IndentingPrinter,
pipeline_snapshot: PipelineSnapshot,
solid_invocation_snap: SolidInvocationSnap,
using_job_op_graph_apis: bool,
) -> None:
check.inst_param(pipeline_snapshot, "pipeline_snapshot", PipelineSnapshot)
check.inst_param(solid_invocation_snap, "solid_invocation_snap",
SolidInvocationSnap)
printer.line(
f"{'Op' if using_job_op_graph_apis else 'Solid'}: {solid_invocation_snap.solid_name}"
)
with printer.with_indent():
printer.line("Inputs:")
for input_dep_snap in solid_invocation_snap.input_dep_snaps:
with printer.with_indent():
printer.line(
"Input: {name}".format(name=input_dep_snap.input_name))
printer.line("Outputs:")
for output_def_snap in pipeline_snapshot.get_node_def_snap(
solid_invocation_snap.solid_def_name).output_def_snaps:
printer.line(output_def_snap.name)
def print_graph(graph, printer=print):
check.inst_param(graph, 'graph', ComputeNodeGraph)
printer = IndentingPrinter(printer=printer)
for node in graph.topological_nodes():
with printer.with_indent(
'Node {node.friendly_name} Id: {node.guid}'.format(node=node)):
for node_input in node.node_inputs:
with printer.with_indent('Input: {node_input.name}'.format(
node_input=node_input)):
printer.line('Type: {node_input.dagster_type.name}'.format(
node_input=node_input))
printer.line(
'From: {node_input.prev_output_handle}'.format(
node_input=node_input))
for node_output in node.node_outputs:
with printer.with_indent('Output: {node_output.name}'.format(
node_output=node_output)):
printer.line(
'Type: {node_output.dagster_type.name}'.format(
node_output=node_output))