def get_inputs_field(creation_data, solid):
check.inst_param(creation_data, 'creation_data', EnvironmentClassCreationData)
check.inst_param(solid, 'solid', Solid)
if not solid_has_configurable_inputs(solid.definition):
return None
inputs_field_fields = {}
for inp in [inp for inp in solid.definition.input_defs if inp.runtime_type.input_schema]:
inp_handle = SolidInputHandle(solid, inp)
# If this input is not satisfied by a dependency you must
# provide it via config
if not creation_data.dependency_structure.has_dep(inp_handle):
inputs_field_fields[inp.name] = FieldImpl(inp.runtime_type.input_schema.schema_type)
if not inputs_field_fields:
return None
return Field(
SystemNamedDict(
'{pipeline_name}.{solid_name}.Inputs'.format(
pipeline_name=camelcase(creation_data.pipeline_name),
solid_name=camelcase(solid.name),
),
inputs_field_fields,
)
)
def get_outputs_field(creation_data, solid):
check.inst_param(creation_data, 'creation_data',
EnvironmentClassCreationData)
check.inst_param(solid, 'solid', Solid)
solid_def = solid.definition
if not solid_has_configurable_outputs(solid_def):
return None
output_dict_fields = {}
for out in [
out for out in solid_def.output_defs
if out.runtime_type.output_schema
]:
output_dict_fields[out.name] = Field(type(
out.runtime_type.output_schema.schema_type),
is_optional=True)
output_entry_dict = SystemNamedDict(
'{pipeline_name}.{solid_name}.Outputs'.format(
pipeline_name=camelcase(creation_data.pipeline_name),
solid_name=camelcase(solid.name)),
output_dict_fields,
)
return Field(List(output_entry_dict), is_optional=True)
def define_environment_cls(creation_data):
check.inst_param(creation_data, 'creation_data',
EnvironmentClassCreationData)
pipeline_name = camelcase(creation_data.pipeline_name)
return SolidContainerConfigDict(
name='{pipeline_name}.Mode.{mode_name}.Environment'.format(
pipeline_name=pipeline_name,
mode_name=camelcase(creation_data.mode_definition.name))
if creation_data.mode_definition else
'{pipeline_name}.Environment'.format(pipeline_name=pipeline_name),
fields=remove_none_entries({
'solids':
Field(
define_solid_dictionary_cls(
'{pipeline_name}.SolidsConfigDictionary'.format(
pipeline_name=pipeline_name),
creation_data.solids,
creation_data.dependency_structure,
creation_data.pipeline_name,
)),
'storage':
Field(
define_storage_config_cls(
'{pipeline_name}.{mode_name}.StorageConfig'.format(
pipeline_name=pipeline_name,
mode_name=camelcase(
creation_data.mode_definition.name),
),
creation_data.mode_definition,
),
is_optional=True,
),
'execution':
Field(
define_executor_config_cls(
'{pipeline_name}.{mode_name}.ExecutionConfig'.format(
pipeline_name=pipeline_name,
mode_name=camelcase(
creation_data.mode_definition.name),
),
creation_data.mode_definition,
),
is_optional=True,
),
'loggers':
Field(
define_logger_dictionary_cls(
'{pipeline_name}.LoggerConfig'.format(
pipeline_name=pipeline_name),
creation_data,
)),
'resources':
Field(
define_mode_resources_dictionary_cls(
pipeline_name, creation_data.mode_definition)),
}),
)
def define_isolid_field(solid, handle, dependency_structure, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.str_param(pipeline_name, 'pipeline_name')
if isinstance(solid.definition, CompositeSolidDefinition):
composite_def = solid.definition
child_solids_config_field = Field(
define_solid_dictionary_cls(
'{pipeline_name}.CompositeSolidsDict.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
composite_def.solids,
composite_def.dependency_structure,
pipeline_name,
handle,
))
composite_config_dict = {
'inputs':
get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
'outputs':
get_outputs_field(solid, handle, pipeline_name),
}
# Mask solid config for solids beneath this level if config mapping is provided
if composite_def.has_config_mapping:
composite_config_dict[
'config'] = composite_def.config_mapping.config_field
else:
composite_config_dict['solids'] = child_solids_config_field
return Field(
SolidContainerConfigDict(
'{name}CompositeSolidConfig'.format(name=str(handle)),
remove_none_entries(composite_config_dict),
handle=handle,
child_solids_config_field=child_solids_config_field,
))
elif isinstance(solid.definition, SolidDefinition):
solid_config_type = define_solid_config_cls(
'{pipeline_name}.SolidConfig.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
solid.definition.config_field,
inputs_field=get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
outputs_field=get_outputs_field(solid, handle, pipeline_name),
)
return Field(solid_config_type)
else:
check.invariant('Unexpected ISolidDefinition type {type}'.format(
type=type(solid.definition)))
def get_inputs_field(solid, handle, dependency_structure, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.inst_param(dependency_structure, 'dependency_structure',
DependencyStructure)
check.str_param(pipeline_name, 'pipeline_name')
if not solid.definition.has_configurable_inputs:
return None
inputs_field_fields = {}
for name, inp in solid.definition.input_dict.items():
if inp.runtime_type.input_schema:
inp_handle = SolidInputHandle(solid, inp)
# If this input is not satisfied by a dependency you must
# provide it via config
if not dependency_structure.has_deps(
inp_handle) and not solid.parent_maps_input(name):
inputs_field_fields[name] = FieldImpl(
inp.runtime_type.input_schema.schema_type)
if not inputs_field_fields:
return None
return Field(
SystemNamedDict(
'{pipeline_name}.{solid_handle}.Inputs'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
inputs_field_fields,
))
def define_resource_cls(parent_name, resource_name, resource_def):
return SystemNamedDict(
'{parent_name}.{resource_name}'.format(
parent_name=parent_name, resource_name=camelcase(resource_name)
),
{'config': resource_def.config_field},
)
def define_environment_cls(creation_data):
check.inst_param(creation_data, 'creation_data', EnvironmentClassCreationData)
pipeline_name = camelcase(creation_data.pipeline_name)
return SystemNamedDict(
name='{pipeline_name}.Environment'.format(pipeline_name=pipeline_name),
fields={
'context': define_maybe_optional_selector_field(
define_context_context_cls(pipeline_name, creation_data.context_definitions)
),
'solids': Field(
define_solid_dictionary_cls(
'{pipeline_name}.SolidsConfigDictionary'.format(pipeline_name=pipeline_name),
creation_data,
)
),
'expectations': Field(
define_expectations_config_cls(
'{pipeline_name}.ExpectationsConfig'.format(pipeline_name=pipeline_name)
)
),
'storage': Field(
define_storage_config_cls(
'{pipeline_name}.StorageConfig'.format(pipeline_name=pipeline_name)
),
is_optional=True,
),
'execution': Field(
define_execution_config_cls(
'{pipeline_name}.ExecutionConfig'.format(pipeline_name=pipeline_name)
)
),
},
)
def get_outputs_field(solid, handle, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.str_param(pipeline_name, 'pipeline_name')
solid_def = solid.definition
if not solid_def.has_configurable_outputs:
return None
output_dict_fields = {}
for name, out in solid_def.output_dict.items():
if out.runtime_type.output_schema:
output_dict_fields[name] = Field(type(
out.runtime_type.output_schema.schema_type),
is_optional=True)
output_entry_dict = SystemNamedDict(
'{pipeline_name}.{solid_handle}.Outputs'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
output_dict_fields,
)
return Field(List(output_entry_dict), is_optional=True)
def define_environment_cls(pipeline_def):
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
pipeline_name = camelcase(pipeline_def.name)
return SystemNamedDict(
name='{pipeline_name}.Environment'.format(pipeline_name=pipeline_name),
fields={
'context':
define_maybe_optional_selector_field(
define_context_context_cls(pipeline_name,
pipeline_def.context_definitions)),
'solids':
Field(
define_solid_dictionary_cls(
'{pipeline_name}.SolidsConfigDictionary'.format(
pipeline_name=pipeline_name),
pipeline_def,
)),
'expectations':
Field(
define_expectations_config_cls(
'{pipeline_name}.ExpectationsConfig'.format(
pipeline_name=pipeline_name))),
'execution':
Field(
define_execution_config_cls(
'{pipeline_name}.ExecutionConfig'.format(
pipeline_name=pipeline_name))),
},
)
def define_isolid_field(solid, handle, dependency_structure, pipeline_name):
check.inst_param(solid, 'solid', Solid)
check.inst_param(handle, 'handle', SolidHandle)
check.str_param(pipeline_name, 'pipeline_name')
if isinstance(solid.definition, CompositeSolidDefinition):
composite_def = solid.definition
solid_cfg = Field(
define_solid_dictionary_cls(
'{pipeline_name}.CompositeSolidsDict.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
composite_def.solids,
composite_def.dependency_structure,
pipeline_name,
handle,
))
return Field(
SystemNamedDict(
'{name}CompositeSolidConfig'.format(name=str(handle)),
remove_none_entries({
'solids':
solid_cfg,
'inputs':
get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
'outputs':
get_outputs_field(solid, handle, pipeline_name),
}),
))
elif isinstance(solid.definition, SolidDefinition):
solid_config_type = define_solid_config_cls(
'{pipeline_name}.SolidConfig.{solid_handle}'.format(
pipeline_name=camelcase(pipeline_name),
solid_handle=handle.camelcase()),
solid.definition.config_field,
inputs_field=get_inputs_field(solid, handle, dependency_structure,
pipeline_name),
outputs_field=get_outputs_field(solid, handle, pipeline_name),
)
return Field(solid_config_type)
else:
check.invariant('Unexpected ISolidDefinition type {type}'.format(
type=type(solid.definition)))
def define_specific_context_cls(pipeline_name, context_name, context_definition):
return define_specific_context_config_cls(
'{pipeline_name}.ContextDefinitionConfig.{context_name}'.format(
pipeline_name=pipeline_name, context_name=camelcase(context_name)
),
context_definition.config_field,
context_definition.resources,
)
def define_solid_dictionary_cls(name, pipeline_def):
check.str_param(name, 'name')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
fields = {}
for solid in pipeline_def.solids:
if solid_has_config_entry(solid.definition):
solid_config_type = define_solid_config_cls(
'{pipeline_name}.SolidConfig.{solid_name}'.format(
pipeline_name=camelcase(pipeline_def.name),
solid_name=camelcase(solid.name)),
solid.definition.config_field,
inputs_field=get_inputs_field(pipeline_def, solid),
outputs_field=get_outputs_field(pipeline_def, solid),
)
fields[solid.name] = Field(solid_config_type)
return SystemNamedDict(name, fields)
def define_mode_resources_dictionary_cls(pipeline_name, mode_definition):
check.str_param(pipeline_name, 'pipeline_name')
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
return define_resource_dictionary_cls(
'{pipeline_name}.Mode.{mode}.Resources'.format(
pipeline_name=pipeline_name, mode=camelcase(mode_definition.name)),
mode_definition.resource_defs,
)
def define_solid_dictionary_cls(name, creation_data):
check.str_param(name, 'name')
check.inst_param(creation_data, 'creation_data', EnvironmentClassCreationData)
fields = {}
for solid in creation_data.solids:
if solid_has_config_entry(solid.definition):
solid_config_type = define_solid_config_cls(
'{pipeline_name}.SolidConfig.{solid_name}'.format(
pipeline_name=camelcase(creation_data.pipeline_name),
solid_name=camelcase(solid.name),
),
solid.definition.config_field,
inputs_field=get_inputs_field(creation_data, solid),
outputs_field=get_outputs_field(creation_data, solid),
)
fields[solid.name] = Field(solid_config_type)
return SystemNamedDict(name, fields)
def define_logger_dictionary_cls(name, creation_data):
check.str_param(name, 'name')
check.inst_param(creation_data, 'creation_data', EnvironmentClassCreationData)
fields = {}
for logger_name, logger_definition in creation_data.logger_defs.items():
fields[logger_name] = Field(
SystemNamedDict(
'{pipeline_name}.LoggerConfig.{logger_name}'.format(
pipeline_name=camelcase(creation_data.pipeline_name),
logger_name=camelcase(logger_name),
),
remove_none_entries({'config': logger_definition.config_field}),
),
is_optional=True,
)
return SystemNamedDict(name, fields)
def define_context_cls(pipeline_def):
from dagster import PipelineDefinition
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
pipeline_name = camelcase(pipeline_def.name)
return SystemNamedDict(
name='{pipeline_name}.Context'.format(pipeline_name=pipeline_name),
fields={
'context': define_maybe_optional_selector_field(
define_context_context_cls(pipeline_name, pipeline_def.context_definitions)
)
},
)
def define_executor_config_cls(type_name, mode_definition):
check.str_param(type_name, 'type_name')
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
fields = {}
for executor_def in mode_definition.executor_defs:
fields[executor_def.name] = Field(
SystemNamedDict(
name='{type_name}.{executor_name}'.format(
type_name=type_name, executor_name=camelcase(executor_def.name)
),
fields={'config': executor_def.config_field} if executor_def.config_field else {},
)
)
return SystemNamedSelector(type_name, fields)
def define_storage_config_cls(type_name, mode_definition):
check.str_param(type_name, 'type_name')
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
fields = {}
for storage_def in mode_definition.system_storage_defs:
fields[storage_def.name] = Field(
SystemNamedDict(
name='{type_name}.{storage_name}'.format(
type_name=type_name, storage_name=camelcase(storage_def.name)
),
fields={'config': storage_def.config_field} if storage_def.config_field else {},
)
)
return SystemNamedSelector(type_name, fields)
def camelcase(self):
return (self.parent.camelcase() + '.' +
camelcase(self.name) if self.parent else camelcase(self.name))
def test_camelcase():
assert camelcase('foo') == 'Foo'
assert camelcase('foo_bar') == 'FooBar'
assert camelcase('foo.bar') == 'FooBar'
assert camelcase('foo-bar') == 'FooBar'
assert camelcase('') == ''
def test_camelcase():
assert camelcase("foo") == "Foo"
assert camelcase("foo_bar") == "FooBar"
assert camelcase("foo.bar") == "FooBar"
assert camelcase("foo-bar") == "FooBar"
assert camelcase("") == ""