def test_invalid_named_selector_field():
with pytest.raises(DagsterInvalidDefinitionError) as exc_info:
NamedSelector('some_selector', {'val': Int})
assert str(exc_info.value) == (
'You have passed a config type "Int" in the parameter "fields" and it is '
'in the "val" entry of that dict. It is from a NamedSelector named '
'"some_selector" with fields [\'val\']. You '
'have likely forgot to wrap this type in a Field.')
def define_csv_dict_field():
return Field(
Dict({'path': Field(Path), 'sep': Field(String, is_optional=True, default_value=',')})
)
def dict_without_keys(ddict, *keys):
return {key: value for key, value in ddict.items() if key not in set(keys)}
@output_selector_schema(
NamedSelector(
'DataFrameOutputSchema',
{
'csv': define_csv_dict_field(),
'parquet': define_path_dict_field(),
'table': define_path_dict_field(),
},
)
)
def dataframe_output_schema(file_type, file_options, pandas_df):
check.str_param(file_type, 'file_type')
check.dict_param(file_options, 'file_options')
check.inst_param(pandas_df, 'pandas_df', DataFrame)
if file_type == 'csv':
path = file_options['path']
return pandas_df.to_csv(path, index=False, **dict_without_keys(file_options, 'path'))
elif file_type == 'parquet':
return pandas_df.to_parquet(file_options['path'])
elif file_type == 'table':