def test_dict_elem():
dict_value = {"blah": "blahblah"}
ddict = {"dictkey": dict_value, "stringkey": "A", "nonekey": None}
assert check.dict_elem(ddict, "dictkey") == dict_value
with pytest.raises(CheckError):
check.dict_elem(ddict, "stringkey")
with pytest.raises(CheckError):
check.dict_elem(ddict, "nonekey")
with pytest.raises(CheckError):
check.dict_elem(ddict, "nonexistantkey")
assert check.dict_elem(
{"foo": {
"str": 1,
"str2": "str",
1: "str",
2: "str"
}},
"foo",
key_type=(str, int),
value_type=(str, int),
)
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult":
"""Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary.
Args:
d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult
<dagster_dbt.NodeResult>`.
Returns:
NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`.
"""
node = check.dict_elem(d, "node")
error = check.opt_str_elem(d, "error")
execution_time = check.float_elem(d, "execution_time")
thread_id = check.opt_str_elem(d, "thread_id")
check.list_elem(d, "timing")
step_timings = [
StepTiming(
name=d["name"],
started_at=parser.isoparse(d["started_at"]),
completed_at=parser.isoparse(d["completed_at"]),
)
for d in check.is_list(d["timing"], of_type=Dict)
]
table = check.opt_dict_elem(d, "table")
return cls(
step_timings=step_timings,
node=node,
error=error,
execution_time=execution_time,
thread_id=thread_id,
table=table,
)
def from_yaml(cls, file_path):
check.str_param(file_path, 'file_path')
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
file_name = check.opt_str_elem(repository_config, 'file')
fn_name = check.str_elem(repository_config, 'fn')
scheduler_pointer, partitions_pointer = _handle_backcompat_pointers(
config, file_path)
if module_name:
pointer = ModuleCodePointer(module_name, fn_name)
else:
# rebase file in config off of the path in the config file
file_name = os.path.join(
os.path.dirname(os.path.abspath(file_path)), file_name)
pointer = FileCodePointer(file_name, fn_name)
return cls(
pointer=pointer,
yaml_path=file_path,
scheduler_pointer=scheduler_pointer,
partitions_pointer=partitions_pointer,
)
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult":
"""Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary.
Args:
d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult
<dagster_dbt.NodeResult>`.
Returns:
NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`.
"""
check.dict_elem(d, "node")
check.opt_str_elem(d, "error")
check.float_elem(d, "execution_time")
check.opt_str_elem(d, "thread_id")
check.list_elem(d, "timing")
check.is_list(d["timing"], of_type=Dict)
check.opt_dict_elem(d, "table")
return cls(step_timings=d.get("timing"), **d)
def from_legacy_repository_yaml(file_path):
check.str_param(file_path, "file_path")
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, "repository")
module_name = check.opt_str_elem(repository_config, "module")
file_name = check.opt_str_elem(repository_config, "file")
fn_name = check.str_elem(repository_config, "fn")
return (CodePointer.from_module(module_name, fn_name) if module_name
# rebase file in config off of the path in the config file
else CodePointer.from_python_file(
rebase_file(file_name, file_path), fn_name, None))
def from_yaml(file_path, from_handle=None):
check.str_param(file_path, 'file_path')
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
file_name = check.opt_str_elem(repository_config, 'file')
fn_name = check.str_elem(repository_config, 'fn')
if module_name:
return LoaderEntrypoint.from_module_target(module_name, fn_name, from_handle)
else:
# rebase file in config off of the path in the config file
file_name = os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name)
return LoaderEntrypoint.from_file_target(file_name, fn_name, from_handle)
def test_dict_elem():
dict_value = {'blah': 'blahblah'}
ddict = {'dictkey': dict_value, 'stringkey': 'A', 'nonekey': None}
assert check.dict_elem(ddict, 'dictkey') == dict_value
with pytest.raises(CheckError):
check.dict_elem(ddict, 'stringkey')
with pytest.raises(CheckError):
check.dict_elem(ddict, 'nonekey')
with pytest.raises(CheckError):
check.dict_elem(ddict, 'nonexistantkey')
def test_dict_elem():
dict_value = {"blah": "blahblah"}
ddict = {"dictkey": dict_value, "stringkey": "A", "nonekey": None}
assert check.dict_elem(ddict, "dictkey") == dict_value
with pytest.raises(CheckError):
check.dict_elem(ddict, "stringkey")
with pytest.raises(CheckError):
check.dict_elem(ddict, "nonekey")
with pytest.raises(CheckError):
check.dict_elem(ddict, "nonexistantkey")
def from_yaml(file_path):
check.str_param(file_path, 'file_path')
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
file_name = check.opt_str_elem(repository_config, 'file')
fn_name = check.str_elem(repository_config, 'fn')
return (
ModuleCodePointer(module_name, fn_name)
if module_name
else FileCodePointer(
# rebase file in config off of the path in the config file
python_file=os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name),
fn_name=fn_name,
)
)
def load_repository_from_file(file_path):
check.str_param(file_path, 'file_path')
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
file_name = check.opt_str_elem(repository_config, 'file')
fn_name = check.str_elem(repository_config, 'fn')
if module_name:
return load_module_target_function(
ModuleTargetFunction(module_name, fn_name))
else:
# rebase file in config off of the path in the config file
file_name = os.path.join(os.path.dirname(os.path.abspath(file_path)),
file_name)
return load_file_target_function(FileTargetFunction(
file_name, fn_name))
def get_acceptable_entrypoint(repo_target_info):
check.inst_param(repo_target_info, 'repo_target_info',
RepositoryTargetInfo)
if repo_target_info.repository_yaml:
check.str_param(repo_target_info.repository_yaml, 'repository_yaml')
config = load_yaml_from_path(repo_target_info.repository_yaml)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
fn_name = check.str_elem(repository_config, 'fn')
if module_name:
return entrypoint_from_module_target(module_name, fn_name)
return None
elif repo_target_info.module_name and repo_target_info.fn_name:
return entrypoint_from_module_target(repo_target_info.module_name,
repo_target_info.fn_name)
elif repo_target_info.python_file and repo_target_info.fn_name:
return None
else:
raise InvalidRepositoryLoadingComboError()
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult":
"""Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary.
Args:
d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult
<dagster_dbt.NodeResult>`.
Returns:
NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`.
"""
# When executing from the CLI in 0.19.x, we get unique_id as a top level attribute
if "unique_id" in d:
unique_id = check.str_elem(d, "unique_id")
node = None
# When executing via RPC server or via CLI in 0.18.x or lower, we get unique id within
# "node" schema
else:
node = check.dict_elem(d, "node")
unique_id = check.str_elem(node, "unique_id")
error = check.opt_str_elem(d, "error")
execution_time = check.float_elem(d, "execution_time")
thread_id = check.opt_str_elem(d, "thread_id")
check.list_elem(d, "timing")
step_timings = [
StepTiming(
name=d["name"],
started_at=parser.isoparse(d["started_at"]),
completed_at=parser.isoparse(d["completed_at"]),
) for d in check.is_list(d["timing"], of_type=Dict)
]
table = check.opt_dict_elem(d, "table")
return cls(
node=node,
unique_id=unique_id,
step_timings=step_timings,
error=error,
execution_time=execution_time,
thread_id=thread_id,
table=table,
)
def get_module_target_function(info):
check.inst_param(info, 'info', RepositoryTargetInfo)
if info.repository_yaml:
mode_data = create_repository_loading_mode_data(info)
file_path = mode_data.data
check.str_param(file_path, 'file_path')
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
fn_name = check.str_elem(repository_config, 'fn')
if module_name:
return ModuleTargetFunction(module_name=module_name,
fn_name=fn_name)
return None
elif info.module_name and info.fn_name:
return ModuleTargetFunction(module_name=info.module_name,
fn_name=info.fn_name)
elif info.python_file and info.fn_name:
return None
else:
raise InvalidRepositoryLoadingComboError()
