def from_yaml(cls, file_path): check.str_param(file_path, 'file_path') config = load_yaml_from_path(file_path) repository_config = check.dict_elem(config, 'repository') module_name = check.opt_str_elem(repository_config, 'module') file_name = check.opt_str_elem(repository_config, 'file') fn_name = check.str_elem(repository_config, 'fn') scheduler_pointer, partitions_pointer = _handle_backcompat_pointers( config, file_path) if module_name: pointer = ModuleCodePointer(module_name, fn_name) else: # rebase file in config off of the path in the config file file_name = os.path.join( os.path.dirname(os.path.abspath(file_path)), file_name) pointer = FileCodePointer(file_name, fn_name) return cls( pointer=pointer, yaml_path=file_path, scheduler_pointer=scheduler_pointer, partitions_pointer=partitions_pointer, )
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult": """Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary. Args: d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult <dagster_dbt.NodeResult>`. Returns: NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`. """ node = check.dict_elem(d, "node") error = check.opt_str_elem(d, "error") execution_time = check.float_elem(d, "execution_time") thread_id = check.opt_str_elem(d, "thread_id") check.list_elem(d, "timing") step_timings = [ StepTiming( name=d["name"], started_at=parser.isoparse(d["started_at"]), completed_at=parser.isoparse(d["completed_at"]), ) for d in check.is_list(d["timing"], of_type=Dict) ] table = check.opt_dict_elem(d, "table") return cls( step_timings=step_timings, node=node, error=error, execution_time=execution_time, thread_id=thread_id, table=table, )
def test_opt_string_elem(): ddict = {'a_str': 'a', 'a_num': 1, 'a_none': None} assert check.opt_str_elem(ddict, 'a_str') == 'a' assert check.opt_str_elem(ddict, 'a_none') == None assert check.opt_str_elem(ddict, 'nonexistentkey') == None with pytest.raises(ElementCheckError): check.opt_str_elem(ddict, 'a_num')
def test_opt_string_elem(): ddict = {"a_str": "a", "a_num": 1, "a_none": None} assert check.opt_str_elem(ddict, "a_str") == "a" assert check.opt_str_elem(ddict, "a_none") == None assert check.opt_str_elem(ddict, "nonexistentkey") == None with pytest.raises(ElementCheckError): check.opt_str_elem(ddict, "a_num")
def from_legacy_repository_yaml(file_path): check.str_param(file_path, "file_path") config = load_yaml_from_path(file_path) repository_config = check.dict_elem(config, "repository") module_name = check.opt_str_elem(repository_config, "module") file_name = check.opt_str_elem(repository_config, "file") fn_name = check.str_elem(repository_config, "fn") return (CodePointer.from_module(module_name, fn_name) if module_name # rebase file in config off of the path in the config file else CodePointer.from_python_file( rebase_file(file_name, file_path), fn_name, None))
def from_yaml(file_path, from_handle=None): check.str_param(file_path, 'file_path') config = load_yaml_from_path(file_path) repository_config = check.dict_elem(config, 'repository') module_name = check.opt_str_elem(repository_config, 'module') file_name = check.opt_str_elem(repository_config, 'file') fn_name = check.str_elem(repository_config, 'fn') if module_name: return LoaderEntrypoint.from_module_target(module_name, fn_name, from_handle) else: # rebase file in config off of the path in the config file file_name = os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name) return LoaderEntrypoint.from_file_target(file_name, fn_name, from_handle)
def from_dict(cls, d: Dict[str, Any]) -> "DbtResult": """Constructs an instance of :class:`DbtResult <dagster_dbt.DbtResult>` from a dictionary. Args: d (Dict[str, Any]): A dictionary with key-values to construct a :class:`DbtResult <dagster_dbt.DbtResult>`. Returns: DbtResult: An instance of :class:`DbtResult <dagster_dbt.DbtResult>`. """ check.list_elem(d, "logs") logs = check.is_list(d["logs"], of_type=Dict) check.list_elem(d, "results") results = [ NodeResult.from_dict(d) for d in check.is_list(d["results"], of_type=Dict) ] generated_at = check.opt_str_elem(d, "generated_at") elapsed_time = check.float_elem(d, "elapsed_time") return cls( logs=logs, results=results, generated_at=generated_at, elapsed_time=elapsed_time, )
def from_yaml(file_path): check.str_param(file_path, 'file_path') config = load_yaml_from_path(file_path) repository_config = check.dict_elem(config, 'repository') module_name = check.opt_str_elem(repository_config, 'module') file_name = check.opt_str_elem(repository_config, 'file') fn_name = check.str_elem(repository_config, 'fn') return ( ModuleCodePointer(module_name, fn_name) if module_name else FileCodePointer( # rebase file in config off of the path in the config file python_file=os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name), fn_name=fn_name, ) )
def _construct_context(yml_config_object): context_obj = check.opt_dict_elem(yml_config_object, 'context') if context_obj: return Context( check.opt_str_elem(context_obj, 'name'), context_obj.get('config'), ) else: return None
def load_repository_from_file(file_path): check.str_param(file_path, 'file_path') config = load_yaml_from_path(file_path) repository_config = check.dict_elem(config, 'repository') module_name = check.opt_str_elem(repository_config, 'module') file_name = check.opt_str_elem(repository_config, 'file') fn_name = check.str_elem(repository_config, 'fn') if module_name: return load_module_target_function( ModuleTargetFunction(module_name, fn_name)) else: # rebase file in config off of the path in the config file file_name = os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name) return load_file_target_function(FileTargetFunction( file_name, fn_name))
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult": """Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary. Args: d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult <dagster_dbt.NodeResult>`. Returns: NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`. """ check.dict_elem(d, "node") check.opt_str_elem(d, "error") check.float_elem(d, "execution_time") check.opt_str_elem(d, "thread_id") check.list_elem(d, "timing") check.is_list(d["timing"], of_type=Dict) check.opt_dict_elem(d, "table") return cls(step_timings=d.get("timing"), **d)
def from_dict(cls, d: Dict[str, Any]) -> "NodeResult": """Constructs an instance of :class:`NodeResult <dagster_dbt.NodeResult>` from a dictionary. Args: d (Dict[str, Any]): A dictionary with key-values to construct a :class:`NodeResult <dagster_dbt.NodeResult>`. Returns: NodeResult: An instance of :class:`NodeResult <dagster_dbt.NodeResult>`. """ # When executing from the CLI in 0.19.x, we get unique_id as a top level attribute if "unique_id" in d: unique_id = check.str_elem(d, "unique_id") node = None # When executing via RPC server or via CLI in 0.18.x or lower, we get unique id within # "node" schema else: node = check.dict_elem(d, "node") unique_id = check.str_elem(node, "unique_id") error = check.opt_str_elem(d, "error") execution_time = check.float_elem(d, "execution_time") thread_id = check.opt_str_elem(d, "thread_id") check.list_elem(d, "timing") step_timings = [ StepTiming( name=d["name"], started_at=parser.isoparse(d["started_at"]), completed_at=parser.isoparse(d["completed_at"]), ) for d in check.is_list(d["timing"], of_type=Dict) ] table = check.opt_dict_elem(d, "table") return cls( node=node, unique_id=unique_id, step_timings=step_timings, error=error, execution_time=execution_time, thread_id=thread_id, table=table, )
def _handle_backcompat_pointers(config, file_path): check.dict_param(config, 'config') partitions = config.get('partitions') scheduler = config.get('scheduler') if not (partitions or scheduler): return None, None warnings.warn( '"scheduler" and "partitions" keys in repository.yaml are deprecated. ' 'Add definitions directly via RepositoryDefinition') scheduler_pointer = None partitions_pointer = None if scheduler: module_name = check.opt_str_elem(scheduler, 'module') file_name = check.opt_str_elem(scheduler, 'file') fn_name = check.str_elem(scheduler, 'fn') if module_name: scheduler_pointer = ModuleCodePointer(module_name, fn_name) else: # rebase file in config off of the path in the config file file_name = os.path.join( os.path.dirname(os.path.abspath(file_path)), file_name) scheduler_pointer = FileCodePointer(file_name, fn_name) if partitions: module_name = check.opt_str_elem(partitions, 'module') file_name = check.opt_str_elem(partitions, 'file') fn_name = check.str_elem(partitions, 'fn') if module_name: partitions_pointer = ModuleCodePointer(module_name, fn_name) else: # rebase file in config off of the path in the config file file_name = os.path.join( os.path.dirname(os.path.abspath(file_path)), file_name) partitions_pointer = FileCodePointer(file_name, fn_name) return (scheduler_pointer, partitions_pointer)
def _handle_backcompat_loaders(config, file_path): check.dict_param(config, 'config') partitions = config.get('partitions') scheduler = config.get('scheduler') if not (partitions or scheduler): return None warnings.warn( '"scheduler" and "partitions" keys in repository.yaml are deprecated. ' 'Add definitions directly via RepositoryDefinition' ) backcompat_loaders = {} if scheduler: module_name = check.opt_str_elem(scheduler, 'module') file_name = check.opt_str_elem(scheduler, 'file') fn_name = check.str_elem(scheduler, 'fn') if module_name: backcompat_loaders['scheduler'] = LoaderEntrypoint.from_module_target( module_name, fn_name ) else: # rebase file in config off of the path in the config file file_name = os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name) backcompat_loaders['scheduler'] = LoaderEntrypoint.from_file_target(file_name, fn_name) if partitions: module_name = check.opt_str_elem(partitions, 'module') file_name = check.opt_str_elem(partitions, 'file') fn_name = check.str_elem(partitions, 'fn') if module_name: return LoaderEntrypoint.from_module_target(module_name, fn_name) else: # rebase file in config off of the path in the config file file_name = os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name) backcompat_loaders['partitions'] = LoaderEntrypoint.from_file_target(file_name, fn_name) return backcompat_loaders
def get_acceptable_entrypoint(repo_target_info): check.inst_param(repo_target_info, 'repo_target_info', RepositoryTargetInfo) if repo_target_info.repository_yaml: check.str_param(repo_target_info.repository_yaml, 'repository_yaml') config = load_yaml_from_path(repo_target_info.repository_yaml) repository_config = check.dict_elem(config, 'repository') module_name = check.opt_str_elem(repository_config, 'module') fn_name = check.str_elem(repository_config, 'fn') if module_name: return entrypoint_from_module_target(module_name, fn_name) return None elif repo_target_info.module_name and repo_target_info.fn_name: return entrypoint_from_module_target(repo_target_info.module_name, repo_target_info.fn_name) elif repo_target_info.python_file and repo_target_info.fn_name: return None else: raise InvalidRepositoryLoadingComboError()
def get_module_target_function(info): check.inst_param(info, 'info', RepositoryTargetInfo) if info.repository_yaml: mode_data = create_repository_loading_mode_data(info) file_path = mode_data.data check.str_param(file_path, 'file_path') config = load_yaml_from_path(file_path) repository_config = check.dict_elem(config, 'repository') module_name = check.opt_str_elem(repository_config, 'module') fn_name = check.str_elem(repository_config, 'fn') if module_name: return ModuleTargetFunction(module_name=module_name, fn_name=fn_name) return None elif info.module_name and info.fn_name: return ModuleTargetFunction(module_name=info.module_name, fn_name=info.fn_name) elif info.python_file and info.fn_name: return None else: raise InvalidRepositoryLoadingComboError()
def get_working_directory_from_kwargs(kwargs: Dict[str, str]) -> Optional[str]: return check.opt_str_elem(kwargs, "working_directory") or os.getcwd()
def get_workspace_load_target(kwargs: Dict[str, str]): check.dict_param(kwargs, "kwargs") if are_all_keys_empty(kwargs, WORKSPACE_CLI_ARGS): if kwargs.get("empty_workspace"): return EmptyWorkspaceTarget() if os.path.exists("workspace.yaml"): return WorkspaceFileTarget(paths=["workspace.yaml"]) raise click.UsageError( "No arguments given and workspace.yaml not found.") if kwargs.get("workspace"): _check_cli_arguments_none( kwargs, "python_file", "working_directory", "module_name", "package_name", "attribute", "grpc_host", "grpc_port", "grpc_socket", ) return WorkspaceFileTarget( paths=list(cast(Union[List, Tuple], kwargs.get("workspace")))) if kwargs.get("python_file"): _check_cli_arguments_none( kwargs, "module_name", "package_name", "grpc_host", "grpc_port", "grpc_socket", ) working_directory = get_working_directory_from_kwargs(kwargs) return PythonFileTarget( python_file=check.str_elem(kwargs, "python_file"), attribute=check.opt_str_elem(kwargs, "attribute"), working_directory=working_directory, location_name=None, ) if kwargs.get("module_name"): _check_cli_arguments_none( kwargs, "package_name", "grpc_host", "grpc_port", "grpc_socket", ) working_directory = get_working_directory_from_kwargs(kwargs) return ModuleTarget( module_name=check.str_elem(kwargs, "module_name"), attribute=check.opt_str_elem(kwargs, "attribute"), working_directory=working_directory, location_name=None, ) if kwargs.get("package_name"): _check_cli_arguments_none( kwargs, "grpc_host", "grpc_port", "grpc_socket", ) working_directory = get_working_directory_from_kwargs(kwargs) return PackageTarget( package_name=check.str_elem(kwargs, "package_name"), attribute=check.opt_str_elem(kwargs, "attribute"), working_directory=working_directory, location_name=None, ) if kwargs.get("grpc_port"): _check_cli_arguments_none( kwargs, "attribute", "working_directory", "grpc_socket", ) return GrpcServerTarget( port=check.int_elem(kwargs, "grpc_port"), socket=None, host=check.opt_str_elem(kwargs, "grpc_host") or "localhost", location_name=None, ) elif kwargs.get("grpc_socket"): _check_cli_arguments_none( kwargs, "attribute", "working_directory", ) return GrpcServerTarget( port=None, socket=check.str_elem(kwargs, "grpc_socket"), host=check.opt_str_elem(kwargs, "grpc_host") or "localhost", location_name=None, ) else: _cli_load_invariant(False) # necessary for pyright, does not understand _cli_load_invariant(False) never returns assert False