def dagster_instance_config( base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None, ): check.str_param(base_dir, "base_dir") check.invariant(os.path.isdir(base_dir), "base_dir should be a directory") overrides = check.opt_dict_param(overrides, "overrides") config_yaml_path = os.path.join(base_dir, config_filename) if not os.path.exists(config_yaml_path): warnings.warn(( "The dagster instance configuration file ({config_filename}) is not present at " "{base_dir}. Dagster uses this file to know where and how to store " "local artifacts, information about past runs, and structured events.\n" "If nothing is specified, Dagster will store this information " "in the local filesystem in the {base_dir} directory.").format( config_filename=config_filename, base_dir=base_dir)) dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path), overrides) if "custom_instance_class" in dagster_config_dict: custom_instance_class_data = dagster_config_dict[ "custom_instance_class"] validate_custom_config = validate_config( configurable_class_schema(), custom_instance_class_data, ) if not validate_custom_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster custom class config at {}". format(config_filename), validate_custom_config.errors, custom_instance_class_data, ) custom_instance_class = class_from_code_pointer( custom_instance_class_data["module"], custom_instance_class_data["class"]) schema = merge_dicts(dagster_instance_config_schema(), custom_instance_class.config_schema()) else: custom_instance_class = None schema = dagster_instance_config_schema() dagster_config = validate_config(schema, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster instance config at {}.".format( config_filename), dagster_config.errors, dagster_config_dict, ) return (dagster_config.value, custom_instance_class)
def custom_instance_class(self): return ( class_from_code_pointer( self.custom_instance_class_data.module_name, self.custom_instance_class_data.class_name, ) if self.custom_instance_class_data else None )
def dagster_instance_config( base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None, ): check.str_param(base_dir, "base_dir") check.invariant(os.path.isdir(base_dir), "base_dir should be a directory") overrides = check.opt_dict_param(overrides, "overrides") config_yaml_path = os.path.join(base_dir, config_filename) if not os.path.exists(config_yaml_path) and is_dagster_home_set(): warnings.warn( f"No dagster instance configuration file ({config_filename}) found at " f"{base_dir}. Defaulting to loading and storing all metadata with {base_dir}. " f"If this is the desired behavior, create an empty {config_filename} file in {base_dir}." ) dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path), overrides) if "instance_class" in dagster_config_dict: custom_instance_class_data = dagster_config_dict["instance_class"] validate_custom_config = validate_config( configurable_class_schema(), custom_instance_class_data, ) if not validate_custom_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster custom class config at {}". format(config_filename), validate_custom_config.errors, custom_instance_class_data, ) custom_instance_class = class_from_code_pointer( custom_instance_class_data["module"], custom_instance_class_data["class"]) schema = merge_dicts(dagster_instance_config_schema(), custom_instance_class.config_schema()) else: custom_instance_class = None schema = dagster_instance_config_schema() dagster_config = validate_config(schema, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster instance config at {}.".format( config_filename), dagster_config.errors, dagster_config_dict, ) return (dagster_config.value, custom_instance_class)