def from_config(cls, config_path):
"""
Method creates cls instance from given config_path.
Args:
config_path: Path to file with clusterizer config.
Returns:
clusterizer: Object that can do clustering.
"""
clusterizer, _ = build_from_config_path(config_path)
return clusterizer
def from_config(cls, config_path):
"""
Method for creating dataset reader instance of 'cls' class.
Args:
config_path: Path to file with dataset reader config.
Returns:
dataset_reader: Instance of class 'cls'
"""
dataset_reader, _ = build_from_config_path(config_path)
return dataset_reader
def from_config(cls, config_path: str):
"""
Builds an object of a class that inherits from this class
using parameters described in a config file
Args:
config_path: path to .jsonnet config.
For example, see this config for LexSubEvaluation class:
"configs/evaluations/lexsub/semeval_all_elmo.jsonnet"
Returns: an object that was created with the parameters described in the given config
"""
evaluation_object, _ = build_from_config_path(config_path)
return evaluation_object
def from_config(cls, config_path):
"""
Create substitute generator from configuration.
Args:
config_path: path to configuration file.
Returns:
object of the SubstituteGenerator class.
"""
subst_generator, _ = build_from_config_path(config_path)
return subst_generator
def evaluate(
self,
config_path: str = None,
config: Optional[Dict] = None,
additional_modules: Optional[List[str]] = None,
experiment_name: Optional[str] = None,
run_name: Optional[str] = None,
) -> NoReturn:
"""
Evaluates task defined by configuration file.
Args:
config_path: path to a configuration file.
config: configuration of a task.
additional_modules: path to directories with modules that should be registered in global Registry.
experiment_name: results of the run will be added to 'experiment_name' experiment in MLflow.
run_name: this run will be marked as 'run_name' in MLflow.
"""
# Instantiate objects from config
task, config = build_from_config_path(config_path, config)
self.import_additional_modules(additional_modules)
# Create experiment with given name or get already existing
if experiment_name is None:
experiment_name = config["class_name"]
experiment_id = get_experiment_id(self.mlflow_client, experiment_name)
# Add Run name in MLFlow tags
tags = copy(self.git_tags)
if config_path is not None and run_name is None:
run_name = Path(config_path).stem
if run_name is not None:
tags[MLFLOW_RUN_NAME] = run_name
# Create Run entity for tracking
run_entity = self.mlflow_client.create_run(experiment_id=experiment_id,
tags=tags)
saved_params = dict()
generator_params = config["substitute_generator"]
log_params(self.mlflow_client, run_entity, generator_params,
saved_params)
dump_json(self.run_dir / "config.json", config)
logger.info("Evaluating...")
metrics = task.evaluate(run_dir=self.run_dir)
metrics = metrics["mean_metrics"]
log_metrics(self.mlflow_client, run_entity, metrics)
self.mlflow_client.log_artifacts(run_entity.info.run_uuid,
local_dir=self.run_dir)
logger.info("Evaluation performed.")
def augment(self,
dataset_name: str,
config_path: str = None,
config: Optional[Dict] = None):
"""
Performs dataset augmentation.
Args:
dataset_name: name of the dataset to augment
config_path: path to a configuration file.
config: configuration of a task
"""
augmenter, config = build_from_config_path(config_path, config)
dump_json(self.run_dir / "config.json", config)
logger.info(f"Augmenting {dataset_name}...")
augmented_dataset = augmenter.augment_dataset(
dataset_name=dataset_name)
augmented_dataset.to_csv(self.run_dir / "augmented_dataset.tsv",
sep="\t",
index=False)
logger.info(
f"Augmentation performed. Results was saved in {self.run_dir}")
def test_loading_all_configs(config_path: str):
print(f"Trying to load '{config_path}' config")
build_from_config_path(str(config_path))