def setup_logger(self) -> None:
"""
Construct a logger for a command line run
"""
from .utils import configure_logger, get_mfa_version
current_version = get_mfa_version()
# Remove previous directory if versions are different
if os.path.exists(self.worker_config_path):
conf = load_configuration(self.worker_config_path)
if conf.get("version", current_version) != current_version:
self.clean = True
if self.clean:
shutil.rmtree(self.output_directory, ignore_errors=True)
os.makedirs(self.workflow_directory, exist_ok=True)
logger = configure_logger(
self.identifier, log_file=self.log_file, quiet=self.quiet, verbose=self.verbose
)
logger.debug(
f"Beginning run for {self.workflow_identifier} on {self.data_source_identifier}"
)
if self.use_mp:
logger.debug(f"Using multiprocessing with {self.num_jobs}")
else:
logger.debug(f"NOT using multiprocessing with {self.num_jobs}")
logger.debug(f"Set up logger for MFA version: {current_version}")
if self.clean:
logger.debug("Cleaned previous run")
def parse_parameters(
cls,
config_path: Optional[str] = None,
args: Optional[Namespace] = None,
unknown_args: Optional[List[str]] = None,
) -> MetaDict:
"""
Parse configuration parameters from a config file and command line arguments
Parameters
----------
config_path: str, optional
Path to yaml configuration file
args: :class:`~argparse.Namespace`, optional
Arguments parsed by argparse
unknown_args: list[str], optional
List of unknown arguments from argparse
Returns
-------
dict[str, Any]
Dictionary of specified configuration parameters
"""
global_params = {}
if config_path and os.path.exists(config_path):
data = load_configuration(config_path)
for k, v in data.items():
if v is None and k in cls.nullable_fields:
v = []
global_params[k] = v
global_params.update(cls.parse_args(args, unknown_args))
return global_params
def parse_parameters(
cls,
config_path: Optional[str] = None,
args: Optional[Namespace] = None,
unknown_args: Optional[List[str]] = None,
) -> MetaDict:
"""
Parse configuration parameters from a config file and command line arguments
Parameters
----------
config_path: str, optional
Path to yaml configuration file
args: :class:`~argparse.Namespace`, optional
Arguments parsed by argparse
unknown_args: list[str], optional
List of unknown arguments from argparse
Returns
-------
dict[str, Any]
Dictionary of specified configuration parameters
"""
global_params = {}
training_params = []
use_default = True
if config_path is not None:
data = load_configuration(config_path)
training_params = []
for k, v in data.items():
if k == "training":
for t in v:
for k2, v2 in t.items():
if "features" in v2:
global_params.update(v2["features"])
del v2["features"]
training_params.append((k2, v2))
elif k == "features":
if "type" in v:
v["feature_type"] = v["type"]
del v["type"]
global_params.update(v)
else:
if v is None and k in cls.nullable_fields:
v = []
global_params[k] = v
if training_params:
use_default = False
if use_default: # default training configuration
training_params.append(("dubm", {}))
# training_params.append(("ubm", {}))
training_params.append(("ivector", {}))
if training_params:
if training_params[0][0] != "dubm":
raise ConfigError("The first round of training must be dubm.")
global_params["training_configuration"] = training_params
global_params.update(cls.parse_args(args, unknown_args))
return global_params
def check_previous_run(self) -> bool:
"""
Check whether a previous run has any conflicting settings with the current run.
Returns
-------
bool
Flag for whether the current run is compatible with the previous one
"""
if not os.path.exists(self.worker_config_path):
return True
conf = load_configuration(self.worker_config_path)
clean = self._validate_previous_configuration(conf)
if not clean:
self.log_warning(
"The previous run had a different configuration than the current, which may cause issues."
" Please see the log for details or use --clean flag if issues are encountered."
)
return clean
def parse_parameters(
cls,
config_path: Optional[str] = None,
args: Optional[Namespace] = None,
unknown_args: Optional[List[str]] = None,
) -> MetaDict:
"""
Parse parameters for segmentation from a config path or command-line arguments
Parameters
----------
config_path: str
Config path
args: :class:`~argparse.Namespace`
Command-line arguments from argparse
unknown_args: list[str], optional
Extra command-line arguments
Returns
-------
dict[str, Any]
Configuration parameters
"""
global_params = {}
if config_path and os.path.exists(config_path):
data = load_configuration(config_path)
for k, v in data.items():
if k == "features":
if "type" in v:
v["feature_type"] = v["type"]
del v["type"]
global_params.update(v)
else:
if v is None and k in cls.nullable_fields:
v = []
global_params[k] = v
global_params.update(cls.parse_args(args, unknown_args))
return global_params