def get_data(path: str,
smiles_columns: Union[str, List[str]] = None,
target_columns: List[str] = None,
ignore_columns: List[str] = None,
skip_invalid_smiles: bool = True,
args: Union[TrainArgs, PredictArgs] = None,
data_weights_path: str = None,
features_path: List[str] = None,
features_generator: List[str] = None,
phase_features_path: str = None,
atom_descriptors_path: str = None,
bond_features_path: str = None,
max_data_size: int = None,
store_row: bool = False,
logger: Logger = None,
loss_function: str = None,
skip_none_targets: bool = False) -> MoleculeDataset:
"""
Gets SMILES and target values from a CSV file.
:param path: Path to a CSV file.
:param smiles_columns: The names of the columns containing SMILES.
By default, uses the first :code:`number_of_molecules` columns.
:param target_columns: Name of the columns containing target values. By default, uses all columns
except the :code:`smiles_column` and the :code:`ignore_columns`.
:param ignore_columns: Name of the columns to ignore when :code:`target_columns` is not provided.
:param skip_invalid_smiles: Whether to skip and filter out invalid smiles using :func:`filter_invalid_smiles`.
:param args: Arguments, either :class:`~chemprop.args.TrainArgs` or :class:`~chemprop.args.PredictArgs`.
:param data_weights_path: A path to a file containing weights for each molecule in the loss function.
:param features_path: A list of paths to files containing features. If provided, it is used
in place of :code:`args.features_path`.
:param features_generator: A list of features generators to use. If provided, it is used
in place of :code:`args.features_generator`.
:param phase_features_path: A path to a file containing phase features as applicable to spectra.
:param atom_descriptors_path: The path to the file containing the custom atom descriptors.
:param bond_features_path: The path to the file containing the custom bond features.
:param max_data_size: The maximum number of data points to load.
:param logger: A logger for recording output.
:param store_row: Whether to store the raw CSV row in each :class:`~chemprop.data.data.MoleculeDatapoint`.
:param skip_none_targets: Whether to skip targets that are all 'None'. This is mostly relevant when --target_columns
are passed in, so only a subset of tasks are examined.
:param loss_function: The loss function to be used in training.
:return: A :class:`~chemprop.data.MoleculeDataset` containing SMILES and target values along
with other info such as additional features when desired.
"""
debug = logger.debug if logger is not None else print
if args is not None:
# Prefer explicit function arguments but default to args if not provided
smiles_columns = smiles_columns if smiles_columns is not None else args.smiles_columns
target_columns = target_columns if target_columns is not None else args.target_columns
ignore_columns = ignore_columns if ignore_columns is not None else args.ignore_columns
features_path = features_path if features_path is not None else args.features_path
features_generator = features_generator if features_generator is not None else args.features_generator
phase_features_path = phase_features_path if phase_features_path is not None else args.phase_features_path
atom_descriptors_path = atom_descriptors_path if atom_descriptors_path is not None \
else args.atom_descriptors_path
bond_features_path = bond_features_path if bond_features_path is not None \
else args.bond_features_path
max_data_size = max_data_size if max_data_size is not None else args.max_data_size
loss_function = loss_function if loss_function is not None else args.loss_function
if not isinstance(smiles_columns, list):
smiles_columns = preprocess_smiles_columns(path=path, smiles_columns=smiles_columns)
max_data_size = max_data_size or float('inf')
# Load features
if features_path is not None:
features_data = []
for feat_path in features_path:
features_data.append(load_features(feat_path)) # each is num_data x num_features
features_data = np.concatenate(features_data, axis=1)
else:
features_data = None
if phase_features_path is not None:
phase_features = load_features(phase_features_path)
for d_phase in phase_features:
if not (d_phase.sum() == 1 and np.count_nonzero(d_phase) == 1):
raise ValueError('Phase features must be one-hot encoded.')
if features_data is not None:
features_data = np.concatenate((features_data,phase_features), axis=1)
else: # if there are no other molecular features, phase features become the only molecular features
features_data = np.array(phase_features)
else:
phase_features = None
# Load data weights
if data_weights_path is not None:
data_weights = get_data_weights(data_weights_path)
else:
data_weights = None
# By default, the targets columns are all the columns except the SMILES column
if target_columns is None:
target_columns = get_task_names(
path=path,
smiles_columns=smiles_columns,
target_columns=target_columns,
ignore_columns=ignore_columns,
)
# Find targets provided as inequalities
if loss_function == 'bounded_mse':
gt_targets, lt_targets = get_inequality_targets(path=path, target_columns=target_columns)
else:
gt_targets, lt_targets = None, None
# Load data
with open(path) as f:
reader = csv.DictReader(f)
all_smiles, all_targets, all_rows, all_features, all_phase_features, all_weights, all_gt, all_lt = [], [], [], [], [], [], [], []
for i, row in enumerate(tqdm(reader)):
smiles = [row[c] for c in smiles_columns]
targets = []
for column in target_columns:
value = row[column]
if value in ['','nan']:
targets.append(None)
elif '>' in value or '<' in value:
if loss_function == 'bounded_mse':
targets.append(float(value.strip('<>')))
else:
raise ValueError('Inequality found in target data. To use inequality targets (> or <), the regression loss function bounded_mse must be used.')
else:
targets.append(float(value))
# Check whether all targets are None and skip if so
if skip_none_targets and all(x is None for x in targets):
continue
all_smiles.append(smiles)
all_targets.append(targets)
if features_data is not None:
all_features.append(features_data[i])
if phase_features is not None:
all_phase_features.append(phase_features[i])
if data_weights is not None:
all_weights.append(data_weights[i])
if gt_targets is not None:
all_gt.append(gt_targets[i])
if lt_targets is not None:
all_lt.append(lt_targets[i])
if store_row:
all_rows.append(row)
if len(all_smiles) >= max_data_size:
break
atom_features = None
atom_descriptors = None
if args is not None and args.atom_descriptors is not None:
try:
descriptors = load_valid_atom_or_bond_features(atom_descriptors_path, [x[0] for x in all_smiles])
except Exception as e:
raise ValueError(f'Failed to load or validate custom atomic descriptors or features: {e}')
if args.atom_descriptors == 'feature':
atom_features = descriptors
elif args.atom_descriptors == 'descriptor':
atom_descriptors = descriptors
bond_features = None
if args is not None and args.bond_features_path is not None:
try:
bond_features = load_valid_atom_or_bond_features(bond_features_path, [x[0] for x in all_smiles])
except Exception as e:
raise ValueError(f'Failed to load or validate custom bond features: {e}')
data = MoleculeDataset([
MoleculeDatapoint(
smiles=smiles,
targets=targets,
row=all_rows[i] if store_row else None,
data_weight=all_weights[i] if data_weights is not None else None,
gt_targets=all_gt[i] if gt_targets is not None else None,
lt_targets=all_lt[i] if lt_targets is not None else None,
features_generator=features_generator,
features=all_features[i] if features_data is not None else None,
phase_features=all_phase_features[i] if phase_features is not None else None,
atom_features=atom_features[i] if atom_features is not None else None,
atom_descriptors=atom_descriptors[i] if atom_descriptors is not None else None,
bond_features=bond_features[i] if bond_features is not None else None,
overwrite_default_atom_features=args.overwrite_default_atom_features if args is not None else False,
overwrite_default_bond_features=args.overwrite_default_bond_features if args is not None else False
) for i, (smiles, targets) in tqdm(enumerate(zip(all_smiles, all_targets)),
total=len(all_smiles))
])
# Filter out invalid SMILES
if skip_invalid_smiles:
original_data_len = len(data)
data = filter_invalid_smiles(data)
if len(data) < original_data_len:
debug(f'Warning: {original_data_len - len(data)} SMILES are invalid.')
return data
def get_data(
path: str,
smiles_columns: Union[str, List[str]] = None,
target_columns: List[str] = None,
ignore_columns: List[str] = None,
skip_invalid_smiles: bool = True,
args: Union[TrainArgs, PredictArgs] = None,
features_path: List[str] = None,
features_generator: List[str] = None,
atom_descriptors_path: str = None,
bond_features_path: str = None,
max_data_size: int = None,
store_row: bool = False,
logger: Logger = None,
skip_none_targets: bool = False,
) -> MoleculeDataset:
"""
Gets SMILES and target values from a CSV file.
:param path: Path to a CSV file.
:param smiles_columns: The names of the columns containing SMILES.
By default, uses the first :code:`number_of_molecules` columns.
:param target_columns: Name of the columns containing target values. By default, uses all columns
except the :code:`smiles_column` and the :code:`ignore_columns`.
:param ignore_columns: Name of the columns to ignore when :code:`target_columns` is not provided.
:param skip_invalid_smiles: Whether to skip and filter out invalid smiles using :func:`filter_invalid_smiles`.
:param args: Arguments, either :class:`~chemprop.args.TrainArgs` or :class:`~chemprop.args.PredictArgs`.
:param features_path: A list of paths to files containing features. If provided, it is used
in place of :code:`args.features_path`.
:param features_generator: A list of features generators to use. If provided, it is used
in place of :code:`args.features_generator`.
:param atom_descriptors_path: The path to the file containing the custom atom descriptors.
:param bond_features_path: The path to the file containing the custom bond features.
:param max_data_size: The maximum number of data points to load.
:param logger: A logger for recording output.
:param store_row: Whether to store the raw CSV row in each :class:`~chemprop.data.data.MoleculeDatapoint`.
:param skip_none_targets: Whether to skip targets that are all 'None'. This is mostly relevant when --target_columns
are passed in, so only a subset of tasks are examined.
:return: A :class:`~chemprop.data.MoleculeDataset` containing SMILES and target values along
with other info such as additional features when desired.
"""
debug = logger.debug if logger is not None else print
if args is not None:
# Prefer explicit function arguments but default to args if not provided
smiles_columns = smiles_columns if smiles_columns is not None else args.smiles_columns
target_columns = target_columns if target_columns is not None else args.target_columns
ignore_columns = ignore_columns if ignore_columns is not None else args.ignore_columns
features_path = features_path if features_path is not None else args.features_path
features_generator = (features_generator if features_generator
is not None else args.features_generator)
atom_descriptors_path = (atom_descriptors_path if atom_descriptors_path
is not None else args.atom_descriptors_path)
bond_features_path = (bond_features_path if bond_features_path
is not None else args.bond_features_path)
max_data_size = max_data_size if max_data_size is not None else args.max_data_size
if not isinstance(smiles_columns, list):
smiles_columns = preprocess_smiles_columns(
path=path, smiles_columns=smiles_columns)
max_data_size = max_data_size or float("inf")
# Load features
if features_path is not None:
features_data = []
for feat_path in features_path:
features_data.append(
load_features(feat_path)) # each is num_data x num_features
features_data = np.concatenate(features_data, axis=1)
else:
features_data = None
# Load data
with open(path) as f:
reader = csv.DictReader(f)
# By default, the targets columns are all the columns except the SMILES column
if target_columns is None:
target_columns = get_task_names(
path=path,
smiles_columns=smiles_columns,
target_columns=target_columns,
ignore_columns=ignore_columns,
)
all_smiles, all_targets, all_rows, all_features = [], [], [], []
for i, row in enumerate(tqdm(reader)):
smiles = [row[c] for c in smiles_columns]
targets = [
float(row[column]) if row[column] != "" else None
for column in target_columns
]
# Check whether all targets are None and skip if so
if skip_none_targets and all(x is None for x in targets):
continue
all_smiles.append(smiles)
all_targets.append(targets)
if features_data is not None:
all_features.append(features_data[i])
if store_row:
all_rows.append(row)
if len(all_smiles) >= max_data_size:
break
atom_features = None
atom_descriptors = None
if args is not None and args.atom_descriptors is not None:
try:
descriptors = load_valid_atom_or_bond_features(
atom_descriptors_path, [x[0] for x in all_smiles])
except Exception as e:
raise ValueError(
f"Failed to load or validate custom atomic descriptors or features: {e}"
)
if args.atom_descriptors == "feature":
atom_features = descriptors
elif args.atom_descriptors == "descriptor":
atom_descriptors = descriptors
bond_features = None
if args is not None and args.bond_features_path is not None:
try:
bond_features = load_valid_atom_or_bond_features(
bond_features_path, [x[0] for x in all_smiles])
except Exception as e:
raise ValueError(
f"Failed to load or validate custom bond features: {e}")
data = MoleculeDataset([
MoleculeDatapoint(
smiles=smiles,
targets=targets,
row=all_rows[i] if store_row else None,
features_generator=features_generator,
features=all_features[i]
if features_data is not None else None,
atom_features=atom_features[i]
if atom_features is not None else None,
atom_descriptors=atom_descriptors[i]
if atom_descriptors is not None else None,
bond_features=bond_features[i]
if bond_features is not None else None,
overwrite_default_atom_features=args.
overwrite_default_atom_features if args is not None else False,
overwrite_default_bond_features=args.
overwrite_default_bond_features if args is not None else False,
) for i, (smiles, targets) in tqdm(
enumerate(zip(all_smiles, all_targets)), total=len(all_smiles))
])
# Filter out invalid SMILES
if skip_invalid_smiles:
original_data_len = len(data)
data = filter_invalid_smiles(data)
if len(data) < original_data_len:
debug(
f"Warning: {original_data_len - len(data)} SMILES are invalid."
)
return data