def get_data(path: str, smiles_columns: Union[str, List[str]] = None, target_columns: List[str] = None, ignore_columns: List[str] = None, skip_invalid_smiles: bool = True, args: Union[TrainArgs, PredictArgs] = None, data_weights_path: str = None, features_path: List[str] = None, features_generator: List[str] = None, phase_features_path: str = None, atom_descriptors_path: str = None, bond_features_path: str = None, max_data_size: int = None, store_row: bool = False, logger: Logger = None, loss_function: str = None, skip_none_targets: bool = False) -> MoleculeDataset: """ Gets SMILES and target values from a CSV file. :param path: Path to a CSV file. :param smiles_columns: The names of the columns containing SMILES. By default, uses the first :code:`number_of_molecules` columns. :param target_columns: Name of the columns containing target values. By default, uses all columns except the :code:`smiles_column` and the :code:`ignore_columns`. :param ignore_columns: Name of the columns to ignore when :code:`target_columns` is not provided. :param skip_invalid_smiles: Whether to skip and filter out invalid smiles using :func:`filter_invalid_smiles`. :param args: Arguments, either :class:`~chemprop.args.TrainArgs` or :class:`~chemprop.args.PredictArgs`. :param data_weights_path: A path to a file containing weights for each molecule in the loss function. :param features_path: A list of paths to files containing features. If provided, it is used in place of :code:`args.features_path`. :param features_generator: A list of features generators to use. If provided, it is used in place of :code:`args.features_generator`. :param phase_features_path: A path to a file containing phase features as applicable to spectra. :param atom_descriptors_path: The path to the file containing the custom atom descriptors. :param bond_features_path: The path to the file containing the custom bond features. :param max_data_size: The maximum number of data points to load. :param logger: A logger for recording output. :param store_row: Whether to store the raw CSV row in each :class:`~chemprop.data.data.MoleculeDatapoint`. :param skip_none_targets: Whether to skip targets that are all 'None'. This is mostly relevant when --target_columns are passed in, so only a subset of tasks are examined. :param loss_function: The loss function to be used in training. :return: A :class:`~chemprop.data.MoleculeDataset` containing SMILES and target values along with other info such as additional features when desired. """ debug = logger.debug if logger is not None else print if args is not None: # Prefer explicit function arguments but default to args if not provided smiles_columns = smiles_columns if smiles_columns is not None else args.smiles_columns target_columns = target_columns if target_columns is not None else args.target_columns ignore_columns = ignore_columns if ignore_columns is not None else args.ignore_columns features_path = features_path if features_path is not None else args.features_path features_generator = features_generator if features_generator is not None else args.features_generator phase_features_path = phase_features_path if phase_features_path is not None else args.phase_features_path atom_descriptors_path = atom_descriptors_path if atom_descriptors_path is not None \ else args.atom_descriptors_path bond_features_path = bond_features_path if bond_features_path is not None \ else args.bond_features_path max_data_size = max_data_size if max_data_size is not None else args.max_data_size loss_function = loss_function if loss_function is not None else args.loss_function if not isinstance(smiles_columns, list): smiles_columns = preprocess_smiles_columns(path=path, smiles_columns=smiles_columns) max_data_size = max_data_size or float('inf') # Load features if features_path is not None: features_data = [] for feat_path in features_path: features_data.append(load_features(feat_path)) # each is num_data x num_features features_data = np.concatenate(features_data, axis=1) else: features_data = None if phase_features_path is not None: phase_features = load_features(phase_features_path) for d_phase in phase_features: if not (d_phase.sum() == 1 and np.count_nonzero(d_phase) == 1): raise ValueError('Phase features must be one-hot encoded.') if features_data is not None: features_data = np.concatenate((features_data,phase_features), axis=1) else: # if there are no other molecular features, phase features become the only molecular features features_data = np.array(phase_features) else: phase_features = None # Load data weights if data_weights_path is not None: data_weights = get_data_weights(data_weights_path) else: data_weights = None # By default, the targets columns are all the columns except the SMILES column if target_columns is None: target_columns = get_task_names( path=path, smiles_columns=smiles_columns, target_columns=target_columns, ignore_columns=ignore_columns, ) # Find targets provided as inequalities if loss_function == 'bounded_mse': gt_targets, lt_targets = get_inequality_targets(path=path, target_columns=target_columns) else: gt_targets, lt_targets = None, None # Load data with open(path) as f: reader = csv.DictReader(f) all_smiles, all_targets, all_rows, all_features, all_phase_features, all_weights, all_gt, all_lt = [], [], [], [], [], [], [], [] for i, row in enumerate(tqdm(reader)): smiles = [row[c] for c in smiles_columns] targets = [] for column in target_columns: value = row[column] if value in ['','nan']: targets.append(None) elif '>' in value or '<' in value: if loss_function == 'bounded_mse': targets.append(float(value.strip('<>'))) else: raise ValueError('Inequality found in target data. To use inequality targets (> or <), the regression loss function bounded_mse must be used.') else: targets.append(float(value)) # Check whether all targets are None and skip if so if skip_none_targets and all(x is None for x in targets): continue all_smiles.append(smiles) all_targets.append(targets) if features_data is not None: all_features.append(features_data[i]) if phase_features is not None: all_phase_features.append(phase_features[i]) if data_weights is not None: all_weights.append(data_weights[i]) if gt_targets is not None: all_gt.append(gt_targets[i]) if lt_targets is not None: all_lt.append(lt_targets[i]) if store_row: all_rows.append(row) if len(all_smiles) >= max_data_size: break atom_features = None atom_descriptors = None if args is not None and args.atom_descriptors is not None: try: descriptors = load_valid_atom_or_bond_features(atom_descriptors_path, [x[0] for x in all_smiles]) except Exception as e: raise ValueError(f'Failed to load or validate custom atomic descriptors or features: {e}') if args.atom_descriptors == 'feature': atom_features = descriptors elif args.atom_descriptors == 'descriptor': atom_descriptors = descriptors bond_features = None if args is not None and args.bond_features_path is not None: try: bond_features = load_valid_atom_or_bond_features(bond_features_path, [x[0] for x in all_smiles]) except Exception as e: raise ValueError(f'Failed to load or validate custom bond features: {e}') data = MoleculeDataset([ MoleculeDatapoint( smiles=smiles, targets=targets, row=all_rows[i] if store_row else None, data_weight=all_weights[i] if data_weights is not None else None, gt_targets=all_gt[i] if gt_targets is not None else None, lt_targets=all_lt[i] if lt_targets is not None else None, features_generator=features_generator, features=all_features[i] if features_data is not None else None, phase_features=all_phase_features[i] if phase_features is not None else None, atom_features=atom_features[i] if atom_features is not None else None, atom_descriptors=atom_descriptors[i] if atom_descriptors is not None else None, bond_features=bond_features[i] if bond_features is not None else None, overwrite_default_atom_features=args.overwrite_default_atom_features if args is not None else False, overwrite_default_bond_features=args.overwrite_default_bond_features if args is not None else False ) for i, (smiles, targets) in tqdm(enumerate(zip(all_smiles, all_targets)), total=len(all_smiles)) ]) # Filter out invalid SMILES if skip_invalid_smiles: original_data_len = len(data) data = filter_invalid_smiles(data) if len(data) < original_data_len: debug(f'Warning: {original_data_len - len(data)} SMILES are invalid.') return data
def get_data( path: str, smiles_columns: Union[str, List[str]] = None, target_columns: List[str] = None, ignore_columns: List[str] = None, skip_invalid_smiles: bool = True, args: Union[TrainArgs, PredictArgs] = None, features_path: List[str] = None, features_generator: List[str] = None, atom_descriptors_path: str = None, bond_features_path: str = None, max_data_size: int = None, store_row: bool = False, logger: Logger = None, skip_none_targets: bool = False, ) -> MoleculeDataset: """ Gets SMILES and target values from a CSV file. :param path: Path to a CSV file. :param smiles_columns: The names of the columns containing SMILES. By default, uses the first :code:`number_of_molecules` columns. :param target_columns: Name of the columns containing target values. By default, uses all columns except the :code:`smiles_column` and the :code:`ignore_columns`. :param ignore_columns: Name of the columns to ignore when :code:`target_columns` is not provided. :param skip_invalid_smiles: Whether to skip and filter out invalid smiles using :func:`filter_invalid_smiles`. :param args: Arguments, either :class:`~chemprop.args.TrainArgs` or :class:`~chemprop.args.PredictArgs`. :param features_path: A list of paths to files containing features. If provided, it is used in place of :code:`args.features_path`. :param features_generator: A list of features generators to use. If provided, it is used in place of :code:`args.features_generator`. :param atom_descriptors_path: The path to the file containing the custom atom descriptors. :param bond_features_path: The path to the file containing the custom bond features. :param max_data_size: The maximum number of data points to load. :param logger: A logger for recording output. :param store_row: Whether to store the raw CSV row in each :class:`~chemprop.data.data.MoleculeDatapoint`. :param skip_none_targets: Whether to skip targets that are all 'None'. This is mostly relevant when --target_columns are passed in, so only a subset of tasks are examined. :return: A :class:`~chemprop.data.MoleculeDataset` containing SMILES and target values along with other info such as additional features when desired. """ debug = logger.debug if logger is not None else print if args is not None: # Prefer explicit function arguments but default to args if not provided smiles_columns = smiles_columns if smiles_columns is not None else args.smiles_columns target_columns = target_columns if target_columns is not None else args.target_columns ignore_columns = ignore_columns if ignore_columns is not None else args.ignore_columns features_path = features_path if features_path is not None else args.features_path features_generator = (features_generator if features_generator is not None else args.features_generator) atom_descriptors_path = (atom_descriptors_path if atom_descriptors_path is not None else args.atom_descriptors_path) bond_features_path = (bond_features_path if bond_features_path is not None else args.bond_features_path) max_data_size = max_data_size if max_data_size is not None else args.max_data_size if not isinstance(smiles_columns, list): smiles_columns = preprocess_smiles_columns( path=path, smiles_columns=smiles_columns) max_data_size = max_data_size or float("inf") # Load features if features_path is not None: features_data = [] for feat_path in features_path: features_data.append( load_features(feat_path)) # each is num_data x num_features features_data = np.concatenate(features_data, axis=1) else: features_data = None # Load data with open(path) as f: reader = csv.DictReader(f) # By default, the targets columns are all the columns except the SMILES column if target_columns is None: target_columns = get_task_names( path=path, smiles_columns=smiles_columns, target_columns=target_columns, ignore_columns=ignore_columns, ) all_smiles, all_targets, all_rows, all_features = [], [], [], [] for i, row in enumerate(tqdm(reader)): smiles = [row[c] for c in smiles_columns] targets = [ float(row[column]) if row[column] != "" else None for column in target_columns ] # Check whether all targets are None and skip if so if skip_none_targets and all(x is None for x in targets): continue all_smiles.append(smiles) all_targets.append(targets) if features_data is not None: all_features.append(features_data[i]) if store_row: all_rows.append(row) if len(all_smiles) >= max_data_size: break atom_features = None atom_descriptors = None if args is not None and args.atom_descriptors is not None: try: descriptors = load_valid_atom_or_bond_features( atom_descriptors_path, [x[0] for x in all_smiles]) except Exception as e: raise ValueError( f"Failed to load or validate custom atomic descriptors or features: {e}" ) if args.atom_descriptors == "feature": atom_features = descriptors elif args.atom_descriptors == "descriptor": atom_descriptors = descriptors bond_features = None if args is not None and args.bond_features_path is not None: try: bond_features = load_valid_atom_or_bond_features( bond_features_path, [x[0] for x in all_smiles]) except Exception as e: raise ValueError( f"Failed to load or validate custom bond features: {e}") data = MoleculeDataset([ MoleculeDatapoint( smiles=smiles, targets=targets, row=all_rows[i] if store_row else None, features_generator=features_generator, features=all_features[i] if features_data is not None else None, atom_features=atom_features[i] if atom_features is not None else None, atom_descriptors=atom_descriptors[i] if atom_descriptors is not None else None, bond_features=bond_features[i] if bond_features is not None else None, overwrite_default_atom_features=args. overwrite_default_atom_features if args is not None else False, overwrite_default_bond_features=args. overwrite_default_bond_features if args is not None else False, ) for i, (smiles, targets) in tqdm( enumerate(zip(all_smiles, all_targets)), total=len(all_smiles)) ]) # Filter out invalid SMILES if skip_invalid_smiles: original_data_len = len(data) data = filter_invalid_smiles(data) if len(data) < original_data_len: debug( f"Warning: {original_data_len - len(data)} SMILES are invalid." ) return data