def test_load_dataframe_from_json(self):
df = load_dataframe_from_json(os.path.join(test_dir, 'dataframe.json'))
self.assertTrue(self.diamond == df['structure'][0],
"Dataframe contents do not match")
df = load_dataframe_from_json(os.path.join(test_dir, 'dataframe.json.gz'))
self.assertTrue(self.diamond == df['structure'][0],
"Dataframe contents do not match")
df = load_dataframe_from_json(os.path.join(test_dir, 'dataframe.json.bz2'))
self.assertTrue(self.diamond == df['structure'][0],
"Dataframe contents do not match")
def test_load_dataframe_from_json(self):
df = load_dataframe_from_json(os.path.join(test_dir, 'dataframe.json'))
self.assertTrue(self.diamond == df['structure'][0],
"Dataframe contents do not match")
df = load_dataframe_from_json(
os.path.join(test_dir, 'dataframe.json.gz'))
self.assertTrue(self.diamond == df['structure'][0],
"Dataframe contents do not match")
df = load_dataframe_from_json(
os.path.join(test_dir, 'dataframe.json.bz2'))
self.assertTrue(self.diamond == df['structure'][0],
"Dataframe contents do not match")
def run_task(self, fw_spec):
# Read data from fw_spec
pipe_config_dict = fw_spec["pipe_config"]
target = fw_spec["target"]
data_file = fw_spec["data_file"]
learner_name = pipe_config_dict["learner_name"]
learner_kwargs = pipe_config_dict["learner_kwargs"]
reducer_kwargs = pipe_config_dict["reducer_kwargs"]
cleaner_kwargs = pipe_config_dict["cleaner_kwargs"]
autofeaturizer_kwargs = pipe_config_dict["autofeaturizer_kwargs"]
# Modify data_file based on computing resource
data_dir = os.environ["AMM_DATASET_DIR"]
data_file = os.path.join(data_dir, data_file)
# Modify save_dir based on computing resource
bench_dir = os.environ["AMM_SINGLE_FIT_DIR"]
base_save_dir = fw_spec["base_save_dir"]
base_save_dir = os.path.join(bench_dir, base_save_dir)
if not os.path.exists(base_save_dir):
os.makedirs(base_save_dir)
# Set up pipeline config
if learner_name == "TPOTAdaptor":
learner = TPOTAdaptor(**learner_kwargs)
elif learner_name == "rf":
warnings.warn(
"Learner kwargs passed into RF regressor/classifiers bc. rf being used."
)
learner = SinglePipelineAdaptor(
regressor=RandomForestRegressor(**learner_kwargs),
classifier=RandomForestClassifier(**learner_kwargs),
)
else:
raise ValueError("{} not supported yet!" "".format(learner_name))
pipe_config = {
"learner": learner,
"reducer": FeatureReducer(**reducer_kwargs),
"cleaner": DataCleaner(**cleaner_kwargs),
"autofeaturizer": AutoFeaturizer(**autofeaturizer_kwargs),
}
pipe = MatPipe(**pipe_config)
# Set up dataset
# Dataset should already be set up correctly as json beforehand.
# this includes targets being converted to classification, removing
# extra columns, having the names of featurization cols set to the
# same as the matpipe config, etc.
df = load_dataframe_from_json(data_file)
pipe.fit(df, target)
pipe.save(os.path.join(base_save_dir, "pipe.p"))
def load_dataset(name, data_home=None, download_if_missing=True):
"""
Loads a dataframe containing the dataset specified with the 'name' field.
Dataset file is stored/loaded from data_home if specified, otherwise at
the MATMINER_DATA environment variable if set or at matminer/datasets
by default.
Args:
name (str): keyword specifying what dataset to load, run
matminer.datasets.get_available_datasets() for options
data_home (str): path to folder to look for dataset file
download_if_missing (bool): whether to download the dataset if is not
found on disk
Returns: (pd.DataFrame,
tuple -> (pd.DataFrame, pd.DataFrame) if return_lumo = True)
"""
global _dataset_dict
if _dataset_dict is None:
_dataset_dict = _load_dataset_dict()
if name not in _dataset_dict:
error_string = "Unrecognized dataset name: {}. \n" \
"Use matminer.datasets.get_available_datasets() " \
"to see a list of currently available " \
"datasets".format(name)
# Very simple attempt to match unrecognized keyword to existing
# dataset names in an attempt to give the user immediate feedback
possible_matches = [
x for x in _dataset_dict.keys() if name.lower() in x.lower()
]
if possible_matches:
error_string += "\nCould you have been looking for these similar " \
"matches?:\n{}".format(possible_matches)
raise ValueError(error_string)
dataset_metadata = _dataset_dict[name]
data_path = os.path.join(_get_data_home(data_home),
name + "." + dataset_metadata['file_type'])
_validate_dataset(data_path, dataset_metadata['url'],
dataset_metadata['hash'], download_if_missing)
df = load_dataframe_from_json(data_path)
return df
def load_dataset(name, data_home=None, download_if_missing=True):
"""
Loads a dataframe containing the dataset specified with the 'name' field.
Dataset file is stored/loaded from data_home if specified, otherwise at
the MATMINER_DATA environment variable if set or at matminer/datasets
by default.
Args:
name (str): keyword specifying what dataset to load, run
matminer.datasets.get_available_datasets() for options
data_home (str): path to folder to look for dataset file
download_if_missing (bool): whether to download the dataset if is not
found on disk
Returns: (pd.DataFrame,
tuple -> (pd.DataFrame, pd.DataFrame) if return_lumo = True)
"""
global _dataset_dict
if _dataset_dict is None:
_dataset_dict = _load_dataset_dict()
if name not in _dataset_dict:
error_string = "Unrecognized dataset name: {}. \n" \
"Use matminer.datasets.get_available_datasets() " \
"to see a list of currently available " \
"datasets".format(name)
# Very simple attempt to match unrecognized keyword to existing
# dataset names in an attempt to give the user immediate feedback
possible_matches = [
x for x in _dataset_dict.keys() if name.lower() in x.lower()
]
if possible_matches:
error_string += "\nCould you have been looking for these similar " \
"matches?:\n{}".format(possible_matches)
raise ValueError(error_string)
dataset_metadata = _dataset_dict[name]
data_path = os.path.join(_get_data_home(data_home),
name + "." + dataset_metadata['file_type'])
_validate_dataset(data_path, dataset_metadata['url'],
dataset_metadata['hash'], download_if_missing)
df = load_dataframe_from_json(data_path)
return df
def __init__(self,
initial_ltol: float = 0.2,
initial_stol: float = 0.3,
initial_angle_tol: float = 5.,
use_fingerprint_matching: bool = True,
fingerprint_distance_cutoff: float = 0.4):
db_file = resource_filename('robocrys.condense', 'mineral_db.json.gz')
self.mineral_db = load_dataframe_from_json(db_file)
self.initial_ltol = initial_ltol
self.initial_stol = initial_stol
self.initial_angle_tol = initial_angle_tol
self.fingerprint_distance_cutoff = fingerprint_distance_cutoff
self.use_fingerprint_matching = use_fingerprint_matching
self._structure = None
self._mineral_db = None
# -- start
import os
import pandas as pd
import numpy as np
import keras
import matminer
import pickle
from keras.models import load_model
from matminer.utils.io import load_dataframe_from_json
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from supplement import searchNN, trainNN, evaluateModel, createRF #stores custom methods/functions
fdf = load_dataframe_from_json('Batteries_feat.json')
print("The starting dataset has {}".format(fdf.shape))
print(fdf.head())
'''
Block 1 - Random Forest
'''
print('\n---\n')
print('Selecting target variable...')
excluded = [
'Spacegroup', 'Capacity Grav', 'Capacity Vol', 'Specific E Wh/kg',
'E Density Wh/l', 'Stability Charge', 'Stability Discharge', 'Ion',
'Reduced Formula', 'Id', 'composition', 'composition_oxid',
'HOMO_character', 'HOMO_element', 'LUMO_character', 'LUMO_element'
]
def load_dataset(name,
data_home=None,
download_if_missing=True,
include_metadata=False):
"""
Loads a dataframe containing the dataset specified with the 'name' field.
Dataset file is stored/loaded from data_home if specified, otherwise at
the MATMINER_DATA environment variable if set or at matminer/datasets
by default.
Args:
name (str): keyword specifying what dataset to load, run
matminer.datasets.available_datasets() for options
data_home (str): path to folder to look for dataset file
download_if_missing (bool): whether to download the dataset if is not
found on disk
include_metadata (bool): optional argument for some datasets with
metadata fields
Returns: (pd.DataFrame)
"""
dataset_dict = _load_dataset_dict()
if name not in dataset_dict:
error_string = "Unrecognized dataset name: {}. \n" \
"Use matminer.datasets.available_datasets() " \
"to see a list of currently available " \
"datasets".format(name)
# Very simple attempt to match unrecognized keyword to existing
# dataset names in an attempt to give the user immediate feedback
possible_matches = [
x for x in dataset_dict.keys() if name.lower() in x.lower()
]
if possible_matches:
error_string += "\nCould you have been looking for these similar " \
"matches?:\n{}".format(possible_matches)
raise ValueError(error_string)
dataset_metadata = dataset_dict[name]
data_path = os.path.join(_get_data_home(data_home),
name + "." + dataset_metadata['file_type'])
_validate_dataset(data_path, dataset_metadata['url'],
dataset_metadata['hash'], download_if_missing)
df = load_dataframe_from_json(data_path)
if not include_metadata:
if name == "elastic_tensor_2015":
df = df.drop(['cif', 'kpoint_density', 'poscar'], axis=1)
elif name in {"piezoelectric_tensor", "dielectric_constant"}:
df = df.drop(['cif', 'meta', 'poscar'], axis=1)
return df
from sklearn.dummy import DummyClassifier, DummyRegressor
from sklearn.model_selection import cross_val_score, KFold, StratifiedKFold
from automatminer.utils.ml import regression_or_classification
from automatminer.utils.ml import AMM_CLF_NAME, AMM_REG_NAME
from automatminer_dev.config import BENCHMARK_FULL_SET, GLASS, EXPT_IS_METAL, EXPT_GAP
from matminer.utils.io import load_dataframe_from_json
benchmark_dir = os.environ["AMM_DATASET_DIR"]
bmarks = BENCHMARK_FULL_SET
bmarks = [GLASS, EXPT_GAP, EXPT_IS_METAL]
for p in bmarks:
pname = p["name"]
print("Loading {}".format(pname))
df = load_dataframe_from_json(os.path.join(benchmark_dir, p["data_file"]))
target = p["target"]
ltype = p["problem_type"]
if ltype == AMM_REG_NAME:
kf = KFold(n_splits=5, random_state=18012019, shuffle=True)
estimator = DummyRegressor(strategy="mean")
scoring = "neg_mean_absolute_error"
multiplier = -1
elif ltype == AMM_CLF_NAME:
kf = StratifiedKFold(n_splits=5, random_state=18012019, shuffle=True)
estimator = DummyClassifier(strategy="stratified")
multiplier = 1
scoring = "roc_auc"
else:
raise ValueError("problem type {} is not known.".format(ltype))
import pandas as pd
import numpy as np
import os
from pymatgen import MPRester
import matminer
from matminer.data_retrieval.retrieve_MP import MPDataRetrieval
from matminer.utils.io import store_dataframe_as_json
from matminer.utils.io import load_dataframe_from_json
from matminer.figrecipes.plot import PlotlyFig
'''
#Block 1 - Loading and filtering the experimental dataframe
'''
df = load_dataframe_from_json('data/Batteries_raw.json')
# Select the working ion among {Li, Al, Zr, Mg}
select = 'Li'
# Initial filter based on the selected element
from matminer.featurizers.conversions import StrToComposition
fdf = StrToComposition().featurize_dataframe(df, 'Ion')
select_at = fdf["composition"].apply(lambda x: x.get_atomic_fraction(select))
fdf = fdf[select_at == 1]
# Debug
print("Remaining samples: {}".format(fdf.describe))
fdf = fdf.drop(['composition'], axis=1)
def test_featurize_bsdos(self, refresh_df_init=False, limit=1):
"""
Tests featurize_dos and featurize_bandstructure.
Args:
refresh_df_init (bool): for developers, if the test need to be
updated set to True. Otherwise set to False to make the final
test independent of MPRester and faster.
limit (int): the maximum final number of entries.
Returns (None):
"""
target = "color"
df_bsdos_pickled = "mp_data_with_dos_bandstructure.pickle"
save_path = os.path.join(TEST_DIR, df_bsdos_pickled)
if refresh_df_init:
mpdr = MPDataRetrieval()
df = mpdr.get_dataframe(
criteria={"material_id": "mp-149"},
properties=[
"pretty_formula",
"dos",
"bandstructure",
"bandstructure_uniform",
],
)
store_dataframe_as_json(df, save_path)
else:
df = load_dataframe_from_json(save_path)
df = df.dropna(axis=0)
df = df.rename(
columns={
"bandstructure_uniform": "bandstructure",
"bandstructure": "line bandstructure",
}
)
df[target] = [["red"]]
n_cols_init = df.shape[1]
featurizer = AutoFeaturizer(
preset="express", ignore_errors=False, multiindex=False
)
df = featurizer.fit_transform(df, target)
# sanity checks
self.assertTrue(len(df), limit)
self.assertGreater(len(df.columns), n_cols_init)
# DOSFeaturizer:
self.assertEqual(df["cbm_character_1"][0], "p")
# DopingFermi:
self.assertAlmostEqual(df["fermi_c1e+20T300"][0], -0.539, 3)
# Hybridization:
self.assertAlmostEqual(df["vbm_sp"][0], 0.181, 3)
self.assertAlmostEqual(df["cbm_s"][0], 0.4416, 3)
self.assertAlmostEqual(df["cbm_sp"][0], 0.9864, 3)
# BandFeaturizer:
self.assertAlmostEqual(df["direct_gap"][0], 2.556, 3)
self.assertAlmostEqual(df["n_ex1_norm"][0], 0.6285, 4)
# BranchPointEnergy:
self.assertAlmostEqual(df["branch_point_energy"][0], 5.7677, 4)
def transform(self, df, target, prevent_cache_overwrite=False):
"""
Decorate a dataframe containing composition, structure, bandstructure,
and/or DOS objects with descriptors.
Args:
df (pandas.DataFrame): The dataframe not containing features.
target (str): The ML-target property contained in the df.
prevent_cache_overwrite (bool): If True, does not try to write any
new features to the cache.
Returns:
df (pandas.DataFrame): Transformed dataframe containing features.
"""
if self.cache_src and os.path.exists(self.cache_src):
logger.debug(self._log_prefix +
"Reading cache_src {}".format(self.cache_src))
cached_df = load_dataframe_from_json(self.cache_src)
if not all([loc in cached_df.index for loc in df.index]):
raise AutomatminerError("Feature cache does not contain all "
"entries (by DataFrame index) needed "
"to transform the input df.")
else:
cached_subdf = cached_df.loc[df.index]
if target in cached_subdf.columns:
if target not in df.columns:
logger.warn(
self._log_prefix +
"Target not present in both cached df and input df."
" Cannot perform comparison to ensure index match."
)
else:
cached_targets = cached_subdf[target]
input_targets = df[target]
cached_type = regression_or_classification(
cached_targets)
input_type = regression_or_classification(
input_targets)
if cached_type != input_type:
raise AutomatminerError(
"Cached targets appear to be '{}' type, while "
"input targets appear to be '{}'."
"".format(cached_type, input_type))
problems = {}
for ix in input_targets.index:
iv = input_targets[ix]
cv = cached_targets[ix]
if iv != cv:
try:
if not math.isclose(iv, cv):
problems[ix] = [iv, cv]
except TypeError:
pass
if problems:
logger.warning(
self._log_prefix +
"Mismatch between cached targets and input "
"targets: \n{}".format(problems))
logger.info(self._log_prefix +
"Restored {} features on {} samples from "
"cache {}".format(len(cached_subdf.columns),
len(df.index), self.cache_src))
return cached_subdf
else:
transforming_on_fitted = df is self.fitted_input_df
df = self._prescreen_df(df, inplace=True)
if transforming_on_fitted:
df = self.converted_input_df
else:
df = self._add_composition_from_structure(df)
for featurizer_type, featurizers in self.featurizers.items():
if featurizer_type in df.columns:
if not transforming_on_fitted:
df = self._tidy_column(df, featurizer_type)
for f in featurizers:
logger.info(self._log_prefix + "Featurizing with {}."
"".format(f.__class__.__name__))
df = f.featurize_dataframe(
df,
featurizer_type,
ignore_errors=self.ignore_errors,
multiindex=self.multiindex,
inplace=False,
)
if self.drop_inputs:
df = df.drop(columns=[featurizer_type])
else:
logger.info(self._log_prefix +
"Featurizer type {} not in the dataframe. "
"Skipping...".format(featurizer_type))
if self.functionalize:
ff = FunctionFeaturizer()
ff.set_n_jobs(self.n_jobs)
cols = df.columns.tolist()
for ft in self.featurizers.keys():
if ft in cols:
cols.pop(ft)
df = ff.fit_featurize_dataframe(
df,
cols,
ignore_errors=self.ignore_errors,
multiindex=self.multiindex,
inplace=False,
)
if (self.cache_src and not os.path.exists(self.cache_src)
and not prevent_cache_overwrite):
store_dataframe_as_json(df, self.cache_src)
return df
# %% where there are several mp_ids, pick the one with lowest energy above convex hull
def get_e_above_hull(mp_id: str) -> float:
return mpr.query(mp_id, ["e_above_hull"])["e_above_hull"]
phonons["es_above_hull"] = phonons.likely_mp_ids.progress_apply(
lambda ids: [get_e_above_hull(id) for id in ids])
phonons["likely_mp_id"] = phonons.apply(
lambda row: row.likely_mp_ids[np.argmin(row.es_above_hull)], axis=1)
# %%
cols = ["structure", "last phdos peak", "likely_mp_id"]
store_dataframe_as_json(phonons[cols], "matbench-phonons-with-mp-id.json.gz")
phonons[cols] = load_dataframe_from_json("matbench-phonons-with-mp-id.json.gz")
# %%
phonons[["sg_symbol", "sg_number"]] = phonons.progress_apply(
lambda row: row.structure.get_space_group_info(),
axis=1,
result_type="expand")
phonons["crystal_system"] = phonons.structure.progress_apply(
lambda struct: SpacegroupAnalyzer(struct).get_crystal_system())
phonons[["sg_symbol", "sg_number", "crystal_system", "volume",
"formula"]].to_csv("additional-df-cols.csv", index=False)
# %%
phonons[["sg_symbol", "sg_number", "crystal_system", "volume",
def run_task(self, fw_spec):
# Read data from fw_spec
pipe_config_dict = fw_spec["pipe_config"]
fold = fw_spec["fold"]
kfold_config = fw_spec["kfold_config"]
target = fw_spec["target"]
data_file = fw_spec["data_file"]
clf_pos_label = fw_spec["clf_pos_label"]
problem_type = fw_spec["problem_type"]
learner_name = pipe_config_dict["learner_name"]
cache = fw_spec["cache"]
learner_kwargs = pipe_config_dict["learner_kwargs"]
reducer_kwargs = pipe_config_dict["reducer_kwargs"]
cleaner_kwargs = pipe_config_dict["cleaner_kwargs"]
autofeaturizer_kwargs = pipe_config_dict["autofeaturizer_kwargs"]
# Modify data_file based on computing resource
data_dir = os.environ["AMM_DATASET_DIR"]
data_file = os.path.join(data_dir, data_file)
# Modify save_dir based on computing resource
bench_dir = os.environ["AMM_BENCH_DIR"]
base_save_dir = fw_spec["base_save_dir"]
base_save_dir = os.path.join(bench_dir, base_save_dir)
save_dir = fw_spec.pop("save_dir")
save_dir = os.path.join(base_save_dir, save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# Set up pipeline config
if learner_name == "TPOTAdaptor":
learner = TPOTAdaptor(**learner_kwargs)
elif learner_name == "rf":
warnings.warn(
"Learner kwargs passed into RF regressor/classifiers bc. rf being used."
)
learner = SinglePipelineAdaptor(
regressor=RandomForestRegressor(**learner_kwargs),
classifier=RandomForestClassifier(**learner_kwargs),
)
else:
raise ValueError("{} not supported by RunPipe yet!"
"".format(learner_name))
if cache:
autofeaturizer_kwargs["cache_src"] = os.path.join(
base_save_dir, "features.json")
pipe_config = {
"learner": learner,
"reducer": FeatureReducer(**reducer_kwargs),
"cleaner": DataCleaner(**cleaner_kwargs),
"autofeaturizer": AutoFeaturizer(**autofeaturizer_kwargs),
}
pipe = MatPipe(**pipe_config)
# Set up dataset
# Dataset should already be set up correctly as json beforehand.
# this includes targets being converted to classification, removing
# extra columns, having the names of featurization cols set to the
# same as the matpipe config, etc.
df = load_dataframe_from_json(data_file)
# Check other parameters that would otherwise not be checked until after
# benchmarking, hopefully saves some errors at the end during scoring.
if problem_type not in [AMM_CLF_NAME, AMM_REG_NAME]:
raise ValueError("Problem must be either classification or "
"regression.")
elif problem_type == AMM_CLF_NAME:
if not isinstance(clf_pos_label, (str, bool)):
raise TypeError(
"The classification positive label should be a "
"string, or bool not {}."
"".format(type(clf_pos_label)))
elif clf_pos_label not in df[target]:
raise ValueError("The classification positive label should be"
"present in the target column.")
elif len(df[target].unique()) > 2:
raise ValueError("Only binary classification scoring available"
"at this time.")
# Set up testing scheme
if problem_type == AMM_REG_NAME:
kfold = KFold(**kfold_config)
else:
kfold = StratifiedKFold(**kfold_config)
if fold >= kfold.n_splits:
raise ValueError("{} is out of range for KFold with n_splits="
"{}".format(fold, kfold))
# Run the benchmark
t1 = time.time()
results = pipe.benchmark(df,
target,
kfold,
fold_subset=[fold],
cache=True)
result_df = results[0]
elapsed_time = time.time() - t1
# Save everything
pipe.save(os.path.join(save_dir, "pipe.p"))
pipe.inspect(filename=os.path.join(save_dir, "digest.txt"))
result_df.to_csv(os.path.join(save_dir, "test_df.csv"))
pipe.post_fit_df.to_csv(os.path.join(save_dir, "fitted_df.csv"))
# Evaluate model
true = result_df[target]
test = result_df[target + " predicted"]
pass_to_storage = {}
if problem_type == AMM_REG_NAME:
pass_to_storage["r2"] = r2_score(true, test)
pass_to_storage["mae"] = mean_absolute_error(true, test)
pass_to_storage["rmse"] = sqrt(mean_squared_error(true, test))
elif problem_type == AMM_CLF_NAME:
pass_to_storage["f1"] = f1_score(true,
test,
pos_label=clf_pos_label)
pass_to_storage["roc_auc"] = roc_auc_score(true, test)
pass_to_storage["accuracy"] = accuracy_score(true, test)
else:
raise ValueError("Scoring method for problem type {} not supported"
"".format(problem_type))
# Extract important inspect for storage
try:
# TPOT Adaptor
best_pipeline = [
str(step) for step in pipe.learner.best_pipeline.steps
]
except AttributeError:
best_pipeline = str(pipe.learner.best_pipeline)
features = pipe.learner.features
n_features = len(features)
fold_orig = list(kfold.split(df, y=df[target]))[fold]
n_samples_train_original = len(fold_orig[0])
n_samples_test_original = len(fold_orig[1])
pass_to_storage.update({
"target": target,
"best_pipeline": best_pipeline,
"elapsed_time": elapsed_time,
"features": features,
"n_features": n_features,
"n_test_samples_original": n_samples_test_original,
"n_train_samples_original": n_samples_train_original,
"n_train_samples": len(pipe.post_fit_df),
"n_test_samples": len(test),
"test_sample_frac_retained": len(test) / n_samples_test_original,
"completion_time": datetime.datetime.now(),
"base_save_dir": base_save_dir,
"save_dir": save_dir,
})
fw_spec.update(pass_to_storage)