def generate_citrine_features(self, save_to_csv=True):
logging.info(
'WARNING: You have specified generation of features from Citrine. Based on which materials you are'
'interested in, there may be many records to parse through, thus this routine may take a long time to complete!'
)
compositions = self.dataframe['Material compositions'].tolist()
citrine_dict_property_min = dict()
citrine_dict_property_max = dict()
citrine_dict_property_avg = dict()
for composition in compositions:
pifquery = self._get_pifquery(composition=composition)
property_name_list, property_value_list = self._get_pifquery_property_list(
pifquery=pifquery)
property_names_unique, parsed_property_min, parsed_property_max, parsed_property_avg = self._parse_pifquery_property_list(
property_name_list=property_name_list,
property_value_list=property_value_list)
citrine_dict_property_min[composition] = parsed_property_min
citrine_dict_property_max[composition] = parsed_property_max
citrine_dict_property_avg[composition] = parsed_property_avg
dataframe = self.dataframe
citrine_dict_list = [
citrine_dict_property_min, citrine_dict_property_max,
citrine_dict_property_avg
]
for citrine_dict in citrine_dict_list:
dataframe_citrine = pd.DataFrame.from_dict(data=citrine_dict,
orient='index')
# Need to reorder compositions in new dataframe to match input dataframe
dataframe_citrine = dataframe_citrine.reindex(
self.dataframe['Material compositions'].tolist())
# Need to make compositions the first column, instead of the row names
dataframe_citrine.index.name = 'Material compositions'
dataframe_citrine.reset_index(inplace=True)
# Need to delete duplicate column before merging dataframes
del dataframe_citrine['Material compositions']
# Merge magpie feature dataframe with originally supplied dataframe
dataframe = DataframeUtilities().merge_dataframe_columns(
dataframe1=dataframe, dataframe2=dataframe_citrine)
if save_to_csv == bool(True):
# Get y_feature in this dataframe, attach it to save path
for column in dataframe.columns.values:
if column in self.configdict['General Setup'][
'target_feature']:
filetag = column
dataframe.to_csv(self.configdict['General Setup']['save_path'] +
"/" + 'input_with_citrine_features' + '_' +
str(filetag) + '.csv',
index=False)
return dataframe
def generate_materialsproject_features(self, save_to_csv=True):
try:
compositions = self.dataframe['Material compositions']
except KeyError:
print(
'No column called "Material compositions" exists in the supplied dataframe.'
)
sys.exit()
mpdata_dict_composition = {}
for composition in compositions:
composition_data_mp = self._get_data_from_materials_project(
composition=composition)
mpdata_dict_composition[composition] = composition_data_mp
dataframe = self.dataframe
dataframe_mp = pd.DataFrame.from_dict(data=mpdata_dict_composition,
orient='index')
# Need to reorder compositions in new dataframe to match input dataframe
dataframe_mp = dataframe_mp.reindex(
self.dataframe['Material compositions'].tolist())
# Need to make compositions the first column, instead of the row names
dataframe_mp.index.name = 'Material compositions'
dataframe_mp.reset_index(inplace=True)
# Need to delete duplicate column before merging dataframes
del dataframe_mp['Material compositions']
# Merge magpie feature dataframe with originally supplied dataframe
dataframe = DataframeUtilities().merge_dataframe_columns(
dataframe1=dataframe, dataframe2=dataframe_mp)
if save_to_csv == bool(True):
# Get y_feature in this dataframe, attach it to save path
for column in dataframe.columns.values:
if column in self.configdict['General Setup'][
'target_feature']:
filetag = column
dataframe.to_csv(self.configdict['General Setup']['save_path'] +
"/" + 'input_with_matproj_features' + '_' +
str(filetag) + '.csv',
index=False)
return dataframe
def generate_magpie_features(self, save_to_csv=True):
compositions = []
composition_components = []
# Replace empty composition fields with empty string instead of NaN
self.dataframe = self.dataframe.fillna('')
for column in self.dataframe.columns:
if 'Material composition' in column:
composition_components.append(self.dataframe[column].tolist())
if len(composition_components) < 1:
logging.info(
'ERROR: No column with "Material composition xx" was found in the supplied dataframe'
)
sys.exit()
row = 0
while row < len(composition_components[0]):
composition = ''
for composition_component in composition_components:
composition += str(composition_component[row])
compositions.append(composition)
row += 1
# Add the column of combined material compositions into the dataframe
self.dataframe['Material compositions'] = compositions
# Assign each magpiedata feature set to appropriate composition name
magpiedata_dict_composition_average = {}
magpiedata_dict_arithmetic_average = {}
magpiedata_dict_max = {}
magpiedata_dict_min = {}
magpiedata_dict_difference = {}
magpiedata_dict_atomic_bysite = {}
for composition in compositions:
magpiedata_composition_average, magpiedata_arithmetic_average, magpiedata_max, magpiedata_min, magpiedata_difference = self._get_computed_magpie_features(
composition=composition)
magpiedata_atomic_notparsed = self._get_atomic_magpie_features(
composition=composition)
magpiedata_dict_composition_average[
composition] = magpiedata_composition_average
magpiedata_dict_arithmetic_average[
composition] = magpiedata_arithmetic_average
magpiedata_dict_max[composition] = magpiedata_max
magpiedata_dict_min[composition] = magpiedata_min
magpiedata_dict_difference[composition] = magpiedata_difference
# Add site-specific elemental features
count = 1
magpiedata_atomic_bysite = {}
for entry in magpiedata_atomic_notparsed.keys():
for magpiefeature, featurevalue in magpiedata_atomic_notparsed[
entry].items():
magpiedata_atomic_bysite["Site" + str(count) + "_" +
str(magpiefeature)] = featurevalue
count += 1
magpiedata_dict_atomic_bysite[
composition] = magpiedata_atomic_bysite
magpiedata_dict_list = [
magpiedata_dict_composition_average,
magpiedata_dict_arithmetic_average, magpiedata_dict_max,
magpiedata_dict_min, magpiedata_dict_difference,
magpiedata_dict_atomic_bysite
]
dataframe = self.dataframe
for magpiedata_dict in magpiedata_dict_list:
dataframe_magpie = pd.DataFrame.from_dict(data=magpiedata_dict,
orient='index')
# Need to reorder compositions in new dataframe to match input dataframe
dataframe_magpie = dataframe_magpie.reindex(
self.dataframe['Material compositions'].tolist())
# Need to make compositions the first column, instead of the row names
dataframe_magpie.index.name = 'Material compositions'
dataframe_magpie.reset_index(inplace=True)
# Need to delete duplicate column before merging dataframes
del dataframe_magpie['Material compositions']
# Merge magpie feature dataframe with originally supplied dataframe
dataframe = DataframeUtilities().merge_dataframe_columns(
dataframe1=dataframe, dataframe2=dataframe_magpie)
if save_to_csv == bool(True):
# Get y_feature in this dataframe, attach it to save path
for column in dataframe.columns.values:
if column in self.configdict['General Setup'][
'target_feature']:
filetag = column
fname = self.configdict['General Setup'][
'save_path'] + "/" + 'input_with_magpie_features' + '_' + str(
filetag) + '.csv'
dataframe.to_csv(fname, index=False)
return dataframe
def normalize_features(self,
x_features,
y_feature,
normalize_x_features,
normalize_y_feature,
to_csv=True):
if normalize_x_features == bool(True) and normalize_y_feature == bool(
False):
scaler = StandardScaler().fit(X=self.dataframe[x_features])
array_normalized = scaler.fit_transform(
X=self.dataframe[x_features])
array_normalized = DataframeUtilities().concatenate_arrays(
X_array=array_normalized,
y_array=np.asarray(self.dataframe[y_feature]).reshape([-1, 1]))
elif normalize_x_features == bool(
False) and normalize_y_feature == bool(True):
scaler = StandardScaler().fit(
X=np.asarray(self.dataframe[y_feature]).reshape([-1, 1]))
array_normalized = scaler.fit_transform(
X=np.asarray(self.dataframe[y_feature]).reshape([-1, 1]))
array_normalized = DataframeUtilities().concatenate_arrays(
X_array=np.asarray(self.dataframe[x_features]),
y_array=array_normalized.reshape([-1, 1]))
elif normalize_x_features == bool(
True) and normalize_y_feature == bool(True):
scaler_x = StandardScaler().fit(X=self.dataframe[x_features])
scaler_y = StandardScaler().fit(
X=np.asarray(self.dataframe[y_feature]).reshape([-1, 1]))
array_normalized_x = scaler_x.fit_transform(
X=self.dataframe[x_features])
array_normalized_y = scaler_y.fit_transform(
X=np.asarray(self.dataframe[y_feature]).reshape([-1, 1]))
array_normalized = DataframeUtilities().concatenate_arrays(
X_array=array_normalized_x, y_array=array_normalized_y)
else:
"You must specify to normalize either x_features, y_feature, or both, or set perform_feature_normalization=False in the input file"
sys.exit()
dataframe_normalized = DataframeUtilities().array_to_dataframe(
array=array_normalized)
dataframe_normalized = DataframeUtilities().assign_columns_as_features(
dataframe=dataframe_normalized,
x_features=x_features,
y_feature=y_feature,
remove_first_row=False)
if to_csv == True:
# Need configdict to get save path
#configdict = ConfigFileParser(configfile=sys.argv[1]).get_config_dict(path_to_file=os.getcwd())
# Get y_feature in this dataframe, attach it to save path
for column in dataframe_normalized.columns.values:
if column in self.configdict['General Setup'][
'target_feature']:
filetag = column
dataframe_normalized.to_csv(
self.configdict['General Setup']['save_path'] + "/" +
'input_data_normalized' + '_' + str(filetag) + '.csv',
index=False)
if not (normalize_x_features == bool(True)
and normalize_y_feature == bool(True)):
return dataframe_normalized, scaler
else:
return dataframe_normalized, (scaler_x, scaler_y)