def test_stats_in_parallel():
from pymks.bases import PrimitiveBasis
from pymks.stats import correlate
from pymks.datasets import make_microstructure
p_basis = PrimitiveBasis(5)
X = make_microstructure(n_samples=5, n_phases=3)
X_corr_actual = correlate(X, p_basis)
for i in range(1, 4):
X_corr_test = correlate(X, p_basis, n_jobs=i)
assert np.allclose(X_corr_actual, X_corr_test)
def test_stats_in_parallel():
import time
from pymks.bases import PrimitiveBasis
from pymks.stats import correlate
from pymks.datasets import make_microstructure
p_basis = PrimitiveBasis(5)
if p_basis._pyfftw:
X = make_microstructure(n_samples=5, n_phases=3)
t = []
for i in range(1, 4):
t_start = time.time()
correlate(X, p_basis, n_jobs=i)
t.append(time.time() - t_start)
assert t == sorted(t, reverse=True)
else:
pass
def test_nonperiodic_correlate():
'''
test corrleate for non-periodic microstructures
'''
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
X = np.array([[[0, 0, 1, 0],
[0, 0, 1, 0],
[0, 0, 1, 0],
[0, 0, 0, 0],
[0, 0, 1, 0]],
[[0, 1, 0, 0],
[0, 1, 0, 0],
[0, 1, 0, 0],
[0, 0, 0, 0],
[0, 1, 0, 0]]])
basis = DiscreteIndicatorBasis(n_states=2)
X_corr = correlate(X, basis)
X_result = [[2 / 3., 4 / 9., 0.75, 4 / 9.],
[5 / 8., 0.5, 0.75, 0.5],
[0.6, 7 / 15., 0.8, 7 / 15.],
[5 / 8., 0.5, 0.75, 0.5],
[0.5, 4 / 9., 0.75, 4 / 9.]]
assert(np.allclose(X_result, X_corr[0, ..., 0]))
def compute_correlations(x, correlations=None, compute_flat=True):
print "-->Constructing Correlations"
prim_basis = PrimitiveBasis(n_states=3, domain=[0,2])
x_ = prim_basis.discretize(x)
if correlations == None:
x_corr = correlate(x_, periodic_axes=[0, 1])
else:
x_corr = correlate(x_, periodic_axes=[0, 1], correlations=correlations)
if compute_flat:
x_corr_flat = np.ndarray(shape=(x.shape[0], x_corr.shape[1]*x_corr.shape[2]*x_corr.shape[3]))
row_ctr = 0
for row in x_corr:
x_corr_flat[row_ctr] = row.flatten()
row_ctr += 1
return x_corr, x_corr_flat
return x_corr
def test_periodic_correlate():
'''
test corrleate for periodic microstructures
'''
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
X = np.array([[[0, 0, 1, 0],
[0, 0, 1, 0],
[0, 0, 1, 0],
[0, 0, 0, 0],
[0, 0, 1, 0]],
[[0, 1, 0, 0],
[0, 1, 0, 0],
[0, 1, 0, 0],
[0, 0, 0, 0],
[0, 1, 0, 0]]])
basis = DiscreteIndicatorBasis(n_states=2)
X_corr = correlate(X, basis, periodic_axes=(0, 1))
X_result = [[0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.8, 0.6],
[0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.75, 0.6]]
assert(np.allclose(X_result, X_corr[0, ..., 0]))
def test_mask_two_samples():
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
from pymks.datasets import make_microstructure
X = make_microstructure(n_samples=2, n_phases=2, size=(3, 3),
grain_size=(2, 2), seed=99)
basis = DiscreteIndicatorBasis(n_states=2)
mask = np.ones(X.shape)
mask[:, 0, 0] = 0.
X_corr = correlate(X, basis, confidence_index=mask)
X_result = np.array([[[[1 / 3., 1 / 3., 1 / 3.],
[1 / 5., 1 / 5., 1 / 5.],
[1 / 4., 1 / 4., 0]],
[[1 / 5., 1 / 5., 2 / 5.],
[1 / 2., 1 / 2., 0],
[1 / 5., 1 / 5., 1 / 5.]],
[[1 / 4., 1 / 4., 1 / 2.],
[1 / 5., 1 / 5., 2 / 5.],
[1 / 3., 1 / 3., 0]]],
[[[0., 0., 1 / 3.],
[2 / 5., 3 / 5., 0.],
[0., 0., 1 / 2.]],
[[0., 0., 2 / 5.],
[3 / 8., 5 / 8., 0],
[0., 0., 3 / 5.]],
[[0., 0., 1 / 2.],
[2 / 5., 3 / 5., 0.],
[0., 0., 2 / 3.]]]])
assert np.allclose(X_corr, X_result)
def get_correlations_for_slice(al_data_slice): # prim basis tellis it to use 0,1,2 as the 3 Stats prim_basis = PrimitiveBasis(n_states=3, domain=[0,2]) disc_basis = prim_basis.discretize(al_data_slice) # get the correlations correlations = correlate(disc_basis, periodic_axes=(0, 1)) return correlations
def _compute_stats(self, X, confidence_index):
"""
Helper function used to calculated 2-point statistics from `X` and
reshape them appropriately for fit and predict methods.
Args:
X (ND array): The microstructure, an `(n_samples, n_x, ...)`
shaped array where `n_samples` is the number of samples and
`n_x` is the spatial discretization..
confidence_index (ND array, optional): array with same shape as X
used to assign a confidence value for each data point.
Returns:
Spatial correlations for each sample formated with dimensions
(n_samples, n_features).
Example
"""
if self.basis is None:
raise AttributeError('basis must be specified')
X_stats = correlate(X,
self.basis,
periodic_axes=self.periodic_axes,
confidence_index=confidence_index,
correlations=self.correlations)
return X_stats
def test_mask_two_samples():
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
from pymks.datasets import make_microstructure
X = make_microstructure(n_samples=2, n_phases=2, size=(3, 3), grain_size=(2, 2), seed=99)
basis = DiscreteIndicatorBasis(n_states=2)
X_ = basis.discretize(X)
mask = np.ones(X.shape)
mask[:, 0, 0] = 0.0
X_corr = correlate(X_, confidence_index=mask)
X_result = np.array(
[
[
[[1 / 3.0, 1 / 3.0, 1 / 3.0], [1 / 5.0, 1 / 5.0, 1 / 5.0], [1 / 4.0, 1 / 4.0, 0]],
[[1 / 5.0, 1 / 5.0, 2 / 5.0], [1 / 2.0, 1 / 2.0, 0], [1 / 5.0, 1 / 5.0, 1 / 5.0]],
[[1 / 4.0, 1 / 4.0, 1 / 2.0], [1 / 5.0, 1 / 5.0, 2 / 5.0], [1 / 3.0, 1 / 3.0, 0]],
],
[
[[0.0, 0.0, 1 / 3.0], [2 / 5.0, 3 / 5.0, 0.0], [0.0, 0.0, 1 / 2.0]],
[[0.0, 0.0, 2 / 5.0], [3 / 8.0, 5 / 8.0, 0], [0.0, 0.0, 3 / 5.0]],
[[0.0, 0.0, 1 / 2.0], [2 / 5.0, 3 / 5.0, 0.0], [0.0, 0.0, 2 / 3.0]],
],
]
)
print np.round(X_corr, decimals=4)
print X_result
assert np.allclose(X_corr, X_result)
def test_stats_in_parallel():
import time
from pymks.bases import PrimitiveBasis
from pymks.stats import correlate
from pymks.datasets import make_microstructure
p_basis = PrimitiveBasis(5)
if p_basis._pyfftw:
X = make_microstructure(n_samples=5, n_phases=3)
t = []
for i in range(1, 4):
t_start = time.time()
correlate(X, p_basis, n_jobs=i)
t.append(time.time() - t_start)
assert t == sorted(t, reverse=True)
else:
pass
def test_mask_two_samples():
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
from pymks.datasets import make_microstructure
X = make_microstructure(n_samples=2,
n_phases=2,
size=(3, 3),
grain_size=(2, 2),
seed=99)
basis = DiscreteIndicatorBasis(n_states=2)
mask = np.ones(X.shape)
mask[:, 0, 0] = 0.
X_corr = correlate(X, basis, confidence_index=mask)
X_result = np.array([[[[1 / 3., 1 / 3., 1 / 3.], [1 / 5., 1 / 5., 1 / 5.],
[1 / 4., 1 / 4., 0]],
[[1 / 5., 1 / 5., 2 / 5.], [1 / 2., 1 / 2., 0],
[1 / 5., 1 / 5., 1 / 5.]],
[[1 / 4., 1 / 4., 1 / 2.], [1 / 5., 1 / 5., 2 / 5.],
[1 / 3., 1 / 3., 0]]],
[[[0., 0., 1 / 3.], [2 / 5., 3 / 5., 0.],
[0., 0., 1 / 2.]],
[[0., 0., 2 / 5.], [3 / 8., 5 / 8., 0],
[0., 0., 3 / 5.]],
[[0., 0., 1 / 2.], [2 / 5., 3 / 5., 0.],
[0., 0., 2 / 3.]]]])
assert np.allclose(X_corr, X_result)
def test_store_correlations():
from pymks import MKSStructureAnalysis
from pymks import PrimitiveBasis
from pymks.stats import correlate
p_basis = PrimitiveBasis(2)
model = MKSStructureAnalysis(basis=p_basis, store_correlations=True)
X = np.random.randint(2, size=(2, 4, 4))
model.fit(X)
X = correlate(X, p_basis, correlations=[(0, 0), (0, 1)])
assert np.allclose(X, model.fit_correlations)
X_0 = np.random.randint(2, size=(2, 4, 4))
model.transform(X_0)
X_corr_0 = correlate(X_0, p_basis, correlations=[(0, 0), (0, 1)])
assert np.allclose(X_corr_0, model.transform_correlations)
X_1 = np.random.randint(2, size=(2, 4, 4))
model.transform(X_1)
X_corr_1 = correlate(X_1, p_basis, correlations=[(0, 0), (0, 1)])
X_corr_ = np.concatenate((X_corr_0, X_corr_1))
assert np.allclose(X_corr_, model.transform_correlations)
def test_periodic_correlate():
'''
test corrleate for periodic microstructures
'''
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
X = np.array([[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0],
[0, 0, 1, 0]],
[[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0],
[0, 1, 0, 0]]])
basis = DiscreteIndicatorBasis(n_states=2)
X_corr = correlate(X, basis, periodic_axes=(0, 1))
X_result = [[0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.8, 0.6], [0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.75, 0.6]]
assert (np.allclose(X_result, X_corr[0, ..., 0]))
def test_nonperiodic_correlate():
'''
test corrleate for non-periodic microstructures
'''
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
X = np.array([[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0],
[0, 0, 1, 0]],
[[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0],
[0, 1, 0, 0]]])
basis = DiscreteIndicatorBasis(n_states=2)
X_corr = correlate(X, basis)
X_result = [[2 / 3., 4 / 9., 0.75, 4 / 9.], [5 / 8., 0.5, 0.75, 0.5],
[0.6, 7 / 15., 0.8, 7 / 15.], [5 / 8., 0.5, 0.75, 0.5],
[0.5, 4 / 9., 0.75, 4 / 9.]]
assert (np.allclose(X_result, X_corr[0, ..., 0]))
def _correlate(self, X, periodic_axes, confidence_index):
"""
Helper function used to calculated 2-point statistics from `X` and
reshape them appropriately for fit and predict methods.
Args:
X (ND array): The microstructure, an `(n_samples, n_x, ...)`
shaped array where `n_samples` is the number of samples and
`n_x` is the spatial discretization..
periodic_axes (list, optional): axes that are periodic. (0, 2)
would indicate that axes x and z are periodic in a 3D
microstrucure.
confidence_index (ND array, optional): array with same shape as X
used to assign a confidence value for each data point.
Returns:
Spatial correlations for each sample formated with dimensions
(n_samples, n_features).
Example
>>> from sklearn.manifold import Isomap
>>> from sklearn.linear_model import ARDRegression
>>> from pymks.bases import PrimitiveBasis
>>> reducer = Isomap()
>>> linker = ARDRegression()
>>> prim_basis = PrimitiveBasis(2, [0, 1])
>>> model = MKSHomogenizationModel(prim_basis, reducer, linker)
>>> X = np.array([[0, 1],
... [1, 0]])
>>> X_stats = model._correlate(X, [], None)
>>> X_test = np.array([[[ 0, 0],
... [0.5, 0]],
... [[0, 1,],
... [0.5, 0]]])
>>> assert np.allclose(X_test, X_stats)
"""
if self.basis is None:
raise AttributeError('basis must be specified')
X_ = self.basis.discretize(X)
X_stats = correlate(X_,
periodic_axes=periodic_axes,
confidence_index=confidence_index,
correlations=self.correlations)
return X_stats
def _correlate(self, X, periodic_axes, confidence_index):
"""
Helper function used to calculated 2-point statistics from `X` and
reshape them appropriately for fit and predict methods.
Args:
X (ND array): The microstructure, an `(n_samples, n_x, ...)`
shaped array where `n_samples` is the number of samples and
`n_x` is the spatial discretization..
periodic_axes (list, optional): axes that are periodic. (0, 2)
would indicate that axes x and z are periodic in a 3D
microstrucure.
confidence_index (ND array, optional): array with same shape as X
used to assign a confidence value for each data point.
Returns:
Spatial correlations for each sample formated with dimensions
(n_samples, n_features).
Example
>>> from sklearn.manifold import Isomap
>>> from sklearn.linear_model import ARDRegression
>>> from pymks.bases import PrimitiveBasis
>>> reducer = Isomap()
>>> linker = ARDRegression()
>>> prim_basis = PrimitiveBasis(2, [0, 1])
>>> model = MKSHomogenizationModel(prim_basis, reducer, linker)
>>> X = np.array([[0, 1],
... [1, 0]])
>>> X_stats = model._correlate(X, [], None)
>>> X_test = np.array([[[ 0, 0],
... [0.5, 0]],
... [[0, 1,],
... [0.5, 0]]])
>>> assert np.allclose(X_test, X_stats)
"""
if self.basis is None:
raise AttributeError('basis must be specified')
X_ = self.basis.discretize(X)
X_stats = correlate(X_, periodic_axes=periodic_axes,
confidence_index=confidence_index,
correlations=self.correlations)
return X_stats
def test_periodic_correlate():
"""
test corrleate for non-periodic microstructures
"""
from pymks import DiscreteIndicatorBasis
from pymks.stats import correlate
X = np.array(
[
[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 1, 0]],
[[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0]],
]
)
basis = DiscreteIndicatorBasis(n_states=2)
X_ = basis.discretize(X)
X_corr = correlate(X_, periodic_axes=(0, 1))
X_result = [
[0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.8, 0.6],
[0.6, 0.6, 0.75, 0.6],
[0.6, 0.6, 0.75, 0.6],
]
assert np.allclose(X_result, X_corr[0, ..., 0])
def _compute_stats(self, X, confidence_index):
"""
Helper function used to calculated 2-point statistics from `X` and
reshape them appropriately for fit and predict methods.
Args:
X (ND array): The microstructure, an `(n_samples, n_x, ...)`
shaped array where `n_samples` is the number of samples and
`n_x` is the spatial discretization..
confidence_index (ND array, optional): array with same shape as X
used to assign a confidence value for each data point.
Returns:
Spatial correlations for each sample formated with dimensions
(n_samples, n_features).
Example
"""
if self.basis is None:
raise AttributeError('basis must be specified')
X_stats = correlate(X, self.basis, periodic_axes=self.periodic_axes,
confidence_index=confidence_index,
correlations=self.correlations)
return X_stats
def analyze_data_slice(al_data_slice):
# prim basis
prim_basis = PrimitiveBasis(n_states=3, domain=[0, 2])
disc_basis = prim_basis.discretize(al_data_slice)
correlations = correlate(disc_basis)
return correlations
linker = RandomForestClassifier()
params = {'n_estimators':range(1,100,10)}
opt_model = run_gridcv_linkage(y,x_pca,linker,params)
print('---->n_est:'), (opt_model.best_estimator_.n_estimators)
r2_mean, r2_std, mse_mean, mse_std = run_conventional_linkage(y,x_pca,5,opt_model)
quit()
print "-->Constructing Correlations"
prim_basis = PrimitiveBasis(n_states=3, domain=[0,2])
x_ = prim_basis.discretize(x)
x_corr = correlate(x_, periodic_axes=[0, 1])
x_corr_flat = np.ndarray(shape=(samples, x_corr.shape[1]*x_corr.shape[2]*x_corr.shape[3]))
row_ctr = 0
for row in x_corr:
x_corr_flat[row_ctr] = row.flatten()
print x.shape
flat_len = (x.shape[0],) + (np.prod(x.shape[1:]),)
X_train, X_test, y_train, y_test = train_test_split(x.reshape(flat_len), y,
test_size=0.2, random_state=3)
print(x_corr.shape)
print(X_test.shape)
# uncomment to view one containers
#draw_correlations(x_corr[0].real)
# Reduce all 2-pt Stats via PCA
def analyze_data_slice(al_data_slice): # prim basis prim_basis = PrimitiveBasis(n_states=3, domain=[0,2]) disc_basis = prim_basis.discretize(al_data_slice) correlations = correlate(disc_basis) return correlations
def predict(bq, log, table_url, predictor_url, reducer_url, ms_path, **kw):
'''
Predicts effective strength of 3-D RVE of a 2-phase composite with strength contrast s2/s1 = 5
Args:
- table_path - path to dream3d file containing microstructure data (phase labels)
- predictor_path - path to sav file containing calibrated model (LinearRegression)
- reducer_path - path to sav file containing dimensionality reducer (Principal Component Basis)
- ms_path - path to microstructure data (phase lables) inside dream3d file
Returns:
- y - predicted effective strength
'''
log.debug('kw is: %s', str(kw))
predictor_uniq = predictor_url.split('/')[-1]
reducer_uniq = reducer_url.split('/')[-1]
table_uniq = table_url.split('/')[-1]
predictor_url = bq.service_url('blob_service', path=predictor_uniq)
predictor_path = os.path.join(kw.get('stagingPath', ''), 'predictor.sav')
predictor_path = bq.fetchblob(predictor_url, path=predictor_path)
reducer_url = bq.service_url('blob_service', path=reducer_uniq)
reducer_path = os.path.join(kw.get('stagingPath', ''), 'reducer.sav')
reducer_path = bq.fetchblob(reducer_url, path=reducer_path)
# ms_path default: '/DataContainers/SyntheticVolumeDataContainer/CellData/Phases'
# Default settings for 2-pt stats
p_axes = (0, 1, 2)
corrs = [(1, 1)]
# Read hdf5 table
table_service = bq.service('table')
# Get dataset
data = table_service.load_array(table_uniq, ms_path.lstrip('/'))
ms = np.squeeze(data)
# f = h5py.File(table_path, 'r')
# data = f[ms_path].value
# ms = np.squeeze(data)
# Get phase labels as local states
states = np.unique(ms)
if len(states) > 2:
log.warn(
'WARNING: Model is only for two-phase materials! All extra phases will be considered as the second (hard) phase'
)
ms[ms > states[0]] = states[0]
ph_1 = np.min(states)
ph_2 = np.max(states)
s1 = 0.2
s2 = 1.0
eta = s2 / s1
f1 = np.count_nonzero(ms == ph_1) * 1.0 / np.prod(ms.shape)
f2 = np.count_nonzero(ms == ph_2) * 1.0 / np.prod(ms.shape)
sbar_up = (f1 * s1) + (f2 * s2)
sbar_low = (f1 / s1) + (f2 / s2)
sbar_low = 1.0 / sbar_low
# Get the size of the RVE
if len(ms.shape) == 4:
dims = ms.shape[1:4]
elif len(ms.shape) == 3:
dims = ms.shape
ms = np.expand_dims(ms, 0)
else:
log.error('ERROR: 3-D RVE(s) are expected!')
return None
# Load model and dimensionality reducer
predictor = joblib.load(predictor_path)
reducer = joblib.load(reducer_path)
# Get the number of PC components used
n_comps = predictor.named_steps['poly'].n_input_features_
# Get the size of the calibration RVE
nx_cal = int(np.round((reducer.components_.shape[1])**(1.0 / 3.0)))
dims_cal = np.array((nx_cal, nx_cal, nx_cal))
# Compute 2-pt stats
n_states = len(states)
p_basis = PrimitiveBasis(n_states=n_states, domain=states)
tps = correlate(ms, p_basis, periodic_axes=p_axes, correlations=corrs)
# Check size of the provided MVE: truncate if large, pad if small
if np.prod(dims) > reducer.components_.shape[1]:
tps = truncate(tps,
[len(ms), dims_cal[0], dims_cal[1], dims_cal[2], 1])
dims = dims_cal
log.info(
'Microstructure volume is larger than calibration RVE. 2-pt correlation function is truncated'
)
elif np.prod(dims) < reducer.components_.shape[1]:
tps = pad(tps, [len(ms), dims_cal[0], dims_cal[1], dims_cal[2], 1])
dims = dims_cal
log.info(
'Microstructure volume is smaller than calibration RVE. 2-pt correlation function is padded'
)
# Convert 2-pt stats to a vector
tps_v = np.reshape(tps, (len(ms), np.prod(dims)))
# Get low-dimensional representation
x = reducer.transform(tps_v)
# Get the property prediction
y = predictor.predict(x[:, 0:n_comps])
# outtable_xml = table_service.store_array(y, name='predicted_strength')
# return [ outtable_xml ]
out_strength_xml = """<tag name="Strength">
<tag name="Strength" type="string" value="%s"/>
<tag name="sbar_up" type="string" value="%s"/>
<tag name="sbar_low" type="string" value="%s"/>
<tag name="Volume Fraction" type="string" value="%s"/>
<tag name="link" type="resource" value="%s"/>
</tag>""" % (str(y[0] * eta), str(
sbar_up * eta), str(
sbar_low * eta), str(f1) + ', ' + str(f2), table_url)
return [out_strength_xml]
# Get a representative slice from the block (or ave or whatever we decide on)
best_slice = get_best_slice(metadatum['data'])
# Get 2-pt Stats for the best slice
print "--->Getting 2pt stats"
metadatum['stats'] = get_correlations_for_slice(best_slice)
print metadata[0]['stats'].shape
# Construct X and Y for PCA and linkage
print "-->Creating X and Y"
i = 0
for metadatum in metadata:
x[i,0:6*metadatum['x']**2] = metadatum['stats'].flatten()
prim_basis = PrimitiveBasis(n_states=3, domain=[0,2])
x_ = prim_basis.discretize(metadata[0]['data'])
x_corr = correlate(x_)
draw_correlations(x_corr.real)
quit()
# Reduce all 2-pt Stats via PCA
# Try linear reg on inputs and outputs
reducer = PCA(n_components=3)
linker = LinearRegression()
model = MKSHomogenizationModel(dimension_reducer=reducer,
property_linker=linker,
compute_correlations=False)
model.n_components = 40
model.fit(metadatum['stats'], y, periodic_axes=[0, 1])
print model.reduced_fit_data
