def test_stats_in_parallel(): from pymks.bases import PrimitiveBasis from pymks.stats import correlate from pymks.datasets import make_microstructure p_basis = PrimitiveBasis(5) X = make_microstructure(n_samples=5, n_phases=3) X_corr_actual = correlate(X, p_basis) for i in range(1, 4): X_corr_test = correlate(X, p_basis, n_jobs=i) assert np.allclose(X_corr_actual, X_corr_test)
def test_stats_in_parallel(): import time from pymks.bases import PrimitiveBasis from pymks.stats import correlate from pymks.datasets import make_microstructure p_basis = PrimitiveBasis(5) if p_basis._pyfftw: X = make_microstructure(n_samples=5, n_phases=3) t = [] for i in range(1, 4): t_start = time.time() correlate(X, p_basis, n_jobs=i) t.append(time.time() - t_start) assert t == sorted(t, reverse=True) else: pass
def test_nonperiodic_correlate(): ''' test corrleate for non-periodic microstructures ''' from pymks import DiscreteIndicatorBasis from pymks.stats import correlate X = np.array([[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0]]]) basis = DiscreteIndicatorBasis(n_states=2) X_corr = correlate(X, basis) X_result = [[2 / 3., 4 / 9., 0.75, 4 / 9.], [5 / 8., 0.5, 0.75, 0.5], [0.6, 7 / 15., 0.8, 7 / 15.], [5 / 8., 0.5, 0.75, 0.5], [0.5, 4 / 9., 0.75, 4 / 9.]] assert(np.allclose(X_result, X_corr[0, ..., 0]))
def compute_correlations(x, correlations=None, compute_flat=True): print "-->Constructing Correlations" prim_basis = PrimitiveBasis(n_states=3, domain=[0,2]) x_ = prim_basis.discretize(x) if correlations == None: x_corr = correlate(x_, periodic_axes=[0, 1]) else: x_corr = correlate(x_, periodic_axes=[0, 1], correlations=correlations) if compute_flat: x_corr_flat = np.ndarray(shape=(x.shape[0], x_corr.shape[1]*x_corr.shape[2]*x_corr.shape[3])) row_ctr = 0 for row in x_corr: x_corr_flat[row_ctr] = row.flatten() row_ctr += 1 return x_corr, x_corr_flat return x_corr
def test_periodic_correlate(): ''' test corrleate for periodic microstructures ''' from pymks import DiscreteIndicatorBasis from pymks.stats import correlate X = np.array([[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0]]]) basis = DiscreteIndicatorBasis(n_states=2) X_corr = correlate(X, basis, periodic_axes=(0, 1)) X_result = [[0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.8, 0.6], [0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.75, 0.6]] assert(np.allclose(X_result, X_corr[0, ..., 0]))
def test_mask_two_samples(): from pymks import DiscreteIndicatorBasis from pymks.stats import correlate from pymks.datasets import make_microstructure X = make_microstructure(n_samples=2, n_phases=2, size=(3, 3), grain_size=(2, 2), seed=99) basis = DiscreteIndicatorBasis(n_states=2) mask = np.ones(X.shape) mask[:, 0, 0] = 0. X_corr = correlate(X, basis, confidence_index=mask) X_result = np.array([[[[1 / 3., 1 / 3., 1 / 3.], [1 / 5., 1 / 5., 1 / 5.], [1 / 4., 1 / 4., 0]], [[1 / 5., 1 / 5., 2 / 5.], [1 / 2., 1 / 2., 0], [1 / 5., 1 / 5., 1 / 5.]], [[1 / 4., 1 / 4., 1 / 2.], [1 / 5., 1 / 5., 2 / 5.], [1 / 3., 1 / 3., 0]]], [[[0., 0., 1 / 3.], [2 / 5., 3 / 5., 0.], [0., 0., 1 / 2.]], [[0., 0., 2 / 5.], [3 / 8., 5 / 8., 0], [0., 0., 3 / 5.]], [[0., 0., 1 / 2.], [2 / 5., 3 / 5., 0.], [0., 0., 2 / 3.]]]]) assert np.allclose(X_corr, X_result)
def get_correlations_for_slice(al_data_slice): # prim basis tellis it to use 0,1,2 as the 3 Stats prim_basis = PrimitiveBasis(n_states=3, domain=[0,2]) disc_basis = prim_basis.discretize(al_data_slice) # get the correlations correlations = correlate(disc_basis, periodic_axes=(0, 1)) return correlations
def _compute_stats(self, X, confidence_index): """ Helper function used to calculated 2-point statistics from `X` and reshape them appropriately for fit and predict methods. Args: X (ND array): The microstructure, an `(n_samples, n_x, ...)` shaped array where `n_samples` is the number of samples and `n_x` is the spatial discretization.. confidence_index (ND array, optional): array with same shape as X used to assign a confidence value for each data point. Returns: Spatial correlations for each sample formated with dimensions (n_samples, n_features). Example """ if self.basis is None: raise AttributeError('basis must be specified') X_stats = correlate(X, self.basis, periodic_axes=self.periodic_axes, confidence_index=confidence_index, correlations=self.correlations) return X_stats
def test_mask_two_samples(): from pymks import DiscreteIndicatorBasis from pymks.stats import correlate from pymks.datasets import make_microstructure X = make_microstructure(n_samples=2, n_phases=2, size=(3, 3), grain_size=(2, 2), seed=99) basis = DiscreteIndicatorBasis(n_states=2) X_ = basis.discretize(X) mask = np.ones(X.shape) mask[:, 0, 0] = 0.0 X_corr = correlate(X_, confidence_index=mask) X_result = np.array( [ [ [[1 / 3.0, 1 / 3.0, 1 / 3.0], [1 / 5.0, 1 / 5.0, 1 / 5.0], [1 / 4.0, 1 / 4.0, 0]], [[1 / 5.0, 1 / 5.0, 2 / 5.0], [1 / 2.0, 1 / 2.0, 0], [1 / 5.0, 1 / 5.0, 1 / 5.0]], [[1 / 4.0, 1 / 4.0, 1 / 2.0], [1 / 5.0, 1 / 5.0, 2 / 5.0], [1 / 3.0, 1 / 3.0, 0]], ], [ [[0.0, 0.0, 1 / 3.0], [2 / 5.0, 3 / 5.0, 0.0], [0.0, 0.0, 1 / 2.0]], [[0.0, 0.0, 2 / 5.0], [3 / 8.0, 5 / 8.0, 0], [0.0, 0.0, 3 / 5.0]], [[0.0, 0.0, 1 / 2.0], [2 / 5.0, 3 / 5.0, 0.0], [0.0, 0.0, 2 / 3.0]], ], ] ) print np.round(X_corr, decimals=4) print X_result assert np.allclose(X_corr, X_result)
def test_store_correlations(): from pymks import MKSStructureAnalysis from pymks import PrimitiveBasis from pymks.stats import correlate p_basis = PrimitiveBasis(2) model = MKSStructureAnalysis(basis=p_basis, store_correlations=True) X = np.random.randint(2, size=(2, 4, 4)) model.fit(X) X = correlate(X, p_basis, correlations=[(0, 0), (0, 1)]) assert np.allclose(X, model.fit_correlations) X_0 = np.random.randint(2, size=(2, 4, 4)) model.transform(X_0) X_corr_0 = correlate(X_0, p_basis, correlations=[(0, 0), (0, 1)]) assert np.allclose(X_corr_0, model.transform_correlations) X_1 = np.random.randint(2, size=(2, 4, 4)) model.transform(X_1) X_corr_1 = correlate(X_1, p_basis, correlations=[(0, 0), (0, 1)]) X_corr_ = np.concatenate((X_corr_0, X_corr_1)) assert np.allclose(X_corr_, model.transform_correlations)
def test_periodic_correlate(): ''' test corrleate for periodic microstructures ''' from pymks import DiscreteIndicatorBasis from pymks.stats import correlate X = np.array([[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0]]]) basis = DiscreteIndicatorBasis(n_states=2) X_corr = correlate(X, basis, periodic_axes=(0, 1)) X_result = [[0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.8, 0.6], [0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.75, 0.6]] assert (np.allclose(X_result, X_corr[0, ..., 0]))
def test_nonperiodic_correlate(): ''' test corrleate for non-periodic microstructures ''' from pymks import DiscreteIndicatorBasis from pymks.stats import correlate X = np.array([[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0]]]) basis = DiscreteIndicatorBasis(n_states=2) X_corr = correlate(X, basis) X_result = [[2 / 3., 4 / 9., 0.75, 4 / 9.], [5 / 8., 0.5, 0.75, 0.5], [0.6, 7 / 15., 0.8, 7 / 15.], [5 / 8., 0.5, 0.75, 0.5], [0.5, 4 / 9., 0.75, 4 / 9.]] assert (np.allclose(X_result, X_corr[0, ..., 0]))
def _correlate(self, X, periodic_axes, confidence_index): """ Helper function used to calculated 2-point statistics from `X` and reshape them appropriately for fit and predict methods. Args: X (ND array): The microstructure, an `(n_samples, n_x, ...)` shaped array where `n_samples` is the number of samples and `n_x` is the spatial discretization.. periodic_axes (list, optional): axes that are periodic. (0, 2) would indicate that axes x and z are periodic in a 3D microstrucure. confidence_index (ND array, optional): array with same shape as X used to assign a confidence value for each data point. Returns: Spatial correlations for each sample formated with dimensions (n_samples, n_features). Example >>> from sklearn.manifold import Isomap >>> from sklearn.linear_model import ARDRegression >>> from pymks.bases import PrimitiveBasis >>> reducer = Isomap() >>> linker = ARDRegression() >>> prim_basis = PrimitiveBasis(2, [0, 1]) >>> model = MKSHomogenizationModel(prim_basis, reducer, linker) >>> X = np.array([[0, 1], ... [1, 0]]) >>> X_stats = model._correlate(X, [], None) >>> X_test = np.array([[[ 0, 0], ... [0.5, 0]], ... [[0, 1,], ... [0.5, 0]]]) >>> assert np.allclose(X_test, X_stats) """ if self.basis is None: raise AttributeError('basis must be specified') X_ = self.basis.discretize(X) X_stats = correlate(X_, periodic_axes=periodic_axes, confidence_index=confidence_index, correlations=self.correlations) return X_stats
def test_periodic_correlate(): """ test corrleate for non-periodic microstructures """ from pymks import DiscreteIndicatorBasis from pymks.stats import correlate X = np.array( [ [[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0]], ] ) basis = DiscreteIndicatorBasis(n_states=2) X_ = basis.discretize(X) X_corr = correlate(X_, periodic_axes=(0, 1)) X_result = [ [0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.8, 0.6], [0.6, 0.6, 0.75, 0.6], [0.6, 0.6, 0.75, 0.6], ] assert np.allclose(X_result, X_corr[0, ..., 0])
def analyze_data_slice(al_data_slice): # prim basis prim_basis = PrimitiveBasis(n_states=3, domain=[0, 2]) disc_basis = prim_basis.discretize(al_data_slice) correlations = correlate(disc_basis) return correlations
linker = RandomForestClassifier() params = {'n_estimators':range(1,100,10)} opt_model = run_gridcv_linkage(y,x_pca,linker,params) print('---->n_est:'), (opt_model.best_estimator_.n_estimators) r2_mean, r2_std, mse_mean, mse_std = run_conventional_linkage(y,x_pca,5,opt_model) quit() print "-->Constructing Correlations" prim_basis = PrimitiveBasis(n_states=3, domain=[0,2]) x_ = prim_basis.discretize(x) x_corr = correlate(x_, periodic_axes=[0, 1]) x_corr_flat = np.ndarray(shape=(samples, x_corr.shape[1]*x_corr.shape[2]*x_corr.shape[3])) row_ctr = 0 for row in x_corr: x_corr_flat[row_ctr] = row.flatten() print x.shape flat_len = (x.shape[0],) + (np.prod(x.shape[1:]),) X_train, X_test, y_train, y_test = train_test_split(x.reshape(flat_len), y, test_size=0.2, random_state=3) print(x_corr.shape) print(X_test.shape) # uncomment to view one containers #draw_correlations(x_corr[0].real) # Reduce all 2-pt Stats via PCA
def analyze_data_slice(al_data_slice): # prim basis prim_basis = PrimitiveBasis(n_states=3, domain=[0,2]) disc_basis = prim_basis.discretize(al_data_slice) correlations = correlate(disc_basis) return correlations
def predict(bq, log, table_url, predictor_url, reducer_url, ms_path, **kw): ''' Predicts effective strength of 3-D RVE of a 2-phase composite with strength contrast s2/s1 = 5 Args: - table_path - path to dream3d file containing microstructure data (phase labels) - predictor_path - path to sav file containing calibrated model (LinearRegression) - reducer_path - path to sav file containing dimensionality reducer (Principal Component Basis) - ms_path - path to microstructure data (phase lables) inside dream3d file Returns: - y - predicted effective strength ''' log.debug('kw is: %s', str(kw)) predictor_uniq = predictor_url.split('/')[-1] reducer_uniq = reducer_url.split('/')[-1] table_uniq = table_url.split('/')[-1] predictor_url = bq.service_url('blob_service', path=predictor_uniq) predictor_path = os.path.join(kw.get('stagingPath', ''), 'predictor.sav') predictor_path = bq.fetchblob(predictor_url, path=predictor_path) reducer_url = bq.service_url('blob_service', path=reducer_uniq) reducer_path = os.path.join(kw.get('stagingPath', ''), 'reducer.sav') reducer_path = bq.fetchblob(reducer_url, path=reducer_path) # ms_path default: '/DataContainers/SyntheticVolumeDataContainer/CellData/Phases' # Default settings for 2-pt stats p_axes = (0, 1, 2) corrs = [(1, 1)] # Read hdf5 table table_service = bq.service('table') # Get dataset data = table_service.load_array(table_uniq, ms_path.lstrip('/')) ms = np.squeeze(data) # f = h5py.File(table_path, 'r') # data = f[ms_path].value # ms = np.squeeze(data) # Get phase labels as local states states = np.unique(ms) if len(states) > 2: log.warn( 'WARNING: Model is only for two-phase materials! All extra phases will be considered as the second (hard) phase' ) ms[ms > states[0]] = states[0] ph_1 = np.min(states) ph_2 = np.max(states) s1 = 0.2 s2 = 1.0 eta = s2 / s1 f1 = np.count_nonzero(ms == ph_1) * 1.0 / np.prod(ms.shape) f2 = np.count_nonzero(ms == ph_2) * 1.0 / np.prod(ms.shape) sbar_up = (f1 * s1) + (f2 * s2) sbar_low = (f1 / s1) + (f2 / s2) sbar_low = 1.0 / sbar_low # Get the size of the RVE if len(ms.shape) == 4: dims = ms.shape[1:4] elif len(ms.shape) == 3: dims = ms.shape ms = np.expand_dims(ms, 0) else: log.error('ERROR: 3-D RVE(s) are expected!') return None # Load model and dimensionality reducer predictor = joblib.load(predictor_path) reducer = joblib.load(reducer_path) # Get the number of PC components used n_comps = predictor.named_steps['poly'].n_input_features_ # Get the size of the calibration RVE nx_cal = int(np.round((reducer.components_.shape[1])**(1.0 / 3.0))) dims_cal = np.array((nx_cal, nx_cal, nx_cal)) # Compute 2-pt stats n_states = len(states) p_basis = PrimitiveBasis(n_states=n_states, domain=states) tps = correlate(ms, p_basis, periodic_axes=p_axes, correlations=corrs) # Check size of the provided MVE: truncate if large, pad if small if np.prod(dims) > reducer.components_.shape[1]: tps = truncate(tps, [len(ms), dims_cal[0], dims_cal[1], dims_cal[2], 1]) dims = dims_cal log.info( 'Microstructure volume is larger than calibration RVE. 2-pt correlation function is truncated' ) elif np.prod(dims) < reducer.components_.shape[1]: tps = pad(tps, [len(ms), dims_cal[0], dims_cal[1], dims_cal[2], 1]) dims = dims_cal log.info( 'Microstructure volume is smaller than calibration RVE. 2-pt correlation function is padded' ) # Convert 2-pt stats to a vector tps_v = np.reshape(tps, (len(ms), np.prod(dims))) # Get low-dimensional representation x = reducer.transform(tps_v) # Get the property prediction y = predictor.predict(x[:, 0:n_comps]) # outtable_xml = table_service.store_array(y, name='predicted_strength') # return [ outtable_xml ] out_strength_xml = """<tag name="Strength"> <tag name="Strength" type="string" value="%s"/> <tag name="sbar_up" type="string" value="%s"/> <tag name="sbar_low" type="string" value="%s"/> <tag name="Volume Fraction" type="string" value="%s"/> <tag name="link" type="resource" value="%s"/> </tag>""" % (str(y[0] * eta), str( sbar_up * eta), str( sbar_low * eta), str(f1) + ', ' + str(f2), table_url) return [out_strength_xml]
# Get a representative slice from the block (or ave or whatever we decide on) best_slice = get_best_slice(metadatum['data']) # Get 2-pt Stats for the best slice print "--->Getting 2pt stats" metadatum['stats'] = get_correlations_for_slice(best_slice) print metadata[0]['stats'].shape # Construct X and Y for PCA and linkage print "-->Creating X and Y" i = 0 for metadatum in metadata: x[i,0:6*metadatum['x']**2] = metadatum['stats'].flatten() prim_basis = PrimitiveBasis(n_states=3, domain=[0,2]) x_ = prim_basis.discretize(metadata[0]['data']) x_corr = correlate(x_) draw_correlations(x_corr.real) quit() # Reduce all 2-pt Stats via PCA # Try linear reg on inputs and outputs reducer = PCA(n_components=3) linker = LinearRegression() model = MKSHomogenizationModel(dimension_reducer=reducer, property_linker=linker, compute_correlations=False) model.n_components = 40 model.fit(metadatum['stats'], y, periodic_axes=[0, 1]) print model.reduced_fit_data