class HasCollections(properties.HasProperties):
tuple_unobs = properties.Tuple('')
dict_unobs = properties.Dictionary('')
dict_obs = properties.Dictionary('', observe_mutations=True)
list_unobs = properties.List('')
list_obs = properties.List('', observe_mutations=True)
set_unobs = properties.Set('')
set_obs = properties.Set('', observe_mutations=True)
class Projection(properties.HasProperties):
projection = properties.String('The coordinate projection')
datum = properties.String('The projection datum')
units = properties.String('The projection units')
corner_point = properties.List('The corner points', prop=CornerPoint)
grid_origin = properties.String('The grid origin')
utm_proj_params = properties.Dictionary('The UTM projection parameters', required=False)
ps_proj_params = properties.Dictionary('The PS projection parameters', required=False)
albers_proj_params = properties.Dictionary('The Albers projection parameters', required=False)
sin_proj_params = properties.Dictionary('The Sin projection parameters', required=False)
def test_with_None_and_no_default(self):
target = properties.Dictionary()
with self.assertRaises(properties.ValidationError) as e:
value = target('Test', None)
self.assertIn('Test', e.exception.message)
def test_with_non_dict(self):
target = properties.Dictionary()
with self.assertRaises(properties.ValidationError) as e:
value = target('Test', 'string')
self.assertIn('Test', e.exception.message)
class Band(properties.HasProperties):
"""Contains raster metadata and data for a single band."""
# metadata attributes
name = properties.String('Name of the band')
data_type = properties.String('Band data type')
nlines = properties.Integer('number of lines')
nsamps = properties.Integer('number of samples')
product = properties.String('Data product')
# Not required
app_version = properties.String('app version', required=False)
production_date = properties.String('production date', required=False)
resample_method = properties.String('resample method', required=False)
category = properties.String('Band category', required=False)
source = properties.String('Band source', required=False)
qa_description = properties.String('QA description', required=False)
# TODO: class_values
percent_coverage = properties.Float('percent coverage', required=False)
# metadata: All required
short_name = properties.String('Short name')
long_name = properties.String('Long display name')
file_name = properties.String('Original file name')
pixel_size = properties.Instance('The pixel size', PixelSize)
# data information
fill_value = properties.Integer('fill value', default=-9999)
saturate_value = properties.Integer('Saturate value', required=False)
add_offset = properties.Float('Add offset', required=False)
data_units = properties.String('Data units', required=False)
scale_factor = properties.Float('Scaling factor', required=False)
valid_range = properties.Instance('The valid data range',
ValidRange,
required=False)
radiance = properties.Instance('The radiance', Lum, required=False)
reflectance = properties.Instance('The reflectance', Lum, required=False)
thermal_const = properties.Instance('The thermal const',
ThermalConst,
required=False)
bitmap_description = properties.Dictionary(
'band bitmap description (not always present)',
required=False,
key_prop=properties.String('Key value'),
value_prop=properties.String('Bitmap value description'))
# TODO: data validation causes a MAJOR slowdown. WAAAAYYY faster to not set
# the data as a `properties` attribute.
# data = properties.Array(
# 'The band data as a 2D NumPy data',
# shape=('*','*'),
# )
data = None
class USNT(MeshSpecifications):
"""The base object to instantiate."""
rocktab = properties.Dictionary(
'Porous medium properties',
key_prop=properties.String('The material type name'),
value_prop=RockType)
@property
def attributes(self):
atts = []
for k, v in RockType._props.items():
if isinstance(v, properties.Float):
atts.append(k)
return atts
def model(self, attribute):
"""Gets a rocktab attribute as a NumPy array ready for discretize or PVGeo"""
mod = np.empty(self.shape)
mod[:] = np.nan
for el_pref in self.mat.keys():
for mat_type in self.mat[el_pref].keys():
for mc in self.mat[el_pref][mat_type]:
mod[mc.i[0]:mc.i[1] + 1, mc.j[0]:mc.j[1] + 1,
mc.k[0]:mc.k[1] +
1] = self.rocktab[mat_type]._get(attribute)
return mod.flatten(order='f')
def all_models(self, dataframe=True):
"""Returns all attributes in a Pandas DataFrame"""
df = pd.DataFrame()
for key in self.attributes:
df[key] = self.model(key)
if dataframe:
return df
return df.to_dict()
def allModels(self, dataframe=True):
return self.all_models(dataframe=dataframe)
def save_lith_lookup_table(self, filename):
atts = ['K0', 'K1', 'K2', 'porosity', 'solid_density']
lootbl = self.lookup_table.set_index('material')
for at in atts:
lootbl[at] = np.full(len(lootbl), np.nan)
for k in lootbl.index:
lootbl.loc[k, at] = self.rocktab[k]._get(at)
lootbl = lootbl.reset_index()
lootbl = lootbl.set_index('id')
return lootbl.to_csv(filename, index_label='Index')
def saveLithLookupTable(self, filename):
return self.save_lith_lookup_table(filename)
def handler(event, context):
props = properties.load(event, {
'Input': properties.Dictionary(),
})
stack_name = aws_utils.get_stack_name_from_stack_arn(event['StackId'])
physical_resource_id = stack_name + '-' + event['LogicalResourceId']
output = _process_dict(props.Input)
custom_resource_response.succeed(event, context, output,
physical_resource_id)
class Models(properties.HasProperties):
"""A container for static models"""
_models = properties.Dictionary(
'The data dictionary.',
key_prop=properties.String('The model names'),
value_prop=properties.Array('The data values',
dtype=(float, int, bool),
shape=None))
@property
def shape(self):
return list(self._models.values())[0].shape
@properties.validator
def _validate_models(self):
shp = self.shape
for k, d in self._models.items():
if d.shape != shp:
raise RuntimeError(
'Validation Failed: dimesnion mismatch between models.')
return True
def __getitem__(self, key):
"""Get a model by its string name and time index"""
return self._models[key]
def __setitem__(self, key, value):
if self._models is None:
self._models = {}
self._models[key] = value
def keys(self):
return self._models.keys()
def values(self):
return self._models.values()
def items(self):
return self._models.items()
def save(self, filename):
with open(filename, 'w') as f:
json.dump(self.serialize(), f)
return filename
class RasterSet(properties.HasProperties):
"""The main class to hold a set of raster data. This contains all of the bands
for a given set of rasters. This is generated by the ``RasterSetReader``.
"""
version = properties.String('version', required=False)
global_metadata = properties.Instance('Raster metadata', RasterMetaData)
# Bands
bands = properties.Dictionary('A dictionary of bands for the swath',
key_prop=properties.String('Band name'),
value_prop=Band)
nlines = properties.Integer('The number of lines')
nsamps = properties.Integer('The number of samples')
pixel_size = properties.Instance('The pixel size', PixelSize)
RGB_SCHEMES = dict(
true=ColorSchemes.LOOKUP_TRUE_COLOR,
infrared=ColorSchemes.LOOKUP_INFRARED,
false_a=ColorSchemes.LOOKUP_FLASE_COLOR_A,
false_b=ColorSchemes.LOOKUP_FLASE_COLOR_B,
false_c=ColorSchemes.LOOKUP_FLASE_COLOR_C,
)
def get_rgb(self, scheme='infrared', names=None):
"""Get an RGB color scheme based on predefined presets or specify your
own band names to use. A given set of names always overrides a scheme.
Note:
Available schemes are defined in ``RGB_SCHEMES`` and include:
- ``true``
- ``infrared``
- ``false_a``
- ``false_b``
- ``false_c``
"""
if names is not None:
if not isinstance(names, (list, tuple)) or len(names) != 3:
raise RuntimeError('RGB band names improperly defined.')
else:
lookup = self.RGB_SCHEMES[scheme]
names = lookup[self.global_metadata.satellite]
# Now check that all bands are available:
for nm in names:
if nm not in self.bands.keys():
raise RuntimeError('Band (%s) unavailable.' % nm)
# Get the RGB bands
r = self.bands[names[0]].data
g = self.bands[names[1]].data
b = self.bands[names[2]].data
# Note that the bands dhould already be masked from read.
# If casted then there are np.nans present
r = ((r - np.nanmin(r)) *
(1 / (np.nanmax(r) - np.nanmin(r)) * 255)).astype('uint8')
g = ((g - np.nanmin(g)) *
(1 / (np.nanmax(g) - np.nanmin(g)) * 255)).astype('uint8')
b = ((b - np.nanmin(b)) *
(1 / (np.nanmax(b) - np.nanmin(b)) * 255)).astype('uint8')
return np.dstack([r, g, b])
def GetRGB(self, *args, **kwargs):
return self.get_rgb(*args, **kwargs)
def validate(self):
b = self.bands.get(list(self.bands.keys())[0])
ny, nx = b.nlines, b.nsamps
dx, dy = b.pixel_size.x, b.pixel_size.y
for name, band in self.bands.items():
if band.nlines != ny or band.nsamps != nx:
raise RuntimeError('Band size mismatch.')
if band.pixel_size.x != dx or band.pixel_size.y != dy:
raise RuntimeError('Pixel size mismatch.')
self.nlines = ny
self.nsamps = nx
self.pixel_size = b.pixel_size
return properties.HasProperties.validate(self)
def to_pyvista(self, z=0.0):
"""Create a :class:`pyvista.UniformGrid` of this raster. Use the ``z``
argument to control the dataset's Z spatial reference.
"""
try:
import pyvista as pv
except ImportError:
raise ImportError("Please install PyVista.")
# Build the spatial reference
output = pv.UniformGrid()
output.dimensions = self.nsamps, self.nlines, 1
output.spacing = self.pixel_size.x, self.pixel_size.y, 1
corner = self.global_metadata.projection_information.corner_point[0]
output.origin = corner.x, corner.y, z
# Add data arrays
clean = lambda arr: np.flip(arr, axis=0)
for name, band in self.bands.items():
output[name] = clean(band.data).ravel()
for scheme in list(self.RGB_SCHEMES.keys()):
output[scheme] = clean(self.get_rgb(scheme=scheme)).reshape(
(-1, 3))
# Add an array for the mask
try:
mask = clean(~band.data.mask).ravel()
output["valid_mask"] = mask
except NameError:
pass
# Reutrn the dataset
return output
class TimeModels(Models):
"""A container for time varying models"""
_models = properties.Dictionary(
'The data dictionary.',
key_prop=properties.String('The model names'),
value_prop=properties.List(
'The model data as a time series list of arrays',
properties.Array('The data values',
dtype=(float, int, bool),
shape=None)))
@property
def shape(self):
return np.array(list(self._models.values())[0][0]).shape
@property
def nt(self):
return len(list(self._models.values())[0])
@properties.validator
def _validate_models(self):
nt = self.nt
shp = self.shape
for key, data in self._models.items():
if len(data) != nt:
raise RuntimeError(
'Validation Failed: time step mismatch on `{}`.'.format(
key))
for arr in data:
if arr.shape != shp:
raise RuntimeError(
'Validation Failed: dimesnion mismatch between models on `{}`.'
.format(key))
return True
def __getitem__(self, pos):
"""Get a model by its string name and time index"""
if not isinstance(pos, (list, tuple)):
pos = [pos]
if len(pos) == 1:
return self._models[pos[0]]
elif len(pos) == 2:
return self._models[pos[0]][pos[1]]
else:
raise RuntimeError('Get at [{}] not understood.'.format(pos))
def getTable(self, idx):
"""Returns a pandas dataframe table of all the models at a specified timestep"""
df = pd.DataFrame()
for k in self.keys():
df[k] = self[k, idx].flatten(order='f')
return df
def applyMethod(self, method, inplace=True):
mods = self
if not inplace:
mods = copy.deepcopy(self)
for key, data in mods._models.items():
for i, arr in enumerate(data):
data[i] = method(arr)
mods._models[key] = data
if not inplace:
return mods
def test_with_None_and_default(self):
target = properties.Dictionary(default={ 'a': 2 })
value = target('Test', None)
self.assertEqual(value, { 'a': 2 })
def test_with_dict_and_no_default(self):
target = properties.Dictionary()
value = target('Test', { 'a': 1 })
self.assertEqual(value, { 'a': 1 })
class BaseSimulation(props.HasModel):
"""
BaseSimulation is the base class for all geophysical forward simulations in
SimPEG.
"""
###########################################################################
# Properties
_REGISTRY = {}
mesh = properties.Instance("a discretize mesh instance", BaseMesh)
survey = properties.Instance("a survey object", BaseSurvey)
counter = properties.Instance("A SimPEG.utils.Counter object", Counter)
sensitivity_path = properties.String("path to store the sensitivty",
default="./sensitivity/")
# TODO: need to implement a serializer for this & setter
solver = Class("Linear algebra solver (e.g. from pymatsolver)",
# default=pymatsolver.Solver
)
solver_opts = properties.Dictionary("solver options as a kwarg dict",
default={})
def _reset(self, name=None):
"""Revert specified property to default value
If no property is specified, all properties are returned to default.
"""
if name is None:
for key in self._props:
if isinstance(self._props[key], properties.basic.Property):
self._reset(key)
return
if name not in self._props:
raise AttributeError("Input name '{}' is not a known "
"property or attribute".format(name))
if not isinstance(self._props[name], properties.basic.Property):
raise AttributeError("Cannot reset GettableProperty "
"'{}'".format(name))
if name in self._defaults:
val = self._defaults[name]
else:
val = self._props[name].default
# if callable(val):
# val = val()
setattr(self, name, val)
###########################################################################
# Properties and observers
@properties.observer("mesh")
def _update_registry(self, change):
self._REGISTRY.update(change["value"]._REGISTRY)
#: List of strings, e.g. ['_MeSigma', '_MeSigmaI']
# TODO: rename to _delete_on_model_update
deleteTheseOnModelUpdate = []
#: List of matrix names to have their factors cleared on a model update
clean_on_model_update = []
@properties.observer("model")
def _on_model_update(self, change):
if change["previous"] is change["value"]:
return
if (isinstance(change["previous"], np.ndarray)
and isinstance(change["value"], np.ndarray)
and np.allclose(change["previous"], change["value"])):
return
# cached properties to delete
for prop in self.deleteTheseOnModelUpdate:
if hasattr(self, prop):
delattr(self, prop)
# matrix factors to clear
for mat in self.clean_on_model_update:
if getattr(self, mat, None) is not None:
getattr(self, mat).clean() # clean factors
setattr(self, mat, None) # set to none
Solver = deprecate_property(
solver,
"Solver",
new_name="simulation.solver",
removal_version="0.16.0",
error=True,
)
solverOpts = deprecate_property(
solver_opts,
"solverOpts",
new_name="solver_opts",
removal_version="0.16.0",
error=True,
)
###########################################################################
# Instantiation
def __init__(self, mesh=None, **kwargs):
# raise exception if user tries to set "mapping"
if "mapping" in kwargs.keys():
raise Exception("Deprecated (in 0.4.0): use one of {}".format(
[p for p in self._props.keys() if "Map" in p]))
if mesh is not None:
kwargs["mesh"] = mesh
super(BaseSimulation, self).__init__(**kwargs)
if "solver" not in kwargs.keys() and "Solver" not in kwargs.keys():
self.solver = DefaultSolver
###########################################################################
# Methods
def fields(self, m=None):
"""
u = fields(m)
The field given the model.
:param numpy.ndarray m: model
:rtype: numpy.ndarray
:return: u, the fields
"""
raise NotImplementedError("fields has not been implemented for this ")
def dpred(self, m=None, f=None):
"""
dpred(m, f=None)
Create the projected data from a model.
The fields, f, (if provided) will be used for the predicted data
instead of recalculating the fields (which may be expensive!).
.. math::
d_\\text{pred} = P(f(m))
Where P is a projection of the fields onto the data space.
"""
if self.survey is None:
raise AttributeError(
"The survey has not yet been set and is required to compute "
"data. Please set the survey for the simulation: "
"simulation.survey = survey")
if f is None:
if m is None:
m = self.model
f = self.fields(m)
data = Data(self.survey)
for src in self.survey.source_list:
for rx in src.receiver_list:
data[src, rx] = rx.eval(src, self.mesh, f)
return mkvc(data)
@timeIt
def Jvec(self, m, v, f=None):
"""
Jv = Jvec(m, v, f=None)
Effect of J(m) on a vector v.
:param numpy.ndarray m: model
:param numpy.ndarray v: vector to multiply
:param Fields f: fields
:rtype: numpy.ndarray
:return: Jv
"""
raise NotImplementedError("Jvec is not yet implemented.")
@timeIt
def Jtvec(self, m, v, f=None):
"""
Jtv = Jtvec(m, v, f=None)
Effect of transpose of J(m) on a vector v.
:param numpy.ndarray m: model
:param numpy.ndarray v: vector to multiply
:param Fields f: fields
:rtype: numpy.ndarray
:return: JTv
"""
raise NotImplementedError("Jt is not yet implemented.")
@timeIt
def Jvec_approx(self, m, v, f=None):
"""Jvec_approx(m, v, f=None)
Approximate effect of J(m) on a vector v
:param numpy.ndarray m: model
:param numpy.ndarray v: vector to multiply
:param Fields f: fields
:rtype: numpy.ndarray
:return: approxJv
"""
return self.Jvec(m, v, f)
@timeIt
def Jtvec_approx(self, m, v, f=None):
"""Jtvec_approx(m, v, f=None)
Approximate effect of transpose of J(m) on a vector v.
:param numpy.ndarray m: model
:param numpy.ndarray v: vector to multiply
:param Fields f: fields
:rtype: numpy.ndarray
:return: JTv
"""
return self.Jtvec(m, v, f)
@count
def residual(self, m, dobs, f=None):
"""residual(m, dobs, f=None)
The data residual:
.. math::
\mu_\\text{data} = \mathbf{d}_\\text{pred} - \mathbf{d}_\\text{obs}
:param numpy.ndarray m: geophysical model
:param numpy.ndarray f: fields
:rtype: numpy.ndarray
:return: data residual
"""
return mkvc(self.dpred(m, f=f) - dobs)
def make_synthetic_data(self,
m,
relative_error=0.05,
noise_floor=0.0,
f=None,
add_noise=False,
**kwargs):
"""
Make synthetic data given a model, and a standard deviation.
:param numpy.ndarray m: geophysical model
:param numpy.ndarray relative_error: standard deviation
:param numpy.ndarray noise_floor: noise floor
:param numpy.ndarray f: fields for the given model (if pre-calculated)
"""
std = kwargs.pop("std", None)
if std is not None:
raise TypeError("The std parameter has been removed. "
"Please use relative_error.")
if f is None:
f = self.fields(m)
dclean = self.dpred(m, f=f)
if add_noise is True:
std = np.sqrt((relative_error * np.abs(dclean))**2 +
noise_floor**2)
noise = std * np.random.randn(*dclean.shape)
dobs = dclean + noise
else:
dobs = dclean
return SyntheticData(
survey=self.survey,
dobs=dobs,
dclean=dclean,
relative_error=relative_error,
noise_floor=noise_floor,
)
def pair(self, survey):
"""
Removed pairing method. Please use :code:`simulation.survey=survey`
instead.
"""
raise TypeError(
"Simulation.pair(survey) has been removed. Please update your code "
"to instead use simulation.survey = survey, or pass it upon intialization "
"of the simulation object.")
class HasFunnyDict(properties.HasProperties):
mydict = properties.Dictionary('my dict',
key_prop=properties.Color(''),
value_prop=HasInt,
observe_mutations=om)
class HasDummyDict(properties.HasProperties):
mydict = properties.Dictionary('dummy has properties set',
key_prop=properties.String(''),
value_prop=HasPropsDummy,
observe_mutations=om)
class Grid(discretize.TensorMesh, GridFileIO):
"""
A data structure to store a model space discretization and different
attributes of that model space.
Example:
>>> import wtools
>>> import numpy as np
>>> models = {
'rand': np.random.random(1000).reshape((10,10,10)),
'spatial': np.arange(1000).reshape((10,10,10)),
}
>>> grid = wtools.Grid(models=models)
>>> grid.validate() # Make sure the data object was created successfully
True
Note:
See Jupyter notebooks under the ``examples`` directory
"""
def __init__(self, h=None, x0=(0., 0., 0.), models=None, **kwargs):
if models is not None:
self.models = models
if h is None:
h = []
shp = list(models.values())[0].shape
# Now create tensors if not present
if len(shp) > 0:
h.append(np.ones(shp[0]))
if len(shp) > 1:
h.append(np.ones(shp[1]))
if len(shp) > 2:
h.append(np.ones(shp[2]))
discretize.TensorMesh.__init__(self, h=h, x0=x0, **kwargs)
models = properties.Dictionary(
'The volumetric data as a 3D NumPy arrays in <X,Y,Z> or <i,j,k> ' +
'coordinates. Each key value pair represents a different model for ' +
'the gridded model space. Keys will be treated as the string name of '
+ 'the model.',
key_prop=properties.String('Model name'),
value_prop=properties.Array(
'The volumetric data as a 3D NumPy array in <X,Y,Z> or <i,j,k> coordinates.',
shape=('*', '*', '*')),
required=False)
@properties.validator
def _validate_models(self):
# Check the models
if self.models is not None:
shp = list(self.models.values())[0].shape
for k, d in self.models.items():
if d.shape != shp:
raise RuntimeError(
'Validation Failed: dimesnion mismatch between models.'
)
return True
@property
def keys(self):
"""List of the string names for each of the models"""
return list(self.models.keys())
@property
def shape(self):
"""3D shape of the grid (number of cells in all three directions)"""
return (self.nCx, self.nCy, self.nCz)
@property
def bounds(self):
"""The bounds of the grid"""
grid = self.gridN
try:
x0, x1 = np.min(grid[:, 0]), np.max(grid[:, 0])
except:
x0, x1 = 0., 0.
try:
y0, y1 = np.min(grid[:, 1]), np.max(grid[:, 1])
except:
y0, y1 = 0., 0.
try:
z0, z1 = np.min(grid[:, 2]), np.max(grid[:, 2])
except:
z0, z1 = 0., 0.
return (x0, x1, y0, y1, z0, z1)
def get_data_range(self, key):
"""Get the data range for a given model"""
data = self.models[key]
return get_data_range(data)
def equal(self, other):
"""Compare this Grid to another Grid"""
return properties.equal(self, other)
def __str__(self):
"""Print this onject as a human readable string"""
self.validate()
fmt = ["<%s instance>" % (self.__class__.__name__)]
fmt.append(" Shape: {}".format(self.shape))
fmt.append(" Origin: {}".format(tuple(self.x0)))
bds = self.bounds
fmt.append(" X Bounds: {}".format((bds[0], bds[1])))
fmt.append(" Y Bounds: {}".format((bds[2], bds[3])))
fmt.append(" Z Bounds: {}".format((bds[4], bds[5])))
if self.models is not None:
fmt.append(" Models: ({})".format(len(self.models.keys())))
for key, val in self.models.items():
dl, dh = self.get_data_range(key)
fmt.append(" '{}' ({}): ({:.3e}, {:.3e})".format(
key, val.dtype, dl, dh))
return '\n'.join(fmt)
def __repr__(self):
return self.__str__()
def _repr_html_(self):
self.validate()
fmt = ""
if self.models is not None:
fmt += "<table>"
fmt += "<tr><th>Grid Attributes</th><th>Models</th></tr>"
fmt += "<tr><td>"
fmt += "\n"
fmt += "<table>\n"
fmt += "<tr><th>Attribute</th><th>Values</th></tr>\n"
row = "<tr><td>{}</td><td>{}</td></tr>\n"
fmt += row.format("Shape", self.shape)
fmt += row.format('Origin', tuple(self.x0))
bds = self.bounds
fmt += row.format("X Bounds", (bds[0], bds[1]))
fmt += row.format("Y Bounds", (bds[2], bds[3]))
fmt += row.format("Z Bounds", (bds[4], bds[5]))
num = 0
if self.models is not None:
num = len(self.models.keys())
fmt += row.format("Models", num)
fmt += "</table>\n"
fmt += "\n"
if self.models is not None:
fmt += "</td><td>"
fmt += "\n"
fmt += "<table>\n"
row = "<tr><th>{}</th><th>{}</th><th>{}</th><th>{}</th></tr>\n"
fmt += row.format("Name", "Type", "Min", "Max")
row = "<tr><td>{}</td><td>{}</td><td>{:.3e}</td><td>{:.3e}</td></tr>\n"
for key, val in self.models.items():
dl, dh = self.get_data_range(key)
fmt += row.format(key, val.dtype, dl, dh)
fmt += "</table>\n"
fmt += "\n"
fmt += "</td></tr> </table>"
return fmt
def __getitem__(self, key):
"""Get a model of this grid by its string name"""
return self.models[key]
def to_data_frame(self, order='C'):
"""Returns the models in this Grid to a Pandas DataFrame with all arrays
flattened in the specified order. A header attribute is added to the
DataFrame to specified the grid extents. Much metadata is lost in this
conversion.
"""
self.validate()
t**s = self.models.keys()
data = {k: v.flatten(order=order) for k, v in self.models.items()}
df = pd.DataFrame.from_dict(data)
df.header = '{} {} {}'.format(self.nCx, self.nCy, self.nCz)
return df
def plot_3d_slicer(self, key, **kwargs):
model = self.models[key]
return discretize.TensorMesh.plot_3d_slicer(self, model, **kwargs)
def plotSlice(self, key, **kwargs):
"""Plots a 2D slice of the mesh
Args:
key (str): the model name to plot
Note:
See the `discretize code docs`_ for more details.
.. _discretize code docs: http://discretize.simpeg.xyz/en/latest/content/mesh_tensor.html?highlight=plotSlice#discretize.View.TensorView.plotSlice
"""
return discretize.TensorMesh.plotSlice(self,
v=self.models[key],
**kwargs)
@property
def models_flat(self):
"""Returns flattened model dictionary in Fortran ordering"""
return {k: v.flatten(order='F') for k, v in self.models.items()}
def toVTK(self):
return discretize.TensorMesh.toVTK(self, models=self.models_flat)
def writeVTK(self):
return discretize.TensorMesh.writeVTK(self, models=self.models_flat)
class MeshSpecifications(properties.HasProperties):
"""specifies the mesh geometry, element material types and names as well as
dual permeability parameters and radiation parameters.
"""
coord = properties.String(
'specifies the type of mesh that will be generated',
change_case='lower',
)
@properties.validator
def _validate_mesh_type(self):
"""Check mesh type is one of two valid options: 'rect' or 'cylind'"""
if self.coord not in ['rect', 'cylind']:
raise properties.ValidationError(
"Mesh type ('{}') not a valid mesh type. Only 'rect' and 'cylind' are suppurted"
.format(self.coord))
return True
down = properties.Array(
'Vector orientation of down direction and in the direction of ' +
'the gravity vector.',
dtype=float,
shape=(3, ))
dx = properties.Array("widths of the mesh in the X-dimension",
dtype=float,
shape=("*", ))
dy = properties.Array("widths of the mesh in the Y-dimension",
dtype=float,
shape=("*", ))
dz = properties.Array("widths of the mesh in the Z-dimension",
dtype=float,
shape=("*", ))
mat = properties.Dictionary(
'The materials int the model space.',
key_prop=properties.String('The element name prefix',
change_case='lower'),
value_prop=properties.Dictionary(
'The material types',
key_prop=properties.String(
'The material type. Must have a corresponding entry in the material type the ``rocktab``.'
),
value_prop=properties.List('ther material components',
MaterialComponent)))
@property
def nx(self):
return len(self.dx)
@property
def ny(self):
return len(self.dy)
@property
def nz(self):
return len(self.dz)
@property
def n_cells(self):
return (self.nx * self.ny * self.nz)
@property
def nC(self):
return self.n_cells
@property
def shape(self):
return (self.nx, self.ny, self.nz)
# Optional
#anistotrpoic
#wrap_around
#radcon
@staticmethod
def _pasrse_cell_list(line):
line_list = []
for seg in line:
if '*' in seg:
sp = seg.split('*')
seg_arr = np.ones((int(sp[0]), ), dtype=float) * float(sp[1])
else:
seg_arr = np.array([float(seg)], dtype=float)
line_list.append(seg_arr)
return np.concatenate(line_list)
@staticmethod
def __pasrseCellList(line):
return MeshSpecifications._pasrse_cell_list(line)
@classmethod
def _create(cls, values, validate=False):
if not isinstance(values, dict):
raise RuntimeError('Input values must be a dictionary')
start_time = time.time()
#print('Creating Mesh Specs...', end='\r')
props = cls()
mat = values.pop('mat')
save = copy.deepcopy(mat)
for k, v in values.items():
if k in cls._props:
# If the value is simple, set it!
if isinstance(cls._props[k], properties.String):
p = props._props.get(k)
props._set(k, p.from_json(v))
else:
# anything that isnt a mat is a list of floats
# make sure to fix any repetitve values us '*' notations
vals = MeshSpecifications.__pasrseCellList(v)
props._set(k, vals)
else:
#warnings.warn("({}:{}) property is not valid.".format(k, v))
pass
# Now handle the material matrix since all other parameters are set
for pref, comp in mat.items():
for name, indices in comp.items():
for idx, ind in enumerate(indices):
ind = [i.replace('nx', '%d' % (props.nx + 1)) for i in ind]
ind = [i.replace('ny', '%d' % (props.ny + 1)) for i in ind]
ind = [i.replace('nz', '%d' % (props.nz + 1)) for i in ind]
# Now shift the indices by one because someone chose +1 indexing :(
ind = [int(i) - 1 for i in ind]
indices[idx] = MaterialComponent(i=ind[0:2],
j=ind[2:4],
k=ind[4:6])
mat[pref][name] = indices
props._set('mat', mat)
# Reutrn the object
values['mat'] = save
#print('Created Mesh Specs in {} seconds.'.format(time.time() - start_time))
if validate:
start_time = time.time()
print('Validating Mesh Specs...', end='\r')
props.validate()
print('Validated Mesh Specs in {} seconds.'.format(time.time() -
start_time))
return props
@property
def definitions(self):
"""Gets the ``mat_type`` definitions as integers to be matched with any
given rocktab file via the lookup table."""
mod = np.empty(self.shape, dtype=int)
lootbl = self.lookup_table.set_index('material')
for el_pref in self.mat.keys():
for mat_type in self.mat[el_pref].keys():
for mc in self.mat[el_pref][mat_type]:
mod[mc.i[0]:mc.i[1] + 1, mc.j[0]:mc.j[1] + 1,
mc.k[0]:mc.k[1] + 1] = lootbl.loc[mat_type]['id']
return mod.flatten(order='f')
@property
def injector(self):
mod = np.full(self.shape, False, dtype=bool)
for mat_type in self.mat['wb1'].keys():
for mc in self.mat['wb1'][mat_type]:
mod[mc.i[0]:mc.i[1] + 1, mc.j[0]:mc.j[1] + 1,
mc.k[0]:mc.k[1] + 1] = True
return mod.flatten(order='f')
@property
def materials(self):
mats = []
for el_pref in self.mat.keys():
for mat_type in self.mat[el_pref].keys():
mats.append(mat_type)
return list(set(mats))
@property
def lookup_table(self):
mats = self.materials
df = pd.DataFrame(data=mats, columns=['material'])
df['id'] = pd.factorize(df['material'])[0]
return df
def to_tensor_mesh(self):
return discretize.TensorMesh(h=[self.dx, self.dy, self.dz])
def toTensorMesh(self):
return self.to_tensor_mesh()
def to_rectilinear_grid(self):
"""Create a PyVista ``RectilinearGrid``."""
import pyvista
return pyvista.RectilinearGrid(self.dx, self.dy, self.dz)
def handler(event, context):
props = properties.load(
event,
{
'ClientApps':
properties.StringOrListOfString(),
'ExplicitAuthFlows':
properties.StringOrListOfString(default=[]),
'RefreshTokenValidity':
properties.String('30'),
'ConfigurationKey':
properties.String(
), ##this is only here to force the resource handler to execute on each update to the deployment
'LambdaConfig':
properties.Dictionary({}),
'PoolName':
properties.String(),
'Groups':
properties.ObjectOrListOfObject(
default=[],
schema={
'Name': properties.String(),
'Description': properties.String(''),
'Role': properties.String(),
'Precedence': properties.String('99')
}),
'AllowAdminCreateUserOnly':
properties.String('')
})
#give the identity pool a unique name per stack
stack_name = aws_utils.get_stack_name_from_stack_arn(event['StackId'])
stack_name = stack_name.replace(
'-', ' '
) # Prepare stack_name to be used by _associate_user_pool_with_player_access
pool_name = props.PoolName.replace('-', ' ')
pool_name = stack_name + pool_name
cognito_idp_client = user_pool.get_idp_client()
pool_id = event.get('PhysicalResourceId')
found_pool = user_pool.get_user_pool(pool_id)
request_type = event['RequestType']
if request_type == 'Delete':
if found_pool != None:
cognito_idp_client.delete_user_pool(UserPoolId=pool_id)
data = {}
else:
#if the pool exists just update it, otherwise create a new one
mfaConfig = 'OFF' # MFA is currently unsupported by Lumberyard
# Users are automatically prompted to verify these things.
# At least one auto-verified thing (email or phone) is required to allow password recovery.
auto_verified_attributes = ['email']
client_app_data = {}
lambda_config = props.LambdaConfig
user_pool.validate_identity_metadata(event['StackId'],
event['LogicalResourceId'],
props.ClientApps)
admin_create_user_config = __get_admin_create_user_config(
props.AllowAdminCreateUserOnly)
print json.dumps(admin_create_user_config)
if found_pool != None: # Update
response = cognito_idp_client.update_user_pool(
UserPoolId=pool_id,
MfaConfiguration=mfaConfig,
AutoVerifiedAttributes=auto_verified_attributes,
LambdaConfig=lambda_config,
AdminCreateUserConfig=admin_create_user_config)
existing_client_apps = user_pool.get_client_apps(pool_id)
client_app_data = update_client_apps(pool_id, props.ClientApps,
existing_client_apps, False,
props.ExplicitAuthFlows,
props.RefreshTokenValidity)
response = cognito_idp_client.list_groups(UserPoolId=pool_id)
found_groups = {}
for actual_group in response['Groups']:
group_name = actual_group['GroupName']
for requested_group in props.Groups:
# does the group exist in the resource template
if group_name == requested_group.Name:
found_groups.update({group_name: True})
break
#delete the group as it is no longer in the resource template
if group_name not in found_groups:
cognito_idp_client.delete_group(
GroupName=actual_group['GroupName'],
UserPoolId=pool_id)
print "Found groups=>", json.dumps(found_groups)
#iterate the groups defined in the user pool resource template
for group in props.Groups:
#update the group as it is currently a group in the user pool
group_definition = __generate_group_defintion(pool_id, group)
print "Group '{}' is defined by {}".format(
group.Name, json.dumps(group_definition))
if group.Name in found_groups:
cognito_idp_client.update_group(**group_definition)
else:
#group is a new group on the user pool
cognito_idp_client.create_group(**group_definition)
else: # Create
response = cognito_idp_client.create_user_pool(
PoolName=pool_name,
MfaConfiguration=mfaConfig,
AutoVerifiedAttributes=auto_verified_attributes,
LambdaConfig=lambda_config,
AdminCreateUserConfig=admin_create_user_config)
pool_id = response['UserPool']['Id']
print 'User pool creation response: ', response
for group in props.Groups:
group_definition = __generate_group_defintion(pool_id, group)
print "Group '{}' is defined by {}".format(
group.Name, json.dumps(group_definition))
cognito_idp_client.create_group(**group_definition)
client_app_data = update_client_apps(pool_id, props.ClientApps, [],
False,
props.ExplicitAuthFlows,
props.RefreshTokenValidity)
updated_resources = {
event['StackId']: {
event['LogicalResourceId']: {
'physical_id': pool_id,
'client_apps': {
client_app['ClientName']: {
'client_id': client_app['ClientId']
}
for client_app in client_app_data['Created'] +
client_app_data['Updated']
}
}
}
}
identity_pool.update_cognito_identity_providers(
event['StackId'], pool_id, updated_resources)
data = {
'UserPoolName': pool_name,
'UserPoolId': pool_id,
'ClientApps': client_app_data,
}
physical_resource_id = pool_id
custom_resource_response.succeed(event, context, data,
physical_resource_id)
def _test_dict(self, om):
with self.assertRaises(TypeError):
properties.Dictionary('bad string set', key_prop=str)
with self.assertRaises(TypeError):
properties.Dictionary('bad string set', value_prop=str)
with self.assertRaises(TypeError):
properties.Dictionary('bad observe',
properties.Integer(''),
observe_mutations=5)
class HasPropsDummy(properties.HasProperties):
pass
mydict = properties.Dictionary('dummy has properties set',
key_prop=properties.String(''),
value_prop=HasPropsDummy,
observe_mutations=om)
assert isinstance(mydict.key_prop, properties.String)
assert isinstance(mydict.value_prop, properties.Instance)
assert mydict.value_prop.instance_class is HasPropsDummy
class HasDummyDict(properties.HasProperties):
mydict = properties.Dictionary('dummy has properties set',
key_prop=properties.String(''),
value_prop=HasPropsDummy,
observe_mutations=om)
assert HasDummyDict()._props['mydict'].name == 'mydict'
assert HasDummyDict()._props['mydict'].key_prop.name == 'mydict'
assert HasDummyDict()._props['mydict'].value_prop.name == 'mydict'
class HasDict(properties.HasProperties):
aaa = properties.Dictionary('dictionary', default=dict)
li = HasDict()
li.aaa = {1: 2}
with self.assertRaises(ValueError):
li.aaa = (1, 2, 3)
li.aaa = {'hi': HasPropsDummy()}
with self.assertRaises(ValueError):
li.aaa = 4
with self.assertRaises(ValueError):
li.aaa = {'a', 'b', 'c'}
li1 = HasDict()
li2 = HasDict()
assert li1.aaa == li2.aaa
assert li1.aaa is not li2.aaa
class HasInt(properties.HasProperties):
myint = properties.Integer('my integer')
class HasFunnyDict(properties.HasProperties):
mydict = properties.Dictionary('my dict',
key_prop=properties.Color(''),
value_prop=HasInt,
observe_mutations=om)
hfd = HasFunnyDict()
hfd.mydict = {'red': HasInt(myint=5)}
hfd.mydict.update({'green': HasInt(myint=1)})
if om:
hfd.validate()
else:
with self.assertRaises(ValueError):
hfd.validate()
with self.assertRaises(ValueError):
hfd.mydict.update({1: HasInt(myint=1)})
hfd.validate()
class HasCoercedDict(properties.HasProperties):
my_coerced_dict = properties.Dictionary('my dict', coerce=True)
my_uncoerced_dict = properties.Dictionary('my dict')
key_val_list = [('a', 1), ('b', 2), ('c', 3)]
hcd = HasCoercedDict()
with self.assertRaises(ValueError):
hcd.my_uncoerced_dict = key_val_list
hcd.my_coerced_dict = key_val_list
assert hcd.my_coerced_dict == {'a': 1, 'b': 2, 'c': 3}
with self.assertRaises(properties.ValidationError):
hcd.my_coerced_dict = 'a'
class Fields(properties.HasProperties):
"""Fancy Field Storage
.. code::python
fields = Fields(
simulation=simulation, knownFields={"phi": "CC"}
)
fields[:,'phi'] = phi
print(fields[src0,'phi'])
"""
simulation = properties.Instance("a SimPEG simulation", BaseSimulation)
knownFields = properties.Dictionary(
"""
a dictionary with the names of the know fields and their location on
a mesh e.g. {"e": "E", "phi": "CC"}
""",
required=True,
)
aliasFields = properties.Dictionary(
"""
a dictionary of the aliased fields with [alias, location, function],
e.g. {"b":["e","F",lambda(F,e,ind)]}
""",
default={},
)
#: dtype is the type of the storage matrix. This can be a dictionary.
dtype = float
def __init__(self, simulation=None, **kwargs):
super(Fields, self).__init__(**kwargs)
if simulation is not None:
self.simulation = simulation
self._fields = {}
self.startup()
@properties.validator("knownFields")
def _check_overlap_with_aliased(self, change):
allFields = [k
for k in change["value"]] + [a for a in self.aliasFields]
assert len(allFields) == len(
set(allFields)
), "Aliased fields and Known Fields have overlapping definitions."
@properties.validator("aliasFields")
def _check_overlap_with_known(self, change):
allFields = [k
for k in self.knownFields] + [a for a in change["value"]]
assert len(allFields) == len(
set(allFields)
), "Aliased fields and Known Fields have overlapping definitions."
@property
def mesh(self):
return self.simulation.mesh
@property
def survey(self):
return self.simulation.survey
def startup(self):
pass
@property
def approxSize(self):
"""The approximate cost to storing all of the known fields."""
sz = 0.0
for f in self.knownFields:
loc = self.knownFields[f]
sz += np.array(self._storageShape(loc)).prod() * 8.0 / (1024**2)
return "{0:e} MB".format(sz)
def _storageShape(self, loc):
nSrc = self.survey.nSrc
nP = {
"CC": self.mesh.nC,
"N": self.mesh.nN,
"F": self.mesh.nF,
"E": self.mesh.nE,
}[loc]
return (nP, nSrc)
def _initStore(self, name):
if name in self._fields:
return self._fields[name]
assert name in self.knownFields, "field name is not known."
loc = self.knownFields[name]
if isinstance(self.dtype, dict):
dtype = self.dtype[name]
else:
dtype = self.dtype
# field = zarr.create(self._storageShape(loc), dtype=dtype)
field = np.zeros(self._storageShape(loc), dtype=dtype)
self._fields[name] = field
return field
def _srcIndex(self, srcTestList):
if type(srcTestList) is slice:
ind = srcTestList
else:
ind = self.survey.getSourceIndex(srcTestList)
return ind
def _nameIndex(self, name, accessType):
if type(name) is slice:
assert name == slice(
None, None,
None), "Fancy field name slicing is not supported... yet."
name = None
if name is None:
return
if accessType == "set" and name not in self.knownFields:
if name in self.aliasFields:
raise KeyError(
"Invalid field name ({0!s}) for setter, you can't "
"set an aliased property".format(name))
else:
raise KeyError(
"Invalid field name ({0!s}) for setter".format(name))
elif accessType == "get" and (name not in self.knownFields
and name not in self.aliasFields):
raise KeyError(
"Invalid field name ({0!s}) for getter".format(name))
return name
def _indexAndNameFromKey(self, key, accessType):
if not isinstance(key, tuple):
key = (key, )
if len(key) == 1:
key += (None, )
assert len(key) == 2, "must be [Src, fieldName]"
srcTestList, name = key
name = self._nameIndex(name, accessType)
ind = self._srcIndex(srcTestList)
return ind, name
def __setitem__(self, key, value):
ind, name = self._indexAndNameFromKey(key, "set")
if name is None:
assert (
isinstance(value, dict)
), "New fields must be a dictionary, if field is not specified."
newFields = value
elif name in self.knownFields:
newFields = {name: value}
else:
raise Exception("Unknown setter")
for name in newFields:
field = self._initStore(name)
self._setField(field, newFields[name], name, ind)
def __getitem__(self, key):
ind, name = self._indexAndNameFromKey(key, "get")
if name is None:
out = {}
for name in self._fields:
out[name] = self._getField(name, ind)
return out
return self._getField(name, ind)
def _setField(self, field, val, name, ind):
if isinstance(val, np.ndarray) and (field.shape[0] == field.size
or val.ndim == 1):
val = mkvc(val, 2)
field[:, ind] = val
def _getField(self, name, ind):
if name in self._fields:
out = self._fields[name][:, ind]
else:
# Aliased fields
alias, loc, func = self.aliasFields[name]
srcII = np.array(self.survey.source_list)[ind]
srcII = srcII.tolist()
if isinstance(func, string_types):
assert hasattr(self, func), (
"The alias field function is a string, but it does not "
"exist in the Fields class.")
func = getattr(self, func)
out = func(self._fields[alias][:, ind], srcII)
if out.shape[0] == out.size or out.ndim == 1:
out = mkvc(out, 2)
return out
def __contains__(self, other):
if other in self.aliasFields:
other = self.aliasFields[other][0]
return self._fields.__contains__(other)
class HasDict(properties.HasProperties):
aaa = properties.Dictionary('dictionary', default=dict)
class BaseRx(properties.HasProperties):
"""SimPEG Receiver Object"""
# TODO: write a validator that checks against mesh dimension in the
# BaseSimulation
# TODO: location
locations = RxLocationArray("Locations of the receivers (nRx x nDim)",
shape=("*", "*"),
required=True)
# TODO: project_grid?
projGLoc = properties.StringChoice(
"Projection grid location, default is CC",
choices=["CC", "Fx", "Fy", "Fz", "Ex", "Ey", "Ez", "N"],
default="CC",
)
# TODO: store_projections
storeProjections = properties.Bool(
"Store calls to getP (organized by mesh)", default=True)
_uid = properties.Uuid("unique ID for the receiver")
_Ps = properties.Dictionary("dictonary for storing projections", )
def __init__(self, locations=None, **kwargs):
super(BaseRx, self).__init__(**kwargs)
if locations is not None:
self.locations = locations
rxType = kwargs.pop("rxType", None)
if rxType is not None:
warnings.warn(
"BaseRx no longer has an rxType. Each rxType should instead "
"be a different receiver class.")
if getattr(self, "_Ps", None) is None:
self._Ps = {}
locs = deprecate_property(locations,
"locs",
new_name="locations",
removal_version="0.15.0")
@property
def nD(self):
"""Number of data in the receiver."""
return self.locations.shape[0]
def getP(self, mesh, projGLoc=None):
"""
Returns the projection matrices as a
list for all components collected by
the receivers.
.. note::
Projection matrices are stored as a dictionary listed by meshes.
"""
if projGLoc is None:
projGLoc = self.projGLoc
if (mesh, projGLoc) in self._Ps:
return self._Ps[(mesh, projGLoc)]
P = mesh.getInterpolationMat(self.locations, projGLoc)
if self.storeProjections:
self._Ps[(mesh, projGLoc)] = P
return P
def eval(self, **kwargs):
raise NotImplementedError(
"the eval method for {} has not been implemented".format(self))
def evalDeriv(self, **kwargs):
raise NotImplementedError(
"the evalDeriv method for {} has not been implemented".format(
self))
class HasCoercedDict(properties.HasProperties):
my_coerced_dict = properties.Dictionary('my dict', coerce=True)
my_uncoerced_dict = properties.Dictionary('my dict')
api_gateway = aws_utils.ClientWrapper(boto3.client('apigateway'))
API_GATEWAY_SERVICE_NAME = 'apigateway.amazonaws.com'
STAGE_NAME = 'api'
PROPERTY_SCHEMA = {
'ConfigurationBucket':
properties.String(),
'ConfigurationKey':
properties.String(),
'CacheClusterEnabled':
properties.Boolean(default=False),
'CacheClusterSize':
properties.String(default='0.5'),
'SwaggerSettings':
properties.Dictionary(default={}),
'MethodSettings':
properties.Object(
default={},
schema={
'*':
properties.Object(
default={}, # path, can be *
schema={
'*':
properties.Object(
default={}, # method, can be *
schema={
'CacheDataEncrypted':
properties.Boolean(default=False),
'CacheTtlInSeconds':
