def rf_from_lib_data(self, lib_data):
"""
Construct RF object from `lib_data`.
Parameters
----------
lib_data : list
RF envelope.
Returns
-------
rf : SimpleNamespace
RF object constructed from lib_data.
"""
rf = SimpleNamespace()
rf.type = 'rf'
amplitude, mag_shape, phase_shape = lib_data[0], lib_data[1], lib_data[2]
shape_data = self.shape_library.data[mag_shape]
compressed = SimpleNamespace()
compressed.num_samples = shape_data[0]
compressed.data = shape_data[1:]
mag = decompress_shape(compressed)
shape_data = self.shape_library.data[phase_shape]
compressed.num_samples = shape_data[0]
compressed.data = shape_data[1:]
phase = decompress_shape(compressed)
rf.signal = 1j * 2 * np.pi * phase
rf.signal = amplitude * mag * np.exp(rf.signal)
rf.t = np.arange(1, max(mag.shape) + 1) * self.rf_raster_time
if max(lib_data.shape) < 6:
rf.delay = 0
rf.freq_offset = lib_data[3]
rf.phase_offset = lib_data[4]
lib_data = np.append(lib_data, 0)
else:
rf.delay = lib_data[3]
rf.freq_offset = lib_data[4]
rf.phase_offset = lib_data[5]
if max(lib_data.shape) < 7:
lib_data = np.append(lib_data, 0)
rf.dead_time = lib_data[6]
if max(lib_data.shape) < 8:
lib_data = np.append(lib_data, 0)
rf.ringdown_time = lib_data[7]
if max(lib_data.shape) < 9:
lib_data = np.append(lib_data, 0)
use_cases = {1: 'excitation', 2: 'refocusing', 3: 'inversion'}
if lib_data[8] in use_cases:
rf.use = use_cases[lib_data[8]]
return rf
def get_image_data(imgfile, first_row):
with Image.open(imgfile) as img:
out = SimpleNamespace()
out.width = img.width
data = list(img.getdata())
if len(data[0]) > 3:
data = [d[:3] for d in data]
if first_row:
out.height = 1
out.data = data[:out.width]
else:
out.height = img.height
out.data = data
return out
def parseCoro():
context = SimpleNamespace(
round=False, turn=0,
round_id=0, action_order=0,
data=None
)
pid = None
while True:
line = yield (pid, context.data)
context.data = None
pid, mch = parseLine(line)
if pid in state_change:
updateContext(pid, context)
if pid in gather and context.round:
context.data = extractData(pid, mch, context)
def get_paths(self, l2a_outdir: str):
"""Get all file paths associated with the current instance of EnMAP_Detector_SensorGeo.
NOTE: This information is read from the detector_meta.
:param l2a_outdir: output directory of EnMAP Level-2A dataset
:return: paths as SimpleNamespace
"""
paths = SimpleNamespace()
paths.root_dir = l2a_outdir
paths.metaxml = path.join(l2a_outdir, self.meta.filename_metaxml)
paths.data = path.join(l2a_outdir, self.meta.filename_data)
paths.mask_landwater = path.join(l2a_outdir,
self.meta.filename_mask_landwater)
paths.mask_clouds = path.join(l2a_outdir,
self.meta.filename_mask_clouds)
paths.mask_cloudshadow = path.join(l2a_outdir,
self.meta.filename_mask_cloudshadow)
paths.mask_haze = path.join(l2a_outdir, self.meta.filename_mask_haze)
paths.mask_snow = path.join(l2a_outdir, self.meta.filename_mask_snow)
paths.mask_cirrus = path.join(l2a_outdir,
self.meta.filename_mask_cirrus)
paths.deadpixelmap_vnir = path.join(
l2a_outdir, self.meta.filename_deadpixelmap_vnir)
paths.deadpixelmap_swir = path.join(
l2a_outdir, self.meta.filename_deadpixelmap_swir)
paths.quicklook_vnir = path.join(l2a_outdir,
self.meta.filename_quicklook_vnir)
paths.quicklook_swir = path.join(l2a_outdir,
self.meta.filename_quicklook_swir)
return paths
def read_data(csv, config):
with open(config) as fin:
use = fin.readline().strip().split(',')
ref = fin.readline().strip()
with open(csv) as fin:
header = fin.readline().strip().split(',')
use.sort(key=lambda i: header.index(i))
data_matrix = np.array([], dtype=np.float).reshape(0, len(use))
ref_array = []
ref_data = None
for line in fin:
entry = line.strip().split(',')
use_data = []
del ref_data
for i, val in enumerate(entry):
label = header[i]
if label in use:
use_data.append(np.log(float(val)))
elif label == ref:
ref_data = int(val)
data_matrix = np.vstack((data_matrix, use_data))
ref_array.append(ref_data)
ret = SimpleNamespace()
ret.data_header = use
ret.data = data_matrix
ret.flag = ref_array
ret.samples = len(ref_array)
return ret
def datagen(savedir=None):
# img = imread(f'/lustre/projects/project-broaddus/rawdata/artifacts/flower.tif')[:10]
img = imread(
f'/lustre/projects/project-broaddus/denoise_experiments/flower/e02/pred_flower.tif'
)[:10]
# img = imread(f'/lustre/projects/project-broaddus/rawdata/artifacts/shutterclosed.tif')[0]
print(img.shape)
# pmin, pmax = np.random.uniform(1,3), np.random.uniform(99.5,99.8)
pmin, pmax = 2, 99.6
print(f"pmin = {pmin}; pmax = {pmax}")
img = normalize3(img, pmin, pmax).astype(np.float32, copy=False)
data = img.reshape((-1, 4, 256, 4, 256)).transpose(
(0, 1, 3, 2, 4)).reshape((-1, 1, 256, 256))
# patch_size = (256,256)
# slicelist = []
# def random_patch():
# ss = random_slice(img.shape, patch_size)
# ## select patches with interesting content. FIXME
# while img[ss].mean() < 0.0:
# ss = random_slice(img.shape, patch_size)
# x = img[ss].copy()
# slicelist.append(ss)
# ## augment
# # noiselevel = 0.2
# # x += np.random.uniform(0,noiselevel,(1,)*3)*np.random.uniform(-1,1,x.shape)
# # for d in [0,1,2]:
# # if np.random.rand() < 0.5:
# # x = np.flip(x,d)
# return (x,)
# data = np.array([random_patch() for _ in range(24)])
# data = np.load('../../devseg_data/cl_datagen/d003/data.npz')
print("data.shape: ", data.shape)
#SCZYX
if savedir:
rgb = collapse2(data[:, :], 'scyx', 's,y,x,c')[..., [0, 0, 0]]
rgb = normalize3(rgb)
rgb = plotgrid([rgb], 10)
io.imsave(savedir / 'data_xy_flower.png', rgb)
np.savez_compressed(savedir / 'data_flower.npz',
data=data,
pmin=pmin,
pmax=pmax)
# pklsave(slicelist, savedir/'slicelist2.pkl')
dg = SimpleNamespace()
dg.data = data
dg.pmin = pmin
dg.pmax = pmax
return dg
def get_data(path, style=dict(), with_info=False):
dataset = SN()
if with_info:
data, info = fym.logging.load(path, with_info=with_info)
dataset.info = info
dataset.style = style | dict(label=info["cfg"].label)
else:
data = fym.logging.load(path)
dataset.style = style
dataset.data = data
return dataset
def parseRSRCVariant(self, bldata):
flags = int.from_bytes(bldata.read(2), byteorder='big', signed=False)
self.vType = flags & 0x1FFF
self.vFlags = flags & ~0x1FFF
vSize = OleVariant.vTypeToSize(self.vType)
self.dimensions = []
self.vData = []
if (self.vFlags & 0x2000) != 0:
ndim = int.from_bytes(bldata.read(2), byteorder='big', signed=False)
totlen = 1
for i in range(ndim):
client = SimpleNamespace()
client.prop1 = int.from_bytes(bldata.read(4), byteorder='big', signed=False)
client.prop2 = int.from_bytes(bldata.read(4), byteorder='big', signed=False)
self.dimensions.append(client)
totlen *= client.prop2
if ndim == 0: totlen = 0
else:
totlen = 1
if self.vType == 8:
for i in range(totlen):
itmlen = int.from_bytes(bldata.read(4), byteorder='big', signed=False)
client = SimpleNamespace()
client.data = bldata.read(2*itmlen)
self.vData.append(client)
elif self.vType == 12:
for i in range(totlen):
# Getting recursive
client = SimpleNamespace()
client.obj = LVclasses.OleVariant(0, self.vi, self.po)
client.obj.parseRSRCData(bldata)
self.vData.append(client)
else:
for i in range(totlen):
client = SimpleNamespace()
client.data = bldata.read(vSize)
self.vData.append(client)
pass
def init(config):
# This gives dot access to all paths, hyperparameters, etc
args = SimpleNamespace(**config)
assertions.validate_hypers(args)
args = mlh.detect_cuda(args)
mlh.seed_all(args.seed)
# get data
args.data = data_handler.get_dataset(args)
# get model
args.model = model_handler.get_model(args)
return args
def load_config(config_filepath, scale_factor=1):
# load file
with open(config_filepath, "r") as infile:
cfg_dict = json.load(infile)
print(config_filepath)
print(cfg_dict)
cfg = SimpleNamespace()
cfg.data = SimpleNamespace(**cfg_dict["data"])
cfg.graph = SimpleNamespace(**cfg_dict["graph"])
(cfg.graph.pylon_dist_min,
cfg.graph.pylon_dist_max) = compute_pylon_dists(
cfg.data.pylon_dist_min, cfg.data.pylon_dist_max, cfg.data.raster,
scale_factor
)
return cfg
def fig_as_data(plt, fig, with_alpha=False):
plt.axis('off')
ns = SimpleNamespace()
try:
yield ns
except:
raise
else:
fig.canvas.draw()
# noinspection PyProtectedMember
data = np.array(fig.canvas.renderer._renderer)
if not with_alpha:
data = data[..., :3]
ns.data = data
finally:
plt.close()
def compress_shape(decompressed_shape: np.ndarray) -> SimpleNamespace:
"""
Returns a run-length encoded compressed shape.
Parameters
----------
decompressed_shape : numpy.ndarray
Decompressed shape.
Returns
-------
compressed_shape : SimpleNamespace
A `SimpleNamespace` object containing the compressed data and corresponding shape.
"""
quant_factor = 1e-7
decompressed_shape_scaled = decompressed_shape / quant_factor
datq = np.round(
np.insert(np.diff(decompressed_shape_scaled), 0,
decompressed_shape_scaled[0]))
qerr = decompressed_shape_scaled - np.cumsum(datq)
qcor = np.insert(np.diff(np.round(qerr)), 0, 0)
datd = datq + qcor
mask_changes = np.insert(np.asarray(np.diff(datd) != 0, dtype=np.int), 0,
1)
vals = datd[mask_changes.nonzero()[0]] * quant_factor
k = np.append(mask_changes, 1).nonzero()[0]
n = np.diff(k)
n_extra = (n - 2).astype(
np.float32) # Cast as float for nan assignment to work
vals2 = np.copy(vals)
vals2[n_extra < 0] = np.nan
n_extra[n_extra < 0] = np.nan
v = np.stack((vals, vals2, n_extra))
v = v.T[np.isfinite(
v).T] # Use transposes to match Matlab's Fortran indexing order
v[abs(v) < 1e-10] = 0
compressed_shape = SimpleNamespace()
compressed_shape.num_samples = len(decompressed_shape)
compressed_shape.data = v
return compressed_shape
def compress_shape(decompressed_shape):
"""
Returns a run-length encoded compressed shape.
Parameters
----------
decompressed_shape : ndarray
Decompressed shape.
Returns
-------
compressed_shape : Holder
A Holder object containing the shape of the compressed shape ndarray and the compressed shape ndarray itself.
"""
quant_factor = 1e-7
decompressed_shape_scaled = decompressed_shape / quant_factor
datq = np.round(
np.insert(np.diff(decompressed_shape_scaled), 0,
decompressed_shape_scaled[0]))
qerr = decompressed_shape_scaled - np.cumsum(datq)
qcor = np.insert(np.diff(np.round(qerr)), 0, 0)
datd = datq + qcor
mask_changes = np.insert(np.diff(datd) != 0, 0, 1)
vals = np.multiply(datd[mask_changes], quant_factor).astype(np.float)
k = np.where(np.append(mask_changes, 1) != 0)[0]
n = np.diff(k)
n_extra = (n - 2).astype(
np.float16) # Cast as float for nan assignment to work
vals2 = np.copy(vals)
vals2[n_extra < 0] = np.nan
n_extra[n_extra < 0] = np.nan
v = np.stack((vals, vals2, n_extra))
v = v.T[np.isfinite(
v).T] # Use transposes to match Matlab's Fortran indexing order
v[abs(v) < 1e-10] = 0
compressed_shape = SimpleNamespace()
compressed_shape.num_samples = len(decompressed_shape)
compressed_shape.data = v
return compressed_shape
def main():
args, protoc_args = parse_args()
if args.decode_raw or args.encode_raw:
message = Empty()
else:
message_name, protofile = args.encode or [] + args.decode or []
protofile = Path(protofile)
os.makedirs(str(TMP_PROTOD_DIR), exist_ok=True)
check_output(['protoc', f'--python_out={TMP_PROTOD_DIR}'] +
protoc_args + [str(protofile)])
pb2_path = TMP_PROTOD_DIR / (protofile.with_suffix('').name +
'_pb2.py')
spec = importlib.util.spec_from_file_location(
protofile.with_suffix('').name, pb2_path)
pb2 = importlib.util.module_from_spec(spec)
spec.loader.exec_module(pb2)
message = getattr(pb2, message_name)()
if args.decode_raw or args.decode:
message.ParseFromString(sys.stdin.buffer.read())
else:
text_format.Parse(sys.stdin.read(),
message,
allow_field_number=True,
allow_unknown_field=True)
if args.proto_out:
unf_like = SimpleNamespace()
unf_like.wire_type = wire_format.WIRETYPE_LENGTH_DELIMITED
unf_like.data = message.SerializeToString()
unf_like.field_number = 0
print(render_proto(merge_definition(define(unf_like)),
args.proto_out == '3'),
end='')
elif args.decode_raw or args.decode:
print(text_format.MessageToString(message, print_unknown_fields=True),
end='')
else:
sys.stdout.buffer.write(message.SerializeToString())
def __call__(self, model, optimiser, niters, device, hooks=None):
"""Train the `model` on `device` for `niters`.
Args:
model:
device:
optimiser:
niters:
Returns:
"""
state = SimpleNamespace(
trainer=self,
model=model,
device=device,
niters=niters,
iter=0,
epoch=0,
data=None,
target=None,
loss=None,
model_outputs=None,
)
try:
model.train()
i = 0
epoch = 0
while i < niters:
epoch += 1
for data, target in self.dataloader:
data = data.to(device)
target = target.to(device)
loss, outputs = self.step(
model=model, optimiser=optimiser, data=data, target=target
)
if hooks is not None:
hooks_to_exec = tuple(
hook for hook, freq in hooks if i % freq == 0
)
if hooks_to_exec:
state.iter = i
state.data = data
state.target = target
state.epoch = epoch
state.loss = loss
state.model_outputs = outputs
for hook in hooks_to_exec:
hook(state)
state.data = None
state.target = None
state.loss = None
state.model_outputs = None
i += 1
if i == niters:
break
except StopTraining:
return
def render_class(state: State, path, class_, env):
logging.debug("generating %s.html", '.'.join(path))
# Call all registered page begin hooks
for hook in state.hooks_pre_page: hook()
url_base = ''
breadcrumb = []
for i in path:
url_base += i + '.'
breadcrumb += [(i, url_base + 'html')]
page = Empty()
page.summary = extract_summary(state, state.class_docs, path, class_.__doc__)
page.url = breadcrumb[-1][1]
page.breadcrumb = breadcrumb
page.prefix_wbr = '.<wbr />'.join(path + [''])
page.classes = []
page.enums = []
page.classmethods = []
page.staticmethods = []
page.dunder_methods = []
page.methods = []
page.properties = []
page.data = []
page.has_enum_details = False
# External page content, if provided
path_str = '.'.join(path)
if path_str in state.class_docs:
page.content = render_rst(state, state.class_docs[path_str]['content'])
state.class_docs[path_str]['used'] = True
# Index entry for this module, returned together with children at the end
index_entry = IndexEntry()
index_entry.kind = 'class'
index_entry.name = breadcrumb[-1][0]
index_entry.url = page.url
index_entry.summary = page.summary
# List of inner classes to render, these will be done after the current
# class introspection is done to have some better memory allocation pattern
classes_to_render = []
# Get inner classes
for name, object in inspect.getmembers(class_, lambda o: inspect.isclass(o) and not is_enum(state, o)):
if name in ['__base__', '__class__']: continue # TODO
if name.startswith('_'): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
page.classes += [extract_class_doc(state, subpath, object)]
classes_to_render += [(subpath, object)]
# Get enums
for name, object in inspect.getmembers(class_, lambda o: is_enum(state, o)):
if name.startswith('_'): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
enum_ = extract_enum_doc(state, subpath, object)
page.enums += [enum_]
if enum_.has_details: page.has_enum_details = True
# Get methods
for name, object in inspect.getmembers(class_, inspect.isroutine):
# Filter out underscored methods (but not dunder methods)
if is_internal_function_name(name): continue
# Filter out dunder methods that don't have their own docs
if name.startswith('__') and (name, object.__doc__) in _filtered_builtin_functions: continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s() is undocumented", '.'.join(subpath))
for function in extract_function_doc(state, class_, subpath, object):
if name.startswith('__'):
page.dunder_methods += [function]
elif function.is_classmethod:
page.classmethods += [function]
elif function.is_staticmethod:
page.staticmethods += [function]
else:
page.methods += [function]
# Get properties
for name, object in inspect.getmembers(class_, inspect.isdatadescriptor):
if (name, object.__doc__) in _filtered_builtin_properties:
continue
if name.startswith('_'): continue # TODO: are there any dunder props?
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
page.properties += [extract_property_doc(state, subpath, object)]
# Get data
# TODO: unify this query
for name, object in inspect.getmembers(class_, lambda o: not inspect.ismodule(o) and not inspect.isclass(o) and not inspect.isroutine(o) and not inspect.isframe(o) and not inspect.istraceback(o) and not inspect.iscode(o) and not inspect.isdatadescriptor(o)):
if name.startswith('_'): continue
subpath = path + [name]
page.data += [extract_data_doc(state, class_, subpath, object)]
# Render the class, free the page data to avoid memory rising indefinitely
render(state.config, 'class.html', page, env)
del page
# Render subclasses
for subpath, object in classes_to_render:
index_entry.children += [render_class(state, subpath, object, env)]
return index_entry
def render_module(state: State, path, module, env):
logging.debug("generating %s.html", '.'.join(path))
# Call all registered page begin hooks
for hook in state.hooks_pre_page: hook()
url_base = ''
breadcrumb = []
for i in path:
url_base += i + '.'
breadcrumb += [(i, url_base + 'html')]
page = Empty()
page.summary = extract_summary(state, state.module_docs, path, module.__doc__)
page.url = breadcrumb[-1][1]
page.breadcrumb = breadcrumb
page.prefix_wbr = '.<wbr />'.join(path + [''])
page.modules = []
page.classes = []
page.enums = []
page.functions = []
page.data = []
page.has_enum_details = False
# External page content, if provided
path_str = '.'.join(path)
if path_str in state.module_docs:
page.content = render_rst(state, state.module_docs[path_str]['content'])
state.module_docs[path_str]['used'] = True
# Index entry for this module, returned together with children at the end
index_entry = IndexEntry()
index_entry.kind = 'module'
index_entry.name = breadcrumb[-1][0]
index_entry.url = page.url
index_entry.summary = page.summary
# List of inner modules and classes to render, these will be done after the
# current class introspection is done to have some better memory allocation
# pattern
modules_to_render = []
classes_to_render = []
# This is actually complicated -- if the module defines __all__, use that.
# The __all__ is meant to expose the public API, so we don't filter out
# underscored things.
if hasattr(module, '__all__'):
# Names exposed in __all__ could be also imported from elsewhere, for
# example this is a common pattern with native libraries and we want
# Foo, Bar, submodule and *everything* in submodule to be referred to
# as `library.RealName` (`library.submodule.func()`, etc.) instead of
# `library._native.Foo`, `library._native.sub.func()` etc.
#
# from ._native import Foo as PublicName
# from ._native import sub as submodule
# __all__ = ['PublicName', 'submodule']
#
# The name references can be cyclic so extract the mapping in a
# separate pass before everything else.
for name in module.__all__:
# Everything available in __all__ is already imported, so get those
# directly
object = getattr(module, name)
subpath = path + [name]
# Modules have __name__ while other objects have __module__, need
# to check both.
if inspect.ismodule(object) and object.__name__ != '.'.join(subpath):
assert object.__name__ not in state.module_mapping
state.module_mapping[object.__name__] = '.'.join(subpath)
elif hasattr(object, '__module__'):
subname = object.__module__ + '.' + object.__name__
if subname != '.'.join(subpath):
assert subname not in state.module_mapping
state.module_mapping[subname] = '.'.join(subpath)
# Now extract the actual docs
for name in module.__all__:
object = getattr(module, name)
subpath = path + [name]
# We allow undocumented submodules (since they're often in the
# standard lib), but not undocumented classes etc. Render the
# submodules and subclasses recursively.
if inspect.ismodule(object):
page.modules += [extract_module_doc(state, subpath, object)]
index_entry.children += [render_module(state, subpath, object, env)]
elif inspect.isclass(object) and not is_enum(state, object):
page.classes += [extract_class_doc(state, subpath, object)]
index_entry.children += [render_class(state, subpath, object, env)]
elif inspect.isclass(object) and is_enum(state, object):
enum_ = extract_enum_doc(state, subpath, object)
page.enums += [enum_]
if enum_.has_details: page.has_enum_details = True
elif inspect.isfunction(object) or inspect.isbuiltin(object):
page.functions += extract_function_doc(state, module, subpath, object)
# Assume everything else is data. The builtin help help() (from
# pydoc) does the same:
# https://github.com/python/cpython/blob/d29b3dd9227cfc4a23f77e99d62e20e063272de1/Lib/pydoc.py#L113
# TODO: unify this query
elif not inspect.isframe(object) and not inspect.istraceback(object) and not inspect.iscode(object):
page.data += [extract_data_doc(state, module, subpath, object)]
else: # pragma: no cover
logging.warning("unknown symbol %s in %s", name, '.'.join(path))
# Otherwise, enumerate the members using inspect. However, inspect lists
# also imported modules, functions and classes, so take only those which
# have __module__ equivalent to `path`.
else:
# Get (and render) inner modules
for name, object in inspect.getmembers(module, inspect.ismodule):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
page.modules += [extract_module_doc(state, subpath, object)]
modules_to_render += [(subpath, object)]
# Get (and render) inner classes
for name, object in inspect.getmembers(module, lambda o: inspect.isclass(o) and not is_enum(state, o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
page.classes += [extract_class_doc(state, subpath, object)]
classes_to_render += [(subpath, object)]
# Get enums
for name, object in inspect.getmembers(module, lambda o: is_enum(state, o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s is undocumented", '.'.join(subpath))
enum_ = extract_enum_doc(state, subpath, object)
page.enums += [enum_]
if enum_.has_details: page.has_enum_details = True
# Get inner functions
for name, object in inspect.getmembers(module, lambda o: inspect.isfunction(o) or inspect.isbuiltin(o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
subpath = path + [name]
if not object.__doc__: logging.warning("%s() is undocumented", '.'.join(subpath))
page.functions += extract_function_doc(state, module, subpath, object)
# Get data
# TODO: unify this query
for name, object in inspect.getmembers(module, lambda o: not inspect.ismodule(o) and not inspect.isclass(o) and not inspect.isroutine(o) and not inspect.isframe(o) and not inspect.istraceback(o) and not inspect.iscode(o)):
if is_internal_or_imported_module_member(state, module, path, name, object): continue
page.data += [extract_data_doc(state, module, path + [name], object)]
# Render the module, free the page data to avoid memory rising indefinitely
render(state.config, 'module.html', page, env)
del page
# Render submodules and subclasses
for subpath, object in modules_to_render:
index_entry.children += [render_module(state, subpath, object, env)]
for subpath, object in classes_to_render:
index_entry.children += [render_class(state, subpath, object, env)]
return index_entry
from pyproj import CRS, Transformer
import numpy as np
import rasterio as rio
from rasterio.warp import calculate_default_transform, reproject, Resampling
import geopandas as gpd
from osgeo import gdal
from component.message import cm
from component import parameter as cp
planet = SimpleNamespace()
# parameters
planet.url = 'https://api.planet.com/auth/v1/experimental/public/my/subscriptions'
planet.mosaic_name = "planet_medres_normalized_analytic_{}_mosaic"
planet.data = "Planet MedRes"
# attributes
planet.valid = False
planet.key = None
planet.client = None
def check_key():
"""raise an error if the key is not validataed"""
if not planet.valid:
raise Exception(cm.planet.invalid_key)
return
def options(*args, **kw):
"""
Get various options (configuration) as a nested namespace, so that
something like ``opts.a.b.c`` could be used. Note that the `data` top-level
attribute contains the initial inputs (starting grids) of the
:param args: Mappings (dictionaries) that may contain options. Non-mapping
arguments are silently discarded.
:param kw: Additional keywords. They override any items of the same name
present in any of the *args dictionaries.
:return: A nested namespace of options.
"""
# Attribute names for sub-namespaces.
np_cp_s = 'np', 'cp' # NumPy, CuPy
tps_s = 'u', 's' # unsigned, signed
o = Sns(**default_options()) # initialize namespace
od = vars(o)
np_cp = dict(np=np, cp=cp)
for a in (*args, kw):
if isinstance(a, Mapping):
od.update(a)
od.update(all_func_speeds(**od))
for sg in ('shape', 'gens'):
if sg not in od:
od[sg] = od[f'{sg}_grades'][o.grade]
print('setting', sg, 'to', getattr(o, sg))
o.h, o.w = o.shape
o.data = Sns(np=Sns(), cp=Sns())
odd = vars(o.data)
o.blocks_per_grid = *(ceil(tpb / hw)
for (tpb, hw) in zip(o.shape, o.threads_per_block)),
npr = np.random
npr.seed(o.random_seed)
init_np = npr.randint(0, 2, o.shape, dtype=o.int_types[0])
for xp_s in np_cp_s: # for each ('np', 'cp')
xp = np_cp[xp_s] # actual NumPy or CuPy module
odxp = vars(odd[xp_s])
for i, tp_s in enumerate(tps_s): # for each 'u' (unsigned) or 's'
odxp[tp_s] = data = Sns()
data.dtype = dtp = xp.dtype(o.int_types[i])
data.init = xp.array(init_np).astype(dtp)
data.rng = xp.arange(5, 8).astype(dtp)
data.isin = xp.isin
data.ker = ker = xp.full((3, 3), 2 - i, dtype=dtp)
ker[1, 1] = 1
data.ker_flat = ker.flatten()
data.grid = xp.mgrid[:o.h, :o.w]
#
data.ker_idxs = map(sub, xp.indices(ker.shape),
map(rshift, ker.shape, (1, 1)))
data.idxs = *((idx + xp.broadcast_to(kidx.flatten(),
(o.h, o.w, ker.size)).T) % d
for (idx, kidx,
d) in zip(data.grid, data.ker_idxs, o.shape)),
global conv_opts
if 'conv_opts' not in globals() or not isinstance(conv_opts, Sns):
conv_opts = o
vars(conv_opts).update(od)
return conv_opts
