class Goblin(Unit):
"""A Goblin, sworn enemy of the Christmas Elf"""
members = []
# note that using the traitlets type we can defer the dependency on the
# Elf class until the opponents attribute is accessed
opponents = traitlets.Type('__main__.Elf')
class PidginKernel(ipykernel.kernelapp.IPythonKernel):
input = traitlets.Enum(input_formats, default_value="markdown")
def __init__(self, *a, **k):
super().__init__(*a, **k), traitlets.link((self, "input"),
(self.shell, "input"))
shell_class = traitlets.Type(PidginShell)
class VoilaExporter(HTMLExporter):
"""Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer"""
markdown_renderer_class = traitlets.Type('mistune.Renderer').tag(config=True)
# The voila exporter overrides the markdown renderer from the HTMLExporter
# to inline images.
@contextfilter
def markdown2html(self, context, source):
cell = context['cell']
attachments = cell.get('attachments', {})
cls = self.markdown_renderer_class
renderer = cls(escape=False, attachments=attachments,
contents_manager=self.contents_manager,
anchor_link_text=self.anchor_link_text)
return MarkdownWithMath(renderer=renderer).render(source)
# The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter.
@property
def default_config(self):
c = Config({
'CSSHTMLHeaderPreprocessor': {
'enabled': False
},
'VoilaExporter': {
'markdown_renderer_class': 'voila.exporter.VoilaMarkdownRenderer'
}
})
c.merge(super(VoilaExporter, self).default_config)
return c
# Instead, we use the VoilaCSSPreprocessor.
@traitlets.default('preprocessors')
def _default_preprocessors(self):
return ['voila.csspreprocessor.VoilaCSSPreprocessor']
# Overriding the default template file.
@traitlets.default('template_file')
def default_template_file(self):
return 'voila.tpl'
class VoilaExporter(HTMLExporter):
"""Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer"""
base_url = traitlets.Unicode(help="Base url for resources").tag(config=True)
markdown_renderer_class = traitlets.Type('mistune.Renderer').tag(config=True)
# Can be a ContentsManager from notebook or jupyter_server, so Any will have to do for now
contents_manager = traitlets.Any()
# The voila exporter overrides the markdown renderer from the HTMLExporter
# to inline images.
@contextfilter
def markdown2html(self, context, source):
cell = context['cell']
attachments = cell.get('attachments', {})
cls = self.markdown_renderer_class
renderer = cls(escape=False, attachments=attachments,
contents_manager=self.contents_manager,
anchor_link_text=self.anchor_link_text)
return MarkdownWithMath(renderer=renderer).render(source)
# The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter.
@property
def default_config(self):
c = Config({
'VoilaExporter': {
'markdown_renderer_class': 'voila.exporter.VoilaMarkdownRenderer'
}
})
c.merge(super(VoilaExporter, self).default_config)
return c
# Overriding the default template file.
@traitlets.default('template_file')
def default_template_file(self):
return 'index.html.j2'
async def generate_from_notebook_node(self, nb, resources=None, extra_context={}, **kw):
# this replaces from_notebook_node, but calls template.generate instead of template.render
langinfo = nb.metadata.get('language_info', {})
lexer = langinfo.get('pygments_lexer', langinfo.get('name', None))
highlight_code = self.filters.get('highlight_code', Highlight2HTML(pygments_lexer=lexer, parent=self))
self.register_filter('highlight_code', highlight_code)
# NOTE: we don't call HTML or TemplateExporter' from_notebook_node
nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw)
resources.setdefault('raw_mimetypes', self.raw_mimetypes)
resources['global_content_filter'] = {
'include_code': not self.exclude_code_cell,
'include_markdown': not self.exclude_markdown,
'include_raw': not self.exclude_raw,
'include_unknown': not self.exclude_unknown,
'include_input': not self.exclude_input,
'include_output': not self.exclude_output,
'include_input_prompt': not self.exclude_input_prompt,
'include_output_prompt': not self.exclude_output_prompt,
'no_prompt': self.exclude_input_prompt and self.exclude_output_prompt,
}
async for output in self.template.generate_async(nb=nb_copy, resources=resources, **extra_context, static_url=self.static_url):
yield (output, resources)
@property
def environment(self):
# enable Jinja async template execution
self.enable_async = True
env = super(type(self), self).environment
if 'jinja2.ext.do' not in env.extensions:
env.add_extension('jinja2.ext.do')
return env
def get_template_paths(self):
return self.template_path
def static_url(self, path):
"""Mimics tornado.web.RequestHandler.static_url"""
settings = {
'static_url_prefix': f'{self.base_url}voila/templates/',
'static_path': None # not used in TemplateStaticFileHandler.get_absolute_path
}
return TemplateStaticFileHandler.make_static_url(settings, f'{self.template_name}/static/{path}')
def _init_resources(self, resources):
resources = super(VoilaExporter, self)._init_resources(resources)
include_assets_functions = create_include_assets_functions(self.template_name, self.base_url)
resources.update(include_assets_functions)
return resources
class SMAP(podpac.compositor.OrderedCompositor):
"""Compositor of all the SMAPDateFolder's for every available SMAP date. Essentially a compositor of all SMAP data
for a particular product.
Attributes
----------
auth_class : {auth_class}
auth_session : {auth_session}
base_url : {base_url}
date_url_re : SRE_Pattern
Regular expression used to extract all folder dates (or folder names) for the particular SMAP product.
layerkey : {layerkey}
password : {password}
product : str
{product}
username : {username}
"""
base_url = tl.Unicode().tag(attr=True)
@tl.default('base_url')
def _base_url_default(self):
return SMAP_BASE_URL()
product = tl.Enum(SMAP_PRODUCT_MAP.coords['product'].data.tolist(),
default_value='SPL4SMAU').tag(attr=True)
version = tl.Int(allow_none=True).tag(attr=True)
@tl.default('version')
def _detect_product_version(self):
return _infer_SMAP_product_version(self.product, self.base_url, self.auth_session)
date_url_re = re.compile(r'[0-9]{4}\.[0-9]{2}\.[0-9]{2}')
auth_session = tl.Instance(authentication.EarthDataSession)
auth_class = tl.Type(authentication.EarthDataSession)
username = tl.Unicode(None, allow_none=True)
password = tl.Unicode(None, allow_none=True)
cache_type = tl.Enum([None, 'disk', 'ram'], allow_none=True, default_value='disk')
@tl.default('auth_session')
def _auth_session_default(self):
return self.auth_class(username=self.username, password=self.password, product_url=SMAP_BASE_URL())
layerkey = tl.Unicode()
@tl.default('layerkey')
def _layerkey_default(self):
return SMAP_PRODUCT_MAP.sel(
product=self.product,
attr='layerkey').item()
@tl.observe('layerkey')
def _layerkey_change(self, change):
if change['old'] != change['new'] and change['old'] != '':
for s in self.sources:
s.layerkey = change['new']
@property
def source(self):
"""The source is used for a unique name to cache SMAP products.
Returns
-------
str
The SMAP product name.
"""
return '%s.%03d' % (self.product, self.version)
@tl.default('sources')
def sources_default(self):
"""SMAPDateFolder objects pointing to specific SMAP folders
Returns
-------
np.ndarray(dtype=object(SMAPDateFolder))
Array of SMAPDateFolder instances tied to specific SMAP folders
"""
dates = self.get_available_times_dates()[1]
src_objs = np.array([
SMAPDateFolder(product=self.product, version=self.version, folder_date=date,
shared_coordinates=self.shared_coordinates,
auth_session=self.auth_session,
layerkey=self.layerkey)
for date in dates])
return src_objs
@common_doc(COMMON_DOC)
def find_coordinates(self):
'''
{native_coordinates}
Notes
-----
These coordinates are computed, assuming dataset is regular.
'''
if self.product in SMAP_IRREGULAR_COORDINATES:
raise Exception("Native coordinates too large. Try using get_filename_coordinates_sources().")
shared = self.get_shared_coordinates()
partial_sources = self.get_source_coordinates()['time'].coordinates
complete_source_0 = self.sources[0].get_source_coordinates()['time'].coordinates
offset = complete_source_0 - partial_sources[0]
full_times = (partial_sources[:, None] + offset[None, :]).ravel()
return [merge_dims([podpac.Coordinates([full_times], ['time']), shared])]
@common_doc(COMMON_DOC)
def get_source_coordinates(self):
"""{source_coordinates}
"""
return podpac.Coordinates([self.get_available_times_dates()[0]], dims=['time'])
def get_available_times_dates(self):
"""Returns the available folder dates in the SMAP product
Returns
-------
np.ndarray
Array of dates in numpy datetime64 format
list
list of dates in SMAP date format
Raises
------
RuntimeError
If the http resource could not be accessed (check Earthdata login credentials)
"""
url = '/'.join([self.base_url, '%s.%03d' % (self.product, self.version)])
r = _get_from_url(url, self.auth_session)
if r is None:
_logger.warning("Could not contact {} to retrieve source coordinates".format(url))
return np.array([]), []
soup = bs4.BeautifulSoup(r.text, 'lxml')
a = soup.find_all('a')
regex = self.date_url_re
times = []
dates = []
for aa in a:
m = regex.match(aa.get_text())
if m:
times.append(np.datetime64(m.group().replace('.', '-')))
dates.append(m.group())
times.sort()
dates.sort()
return np.array(times), dates
@cache_func('shared.coordinates')
def get_shared_coordinates(self):
"""Coordinates that are shared by all files in the SMAP product family.
Returns
-------
podpac.Coordinates
Coordinates shared by all files in the SMAP product.
Notes
------
For example, the gridded SMAP data have the same lat-lon coordinates in every file (global at some resolution),
and the only difference between files is the time coordinate.
This is not true for the SMAP-Sentinel product, in which case this function returns None
"""
if self.product in SMAP_INCOMPLETE_SOURCE_COORDINATES:
return None
coords = SMAPDateFolder(product=self.product, version=self.version,
folder_date=self.get_available_times_dates()[1][0],
auth_session=self.auth_session,
).shared_coordinates
return coords
def get_filename_coordinates_sources(self, bounds=None):
"""Returns coordinates solely based on the filenames of the sources. This function was motivated by the
SMAP-Sentinel product, which does not have regularly stored tiles (in space and time).
Parameters
-----------
Bounds: podpac.Coordinates, Optional
Default is None. Return the coordinates based on filenames of the source only within the specified bounds. When
not None, the result is not cached.
Returns
-------
podpac.Coordinates
Coordinates of all the sources in the product family
np.ndarray(dtype=object(SMAPSource))
Array of all the SMAPSources pointing to unique OpenDAP urls corresponding to the partial native coordinates
Notes
------
The outputs of this function can be used to find source that overlap spatially or temporally with a subset
region specified by the user.
If 'bounds' is not specified, the result is cached for faster future access.
"""
try:
crds, sources = (self.get_cache('filename.coordinates'),
self.get_cache('filename.sources'))
if bounds:
crds, I = crds.intersect(bounds, outer=True, return_indices=True)
sources = np.array(sources)[I].tolist()
return crds, sources
except NodeException: # Not in cache
pass
if bounds is None:
active_sources = self.sources
else:
crds, I = self.source_coordinates.intersect(bounds, outer=True, return_indices=True)
active_sources = self.sources[I]
crds = active_sources[0].source_coordinates
sources = [active_sources[0].sources]
for s in active_sources[1:]:
if np.prod(s.source_coordinates.shape) > 0:
crds = concat([crds, s.source_coordinates])
sources.append(s.sources)
#if self.shared_coordinates is not None:
#crds = crds + self.shared_coordinates
sources = np.concatenate(sources)
if bounds is None:
self.put_cache(crds, 'filename.coordinates')
self.put_cache(sources, 'filename.sources')
return crds, sources
@property
def base_ref(self):
"""Summary
Returns
-------
TYPE
Description
"""
return '{0}_{1}'.format(self.__class__.__name__, self.product)
@property
def base_definition(self):
""" Definition for SMAP node. Sources not required as these are computed.
"""
d = super(podpac.compositor.Compositor, self).base_definition
d['interpolation'] = self.interpolation
return d
@property
@common_doc(COMMON_DOC)
def keys(self):
"""{keys}
"""
return self.sources[0].keys
class SMAPDateFolder(podpac.compositor.OrderedCompositor):
"""Compositor of all the SMAP source urls present in a particular folder which is defined for a particular date
Attributes
----------
auth_class : {auth_class}
auth_session : {auth_session}
base_url : {base_url}
cache_native_coordinates : bool, optional
Default is False. If True, the native_coordinates will be cached to disk after being computed the first time
date_time_url_re : SRE_Pattern
Regular expression used to retrieve the date and time from the filename if file_url_re matches
date_url_re : SRE_Pattern
Regular expression used to retrieve the date from the filename if file_url_re2 matches
file_url_re : SRE_Pattern
Regular expression used to find files in a folder that match the expected format of a SMAP source file
file_url_re2 : SRE_Pattern
Same as file_url_re, but for variation of SMAP files that do not contain time in the filename
folder_date : str
The name of the folder. This is used to construct the OpenDAP URL from the base_url
latlon_delta : float, optional
Default is 1.5 degrees. For SMAP files that contain LAT-LON data (i.e. SMAP-Sentinel), how many degrees does the
tile cover?
latlon_url_re : SRE_Pattern
Regular expression used to find the lat-lon coordinates associated with the file from the file name
layerkey : {layerkey}
password : {password}
product : str
{product}
version : int
{version}
username : {username}
"""
auth_session = tl.Instance(authentication.EarthDataSession)
auth_class = tl.Type(authentication.EarthDataSession)
username = tl.Unicode(None, allow_none=True)
password = tl.Unicode(None, allow_none=True)
cache_type = tl.Enum([None, 'disk', 'ram'], allow_none=True, default_value='disk')
@tl.default('auth_session')
def _auth_session_default(self):
return self.auth_class(username=self.username, password=self.password, product_url=SMAP_BASE_URL())
base_url = tl.Unicode().tag(attr=True)
@tl.default('base_url')
def _base_url_default(self):
return SMAP_BASE_URL()
product = tl.Enum(SMAP_PRODUCT_MAP.coords['product'].data.tolist()).tag(attr=True)
version = tl.Int(allow_none=True).tag(attr=True)
@tl.default('version')
def _detect_product_version(self):
return _infer_SMAP_product_version(self.product, self.base_url, self.auth_session)
folder_date = tl.Unicode(u'').tag(attr=True)
file_url_re = re.compile(r'.*_[0-9]{8}T[0-9]{6}_.*\.h5')
file_url_re2 = re.compile(r'.*_[0-9]{8}_.*\.h5')
date_time_url_re = re.compile(r'[0-9]{8}T[0-9]{6}')
date_url_re = re.compile(r'[0-9]{8}')
latlon_url_re = re.compile(r'[0-9]{3}[E,W][0-9]{2}[N,S]')
latlon_delta = tl.Float(default_value=1.5).tag(attr=True)
cache_native_coordinates = tl.Bool(False)
layerkey = tl.Unicode()
@tl.default('layerkey')
def _layerkey_default(self):
return SMAP_PRODUCT_MAP.sel(product=self.product, attr='layerkey').item()
@tl.observe('layerkey')
def _layerkey_change(self, change):
if change['old'] != change['new'] and change['old'] != '':
for s in self.sources:
s.layerkey = change['new']
@property
def source(self):
"""URL to OpenDAP dataset folder
Returns
-------
str
URL to OpenDAP dataset folder
"""
return '/'.join([self.base_url, '%s.%03d' %(self.product, self.version), self.folder_date])
@tl.default('sources')
def sources_default(self):
"""SMAPSource objects pointing to URLs of specific SMAP files in the folder
Returns
-------
np.ndarray(dtype=object(SMAPSource))
Array of SMAPSource instances tied to specific SMAP files
"""
# Swapped the try and except blocks. SMAP filenames may change version numbers, which causes cached source to
# break. Hence, try to get the new source everytime, unless data is offline, in which case rely on the cache.
try:
_, _, sources = self.get_available_coords_sources()
self.put_cache(sources, 'sources', overwrite=True)
except: # No internet or authentication error
try:
sources = self.get_cache('sources')
except NodeException as e:
raise NodeException("Connection or Authentication error, and no disk cache to fall back on for determining sources.")
b = self.source + '/'
time_crds = self.source_coordinates['time']
if time_crds.is_monotonic and time_crds.is_uniform and time_crds.size > 1:
tol = time_crds.coordinates[1] - time_crds.coordinates[0]
else:
tol = self.source_coordinates['time'].coordinates[0]
tol = tol - tol
tol = np.timedelta64(1, dtype=(tol.dtype))
src_objs = [
SMAPSource(source=b+s, auth_session=self.auth_session, layerkey=self.layerkey, interpolation={
'method': 'nearest',
'time_tolerance': tol
})
for s in sources]
return np.array(src_objs)
@tl.default('is_source_coordinates_complete')
def src_crds_complete_default(self):
"""Flag use to optimize creation of native_coordinates. If the source_coordinates are complete,
native_coordinates can easily be reconstructed, and same with shared coordinates.
Returns
-------
bool
Flag indicating whether the source coordinates completely describe the source's coordinates for that dimension
"""
return self.product not in SMAP_INCOMPLETE_SOURCE_COORDINATES
def get_source_coordinates(self):
"""{source_coordinates}
"""
try:
times, latlon, _ = self.get_available_coords_sources()
except:
try:
return self.get_cache('source.coordinates')
except NodeException as e:
raise NodeException("Connection or Authentication error, and no disk cache to fall back on for determining sources.")
if latlon is not None and latlon.size > 0:
crds = podpac.Coordinates([[times, latlon[:, 0], latlon[:, 1]]], dims=['time_lat_lon'])
else:
crds = podpac.Coordinates([times], dims=['time'])
self.put_cache(crds, 'source.coordinates', overwrite=True)
return crds
@cache_func('shared.coordinates')
def get_shared_coordinates(self):
"""Coordinates that are shared by all files in the folder.
Returns
-------
podpac.Coordinates
Coordinates shared by all files in the folder
"""
if self.product in SMAP_INCOMPLETE_SOURCE_COORDINATES:
return None
coords = copy.deepcopy(self.sources[0].native_coordinates)
return coords.drop('time')
def get_available_coords_sources(self):
"""Read NSIDC site for available coordinate sources
Returns
-------
np.ndarray
Available times of sources in the folder
np.ndarray
Available lat lon coordinates of sources in the folder, None if empty
np.ndarray
The url's of the sources
Raises
------
RuntimeError
If the NSIDC website cannot be accessed
"""
url = self.source
r = _get_from_url(url, self.auth_session)
if r is None:
_logger.warning("Could not contact {} to retrieve source coordinates".format(url))
return np.array([]), None, np.array([])
soup = bs4.BeautifulSoup(r.text, 'lxml')
a = soup.find_all('a')
file_regex = self.file_url_re
file_regex2 = self.file_url_re2
date_time_regex = self.date_time_url_re
date_regex = self.date_url_re
latlon_regex = self.latlon_url_re
times = []
latlons = []
sources = []
for aa in a:
t = aa.get_text().strip('\n')
if 'h5.iso.xml' in t:
continue
m = file_regex.match(t)
m2 = file_regex2.match(t)
lonlat = None
if m:
date_time = date_time_regex.search(m.group()).group()
times.append(np.datetime64(
'%s-%s-%sT%s:%s:%s' % (date_time[:4], date_time[4:6], date_time[6:8], date_time[9:11],
date_time[11:13], date_time[13:15])
))
elif m2:
m = m2
date = date_regex.search(m.group()).group()
times.append(np.datetime64('%s-%s-%s' % (date[:4], date[4:6], date[6:8])))
if m:
sources.append(m.group())
lonlat = latlon_regex.search(m.group())
if lonlat:
lonlat = lonlat.group()
latlons.append((float(lonlat[4:6]) * (1 - 2 * (lonlat[6] == 'S')),
float(lonlat[:3]) * (1 - 2 * (lonlat[3] == 'W'))
))
times = np.array(times)
latlons = np.array(latlons)
sources = np.array(sources)
I = np.argsort(times)
if latlons.shape[0] == times.size:
return times[I], latlons[I], sources[I]
return times[I], None, sources[I]
@property
@common_doc(COMMON_DOC)
def keys(self):
"""{keys}
"""
return self.sources[0].keys
@property
def base_definition(self):
""" Definition for SMAP node. Sources not required as these are computed.
"""
d = super(podpac.compositor.Compositor, self).base_definition
d['interpolation'] = self.interpolation
return d
class SMAPSource(datatype.PyDAP):
"""Accesses SMAP data given a specific openDAP URL. This is the base class giving access to SMAP data, and knows how
to extract the correct coordinates and data keys for the soil moisture data.
Attributes
----------
auth_class : {auth_class}
auth_session : {auth_session}
date_file_url_re : SRE_Pattern
Regular expression used to retrieve date from self.source (OpenDAP Url)
date_time_file_url_re : SRE_Pattern
Regular expression used to retrieve date and time from self.source (OpenDAP Url)
layerkey : str
Key used to retrieve data from OpenDAP dataset. This specifies the key used to retrieve the data
nan_vals : list
List of values that should be treated as no-data (these are replaced by np.nan)
rootdatakey : str
String the prepends every or most keys for data in the OpenDAP dataset
"""
auth_session = tl.Instance(authentication.EarthDataSession)
auth_class = tl.Type(authentication.EarthDataSession)
@tl.default('auth_session')
def _auth_session_default(self):
session = self.auth_class(username=self.username, password=self.password, product_url=SMAP_BASE_URL())
# check url
try:
session.get(SMAP_BASE_URL())
except Exception as e:
_logger.warning("Unknown exception: ", e)
return session
#date_url_re = re.compile('[0-9]{4}\.[0-9]{2}\.[0-9]{2}')
date_time_file_url_re = re.compile('[0-9]{8}T[0-9]{6}')
date_file_url_re = re.compile('[0-9]{8}')
rootdatakey = tl.Unicode()
@tl.default('rootdatakey')
def _rootdatakey_default(self):
return SMAP_PRODUCT_MAP.sel(product=self.product,
attr='rootdatakey').item()
layerkey = tl.Unicode()
@tl.default('layerkey')
def _layerkey_default(self):
return SMAP_PRODUCT_MAP.sel(
product=self.product,
attr='layerkey').item()
nan_vals = [-9999.0]
@property
def product(self):
"""Returns the SMAP product from the OpenDAP Url
Returns
-------
str
{product}
"""
src = self.source.split('/')
return src[src.index('SMAP')+1].split('.')[0]
@property
def version(self):
"""Returns the SMAP product version from the OpenDAP Url
Returns
-------
int
{version}
"""
src = self.source.split('/')
return int(src[src.index('SMAP')+1].split('.')[1])
@tl.default('datakey')
def _datakey_default(self):
return self.layerkey.format(rdk=self.rootdatakey)
@property
def latkey(self):
"""The key used to retrieve the latitude
Returns
-------
str
OpenDap dataset key for latitude
"""
return SMAP_PRODUCT_MAP.sel(product=self.product, attr='latkey') \
.item().format(rdk=self.rootdatakey)
@property
def lonkey(self):
"""The key used to retrieve the latitude
Returns
-------
str
OpenDap dataset key for longitude
"""
return SMAP_PRODUCT_MAP.sel(product=self.product, attr='lonkey').item().format(rdk=self.rootdatakey)
@common_doc(COMMON_DOC)
@cache_func('native.coordinates')
def get_native_coordinates(self):
"""{get_native_coordinates}
"""
times = self.get_available_times()
ds = self.dataset
lons = np.array(ds[self.lonkey][:, :])
lats = np.array(ds[self.latkey][:, :])
lons[lons == self.nan_vals[0]] = np.nan
lats[lats == self.nan_vals[0]] = np.nan
lons = np.nanmean(lons, axis=0)
lats = np.nanmean(lats, axis=1)
coords = podpac.Coordinates([times, lats, lons], dims=['time', 'lat', 'lon'])
return coords
def get_available_times(self):
"""Retrieve the available times from the SMAP file. This is primarily based on the filename, but some products
have multiple times stored in a single file.
Returns
-------
np.ndarray(dtype=np.datetime64)
Available times in the SMAP source
"""
m = self.date_time_file_url_re.search(self.source)
if not m:
m = self.date_file_url_re.search(self.source)
times = m.group()
times = smap2np_date(times)
if 'SM_P_' in self.source:
times = times + np.array([6, 18], 'timedelta64[h]')
return times
@common_doc(COMMON_DOC)
def get_data(self, coordinates, coordinates_index):
"""{get_data}
"""
# We actually ignore the time slice
s = tuple([slc for d, slc in zip(coordinates.dims, coordinates_index)
if 'time' not in d])
if 'SM_P_' in self.source:
d = self.create_output_array(coordinates)
am_key = self.layerkey.format(rdk=self.rootdatakey + 'AM')
pm_key = self.layerkey.format(rdk=self.rootdatakey + 'PM') + '_pm'
try:
t = self.native_coordinates.coords['time'][0]
d.loc[dict(time=t)] = np.array(self.dataset[am_key][s])
except:
pass
try:
t = self.native_coordinates.coords['time'][1]
d.loc[dict(time=t)] = np.array(self.dataset[pm_key][s])
except:
pass
else:
data = np.array(self.dataset[self.datakey][s])
d = self.create_output_array(coordinates, data=data.reshape(coordinates.shape))
return d
class Goblin(Unit):
members = []
opponents = traitlets.Type('__main__.Elf')
class VoilaExporter(HTMLExporter):
"""Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer"""
markdown_renderer_class = traitlets.Type('mistune.Renderer').tag(
config=True)
# The voila exporter overrides the markdown renderer from the HTMLExporter
# to inline images.
@contextfilter
def markdown2html(self, context, source):
cell = context['cell']
attachments = cell.get('attachments', {})
cls = self.markdown_renderer_class
renderer = cls(escape=False,
attachments=attachments,
contents_manager=self.contents_manager,
anchor_link_text=self.anchor_link_text)
return MarkdownWithMath(renderer=renderer).render(source)
# The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter.
@property
def default_config(self):
c = Config({
'CSSHTMLHeaderPreprocessor': {
'enabled': False
},
'VoilaExporter': {
'markdown_renderer_class':
'voila.exporter.VoilaMarkdownRenderer'
}
})
c.merge(super(VoilaExporter, self).default_config)
return c
# Instead, we use the VoilaCSSPreprocessor.
@traitlets.default('preprocessors')
def _default_preprocessors(self):
return ['voila.csspreprocessor.VoilaCSSPreprocessor']
# Overriding the default template file.
@traitlets.default('template_file')
def default_template_file(self):
return 'voila.tpl'
async def generate_from_notebook_node(self,
nb,
resources=None,
extra_context={},
**kw):
# this replaces from_notebook_node, but calls template.generate instead of template.render
langinfo = nb.metadata.get('language_info', {})
lexer = langinfo.get('pygments_lexer', langinfo.get('name', None))
highlight_code = self.filters.get(
'highlight_code', Highlight2HTML(pygments_lexer=lexer,
parent=self))
self.register_filter('highlight_code', highlight_code)
# NOTE: we don't call HTML or TemplateExporter' from_notebook_node
nb_copy, resources = super(TemplateExporter, self).from_notebook_node(
nb, resources, **kw)
resources.setdefault('raw_mimetypes', self.raw_mimetypes)
resources['global_content_filter'] = {
'include_code': not self.exclude_code_cell,
'include_markdown': not self.exclude_markdown,
'include_raw': not self.exclude_raw,
'include_unknown': not self.exclude_unknown,
'include_input': not self.exclude_input,
'include_output': not self.exclude_output,
'include_input_prompt': not self.exclude_input_prompt,
'include_output_prompt': not self.exclude_output_prompt,
'no_prompt': self.exclude_input_prompt
and self.exclude_output_prompt,
}
async for output in self.template.generate_async(nb=nb_copy,
resources=resources,
**extra_context):
yield (output, resources)
@property
def environment(self):
# enable Jinja async template execution
self.enable_async = True
env = super(type(self), self).environment
if 'jinja2.ext.do' not in env.extensions:
env.add_extension('jinja2.ext.do')
return env
class Unit(traitlets.HasTraits):
"""
A generic class to represent units in the dungeon.
Eeally the only difference is what side the units take, so (just about)
everything can be defined here.
"""
attack_power = traitlets.Integer(default_value=3)
hit_points = traitlets.Integer(default_value=200)
location = traitlets.Tuple(traitlets.Integer(),
traitlets.Integer()) # y, x
dead = traitlets.Bool(default_value=False)
members = [] # here to store class instances
opponents = traitlets.Type('__main__.Unit')
def __new__(cls, *args, **kwargs):
instance = super().__new__(cls, *args, **kwargs)
cls.members.append(instance)
return instance
def attack(self, other):
other.hit_points -= self.attack_power
if other.hit_points <= 0:
other.dead = True
self.opponents.members.remove(other)
def distance(self, other):
return cityblock(self.location, other.location)
@property
def target(self):
"""
Find the nearest target for attack assuming one is available.
:rtype: Unit
"""
opponent_distances = [
self.distance(foe) for foe in self.opponents.members
]
potential_targets = [
foe for foe, distance in zip(self.opponents.members,
opponent_distances) if distance == 1
]
if not potential_targets:
return None
elif len(potential_targets) == 1:
return potential_targets[0]
else:
return sorted(potential_targets,
key=lambda u: (u.hit_points, *u.location))[0]
def move(self):
"""
Move the current unit to the closest valid target
Use a minimum cost path through the grid, after removing path through
allies spaces (you can ignore blocking out enemies because if a path
would go through an enemy it's going to end up closer).
:rtype: None
"""
# first, block out your buddies
current_dungeon = DUNGEON.copy()
allies = np.array(
[friend.location for friend in self.members if friend is not self])
if allies.size: # assuming there are any allies left
# locations are stored as y, x, so:
current_dungeon[allies[:, 0], allies[:, 1]] = -1
foe_locations = np.array(
[foe.location for foe in self.opponents.members])
# and now find the costs
mcp = MCP(current_dungeon, fully_connected=False)
cum_costs, traceback = mcp.find_costs(starts=[self.location],
find_all_ends=True)
foe_distances = cum_costs[foe_locations[:, 0], foe_locations[:, 1]]
if np.isinf(foe_distances.min()):
return # no route available to any foe
closest_foes = np.arange(
len(foe_distances))[foe_distances == foe_distances.min()]
closest_foe = sorted(self.opponents.members[i]
for i in closest_foes)[0]
# now you have one closest foe, reverse the distance calc
# and move one step closer
mcp = MCP(current_dungeon, fully_connected=False)
cum_costs, traceback = mcp.find_costs(ends=[self.location],
starts=[closest_foe.location],
find_all_ends=False)
# the minimum foe distance will be the location of self, so decrease
# by one
target_locations = np.argwhere(cum_costs == foe_distances.min() - 1)
# the MCP algorithm will expand out in many directions, so make sure
# to filter out only those points around self.
valid_locations = target_locations[(
(target_locations >= np.array(self.location) - 1) &
(target_locations <= np.array(self.location) + 1)).all(axis=1)]
# this is _ugly_, but I couldn't quickly think of a better way to sort
# the locations
y, x = (sorted(tuple(coords) for coords in valid_locations))[0]
self.location = (int(y), int(x))
# define comparison methods for sorting:
def __eq__(self, other):
return self.location == other.location
def __lt__(self, other):
return self.location < other.location
def __gt__(self, other):
return self.location == other.location
def __repr__(self):
"""Nice string representation"""
return f'<{self.__class__.__name__} ap{self.attack_power} hp{self.hit_points} loc{self.location}>'
# define add and radd so you can easily sum the list of units
def __add__(self, other):
return self.hit_points + other.hit_points
def __radd__(self, other):
return self.hit_points + other
class Elf(Unit):
"""A Christmas Elf"""
members = []
# likewise access to the Goblins is deferred until required.
opponents = traitlets.Type('__main__.Goblin')
class Cmd(trtc.Application):
"""Common machinery for all (sub-)commands. """
## INFO: Do not use it directly; inherit it.
# See module documentation for developer's guidelines.
@trt.default('name')
def _name(self):
name = class2cmd_name(type(self))
return name
@trt.default('description')
def _description(self):
return __doc__ or '<no description>'
config_files = trt.Unicode(None,
allow_none=True,
help="""
Absolute/relative path(s) to config files to OVERRIDE default configs.
Multiple paths are separated by '{pathsep}' in descending order.
Any extensions are ignored, and '.json' or '.py' are searched (in this order).
If the path specified resolves to a folder, the filename `{appname}_config.[json | py]` is appended;
Any command-line values take precendance over the `{confvar}` envvar.
Use `gen-config` sub-command to produce a skeleton of the config-file.
""".format(appname=APPNAME,
confvar=CONF_VAR_NAME,
pathsep=osp.pathsep)).tag(config=True)
@trt.default('log')
def _log(self):
## Use a regular logger.
return logging.getLogger(type(self).__name__)
@property
def user_config_fpaths(self):
fpaths = []
config_files = os.environ.get(CONF_VAR_NAME, self.config_files)
if config_files:
def _procfpath(p):
p = pndlu.convpath(p)
if osp.isdir(p):
p = osp.join(p, default_config_fname())
else:
p = osp.splitext(p)[0]
return p
fpaths = config_files.split(osp.pathsep)
fpaths = [_procfpath(p) for p in fpaths]
return fpaths
def load_config_files(self):
"""Load default user-specified overrides config files.
Config-files in descending orders:
- user-overrides:
- :envvar:`<APPNAME>_CONFIG_FILE`, or if not set,
- :attr:`config_file`;
- default config-files:
- ~/.<appname>/<appname>_config.{json,py} and
- <this-file's-folder>/<appname>_config.{json,py}.
"""
# Load "standard" configs,
# path-list in descending priority order.
#
paths = list(iset([default_config_dir(), _mydir]))
self.load_config_file(default_config_fname(), path=paths)
# Load "user" configs.
#
user_conf_fpaths = self.user_config_fpaths
for fp in user_conf_fpaths[::-1]:
cdir, cfname = osp.split(fp)
self.load_config_file(cfname, path=cdir)
def write_default_config(self, config_file=None):
if not config_file:
config_file = default_config_fpath()
else:
config_file = pndlu.convpath(config_file)
if osp.isdir(config_file):
config_file = osp.join(config_file, default_config_fname())
config_file = pndlu.ensure_file_ext(config_file, '.py')
op = 'Over-writting' if osp.isfile(config_file) else 'Writting'
self.log.info('%s config-file %r...', op, config_file)
pndlu.ensure_dir_exists(os.path.dirname(config_file), 0o700)
config_text = self.generate_config_file()
with io.open(config_file, mode='wt') as fp:
fp.write(config_text)
def print_subcommands(self):
"""Print the subcommand part of the help."""
## Overridden, to print "default" sub-cmd.
if not self.subcommands:
return
lines = ["Subcommands"]
lines.append('-' * len(lines[0]))
lines.append('')
for p in wrap_paragraphs(
self.subcommand_description.format(app=self.name)):
lines.append(p)
lines.append('')
for subc, (cls, hlp) in self.subcommands.items():
if self.default_subcmd == subc:
subc = '%s[*]' % subc
lines.append(subc)
if hlp:
lines.append(indent(dedent(hlp.strip())))
if self.default_subcmd:
lines.append('')
lines.append(
"""Note: The asterisk '[*]' marks the "default" sub-command to run when none specified."""
)
lines.append('')
print(os.linesep.join(lines))
@trtc.catch_config_error
def initialize_subcommand(self, subc, argv=None):
"""Initialize a subcommand named `subc` with `argv`, or `sys.argv` if `None` (default)."""
## INFO: Overriden to set parent on subcmds and inherit config,
# see https://github.com/ipython/traitlets/issues/286
subcmd_tuple = self.subcommands.get(subc)
assert subcmd_tuple, "Cannot find sub-cmd %r in sub-cmds of %r: %s" % (
subc, self.name, list(self.subcommands.keys()))
subapp, _ = subcmd_tuple
type(self).clear_instance()
self.subapp = subapp.instance(parent=self)
self.subapp.initialize(argv)
default_subcmd = trt.Unicode(
None,
allow_none=True,
help="The name of the sub-command to use if unspecified.")
conf_classes = trt.List(trt.Type(trtc.Configurable),
default_value=[],
help="""
Any *configurables* found in this prop up the cmd-chain are merged,
along with any subcommands, into :attr:`classes`.
""")
cmd_aliases = trt.Dict(
{},
help=
"Any *flags* found in this prop up the cmd-chain are merged into :attr:`aliases`. "
"")
cmd_flags = trt.Dict(
{},
help=
"Any *flags* found in this prop up the cmd-chain are merged into :attr:`flags`. "
"")
def my_cmd_chain(self):
"""Return the chain of cmd-classes starting from my self or subapp."""
cmd_chain = []
pcl = self.subapp if self.subapp else self
while pcl:
cmd_chain.append(pcl)
pcl = pcl.parent
return cmd_chain
@trt.observe('parent', 'conf_classes', 'cmd_aliases', 'cmd_flags',
'subapp', 'subcommands')
def _inherit_parent_cmd(self, change):
""" Inherit config-related stuff from up the cmd-chain. """
if self.parent:
## Collect parents, ordered like that:
# subapp, self, parent1, ...
#
cmd_chain = self.my_cmd_chain()
## Collect separately and merge SPECs separately,
# to prepend them before SPECs at the end.
#
conf_classes = list(
itz.concat(cmd.conf_classes for cmd in cmd_chain))
## Merge aliases/flags reversed.
#
cmd_aliases = dtz.merge(cmd.cmd_aliases for cmd in cmd_chain[::-1])
cmd_flags = dtz.merge(cmd.cmd_flags for cmd in cmd_chain[::-1])
else:
## We are root.
cmd_chain = [self]
conf_classes = list(self.conf_classes)
cmd_aliases = self.cmd_aliases
cmd_flags = self.cmd_flags
cmd_classes = [type(cmd) for cmd in cmd_chain]
self.classes = list(iset(cmd_classes + conf_classes))
self.aliases.update(cmd_aliases)
self.flags.update(cmd_flags)
@trt.observe('log_level')
def _init_logging(self, change):
log_level = change['new']
if isinstance(log_level, str):
log_level = getattr(logging, log_level)
init_logging(level=log_level)
def __init__(self, **kwds):
cls = type(self)
dkwds = {
## Traits defaults are always applied...??
#
'description': cls.__doc__,
'name': class2cmd_name(cls),
## Set some nice defaults for root-CMDs.
#
'cmd_aliases': {
'config-files': 'Cmd.config_files',
},
'cmd_flags': {
('d', 'debug'):
({
'Application': {
'log_level': 0
},
'Spec': {
'log_level': 0
},
'Cmd': {
'raise_config_file_errors': True,
'print_config': True,
},
}, "Log more logging, fail on configuration errors, "
"and print configuration on each cmd startup."),
('v', 'verbose'): ({
'Spec': {
'verbose': True
},
}, pndlu.first_line(Spec.verbose.help)),
('f', 'force'): ({
'Spec': {
'force': True
},
}, pndlu.first_line(Spec.force.help))
},
}
dkwds.update(kwds)
super().__init__(**dkwds)
def _is_dispatching(self):
"""True if dispatching to another command."""
return bool(self.subapp)
@trtc.catch_config_error
def initialize(self, argv=None):
## Invoked after __init__() by Cmd.launch_instance() to read configs.
# It parses cl-args before file-configs, to detect sub-commands
# and update any :attr:`config_file`,
# load file-configs, and then re-apply cmd-line configs as overrides
# (trick copied from `jupyter-core`).
self.parse_command_line(argv)
if self._is_dispatching():
## Only the final child gets file-configs.
# Also avoid contaminations with user if generating-config.
return
cl_config = copy.deepcopy(self.config)
self.load_config_files()
self.update_config(cl_config)
print_config = trt.Bool(
False,
help=
"""Enable it to print the configurations before launching any command."""
).tag(config=True)
def start(self):
"""Dispatches into sub-cmds (if any), and then delegates to :meth:`run().
If overriden, better invoke :func:`super()`, but even better
to override :meth:``run()`.
"""
if self.print_config:
self.log.info('Running cmd %r with config: \n %s', self.name,
self.config)
if self.subapp is not None:
pass
elif self.default_subcmd:
self.initialize_subcommand(self.default_subcmd, self.argv)
else:
return self.run(*self.extra_args)
return self.subapp.start()
def run(self, *args):
"""Leaf sub-commands must inherit this instead of :meth:`start()` without invoking :func:`super()`.
By default, screams about using sub-cmds, or about doing nothing!
:param args: Invoked by :meth:`start()` with :attr:`extra_args`.
"""
if self.subcommands:
cmd_line = ' '.join(cl.name
for cl in reversed(self.my_cmd_chain()))
raise CmdException("Specify one of the sub-commands: "
"\n %s\nor type: \n %s -h" %
(', '.join(self.subcommands.keys()), cmd_line))
assert False, "Override run() method in cmd subclasses."
class Unit(traitlets.HasTraits):
attack_power = traitlets.Integer(default_value=3)
hit_points = traitlets.Integer(default_value=200)
location = traitlets.Tuple(traitlets.Integer(),
traitlets.Integer()) # y, x
dead = traitlets.Bool(default_value=False)
members = []
opponents = traitlets.Type('__main__.Unit')
@classmethod
def append(cls, other):
cls.members.append(other)
def attack(self, other):
other.hit_points -= self.attack_power
if other.hit_points <= 0:
other.dead = True
self.opponents.members.remove(other)
print(self, 'killed', other)
def distance(self, other):
return cityblock(self.location, other.location)
@property
def target(self):
opponent_distances = [
self.distance(foe) for foe in self.opponents.members
]
potential_targets = [
foe for foe, distance in zip(self.opponents.members,
opponent_distances) if distance == 1
]
if not potential_targets:
return None
elif len(potential_targets) == 1:
return potential_targets[0]
else:
return sorted(potential_targets,
key=lambda u: (u.hit_points, *u.location))[0]
def move(self):
# first, block out your buddies
current_dungeon = DUNGEON.copy()
allies = np.array(
[friend.location for friend in self.members if friend is not self])
if allies.size:
# locations are stored as y, x, so:
current_dungeon[allies[:, 0], allies[:, 1]] = -1
foe_locations = np.array(
[foe.location for foe in self.opponents.members])
# and now find the costs
mcp = MCP(current_dungeon, fully_connected=False)
cum_costs, traceback = mcp.find_costs(
starts=[self.location],
#ends=foe_locations,
find_all_ends=True)
foe_distances = cum_costs[foe_locations[:, 0], foe_locations[:, 1]]
if np.isinf(foe_distances.min()):
return # no route available to any foe
closest_foes = np.arange(
len(foe_distances))[foe_distances == foe_distances.min()]
closest_foe = sorted(self.opponents.members[i]
for i in closest_foes)[0]
# now you have one closest foe, reverse the distance calc
# and move one step closer
mcp = MCP(current_dungeon, fully_connected=False)
cum_costs, traceback = mcp.find_costs(ends=[self.location],
starts=[closest_foe.location],
find_all_ends=False)
target_locations = np.argwhere(cum_costs == foe_distances.min() - 1)
valid_locations = target_locations[(
(target_locations >= np.array(self.location) - 1) &
(target_locations <= np.array(self.location) + 1)).all(axis=1)]
if valid_locations.size == 0:
print(valid_locations)
y, x = (sorted(tuple(coords) for coords in valid_locations))[0]
print(self, 'moving to', y, x)
self.location = (int(y), int(x))
def __eq__(self, other):
return (*self.location, self.hit_points) == (*other.location,
other.hit_points)
def __lt__(self, other):
return (*self.location, self.hit_points) < (*other.location,
other.hit_points)
def __gt__(self, other):
return (*self.location, self.hit_points) == (*other.location,
other.hit_points)
def __repr__(self):
return f'<{self.__class__.__name__} ap{self.attack_power} hp{self.hit_points} loc{self.location}>'
def __add__(self, other):
return self.hit_points + other.hit_points
def __radd__(self, other):
return self.hit_points + other
class Elf(Unit):
members = []
opponents = traitlets.Type('__main__.Goblin')
class PyDAP(DataSource):
"""Create a DataSource from an OpenDAP server feed.
Attributes
----------
auth_class : :class:`podpac.authentication.Session`
:class:`requests.Session` derived class providing authentication credentials.
When username and password are provided, an auth_session is created using this class.
auth_session : :class:`podpac.authentication.Session`
Instance of the auth_class. This is created if username and password is supplied, but this object can also be
supplied directly
datakey : str
Pydap 'key' for the data to be retrieved from the server. Datasource may have multiple keys, so this key
determines which variable is returned from the source.
dataset : pydap.model.DatasetType
The open pydap dataset. This is provided for troubleshooting.
native_coordinates : Coordinates
{native_coordinates}
password : str, optional
Password used for authenticating against OpenDAP server. WARNING: this is stored as plain-text, provide
auth_session instead if you have security concerns.
source : str
URL of the OpenDAP server.
username : str, optional
Username used for authenticating against OpenDAP server. WARNING: this is stored as plain-text, provide
auth_session instead if you have security concerns.
"""
# required inputs
source = tl.Unicode(allow_none=False, default_value='')
datakey = tl.Unicode(allow_none=False).tag(attr=True)
# optional inputs and later defined traits
auth_session = tl.Instance(authentication.Session, allow_none=True)
auth_class = tl.Type(authentication.Session)
username = tl.Unicode(None, allow_none=True)
password = tl.Unicode(None, allow_none=True)
dataset = tl.Instance('pydap.model.DatasetType', allow_none=False)
@tl.default('auth_session')
def _auth_session_default(self):
# requires username and password
if not self.username or not self.password:
return None
# requires auth_class
# TODO: default auth_class?
if not self.auth_class:
return None
# instantiate and check utl
try:
session = self.auth_class(username=self.username,
password=self.password)
session.get(self.source + '.dds')
except:
# TODO: catch a 403 error
return None
return session
@tl.default('dataset')
def _open_dataset(self, source=None):
"""Summary
Parameters
----------
source : str, optional
Description
Returns
-------
TYPE
Description
"""
# TODO: is source ever None?
# TODO: enforce string source
if source is None:
source = self.source
else:
self.source = source
# auth session
# if self.auth_session:
try:
dataset = pydap.client.open_url(source, session=self.auth_session)
except Exception:
# TODO handle a 403 error
# TODO: Check Url (probably inefficient...)
try:
self.auth_session.get(self.source + '.dds')
dataset = pydap.client.open_url(source,
session=self.auth_session)
except Exception:
# TODO: handle 403 error
print(
"Warning, dataset could not be opened. Check login credentials."
)
dataset = None
return dataset
@tl.observe('source')
def _update_dataset(self, change=None):
if change is None:
return
if change['old'] == None or change['old'] == '':
return
if self.dataset is not None and 'new' in change:
self.dataset = self._open_dataset(source=change['new'])
try:
if self.native_coordinates is not None:
self.native_coordinates = self.get_native_coordinates()
except NotImplementedError:
pass
@common_doc(COMMON_DATA_DOC)
def get_native_coordinates(self):
"""{get_native_coordinates}
Raises
------
NotImplementedError
DAP has no mechanism for creating coordinates automatically, so this is left up to child classes.
"""
raise NotImplementedError(
"DAP has no mechanism for creating coordinates" +
", so this is left up to child class " + "implementations.")
@common_doc(COMMON_DATA_DOC)
def get_data(self, coordinates, coordinates_index):
"""{get_data}
"""
data = self.dataset[self.datakey][tuple(coordinates_index)]
# PyDAP 3.2.1 gives a numpy array for the above, whereas 3.2.2 needs the .data attribute to get a numpy array
if not isinstance(data, np.ndarray) and hasattr(data, 'data'):
data = data.data
d = self.create_output_array(coordinates,
data=data.reshape(coordinates.shape))
return d
@property
def keys(self):
"""The list of available keys from the OpenDAP dataset.
Returns
-------
List
The list of available keys from the OpenDAP dataset. Any of these keys can be set as self.datakey
"""
return self.dataset.keys()
class ParallelAsync(Parallel):
"""
This class launches the parallel node evaluations in threads up to n_workers, and expects the node.eval to return
quickly for parallel execution. This Node was written with aws.Lambda(eval_timeout=1.25<small>) Nodes in mind.
Users can implement the `check_worker_available` method or specify the `no_worker_exception` attribute, which is an
exception thrown if workers are not available.
Attributes
-----------
chunks: dict
Dictionary of dimensions and sizes that will be iterated over. If a dimension is not in this dictionary, the
size of the eval coordinates will be used for the chunk. In this case, it may not be possible to automatically
set the coordinates of missing dimensions in the final file.
fill_output: bool
Default is True. When True, the final results will be assembled and returned to the user. If False, the final
results should be written to a file by specifying the output_format in a Process or Lambda node.
See note below.
source: podpac.Node
The source dataset for the computation
sleep_time: float
Default is 1 second. Number of seconds to sleep between trying to submit new workers
no_worker_exception: Exception, optional
Default is .Exception class used to identify when a submission failed due to no available workers. The default
is chosen to work with the podpac.managers.Lambda node.
async_exception: Exception
Default is botocore.exceptions.ReadTimeoutException. This is an exception thrown by the async function in case
it time out waiting for a return. In our case, this is a success. The default is chosen to work with the
podpac.managers.Lambda node.
Notes
------
In some cases where the input and output coordinates of the source node is not the same (such as reduce nodes)
and fill_output is True, the user may need to specify 'output' as part of the eval call.
"""
source = NodeTrait().tag(attr=True)
chunks = tl.Dict().tag(attr=True)
fill_output = tl.Bool(True).tag(attr=True)
sleep_time = tl.Float(1).tag(attr=True)
no_worker_exception = tl.Type(
botocore.exceptions.ClientError).tag(attr=True)
async_exception = tl.Type(
botocore.exceptions.ReadTimeoutError).tag(attr=True)
def check_worker_available(self):
return True
def eval_source(self, coordinates, coordinates_index, out, i, source=None):
if source is None:
source = self.source
# Make a copy to prevent any possibility of memory corruption
source = Node.from_definition(source.definition)
success = False
o = None
while not success:
if self.check_worker_available():
try:
o = source.eval(coordinates, output=out)
success = True
except self.async_exception:
# This exception is fine and constitutes a success
o = None
success = True
except self.no_worker_exception as e:
response = e.response
if not (response and response.get("Error", {}).get("Code")
== "TooManyRequestsException"):
raise e # Raise error again, not the right error
_log.debug("Worker {} exception {}".format(i, e))
success = False
time.sleep(self.sleep_time)
else:
_log.debug("Worker unavailable for {}".format(i, e))
time.sleep(self.sleep_time)
_log.info("Submitting source {}".format(i))
return (o, coordinates_index)
class S3(DataSource):
"""Create a DataSource from a file on an S3 Bucket.
Attributes
----------
node : Node, optional
The DataSource node used to interpret the S3 file
node_class : DataSource, optional
The class type of self.node. This is used to create self.node if self.node is not specified
node_kwargs : dict, optional
Keyword arguments passed to `node_class` when automatically creating `node`
return_type : str, optional
Either: 'file_handle' (for files downloaded to RAM); or
the default option 'path' (for files downloaded to disk)
s3_bucket : str, optional
Name of the S3 bucket. Uses ``podpac.settings['S3_BUCKET_NAME']`` by default.
s3_data : file/str
If return_type == 'file_handle' returns a file pointer object
If return_type == 'path' returns a string to the data
source : str
Path to the file residing in the S3 bucket that will be loaded
"""
source = tl.Unicode()
node = tl.Instance(Node)
node_class = tl.Type(DataSource) # A class
node_kwargs = tl.Dict(default_value={})
s3_bucket = tl.Unicode(allow_none=True)
s3_data = tl.Any(allow_none=True)
_temp_file_cleanup = tl.List()
return_type = tl.Enum(['file_handle', 'path'], default_value='path')
# TODO: handle s3 auth setup
@tl.default('node')
def node_default(self):
"""Creates the default node using the node_class and node_kwargs
Returns
-------
self.node_class
Instance of self.node_class
Raises
------
Exception
This function sets the source in the node, so 'source' cannot be present in node_kwargs
"""
if 'source' in self.node_kwargs:
raise Exception("'source' present in node_kwargs for S3")
return self.node_class(source=self.s3_data, **self.node_kwargs)
@tl.default('s3_bucket')
def s3_bucket_default(self):
"""Retrieves default S3 Bucket from settings
Returns
-------
Str
Name of the S3 bucket
"""
return settings['S3_BUCKET_NAME']
@tl.default('s3_data')
def s3_data_default(self):
"""Returns the file handle or path to the S3 bucket
Returns
-------
str/file
Either a string to the downloaded file path, or a file handle
"""
if self.s3_bucket is None:
raise ValueError('No s3 bucket set')
s3 = boto3.resource('s3').Bucket(self.s3_bucket)
if self.return_type == 'file_handle':
# TODO: should this use the with/as syntax
# https://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.download_fileobj
# download into memory
io = BytesIO()
s3.download_fileobj(self.source, io)
io.seek(0)
return io
elif self.return_type == 'path':
# Download the file to cache directory
#tmppath = os.path.join(tempfile.gettempdir(),
#self.source.replace('\\', '').replace(':','')\
#.replace('/', ''))
tmppath = os.path.join(
settings['CACHE_DIR'],
self.source.replace('\\', '').replace(':',
'').replace('/', ''))
rootpath = os.path.split(tmppath)[0]
if not os.path.exists(rootpath):
os.makedirs(rootpath)
#i = 0
#while os.path.exists(tmppath):
#tmppath = os.path.join(tempfile.gettempdir(),
#self.source + '.%d' % i)
if not os.path.exists(tmppath):
s3.download_file(self.source, tmppath)
# TODO: should we handle temp files here?
#self._temp_file_cleanup.append(tmppath)
return tmppath
@common_doc(COMMON_DATA_DOC)
def get_data(self, coordinates, coordinates_index):
"""{get_data}
"""
self.nan_vals = getattr(self.node, 'nan_vals', [])
return self.node.get_data(coordinates, coordinates_index)
@property
@common_doc(COMMON_DATA_DOC)
def native_coordinates(self):
"""{native_coordinates}
"""
return self.node.native_coordinates
def __del__(self):
if hasattr(super(S3), '__del__'):
super(S3).__del__(self)
for f in self._temp_file_cleanup:
os.remove(f)
class WorkflowIPythonKernel(IPythonKernel):
# Set StreamFlow shell
shell_class = traitlets.Type(StreamFlowInteractiveShell)
# Kernel info fields
implementation = 'sf-ipython'
implementation_version = streamflow.version.VERSION
language_info = {
'name': 'python',
'version': sys.version.split()[0],
'mimetype': 'text/x-python',
'codemirror_mode': {
'name': 'ipython',
'version': sys.version_info[0]
},
'pygments_lexer': 'ipython%d' % (3 if PY3 else 2),
'nbconvert_exporter': 'python',
'file_extension': '.py'
}
msg_types = IPythonKernel.msg_types + ['workflow_request']
def init_metadata(self, parent):
# Call parent functionse
metadata = super().init_metadata(parent)
# If StreamFlow has been configured for this cell, store its configuration
workflow_config = (
parent['metadata'] if 'workflow' in parent['metadata'] else
parent['content'] if 'workflow' in parent['content'] else None)
if workflow_config is not None:
try:
validate({
k: v
for k, v in workflow_config['workflow'].items()
if k != 'cell_id'
})
except BaseException as e:
self.log.error(str(e))
return metadata
metadata['sf_token'] = self.shell.wf_cell_config.set(
workflow_config['workflow'])
# Return metadata
return metadata
def finish_metadata(self, parent, metadata, reply_content):
# Remove StreamFlow configuration from the `sf_cell_config` attribute if present
if 'sf_token' in metadata:
self.shell.wf_cell_config.reset(metadata['sf_token'])
del metadata['sf_token']
# Call parent function
return super().finish_metadata(parent, metadata, reply_content)
@gen.coroutine
def workflow_request(self, stream, ident, parent):
try:
content = parent['content']
notebook = content['notebook']
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
metadata = self.init_metadata(parent)
reply_content = yield gen.maybe_future(
self.do_workflow(notebook, ident, parent))
sys.stdout.flush()
sys.stderr.flush()
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
metadata = self.finish_metadata(parent, metadata, reply_content)
reply_msg = self.session.send(stream,
'execute_reply',
reply_content,
parent,
metadata=metadata,
ident=ident)
self.log.debug("%s", reply_msg)
@gen.coroutine
def do_workflow(self, notebook, ident, parent):
shell = self.shell
reply_content = {}
if (_asyncio_runner and shell.loop_runner is _asyncio_runner
and asyncio.get_event_loop().is_running()):
coro = shell.run_workflow(notebook)
coro_future = asyncio.ensure_future(coro)
with self._cancel_on_sigint(coro_future):
try:
res = yield coro_future
finally:
shell.events.trigger('post_execute')
else:
coro = shell.run_workflow(notebook)
if shell.trio_runner:
runner = shell.trio_runner
else:
runner = shell.loop_runner
res = runner(coro)
# Send outputs to cell streams
for cell_name, content in res.result.items():
self.session.send(self.iopub_thread,
'stream',
content={
'name': cell_name,
'text': content
},
parent=extract_header(parent),
ident=ident)
# Send reply message
if res.error_before_exec is not None:
err = res.error_before_exec
else:
err = res.error_in_exec
if res.success:
reply_content['status'] = 'ok'
else:
reply_content['status'] = 'error'
# noinspection PyProtectedMember
reply_content.update({
'traceback': shell._last_traceback or [],
'ename': str(type(err).__name__),
'evalue': safe_unicode(err),
})
reply_content['execution_count'] = shell.execution_count - 1
reply_content['payload'] = shell.payload_manager.read_payload()
shell.payload_manager.clear_payload()
return reply_content
class VoilaExporter(HTMLExporter):
"""Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer"""
base_url = traitlets.Unicode(help="Base url for resources").tag(config=True)
markdown_renderer_class = traitlets.Type('mistune.Renderer').tag(config=True)
# The voila exporter overrides the markdown renderer from the HTMLExporter
# to inline images.
@contextfilter
def markdown2html(self, context, source):
cell = context['cell']
attachments = cell.get('attachments', {})
cls = self.markdown_renderer_class
renderer = cls(escape=False, attachments=attachments,
contents_manager=self.contents_manager,
anchor_link_text=self.anchor_link_text)
return MarkdownWithMath(renderer=renderer).render(source)
# The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter.
@property
def default_config(self):
c = Config({
'VoilaExporter': {
'markdown_renderer_class': 'voila.exporter.VoilaMarkdownRenderer'
}
})
c.merge(super(VoilaExporter, self).default_config)
return c
# Overriding the default template file.
@traitlets.default('template_file')
def default_template_file(self):
return 'index.html.j2'
async def generate_from_notebook_node(self, nb, resources=None, extra_context={}, **kw):
# this replaces from_notebook_node, but calls template.generate instead of template.render
langinfo = nb.metadata.get('language_info', {})
lexer = langinfo.get('pygments_lexer', langinfo.get('name', None))
highlight_code = self.filters.get('highlight_code', Highlight2HTML(pygments_lexer=lexer, parent=self))
self.register_filter('highlight_code', highlight_code)
# NOTE: we don't call HTML or TemplateExporter' from_notebook_node
nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw)
resources.setdefault('raw_mimetypes', self.raw_mimetypes)
resources['global_content_filter'] = {
'include_code': not self.exclude_code_cell,
'include_markdown': not self.exclude_markdown,
'include_raw': not self.exclude_raw,
'include_unknown': not self.exclude_unknown,
'include_input': not self.exclude_input,
'include_output': not self.exclude_output,
'include_input_prompt': not self.exclude_input_prompt,
'include_output_prompt': not self.exclude_output_prompt,
'no_prompt': self.exclude_input_prompt and self.exclude_output_prompt,
}
async for output in self.template.generate_async(nb=nb_copy, resources=resources, **extra_context):
yield (output, resources)
@property
def environment(self):
# enable Jinja async template execution
self.enable_async = True
env = super(type(self), self).environment
if 'jinja2.ext.do' not in env.extensions:
env.add_extension('jinja2.ext.do')
return env
def get_template_paths(self):
return self.template_path
def _init_resources(self, resources):
def include_css(name):
code = """<link rel="stylesheet" type="text/css" href="%svoila/%s">""" % (self.base_url, name)
return jinja2.Markup(code)
def include_js(name):
code = """<script src="%svoila/%s"></script>""" % (self.base_url, name)
return jinja2.Markup(code)
def include_url(name):
url = "%svoila/%s" % (self.base_url, name)
return jinja2.Markup(url)
resources = super(VoilaExporter, self)._init_resources(resources)
resources['include_css'] = include_css
resources['include_js'] = include_js
resources['include_url'] = include_url
return resources
class FastDazzlerSetupLoader(DatasetLoader):
type = 'FastDazzler'
trim_to = traitlets.Instance(klass=slice, args=(10, None),
help='range of indexes to throw away when loading a camera file' +\
'; used for removing saturated frames at the beginning of ' +\
'acquisition', config=True)
batch_name = traitlets.Unicode(default_value='test-batch',
help='name of job used in LabView software',
config=True,
allow_none=False)
batch_path = traitlets.Unicode(help='location of the batch folder on disk',
config=True,
allow_none=False)
nwaveforms = traitlets.Int(default_value=1,
help='number of waveforms in the Dazzler table',
config=True)
nrepeat = traitlets.Int(default_value=1,
help='number of times each waveform is repeated in camera file; not '\
+ 'the same as Dazzler NRepeat', config=True)
data_type = traitlets.Type(klass=Dataset, config=True)
allow_parallel = traitlets.Bool(default_value=False, config=True)
def __init__(self, config=None):
super(FastDazzlerSetupLoader, self).__init__(config=config)
def load(self):
pattern = re.compile('(?P<job_name>[\w-]+)-batch(?P<batch_no>\d+)')
p = pathlib.Path(self.batch_path)
log.debug('loading {path!s}'.format(path=p.name))
m = pattern.match(p.name)
if m is None:
raise ValueError('wrong job-name format')
log.debug(
'matched: job_name = `{job_name:s}`, batch_no = `{batch_no:s}`'.
format(job_name=m.group('job_name'), batch_no=m.group('batch_no')))
batch_info = next(
load_job(m.group('job_name'),
when=None,
batch_set=set([int(m.group('batch_no'))]),
data_path=p.parent))
raw_datas = []
# loop over available data
for t2, table, loop in it.product(batch_info['t2_range'],
batch_info['table_range'],
batch_info['loop_range']):
c = Config()
# copy over settings used
c.RawData.batch_name = self.batch_name
c.RawData.batch_path = self.batch_path
c.RawData.nwaveforms = self.nwaveforms
c.RawData.nrepeat = self.nrepeat
c.RawData.t2 = t2
c.RawData.table = table
c.RawData.loop = loop
c.RawData.phase_cycles = self.data_type.phase_cycles
if self.allow_parallel:
raw_datas.append(delayed(RawData)(config=c).process())
else:
raw_datas.append(RawData(config=c).process())
# create dataset and process it
if self.allow_parallel:
dset = delayed(self.data_type)(config=self.config)
else:
dset = self.data_type(config=self.config)
return dset.from_raw_data(raw_datas)
