class Goblin(Unit): """A Goblin, sworn enemy of the Christmas Elf""" members = [] # note that using the traitlets type we can defer the dependency on the # Elf class until the opponents attribute is accessed opponents = traitlets.Type('__main__.Elf')
class PidginKernel(ipykernel.kernelapp.IPythonKernel): input = traitlets.Enum(input_formats, default_value="markdown") def __init__(self, *a, **k): super().__init__(*a, **k), traitlets.link((self, "input"), (self.shell, "input")) shell_class = traitlets.Type(PidginShell)
class VoilaExporter(HTMLExporter): """Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer""" markdown_renderer_class = traitlets.Type('mistune.Renderer').tag(config=True) # The voila exporter overrides the markdown renderer from the HTMLExporter # to inline images. @contextfilter def markdown2html(self, context, source): cell = context['cell'] attachments = cell.get('attachments', {}) cls = self.markdown_renderer_class renderer = cls(escape=False, attachments=attachments, contents_manager=self.contents_manager, anchor_link_text=self.anchor_link_text) return MarkdownWithMath(renderer=renderer).render(source) # The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter. @property def default_config(self): c = Config({ 'CSSHTMLHeaderPreprocessor': { 'enabled': False }, 'VoilaExporter': { 'markdown_renderer_class': 'voila.exporter.VoilaMarkdownRenderer' } }) c.merge(super(VoilaExporter, self).default_config) return c # Instead, we use the VoilaCSSPreprocessor. @traitlets.default('preprocessors') def _default_preprocessors(self): return ['voila.csspreprocessor.VoilaCSSPreprocessor'] # Overriding the default template file. @traitlets.default('template_file') def default_template_file(self): return 'voila.tpl'
class VoilaExporter(HTMLExporter): """Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer""" base_url = traitlets.Unicode(help="Base url for resources").tag(config=True) markdown_renderer_class = traitlets.Type('mistune.Renderer').tag(config=True) # Can be a ContentsManager from notebook or jupyter_server, so Any will have to do for now contents_manager = traitlets.Any() # The voila exporter overrides the markdown renderer from the HTMLExporter # to inline images. @contextfilter def markdown2html(self, context, source): cell = context['cell'] attachments = cell.get('attachments', {}) cls = self.markdown_renderer_class renderer = cls(escape=False, attachments=attachments, contents_manager=self.contents_manager, anchor_link_text=self.anchor_link_text) return MarkdownWithMath(renderer=renderer).render(source) # The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter. @property def default_config(self): c = Config({ 'VoilaExporter': { 'markdown_renderer_class': 'voila.exporter.VoilaMarkdownRenderer' } }) c.merge(super(VoilaExporter, self).default_config) return c # Overriding the default template file. @traitlets.default('template_file') def default_template_file(self): return 'index.html.j2' async def generate_from_notebook_node(self, nb, resources=None, extra_context={}, **kw): # this replaces from_notebook_node, but calls template.generate instead of template.render langinfo = nb.metadata.get('language_info', {}) lexer = langinfo.get('pygments_lexer', langinfo.get('name', None)) highlight_code = self.filters.get('highlight_code', Highlight2HTML(pygments_lexer=lexer, parent=self)) self.register_filter('highlight_code', highlight_code) # NOTE: we don't call HTML or TemplateExporter' from_notebook_node nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw) resources.setdefault('raw_mimetypes', self.raw_mimetypes) resources['global_content_filter'] = { 'include_code': not self.exclude_code_cell, 'include_markdown': not self.exclude_markdown, 'include_raw': not self.exclude_raw, 'include_unknown': not self.exclude_unknown, 'include_input': not self.exclude_input, 'include_output': not self.exclude_output, 'include_input_prompt': not self.exclude_input_prompt, 'include_output_prompt': not self.exclude_output_prompt, 'no_prompt': self.exclude_input_prompt and self.exclude_output_prompt, } async for output in self.template.generate_async(nb=nb_copy, resources=resources, **extra_context, static_url=self.static_url): yield (output, resources) @property def environment(self): # enable Jinja async template execution self.enable_async = True env = super(type(self), self).environment if 'jinja2.ext.do' not in env.extensions: env.add_extension('jinja2.ext.do') return env def get_template_paths(self): return self.template_path def static_url(self, path): """Mimics tornado.web.RequestHandler.static_url""" settings = { 'static_url_prefix': f'{self.base_url}voila/templates/', 'static_path': None # not used in TemplateStaticFileHandler.get_absolute_path } return TemplateStaticFileHandler.make_static_url(settings, f'{self.template_name}/static/{path}') def _init_resources(self, resources): resources = super(VoilaExporter, self)._init_resources(resources) include_assets_functions = create_include_assets_functions(self.template_name, self.base_url) resources.update(include_assets_functions) return resources
class SMAP(podpac.compositor.OrderedCompositor): """Compositor of all the SMAPDateFolder's for every available SMAP date. Essentially a compositor of all SMAP data for a particular product. Attributes ---------- auth_class : {auth_class} auth_session : {auth_session} base_url : {base_url} date_url_re : SRE_Pattern Regular expression used to extract all folder dates (or folder names) for the particular SMAP product. layerkey : {layerkey} password : {password} product : str {product} username : {username} """ base_url = tl.Unicode().tag(attr=True) @tl.default('base_url') def _base_url_default(self): return SMAP_BASE_URL() product = tl.Enum(SMAP_PRODUCT_MAP.coords['product'].data.tolist(), default_value='SPL4SMAU').tag(attr=True) version = tl.Int(allow_none=True).tag(attr=True) @tl.default('version') def _detect_product_version(self): return _infer_SMAP_product_version(self.product, self.base_url, self.auth_session) date_url_re = re.compile(r'[0-9]{4}\.[0-9]{2}\.[0-9]{2}') auth_session = tl.Instance(authentication.EarthDataSession) auth_class = tl.Type(authentication.EarthDataSession) username = tl.Unicode(None, allow_none=True) password = tl.Unicode(None, allow_none=True) cache_type = tl.Enum([None, 'disk', 'ram'], allow_none=True, default_value='disk') @tl.default('auth_session') def _auth_session_default(self): return self.auth_class(username=self.username, password=self.password, product_url=SMAP_BASE_URL()) layerkey = tl.Unicode() @tl.default('layerkey') def _layerkey_default(self): return SMAP_PRODUCT_MAP.sel( product=self.product, attr='layerkey').item() @tl.observe('layerkey') def _layerkey_change(self, change): if change['old'] != change['new'] and change['old'] != '': for s in self.sources: s.layerkey = change['new'] @property def source(self): """The source is used for a unique name to cache SMAP products. Returns ------- str The SMAP product name. """ return '%s.%03d' % (self.product, self.version) @tl.default('sources') def sources_default(self): """SMAPDateFolder objects pointing to specific SMAP folders Returns ------- np.ndarray(dtype=object(SMAPDateFolder)) Array of SMAPDateFolder instances tied to specific SMAP folders """ dates = self.get_available_times_dates()[1] src_objs = np.array([ SMAPDateFolder(product=self.product, version=self.version, folder_date=date, shared_coordinates=self.shared_coordinates, auth_session=self.auth_session, layerkey=self.layerkey) for date in dates]) return src_objs @common_doc(COMMON_DOC) def find_coordinates(self): ''' {native_coordinates} Notes ----- These coordinates are computed, assuming dataset is regular. ''' if self.product in SMAP_IRREGULAR_COORDINATES: raise Exception("Native coordinates too large. Try using get_filename_coordinates_sources().") shared = self.get_shared_coordinates() partial_sources = self.get_source_coordinates()['time'].coordinates complete_source_0 = self.sources[0].get_source_coordinates()['time'].coordinates offset = complete_source_0 - partial_sources[0] full_times = (partial_sources[:, None] + offset[None, :]).ravel() return [merge_dims([podpac.Coordinates([full_times], ['time']), shared])] @common_doc(COMMON_DOC) def get_source_coordinates(self): """{source_coordinates} """ return podpac.Coordinates([self.get_available_times_dates()[0]], dims=['time']) def get_available_times_dates(self): """Returns the available folder dates in the SMAP product Returns ------- np.ndarray Array of dates in numpy datetime64 format list list of dates in SMAP date format Raises ------ RuntimeError If the http resource could not be accessed (check Earthdata login credentials) """ url = '/'.join([self.base_url, '%s.%03d' % (self.product, self.version)]) r = _get_from_url(url, self.auth_session) if r is None: _logger.warning("Could not contact {} to retrieve source coordinates".format(url)) return np.array([]), [] soup = bs4.BeautifulSoup(r.text, 'lxml') a = soup.find_all('a') regex = self.date_url_re times = [] dates = [] for aa in a: m = regex.match(aa.get_text()) if m: times.append(np.datetime64(m.group().replace('.', '-'))) dates.append(m.group()) times.sort() dates.sort() return np.array(times), dates @cache_func('shared.coordinates') def get_shared_coordinates(self): """Coordinates that are shared by all files in the SMAP product family. Returns ------- podpac.Coordinates Coordinates shared by all files in the SMAP product. Notes ------ For example, the gridded SMAP data have the same lat-lon coordinates in every file (global at some resolution), and the only difference between files is the time coordinate. This is not true for the SMAP-Sentinel product, in which case this function returns None """ if self.product in SMAP_INCOMPLETE_SOURCE_COORDINATES: return None coords = SMAPDateFolder(product=self.product, version=self.version, folder_date=self.get_available_times_dates()[1][0], auth_session=self.auth_session, ).shared_coordinates return coords def get_filename_coordinates_sources(self, bounds=None): """Returns coordinates solely based on the filenames of the sources. This function was motivated by the SMAP-Sentinel product, which does not have regularly stored tiles (in space and time). Parameters ----------- Bounds: podpac.Coordinates, Optional Default is None. Return the coordinates based on filenames of the source only within the specified bounds. When not None, the result is not cached. Returns ------- podpac.Coordinates Coordinates of all the sources in the product family np.ndarray(dtype=object(SMAPSource)) Array of all the SMAPSources pointing to unique OpenDAP urls corresponding to the partial native coordinates Notes ------ The outputs of this function can be used to find source that overlap spatially or temporally with a subset region specified by the user. If 'bounds' is not specified, the result is cached for faster future access. """ try: crds, sources = (self.get_cache('filename.coordinates'), self.get_cache('filename.sources')) if bounds: crds, I = crds.intersect(bounds, outer=True, return_indices=True) sources = np.array(sources)[I].tolist() return crds, sources except NodeException: # Not in cache pass if bounds is None: active_sources = self.sources else: crds, I = self.source_coordinates.intersect(bounds, outer=True, return_indices=True) active_sources = self.sources[I] crds = active_sources[0].source_coordinates sources = [active_sources[0].sources] for s in active_sources[1:]: if np.prod(s.source_coordinates.shape) > 0: crds = concat([crds, s.source_coordinates]) sources.append(s.sources) #if self.shared_coordinates is not None: #crds = crds + self.shared_coordinates sources = np.concatenate(sources) if bounds is None: self.put_cache(crds, 'filename.coordinates') self.put_cache(sources, 'filename.sources') return crds, sources @property def base_ref(self): """Summary Returns ------- TYPE Description """ return '{0}_{1}'.format(self.__class__.__name__, self.product) @property def base_definition(self): """ Definition for SMAP node. Sources not required as these are computed. """ d = super(podpac.compositor.Compositor, self).base_definition d['interpolation'] = self.interpolation return d @property @common_doc(COMMON_DOC) def keys(self): """{keys} """ return self.sources[0].keys
class SMAPDateFolder(podpac.compositor.OrderedCompositor): """Compositor of all the SMAP source urls present in a particular folder which is defined for a particular date Attributes ---------- auth_class : {auth_class} auth_session : {auth_session} base_url : {base_url} cache_native_coordinates : bool, optional Default is False. If True, the native_coordinates will be cached to disk after being computed the first time date_time_url_re : SRE_Pattern Regular expression used to retrieve the date and time from the filename if file_url_re matches date_url_re : SRE_Pattern Regular expression used to retrieve the date from the filename if file_url_re2 matches file_url_re : SRE_Pattern Regular expression used to find files in a folder that match the expected format of a SMAP source file file_url_re2 : SRE_Pattern Same as file_url_re, but for variation of SMAP files that do not contain time in the filename folder_date : str The name of the folder. This is used to construct the OpenDAP URL from the base_url latlon_delta : float, optional Default is 1.5 degrees. For SMAP files that contain LAT-LON data (i.e. SMAP-Sentinel), how many degrees does the tile cover? latlon_url_re : SRE_Pattern Regular expression used to find the lat-lon coordinates associated with the file from the file name layerkey : {layerkey} password : {password} product : str {product} version : int {version} username : {username} """ auth_session = tl.Instance(authentication.EarthDataSession) auth_class = tl.Type(authentication.EarthDataSession) username = tl.Unicode(None, allow_none=True) password = tl.Unicode(None, allow_none=True) cache_type = tl.Enum([None, 'disk', 'ram'], allow_none=True, default_value='disk') @tl.default('auth_session') def _auth_session_default(self): return self.auth_class(username=self.username, password=self.password, product_url=SMAP_BASE_URL()) base_url = tl.Unicode().tag(attr=True) @tl.default('base_url') def _base_url_default(self): return SMAP_BASE_URL() product = tl.Enum(SMAP_PRODUCT_MAP.coords['product'].data.tolist()).tag(attr=True) version = tl.Int(allow_none=True).tag(attr=True) @tl.default('version') def _detect_product_version(self): return _infer_SMAP_product_version(self.product, self.base_url, self.auth_session) folder_date = tl.Unicode(u'').tag(attr=True) file_url_re = re.compile(r'.*_[0-9]{8}T[0-9]{6}_.*\.h5') file_url_re2 = re.compile(r'.*_[0-9]{8}_.*\.h5') date_time_url_re = re.compile(r'[0-9]{8}T[0-9]{6}') date_url_re = re.compile(r'[0-9]{8}') latlon_url_re = re.compile(r'[0-9]{3}[E,W][0-9]{2}[N,S]') latlon_delta = tl.Float(default_value=1.5).tag(attr=True) cache_native_coordinates = tl.Bool(False) layerkey = tl.Unicode() @tl.default('layerkey') def _layerkey_default(self): return SMAP_PRODUCT_MAP.sel(product=self.product, attr='layerkey').item() @tl.observe('layerkey') def _layerkey_change(self, change): if change['old'] != change['new'] and change['old'] != '': for s in self.sources: s.layerkey = change['new'] @property def source(self): """URL to OpenDAP dataset folder Returns ------- str URL to OpenDAP dataset folder """ return '/'.join([self.base_url, '%s.%03d' %(self.product, self.version), self.folder_date]) @tl.default('sources') def sources_default(self): """SMAPSource objects pointing to URLs of specific SMAP files in the folder Returns ------- np.ndarray(dtype=object(SMAPSource)) Array of SMAPSource instances tied to specific SMAP files """ # Swapped the try and except blocks. SMAP filenames may change version numbers, which causes cached source to # break. Hence, try to get the new source everytime, unless data is offline, in which case rely on the cache. try: _, _, sources = self.get_available_coords_sources() self.put_cache(sources, 'sources', overwrite=True) except: # No internet or authentication error try: sources = self.get_cache('sources') except NodeException as e: raise NodeException("Connection or Authentication error, and no disk cache to fall back on for determining sources.") b = self.source + '/' time_crds = self.source_coordinates['time'] if time_crds.is_monotonic and time_crds.is_uniform and time_crds.size > 1: tol = time_crds.coordinates[1] - time_crds.coordinates[0] else: tol = self.source_coordinates['time'].coordinates[0] tol = tol - tol tol = np.timedelta64(1, dtype=(tol.dtype)) src_objs = [ SMAPSource(source=b+s, auth_session=self.auth_session, layerkey=self.layerkey, interpolation={ 'method': 'nearest', 'time_tolerance': tol }) for s in sources] return np.array(src_objs) @tl.default('is_source_coordinates_complete') def src_crds_complete_default(self): """Flag use to optimize creation of native_coordinates. If the source_coordinates are complete, native_coordinates can easily be reconstructed, and same with shared coordinates. Returns ------- bool Flag indicating whether the source coordinates completely describe the source's coordinates for that dimension """ return self.product not in SMAP_INCOMPLETE_SOURCE_COORDINATES def get_source_coordinates(self): """{source_coordinates} """ try: times, latlon, _ = self.get_available_coords_sources() except: try: return self.get_cache('source.coordinates') except NodeException as e: raise NodeException("Connection or Authentication error, and no disk cache to fall back on for determining sources.") if latlon is not None and latlon.size > 0: crds = podpac.Coordinates([[times, latlon[:, 0], latlon[:, 1]]], dims=['time_lat_lon']) else: crds = podpac.Coordinates([times], dims=['time']) self.put_cache(crds, 'source.coordinates', overwrite=True) return crds @cache_func('shared.coordinates') def get_shared_coordinates(self): """Coordinates that are shared by all files in the folder. Returns ------- podpac.Coordinates Coordinates shared by all files in the folder """ if self.product in SMAP_INCOMPLETE_SOURCE_COORDINATES: return None coords = copy.deepcopy(self.sources[0].native_coordinates) return coords.drop('time') def get_available_coords_sources(self): """Read NSIDC site for available coordinate sources Returns ------- np.ndarray Available times of sources in the folder np.ndarray Available lat lon coordinates of sources in the folder, None if empty np.ndarray The url's of the sources Raises ------ RuntimeError If the NSIDC website cannot be accessed """ url = self.source r = _get_from_url(url, self.auth_session) if r is None: _logger.warning("Could not contact {} to retrieve source coordinates".format(url)) return np.array([]), None, np.array([]) soup = bs4.BeautifulSoup(r.text, 'lxml') a = soup.find_all('a') file_regex = self.file_url_re file_regex2 = self.file_url_re2 date_time_regex = self.date_time_url_re date_regex = self.date_url_re latlon_regex = self.latlon_url_re times = [] latlons = [] sources = [] for aa in a: t = aa.get_text().strip('\n') if 'h5.iso.xml' in t: continue m = file_regex.match(t) m2 = file_regex2.match(t) lonlat = None if m: date_time = date_time_regex.search(m.group()).group() times.append(np.datetime64( '%s-%s-%sT%s:%s:%s' % (date_time[:4], date_time[4:6], date_time[6:8], date_time[9:11], date_time[11:13], date_time[13:15]) )) elif m2: m = m2 date = date_regex.search(m.group()).group() times.append(np.datetime64('%s-%s-%s' % (date[:4], date[4:6], date[6:8]))) if m: sources.append(m.group()) lonlat = latlon_regex.search(m.group()) if lonlat: lonlat = lonlat.group() latlons.append((float(lonlat[4:6]) * (1 - 2 * (lonlat[6] == 'S')), float(lonlat[:3]) * (1 - 2 * (lonlat[3] == 'W')) )) times = np.array(times) latlons = np.array(latlons) sources = np.array(sources) I = np.argsort(times) if latlons.shape[0] == times.size: return times[I], latlons[I], sources[I] return times[I], None, sources[I] @property @common_doc(COMMON_DOC) def keys(self): """{keys} """ return self.sources[0].keys @property def base_definition(self): """ Definition for SMAP node. Sources not required as these are computed. """ d = super(podpac.compositor.Compositor, self).base_definition d['interpolation'] = self.interpolation return d
class SMAPSource(datatype.PyDAP): """Accesses SMAP data given a specific openDAP URL. This is the base class giving access to SMAP data, and knows how to extract the correct coordinates and data keys for the soil moisture data. Attributes ---------- auth_class : {auth_class} auth_session : {auth_session} date_file_url_re : SRE_Pattern Regular expression used to retrieve date from self.source (OpenDAP Url) date_time_file_url_re : SRE_Pattern Regular expression used to retrieve date and time from self.source (OpenDAP Url) layerkey : str Key used to retrieve data from OpenDAP dataset. This specifies the key used to retrieve the data nan_vals : list List of values that should be treated as no-data (these are replaced by np.nan) rootdatakey : str String the prepends every or most keys for data in the OpenDAP dataset """ auth_session = tl.Instance(authentication.EarthDataSession) auth_class = tl.Type(authentication.EarthDataSession) @tl.default('auth_session') def _auth_session_default(self): session = self.auth_class(username=self.username, password=self.password, product_url=SMAP_BASE_URL()) # check url try: session.get(SMAP_BASE_URL()) except Exception as e: _logger.warning("Unknown exception: ", e) return session #date_url_re = re.compile('[0-9]{4}\.[0-9]{2}\.[0-9]{2}') date_time_file_url_re = re.compile('[0-9]{8}T[0-9]{6}') date_file_url_re = re.compile('[0-9]{8}') rootdatakey = tl.Unicode() @tl.default('rootdatakey') def _rootdatakey_default(self): return SMAP_PRODUCT_MAP.sel(product=self.product, attr='rootdatakey').item() layerkey = tl.Unicode() @tl.default('layerkey') def _layerkey_default(self): return SMAP_PRODUCT_MAP.sel( product=self.product, attr='layerkey').item() nan_vals = [-9999.0] @property def product(self): """Returns the SMAP product from the OpenDAP Url Returns ------- str {product} """ src = self.source.split('/') return src[src.index('SMAP')+1].split('.')[0] @property def version(self): """Returns the SMAP product version from the OpenDAP Url Returns ------- int {version} """ src = self.source.split('/') return int(src[src.index('SMAP')+1].split('.')[1]) @tl.default('datakey') def _datakey_default(self): return self.layerkey.format(rdk=self.rootdatakey) @property def latkey(self): """The key used to retrieve the latitude Returns ------- str OpenDap dataset key for latitude """ return SMAP_PRODUCT_MAP.sel(product=self.product, attr='latkey') \ .item().format(rdk=self.rootdatakey) @property def lonkey(self): """The key used to retrieve the latitude Returns ------- str OpenDap dataset key for longitude """ return SMAP_PRODUCT_MAP.sel(product=self.product, attr='lonkey').item().format(rdk=self.rootdatakey) @common_doc(COMMON_DOC) @cache_func('native.coordinates') def get_native_coordinates(self): """{get_native_coordinates} """ times = self.get_available_times() ds = self.dataset lons = np.array(ds[self.lonkey][:, :]) lats = np.array(ds[self.latkey][:, :]) lons[lons == self.nan_vals[0]] = np.nan lats[lats == self.nan_vals[0]] = np.nan lons = np.nanmean(lons, axis=0) lats = np.nanmean(lats, axis=1) coords = podpac.Coordinates([times, lats, lons], dims=['time', 'lat', 'lon']) return coords def get_available_times(self): """Retrieve the available times from the SMAP file. This is primarily based on the filename, but some products have multiple times stored in a single file. Returns ------- np.ndarray(dtype=np.datetime64) Available times in the SMAP source """ m = self.date_time_file_url_re.search(self.source) if not m: m = self.date_file_url_re.search(self.source) times = m.group() times = smap2np_date(times) if 'SM_P_' in self.source: times = times + np.array([6, 18], 'timedelta64[h]') return times @common_doc(COMMON_DOC) def get_data(self, coordinates, coordinates_index): """{get_data} """ # We actually ignore the time slice s = tuple([slc for d, slc in zip(coordinates.dims, coordinates_index) if 'time' not in d]) if 'SM_P_' in self.source: d = self.create_output_array(coordinates) am_key = self.layerkey.format(rdk=self.rootdatakey + 'AM') pm_key = self.layerkey.format(rdk=self.rootdatakey + 'PM') + '_pm' try: t = self.native_coordinates.coords['time'][0] d.loc[dict(time=t)] = np.array(self.dataset[am_key][s]) except: pass try: t = self.native_coordinates.coords['time'][1] d.loc[dict(time=t)] = np.array(self.dataset[pm_key][s]) except: pass else: data = np.array(self.dataset[self.datakey][s]) d = self.create_output_array(coordinates, data=data.reshape(coordinates.shape)) return d
class Goblin(Unit): members = [] opponents = traitlets.Type('__main__.Elf')
class VoilaExporter(HTMLExporter): """Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer""" markdown_renderer_class = traitlets.Type('mistune.Renderer').tag( config=True) # The voila exporter overrides the markdown renderer from the HTMLExporter # to inline images. @contextfilter def markdown2html(self, context, source): cell = context['cell'] attachments = cell.get('attachments', {}) cls = self.markdown_renderer_class renderer = cls(escape=False, attachments=attachments, contents_manager=self.contents_manager, anchor_link_text=self.anchor_link_text) return MarkdownWithMath(renderer=renderer).render(source) # The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter. @property def default_config(self): c = Config({ 'CSSHTMLHeaderPreprocessor': { 'enabled': False }, 'VoilaExporter': { 'markdown_renderer_class': 'voila.exporter.VoilaMarkdownRenderer' } }) c.merge(super(VoilaExporter, self).default_config) return c # Instead, we use the VoilaCSSPreprocessor. @traitlets.default('preprocessors') def _default_preprocessors(self): return ['voila.csspreprocessor.VoilaCSSPreprocessor'] # Overriding the default template file. @traitlets.default('template_file') def default_template_file(self): return 'voila.tpl' async def generate_from_notebook_node(self, nb, resources=None, extra_context={}, **kw): # this replaces from_notebook_node, but calls template.generate instead of template.render langinfo = nb.metadata.get('language_info', {}) lexer = langinfo.get('pygments_lexer', langinfo.get('name', None)) highlight_code = self.filters.get( 'highlight_code', Highlight2HTML(pygments_lexer=lexer, parent=self)) self.register_filter('highlight_code', highlight_code) # NOTE: we don't call HTML or TemplateExporter' from_notebook_node nb_copy, resources = super(TemplateExporter, self).from_notebook_node( nb, resources, **kw) resources.setdefault('raw_mimetypes', self.raw_mimetypes) resources['global_content_filter'] = { 'include_code': not self.exclude_code_cell, 'include_markdown': not self.exclude_markdown, 'include_raw': not self.exclude_raw, 'include_unknown': not self.exclude_unknown, 'include_input': not self.exclude_input, 'include_output': not self.exclude_output, 'include_input_prompt': not self.exclude_input_prompt, 'include_output_prompt': not self.exclude_output_prompt, 'no_prompt': self.exclude_input_prompt and self.exclude_output_prompt, } async for output in self.template.generate_async(nb=nb_copy, resources=resources, **extra_context): yield (output, resources) @property def environment(self): # enable Jinja async template execution self.enable_async = True env = super(type(self), self).environment if 'jinja2.ext.do' not in env.extensions: env.add_extension('jinja2.ext.do') return env
class Unit(traitlets.HasTraits): """ A generic class to represent units in the dungeon. Eeally the only difference is what side the units take, so (just about) everything can be defined here. """ attack_power = traitlets.Integer(default_value=3) hit_points = traitlets.Integer(default_value=200) location = traitlets.Tuple(traitlets.Integer(), traitlets.Integer()) # y, x dead = traitlets.Bool(default_value=False) members = [] # here to store class instances opponents = traitlets.Type('__main__.Unit') def __new__(cls, *args, **kwargs): instance = super().__new__(cls, *args, **kwargs) cls.members.append(instance) return instance def attack(self, other): other.hit_points -= self.attack_power if other.hit_points <= 0: other.dead = True self.opponents.members.remove(other) def distance(self, other): return cityblock(self.location, other.location) @property def target(self): """ Find the nearest target for attack assuming one is available. :rtype: Unit """ opponent_distances = [ self.distance(foe) for foe in self.opponents.members ] potential_targets = [ foe for foe, distance in zip(self.opponents.members, opponent_distances) if distance == 1 ] if not potential_targets: return None elif len(potential_targets) == 1: return potential_targets[0] else: return sorted(potential_targets, key=lambda u: (u.hit_points, *u.location))[0] def move(self): """ Move the current unit to the closest valid target Use a minimum cost path through the grid, after removing path through allies spaces (you can ignore blocking out enemies because if a path would go through an enemy it's going to end up closer). :rtype: None """ # first, block out your buddies current_dungeon = DUNGEON.copy() allies = np.array( [friend.location for friend in self.members if friend is not self]) if allies.size: # assuming there are any allies left # locations are stored as y, x, so: current_dungeon[allies[:, 0], allies[:, 1]] = -1 foe_locations = np.array( [foe.location for foe in self.opponents.members]) # and now find the costs mcp = MCP(current_dungeon, fully_connected=False) cum_costs, traceback = mcp.find_costs(starts=[self.location], find_all_ends=True) foe_distances = cum_costs[foe_locations[:, 0], foe_locations[:, 1]] if np.isinf(foe_distances.min()): return # no route available to any foe closest_foes = np.arange( len(foe_distances))[foe_distances == foe_distances.min()] closest_foe = sorted(self.opponents.members[i] for i in closest_foes)[0] # now you have one closest foe, reverse the distance calc # and move one step closer mcp = MCP(current_dungeon, fully_connected=False) cum_costs, traceback = mcp.find_costs(ends=[self.location], starts=[closest_foe.location], find_all_ends=False) # the minimum foe distance will be the location of self, so decrease # by one target_locations = np.argwhere(cum_costs == foe_distances.min() - 1) # the MCP algorithm will expand out in many directions, so make sure # to filter out only those points around self. valid_locations = target_locations[( (target_locations >= np.array(self.location) - 1) & (target_locations <= np.array(self.location) + 1)).all(axis=1)] # this is _ugly_, but I couldn't quickly think of a better way to sort # the locations y, x = (sorted(tuple(coords) for coords in valid_locations))[0] self.location = (int(y), int(x)) # define comparison methods for sorting: def __eq__(self, other): return self.location == other.location def __lt__(self, other): return self.location < other.location def __gt__(self, other): return self.location == other.location def __repr__(self): """Nice string representation""" return f'<{self.__class__.__name__} ap{self.attack_power} hp{self.hit_points} loc{self.location}>' # define add and radd so you can easily sum the list of units def __add__(self, other): return self.hit_points + other.hit_points def __radd__(self, other): return self.hit_points + other
class Elf(Unit): """A Christmas Elf""" members = [] # likewise access to the Goblins is deferred until required. opponents = traitlets.Type('__main__.Goblin')
class Cmd(trtc.Application): """Common machinery for all (sub-)commands. """ ## INFO: Do not use it directly; inherit it. # See module documentation for developer's guidelines. @trt.default('name') def _name(self): name = class2cmd_name(type(self)) return name @trt.default('description') def _description(self): return __doc__ or '<no description>' config_files = trt.Unicode(None, allow_none=True, help=""" Absolute/relative path(s) to config files to OVERRIDE default configs. Multiple paths are separated by '{pathsep}' in descending order. Any extensions are ignored, and '.json' or '.py' are searched (in this order). If the path specified resolves to a folder, the filename `{appname}_config.[json | py]` is appended; Any command-line values take precendance over the `{confvar}` envvar. Use `gen-config` sub-command to produce a skeleton of the config-file. """.format(appname=APPNAME, confvar=CONF_VAR_NAME, pathsep=osp.pathsep)).tag(config=True) @trt.default('log') def _log(self): ## Use a regular logger. return logging.getLogger(type(self).__name__) @property def user_config_fpaths(self): fpaths = [] config_files = os.environ.get(CONF_VAR_NAME, self.config_files) if config_files: def _procfpath(p): p = pndlu.convpath(p) if osp.isdir(p): p = osp.join(p, default_config_fname()) else: p = osp.splitext(p)[0] return p fpaths = config_files.split(osp.pathsep) fpaths = [_procfpath(p) for p in fpaths] return fpaths def load_config_files(self): """Load default user-specified overrides config files. Config-files in descending orders: - user-overrides: - :envvar:`<APPNAME>_CONFIG_FILE`, or if not set, - :attr:`config_file`; - default config-files: - ~/.<appname>/<appname>_config.{json,py} and - <this-file's-folder>/<appname>_config.{json,py}. """ # Load "standard" configs, # path-list in descending priority order. # paths = list(iset([default_config_dir(), _mydir])) self.load_config_file(default_config_fname(), path=paths) # Load "user" configs. # user_conf_fpaths = self.user_config_fpaths for fp in user_conf_fpaths[::-1]: cdir, cfname = osp.split(fp) self.load_config_file(cfname, path=cdir) def write_default_config(self, config_file=None): if not config_file: config_file = default_config_fpath() else: config_file = pndlu.convpath(config_file) if osp.isdir(config_file): config_file = osp.join(config_file, default_config_fname()) config_file = pndlu.ensure_file_ext(config_file, '.py') op = 'Over-writting' if osp.isfile(config_file) else 'Writting' self.log.info('%s config-file %r...', op, config_file) pndlu.ensure_dir_exists(os.path.dirname(config_file), 0o700) config_text = self.generate_config_file() with io.open(config_file, mode='wt') as fp: fp.write(config_text) def print_subcommands(self): """Print the subcommand part of the help.""" ## Overridden, to print "default" sub-cmd. if not self.subcommands: return lines = ["Subcommands"] lines.append('-' * len(lines[0])) lines.append('') for p in wrap_paragraphs( self.subcommand_description.format(app=self.name)): lines.append(p) lines.append('') for subc, (cls, hlp) in self.subcommands.items(): if self.default_subcmd == subc: subc = '%s[*]' % subc lines.append(subc) if hlp: lines.append(indent(dedent(hlp.strip()))) if self.default_subcmd: lines.append('') lines.append( """Note: The asterisk '[*]' marks the "default" sub-command to run when none specified.""" ) lines.append('') print(os.linesep.join(lines)) @trtc.catch_config_error def initialize_subcommand(self, subc, argv=None): """Initialize a subcommand named `subc` with `argv`, or `sys.argv` if `None` (default).""" ## INFO: Overriden to set parent on subcmds and inherit config, # see https://github.com/ipython/traitlets/issues/286 subcmd_tuple = self.subcommands.get(subc) assert subcmd_tuple, "Cannot find sub-cmd %r in sub-cmds of %r: %s" % ( subc, self.name, list(self.subcommands.keys())) subapp, _ = subcmd_tuple type(self).clear_instance() self.subapp = subapp.instance(parent=self) self.subapp.initialize(argv) default_subcmd = trt.Unicode( None, allow_none=True, help="The name of the sub-command to use if unspecified.") conf_classes = trt.List(trt.Type(trtc.Configurable), default_value=[], help=""" Any *configurables* found in this prop up the cmd-chain are merged, along with any subcommands, into :attr:`classes`. """) cmd_aliases = trt.Dict( {}, help= "Any *flags* found in this prop up the cmd-chain are merged into :attr:`aliases`. " "") cmd_flags = trt.Dict( {}, help= "Any *flags* found in this prop up the cmd-chain are merged into :attr:`flags`. " "") def my_cmd_chain(self): """Return the chain of cmd-classes starting from my self or subapp.""" cmd_chain = [] pcl = self.subapp if self.subapp else self while pcl: cmd_chain.append(pcl) pcl = pcl.parent return cmd_chain @trt.observe('parent', 'conf_classes', 'cmd_aliases', 'cmd_flags', 'subapp', 'subcommands') def _inherit_parent_cmd(self, change): """ Inherit config-related stuff from up the cmd-chain. """ if self.parent: ## Collect parents, ordered like that: # subapp, self, parent1, ... # cmd_chain = self.my_cmd_chain() ## Collect separately and merge SPECs separately, # to prepend them before SPECs at the end. # conf_classes = list( itz.concat(cmd.conf_classes for cmd in cmd_chain)) ## Merge aliases/flags reversed. # cmd_aliases = dtz.merge(cmd.cmd_aliases for cmd in cmd_chain[::-1]) cmd_flags = dtz.merge(cmd.cmd_flags for cmd in cmd_chain[::-1]) else: ## We are root. cmd_chain = [self] conf_classes = list(self.conf_classes) cmd_aliases = self.cmd_aliases cmd_flags = self.cmd_flags cmd_classes = [type(cmd) for cmd in cmd_chain] self.classes = list(iset(cmd_classes + conf_classes)) self.aliases.update(cmd_aliases) self.flags.update(cmd_flags) @trt.observe('log_level') def _init_logging(self, change): log_level = change['new'] if isinstance(log_level, str): log_level = getattr(logging, log_level) init_logging(level=log_level) def __init__(self, **kwds): cls = type(self) dkwds = { ## Traits defaults are always applied...?? # 'description': cls.__doc__, 'name': class2cmd_name(cls), ## Set some nice defaults for root-CMDs. # 'cmd_aliases': { 'config-files': 'Cmd.config_files', }, 'cmd_flags': { ('d', 'debug'): ({ 'Application': { 'log_level': 0 }, 'Spec': { 'log_level': 0 }, 'Cmd': { 'raise_config_file_errors': True, 'print_config': True, }, }, "Log more logging, fail on configuration errors, " "and print configuration on each cmd startup."), ('v', 'verbose'): ({ 'Spec': { 'verbose': True }, }, pndlu.first_line(Spec.verbose.help)), ('f', 'force'): ({ 'Spec': { 'force': True }, }, pndlu.first_line(Spec.force.help)) }, } dkwds.update(kwds) super().__init__(**dkwds) def _is_dispatching(self): """True if dispatching to another command.""" return bool(self.subapp) @trtc.catch_config_error def initialize(self, argv=None): ## Invoked after __init__() by Cmd.launch_instance() to read configs. # It parses cl-args before file-configs, to detect sub-commands # and update any :attr:`config_file`, # load file-configs, and then re-apply cmd-line configs as overrides # (trick copied from `jupyter-core`). self.parse_command_line(argv) if self._is_dispatching(): ## Only the final child gets file-configs. # Also avoid contaminations with user if generating-config. return cl_config = copy.deepcopy(self.config) self.load_config_files() self.update_config(cl_config) print_config = trt.Bool( False, help= """Enable it to print the configurations before launching any command.""" ).tag(config=True) def start(self): """Dispatches into sub-cmds (if any), and then delegates to :meth:`run(). If overriden, better invoke :func:`super()`, but even better to override :meth:``run()`. """ if self.print_config: self.log.info('Running cmd %r with config: \n %s', self.name, self.config) if self.subapp is not None: pass elif self.default_subcmd: self.initialize_subcommand(self.default_subcmd, self.argv) else: return self.run(*self.extra_args) return self.subapp.start() def run(self, *args): """Leaf sub-commands must inherit this instead of :meth:`start()` without invoking :func:`super()`. By default, screams about using sub-cmds, or about doing nothing! :param args: Invoked by :meth:`start()` with :attr:`extra_args`. """ if self.subcommands: cmd_line = ' '.join(cl.name for cl in reversed(self.my_cmd_chain())) raise CmdException("Specify one of the sub-commands: " "\n %s\nor type: \n %s -h" % (', '.join(self.subcommands.keys()), cmd_line)) assert False, "Override run() method in cmd subclasses."
class Unit(traitlets.HasTraits): attack_power = traitlets.Integer(default_value=3) hit_points = traitlets.Integer(default_value=200) location = traitlets.Tuple(traitlets.Integer(), traitlets.Integer()) # y, x dead = traitlets.Bool(default_value=False) members = [] opponents = traitlets.Type('__main__.Unit') @classmethod def append(cls, other): cls.members.append(other) def attack(self, other): other.hit_points -= self.attack_power if other.hit_points <= 0: other.dead = True self.opponents.members.remove(other) print(self, 'killed', other) def distance(self, other): return cityblock(self.location, other.location) @property def target(self): opponent_distances = [ self.distance(foe) for foe in self.opponents.members ] potential_targets = [ foe for foe, distance in zip(self.opponents.members, opponent_distances) if distance == 1 ] if not potential_targets: return None elif len(potential_targets) == 1: return potential_targets[0] else: return sorted(potential_targets, key=lambda u: (u.hit_points, *u.location))[0] def move(self): # first, block out your buddies current_dungeon = DUNGEON.copy() allies = np.array( [friend.location for friend in self.members if friend is not self]) if allies.size: # locations are stored as y, x, so: current_dungeon[allies[:, 0], allies[:, 1]] = -1 foe_locations = np.array( [foe.location for foe in self.opponents.members]) # and now find the costs mcp = MCP(current_dungeon, fully_connected=False) cum_costs, traceback = mcp.find_costs( starts=[self.location], #ends=foe_locations, find_all_ends=True) foe_distances = cum_costs[foe_locations[:, 0], foe_locations[:, 1]] if np.isinf(foe_distances.min()): return # no route available to any foe closest_foes = np.arange( len(foe_distances))[foe_distances == foe_distances.min()] closest_foe = sorted(self.opponents.members[i] for i in closest_foes)[0] # now you have one closest foe, reverse the distance calc # and move one step closer mcp = MCP(current_dungeon, fully_connected=False) cum_costs, traceback = mcp.find_costs(ends=[self.location], starts=[closest_foe.location], find_all_ends=False) target_locations = np.argwhere(cum_costs == foe_distances.min() - 1) valid_locations = target_locations[( (target_locations >= np.array(self.location) - 1) & (target_locations <= np.array(self.location) + 1)).all(axis=1)] if valid_locations.size == 0: print(valid_locations) y, x = (sorted(tuple(coords) for coords in valid_locations))[0] print(self, 'moving to', y, x) self.location = (int(y), int(x)) def __eq__(self, other): return (*self.location, self.hit_points) == (*other.location, other.hit_points) def __lt__(self, other): return (*self.location, self.hit_points) < (*other.location, other.hit_points) def __gt__(self, other): return (*self.location, self.hit_points) == (*other.location, other.hit_points) def __repr__(self): return f'<{self.__class__.__name__} ap{self.attack_power} hp{self.hit_points} loc{self.location}>' def __add__(self, other): return self.hit_points + other.hit_points def __radd__(self, other): return self.hit_points + other
class Elf(Unit): members = [] opponents = traitlets.Type('__main__.Goblin')
class PyDAP(DataSource): """Create a DataSource from an OpenDAP server feed. Attributes ---------- auth_class : :class:`podpac.authentication.Session` :class:`requests.Session` derived class providing authentication credentials. When username and password are provided, an auth_session is created using this class. auth_session : :class:`podpac.authentication.Session` Instance of the auth_class. This is created if username and password is supplied, but this object can also be supplied directly datakey : str Pydap 'key' for the data to be retrieved from the server. Datasource may have multiple keys, so this key determines which variable is returned from the source. dataset : pydap.model.DatasetType The open pydap dataset. This is provided for troubleshooting. native_coordinates : Coordinates {native_coordinates} password : str, optional Password used for authenticating against OpenDAP server. WARNING: this is stored as plain-text, provide auth_session instead if you have security concerns. source : str URL of the OpenDAP server. username : str, optional Username used for authenticating against OpenDAP server. WARNING: this is stored as plain-text, provide auth_session instead if you have security concerns. """ # required inputs source = tl.Unicode(allow_none=False, default_value='') datakey = tl.Unicode(allow_none=False).tag(attr=True) # optional inputs and later defined traits auth_session = tl.Instance(authentication.Session, allow_none=True) auth_class = tl.Type(authentication.Session) username = tl.Unicode(None, allow_none=True) password = tl.Unicode(None, allow_none=True) dataset = tl.Instance('pydap.model.DatasetType', allow_none=False) @tl.default('auth_session') def _auth_session_default(self): # requires username and password if not self.username or not self.password: return None # requires auth_class # TODO: default auth_class? if not self.auth_class: return None # instantiate and check utl try: session = self.auth_class(username=self.username, password=self.password) session.get(self.source + '.dds') except: # TODO: catch a 403 error return None return session @tl.default('dataset') def _open_dataset(self, source=None): """Summary Parameters ---------- source : str, optional Description Returns ------- TYPE Description """ # TODO: is source ever None? # TODO: enforce string source if source is None: source = self.source else: self.source = source # auth session # if self.auth_session: try: dataset = pydap.client.open_url(source, session=self.auth_session) except Exception: # TODO handle a 403 error # TODO: Check Url (probably inefficient...) try: self.auth_session.get(self.source + '.dds') dataset = pydap.client.open_url(source, session=self.auth_session) except Exception: # TODO: handle 403 error print( "Warning, dataset could not be opened. Check login credentials." ) dataset = None return dataset @tl.observe('source') def _update_dataset(self, change=None): if change is None: return if change['old'] == None or change['old'] == '': return if self.dataset is not None and 'new' in change: self.dataset = self._open_dataset(source=change['new']) try: if self.native_coordinates is not None: self.native_coordinates = self.get_native_coordinates() except NotImplementedError: pass @common_doc(COMMON_DATA_DOC) def get_native_coordinates(self): """{get_native_coordinates} Raises ------ NotImplementedError DAP has no mechanism for creating coordinates automatically, so this is left up to child classes. """ raise NotImplementedError( "DAP has no mechanism for creating coordinates" + ", so this is left up to child class " + "implementations.") @common_doc(COMMON_DATA_DOC) def get_data(self, coordinates, coordinates_index): """{get_data} """ data = self.dataset[self.datakey][tuple(coordinates_index)] # PyDAP 3.2.1 gives a numpy array for the above, whereas 3.2.2 needs the .data attribute to get a numpy array if not isinstance(data, np.ndarray) and hasattr(data, 'data'): data = data.data d = self.create_output_array(coordinates, data=data.reshape(coordinates.shape)) return d @property def keys(self): """The list of available keys from the OpenDAP dataset. Returns ------- List The list of available keys from the OpenDAP dataset. Any of these keys can be set as self.datakey """ return self.dataset.keys()
class ParallelAsync(Parallel): """ This class launches the parallel node evaluations in threads up to n_workers, and expects the node.eval to return quickly for parallel execution. This Node was written with aws.Lambda(eval_timeout=1.25<small>) Nodes in mind. Users can implement the `check_worker_available` method or specify the `no_worker_exception` attribute, which is an exception thrown if workers are not available. Attributes ----------- chunks: dict Dictionary of dimensions and sizes that will be iterated over. If a dimension is not in this dictionary, the size of the eval coordinates will be used for the chunk. In this case, it may not be possible to automatically set the coordinates of missing dimensions in the final file. fill_output: bool Default is True. When True, the final results will be assembled and returned to the user. If False, the final results should be written to a file by specifying the output_format in a Process or Lambda node. See note below. source: podpac.Node The source dataset for the computation sleep_time: float Default is 1 second. Number of seconds to sleep between trying to submit new workers no_worker_exception: Exception, optional Default is .Exception class used to identify when a submission failed due to no available workers. The default is chosen to work with the podpac.managers.Lambda node. async_exception: Exception Default is botocore.exceptions.ReadTimeoutException. This is an exception thrown by the async function in case it time out waiting for a return. In our case, this is a success. The default is chosen to work with the podpac.managers.Lambda node. Notes ------ In some cases where the input and output coordinates of the source node is not the same (such as reduce nodes) and fill_output is True, the user may need to specify 'output' as part of the eval call. """ source = NodeTrait().tag(attr=True) chunks = tl.Dict().tag(attr=True) fill_output = tl.Bool(True).tag(attr=True) sleep_time = tl.Float(1).tag(attr=True) no_worker_exception = tl.Type( botocore.exceptions.ClientError).tag(attr=True) async_exception = tl.Type( botocore.exceptions.ReadTimeoutError).tag(attr=True) def check_worker_available(self): return True def eval_source(self, coordinates, coordinates_index, out, i, source=None): if source is None: source = self.source # Make a copy to prevent any possibility of memory corruption source = Node.from_definition(source.definition) success = False o = None while not success: if self.check_worker_available(): try: o = source.eval(coordinates, output=out) success = True except self.async_exception: # This exception is fine and constitutes a success o = None success = True except self.no_worker_exception as e: response = e.response if not (response and response.get("Error", {}).get("Code") == "TooManyRequestsException"): raise e # Raise error again, not the right error _log.debug("Worker {} exception {}".format(i, e)) success = False time.sleep(self.sleep_time) else: _log.debug("Worker unavailable for {}".format(i, e)) time.sleep(self.sleep_time) _log.info("Submitting source {}".format(i)) return (o, coordinates_index)
class S3(DataSource): """Create a DataSource from a file on an S3 Bucket. Attributes ---------- node : Node, optional The DataSource node used to interpret the S3 file node_class : DataSource, optional The class type of self.node. This is used to create self.node if self.node is not specified node_kwargs : dict, optional Keyword arguments passed to `node_class` when automatically creating `node` return_type : str, optional Either: 'file_handle' (for files downloaded to RAM); or the default option 'path' (for files downloaded to disk) s3_bucket : str, optional Name of the S3 bucket. Uses ``podpac.settings['S3_BUCKET_NAME']`` by default. s3_data : file/str If return_type == 'file_handle' returns a file pointer object If return_type == 'path' returns a string to the data source : str Path to the file residing in the S3 bucket that will be loaded """ source = tl.Unicode() node = tl.Instance(Node) node_class = tl.Type(DataSource) # A class node_kwargs = tl.Dict(default_value={}) s3_bucket = tl.Unicode(allow_none=True) s3_data = tl.Any(allow_none=True) _temp_file_cleanup = tl.List() return_type = tl.Enum(['file_handle', 'path'], default_value='path') # TODO: handle s3 auth setup @tl.default('node') def node_default(self): """Creates the default node using the node_class and node_kwargs Returns ------- self.node_class Instance of self.node_class Raises ------ Exception This function sets the source in the node, so 'source' cannot be present in node_kwargs """ if 'source' in self.node_kwargs: raise Exception("'source' present in node_kwargs for S3") return self.node_class(source=self.s3_data, **self.node_kwargs) @tl.default('s3_bucket') def s3_bucket_default(self): """Retrieves default S3 Bucket from settings Returns ------- Str Name of the S3 bucket """ return settings['S3_BUCKET_NAME'] @tl.default('s3_data') def s3_data_default(self): """Returns the file handle or path to the S3 bucket Returns ------- str/file Either a string to the downloaded file path, or a file handle """ if self.s3_bucket is None: raise ValueError('No s3 bucket set') s3 = boto3.resource('s3').Bucket(self.s3_bucket) if self.return_type == 'file_handle': # TODO: should this use the with/as syntax # https://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.download_fileobj # download into memory io = BytesIO() s3.download_fileobj(self.source, io) io.seek(0) return io elif self.return_type == 'path': # Download the file to cache directory #tmppath = os.path.join(tempfile.gettempdir(), #self.source.replace('\\', '').replace(':','')\ #.replace('/', '')) tmppath = os.path.join( settings['CACHE_DIR'], self.source.replace('\\', '').replace(':', '').replace('/', '')) rootpath = os.path.split(tmppath)[0] if not os.path.exists(rootpath): os.makedirs(rootpath) #i = 0 #while os.path.exists(tmppath): #tmppath = os.path.join(tempfile.gettempdir(), #self.source + '.%d' % i) if not os.path.exists(tmppath): s3.download_file(self.source, tmppath) # TODO: should we handle temp files here? #self._temp_file_cleanup.append(tmppath) return tmppath @common_doc(COMMON_DATA_DOC) def get_data(self, coordinates, coordinates_index): """{get_data} """ self.nan_vals = getattr(self.node, 'nan_vals', []) return self.node.get_data(coordinates, coordinates_index) @property @common_doc(COMMON_DATA_DOC) def native_coordinates(self): """{native_coordinates} """ return self.node.native_coordinates def __del__(self): if hasattr(super(S3), '__del__'): super(S3).__del__(self) for f in self._temp_file_cleanup: os.remove(f)
class WorkflowIPythonKernel(IPythonKernel): # Set StreamFlow shell shell_class = traitlets.Type(StreamFlowInteractiveShell) # Kernel info fields implementation = 'sf-ipython' implementation_version = streamflow.version.VERSION language_info = { 'name': 'python', 'version': sys.version.split()[0], 'mimetype': 'text/x-python', 'codemirror_mode': { 'name': 'ipython', 'version': sys.version_info[0] }, 'pygments_lexer': 'ipython%d' % (3 if PY3 else 2), 'nbconvert_exporter': 'python', 'file_extension': '.py' } msg_types = IPythonKernel.msg_types + ['workflow_request'] def init_metadata(self, parent): # Call parent functionse metadata = super().init_metadata(parent) # If StreamFlow has been configured for this cell, store its configuration workflow_config = ( parent['metadata'] if 'workflow' in parent['metadata'] else parent['content'] if 'workflow' in parent['content'] else None) if workflow_config is not None: try: validate({ k: v for k, v in workflow_config['workflow'].items() if k != 'cell_id' }) except BaseException as e: self.log.error(str(e)) return metadata metadata['sf_token'] = self.shell.wf_cell_config.set( workflow_config['workflow']) # Return metadata return metadata def finish_metadata(self, parent, metadata, reply_content): # Remove StreamFlow configuration from the `sf_cell_config` attribute if present if 'sf_token' in metadata: self.shell.wf_cell_config.reset(metadata['sf_token']) del metadata['sf_token'] # Call parent function return super().finish_metadata(parent, metadata, reply_content) @gen.coroutine def workflow_request(self, stream, ident, parent): try: content = parent['content'] notebook = content['notebook'] except: self.log.error("Got bad msg: ") self.log.error("%s", parent) return metadata = self.init_metadata(parent) reply_content = yield gen.maybe_future( self.do_workflow(notebook, ident, parent)) sys.stdout.flush() sys.stderr.flush() if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) metadata = self.finish_metadata(parent, metadata, reply_content) reply_msg = self.session.send(stream, 'execute_reply', reply_content, parent, metadata=metadata, ident=ident) self.log.debug("%s", reply_msg) @gen.coroutine def do_workflow(self, notebook, ident, parent): shell = self.shell reply_content = {} if (_asyncio_runner and shell.loop_runner is _asyncio_runner and asyncio.get_event_loop().is_running()): coro = shell.run_workflow(notebook) coro_future = asyncio.ensure_future(coro) with self._cancel_on_sigint(coro_future): try: res = yield coro_future finally: shell.events.trigger('post_execute') else: coro = shell.run_workflow(notebook) if shell.trio_runner: runner = shell.trio_runner else: runner = shell.loop_runner res = runner(coro) # Send outputs to cell streams for cell_name, content in res.result.items(): self.session.send(self.iopub_thread, 'stream', content={ 'name': cell_name, 'text': content }, parent=extract_header(parent), ident=ident) # Send reply message if res.error_before_exec is not None: err = res.error_before_exec else: err = res.error_in_exec if res.success: reply_content['status'] = 'ok' else: reply_content['status'] = 'error' # noinspection PyProtectedMember reply_content.update({ 'traceback': shell._last_traceback or [], 'ename': str(type(err).__name__), 'evalue': safe_unicode(err), }) reply_content['execution_count'] = shell.execution_count - 1 reply_content['payload'] = shell.payload_manager.read_payload() shell.payload_manager.clear_payload() return reply_content
class VoilaExporter(HTMLExporter): """Custom HTMLExporter that inlines the images using VoilaMarkdownRenderer""" base_url = traitlets.Unicode(help="Base url for resources").tag(config=True) markdown_renderer_class = traitlets.Type('mistune.Renderer').tag(config=True) # The voila exporter overrides the markdown renderer from the HTMLExporter # to inline images. @contextfilter def markdown2html(self, context, source): cell = context['cell'] attachments = cell.get('attachments', {}) cls = self.markdown_renderer_class renderer = cls(escape=False, attachments=attachments, contents_manager=self.contents_manager, anchor_link_text=self.anchor_link_text) return MarkdownWithMath(renderer=renderer).render(source) # The voila exporter disables the CSSHTMLHeaderPreprocessor from the HTMLExporter. @property def default_config(self): c = Config({ 'VoilaExporter': { 'markdown_renderer_class': 'voila.exporter.VoilaMarkdownRenderer' } }) c.merge(super(VoilaExporter, self).default_config) return c # Overriding the default template file. @traitlets.default('template_file') def default_template_file(self): return 'index.html.j2' async def generate_from_notebook_node(self, nb, resources=None, extra_context={}, **kw): # this replaces from_notebook_node, but calls template.generate instead of template.render langinfo = nb.metadata.get('language_info', {}) lexer = langinfo.get('pygments_lexer', langinfo.get('name', None)) highlight_code = self.filters.get('highlight_code', Highlight2HTML(pygments_lexer=lexer, parent=self)) self.register_filter('highlight_code', highlight_code) # NOTE: we don't call HTML or TemplateExporter' from_notebook_node nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw) resources.setdefault('raw_mimetypes', self.raw_mimetypes) resources['global_content_filter'] = { 'include_code': not self.exclude_code_cell, 'include_markdown': not self.exclude_markdown, 'include_raw': not self.exclude_raw, 'include_unknown': not self.exclude_unknown, 'include_input': not self.exclude_input, 'include_output': not self.exclude_output, 'include_input_prompt': not self.exclude_input_prompt, 'include_output_prompt': not self.exclude_output_prompt, 'no_prompt': self.exclude_input_prompt and self.exclude_output_prompt, } async for output in self.template.generate_async(nb=nb_copy, resources=resources, **extra_context): yield (output, resources) @property def environment(self): # enable Jinja async template execution self.enable_async = True env = super(type(self), self).environment if 'jinja2.ext.do' not in env.extensions: env.add_extension('jinja2.ext.do') return env def get_template_paths(self): return self.template_path def _init_resources(self, resources): def include_css(name): code = """<link rel="stylesheet" type="text/css" href="%svoila/%s">""" % (self.base_url, name) return jinja2.Markup(code) def include_js(name): code = """<script src="%svoila/%s"></script>""" % (self.base_url, name) return jinja2.Markup(code) def include_url(name): url = "%svoila/%s" % (self.base_url, name) return jinja2.Markup(url) resources = super(VoilaExporter, self)._init_resources(resources) resources['include_css'] = include_css resources['include_js'] = include_js resources['include_url'] = include_url return resources
class FastDazzlerSetupLoader(DatasetLoader): type = 'FastDazzler' trim_to = traitlets.Instance(klass=slice, args=(10, None), help='range of indexes to throw away when loading a camera file' +\ '; used for removing saturated frames at the beginning of ' +\ 'acquisition', config=True) batch_name = traitlets.Unicode(default_value='test-batch', help='name of job used in LabView software', config=True, allow_none=False) batch_path = traitlets.Unicode(help='location of the batch folder on disk', config=True, allow_none=False) nwaveforms = traitlets.Int(default_value=1, help='number of waveforms in the Dazzler table', config=True) nrepeat = traitlets.Int(default_value=1, help='number of times each waveform is repeated in camera file; not '\ + 'the same as Dazzler NRepeat', config=True) data_type = traitlets.Type(klass=Dataset, config=True) allow_parallel = traitlets.Bool(default_value=False, config=True) def __init__(self, config=None): super(FastDazzlerSetupLoader, self).__init__(config=config) def load(self): pattern = re.compile('(?P<job_name>[\w-]+)-batch(?P<batch_no>\d+)') p = pathlib.Path(self.batch_path) log.debug('loading {path!s}'.format(path=p.name)) m = pattern.match(p.name) if m is None: raise ValueError('wrong job-name format') log.debug( 'matched: job_name = `{job_name:s}`, batch_no = `{batch_no:s}`'. format(job_name=m.group('job_name'), batch_no=m.group('batch_no'))) batch_info = next( load_job(m.group('job_name'), when=None, batch_set=set([int(m.group('batch_no'))]), data_path=p.parent)) raw_datas = [] # loop over available data for t2, table, loop in it.product(batch_info['t2_range'], batch_info['table_range'], batch_info['loop_range']): c = Config() # copy over settings used c.RawData.batch_name = self.batch_name c.RawData.batch_path = self.batch_path c.RawData.nwaveforms = self.nwaveforms c.RawData.nrepeat = self.nrepeat c.RawData.t2 = t2 c.RawData.table = table c.RawData.loop = loop c.RawData.phase_cycles = self.data_type.phase_cycles if self.allow_parallel: raw_datas.append(delayed(RawData)(config=c).process()) else: raw_datas.append(RawData(config=c).process()) # create dataset and process it if self.allow_parallel: dset = delayed(self.data_type)(config=self.config) else: dset = self.data_type(config=self.config) return dset.from_raw_data(raw_datas)