def addCatNodes(baseNode: CatalogNode, curdir: str): subDirs = [ sdir for sdir in os.listdir(curdir) if path.isdir(path.join(curdir, sdir)) ] if len(set(subDirs).intersection(vars)) > 0: data_collection: DataCollectionGenerator = baseNode.addDataCollection( path.basename(curdir)) for sDir in subDirs: agg_op_params.append( dict(node=data_collection, name=sDir, files=f"{curdir}/{sDir}/*.nc")) else: node_name = path.basename(curdir) current_node = baseNode.addCatalogNodes(node_name) for subDir in glob(f"{curdir}/*"): if os.path.isdir(subDir): addCatNodes(current_node, path.join(curdir, subDir))
from intake.gui import gui from collection_intake.xintake.catalog import CatalogNode base_catalog: CatalogNode = CatalogNode.getCatalogBase() print(f"Starting intake gui with base catalog: {base_catalog}") gui = gui.GUI([base_catalog.catalog]) gui.show()
from glob import glob from collection_intake.xintake.catalog import CatalogNode from collection_intake.xintake.collection import DataCollectionGenerator import os, intake, math, time from multiprocessing import Pool import multiprocessing as mp cBase: CatalogNode = CatalogNode.getCatalogBase() cImage = cBase.addCatalogNode("image", description="NCCS Image collections") cBioclim = cImage.addCatalogNode("bioclim", description="") collection_root = "/nfs4m/css/curated01/bioclim/data/" model_dirs = glob(f"{collection_root}/*") aggs = {} for model_dir in model_dirs: model_name = os.path.basename(model_dir) for index in range(1, 19): files = glob(f"{model_dir}/bio{index}_equiv_*.tiff") aggs[model_name] def createAggregation(agg_dir: str): print( f"Adding aggregation data source to bioclim collection for data path {agg_dir}" ) dsname = os.path.basename(agg_dir) data_collection: DataCollectionGenerator = cBioclim.addDataCollection( dsname) data_collection.addFileCollection(f"{agg_dir}/*.tiff", driver="rasterio",
def getCatalog(cls, cat_path: str) -> Catalog: return intake.open_catalog( CatalogNode.getCatalogURI(cat_path.split('/')))
def __init__(self, **kwargs): self.catalog_path = kwargs.get("cat_path", CatalogNode.getCatalogURI())
from collection_intake.xintake.catalog import CatalogNode from intake.catalog.local import YAMLFileCatalog, Catalog, LocalCatalogEntry from dateutil.parser import parse catNode: CatalogNode = CatalogNode.open( "image/ABoVE/ORNL_AVIRIS_NG/ang_rdn_v2r2" ) sources = catNode.sources source: LocalCatalogEntry = catNode.getSource( sources[0] ) print( source.__class__.__name__ )