def openFromCatalog(self, **kwargs) -> xa.Dataset:
if self.catalog is None:
cat_file = IntakeNode.getCatalogFilePath([], **kwargs)
print( f"Opening aggregation from file {cat_file}" )
self.catalog: YAMLFileCatalog = intake.open_catalog( cat_file, driver="yaml_file_cat")
self.catalog.discover()
ds: xa.Dataset = getattr( self.catalog, self.name ).to_dask()
return ds
def getCatalog(self, **kwargs) -> YAMLFileCatalog:
if self.catalog is None:
cat_nodes = kwargs.pop("cat_nodes", [])
cdir = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
cat_file = os.path.join(cdir, "catalog.json")
print(f"Opening collection from file {cat_file}")
self.catalog = intake.open_catalog(cat_file,
driver="yaml_file_cat")
self.catalog.discover()
return self.catalog
def writeCatalogFile(self, **kwargs) -> Optional[str]:
cat_nodes = kwargs.pop("cat_nodes", [ self.collection, self.name ] )
self.openDataSource( **kwargs )
if self.dataSource is None: return None
catalog_file = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
with open( catalog_file, 'w' ) as f:
yaml = self.dataSource.yaml()
print( f"Writing aggregation {self.name} to {catalog_file}")
f.write( yaml )
return catalog_file
def generate(self, aggs: Dict[str, DataSource], **kwargs):
cat_nodes = kwargs.pop("cat_nodes", [])
catalog_file = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
print(f"Opening collection {self.name} with aggs:\n")
pp(aggs.keys())
catalog: Catalog = Catalog.from_dict(aggs,
name=self.name,
description=self.description,
metadata=str_dict(self.metadata))
with open(catalog_file, 'w') as f:
yaml = catalog.yaml()
print(f"\nWriting collection {self.name} to {catalog_file}")
f.write(yaml)
def generate(self, **kwargs):
cat_nodes = kwargs.pop("cat_nodes",
[] if self.name == "root" else [self.name])
catalog_file = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
cat_items = kwargs.get('cats')
if not cat_items:
cdir = os.path.dirname(catalog_file)
cat_items = glob(f"{cdir}/*/catalog.yaml")
print(f"Opening collection {self.name} with items:\n")
pp(cat_items)
catalogs: YAMLFilesCatalog = intake.open_catalog(
cat_items, driver="yaml_files_cat")
catalogs.name = self.name
catalogs.description = self.description
catalogs.metadata = str_dict(self.metadata)
with open(catalog_file, 'w') as f:
yaml = catalogs.yaml()
print(f"\nWriting collection {self.name} to {catalog_file}")
f.write(yaml)
return catalog_file
def __init__(self, name: str = "root", **kwargs):
IntakeNode.__init__(self, name, **kwargs)
def __init__(self, path_nodes: List[str], **kwargs):
IntakeNode.__init__(self, path_nodes, **kwargs)
def close(self):
IntakeNode.close(self)
if self.dataSource: self.dataSource.close()
def __init__( self, name: str, **kwargs ):
IntakeNode.__init__(self, name, **kwargs)
self.files = kwargs.get( "files", None )
self.collection = kwargs.get( "collection", self.name )
self.dataSource: Optional[DataSource] = None
self.openDataSource( **kwargs )
def saveCatalog(cls, catalog: Catalog) -> str:
cat_path = catalog.path
catalog.save(cat_path)
IntakeNode.patch_yaml(cat_path)
return cat_path