def openFromCatalog(self, **kwargs) -> xa.Dataset:
if self.catalog is None:
cat_file = IntakeNode.getCatalogFilePath([], **kwargs)
print( f"Opening aggregation from file {cat_file}" )
self.catalog: YAMLFileCatalog = intake.open_catalog( cat_file, driver="yaml_file_cat")
self.catalog.discover()
ds: xa.Dataset = getattr( self.catalog, self.name ).to_dask()
return ds
def getCatalog(self, **kwargs) -> YAMLFileCatalog:
if self.catalog is None:
cat_nodes = kwargs.pop("cat_nodes", [])
cdir = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
cat_file = os.path.join(cdir, "catalog.json")
print(f"Opening collection from file {cat_file}")
self.catalog = intake.open_catalog(cat_file,
driver="yaml_file_cat")
self.catalog.discover()
return self.catalog
def writeCatalogFile(self, **kwargs) -> Optional[str]:
cat_nodes = kwargs.pop("cat_nodes", [ self.collection, self.name ] )
self.openDataSource( **kwargs )
if self.dataSource is None: return None
catalog_file = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
with open( catalog_file, 'w' ) as f:
yaml = self.dataSource.yaml()
print( f"Writing aggregation {self.name} to {catalog_file}")
f.write( yaml )
return catalog_file
def generate(self, aggs: Dict[str, DataSource], **kwargs):
cat_nodes = kwargs.pop("cat_nodes", [])
catalog_file = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
print(f"Opening collection {self.name} with aggs:\n")
pp(aggs.keys())
catalog: Catalog = Catalog.from_dict(aggs,
name=self.name,
description=self.description,
metadata=str_dict(self.metadata))
with open(catalog_file, 'w') as f:
yaml = catalog.yaml()
print(f"\nWriting collection {self.name} to {catalog_file}")
f.write(yaml)
def generate(self, **kwargs):
cat_nodes = kwargs.pop("cat_nodes",
[] if self.name == "root" else [self.name])
catalog_file = IntakeNode.getCatalogFilePath(cat_nodes, **kwargs)
cat_items = kwargs.get('cats')
if not cat_items:
cdir = os.path.dirname(catalog_file)
cat_items = glob(f"{cdir}/*/catalog.yaml")
print(f"Opening collection {self.name} with items:\n")
pp(cat_items)
catalogs: YAMLFilesCatalog = intake.open_catalog(
cat_items, driver="yaml_files_cat")
catalogs.name = self.name
catalogs.description = self.description
catalogs.metadata = str_dict(self.metadata)
with open(catalog_file, 'w') as f:
yaml = catalogs.yaml()
print(f"\nWriting collection {self.name} to {catalog_file}")
f.write(yaml)
return catalog_file