def test_cat_with_no_declared_name_gets_name_from_dir_if_file_named_catalog():
fn = abspath("catalog.yml")
cat = open_nested_yaml_cat(fn,
name='name_in_func',
description='Description in func')
assert cat.name == 'name_in_func'
assert cat.description == 'Description in func'
cat = open_nested_yaml_cat(fn)
assert cat.name == 'test_local'
assert cat.description == None
def test_cat_with_declared_name():
fn = abspath("catalog_named.yml")
description = 'Description declared in the open function'
cat = open_nested_yaml_cat(fn,
name='name_in_func',
description=description)
assert cat.name == 'name_in_func'
assert cat.description == description
assert cat.metadata.get('some') == 'thing'
cat = open_nested_yaml_cat(fn)
assert cat.name == 'name_in_spec'
assert cat.description == 'This is a catalog with a description in the yaml'
def test_fsspec_integration():
import fsspec
import pandas as pd
mem = fsspec.filesystem('memory')
with mem.open('cat.yaml', 'wt') as f:
f.write("""
metadata:
hierarchical_catalog: True
implicit:
driver: csv
description: o
args:
urlpath: "{{CATALOG_DIR}}/file.csv"
explicit:
driver: csv
description: o
args:
urlpath: "memory:///file.csv"
extra:
driver: csv
description: o
args:
urlpath: "{{CATALOG_DIR}}/file.csv"
storage_options: {other: option}""")
with mem.open('/file.csv', 'wt') as f:
f.write("a,b\n0,1")
expected = pd.DataFrame({'a': [0], 'b': [1]})
cat = open_nested_yaml_cat("memory://cat.yaml")
assert list(cat) == ['implicit', 'explicit', 'extra']
assert cat.implicit.read().equals(expected)
assert cat.explicit.read().equals(expected)
s = cat.extra()
assert s._storage_options['other']
def test_cat_add(tmpdir):
tmpdir = str(tmpdir)
fn = os.path.join(tmpdir, 'cat.yaml')
with open(fn, 'w') as f:
f.write("""
metadata:
hierarchical_catalog: True
""")
cat = open_nested_yaml_cat(fn)
assert list(cat) == []
# was added in memory
cat.add(cat)
cat._load() # this would happen automatically, but not immediately
assert list(cat) == ['cat']
# was added to the file
cat = open_nested_yaml_cat(fn)
assert list(cat) == ['cat']
def test_multi_plugins():
from intake.source.csv import CSVSource
fn = abspath('multi_plugins.yaml')
cat = open_nested_yaml_cat(fn)
s = cat.level1.tables0()
assert isinstance(s, CSVSource)
s = cat.level1.tables1()
assert isinstance(s, CSVSource)
s = cat.level1.tables2()
assert isinstance(s, CSVSource)
s = cat.level1.tables3()
assert isinstance(s, CSVSource)
assert s._csv_kwargs == {}
s = cat.level1.tables3(plugin='myplug')
assert isinstance(s, CSVSource)
assert s._csv_kwargs == {}
s = cat.level1.tables3(plugin='myplug2')
assert isinstance(s, CSVSource)
assert s._csv_kwargs is True
with pytest.raises(ValueError):
cat.level1.tables4()
with pytest.raises(ValueError):
cat.level1.tables4(plugin='myplug')
with pytest.raises(ValueError):
cat.level1.tables4(plugin='myplug2')
s = cat.level1.tables5()
assert isinstance(s, CSVSource)
with pytest.raises(ValueError):
cat.level1.tables5(plugin='myplug')
fn = abspath('multi_plugins2.yaml')
with pytest.raises(ValueError):
open_nested_yaml_cat(fn)
def test_no_plugins():
fn = abspath('multi_plugins.yaml')
cat = open_nested_yaml_cat(fn)
s = cat.level1.tables6
with pytest.raises(ValueError) as e:
s()
assert 'doesnotexist' in str(e.value)
assert 'plugin-directory' in str(e.value)
s = cat.level1.tables7
with pytest.raises(ValueError) as e:
s()
assert 'doesnotexist' in str(e.value)
def test_dict_save():
from intake.catalog.base import Catalog
fn = os.path.join(tempfile.mkdtemp(), 'mycat.yaml')
entry = LocalCatalogEntry(name='trial',
description='get this back',
driver='csv')
cat = Catalog.from_dict({'trial': entry}, name='mycat')
cat.save(fn)
cat2 = open_nested_yaml_cat(fn)
assert 'trial' in cat2
assert cat2.name == 'mycat'
assert cat2.trial._driver == 'csv'
def test_dot_names():
fn = abspath('dot-nest.yaml')
cat = open_nested_yaml_cat(fn)
assert cat.level1.self._container == 'catalog'
assert cat.level1.self.level1.leaf._description == 'leaf'
assert cat.level1.self['level1.leafdot.dot']._description == 'leaf-dot'
assert cat['level1.selfdot.dot',
'level1.leafdot.dot']._description == 'leaf-dot'
assert cat['level1.self.level1.selfdot.dot',
'level1.leafdot.dot']._description == 'leaf-dot'
assert cat['level1.self.level1.self.dot',
'level1.leafdot.dot']._description == 'leaf-dot'
assert cat['level1.self.level1.self.dot',
'level1.leaf']._description == 'leaf'
assert cat['level1.self.level1.self.dot',
'level1.leaf.dot']._description == 'leaf-dot'
assert cat[
'level1.self.level1.self.dot.level1.leaf.dot']._description == 'leaf-dot'
def test_getitem_and_getattr():
fn = abspath('multi_plugins.yaml')
catalog = open_nested_yaml_cat(fn)
catalog.level1['tables0']
with pytest.raises(KeyError):
catalog.level1['doesnotexist']
with pytest.raises(KeyError):
catalog.level1['_doesnotexist']
with pytest.raises(KeyError):
# This exists as an *attribute* but not as an item.
catalog.level1['metadata']
catalog.level1.tables0 # alias to catalog['tables0']
catalog.metadata # a normal attribute
with pytest.raises(AttributeError):
catalog.doesnotexit
with pytest.raises(AttributeError):
catalog._doesnotexit
assert catalog.level1.tables0 is catalog['level1.tables0']
assert isinstance(catalog.level1.tables0, LocalCatalogEntry)
assert isinstance(catalog.metadata, (dict, type(None)))
def test_intake_nested_yaml_cat():
cat = intake.open_nested_yaml_cat(nested_yaml_cat_path)
assert {'customer', 'user'} == set(cat.entity)
assert {'customer_attributes',
'retention_project'} == set(cat.entity.customer)
assert 'description' == cat.entity.description
entry_description = cat.entity.customer.retention_project.good_customers.describe(
)
assert 'good_customers' == entry_description['name']
assert 'good_customers description' == entry_description['description']
discover = cat.entity.customer.retention_project.good_customers.discover()
assert discover['dtype'].keys() == {'customer_id', 'name'}
df = cat.entity.customer.retention_project.good_customers().read()
assert [101, 'Bob'] == df.loc[0].values.tolist()
df = cat.entity.customer.retention_project.good_customers(
kind="funny").read()
assert [31, 'Donald Duck'] == df.loc[0].values.tolist()
import intake
import os
here = os.path.abspath(os.path.dirname(__file__))
# the catalog is a YAML file in the same directory as this init file
cat = intake.open_nested_yaml_cat(os.path.join(here, "nested_catalog.yaml"))
original_data = (cat.main_catalog.shared.original.capital_projects()
) # <- note the parentheses
# after installation, this will be available as intake.cat.cs109b, with one entry, main_catalog
# and intake.cat.airpred_clean, which is a data source.
# intake.cat.cs109b.main_catalog.shared.original.airpred_clean and intake.cat.airpred_clean are identical.
def test_gui():
pytest.importorskip('panel')
cat = intake.open_nested_yaml_cat(nested_yaml_cat_path)
assert repr(intake.gui).startswith('Row')
intake.gui.add(cat)
def test_nested_cat_walk():
cat = intake.open_nested_yaml_cat(nested_yaml_cat_path)
assert 'entity.customer.retention_project.good_customers' not in list(
cat.walk(depth=3))
assert 'entity.customer.retention_project.good_customers' in list(
cat.walk(depth=4))
def test_missing_hierarchical_catalog():
invalid_nested_yaml_cat_path = str(
Path(__file__).resolve().parent.joinpath(
Path("invalid_nested_yaml_cat.yaml")))
intake.open_nested_yaml_cat(invalid_nested_yaml_cat_path)
def test_version():
cat = intake.open_nested_yaml_cat(nested_yaml_cat_path)
assert cat.version != ''
def catalog1():
path = os.path.dirname(__file__)
return open_nested_yaml_cat(os.path.join(path, 'catalog1.yml'))
