def ingest(config: str):
"""Main command for ingesting metadata into DataHub"""
config_file = pathlib.Path(config)
if not config_file.is_file():
raise ConfigurationError(f"Cannot open config file {config}")
config_mech: ConfigurationMechanism
if config_file.suffix in [".yaml", ".yml"]:
config_mech = YamlConfigurationMechanism()
elif config_file.suffix == ".toml":
config_mech = TomlConfigurationMechanism()
else:
raise ConfigurationError(
"Only .toml and .yml are supported. Cannot process file type {}".
format(config_file.suffix))
with config_file.open() as fp:
pipeline_config = config_mech.load_config(fp)
try:
logger.debug(f"Using config: {pipeline_config}")
pipeline = Pipeline.create(pipeline_config)
except ValidationError as e:
click.echo(e, err=True)
sys.exit(1)
pipeline.run()
ret = pipeline.pretty_print_summary()
sys.exit(ret)
def ingest(config: str):
"""Main command for ingesting metadata into DataHub"""
config_file = pathlib.Path(config)
if not config_file.is_file():
raise ConfigurationError(f"Cannot open config file {config}")
config_mech: ConfigurationMechanism
if config_file.suffix in [".yaml", ".yml"]:
config_mech = YamlConfigurationMechanism()
elif config_file.suffix == ".toml":
config_mech = TomlConfigurationMechanism()
else:
raise ConfigurationError(
"Only .toml and .yml are supported. Cannot process file type {}".format(
config_file.suffix
)
)
with config_file.open() as fp:
pipeline_config = config_mech.load_config(fp)
with nicely_formatted_validation_errors():
logger.debug(f"Using config: {pipeline_config}")
pipeline = Pipeline.create(pipeline_config)
pipeline.run()
def load_config_file(config_file: Union[pathlib.Path, str]) -> dict:
if isinstance(config_file, str):
config_file = pathlib.Path(config_file)
if not config_file.is_file():
raise ConfigurationError(f"Cannot open config file {config_file}")
config_mech: ConfigurationMechanism
if config_file.suffix in [".yaml", ".yml"]:
config_mech = YamlConfigurationMechanism()
elif config_file.suffix == ".toml":
config_mech = TomlConfigurationMechanism()
else:
raise ConfigurationError(
"Only .toml and .yml are supported. Cannot process file type {}".
format(config_file.suffix))
with config_file.open() as raw_config_fp:
raw_config_file = raw_config_fp.read()
config_fp = io.StringIO(raw_config_file)
config = config_mech.load_config(config_fp)
resolve_env_variables(config)
return config
def load_config_file(config_file: pathlib.Path) -> dict:
if not config_file.is_file():
raise ConfigurationError(f"Cannot open config file {config_file}")
config_mech: ConfigurationMechanism
if config_file.suffix in [".yaml", ".yml"]:
config_mech = YamlConfigurationMechanism()
elif config_file.suffix == ".toml":
config_mech = TomlConfigurationMechanism()
else:
raise ConfigurationError(
"Only .toml and .yml are supported. Cannot process file type {}".format(
config_file.suffix
)
)
with config_file.open() as raw_config_fp:
raw_config_file = raw_config_fp.read()
expanded_config_file = expandvars(raw_config_file, nounset=True)
config_fp = io.StringIO(expanded_config_file)
config = config_mech.load_config(config_fp)
return config