def test_kwargs_to_kwargs(self):
self.assertEqual(
{
'input_paths': 'inputs/*.nc',
'input_multi_file': True,
'input_decode_cf': False,
'input_concat_dim': 'time',
'process_rename': {
'lons': 'lon'
},
'output_path': 'my.zarr',
'output_overwrite': False,
'output_append': True,
'dry_run': True,
'verbosity': 2,
},
load_config(return_kwargs=True,
input_paths='inputs/*.nc',
input_decode_cf=False,
input_concat_dim='time',
input_multi_file=True,
process_rename=dict(lons='lon'),
output_path='my.zarr',
output_overwrite=False,
output_append=True,
dry_run=True,
verbosity=2))
def test_many_config_files_to_config(self):
self.assertEqual(
{
'dry_run': False,
'verbosity': 1,
'input': {
'paths': [
'inputs/2009/*.nc',
'inputs/2010/*.nc',
'inputs/2011/*.nc',
],
'concat_dim':
'time',
'decode_cf':
False,
},
'process': {
'rename': {
'longitude': 'lon',
'latitude': 'lat',
},
},
'output': {
'append': True,
'path': 'mybucket/my.zarr',
's3': {
'key': 'mykey',
'secret': 'mysecret'
},
},
},
load_config(config_paths=self.config_paths,
input_paths=['inputs/2011/*.nc'],
dry_run=False,
verbosity=1))
def test_one_config_file_to_kwargs(self):
self.assertEqual(
{
'verbosity': 2,
'input_paths': ['inputs/2009/*.nc'],
'input_concat_dim': 'time',
'input_decode_cf': False,
'output_s3': {
'key': 'mykey',
'secret': 'mysecret'
},
},
load_config(config_paths=[self.config_paths[0]],
return_kwargs=True,
input_decode_cf=False,
verbosity=2))
def test_defaults(self):
self.assertEqual({}, load_config())
def test_not_found(self):
with self.assertRaises(ConverterError) as cm:
load_config(config_paths=['bibo.yml'])
self.assertEqual('Configuration not found: bibo.yml',
f'{cm.exception}')
def nc2zarr(input_paths: Tuple[str], output_path: str,
config_paths: Tuple[str], multi_file: bool,
concat_dim: Optional[str], overwrite: bool, append: bool,
decode_cf: bool, sort_by: str, adjust_metadata: bool,
finalize_only: bool, dry_run: bool, verbose: int, version: bool):
"""
Reads one or more input datasets and writes or appends them to a single
Zarr output dataset.
INPUT_FILE may refer to a NetCDF file, or Zarr dataset, or a glob that
identifies multiple paths, e.g. "L3_SST/**/*.nc".
OUTPUT_PATH must be directory which will contain the output Zarr dataset,
e.g. "L3_SST.zarr".
CONFIG_FILE must be in YAML format. It comprises the optional objects
"input", "process", and "output".
See nc2zarr/res/config-template.yml for a template file that describes the format.
Multiple --config options may be passed as a chain to allow for
reuse of credentials and other common parameters. Contained configuration objects
are recursively merged, lists are appended, and other values overwrite each
other from left to right. For example:
\b
nc2zarr -c s3.yml -c common.yml -c inputs-01.yml -o out-01.zarr
nc2zarr -c s3.yml -c common.yml -c inputs-02.yml -o out-02.zarr
nc2zarr out-01.zarr out-02.zarr -o final.zarr
Command line arguments and options have precedence over other
configurations and thus override settings in any CONFIG_FILE:
\b
[--finalize-only] overrides /finalize_only
[--dry-run] overrides /dry_run
[--verbose] overrides /verbosity
\b
[INPUT_FILE ...] overrides /input/paths in CONFIG_FILE
[--multi-file] overrides /input/multi_file
[--concat-dim] overrides /input/concat_dim
[--decode-cf] overrides /input/decode_cf
[--sort-by] overrides /input/sort_by
\b
[--output OUTPUT_FILE] overrides /output/path
[--overwrite] overrides /output/overwrite
[--append] overrides /output/append
[--adjust-metadata] overrides /output/adjust_metadata
"""
if version:
from nc2zarr.version import version
print(version)
return 0
from nc2zarr.config import load_config
from nc2zarr.converter import Converter
from nc2zarr.error import ConverterError
try:
config_kwargs = load_config(config_paths=config_paths,
return_kwargs=True,
input_paths=input_paths or None,
input_decode_cf=decode_cf,
input_multi_file=multi_file,
input_concat_dim=concat_dim,
input_sort_by=sort_by,
output_path=output_path,
output_overwrite=overwrite,
output_append=append,
output_adjust_metadata=adjust_metadata,
finalize_only=finalize_only,
verbosity=verbose if verbose else None,
dry_run=dry_run)
Converter(**config_kwargs).run()
except ConverterError as e:
raise click.ClickException(e) from e