def dump_to_tar(data_stream, tar):
nonlocal exit_early
fps = RateEstimator()
for d in data_stream:
fps()
fname = d.url[5:]
if d.data is not None:
if verbose:
if fps.every(10):
print('.', file=stderr, end='', flush=True)
if fps.every(100):
print(' {}'.format(str(fps)), file=stderr)
add_txt_file(tar, fname, d.data, last_modified=d.last_modified)
else:
print("Failed %s (%s)" % (d.url, str(d.error)),
file=stderr)
if exit_early:
break
if verbose:
print(' {}'.format(str(fps)), file=stderr)
def cli(input_fname,
env,
product_names,
exclude_product_names,
auto_add_lineage,
verify_lineage,
ignore_lineage,
update,
eo3,
gzip,
xz,
protocol):
# Ensure :// is present in prefix
prefix = protocol.rstrip('://') + '://'
if prefix.startswith('file'):
prefix = prefix + '/'
if ignore_lineage:
auto_add_lineage = False
if eo3:
verify_lineage = False
auto_add_lineage = False
ds_resolve_args = dict(products=product_names,
exclude_products=exclude_product_names,
fail_on_missing_lineage=not auto_add_lineage,
verify_lineage=verify_lineage,
skip_lineage=ignore_lineage)
doc_transform = prep_eo3 if eo3 else None
allowed_changes = {(): allow_any}
def mk_uri(name):
return prefix + name
def report_error(msg):
print(msg, file=sys.stderr)
def process_file(filename, index, fps, mode=None, n_failed=0, doc_transform=None):
for ds, err in from_tar_file(filename, index, mk_uri, doc_transform=doc_transform, mode=mode, **ds_resolve_args):
if ds is not None:
try:
if update:
index.datasets.update(ds, allowed_changes)
else:
index.datasets.add(ds, with_lineage=auto_add_lineage)
except Exception as e:
n_failed += 1
report_error(str(e))
else:
n_failed += 1
report_error(err)
fps()
if fps.every(10):
print('.', end='', flush=True)
if fps.every(100):
print(' {} F:{:d}'.format(str(fps), n_failed))
return n_failed
dc = datacube.Datacube(env=env)
if len(input_fname) == 0:
input_fname = ('-',)
n_failed = 0
fps = RateEstimator()
mode = None
for filename in input_fname:
if filename == '-':
if sys.stdin.isatty():
report_error("Requesting to read from stdin but not redirecting input?")
sys.exit(1)
filename = sys.stdin.buffer
mode = tar_mode(is_pipe=True, gzip=gzip, xz=xz)
n_failed = process_file(filename, dc.index, fps, mode=mode, n_failed=n_failed, doc_transform=doc_transform)
if n_failed > 0:
report_error("**WARNING** there were failures: {}".format(n_failed))