def dump_to_tar(data_stream, tar): nonlocal exit_early fps = RateEstimator() for d in data_stream: fps() fname = d.url[5:] if d.data is not None: if verbose: if fps.every(10): print('.', file=stderr, end='', flush=True) if fps.every(100): print(' {}'.format(str(fps)), file=stderr) add_txt_file(tar, fname, d.data, last_modified=d.last_modified) else: print("Failed %s (%s)" % (d.url, str(d.error)), file=stderr) if exit_early: break if verbose: print(' {}'.format(str(fps)), file=stderr)
def cli(input_fname, env, product_names, exclude_product_names, auto_add_lineage, verify_lineage, ignore_lineage, update, eo3, gzip, xz, protocol): # Ensure :// is present in prefix prefix = protocol.rstrip('://') + '://' if prefix.startswith('file'): prefix = prefix + '/' if ignore_lineage: auto_add_lineage = False if eo3: verify_lineage = False auto_add_lineage = False ds_resolve_args = dict(products=product_names, exclude_products=exclude_product_names, fail_on_missing_lineage=not auto_add_lineage, verify_lineage=verify_lineage, skip_lineage=ignore_lineage) doc_transform = prep_eo3 if eo3 else None allowed_changes = {(): allow_any} def mk_uri(name): return prefix + name def report_error(msg): print(msg, file=sys.stderr) def process_file(filename, index, fps, mode=None, n_failed=0, doc_transform=None): for ds, err in from_tar_file(filename, index, mk_uri, doc_transform=doc_transform, mode=mode, **ds_resolve_args): if ds is not None: try: if update: index.datasets.update(ds, allowed_changes) else: index.datasets.add(ds, with_lineage=auto_add_lineage) except Exception as e: n_failed += 1 report_error(str(e)) else: n_failed += 1 report_error(err) fps() if fps.every(10): print('.', end='', flush=True) if fps.every(100): print(' {} F:{:d}'.format(str(fps), n_failed)) return n_failed dc = datacube.Datacube(env=env) if len(input_fname) == 0: input_fname = ('-',) n_failed = 0 fps = RateEstimator() mode = None for filename in input_fname: if filename == '-': if sys.stdin.isatty(): report_error("Requesting to read from stdin but not redirecting input?") sys.exit(1) filename = sys.stdin.buffer mode = tar_mode(is_pipe=True, gzip=gzip, xz=xz) n_failed = process_file(filename, dc.index, fps, mode=mode, n_failed=n_failed, doc_transform=doc_transform) if n_failed > 0: report_error("**WARNING** there were failures: {}".format(n_failed))