from opensanctions.core import Dataset, Context, setup from opensanctions.exporters.statements import export_statements_path from opensanctions.exporters.statements import import_statements_path from opensanctions.core.audit import audit_resolver from opensanctions.core.loader import Database from opensanctions.core.resolver import AUTO_USER, export_pairs, get_resolver from opensanctions.core.xref import blocking_xref from opensanctions.core.statements import max_last_seen from opensanctions.core.statements import resolve_all_canonical, resolve_canonical from opensanctions.core.analytics import build_analytics from opensanctions.core.db import engine_tx from opensanctions.processing import run_enrich, run_pipeline from opensanctions.util import write_json log = get_logger(__name__) datasets = click.Choice(Dataset.names()) @click.group(help="OpenSanctions ETL toolkit") @click.option("-v", "--verbose", is_flag=True, default=False) @click.option("-q", "--quiet", is_flag=True, default=False) def cli(verbose=False, quiet=False): level = logging.INFO if quiet: level = logging.WARNING if verbose: level = logging.DEBUG setup(log_level=level) @cli.command("crawl", help="Crawl entities into the given dataset")
@click.group(help="OpenSanctions ETL toolkit") @click.option("-v", "--verbose", is_flag=True, default=False) @click.option("-q", "--quiet", is_flag=True, default=False) def cli(verbose=False, quiet=False): level = logging.INFO if quiet: level = logging.ERROR if verbose: level = logging.DEBUG setup(log_level=level) @cli.command("dump", help="Export the entities from a dataset") @click.argument("dataset", default=Dataset.ALL, type=click.Choice(Dataset.names())) @click.option("-o", "--outfile", type=click.File("w"), default="-") def dump_dataset(dataset, outfile): dataset = Dataset.get(dataset) context = Context(dataset) context.normalize() for entity in dataset.store: write_object(outfile, entity) @cli.command("crawl", help="Crawl entities into the given dataset") @click.argument("dataset", default=Dataset.ALL, type=click.Choice(Dataset.names())) def crawl(dataset): dataset = Dataset.get(dataset)
class IndexResponse(BaseModel): datasets: List[str] = Dataset.names() model: ModelToDict terms: int = Field(..., example=23) tokens: int = Field(..., example=42)