from opensanctions.core import Dataset, Context, setup
from opensanctions.exporters.statements import export_statements_path
from opensanctions.exporters.statements import import_statements_path
from opensanctions.core.audit import audit_resolver
from opensanctions.core.loader import Database
from opensanctions.core.resolver import AUTO_USER, export_pairs, get_resolver
from opensanctions.core.xref import blocking_xref
from opensanctions.core.statements import max_last_seen
from opensanctions.core.statements import resolve_all_canonical, resolve_canonical
from opensanctions.core.analytics import build_analytics
from opensanctions.core.db import engine_tx
from opensanctions.processing import run_enrich, run_pipeline
from opensanctions.util import write_json
log = get_logger(__name__)
datasets = click.Choice(Dataset.names())
@click.group(help="OpenSanctions ETL toolkit")
@click.option("-v", "--verbose", is_flag=True, default=False)
@click.option("-q", "--quiet", is_flag=True, default=False)
def cli(verbose=False, quiet=False):
level = logging.INFO
if quiet:
level = logging.WARNING
if verbose:
level = logging.DEBUG
setup(log_level=level)
@cli.command("crawl", help="Crawl entities into the given dataset")
@click.group(help="OpenSanctions ETL toolkit")
@click.option("-v", "--verbose", is_flag=True, default=False)
@click.option("-q", "--quiet", is_flag=True, default=False)
def cli(verbose=False, quiet=False):
level = logging.INFO
if quiet:
level = logging.ERROR
if verbose:
level = logging.DEBUG
setup(log_level=level)
@cli.command("dump", help="Export the entities from a dataset")
@click.argument("dataset",
default=Dataset.ALL,
type=click.Choice(Dataset.names()))
@click.option("-o", "--outfile", type=click.File("w"), default="-")
def dump_dataset(dataset, outfile):
dataset = Dataset.get(dataset)
context = Context(dataset)
context.normalize()
for entity in dataset.store:
write_object(outfile, entity)
@cli.command("crawl", help="Crawl entities into the given dataset")
@click.argument("dataset",
default=Dataset.ALL,
type=click.Choice(Dataset.names()))
def crawl(dataset):
dataset = Dataset.get(dataset)
class IndexResponse(BaseModel):
datasets: List[str] = Dataset.names()
model: ModelToDict
terms: int = Field(..., example=23)
tokens: int = Field(..., example=42)