def __init__(self, dataset: Dataset):
self.dataset = dataset
self.path = settings.DATASET_PATH.joinpath(dataset.name)
self.log = get_logger(dataset.name)
self.cache = Cache(engine, MetaData(bind=engine), dataset)
self._statements: Dict[str, Statement] = {}
self.http = requests.Session()
self.http.headers.update(settings.HEADERS)
def setup(log_level=None):
"""Configure the framework."""
configure_logging(level=log_level, extra_processors=[store_log_event])
log = get_logger(__name__)
log.debug(
"OpenSanctions starting",
database_uri=settings.DATABASE_URI,
data_path=str(settings.DATA_PATH),
datasets=Dataset.names(),
)
create_db()
from zavod.logs import get_logger
from nomenklatura.cache import Cache
from nomenklatura.util import is_qid
from nomenklatura.enrich.wikidata import WikidataEnricher
from nomenklatura.judgement import Judgement
from opensanctions.core.db import engine, metadata, engine_read
from opensanctions.core.statements import entities_datasets
from opensanctions.core.resolver import get_resolver
from opensanctions.core.dataset import Dataset
log = get_logger(__name__)
def get_wikidata_enricher() -> WikidataEnricher:
wikidata = Dataset.require("wikidata")
cache = Cache(engine, metadata, wikidata)
wd: WikidataEnricher = wikidata.get_enricher(cache)
return wd
def audit_resolver():
wd = get_wikidata_enricher()
resolver = get_resolver()
log.info("Loading all entity IDs...")
with engine_read() as conn:
entities = list(entities_datasets(conn))
entity_ids = set([e for e, _ in entities])
log.info("Loaded %d entity IDs..." % len(entity_ids))