def check_links(entity, catalog, wikidata_cache=None):
catalog_terms = _get_vocabulary(catalog)
# Target links
target = gather_target_links(entity, catalog)
# Early stop in case of no target links
if target is None:
return None, None, None
to_deprecate = defaultdict(set)
to_add = defaultdict(set)
if wikidata_cache is None:
wikidata = {}
# Wikidata links
gather_identifiers(entity, catalog, catalog_terms['pid'], wikidata)
url_pids, ext_id_pids_to_urls = gather_relevant_pids()
gather_wikidata_links(wikidata, url_pids, ext_id_pids_to_urls)
else:
wikidata = wikidata_cache
# Check
_assess('links', wikidata, target, to_deprecate, to_add)
# Separate external IDs from URLs
ext_ids_to_add, urls_to_add = extract_ids_from_urls(
to_add, ext_id_pids_to_urls)
LOGGER.info(
'Validation completed. %d %s IDs to be deprecated, %d external IDs to be added, %d URL statements to be added',
len(to_deprecate), catalog, len(ext_ids_to_add), len(urls_to_add))
return to_deprecate, ext_ids_to_add, urls_to_add, wikidata
def links(catalog: str,
entity: str,
wd_cache=None) -> Tuple[DefaultDict, List, List, Dict]:
"""Validate identifiers against available links.
Also generate statements based on additional links
found in the given catalog.
They can be used to enrich Wikidata items.
**How it works:**
1. gather links from the given catalog
2. gather links from relevant Wikidata items
3. look for shared links between pairs of Wikidata and catalog items:
- when the pair does not share any link,
the catalog identifier should be marked with a deprecated rank
- when the catalog item has more links than the Wikidata one,
they should be added to the latter
4. try to extract third-party identifiers from extra links
:param catalog: ``{'discogs', 'imdb', 'musicbrainz'}``.
A supported catalog
:param entity: ``{'actor', 'band', 'director', 'musician', 'producer',
'writer', 'audiovisual_work', 'musical_work'}``.
A supported entity
:param wd_cache: (optional) a ``dict`` of links gathered from Wikidata
in a previous run
:return: 4 objects
1. ``dict`` of identifiers that should be deprecated
2. ``list`` of third-party identifiers that should be added
3. ``list`` of URLs that should be added
4. ``dict`` of links gathered from Wikidata
"""
# Target catalog side first:
# enable early return in case of no target links
target_links = data_gathering.gather_target_links(entity, catalog)
if target_links is None:
return None, None, None, None
to_be_deprecated, to_be_added = defaultdict(set), defaultdict(set)
# Wikidata side
url_pids, ext_id_pids_to_urls = data_gathering.gather_relevant_pids()
if wd_cache is None:
wd_links = {}
data_gathering.gather_target_ids(
entity,
catalog,
target_database.get_catalog_pid(catalog, entity),
wd_links,
)
data_gathering.gather_wikidata_links(wd_links, url_pids,
ext_id_pids_to_urls)
else:
wd_links = wd_cache
# Validation
_validate(keys.LINKS, wd_links, target_links, to_be_deprecated,
to_be_added)
# Separate external IDs from URLs
ext_ids_to_be_added, urls_to_be_added = data_gathering.extract_ids_from_urls(
to_be_added, ext_id_pids_to_urls)
LOGGER.info(
'Validation completed. Target: %s %s. '
'IDs to be deprecated: %d. '
'Third-party IDs to be added: %d. '
'URL statements to be added: %d',
catalog,
entity,
len(to_be_deprecated),
len(ext_ids_to_be_added),
len(urls_to_be_added),
)
return to_be_deprecated, ext_ids_to_be_added, urls_to_be_added, wd_links
def build_wikidata(goal: str, catalog: str, entity: str,
dir_io: str) -> JsonReader:
"""Build a Wikidata dataset for training or classification purposes:
workflow step 1.
Data is gathered from the
`SPARQL endpoint <https://query.wikidata.org/>`_ and the
`Web API <https://www.wikidata.org/w/api.php>`_.
**How it works:**
1. gather relevant Wikidata items that *hold* (for *training*)
or *lack* (for *classification*) identifiers of the given catalog
2. gather relevant item data
3. dump the dataset to a gzipped `JSON Lines <http://jsonlines.org/>`_ file
4. read the dataset into a generator of :class:`pandas.DataFrame` chunks
for memory-efficient processing
:param goal: ``{'training', 'classification'}``.
Whether to build a dataset for training or classification
:param catalog: ``{'discogs', 'imdb', 'musicbrainz'}``.
A supported catalog
:param entity: ``{'actor', 'band', 'director', 'musician', 'producer',
'writer', 'audiovisual_work', 'musical_work'}``.
A supported entity
:param dir_io: input/output directory where working files
will be read/written
:return: the generator yielding :class:`pandas.DataFrame` chunks
"""
qids_and_tids, wd_io_path = _handle_goal(goal, catalog, entity, dir_io)
catalog_pid = target_database.get_catalog_pid(catalog, entity)
if not os.path.isfile(wd_io_path):
LOGGER.info(
"Building Wikidata %s set for %s %s, output file '%s' ...",
goal,
catalog,
entity,
wd_io_path,
)
# Make working folders
os.makedirs(os.path.dirname(wd_io_path), exist_ok=True)
# 1. Gather Wikidata items
if goal == 'training':
# WITH target IDs
data_gathering.gather_target_ids(entity, catalog, catalog_pid,
qids_and_tids)
qids = qids_and_tids.keys()
elif goal == 'classification':
# WITHOUT target IDs
qids = data_gathering.gather_qids(entity, catalog, catalog_pid)
# 2. Collect relevant data, and 3. dump to gzipped JSON Lines
url_pids, ext_id_pids_to_urls = data_gathering.gather_relevant_pids()
with gzip.open(wd_io_path, 'wt') as wd_io:
api_requests.get_data_for_linker(
catalog,
entity,
qids,
url_pids,
ext_id_pids_to_urls,
qids_and_tids,
wd_io,
)
# Cached dataset, for development purposes
else:
LOGGER.info("Will reuse existing Wikidata %s set: '%s'", goal,
wd_io_path)
if goal == 'training':
_reconstruct_qids_and_tids(wd_io_path, qids_and_tids)
LOGGER.info('Wikidata %s set built', goal)
return pd.read_json(wd_io_path, lines=True, chunksize=1000)
def links(
catalog: str,
entity: str,
url_blacklist=False,
wd_cache=None
) -> Optional[Tuple[defaultdict, list, list, list, list, list, dict]]:
"""Validate identifiers against available links.
Also generate statements based on additional links
found in the target catalog.
They can be used to enrich Wikidata items.
**How it works:**
1. gather links from the target catalog
2. gather links from relevant Wikidata items
3. look for shared links between pairs of Wikidata and catalog items:
- when the pair does not share any link,
the catalog identifier should be marked with a deprecated rank
- when the catalog item has more links than the Wikidata one,
they should be added to the latter
4. try to extract third-party identifiers from extra links
:param catalog: ``{'discogs', 'imdb', 'musicbrainz'}``.
A supported catalog
:param entity: ``{'actor', 'band', 'director', 'musician', 'producer',
'writer', 'audiovisual_work', 'musical_work'}``.
A supported entity
:param url_blacklist: (optional) whether to apply a blacklist
of URL domains. Default: ``False``
:param wd_cache: (optional) a ``dict`` of links gathered from Wikidata
in a previous run. Default: ``None``
:return: 7 objects
1. ``dict`` of identifiers that should be deprecated
2. ``list`` of third-party identifiers that should be added
3. ``list`` of URLs that should be added
4. ``list`` of third-party identifiers that should be referenced
5. ``list`` of URLs that should be referenced
6. ``list`` of URLs found in Wikidata but not in the target catalog
7. ``dict`` of links gathered from Wikidata
or ``None`` if the target catalog has no links.
"""
# Target catalog side first:
# enable early return in case of no target links
target_links = data_gathering.gather_target_links(entity, catalog)
if target_links is None:
return None
deprecate, add = defaultdict(set), defaultdict(set)
reference, wd_only = defaultdict(set), defaultdict(set)
# Wikidata side
url_pids, ext_id_pids_to_urls = data_gathering.gather_relevant_pids()
if wd_cache is None:
wd_links = {}
data_gathering.gather_target_ids(
entity,
catalog,
target_database.get_catalog_pid(catalog, entity),
wd_links,
)
data_gathering.gather_wikidata_links(wd_links, url_pids,
ext_id_pids_to_urls)
else:
wd_links = wd_cache
# Validation
_validate(keys.LINKS, wd_links, target_links, deprecate, add, reference,
wd_only)
# URLs to be added:
# 1. Separate external IDs from URLs
add_ext_ids, add_urls = data_gathering.extract_ids_from_urls(
add, ext_id_pids_to_urls)
# 2. Apply URL blacklist
if url_blacklist:
add_urls = _apply_url_blacklist(add_urls)
# URLs to be referenced: separate external IDs from URLs
ref_ext_ids, ref_urls = data_gathering.extract_ids_from_urls(
reference, ext_id_pids_to_urls)
# Wikidata-only URLs: convert into a list of statements
# with complete Wikidata item URLs
wd_only_urls = []
for (qid, tid), urls in wd_only.items():
for url in urls:
wd_only_urls.append((tid, url, QID_PREFIX + qid))
LOGGER.info(
'Validation completed. Target: %s %s. '
'IDs to be deprecated: %d. '
'Third-party IDs to be added: %d. '
'URL statements to be added: %d. '
'Third-party IDs to be referenced: %d. '
'URL statements to be referenced: %d. '
'URL in Wikidata but not in the target: %d',
catalog,
entity,
len(deprecate),
len(add_ext_ids),
len(add_urls),
len(ref_ext_ids),
len(ref_urls),
len(wd_only_urls),
)
return (
deprecate,
add_ext_ids,
add_urls,
ref_ext_ids,
ref_urls,
wd_only_urls,
wd_links,
)