def fetch_data():
url = self.API_PHP_BASE_PATTERN.format(lang=source_lang)
with prometheus.wiki_request_duration("wiki_api", "get_title"):
resp = self.session.get(
url=url,
params={
"action": "query",
"prop": "langlinks",
"titles": title,
"lllang": dest_lang,
"formatversion": 2,
"format": "json",
},
timeout=self.TIMEOUT,
)
resp.raise_for_status()
resp_data = resp.json()
resp_pages = resp_data.get("query", {}).get("pages", [])
if len(resp_pages) > 0:
if len(resp_pages) > 1:
logger.warning(
"Got multiple pages in wikipedia langlinks response: %s",
resp_data)
lang_links = resp_pages[0].get("langlinks", [])
if len(lang_links) > 0:
return lang_links[0].get("title")
return None
def get_wiki_info(cls, wikidata_id, wiki_index):
cls.init_wiki_es()
try:
with prometheus.wiki_request_duration("wiki_es", "get_wiki_info"):
resp = cls._wiki_es.search(
index=wiki_index,
body={
"filter": {
"term": {
"wikibase_item": wikidata_id
}
}
}
).get('hits', {}).get('hits', [])
except ConnectionError:
logger.warning("Wiki ES not available: connection exception raised", exc_info=True)
return None
except NotFoundError:
logger.warning("Wiki ES didn't find wikidata_id '%s' in wiki_index '%s'",
wikidata_id, wiki_index, exc_info=True)
return None
except ElasticsearchException as e:
logger.warning("Wiki ES failure: unknown elastic error", exc_info=True)
return None
if len(resp) == 0:
return None
wiki = resp[0]['_source']
return wiki
def fetch_data():
try:
with prometheus.wiki_request_duration("wiki_es", "get_wiki_info"):
resp = (
self.es.search(
index=es_index,
body={
"query": {
"bool": {"filter": {"term": {"wikibase_item": wikidata_id}}}
}
},
)
.get("hits", {})
.get("hits", [])
)
except ConnectionError:
logger.warning("Wiki ES not available: connection exception raised", exc_info=True)
return None
except NotFoundError:
logger.warning(
"Wiki ES didn't find wikidata_id '%s' in wiki_index '%s'",
wikidata_id,
es_index,
exc_info=True,
)
return None
except ElasticsearchException:
logger.warning("Wiki ES failure: unknown elastic error", exc_info=True)
return None
if not resp:
return None
return resp[0].get("_source")
def get_wiki_info(cls, wikidata_id, lang, wiki_index):
cls.init_wiki_es()
try:
with prometheus.wiki_request_duration("wiki_es", "get_wiki_info"):
resp = cls._wiki_es.search(index=wiki_index,
body={
"filter": {
"term": {
"wikibase_item":
wikidata_id
}
}
}).get('hits', {}).get('hits', [])
except ConnectionError:
logger.warning(
"Wiki ES not available: connection exception raised",
exc_info=True)
return None
if len(resp) == 0:
return None
wiki = resp[0]['_source']
return wiki
def fetch_data():
url = f"{self.API_V1_BASE_PATTERN.format(lang=lang)}/page/summary/{title}"
with prometheus.wiki_request_duration("wiki_api", "get_summary"):
resp = self.session.get(url=url,
params={"redirect": True},
timeout=self.TIMEOUT)
resp.raise_for_status()
return resp.json()
def get_summary(self, title, lang):
url = "{base_url}/page/summary/{title}".format(
base_url=self.API_V1_BASE_PATTERN.format(lang=lang), title=title)
with prometheus.wiki_request_duration("wiki_api", "get_summary"):
resp = self.session.get(url=url,
params={"redirect": True},
timeout=self.timeout)
resp.raise_for_status()
return resp.json()