def gen_inserts_from_logevent(self, logevent):
title = self.db.Title(logevent["title"])
log_deleted = 0
if "actionhidden" in logevent:
log_deleted |= mwconst.DELETED_ACTION
if "commenthidden" in logevent:
log_deleted |= mwconst.DELETED_COMMENT
if "userhidden" in logevent:
log_deleted |= mwconst.DELETED_USER
if "suppressed" in logevent:
log_deleted |= mwconst.DELETED_RESTRICTED
# Do not use title.dbtitle:
# - Interwiki prefix has to be included due to old log entries from
# times when the current interwiki prefixes were not in place.
# - Section anchor has to be included due to old log entries,
# apparently MediaWiki allowed ``#`` in user names at some point.
log_title = title.format(iwprefix=True,
namespace=False,
sectionname=True)
# Hack for the introduction of a new namespace (if the namespace numbers
# don't match, use logevent["title"] verbatim).
if logevent["ns"] == 0 and title.namespacenumber != 0:
log_title = logevent["title"]
# it's not an interwiki prefix -> capitalize first letter
log_title = log_title[0].upper() + log_title[1:]
db_entry = {
"log_id": logevent["logid"],
"log_type": logevent["type"],
"log_action": logevent["action"],
"log_timestamp": logevent["timestamp"],
# This assumes that anonymous users can't create log events, so all "0" from the API are from deleted users
"log_user": value_or_none(logevent["userid"]),
"log_user_text": logevent["user"],
"log_namespace": logevent["ns"],
"log_title": log_title,
# 'logpage' can be different from 'pageid', e.g. if the page was deleted
# in an old MediaWiki that did not preserve pageid and then restored
"log_page": value_or_none(logevent["logpage"]),
"log_comment": logevent["comment"],
"log_params": logevent["params"],
"log_deleted": log_deleted,
}
yield self.sql["insert", "logging"], db_entry
for tag_name in logevent.get("tags", []):
db_entry = {
"b_log_id": logevent["logid"],
"b_tag_name": tag_name,
}
yield self.sql["insert", "tagged_logevent"], db_entry
def gen_revisions(self, page):
for rev in page["revisions"]:
db_entry = {
"rev_id": rev["revid"],
"rev_page": value_or_none(page.get("pageid")),
"rev_comment": rev["comment"],
"rev_user": rev["userid"],
"rev_user_text": rev["user"],
"rev_timestamp": rev["timestamp"],
"rev_minor_edit": "minor" in rev,
# rev_deleted is set separately with an update query, see below
"rev_len": rev["size"],
"rev_parent_id": rev.get("parentid"),
"rev_sha1": rev["sha1"],
"rev_content_model": rev["contentmodel"], # always available
"rev_content_format": rev.get("contentformat"), # available iff content is available
}
if self.with_content is True:
text_id = next(self.text_id_gen)
db_entry["rev_text_id"] = text_id
yield from self.gen_text(rev, text_id)
yield self.sql["insert", "revision"], db_entry
for tag_name in rev.get("tags", []):
db_entry = {
"b_rev_id": rev["revid"],
"b_tag_name": tag_name,
}
yield self.sql["insert", "tagged_revision"], db_entry
def gen_revisions(self, page):
for rev in page["revisions"]:
db_entry = {
"rev_id": rev["revid"],
"rev_page": value_or_none(page.get("pageid")),
"rev_comment": rev["comment"],
"rev_user": rev["userid"],
"rev_user_text": rev["user"],
"rev_timestamp": rev["timestamp"],
"rev_minor_edit": "minor" in rev,
# rev_deleted is set separately with an update query, see below
"rev_len": rev["size"],
"rev_parent_id": rev.get("parentid"),
"rev_sha1": rev["sha1"],
"rev_content_model": rev["contentmodel"], # always available
"rev_content_format":
rev.get("contentformat"), # available iff content is available
}
if self.with_content is True:
text_id = next(self.text_id_gen)
db_entry["rev_text_id"] = text_id
yield from self.gen_text(rev, text_id)
yield self.sql["insert", "revision"], db_entry
for tag_name in rev.get("tags", []):
db_entry = {
"b_rev_id": rev["revid"],
"b_tag_name": tag_name,
}
yield self.sql["insert", "tagged_revision"], db_entry
def gen_inserts_from_logevent(self, logevent):
title = self.db.Title(logevent["title"])
log_deleted = 0
if "actionhidden" in logevent:
log_deleted |= mwconst.DELETED_ACTION
if "commenthidden" in logevent:
log_deleted |= mwconst.DELETED_COMMENT
if "userhidden" in logevent:
log_deleted |= mwconst.DELETED_USER
if "suppressed" in logevent:
log_deleted |= mwconst.DELETED_RESTRICTED
# Do not use title.dbtitle:
# - Interwiki prefix has to be included due to old log entries from
# times when the current interwiki prefixes were not in place.
# - Section anchor has to be included due to old log entries,
# apparently MediaWiki allowed ``#`` in user names at some point.
log_title = title.format(iwprefix=True, namespace=False, sectionname=True)
# it's not an interwiki prefix -> capitalize first letter
log_title = log_title[0].upper() + log_title[1:]
db_entry = {
"log_id": logevent["logid"],
"log_type": logevent["type"],
"log_action": logevent["action"],
"log_timestamp": logevent["timestamp"],
# This assumes that anonymous users can't create log events, so all "0" from the API are from deleted users
"log_user": value_or_none(logevent["userid"]),
"log_user_text": logevent["user"],
"log_namespace": logevent["ns"],
"log_title": log_title,
# 'logpage' can be different from 'pageid', e.g. if the page was deleted
# in an old MediaWiki that did not preserve pageid and then restored
"log_page": value_or_none(logevent["logpage"]),
"log_comment": logevent["comment"],
"log_params": logevent["params"],
"log_deleted": log_deleted,
}
yield self.sql["insert", "logging"], db_entry
for tag_name in logevent.get("tags", []):
db_entry = {
"b_log_id": logevent["logid"],
"b_tag_name": tag_name,
}
yield self.sql["insert", "tagged_logevent"], db_entry
def gen_insert(self):
# Tags from MW extension appear on first use, without any log event,
# so we fetch them the same way as namespaces.
for tag in self.api.site.tags:
db_entry = {
"tag_name": tag["name"],
"tag_displayname": tag["displayname"],
"tag_description": value_or_none(tag["description"]),
"tag_defined": "defined" in tag,
"tag_active": "active" in tag,
"tag_source": tag["source"],
}
yield self.sql["insert", "tag"], db_entry
def gen_insert(self):
# Tags from MW extension appear on first use, without any log event,
# so we fetch them the same way as namespaces.
for tag in self.api.site.tags:
db_entry = {
"tag_name": tag["name"],
"tag_displayname": tag["displayname"],
"tag_description": value_or_none(tag["description"]),
"tag_defined": "defined" in tag,
"tag_active": "active" in tag,
"tag_source": tag["source"],
}
yield self.sql["insert", "tag"], db_entry
def gen_inserts_from_rc(self, rc):
title = self.db.Title(rc["title"])
rc_deleted = 0
if "sha1hidden" in rc:
rc_deleted |= mwconst.DELETED_TEXT
if "actionhidden" in rc:
rc_deleted |= mwconst.DELETED_ACTION
if "commenthidden" in rc:
rc_deleted |= mwconst.DELETED_COMMENT
# FIXME: either this or make the column nullable or require the "viewsuppressed" right for syncing
rc.setdefault("comment", "")
if "userhidden" in rc:
rc_deleted |= mwconst.DELETED_USER
# FIXME: either this or make the column nullable or require the "viewsuppressed" right for syncing
rc.setdefault("user", "")
if "suppressed" in rc:
rc_deleted |= mwconst.DELETED_RESTRICTED
db_entry = {
"rc_id": rc["rcid"],
"rc_timestamp": rc["timestamp"],
"rc_user": rc.get("userid"), # may be hidden due to rc_deleted
"rc_user_text": rc["user"], # may be hidden due to rc_deleted
"rc_namespace": rc["ns"],
"rc_title": title.dbtitle(rc["ns"]),
"rc_comment": rc["comment"], # may be hidden due to rc_deleted
"rc_minor": "minor" in rc,
"rc_bot": "bot" in rc,
"rc_new": "new" in rc,
"rc_cur_id": value_or_none(rc["pageid"]),
"rc_this_oldid": value_or_none(rc["revid"]),
"rc_last_oldid": value_or_none(rc["old_revid"]),
"rc_type": rc["type"],
"rc_patrolled": "patrolled" in rc,
"rc_old_len": rc["oldlen"],
"rc_new_len": rc["newlen"],
"rc_deleted": rc_deleted,
"rc_logid": rc.get("logid"),
"rc_log_type": rc.get("logtype"),
"rc_log_action": rc.get("logaction"),
"rc_params": rc.get("logparams"),
}
yield self.sql["insert", "recentchanges"], db_entry
for tag_name in rc.get("tags", []):
db_entry = {
"b_rc_id": rc["rcid"],
"b_tag_name": tag_name,
}
yield self.sql["insert", "tagged_recentchange"], db_entry
# check logevents and and update rc_deleted of the past changes,
# including the DELETED_TEXT value (which is a MW incompatibility)
if rc.get("logtype") == "delete":
if rc.get("logaction") == "revision":
for revid in rc["logparams"]["ids"]:
yield self.sql["update", "rc_deleted-b_revid"], {
"b_rev_id": revid,
"rc_deleted": rc["logparams"]["new"]["bitmask"]
}
elif rc.get("logaction") == "event":
for logid in rc["logparams"]["ids"]:
yield self.sql["update", "rc_deleted-b_logid"], {
"b_log_id": logid,
"rc_deleted": rc["logparams"]["new"]["bitmask"]
}
def gen_inserts_from_rc(self, rc):
title = self.db.Title(rc["title"])
rc_deleted = 0
if "sha1hidden" in rc:
rc_deleted |= mwconst.DELETED_TEXT
if "actionhidden" in rc:
rc_deleted |= mwconst.DELETED_ACTION
if "commenthidden" in rc:
rc_deleted |= mwconst.DELETED_COMMENT
# FIXME: either this or make the column nullable or require the "viewsuppressed" right for syncing
rc.setdefault("comment", "")
if "userhidden" in rc:
rc_deleted |= mwconst.DELETED_USER
# FIXME: either this or make the column nullable or require the "viewsuppressed" right for syncing
rc.setdefault("user", "")
if "suppressed" in rc:
rc_deleted |= mwconst.DELETED_RESTRICTED
db_entry = {
"rc_id": rc["rcid"],
"rc_timestamp": rc["timestamp"],
"rc_user": rc.get("userid"), # may be hidden due to rc_deleted
"rc_user_text": rc["user"], # may be hidden due to rc_deleted
"rc_namespace": rc["ns"],
"rc_title": title.dbtitle(rc["ns"]),
"rc_comment": rc["comment"], # may be hidden due to rc_deleted
"rc_minor": "minor" in rc,
"rc_bot": "bot" in rc,
"rc_new": "new" in rc,
"rc_cur_id": value_or_none(rc["pageid"]),
"rc_this_oldid": value_or_none(rc["revid"]),
"rc_last_oldid": value_or_none(rc["old_revid"]),
"rc_type": rc["type"],
"rc_patrolled": "patrolled" in rc,
"rc_old_len": rc["oldlen"],
"rc_new_len": rc["newlen"],
"rc_deleted": rc_deleted,
"rc_logid": rc.get("logid"),
"rc_log_type": rc.get("logtype"),
"rc_log_action": rc.get("logaction"),
"rc_params": rc.get("logparams"),
}
yield self.sql["insert", "recentchanges"], db_entry
for tag_name in rc.get("tags", []):
db_entry = {
"b_rc_id": rc["rcid"],
"b_tag_name": tag_name,
}
yield self.sql["insert", "tagged_recentchange"], db_entry
# check logevents and and update rc_deleted of the past changes,
# including the DELETED_TEXT value (which is a MW incompatibility)
if rc.get("logtype") == "delete":
if rc.get("logaction") == "revision":
for revid in rc["logparams"]["ids"]:
yield self.sql["update", "rc_deleted-b_revid"], {"b_rev_id": revid, "rc_deleted": rc["logparams"]["new"]["bitmask"]}
elif rc.get("logaction") == "event":
for logid in rc["logparams"]["ids"]:
yield self.sql["update", "rc_deleted-b_logid"], {"b_log_id": logid, "rc_deleted": rc["logparams"]["new"]["bitmask"]}