def treat_page_and_item(self, unused_page, item):
"""Print out constraint failures
unused_page is always None since use_from_page is False.
See https://doc.wikimedia.org/pywikibot/master/api_ref/pywikibot.html#pywikibot.WikidataBot
"""
item.get(force=True)
typed_item = self.factory.get_typed_item(item.title())
botlogging.output(f"Checking constraints for {typed_item}", toStdout=True)
satisfied = []
not_satisfied = []
for constraint in typed_item.constraints:
if constraint.validate(typed_item):
satisfied.append(constraint)
else:
not_satisfied.append(constraint)
if self.verbose:
self.print_failures(typed_item, not_satisfied)
self.print_successes(typed_item, satisfied)
total = len(typed_item.constraints)
failures = len(not_satisfied)
botlogging.output(
f"Found {failures}/{total} constraint failures", toStdout=True
)
return satisfied, not_satisfied
def _flush(stop: bool = True) -> None:
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from
the throttle log. Called automatically at Python exit.
"""
_logger = 'wiki'
debug('_flush() called', _logger)
def remaining() -> Tuple[int, datetime.timedelta]:
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(
seconds=round(remainingPages * _config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > _config.noisysleep:
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
if _putthread is not threading.current_thread():
while (_putthread.is_alive() and (page_put_queue.qsize() > 0
or page_put_queue_busy.qsize() > 0)):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn(
'There are {} pages remaining in the queue. '
'Estimated time remaining: {}\nReally exit?'.format(
*remaining()),
default=False,
automatic_quit=False):
# delete the put queue
with page_put_queue.mutex:
page_put_queue.all_tasks_done.notify_all()
page_put_queue.queue.clear()
page_put_queue.not_full.notify_all()
break
# only need one drop() call because all throttles use the same global pid
with suppress(IndexError):
list(_sites.values())[0].throttle.drop()
log('Dropped throttle(s).')
def _flush(stop=True):
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from
the throttle log. Called automatically at Python exit.
"""
_logger = "wiki"
debug('_flush() called', _logger)
def remaining():
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(seconds=(remainingPages *
config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > config.noisysleep:
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
while _putthread.isAlive() and page_put_queue.qsize() > 0:
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are {0} pages remaining in the queue. '
'Estimated time remaining: {1}\nReally exit?'
''.format(*remaining()),
default=False,
automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def stopme():
"""Drop this process from the throttle log, after pending threads finish.
Can be called manually if desired, but if not, will be called automatically
at Python exit.
"""
global stopped
_logger = "wiki"
if not stopped:
debug(u"stopme() called", _logger)
def remaining():
remainingPages = page_put_queue.qsize() - 1
# -1 because we added a None element to stop the queue
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
page_put_queue.put((None, [], {}))
stopped = True
if page_put_queue.qsize() > 1:
num, sec = remaining()
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
while (_putthread.isAlive()):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are %i pages remaining in the queue. '
'Estimated time remaining: %s\nReally exit?' %
remaining(),
default=False,
automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def _flush(stop=True):
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from the
throttle log. Called automatically at Python exit.
"""
_logger = "wiki"
debug('_flush() called', _logger)
def remaining():
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > config.noisysleep:
output(color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}', num=num, sec=sec))
while _putthread.isAlive() and page_put_queue.qsize() > 0:
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are {0} pages remaining in the queue. '
'Estimated time remaining: {1}\nReally exit?'
''.format(*remaining()),
default=False, automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def stopme():
"""
Drop this process from the throttle log, after pending threads finish.
Can be called manually if desired, but if not, will be called automatically
at Python exit.
"""
global stopped
_logger = "wiki"
if not stopped:
debug(u"stopme() called", _logger)
def remaining():
remainingPages = page_put_queue.qsize() - 1
# -1 because we added a None element to stop the queue
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
page_put_queue.put((None, [], {}))
stopped = True
if page_put_queue.qsize() > 1:
num, sec = remaining()
output(color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}', num=num, sec=sec))
while(_putthread.isAlive()):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are %i pages remaining in the queue. '
'Estimated time remaining: %s\nReally exit?'
% remaining(), default=False, automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def fixall(self):
if self.sort:
self.fixes = list(sorted(self.fixes, key=lambda f: f.summary))
if self._filters:
self.fixes = [
fix for fix in self.fixes if should_fix(fix, self._filters)
]
for fix in self.fixes:
print(fix.summary)
fixed = 0
for fix in self.fixes:
success = fix.apply(self.user_add_claim)
fixed += int(success)
total = len(self.fixes)
botlogging.output(f"Fixed {fixed}/{total} constraint failures", toStdout=True)
def _win32_extension_command(extension):
"""Get the command from the Win32 registry for an extension."""
fileexts_key = r"Software\Microsoft\Windows\CurrentVersion\Explorer\FileExts"
key_name = fileexts_key + r"\." + extension + r"\OpenWithProgids"
_winreg = winreg # exists for git blame only; do not use
try:
key1 = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_name)
_progID = winreg.EnumValue(key1, 0)[0]
_key2 = _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, "%s\shell\open\command" % _progID)
_cmd = _winreg.QueryValueEx(_key2, None)[0]
# See T102465 for issues relating to using this value.
cmd = _cmd
if cmd.find("%1"):
cmd = cmd[: cmd.find("%1")]
# Remove any trailing characher, which should be a quote or space
# and then remove all whitespace.
return cmd[:-1].strip()
except WindowsError as e:
# Catch any key lookup errors
output('Unable to detect program for file extension "{0}": {1}'.format(extension, e))
def _win32_extension_command(extension):
"""Get the command from the Win32 registry for an extension."""
fileexts_key = \
r'Software\Microsoft\Windows\CurrentVersion\Explorer\FileExts'
key_name = fileexts_key + r'\.' + extension + r'\OpenWithProgids'
try:
key1 = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_name)
_prog_id = winreg.EnumValue(key1, 0)[0]
_key2 = winreg.OpenKey(winreg.HKEY_CLASSES_ROOT,
r'{}\shell\open\command'.format(_prog_id))
_cmd = winreg.QueryValueEx(_key2, None)[0]
# See T102465 for issues relating to using this value.
cmd = _cmd
if cmd.find('%1'):
cmd = cmd[:cmd.find('%1')]
# Remove any trailing character, which should be a quote or space
# and then remove all whitespace.
return cmd[:-1].strip()
except WindowsError as e:
# Catch any key lookup errors
output('Unable to detect program for file extension "{0}": {1!r}'
.format(extension, e))
def treat_page_and_item(self, unused_page, item):
"""Fix items that have constraint failures
unused_page is always None since use_from_page is False.
See https://doc.wikimedia.org/pywikibot/master/api_ref/pywikibot.html#pywikibot.WikidataBot
"""
item.get(force=True)
_, not_satisfied = super().treat_page_and_item(unused_page, item)
typed_item = self.factory.get_typed_item(item.title())
fixes = [
fix for constraint in not_satisfied for fix in constraint.fix(typed_item)
]
fixed = 0
for fix in fixes:
skip_fix = not should_fix(fix, self._filters)
if self._filters and skip_fix:
continue
success = fix.apply(self.user_add_claim)
fixed += success
total = len(not_satisfied)
botlogging.output(f"Fixed {fixed}/{total} constraint failures", toStdout=True)
def get_base_dir(test_directory=None):
r"""Return the directory in which user-specific information is stored.
This is determined in the following order:
1. If the script was called with a -dir: argument, use the directory
provided in this argument.
2. If the user has a PYWIKIBOT2_DIR environment variable, use the value
of it.
3. If user-config is present in current directory, use the current
directory.
4. If user-config is present in pwb.py directory, use that directory
5. Use (and if necessary create) a 'pywikibot' folder under
'Application Data' or 'AppData\Roaming' (Windows) or
'.pywikibot' directory (Unix and similar) under the user's home
directory.
Set PYWIKIBOT2_NO_USER_CONFIG=1 to disable loading user-config.py
@param test_directory: Assume that a user config file exists in this
directory. Used to test whether placing a user config file in this
directory will cause it to be selected as the base directory.
@type test_directory: str or None
@rtype: unicode
"""
def exists(directory):
directory = os.path.abspath(directory)
if directory == test_directory:
return True
else:
return os.path.exists(os.path.join(directory, 'user-config.py'))
if test_directory is not None:
test_directory = os.path.abspath(test_directory)
DIRNAME_WIN = u"Pywikibot"
DIRNAME_WIN_FBCK = u"pywikibot"
DIRNAME_UNIX = u".pywikibot"
base_dir = ""
for arg in sys.argv[1:]:
if arg.startswith(str('-dir:')):
base_dir = arg[5:]
base_dir = os.path.expanduser(base_dir)
break
else:
if ('PYWIKIBOT2_DIR' in os.environ
and exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR']))):
base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR'])
elif exists('.'):
base_dir = os.path.abspath('.')
elif ('PYWIKIBOT2_DIR_PWB' in os.environ
and exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB']))):
base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB'])
else:
base_dir_cand = []
home = os.path.expanduser("~")
if sys.platform == 'win32':
import platform
win_version = int(platform.version()[0])
if win_version == 5:
sub_dir = ["Application Data"]
elif win_version == 6:
sub_dir = ["AppData", "Roaming"]
else:
raise WindowsError(
u'Windows version %s not supported yet.' % win_version)
base_dir_cand.extend([[home] + sub_dir + [DIRNAME_WIN],
[home] + sub_dir + [DIRNAME_WIN_FBCK]])
else:
base_dir_cand.append([home, DIRNAME_UNIX])
for dir in base_dir_cand:
dir = os.path.join(*dir)
if not os.path.isdir(dir):
os.makedirs(dir, mode=private_files_permission)
if exists(dir):
base_dir = dir
break
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
# make sure this path is valid and that it contains user-config file
if not os.path.isdir(base_dir):
raise RuntimeError("Directory '%s' does not exist." % base_dir)
# check if user-config.py is in base_dir
if not exists(base_dir):
exc_text = "No user-config.py found in directory '%s'.\n" % base_dir
if _no_user_config:
if _no_user_config != '2':
output(exc_text)
else:
exc_text += " Please check that user-config.py is stored in the correct location.\n"
exc_text += " Directory where user-config.py is searched is determined as follows:\n\n"
exc_text += " " + get_base_dir.__doc__
raise RuntimeError(exc_text)
return base_dir
def get_base_dir(test_directory: Optional[str] = None) -> str:
r"""Return the directory in which user-specific information is stored.
This is determined in the following order:
1. If the script was called with a -dir: argument, use the directory
provided in this argument.
2. If the user has a PYWIKIBOT_DIR environment variable, use the value
of it.
3. If user-config is present in current directory, use the current
directory.
4. If user-config is present in pwb.py directory, use that directory
5. Use (and if necessary create) a 'pywikibot' folder under
'Application Data' or 'AppData\Roaming' (Windows) or
'.pywikibot' directory (Unix and similar) under the user's home
directory.
Set PYWIKIBOT_NO_USER_CONFIG=1 to disable loading user-config.py
@param test_directory: Assume that a user config file exists in this
directory. Used to test whether placing a user config file in this
directory will cause it to be selected as the base directory.
"""
def exists(directory):
directory = os.path.abspath(directory)
if directory == test_directory:
return True
else:
return os.path.exists(os.path.join(directory, 'user-config.py'))
if test_directory is not None:
test_directory = os.path.abspath(test_directory)
base_dir = ''
for arg in sys.argv[1:]:
if arg.startswith('-dir:'):
base_dir = arg[5:]
base_dir = os.path.expanduser(base_dir)
break
else:
if ('PYWIKIBOT_DIR' in environ
and exists(os.path.abspath(environ['PYWIKIBOT_DIR']))):
base_dir = os.path.abspath(environ['PYWIKIBOT_DIR'])
elif exists('.'):
base_dir = os.path.abspath('.')
elif ('PYWIKIBOT_DIR_PWB' in environ
and exists(os.path.abspath(environ['PYWIKIBOT_DIR_PWB']))):
base_dir = os.path.abspath(environ['PYWIKIBOT_DIR_PWB'])
else:
base_dir_cand = []
home = os.path.expanduser('~')
if OSWIN32:
win_version = int(platform.version().split('.')[0])
if win_version == 5:
sub_dir = ['Application Data']
elif win_version in (6, 10):
sub_dir = ['AppData', 'Roaming']
else:
raise WindowsError('Windows version {} not supported yet.'
.format(win_version))
base_dir_cand.extend([[home] + sub_dir + ['Pywikibot'],
[home] + sub_dir + ['pywikibot']])
else:
base_dir_cand.append([home, '.pywikibot'])
for dir in base_dir_cand:
dir = os.path.join(*dir)
try:
os.makedirs(dir, mode=private_files_permission)
except OSError: # PermissionError or already exists
if exists(dir):
base_dir = dir
break
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
# make sure this path is valid and that it contains user-config file
if not os.path.isdir(base_dir):
raise RuntimeError("Directory '%s' does not exist." % base_dir)
# check if user-config.py is in base_dir
if not exists(base_dir):
exc_text = "No user-config.py found in directory '%s'.\n" % base_dir
if __no_user_config:
if __no_user_config != '2':
output(exc_text)
else:
exc_text += (
' Please check that user-config.py is stored in the correct '
'location.\n'
' Directory where user-config.py is searched is determined '
'as follows:\n\n ') + get_base_dir.__doc__
raise RuntimeError(exc_text)
return base_dir
exc_text += (
' Please check that user-config.py is stored in the correct '
'location.\n'
' Directory where user-config.py is searched is determined '
'as follows:\n\n ') + get_base_dir.__doc__
raise RuntimeError(exc_text)
return base_dir
# Save base_dir for use by other modules
base_dir = get_base_dir()
for arg in sys.argv[1:]:
if arg.startswith('-verbose') or arg == '-v':
output('The base directory is ' + base_dir)
break
family_files = {}
def register_family_file(family_name, file_path):
"""Register a single family class file.
Parameter file_path may be a path or an url.
family.AutoFamily function is used when the url is given.
"""
usernames[family_name] = {}
disambiguation_comment[family_name] = {}
family_files[family_name] = file_path
def output(self):
"""Output the text of the current sequence."""
output(self.format_list())
def print_failures(self, typed_item: BaseType, failed_constraints):
"""Print failed constraints"""
for constraint in failed_constraints:
botlogging.output(f"{constraint} failed for {typed_item}", toStdout=True)
def output(self):
"""Output the text of the current sequence."""
output(self.format_list())
def get_base_dir(test_directory=None):
r"""Return the directory in which user-specific information is stored.
This is determined in the following order:
1. If the script was called with a -dir: argument, use the directory
provided in this argument.
2. If the user has a PYWIKIBOT2_DIR environment variable, use the value
of it.
3. If user-config is present in current directory, use the current
directory.
4. If user-config is present in pwb.py directory, use that directory
5. Use (and if necessary create) a 'pywikibot' folder under
'Application Data' or 'AppData\Roaming' (Windows) or
'.pywikibot' directory (Unix and similar) under the user's home
directory.
Set PYWIKIBOT2_NO_USER_CONFIG=1 to disable loading user-config.py
@param test_directory: Assume that a user config file exists in this
directory. Used to test whether placing a user config file in this
directory will cause it to be selected as the base directory.
@type test_directory: str or None
@rtype: unicode
"""
def exists(directory):
directory = os.path.abspath(directory)
if directory == test_directory:
return True
else:
return os.path.exists(os.path.join(directory, 'user-config.py'))
if test_directory is not None:
test_directory = os.path.abspath(test_directory)
DIRNAME_WIN = u"Pywikibot"
DIRNAME_WIN_FBCK = u"pywikibot"
DIRNAME_UNIX = u".pywikibot"
base_dir = ""
for arg in sys.argv[1:]:
if arg.startswith(str('-dir:')):
base_dir = arg[5:]
base_dir = os.path.expanduser(base_dir)
break
else:
if ('PYWIKIBOT2_DIR' in os.environ and
exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR']))):
base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR'])
elif exists('.'):
base_dir = os.path.abspath('.')
elif ('PYWIKIBOT2_DIR_PWB' in os.environ and
exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB']))):
base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB'])
else:
base_dir_cand = []
home = os.path.expanduser("~")
if OSWIN32:
win_version = int(platform.version().split(".")[0])
if win_version == 5:
sub_dir = ["Application Data"]
elif win_version in (6, 10):
sub_dir = ["AppData", "Roaming"]
else:
raise WindowsError(u'Windows version {0!s} not supported yet.'.format(win_version))
base_dir_cand.extend([[home] + sub_dir + [DIRNAME_WIN],
[home] + sub_dir + [DIRNAME_WIN_FBCK]])
else:
base_dir_cand.append([home, DIRNAME_UNIX])
for dir in base_dir_cand:
dir = os.path.join(*dir)
if not os.path.isdir(dir):
os.makedirs(dir, mode=private_files_permission)
if exists(dir):
base_dir = dir
break
if not os.path.isabs(base_dir):
base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
# make sure this path is valid and that it contains user-config file
if not os.path.isdir(base_dir):
raise RuntimeError("Directory '{0!s}' does not exist.".format(base_dir))
# check if user-config.py is in base_dir
if not exists(base_dir):
exc_text = "No user-config.py found in directory '{0!s}'.\n".format(base_dir)
if __no_user_config:
if __no_user_config != '2':
output(exc_text)
else:
exc_text += " Please check that user-config.py is stored in the correct location.\n"
exc_text += " Directory where user-config.py is searched is determined as follows:\n\n"
exc_text += " " + get_base_dir.__doc__
raise RuntimeError(exc_text)
return base_dir
else:
exc_text += " Please check that user-config.py is stored in the correct location.\n"
exc_text += " Directory where user-config.py is searched is determined as follows:\n\n"
exc_text += " " + get_base_dir.__doc__
raise RuntimeError(exc_text)
return base_dir
_get_base_dir = get_base_dir # for backward compatibility
_base_dir = get_base_dir()
# Save base_dir for use by other modules
base_dir = _base_dir
for arg in sys.argv[1:]:
if arg.startswith(str('-verbose')) or arg == str('-v'):
output('The base directory is {0}'.format(base_dir))
break
family_files = {}
def register_family_file(family_name, file_path):
"""Register a single family class file."""
usernames[family_name] = {}
sysopnames[family_name] = {}
disambiguation_comment[family_name] = {}
family_files[family_name] = file_path
def register_families_folder(folder_path):
"""Register all family class files contained in a directory."""
for file_name in os.listdir(folder_path):
def output(self):
"""Output the text of the current sequence."""
output(self.out)
def print_successes(self, typed_item: BaseType, passed_constraints):
"""Print passed constraints"""
for constraint in passed_constraints:
botlogging.output(f"{constraint} passed for {typed_item}", toStdout=True)
exc_text += " Please check that user-config.py is stored in the correct location.\n"
exc_text += " Directory where user-config.py is searched is determined as follows:\n\n"
exc_text += " " + get_base_dir.__doc__
raise RuntimeError(exc_text)
return base_dir
_get_base_dir = get_base_dir # for backward compatibility
_base_dir = get_base_dir()
# Save base_dir for use by other modules
base_dir = _base_dir
for arg in sys.argv[1:]:
if arg.startswith(str("-verbose")) or arg == str("-v"):
output("The base directory is {0}".format(base_dir))
break
family_files = {}
def register_family_file(family_name, file_path):
"""Register a single family class file."""
usernames[family_name] = {}
sysopnames[family_name] = {}
disambiguation_comment[family_name] = {}
family_files[family_name] = file_path
def register_families_folder(folder_path):
"""Register all family class files contained in a directory."""
for file_name in os.listdir(folder_path):
