def treat_page_and_item(self, unused_page, item): """Print out constraint failures unused_page is always None since use_from_page is False. See https://doc.wikimedia.org/pywikibot/master/api_ref/pywikibot.html#pywikibot.WikidataBot """ item.get(force=True) typed_item = self.factory.get_typed_item(item.title()) botlogging.output(f"Checking constraints for {typed_item}", toStdout=True) satisfied = [] not_satisfied = [] for constraint in typed_item.constraints: if constraint.validate(typed_item): satisfied.append(constraint) else: not_satisfied.append(constraint) if self.verbose: self.print_failures(typed_item, not_satisfied) self.print_successes(typed_item, satisfied) total = len(typed_item.constraints) failures = len(not_satisfied) botlogging.output( f"Found {failures}/{total} constraint failures", toStdout=True ) return satisfied, not_satisfied
def _flush(stop: bool = True) -> None: """ Drop this process from the throttle log, after pending threads finish. Wait for the page-putter to flush its queue. Also drop this process from the throttle log. Called automatically at Python exit. """ _logger = 'wiki' debug('_flush() called', _logger) def remaining() -> Tuple[int, datetime.timedelta]: remainingPages = page_put_queue.qsize() if stop: # -1 because we added a None element to stop the queue remainingPages -= 1 remainingSeconds = datetime.timedelta( seconds=round(remainingPages * _config.put_throttle)) return (remainingPages, remainingSeconds) if stop: # None task element leaves async_manager page_put_queue.put((None, [], {})) num, sec = remaining() if num > 0 and sec.total_seconds() > _config.noisysleep: output( color_format( '{lightblue}Waiting for {num} pages to be put. ' 'Estimated time remaining: {sec}{default}', num=num, sec=sec)) if _putthread is not threading.current_thread(): while (_putthread.is_alive() and (page_put_queue.qsize() > 0 or page_put_queue_busy.qsize() > 0)): try: _putthread.join(1) except KeyboardInterrupt: if input_yn( 'There are {} pages remaining in the queue. ' 'Estimated time remaining: {}\nReally exit?'.format( *remaining()), default=False, automatic_quit=False): # delete the put queue with page_put_queue.mutex: page_put_queue.all_tasks_done.notify_all() page_put_queue.queue.clear() page_put_queue.not_full.notify_all() break # only need one drop() call because all throttles use the same global pid with suppress(IndexError): list(_sites.values())[0].throttle.drop() log('Dropped throttle(s).')
def _flush(stop=True): """ Drop this process from the throttle log, after pending threads finish. Wait for the page-putter to flush its queue. Also drop this process from the throttle log. Called automatically at Python exit. """ _logger = "wiki" debug('_flush() called', _logger) def remaining(): remainingPages = page_put_queue.qsize() if stop: # -1 because we added a None element to stop the queue remainingPages -= 1 remainingSeconds = datetime.timedelta(seconds=(remainingPages * config.put_throttle)) return (remainingPages, remainingSeconds) if stop: # None task element leaves async_manager page_put_queue.put((None, [], {})) num, sec = remaining() if num > 0 and sec.total_seconds() > config.noisysleep: output( color_format( '{lightblue}Waiting for {num} pages to be put. ' 'Estimated time remaining: {sec}{default}', num=num, sec=sec)) while _putthread.isAlive() and page_put_queue.qsize() > 0: try: _putthread.join(1) except KeyboardInterrupt: if input_yn('There are {0} pages remaining in the queue. ' 'Estimated time remaining: {1}\nReally exit?' ''.format(*remaining()), default=False, automatic_quit=False): return # only need one drop() call because all throttles use the same global pid try: list(_sites.values())[0].throttle.drop() log(u"Dropped throttle(s).") except IndexError: pass
def stopme(): """Drop this process from the throttle log, after pending threads finish. Can be called manually if desired, but if not, will be called automatically at Python exit. """ global stopped _logger = "wiki" if not stopped: debug(u"stopme() called", _logger) def remaining(): remainingPages = page_put_queue.qsize() - 1 # -1 because we added a None element to stop the queue remainingSeconds = datetime.timedelta( seconds=(remainingPages * config.put_throttle)) return (remainingPages, remainingSeconds) page_put_queue.put((None, [], {})) stopped = True if page_put_queue.qsize() > 1: num, sec = remaining() output( color_format( '{lightblue}Waiting for {num} pages to be put. ' 'Estimated time remaining: {sec}{default}', num=num, sec=sec)) while (_putthread.isAlive()): try: _putthread.join(1) except KeyboardInterrupt: if input_yn('There are %i pages remaining in the queue. ' 'Estimated time remaining: %s\nReally exit?' % remaining(), default=False, automatic_quit=False): return # only need one drop() call because all throttles use the same global pid try: list(_sites.values())[0].throttle.drop() log(u"Dropped throttle(s).") except IndexError: pass
def _flush(stop=True): """ Drop this process from the throttle log, after pending threads finish. Wait for the page-putter to flush its queue. Also drop this process from the throttle log. Called automatically at Python exit. """ _logger = "wiki" debug('_flush() called', _logger) def remaining(): remainingPages = page_put_queue.qsize() if stop: # -1 because we added a None element to stop the queue remainingPages -= 1 remainingSeconds = datetime.timedelta( seconds=(remainingPages * config.put_throttle)) return (remainingPages, remainingSeconds) if stop: # None task element leaves async_manager page_put_queue.put((None, [], {})) num, sec = remaining() if num > 0 and sec.total_seconds() > config.noisysleep: output(color_format( '{lightblue}Waiting for {num} pages to be put. ' 'Estimated time remaining: {sec}{default}', num=num, sec=sec)) while _putthread.isAlive() and page_put_queue.qsize() > 0: try: _putthread.join(1) except KeyboardInterrupt: if input_yn('There are {0} pages remaining in the queue. ' 'Estimated time remaining: {1}\nReally exit?' ''.format(*remaining()), default=False, automatic_quit=False): return # only need one drop() call because all throttles use the same global pid try: list(_sites.values())[0].throttle.drop() log(u"Dropped throttle(s).") except IndexError: pass
def stopme(): """ Drop this process from the throttle log, after pending threads finish. Can be called manually if desired, but if not, will be called automatically at Python exit. """ global stopped _logger = "wiki" if not stopped: debug(u"stopme() called", _logger) def remaining(): remainingPages = page_put_queue.qsize() - 1 # -1 because we added a None element to stop the queue remainingSeconds = datetime.timedelta( seconds=(remainingPages * config.put_throttle)) return (remainingPages, remainingSeconds) page_put_queue.put((None, [], {})) stopped = True if page_put_queue.qsize() > 1: num, sec = remaining() output(color_format( '{lightblue}Waiting for {num} pages to be put. ' 'Estimated time remaining: {sec}{default}', num=num, sec=sec)) while(_putthread.isAlive()): try: _putthread.join(1) except KeyboardInterrupt: if input_yn('There are %i pages remaining in the queue. ' 'Estimated time remaining: %s\nReally exit?' % remaining(), default=False, automatic_quit=False): return # only need one drop() call because all throttles use the same global pid try: list(_sites.values())[0].throttle.drop() log(u"Dropped throttle(s).") except IndexError: pass
def fixall(self): if self.sort: self.fixes = list(sorted(self.fixes, key=lambda f: f.summary)) if self._filters: self.fixes = [ fix for fix in self.fixes if should_fix(fix, self._filters) ] for fix in self.fixes: print(fix.summary) fixed = 0 for fix in self.fixes: success = fix.apply(self.user_add_claim) fixed += int(success) total = len(self.fixes) botlogging.output(f"Fixed {fixed}/{total} constraint failures", toStdout=True)
def _win32_extension_command(extension): """Get the command from the Win32 registry for an extension.""" fileexts_key = r"Software\Microsoft\Windows\CurrentVersion\Explorer\FileExts" key_name = fileexts_key + r"\." + extension + r"\OpenWithProgids" _winreg = winreg # exists for git blame only; do not use try: key1 = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_name) _progID = winreg.EnumValue(key1, 0)[0] _key2 = _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, "%s\shell\open\command" % _progID) _cmd = _winreg.QueryValueEx(_key2, None)[0] # See T102465 for issues relating to using this value. cmd = _cmd if cmd.find("%1"): cmd = cmd[: cmd.find("%1")] # Remove any trailing characher, which should be a quote or space # and then remove all whitespace. return cmd[:-1].strip() except WindowsError as e: # Catch any key lookup errors output('Unable to detect program for file extension "{0}": {1}'.format(extension, e))
def _win32_extension_command(extension): """Get the command from the Win32 registry for an extension.""" fileexts_key = \ r'Software\Microsoft\Windows\CurrentVersion\Explorer\FileExts' key_name = fileexts_key + r'\.' + extension + r'\OpenWithProgids' try: key1 = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_name) _prog_id = winreg.EnumValue(key1, 0)[0] _key2 = winreg.OpenKey(winreg.HKEY_CLASSES_ROOT, r'{}\shell\open\command'.format(_prog_id)) _cmd = winreg.QueryValueEx(_key2, None)[0] # See T102465 for issues relating to using this value. cmd = _cmd if cmd.find('%1'): cmd = cmd[:cmd.find('%1')] # Remove any trailing character, which should be a quote or space # and then remove all whitespace. return cmd[:-1].strip() except WindowsError as e: # Catch any key lookup errors output('Unable to detect program for file extension "{0}": {1!r}' .format(extension, e))
def treat_page_and_item(self, unused_page, item): """Fix items that have constraint failures unused_page is always None since use_from_page is False. See https://doc.wikimedia.org/pywikibot/master/api_ref/pywikibot.html#pywikibot.WikidataBot """ item.get(force=True) _, not_satisfied = super().treat_page_and_item(unused_page, item) typed_item = self.factory.get_typed_item(item.title()) fixes = [ fix for constraint in not_satisfied for fix in constraint.fix(typed_item) ] fixed = 0 for fix in fixes: skip_fix = not should_fix(fix, self._filters) if self._filters and skip_fix: continue success = fix.apply(self.user_add_claim) fixed += success total = len(not_satisfied) botlogging.output(f"Fixed {fixed}/{total} constraint failures", toStdout=True)
def get_base_dir(test_directory=None): r"""Return the directory in which user-specific information is stored. This is determined in the following order: 1. If the script was called with a -dir: argument, use the directory provided in this argument. 2. If the user has a PYWIKIBOT2_DIR environment variable, use the value of it. 3. If user-config is present in current directory, use the current directory. 4. If user-config is present in pwb.py directory, use that directory 5. Use (and if necessary create) a 'pywikibot' folder under 'Application Data' or 'AppData\Roaming' (Windows) or '.pywikibot' directory (Unix and similar) under the user's home directory. Set PYWIKIBOT2_NO_USER_CONFIG=1 to disable loading user-config.py @param test_directory: Assume that a user config file exists in this directory. Used to test whether placing a user config file in this directory will cause it to be selected as the base directory. @type test_directory: str or None @rtype: unicode """ def exists(directory): directory = os.path.abspath(directory) if directory == test_directory: return True else: return os.path.exists(os.path.join(directory, 'user-config.py')) if test_directory is not None: test_directory = os.path.abspath(test_directory) DIRNAME_WIN = u"Pywikibot" DIRNAME_WIN_FBCK = u"pywikibot" DIRNAME_UNIX = u".pywikibot" base_dir = "" for arg in sys.argv[1:]: if arg.startswith(str('-dir:')): base_dir = arg[5:] base_dir = os.path.expanduser(base_dir) break else: if ('PYWIKIBOT2_DIR' in os.environ and exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR']))): base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR']) elif exists('.'): base_dir = os.path.abspath('.') elif ('PYWIKIBOT2_DIR_PWB' in os.environ and exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB']))): base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB']) else: base_dir_cand = [] home = os.path.expanduser("~") if sys.platform == 'win32': import platform win_version = int(platform.version()[0]) if win_version == 5: sub_dir = ["Application Data"] elif win_version == 6: sub_dir = ["AppData", "Roaming"] else: raise WindowsError( u'Windows version %s not supported yet.' % win_version) base_dir_cand.extend([[home] + sub_dir + [DIRNAME_WIN], [home] + sub_dir + [DIRNAME_WIN_FBCK]]) else: base_dir_cand.append([home, DIRNAME_UNIX]) for dir in base_dir_cand: dir = os.path.join(*dir) if not os.path.isdir(dir): os.makedirs(dir, mode=private_files_permission) if exists(dir): base_dir = dir break if not os.path.isabs(base_dir): base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) # make sure this path is valid and that it contains user-config file if not os.path.isdir(base_dir): raise RuntimeError("Directory '%s' does not exist." % base_dir) # check if user-config.py is in base_dir if not exists(base_dir): exc_text = "No user-config.py found in directory '%s'.\n" % base_dir if _no_user_config: if _no_user_config != '2': output(exc_text) else: exc_text += " Please check that user-config.py is stored in the correct location.\n" exc_text += " Directory where user-config.py is searched is determined as follows:\n\n" exc_text += " " + get_base_dir.__doc__ raise RuntimeError(exc_text) return base_dir
def get_base_dir(test_directory: Optional[str] = None) -> str: r"""Return the directory in which user-specific information is stored. This is determined in the following order: 1. If the script was called with a -dir: argument, use the directory provided in this argument. 2. If the user has a PYWIKIBOT_DIR environment variable, use the value of it. 3. If user-config is present in current directory, use the current directory. 4. If user-config is present in pwb.py directory, use that directory 5. Use (and if necessary create) a 'pywikibot' folder under 'Application Data' or 'AppData\Roaming' (Windows) or '.pywikibot' directory (Unix and similar) under the user's home directory. Set PYWIKIBOT_NO_USER_CONFIG=1 to disable loading user-config.py @param test_directory: Assume that a user config file exists in this directory. Used to test whether placing a user config file in this directory will cause it to be selected as the base directory. """ def exists(directory): directory = os.path.abspath(directory) if directory == test_directory: return True else: return os.path.exists(os.path.join(directory, 'user-config.py')) if test_directory is not None: test_directory = os.path.abspath(test_directory) base_dir = '' for arg in sys.argv[1:]: if arg.startswith('-dir:'): base_dir = arg[5:] base_dir = os.path.expanduser(base_dir) break else: if ('PYWIKIBOT_DIR' in environ and exists(os.path.abspath(environ['PYWIKIBOT_DIR']))): base_dir = os.path.abspath(environ['PYWIKIBOT_DIR']) elif exists('.'): base_dir = os.path.abspath('.') elif ('PYWIKIBOT_DIR_PWB' in environ and exists(os.path.abspath(environ['PYWIKIBOT_DIR_PWB']))): base_dir = os.path.abspath(environ['PYWIKIBOT_DIR_PWB']) else: base_dir_cand = [] home = os.path.expanduser('~') if OSWIN32: win_version = int(platform.version().split('.')[0]) if win_version == 5: sub_dir = ['Application Data'] elif win_version in (6, 10): sub_dir = ['AppData', 'Roaming'] else: raise WindowsError('Windows version {} not supported yet.' .format(win_version)) base_dir_cand.extend([[home] + sub_dir + ['Pywikibot'], [home] + sub_dir + ['pywikibot']]) else: base_dir_cand.append([home, '.pywikibot']) for dir in base_dir_cand: dir = os.path.join(*dir) try: os.makedirs(dir, mode=private_files_permission) except OSError: # PermissionError or already exists if exists(dir): base_dir = dir break if not os.path.isabs(base_dir): base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) # make sure this path is valid and that it contains user-config file if not os.path.isdir(base_dir): raise RuntimeError("Directory '%s' does not exist." % base_dir) # check if user-config.py is in base_dir if not exists(base_dir): exc_text = "No user-config.py found in directory '%s'.\n" % base_dir if __no_user_config: if __no_user_config != '2': output(exc_text) else: exc_text += ( ' Please check that user-config.py is stored in the correct ' 'location.\n' ' Directory where user-config.py is searched is determined ' 'as follows:\n\n ') + get_base_dir.__doc__ raise RuntimeError(exc_text) return base_dir
exc_text += ( ' Please check that user-config.py is stored in the correct ' 'location.\n' ' Directory where user-config.py is searched is determined ' 'as follows:\n\n ') + get_base_dir.__doc__ raise RuntimeError(exc_text) return base_dir # Save base_dir for use by other modules base_dir = get_base_dir() for arg in sys.argv[1:]: if arg.startswith('-verbose') or arg == '-v': output('The base directory is ' + base_dir) break family_files = {} def register_family_file(family_name, file_path): """Register a single family class file. Parameter file_path may be a path or an url. family.AutoFamily function is used when the url is given. """ usernames[family_name] = {} disambiguation_comment[family_name] = {} family_files[family_name] = file_path
def output(self): """Output the text of the current sequence.""" output(self.format_list())
def print_failures(self, typed_item: BaseType, failed_constraints): """Print failed constraints""" for constraint in failed_constraints: botlogging.output(f"{constraint} failed for {typed_item}", toStdout=True)
def get_base_dir(test_directory=None): r"""Return the directory in which user-specific information is stored. This is determined in the following order: 1. If the script was called with a -dir: argument, use the directory provided in this argument. 2. If the user has a PYWIKIBOT2_DIR environment variable, use the value of it. 3. If user-config is present in current directory, use the current directory. 4. If user-config is present in pwb.py directory, use that directory 5. Use (and if necessary create) a 'pywikibot' folder under 'Application Data' or 'AppData\Roaming' (Windows) or '.pywikibot' directory (Unix and similar) under the user's home directory. Set PYWIKIBOT2_NO_USER_CONFIG=1 to disable loading user-config.py @param test_directory: Assume that a user config file exists in this directory. Used to test whether placing a user config file in this directory will cause it to be selected as the base directory. @type test_directory: str or None @rtype: unicode """ def exists(directory): directory = os.path.abspath(directory) if directory == test_directory: return True else: return os.path.exists(os.path.join(directory, 'user-config.py')) if test_directory is not None: test_directory = os.path.abspath(test_directory) DIRNAME_WIN = u"Pywikibot" DIRNAME_WIN_FBCK = u"pywikibot" DIRNAME_UNIX = u".pywikibot" base_dir = "" for arg in sys.argv[1:]: if arg.startswith(str('-dir:')): base_dir = arg[5:] base_dir = os.path.expanduser(base_dir) break else: if ('PYWIKIBOT2_DIR' in os.environ and exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR']))): base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR']) elif exists('.'): base_dir = os.path.abspath('.') elif ('PYWIKIBOT2_DIR_PWB' in os.environ and exists(os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB']))): base_dir = os.path.abspath(os.environ['PYWIKIBOT2_DIR_PWB']) else: base_dir_cand = [] home = os.path.expanduser("~") if OSWIN32: win_version = int(platform.version().split(".")[0]) if win_version == 5: sub_dir = ["Application Data"] elif win_version in (6, 10): sub_dir = ["AppData", "Roaming"] else: raise WindowsError(u'Windows version {0!s} not supported yet.'.format(win_version)) base_dir_cand.extend([[home] + sub_dir + [DIRNAME_WIN], [home] + sub_dir + [DIRNAME_WIN_FBCK]]) else: base_dir_cand.append([home, DIRNAME_UNIX]) for dir in base_dir_cand: dir = os.path.join(*dir) if not os.path.isdir(dir): os.makedirs(dir, mode=private_files_permission) if exists(dir): base_dir = dir break if not os.path.isabs(base_dir): base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) # make sure this path is valid and that it contains user-config file if not os.path.isdir(base_dir): raise RuntimeError("Directory '{0!s}' does not exist.".format(base_dir)) # check if user-config.py is in base_dir if not exists(base_dir): exc_text = "No user-config.py found in directory '{0!s}'.\n".format(base_dir) if __no_user_config: if __no_user_config != '2': output(exc_text) else: exc_text += " Please check that user-config.py is stored in the correct location.\n" exc_text += " Directory where user-config.py is searched is determined as follows:\n\n" exc_text += " " + get_base_dir.__doc__ raise RuntimeError(exc_text) return base_dir
else: exc_text += " Please check that user-config.py is stored in the correct location.\n" exc_text += " Directory where user-config.py is searched is determined as follows:\n\n" exc_text += " " + get_base_dir.__doc__ raise RuntimeError(exc_text) return base_dir _get_base_dir = get_base_dir # for backward compatibility _base_dir = get_base_dir() # Save base_dir for use by other modules base_dir = _base_dir for arg in sys.argv[1:]: if arg.startswith(str('-verbose')) or arg == str('-v'): output('The base directory is {0}'.format(base_dir)) break family_files = {} def register_family_file(family_name, file_path): """Register a single family class file.""" usernames[family_name] = {} sysopnames[family_name] = {} disambiguation_comment[family_name] = {} family_files[family_name] = file_path def register_families_folder(folder_path): """Register all family class files contained in a directory.""" for file_name in os.listdir(folder_path):
def output(self): """Output the text of the current sequence.""" output(self.out)
def print_successes(self, typed_item: BaseType, passed_constraints): """Print passed constraints""" for constraint in passed_constraints: botlogging.output(f"{constraint} passed for {typed_item}", toStdout=True)
exc_text += " Please check that user-config.py is stored in the correct location.\n" exc_text += " Directory where user-config.py is searched is determined as follows:\n\n" exc_text += " " + get_base_dir.__doc__ raise RuntimeError(exc_text) return base_dir _get_base_dir = get_base_dir # for backward compatibility _base_dir = get_base_dir() # Save base_dir for use by other modules base_dir = _base_dir for arg in sys.argv[1:]: if arg.startswith(str("-verbose")) or arg == str("-v"): output("The base directory is {0}".format(base_dir)) break family_files = {} def register_family_file(family_name, file_path): """Register a single family class file.""" usernames[family_name] = {} sysopnames[family_name] = {} disambiguation_comment[family_name] = {} family_files[family_name] = file_path def register_families_folder(folder_path): """Register all family class files contained in a directory.""" for file_name in os.listdir(folder_path):