def _flush(stop: bool = True) -> None:
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from
the throttle log. Called automatically at Python exit.
"""
_logger = 'wiki'
debug('_flush() called', _logger)
def remaining() -> Tuple[int, datetime.timedelta]:
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(
seconds=round(remainingPages * _config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > _config.noisysleep:
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
if _putthread is not threading.current_thread():
while (_putthread.is_alive() and (page_put_queue.qsize() > 0
or page_put_queue_busy.qsize() > 0)):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn(
'There are {} pages remaining in the queue. '
'Estimated time remaining: {}\nReally exit?'.format(
*remaining()),
default=False,
automatic_quit=False):
# delete the put queue
with page_put_queue.mutex:
page_put_queue.all_tasks_done.notify_all()
page_put_queue.queue.clear()
page_put_queue.not_full.notify_all()
break
# only need one drop() call because all throttles use the same global pid
with suppress(IndexError):
list(_sites.values())[0].throttle.drop()
log('Dropped throttle(s).')
def _flush(stop=True):
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from
the throttle log. Called automatically at Python exit.
"""
_logger = "wiki"
debug('_flush() called', _logger)
def remaining():
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(seconds=(remainingPages *
config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > config.noisysleep:
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
while _putthread.isAlive() and page_put_queue.qsize() > 0:
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are {0} pages remaining in the queue. '
'Estimated time remaining: {1}\nReally exit?'
''.format(*remaining()),
default=False,
automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def stopme():
"""Drop this process from the throttle log, after pending threads finish.
Can be called manually if desired, but if not, will be called automatically
at Python exit.
"""
global stopped
_logger = "wiki"
if not stopped:
debug("stopme() called", _logger)
def remaining():
remainingPages = page_put_queue.qsize() - 1
# -1 because we added a None element to stop the queue
remainingSeconds = datetime.timedelta(seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
page_put_queue.put((None, [], {}))
stopped = True
if page_put_queue.qsize() > 1:
num, sec = remaining()
format_values = dict(num=num, sec=sec)
output(
"\03{lightblue}"
"Waiting for %(num)i pages to be put. "
"Estimated time remaining: %(sec)s"
"\03{default}" % format_values
)
while _putthread.isAlive():
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn(
"There are %i pages remaining in the queue. "
"Estimated time remaining: %s\nReally exit?" % remaining(),
default=False,
automatic_quit=False,
):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log("Dropped throttle(s).")
except IndexError:
pass
def stopme():
"""Drop this process from the throttle log, after pending threads finish.
Can be called manually if desired, but if not, will be called automatically
at Python exit.
"""
global stopped
_logger = "wiki"
if not stopped:
debug(u"stopme() called", _logger)
def remaining():
remainingPages = page_put_queue.qsize() - 1
# -1 because we added a None element to stop the queue
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
page_put_queue.put((None, [], {}))
stopped = True
if page_put_queue.qsize() > 1:
num, sec = remaining()
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
while (_putthread.isAlive()):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are %i pages remaining in the queue. '
'Estimated time remaining: %s\nReally exit?' %
remaining(),
default=False,
automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def _flush(stop=True):
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from the
throttle log. Called automatically at Python exit.
"""
_logger = "wiki"
debug('_flush() called', _logger)
def remaining():
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > config.noisysleep:
output(color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}', num=num, sec=sec))
while _putthread.isAlive() and page_put_queue.qsize() > 0:
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are {0} pages remaining in the queue. '
'Estimated time remaining: {1}\nReally exit?'
''.format(*remaining()),
default=False, automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def treat_page(self):
"""Convert all HTML tables in text to wiki syntax and save it."""
text = self.current_page.text
newText, convertedTables, warnings = self.convertAllHTMLTables(text)
# Check if there are any marked tags left
markedTableTagR = re.compile("<##table##|</##table##>", re.IGNORECASE)
if markedTableTagR.search(newText):
pywikibot.error(
u'not all marked table start or end tags processed!')
return
if convertedTables == 0:
pywikibot.output(u"No changes were necessary.")
return
if warnings:
if self.getOption('always') and self.getOption('skipwarning'):
pywikibot.output(
'There were %i replacements that might lead to bad '
'output. Skipping.' % warnings)
return
if not self.getOption('always'):
pywikibot.output(
'There were %i replacements that might lead to bad '
'output.' % warnings)
if not input_yn('Do you want to change the page anyway'):
return
# get edit summary message
if warnings == 0:
editSummaryMessage = i18n.twtranslate(
self.site.code, 'table2wiki-no-warning')
else:
editSummaryMessage = i18n.twntranslate(
self.site.code,
'table2wiki-warnings',
{'count': warnings}
)
self.put_current(newText, summary=editSummaryMessage,
show_diff=not (self.getOption('quiet') and
self.getOption('always')))
def treat_page(self):
"""Convert all HTML tables in text to wiki syntax and save it."""
text = self.current_page.text
newText, convertedTables, warnings = self.convertAllHTMLTables(text)
# Check if there are any marked tags left
markedTableTagR = re.compile("<##table##|</##table##>", re.IGNORECASE)
if markedTableTagR.search(newText):
pywikibot.error(
u'not all marked table start or end tags processed!')
return
if convertedTables == 0:
pywikibot.output(u"No changes were necessary.")
return
if warnings:
if self.getOption('always') and self.getOption('skipwarning'):
pywikibot.output(
'There were %i replacements that might lead to bad '
'output. Skipping.' % warnings)
return
if not self.getOption('always'):
pywikibot.output(
'There were %i replacements that might lead to bad '
'output.' % warnings)
if not input_yn('Do you want to change the page anyway'):
return
# get edit summary message
if warnings == 0:
editSummaryMessage = i18n.twtranslate(
self.site.code, 'table2wiki-no-warning')
else:
editSummaryMessage = i18n.twntranslate(
self.site.code,
'table2wiki-warnings',
{'count': warnings}
)
self.put_current(newText, summary=editSummaryMessage,
show_diff=not (self.getOption('quiet') and
self.getOption('always')))
def treat_page(self):
"""Convert all HTML tables in text to wiki syntax and save it."""
text = self.current_page.text
new_text, converted_tables, warnings = self.convertAllHTMLTables(text)
# Check if there are any marked tags left
if re.search('<##table##|</##table##>', new_text, re.IGNORECASE):
pywikibot.error(
'not all marked table start or end tags processed!')
return
if converted_tables == 0:
pywikibot.output('No changes were necessary.')
return
if warnings:
if self.opt.always and self.opt.skipwarning:
pywikibot.output(
'There were {} replacements that might lead to bad '
'output. Skipping.'.format(warnings))
return
if not self.opt.always:
pywikibot.output(
'There were {} replacements that might lead to bad '
'output.'.format(warnings))
if not input_yn('Do you want to change the page anyway'):
return
# get edit summary message
if warnings == 0:
edit_summary = i18n.twtranslate(
self.site.code, 'table2wiki-no-warning')
else:
edit_summary = i18n.twntranslate(
self.site.code,
'table2wiki-warnings',
{'count': warnings}
)
self.put_current(new_text, summary=edit_summary,
show_diff=not (self.opt.quiet
and self.opt.always))
