def _flush():
log('Closing network session.')
session.close()
if hasattr(sys, 'last_type'):
critical('Exiting due to uncaught exception {}'.format(sys.last_type))
log('Network session closed.')
def _flush():
log('Closing network session.')
session.close()
if hasattr(sys, 'last_type'):
critical('Exiting due to uncaught exception {}'.format(sys.last_type))
log('Network session closed.')
def flush():
"""Close the session object. This is called when the module terminates."""
log('Closing network session.')
session.close()
if hasattr(sys, 'last_type'):
critical('Exiting due to uncaught exception {}'.format(sys.last_type))
log('Network session closed.')
def _flush(stop: bool = True) -> None:
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from
the throttle log. Called automatically at Python exit.
"""
_logger = 'wiki'
debug('_flush() called', _logger)
def remaining() -> Tuple[int, datetime.timedelta]:
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(
seconds=round(remainingPages * _config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > _config.noisysleep:
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
if _putthread is not threading.current_thread():
while (_putthread.is_alive() and (page_put_queue.qsize() > 0
or page_put_queue_busy.qsize() > 0)):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn(
'There are {} pages remaining in the queue. '
'Estimated time remaining: {}\nReally exit?'.format(
*remaining()),
default=False,
automatic_quit=False):
# delete the put queue
with page_put_queue.mutex:
page_put_queue.all_tasks_done.notify_all()
page_put_queue.queue.clear()
page_put_queue.not_full.notify_all()
break
# only need one drop() call because all throttles use the same global pid
with suppress(IndexError):
list(_sites.values())[0].throttle.drop()
log('Dropped throttle(s).')
def _flush():
session.close()
message = 'Closing network session.'
if hasattr(sys, 'last_type'):
# we quit because of an exception
print(sys.last_type)
critical(message)
else:
log(message)
log('Network session closed.')
def _flush():
session.close()
message = 'Closing network session.'
if hasattr(sys, 'last_type'):
# we quit because of an exception
print(sys.last_type) # flake8: disable=T003 (print)
critical(message)
else:
log(message)
log('Network session closed.')
def _flush(stop=True):
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from
the throttle log. Called automatically at Python exit.
"""
_logger = "wiki"
debug('_flush() called', _logger)
def remaining():
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(seconds=(remainingPages *
config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > config.noisysleep:
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
while _putthread.isAlive() and page_put_queue.qsize() > 0:
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are {0} pages remaining in the queue. '
'Estimated time remaining: {1}\nReally exit?'
''.format(*remaining()),
default=False,
automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def stopme():
"""Drop this process from the throttle log, after pending threads finish.
Can be called manually if desired, but if not, will be called automatically
at Python exit.
"""
global stopped
_logger = "wiki"
if not stopped:
debug(u"stopme() called", _logger)
def remaining():
remainingPages = page_put_queue.qsize() - 1
# -1 because we added a None element to stop the queue
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
page_put_queue.put((None, [], {}))
stopped = True
if page_put_queue.qsize() > 1:
num, sec = remaining()
output(
color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}',
num=num,
sec=sec))
while (_putthread.isAlive()):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are %i pages remaining in the queue. '
'Estimated time remaining: %s\nReally exit?' %
remaining(),
default=False,
automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def _flush(stop=True):
"""
Drop this process from the throttle log, after pending threads finish.
Wait for the page-putter to flush its queue. Also drop this process from the
throttle log. Called automatically at Python exit.
"""
_logger = "wiki"
debug('_flush() called', _logger)
def remaining():
remainingPages = page_put_queue.qsize()
if stop:
# -1 because we added a None element to stop the queue
remainingPages -= 1
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
if stop:
# None task element leaves async_manager
page_put_queue.put((None, [], {}))
num, sec = remaining()
if num > 0 and sec.total_seconds() > config.noisysleep:
output(color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}', num=num, sec=sec))
while _putthread.isAlive() and page_put_queue.qsize() > 0:
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are {0} pages remaining in the queue. '
'Estimated time remaining: {1}\nReally exit?'
''.format(*remaining()),
default=False, automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
def stopme():
"""
Drop this process from the throttle log, after pending threads finish.
Can be called manually if desired, but if not, will be called automatically
at Python exit.
"""
global stopped
_logger = "wiki"
if not stopped:
debug(u"stopme() called", _logger)
def remaining():
remainingPages = page_put_queue.qsize() - 1
# -1 because we added a None element to stop the queue
remainingSeconds = datetime.timedelta(
seconds=(remainingPages * config.put_throttle))
return (remainingPages, remainingSeconds)
page_put_queue.put((None, [], {}))
stopped = True
if page_put_queue.qsize() > 1:
num, sec = remaining()
output(color_format(
'{lightblue}Waiting for {num} pages to be put. '
'Estimated time remaining: {sec}{default}', num=num, sec=sec))
while(_putthread.isAlive()):
try:
_putthread.join(1)
except KeyboardInterrupt:
if input_yn('There are %i pages remaining in the queue. '
'Estimated time remaining: %s\nReally exit?'
% remaining(), default=False, automatic_quit=False):
return
# only need one drop() call because all throttles use the same global pid
try:
list(_sites.values())[0].throttle.drop()
log(u"Dropped throttle(s).")
except IndexError:
pass
