def stop(condition, quantum=settings.QUANTUM):
last, tries = None, 0
while 1:
threads = tuple(thread for thread in enumerate_threads()
if isinstance(thread, SmartThread) and condition(thread))
if not threads:
break
for thread in threads:
if not thread.stopping:
thread.stop()
thread = threads[0]
if thread == last:
if tries == SHUTDOWN_TIME_TO_NOTIFY // quantum:
sys.stdout.write("Waiting for %s:\n%s" %
(thread.name, format_thread_trace(thread, indent=" ")))
if SHUTDOWN_CONSECUTIVE_NOTIFICATIONS:
tries = 0
else:
last, tries = thread, 0
try:
thread.join(quantum)
except IOError as error:
if error.errno != EINTR:
raise
tries += 1
def _stop_daemon_threads(self):
"""Notify the user of which threads are going to be killed.
Unfortunately, there is no method right now of stopping command and
task threads safely. This is because there is no way to tell them to
stop like the IRC components can be told; furthermore, they are run as
daemons, and daemon threads automatically stop without calling any
__exit__ or try/finally code when all non-daemon threads stop. They
were originally implemented as regular non-daemon threads, but this
meant there was no way to completely stop the bot if tasks were
running, because all other threads would exit and threading would
absorb KeyboardInterrupts.
The advantage of this is that stopping the bot is truly guarenteed to
*stop* the bot, while the disadvantage is that the threads are given no
advance warning of their forced shutdown.
"""
tasks = []
component_names = self.config.components.keys()
skips = component_names + ["MainThread", "reminder", "irc:quit"]
for thread in enumerate_threads():
if thread.name not in skips and thread.is_alive():
tasks.append(thread.name)
if tasks:
log = "The following commands or tasks will be killed: {0}"
self.logger.warn(log.format(", ".join(tasks)))
def network_shutdown(self):
try:
print >> sys.stderr, "tlm: network_shutdown"
# Arno, 2012-07-04: Obsolete, each thread must close the DBHandler
# it uses in its own shutdown procedure. There is no global close
# of all per-thread cursors/connections.
#
# cachedb.done()
# SWIFTPROC
if self.spm is not None:
self.spm.network_shutdown()
ts = enumerate_threads()
print >> sys.stderr, "tlm: Number of threads still running", len(ts)
for t in ts:
print >> sys.stderr, "tlm: Thread still running", t.getName(), "daemon", t.isDaemon(), "instance:", t
except:
print_exc()
# Stop network thread
self.sessdoneflag.set()
# Arno, 2010-08-09: Stop Session pool threads only after gracetime
self.session.uch.shutdown()
# Shutdown libtorrent session after checkpoints have been made
if self.ltmgr:
self.ltmgr.shutdown()
self.ltmgr.delInstance()
def _stop_daemon_threads(self):
"""Notify the user of which threads are going to be killed.
Unfortunately, there is no method right now of stopping command and
task threads safely. This is because there is no way to tell them to
stop like the IRC components can be told; furthermore, they are run as
daemons, and daemon threads automatically stop without calling any
__exit__ or try/finally code when all non-daemon threads stop. They
were originally implemented as regular non-daemon threads, but this
meant there was no way to completely stop the bot if tasks were
running, because all other threads would exit and threading would
absorb KeyboardInterrupts.
The advantage of this is that stopping the bot is truly guarenteed to
*stop* the bot, while the disadvantage is that the threads are given no
advance warning of their forced shutdown.
"""
tasks = []
component_names = self.config.components.keys()
skips = component_names + ["MainThread", "reminder", "irc:quit"]
for thread in enumerate_threads():
if thread.name not in skips and thread.is_alive():
tasks.append(thread.name)
if tasks:
log = "The following commands or tasks will be killed: {0}"
self.logger.warn(log.format(", ".join(tasks)))
def _launch_job_function(self, job, func_args=None):
# type: (BaseScheduleJob, Optional[Sequence]) -> Optional[Thread]
# make sure this IS a function job
if not job.base_function:
return None
# check if this is a single instance, then we need to abort the Task
if job.single_instance and job.get_last_executed_task_id():
# noinspection PyBroadException
try:
a_thread = [t for t in enumerate_threads() if t.ident == job.get_last_executed_task_id()]
if a_thread:
a_thread = a_thread[0]
except Exception:
a_thread = None
if a_thread and a_thread.is_alive():
self._log(
"Skipping Task '{}' scheduling, previous Thread instance '{}' still running".format(
job.name, a_thread.ident
))
job.run(None)
return None
self._log("Scheduling Job '{}', Task '{}' on background thread".format(
job.name, job.base_function))
t = Thread(target=job.base_function, args=func_args or ())
t.start()
# mark as run
job.run(t.ident)
return t
def tearDown(self):
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
del self.guiUtility
del self.frame
del self.lm
del self.session
time.sleep(1)
gc.collect()
ts = enumerate_threads()
print >> sys.stderr, "teardown: Number of threads still running", len(ts)
for t in ts:
print >> sys.stderr, "teardown: Thread still running", t.getName(), "daemon", t.isDaemon(), "instance:", t
dhtlog = os.path.join(STATE_DIR, 'pymdht.log')
if os.path.exists(dhtlog):
print >> sys.stderr, "teardown: content of pymdht.log"
f = open(dhtlog, 'r')
for line in f:
line = line.strip()
if line:
print >> sys.stderr, line
f.close()
print >> sys.stderr, "teardown: finished printing content of pymdht.log"
self.tearDownCleanup()
for boolean, reason in self.asserts:
assert boolean, reason
def network_shutdown(self):
try:
print >> sys.stderr, "tlm: network_shutdown"
# Arno, 2012-07-04: Obsolete, each thread must close the DBHandler
# it uses in its own shutdown procedure. There is no global close
# of all per-thread cursors/connections.
#
# cachedb.done()
# SWIFTPROC
if self.spm is not None:
self.spm.network_shutdown()
ts = enumerate_threads()
print >> sys.stderr, "tlm: Number of threads still running", len(
ts)
for t in ts:
print >> sys.stderr, "tlm: Thread still running", t.getName(
), "daemon", t.isDaemon(), "instance:", t
except:
print_exc()
# Stop network thread
self.sessdoneflag.set()
# Arno, 2010-08-09: Stop Session pool threads only after gracetime
self.session.uch.shutdown()
# Shutdown libtorrent session after checkpoints have been made
if self.ltmgr:
self.ltmgr.shutdown()
self.ltmgr.delInstance()
def network_shutdown(self):
try:
self._logger.info("tlm: network_shutdown")
ts = enumerate_threads()
self._logger.info("tlm: Number of threads still running %d",
len(ts))
for t in ts:
self._logger.info(
"tlm: Thread still running=%s, daemon=%s, instance=%s",
t.getName(), t.isDaemon(), t)
except:
print_exc()
# Stop network thread
self.sessdoneflag.set()
# Shutdown libtorrent session after checkpoints have been made
if self.ltmgr:
self.ltmgr.shutdown()
self.ltmgr = None
if self.threadpool:
self.threadpool.cancel_all_pending_tasks()
self.threadpool = None
def manager(rest_url = 'statuses/user_timeline'):
# print("manager: waiting tasks...")
# while tq.empty():
# time.sleep(1)
print("manager: start looping all keys, will quit when the main thread quit.")
while 1:
with open('./twitter_api_keys.pydict','r') as f:
api_keys = dict((key['consumer_key'], key) for key in eval(f.read()))
n = 0
for i, consumer_key in enumerate(api_keys):
time.sleep(1)
current_workers = list(t.name for t in enumerate_threads() if "Thread" not in t.name)
if consumer_key in current_workers or tq.qsize() < len(current_workers):
continue
worker = Thread(target = crawler,
kwargs ={'key': api_keys[consumer_key],
'name': '{min}:{max}-{no:>25}'.format(min = min(RG), max = max(RG), no = consumer_key),
'rest_url':rest_url})
worker.daemon = True
worker.name = consumer_key
worker.start()
n = n + 1
# print(("manager: {} workers are working.".format(len(current_workers))))
# if n>0: print(("manager: {} new workers are dispatched.".format(n)))
print("manager: quit.")
def stop(self):
super(VDOM_web_server_thread, self).stop()
log.write("Stop %s\n" % self.name)
if self.__server:
self.__server.shutdown()
for thread in enumerate_threads():
request = getattr(thread, THREAD_ATTRIBUTE_NAME, None)
if request is not None:
print thread, thread.name, request
def _update_execution_plots(self):
# type: () -> None
if not self._task:
return
task_link_template = self._task.get_output_log_web_page() \
.replace('/{}/'.format(self._task.project), '/{project}/') \
.replace('/{}/'.format(self._task.id), '/{task}/')
# plot the already executed Tasks
executed_table = [[
'trigger', 'name', 'task id', 'started', 'finished'
]]
for executed_job in sorted(self._executed_triggers,
key=lambda x: x.started,
reverse=True):
if not executed_job.finished:
if executed_job.task_id:
t = Task.get_task(task_id=executed_job.task_id)
if t.status not in ('in_progress', 'queued'):
executed_job.finished = t.data.completed or datetime.utcnow(
)
elif executed_job.thread_id:
# noinspection PyBroadException
try:
a_thread = [
t for t in enumerate_threads()
if t.ident == executed_job.thread_id
]
if not a_thread or not a_thread[0].is_alive():
executed_job.finished = datetime.utcnow()
except Exception:
pass
executed_table += [[
executed_job.trigger, executed_job.name,
'<a href="{}">{}</a>'.format(
task_link_template.format(project='*',
task=executed_job.task_id),
executed_job.task_id)
if executed_job.task_id else 'function',
str(executed_job.started).split('.', 1)[0],
str(executed_job.finished).split('.', 1)[0]
]]
# plot the schedule definition
self._task.get_logger().report_table(title='Triggers Executed',
series=' ',
iteration=0,
table_plot=executed_table)
self.__report_trigger_table(triggers=self._model_triggers,
title='Model Triggers')
self.__report_trigger_table(triggers=self._dataset_triggers,
title='Dataset Triggers')
self.__report_trigger_table(triggers=self._task_triggers,
title='Task Triggers')
def suspend(self):
self.__server.unavailable = True
allowable = {current_thread()}
while 1:
threads = {thread for thread in enumerate_threads()
if getattr(thread, THREAD_ATTRIBUTE_NAME, None) is not None}
if not threads or threads == allowable:
break
sleep(settings.QUANTUM)
def tearDown(self):
""" unittest test tear down code """
self.tp.joinAll()
time.sleep(2)
self.got.sort()
self.assertEquals(self.exp, self.got)
ts = enumerate_threads()
print >> sys.stderr, "test_threadpool: Number of threads still running", len(ts)
for t in ts:
print >> sys.stderr, "test_threadpool: Thread still running", t.getName(), "daemon", t.isDaemon(), "instance:", t
def tearDown(self):
""" unittest test tear down code """
self.tp.joinAll()
time.sleep(2)
self.got.sort()
self.assertEquals(self.exp, self.got)
ts = enumerate_threads()
print >> sys.stderr, "test_threadpool: Number of threads still running", len(
ts)
for t in ts:
print >> sys.stderr, "test_threadpool: Thread still running", t.getName(
), "daemon", t.isDaemon(), "instance:", t
def tearDown(self):
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
if self.session is not None:
yield self.session.shutdown()
assert self.session.has_shutdown()
self.session = None
yield self.stop_seeder()
ts = enumerate_threads()
self._logger.debug("test_as_server: Number of threads still running %d", len(ts))
for t in ts:
self._logger.debug("Thread still running %s, daemon: %s, instance: %s", t.getName(), t.isDaemon(), t)
yield super(TestAsServer, self).tearDown()
def tearDown(self):
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
if self.session is not None:
self._shutdown_session(self.session)
Session.del_instance()
time.sleep(10)
gc.collect()
ts = enumerate_threads()
self._logger.debug("test_as_server: Number of threads still running %d", len(ts))
for t in ts:
self._logger.debug("Thread still running %s, daemon: %s, instance: %s", t.getName(), t.isDaemon(), t)
super(TestAsServer, self).tearDown(annotate=False)
def tearDown(self, annotate=True):
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
if self.session is not None:
yield self.session.shutdown()
assert self.session.has_shutdown()
self.session = None
yield self.stop_seeder()
ts = enumerate_threads()
self._logger.debug("test_as_server: Number of threads still running %d", len(ts))
for t in ts:
self._logger.debug("Thread still running %s, daemon: %s, instance: %s", t.getName(), t.isDaemon(), t)
yield super(TestAsServer, self).tearDown(annotate=False)
def network_shutdown(self):
try:
self._logger.info("tlm: network_shutdown")
ts = enumerate_threads()
self._logger.info("tlm: Number of threads still running %d", len(ts))
for t in ts:
self._logger.info("tlm: Thread still running=%s, daemon=%s, instance=%s", t.getName(), t.isDaemon(), t)
except:
print_exc()
# Stop network thread
self.sessdoneflag.set()
# Shutdown libtorrent session after checkpoints have been made
if self.ltmgr is not None:
self.ltmgr.shutdown()
self.ltmgr = None
def exit(self):
global RedeemIsRunning
if not RedeemIsRunning:
return
RedeemIsRunning = False
logging.info("Redeem Shutting Down")
printer.path_planner.wait_until_done()
printer.path_planner.force_exit()
# Stops plugins
self.printer.plugins.exit()
for name, stepper in iteritems(self.printer.steppers):
stepper.set_disabled()
Stepper.commit()
for name, heater in iteritems(self.printer.heaters):
logging.debug("closing " + name)
heater.disable()
for name, endstop in iteritems(self.printer.end_stops):
logging.debug("terminating " + name)
endstop.stop()
for name, comm in iteritems(self.printer.comms):
logging.debug("closing " + name)
comm.close()
self.printer.comms_io_manager.stop()
self.printer.enable.set_disabled()
self.printer.swd.stop()
Alarm.executor.stop()
Key_pin.listener.stop()
self.printer.endstop_io_manager.stop()
self.printer.watchdog.stop()
self.printer.enable.set_disabled()
# list all threads that are still running
# note: some of these may be daemons
for t in enumerate_threads():
if t.name != "MainThread":
logging.debug("Thread " + t.name + " is still running")
def print_threads(writer=sys.stdout):
row = "%-28s %-36s %-18s %-8s %-8s %s"
writer.write(row % ("Class", "Name", "Ident", "Alive", "Daemon", ""))
writer.write(os.linesep)
writer.write("-" * 120)
writer.write(os.linesep)
for thread in sorted(enumerate_threads()):
cls = thread.__class__.__name__
name = thread.name
ident = thread.ident
alive = thread.is_alive()
daemon = thread.daemon
extra = ""
#extra = thread._Thread__target
writer.write(row % (cls, name, ident, alive, daemon, extra))
writer.write(os.linesep)
def get_current_threads():
"""Get a list of `ThreadInfo` objects."""
threads = []
for thread in enumerate_threads():
tinfo = ThreadInfo()
try:
tinfo.ident = thread.ident
except AttributeError:
tinfo.ident = None # TODO
try:
tinfo.name = thread.name
except AttributeError:
tinfo.name = thread.getName()
try:
tinfo.daemon = thread.daemon
except AttributeError:
tinfo.daemon = thread.isDaemon()
threads.append(tinfo)
return threads
def tearDown(self):
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
del self.guiUtility
del self.frame
del self.lm
del self.session
from Tribler.Core.CacheDB.sqlitecachedb import unregister
unregister()
sleep(10)
gc.collect()
ts = enumerate_threads()
print >> sys.stderr, "teardown: Number of threads still running", len(
ts)
for t in ts:
print >> sys.stderr, "teardown: Thread still running", t.getName(
), "daemon", t.isDaemon(), "instance:", t
dhtlog = os.path.join('.Tribler', 'pymdht.log')
if os.path.exists(dhtlog):
print >> sys.stderr, "teardown: content of pymdht.log"
f = open(dhtlog, 'r')
for line in f:
line = line.strip()
if line:
print >> sys.stderr, line
f.close()
print >> sys.stderr, "teardown: finished printing content of pymdht.log"
shutil.rmtree(".Tribler")
shutil.rmtree("TriblerDownloads")
for boolean, reason in self.asserts:
assert boolean, reason
assert not os.path.exists(".Tribler"), ".Tribler should not exist"
assert not os.path.exists(
"TriblerDownloads"), "TriblerDownloads should not exist"
def tearDown(self):
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
if self.session is not None:
self._shutdown_session(self.session)
Session.del_instance()
time.sleep(10)
gc.collect()
ts = enumerate_threads()
print >> sys.stderr, "test_as_server: Number of threads still running", len(ts)
for t in ts:
print >> sys.stderr, "test_as_server: Thread still running", t.getName(), "daemon", t.isDaemon(), "instance:", t
if SQLiteCacheDB.hasInstance():
SQLiteCacheDB.getInstance().close_all()
SQLiteCacheDB.delInstance()
self.tearDownCleanup()
def tearDown(self):
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
del self.guiUtility
del self.frame
del self.lm
del self.session
from Tribler.Core.CacheDB.sqlitecachedb import unregister
unregister()
sleep(10)
gc.collect()
ts = enumerate_threads()
print >>sys.stderr, "teardown: Number of threads still running", len(ts)
for t in ts:
print >>sys.stderr, "teardown: Thread still running", t.getName(), "daemon", t.isDaemon(), "instance:", t
dhtlog = os.path.join(".Tribler", "pymdht.log")
if os.path.exists(dhtlog):
print >>sys.stderr, "teardown: content of pymdht.log"
f = open(dhtlog, "r")
for line in f:
line = line.strip()
if line:
print >>sys.stderr, line
f.close()
print >>sys.stderr, "teardown: finished printing content of pymdht.log"
shutil.rmtree(".Tribler")
shutil.rmtree("TriblerDownloads")
for boolean, reason in self.asserts:
assert boolean, reason
assert not os.path.exists(".Tribler"), ".Tribler should not exist"
assert not os.path.exists("TriblerDownloads"), "TriblerDownloads should not exist"
def feed():
status = "<title>VCM STATUS</title>"
execution_time = seconds_to_str(time() - t0, integer=False)
status += f"Execution time: {execution_time}<br>"
status += 'Unfinished <a href="/queue" target="blank" style="text-decoration:none">'
status += f"tasks</a>: {queue.unfinished_tasks}"
if not running.is_set():
status += '<font color="red"> [Shutting down]</font>'
status += f"<br>Items left: {queue.qsize()}<br><br>"
thread_status = "Threads (%d):" % count_threads()
if ErrorCounter.has_errors():
report = f'<font color="red">\t<b>[{ErrorCounter.report()}]</b></font>'
thread_status += report
thread_status += "<br>"
colors, working, idle = get_thread_state_info()
for thread in enumerate_threads():
if not isinstance(thread, Worker):
continue
temp_status = thread.to_log(integer=True)
thread_status += f"\t-{temp_status}<br>"
colors = list(colors.items())
colors.sort(key=lambda x: x[-1], reverse=True)
# status += f"Threads working: {working}<br>"
# status += f"Threads idle: {idle}<br><br>"
status += f"Codes:<br>"
status += "<br>".join(
[f'<font color="{x[0]}">-{x[0]}: {x[1]}</font>' for x in colors]
)
status += "<br><br>"
status += thread_status
return status
def get_thread_state_info():
colors = defaultdict(int, {"green": 0, "orange": 0, "red": 0, "magenta": 0})
working = 0
idle = 0
for thread in enumerate_threads():
if not isinstance(thread, Worker):
continue
if isinstance(thread, Killer):
continue
state = thread.state
color = state_to_color[state]
colors[color.name] += 1
if state == ThreadStates.idle:
idle += 1
if state.alias == "working":
working += 1
colors.pop("blue", None)
return dict(colors), working, idle
def tearDown(self):
self.wx_watchdog = None
self.twisted_watchdog = None
self.watchdog.unregister_event("wx thread")
self.watchdog.unregister_event("twisted thread")
self.annotate(self._testMethodName, start=False)
""" unittest test tear down code """
del self.guiUtility
del self.frame
del self.lm
del self.session
time.sleep(1)
gc.collect()
ts = enumerate_threads()
if ts:
self._logger.debug("Number of threads still running %s", len(ts))
for t in ts:
self._logger.debug("Thread still running %s, daemon %s, instance: %s", t.getName(), t.isDaemon(), t)
dhtlog = os.path.join(self.state_dir, 'pymdht.log')
if os.path.exists(dhtlog):
self._logger.debug("Content of pymdht.log")
f = open(dhtlog, 'r')
for line in f:
line = line.strip()
if line:
self._logger.debug("> %s", line)
f.close()
self._logger.debug("Finished printing content of pymdht.log")
AbstractServer.tearDown(self, annotate=False)
for boolean, reason in self.asserts:
assert boolean, reason
def search(name):
for thread in enumerate_threads():
if not isinstance(thread, SmartThread):
continue
if thread.name == name:
return thread
def condition():
for thread in enumerate_threads():
if thread.name == Logger.name:
return False
return True
def count_threads() -> int:
nthreads = 0
for thread in enumerate_threads():
if isinstance(thread, Worker):
nthreads += 1
return nthreads
def _update_execution_plots(self):
# type: () -> None
"""
Update the configuration and execution table plots
"""
if not self._task:
return
task_link_template = self._task.get_output_log_web_page() \
.replace('/{}/'.format(self._task.project), '/{project}/') \
.replace('/{}/'.format(self._task.id), '/{task}/')
# plot the schedule definition
columns = [
'name', 'base_task_id', 'base_function', 'next_run', 'target_project', 'queue',
'minute', 'hour', 'day', 'month', 'year',
'starting_time', 'execution_limit_hours', 'recurring',
'single_instance', 'task_parameters', 'task_overrides', 'clone_task',
]
scheduler_table = [columns]
for j in self._schedule_jobs:
j_dict = j.to_dict()
j_dict['next_run'] = j.next()
j_dict['base_function'] = "{}.{}".format(
getattr(j.base_function, '__module__', ''),
getattr(j.base_function, '__name__', '')
) if j.base_function else ''
if not j_dict.get('base_task_id'):
j_dict['clone_task'] = ''
row = [
str(j_dict.get(c)).split('.', 1)[0] if isinstance(j_dict.get(c), datetime) else str(j_dict.get(c) or '')
for c in columns
]
if row[1]:
row[1] = '<a href="{}">{}</a>'.format(
task_link_template.format(project='*', task=row[1]), row[1])
scheduler_table += [row]
# plot the already executed Tasks
executed_table = [['name', 'task id', 'started', 'finished']]
for executed_job in sorted(self._executed_jobs, key=lambda x: x.started, reverse=True):
if not executed_job.finished:
if executed_job.task_id:
t = Task.get_task(task_id=executed_job.task_id)
if t.status not in ('in_progress', 'queued'):
executed_job.finished = t.data.completed or datetime.utcnow()
elif executed_job.thread_id:
# noinspection PyBroadException
try:
a_thread = [t for t in enumerate_threads() if t.ident == executed_job.thread_id]
if not a_thread or not a_thread[0].is_alive():
executed_job.finished = datetime.utcnow()
except Exception:
pass
executed_table += [
[executed_job.name,
'<a href="{}">{}</a>'.format(task_link_template.format(
project='*', task=executed_job.task_id), executed_job.task_id)
if executed_job.task_id else 'function',
str(executed_job.started).split('.', 1)[0], str(executed_job.finished).split('.', 1)[0]
]
]
self._task.get_logger().report_table(
title='Schedule Tasks', series=' ', iteration=0,
table_plot=scheduler_table
)
self._task.get_logger().report_table(
title='Executed Tasks', series=' ', iteration=0,
table_plot=executed_table
)
def wait_for_all_child_threads():
threads = enumerate_threads()
threads.remove(current_thread())
for thread in threads:
thread.join()
