def start_workload(self, parallel_threads, **kwargs):
start_monitoring(seconds_frozen=60)
workload_perc = {}
workloads = kwargs.pop("workload").split(",")
wperc = 100 / len(workloads)
for workload in workloads:
wload = workload.split(":")
if len(wload) == 1:
workload_perc[wload[0]] = wperc
continue
workload_perc[wload[0]] = int(wload[1])
quota = {}
for workload_type, workload_per in workload_perc.iteritems():
quota[workload_type] = {"perc": workload_per}
self._task_manager = parallel_execution(parallel_threads, quotas=quota)
self._prepare_stage(**kwargs)
self._exit_marker = self._create_exit_marker()
self._start_and_monitor(workload_perc.keys(), **kwargs)
def stop_gracefully(*_, **__):
global signal_call_counter, trace_threads, monitoring_thread
signal_call_counter += 1
logger = logging.getLogger(__name__)
if trace_threads:
from hanging_threads import start_monitoring
if signal_call_counter == 1:
logger.critical(
"Thread monitoring is started. "
"Hanging threads will be reported every 10 seconds.\n"
"If you see a thread keeps showing for multiple "
"occurrences, please report it to the developer.\n\n"
"Press Ctrl-C again to stop reporting, "
"press another 3 times to force quit EFB."
)
monitoring_thread = start_monitoring()
elif monitoring_thread is not None and monitoring_thread.is_alive():
monitoring_thread.stop()
logger.critical(
"Thread report is now stopped, "
"press Ctrl-C for 3 times to force quit EFB."
)
if signal_call_counter >= MAX_SIG_CALL_BEFORE_FORCE_EXIT:
logger.error(
"5 consequent exit signals detected.\n"
"Force exiting EFB without cleaning up.\n"
"\n"
"If it has taken you too long to quit EFB, "
"it is most likely a bug with EFB or some of the modules enabled. "
"Please consider tracing the hanging thread using --trace-threads "
"argument, and report a bug to the developers."
)
exit(1)
# Stop the main thread from blocking.
if not exit_event.is_set():
exit_event.set()
# Wait for channels to stop polling.
if hasattr(coordinator, "master") and isinstance(coordinator.master, MasterChannel):
coordinator.master.stop_polling()
logger.debug("Stop signal sent to master: %s" % coordinator.master.channel_name)
else:
logger.info("Valid master channel is not found.")
for i in coordinator.slaves:
if isinstance(coordinator.slaves[i], SlaveChannel):
coordinator.slaves[i].stop_polling()
logger.debug("Stop signal sent to slave: %s" % coordinator.slaves[i].channel_name)
if coordinator.master_thread and coordinator.master_thread.is_alive():
coordinator.master_thread.join()
for i in coordinator.slave_threads.values():
if i.is_alive():
i.join()
def daemon():
starts = [int(x) for x in sys.argv[1:]]
threads = []
mv = MasterVault()
for start in starts:
def mv_thread():
print("thread open")
mv.scrape_back(start, start)
print("thread close")
t = threading.Thread(target=mv_thread)
threads.append(t)
t.start()
#t = threading.Thread(target=workers.thread)
#threads.append(t)
#t.start()
print("start monitoring")
monitoring_thread = start_monitoring(seconds_frozen=30)
def runtests(names, options, arg_i, arg_p, runtime_test_params):
log.info("\nNumber of tests initially selected before GROUP filters: " +
str(len(names)))
BEFORE_SUITE = "suite_setUp"
AFTER_SUITE = "suite_tearDown"
xunit = XUnitTestResult()
# Create root logs directory
abs_path = os.path.dirname(os.path.abspath(sys.argv[0]))
# Create testrunner logs subdirectory
str_time = time.strftime("%y-%b-%d_%H-%M-%S", time.localtime())
root_log_dir = os.path.join(
abs_path, "logs{0}testrunner-{1}".format(os.sep, str_time))
if not os.path.exists(root_log_dir):
os.makedirs(root_log_dir)
results = []
case_number = 1
if "GROUP" in runtime_test_params:
print(("Only cases in GROUPs '{0}' will be executed".format(
runtime_test_params["GROUP"])))
if "EXCLUDE_GROUP" in runtime_test_params:
print(("Cases from GROUPs '{0}' will be excluded".format(
runtime_test_params["EXCLUDE_GROUP"])))
if TestInputSingleton.input.param("get-delays", False):
# start measure_sched_delays on all servers
sd = SchedDelays(TestInputSingleton.input.servers)
sd.start_measure_sched_delays()
if TestInputSingleton.input.param("hanging_threads", False):
print("--> hanging_threads: start monitoring...")
from hanging_threads import start_monitoring
hanging_threads_frozen_time = int(
TestInputSingleton.input.param("hanging_threads", 120))
hanging_threads_test_interval = int(
TestInputSingleton.input.param("test_interval", 1000))
monitoring_thread = start_monitoring(
seconds_frozen=hanging_threads_frozen_time,
test_interval=hanging_threads_test_interval)
logs_folder = "."
test_exec_count = 0
for name in names:
start_time = time.time()
argument_split = [
a.strip() for a in re.split("[,]?([^,=]+)=", name)[1:]
]
params = dict(list(zip(argument_split[::2], argument_split[1::2])))
# Note that if ALL is specified at runtime then tests which have no groups are still run - just being
# explicit on this
if "GROUP" in runtime_test_params and "ALL" not in runtime_test_params[
"GROUP"].split(";"):
if 'GROUP' not in params: # params is the .conf file parameters.
# this test is not in any groups so we do not run it
print((
"test '{0}' skipped, a group was requested and this is not any groups"
.format(name)))
continue
# there is a group for this test case, if that group is not specified at run time then do not run it
elif not set(runtime_test_params["GROUP"].split(";")).issubset(
set(params["GROUP"].split(";"))):
print(("test '{0}' skipped, is not in the requested group".
format(name)))
continue
else:
pass # the test was in requested group, will run it
elif "EXCLUDE_GROUP" in runtime_test_params:
if 'GROUP' in params and \
set(runtime_test_params["EXCLUDE_GROUP"].split(";")).issubset(set(params["GROUP"].split(";"))):
print(
("test '{0}' skipped, is in an excluded group".format(name)
))
continue
log.info("--> Running test: {}".format(name))
test_exec_count += 1
# Create Log Directory
logs_folder = os.path.join(root_log_dir, "test_%s" % case_number)
log.info("Logs folder: {}".format(logs_folder))
os.mkdir(logs_folder)
test_log_file = os.path.join(logs_folder, "test.log")
log_config_filename = r'{0}'.format(
os.path.join(logs_folder, "test.logging.conf"))
create_log_file(log_config_filename, test_log_file, options.loglevel)
logging.config.fileConfig(log_config_filename)
print(("Logs will be stored at {0}".format(logs_folder)))
print(("\n.{3}testrunner -i {0} -p {1} -t {2}\n"\
.format(arg_i or "", arg_p or "", name, os.sep)))
name = name.split(",")[0]
# Update the test params for each test
TestInputSingleton.input.test_params = params
TestInputSingleton.input.test_params.update(runtime_test_params)
TestInputSingleton.input.test_params["case_number"] = case_number
TestInputSingleton.input.test_params["logs_folder"] = logs_folder
print("Test Input params:")
print((TestInputSingleton.input.test_params))
if "get-coredumps" in TestInputSingleton.input.test_params:
if TestInputSingleton.input.param("get-coredumps", True):
clear_old_core_dumps(TestInputSingleton.input, logs_folder)
if case_number == 1:
before_suite_name = "%s.%s" % (name[:name.rfind('.')],
BEFORE_SUITE)
try:
print(("Run before suite setup for %s" % name))
suite = unittest.TestLoader().loadTestsFromName(
before_suite_name)
print(("-->before_suite_name:{},suite: {}".format(
before_suite_name, suite)))
result = unittest.TextTestRunner(verbosity=2).run(suite)
print(("-->result: {}".format(result)))
if "get-coredumps" in TestInputSingleton.input.test_params:
if TestInputSingleton.input.param("get-coredumps", True):
if get_core_dumps(TestInputSingleton.input,
logs_folder):
result = unittest.TextTestRunner(
verbosity=2)._makeResult()
result.errors = [
(name, "Failing test : new core dump(s) "
"were found and collected."
" Check testrunner logs folder.")
]
log.info(
"FAIL: New core dump(s) was found and collected"
)
except AttributeError as ex:
traceback.print_exc()
pass
try:
suite = unittest.TestLoader().loadTestsFromName(name)
except AttributeError as e:
print(("Test {0} was not found: {1}".format(name, e)))
result = unittest.TextTestRunner(verbosity=2)._makeResult()
result.errors = [(name, str(e))]
except SyntaxError as e:
print(("SyntaxError in {0}: {1}".format(name, e)))
result = unittest.TextTestRunner(verbosity=2)._makeResult()
result.errors = [(name, str(e))]
else:
test_timeout = TestInputSingleton.input.param("test_timeout", None)
t = StoppableThreadWithResult(
target=unittest.TextTestRunner(verbosity=2).run,
name="test_thread",
args=(suite))
t.start()
result = t.join(timeout=test_timeout)
if "get-coredumps" in TestInputSingleton.input.test_params:
if TestInputSingleton.input.param("get-coredumps", True):
if get_core_dumps(TestInputSingleton.input, logs_folder):
result = unittest.TextTestRunner(
verbosity=2)._makeResult()
result.errors = [(name,
"Failing test : new core dump(s) "
"were found and collected."
" Check testrunner logs folder.")]
log.info(
"FAIL: New core dump(s) was found and collected")
if not result:
for t in threading.enumerate():
if t != threading.current_thread():
t._Thread__stop()
result = unittest.TextTestRunner(verbosity=2)._makeResult()
case_number += 1000
print("========TEST WAS STOPPED DUE TO TIMEOUT=========")
result.errors = [(name, "Test was stopped due to timeout")]
time_taken = time.time() - start_time
# Concat params to test name
# To make tests more readable
params = ''
if TestInputSingleton.input.test_params:
for key, value in list(
TestInputSingleton.input.test_params.items()):
if key and value:
params += "," + str(key) + ":" + str(value)
if result.failures or result.errors:
# Immediately get the server logs, if
# the test has failed or has errors
if "get-logs" in TestInputSingleton.input.test_params:
get_server_logs(TestInputSingleton.input, logs_folder)
if "get-logs-cluster-run" in TestInputSingleton.input.test_params:
if TestInputSingleton.input.param("get-logs-cluster-run",
True):
# Generate path to ns_server directory
ns_server_path = os.path.normpath(abs_path + os.sep +
os.pardir + os.sep +
"ns_server")
get_logs_cluster_run(TestInputSingleton.input, logs_folder,
ns_server_path)
if "get-cbcollect-info" in TestInputSingleton.input.test_params:
if TestInputSingleton.input.param("get-cbcollect-info", True):
get_cbcollect_info(TestInputSingleton.input, logs_folder)
if "get-couch-dbinfo" in TestInputSingleton.input.test_params and \
TestInputSingleton.input.param("get-couch-dbinfo", True):
get_couch_dbinfo(TestInputSingleton.input, logs_folder)
errors = []
for failure in result.failures:
test_case, failure_string = failure
errors.append(failure_string)
break
for error in result.errors:
test_case, error_string = error
errors.append(error_string)
break
xunit.add_test(name=name,
status='fail',
time=time_taken,
errorType='membase.error',
errorMessage=str(errors),
params=params)
results.append({"result": "fail", "name": name})
else:
xunit.add_test(name=name, time=time_taken, params=params)
results.append({
"result": "pass",
"name": name,
"time": time_taken
})
xunit.write("{0}{2}report-{1}".format(os.path.dirname(logs_folder),
str_time, os.sep))
xunit.print_summary()
print(("testrunner logs, diags and results are available under {0}".
format(logs_folder)))
case_number += 1
if (result.failures or result.errors) and \
TestInputSingleton.input.param("stop-on-failure", False):
print("test fails, all of the following tests will be skipped!!!")
break
print("\n*** Tests executed count: {}\n".format(test_exec_count))
if test_exec_count > 0:
after_suite_name = "%s.%s" % (name[:name.rfind('.')], AFTER_SUITE)
try:
print(("Run after suite setup for %s" % name))
suite = unittest.TestLoader().loadTestsFromName(after_suite_name)
result = unittest.TextTestRunner(verbosity=2).run(suite)
except AttributeError as ex:
pass
if "makefile" in TestInputSingleton.input.test_params:
# print out fail for those tests which failed and do sys.exit() error code
fail_count = 0
for result in results:
if result["result"] == "fail":
print((result["name"], " fail "))
fail_count += 1
else:
print((result["name"], " pass"))
if fail_count > 0:
sys.exit(1)
print("During the test, Remote Connections: %s, Disconnections: %s" %
(RemoteMachineShellConnection.connections,
RemoteMachineShellConnection.disconnections))
if TestInputSingleton.input.param("get-delays", False):
sd.stop_measure_sched_delay()
sd.fetch_logs()
# terminate any non main thread - these were causing hangs
for t in threading.enumerate():
if t.name != 'MainThread' and t.is_alive():
print(('Thread', t,
'was not properly terminated, will be terminated now.'))
if hasattr(t, 'shutdown'):
print("Shutting down the thread...")
t.shutdown(True)
else:
print("Stopping the thread...")
try:
t._stop()
except Exception as e:
pass
return results, xunit, "{0}{2}report-{1}".format(
os.path.dirname(logs_folder), str_time, os.sep)
from gui.gui import *
import pmuThreads
import pyshark
from ptpSniffer import ptpSniffer, ptpPacketData
from threading import Thread
from time import sleep
from datetime import datetime
from hanging_threads import start_monitoring
monitoring_thread = start_monitoring()
class PMUrun(Thread):
def __init__(self, pmuid, pmuip, port, buffsize, setTS):
self.pmu_id = pmuid
self.pmu_ip = pmuip
self.port = port
self.buff_size = buffsize
self.set_TS = setTS
self.queue = queue
Thread.__init__(self)
self.daemon = True
self.output = None
# self.start()
def run(self):
print("Starting PMU " + str(self.pmu_id) + "\n")
pmuThreads.pmuThread(self.pmu_id, self.pmu_ip, self.port, self.buff_size, self.set_TS)
class PDCrun(Thread):
# noinspection PyUnresolvedReferences
from mozprocess import processhandler
# XXX We want to convince ourselves that the race condition that importing
# hanging_threads.py works around, tracked by bug 1046873, is gone, and get
# rid of this inclusion entirely before we hook our functional tests up to
# Tbpl, so that we don't introduce an intermittent failure.
import sys
import os
sys.path.append(os.path.dirname(__file__))
import hanging_threads
from config import CONTENT_SERVER_PORT, CONTENT_SERVER_URL, LOOP_SERVER_PORT, LOOP_SERVER_URL, \
FIREFOX_PREFERENCES, TEST_SERVER, USE_LOCAL_STANDALONE
hanging_threads.start_monitoring()
WORKING_DIR = os.getcwd()
CONTENT_SERVER_COMMAND = ["make", "runserver_nowatch"]
CONTENT_SERVER_ENV = os.environ.copy()
# Set PORT so that it does not interfere with any other
# development server that might be running
CONTENT_SERVER_ENV.update({"PORT": str(CONTENT_SERVER_PORT),
"LOOP_SERVER_URL": LOOP_SERVER_URL})
ROOMS_WEB_APP_URL_BASE = CONTENT_SERVER_URL
ROOMS_WEB_APP_URL = ROOMS_WEB_APP_URL_BASE + \
"/{token}"
from layout.annotator.annotation import Annotator, AnnotationQueueRest
from layout.upload_annotation.upload_annotation import UploadAnnotator
from layout.upload_annotation.upload_annotation import UploadAnnotator
from layout.annotation_thread import layout_annotate_train_model, UploadAnnotationQueueRest, sample_pipe, model_pipe, \
upload_pipe, full_model_path
from language.PredictionAlignment2Css import PredictionAlignment2Css
from layout.Layout2ReadingOrder import Layout2ReadingOrder
from language.transformer.ElmoDifference import ElmoDifference, ElmoDifferenceQueueRest, elmo_difference_pipe, \
elmo_difference_model_pipe, annotate_difference_elmo
# from language.heuristic.heuristic_difference import HeurisiticalLogician
from hanging_threads import start_monitoring
monitoring_thread = start_monitoring(seconds_frozen=30, test_interval=1000)
def get_all_routes(api):
routes_list = []
def get_children(node):
if len(node.children):
for child_node in node.children:
get_children(child_node)
else:
routes_list.append((node.uri_template, node.resource))
[get_children(node) for node in api._router._roots]
return routes_list
# noinspection PyUnresolvedReferences
from mozprocess import processhandler
# XXX We want to convince ourselves that the race condition that importing
# hanging_threads.py works around, tracked by bug 1046873, is gone, and get
# rid of this inclusion entirely before we hook our functional tests up to
# Tbpl, so that we don't introduce an intermittent failure.
import sys
import os
sys.path.append(os.path.dirname(__file__))
import hanging_threads
from config import CONTENT_SERVER_PORT, CONTENT_SERVER_URL, LOOP_SERVER_PORT, LOOP_SERVER_URL, \
FIREFOX_PREFERENCES, TEST_SERVER, USE_LOCAL_STANDALONE
hanging_threads.start_monitoring()
WORKING_DIR = os.getcwd()
CONTENT_SERVER_COMMAND = ["make", "runserver_nowatch"]
CONTENT_SERVER_ENV = os.environ.copy()
# Set PORT so that it does not interfere with any other
# development server that might be running
CONTENT_SERVER_ENV.update({
"PORT": str(CONTENT_SERVER_PORT),
"LOOP_SERVER_URL": LOOP_SERVER_URL
})
ROOMS_WEB_APP_URL_BASE = CONTENT_SERVER_URL
ROOMS_WEB_APP_URL = ROOMS_WEB_APP_URL_BASE + \
import argparse
import asyncio
import itertools
import sys
import time
from typing import Iterator, List
import aiohttp
from mtgjson4 import mtg_builder, mtg_global, mtg_storage
try:
import hanging_threads
THREAD_MONITOR = hanging_threads.start_monitoring()
except ImportError:
print(
'INFO: hanging_threads not installed - Thread Monitor is not running.')
async def main(loop: asyncio.AbstractEventLoop, session: aiohttp.ClientSession,
args: dict) -> None:
"""
Main method that starts the entire build process
:param args:
:param loop:
:param session:
:return:
"""
def get_next_batch_of_sets(queue: Iterator[List[str]]) -> List[List[str]]:
"""
To ensure better performance, we limit the number of sets built at a time
import time
from hanging_threads import start_monitoring
def sleep(t):
time.sleep(t)
if __name__ == '__main__':
print("Starting the deadlocks monitoring")
monitoring_thread = start_monitoring(seconds_frozen=1)
print("Sleep 3 seconds in custom func")
sleep(3)
print("Sleep 3 seconds")
time.sleep(3)
print("Sleep 3 seconds")
time.sleep(3)
print("Stopping the deadlocks monitoring")
# This may be useful when working in shell.
monitoring_thread.stop()
print("Sleep 3 seconds")
time.sleep(3)
print("Sleep 3 seconds")
time.sleep(3)
print("Exiting")
