def start_workload(self, parallel_threads, **kwargs): start_monitoring(seconds_frozen=60) workload_perc = {} workloads = kwargs.pop("workload").split(",") wperc = 100 / len(workloads) for workload in workloads: wload = workload.split(":") if len(wload) == 1: workload_perc[wload[0]] = wperc continue workload_perc[wload[0]] = int(wload[1]) quota = {} for workload_type, workload_per in workload_perc.iteritems(): quota[workload_type] = {"perc": workload_per} self._task_manager = parallel_execution(parallel_threads, quotas=quota) self._prepare_stage(**kwargs) self._exit_marker = self._create_exit_marker() self._start_and_monitor(workload_perc.keys(), **kwargs)
def stop_gracefully(*_, **__): global signal_call_counter, trace_threads, monitoring_thread signal_call_counter += 1 logger = logging.getLogger(__name__) if trace_threads: from hanging_threads import start_monitoring if signal_call_counter == 1: logger.critical( "Thread monitoring is started. " "Hanging threads will be reported every 10 seconds.\n" "If you see a thread keeps showing for multiple " "occurrences, please report it to the developer.\n\n" "Press Ctrl-C again to stop reporting, " "press another 3 times to force quit EFB." ) monitoring_thread = start_monitoring() elif monitoring_thread is not None and monitoring_thread.is_alive(): monitoring_thread.stop() logger.critical( "Thread report is now stopped, " "press Ctrl-C for 3 times to force quit EFB." ) if signal_call_counter >= MAX_SIG_CALL_BEFORE_FORCE_EXIT: logger.error( "5 consequent exit signals detected.\n" "Force exiting EFB without cleaning up.\n" "\n" "If it has taken you too long to quit EFB, " "it is most likely a bug with EFB or some of the modules enabled. " "Please consider tracing the hanging thread using --trace-threads " "argument, and report a bug to the developers." ) exit(1) # Stop the main thread from blocking. if not exit_event.is_set(): exit_event.set() # Wait for channels to stop polling. if hasattr(coordinator, "master") and isinstance(coordinator.master, MasterChannel): coordinator.master.stop_polling() logger.debug("Stop signal sent to master: %s" % coordinator.master.channel_name) else: logger.info("Valid master channel is not found.") for i in coordinator.slaves: if isinstance(coordinator.slaves[i], SlaveChannel): coordinator.slaves[i].stop_polling() logger.debug("Stop signal sent to slave: %s" % coordinator.slaves[i].channel_name) if coordinator.master_thread and coordinator.master_thread.is_alive(): coordinator.master_thread.join() for i in coordinator.slave_threads.values(): if i.is_alive(): i.join()
def daemon(): starts = [int(x) for x in sys.argv[1:]] threads = [] mv = MasterVault() for start in starts: def mv_thread(): print("thread open") mv.scrape_back(start, start) print("thread close") t = threading.Thread(target=mv_thread) threads.append(t) t.start() #t = threading.Thread(target=workers.thread) #threads.append(t) #t.start() print("start monitoring") monitoring_thread = start_monitoring(seconds_frozen=30)
def runtests(names, options, arg_i, arg_p, runtime_test_params): log.info("\nNumber of tests initially selected before GROUP filters: " + str(len(names))) BEFORE_SUITE = "suite_setUp" AFTER_SUITE = "suite_tearDown" xunit = XUnitTestResult() # Create root logs directory abs_path = os.path.dirname(os.path.abspath(sys.argv[0])) # Create testrunner logs subdirectory str_time = time.strftime("%y-%b-%d_%H-%M-%S", time.localtime()) root_log_dir = os.path.join( abs_path, "logs{0}testrunner-{1}".format(os.sep, str_time)) if not os.path.exists(root_log_dir): os.makedirs(root_log_dir) results = [] case_number = 1 if "GROUP" in runtime_test_params: print(("Only cases in GROUPs '{0}' will be executed".format( runtime_test_params["GROUP"]))) if "EXCLUDE_GROUP" in runtime_test_params: print(("Cases from GROUPs '{0}' will be excluded".format( runtime_test_params["EXCLUDE_GROUP"]))) if TestInputSingleton.input.param("get-delays", False): # start measure_sched_delays on all servers sd = SchedDelays(TestInputSingleton.input.servers) sd.start_measure_sched_delays() if TestInputSingleton.input.param("hanging_threads", False): print("--> hanging_threads: start monitoring...") from hanging_threads import start_monitoring hanging_threads_frozen_time = int( TestInputSingleton.input.param("hanging_threads", 120)) hanging_threads_test_interval = int( TestInputSingleton.input.param("test_interval", 1000)) monitoring_thread = start_monitoring( seconds_frozen=hanging_threads_frozen_time, test_interval=hanging_threads_test_interval) logs_folder = "." test_exec_count = 0 for name in names: start_time = time.time() argument_split = [ a.strip() for a in re.split("[,]?([^,=]+)=", name)[1:] ] params = dict(list(zip(argument_split[::2], argument_split[1::2]))) # Note that if ALL is specified at runtime then tests which have no groups are still run - just being # explicit on this if "GROUP" in runtime_test_params and "ALL" not in runtime_test_params[ "GROUP"].split(";"): if 'GROUP' not in params: # params is the .conf file parameters. # this test is not in any groups so we do not run it print(( "test '{0}' skipped, a group was requested and this is not any groups" .format(name))) continue # there is a group for this test case, if that group is not specified at run time then do not run it elif not set(runtime_test_params["GROUP"].split(";")).issubset( set(params["GROUP"].split(";"))): print(("test '{0}' skipped, is not in the requested group". format(name))) continue else: pass # the test was in requested group, will run it elif "EXCLUDE_GROUP" in runtime_test_params: if 'GROUP' in params and \ set(runtime_test_params["EXCLUDE_GROUP"].split(";")).issubset(set(params["GROUP"].split(";"))): print( ("test '{0}' skipped, is in an excluded group".format(name) )) continue log.info("--> Running test: {}".format(name)) test_exec_count += 1 # Create Log Directory logs_folder = os.path.join(root_log_dir, "test_%s" % case_number) log.info("Logs folder: {}".format(logs_folder)) os.mkdir(logs_folder) test_log_file = os.path.join(logs_folder, "test.log") log_config_filename = r'{0}'.format( os.path.join(logs_folder, "test.logging.conf")) create_log_file(log_config_filename, test_log_file, options.loglevel) logging.config.fileConfig(log_config_filename) print(("Logs will be stored at {0}".format(logs_folder))) print(("\n.{3}testrunner -i {0} -p {1} -t {2}\n"\ .format(arg_i or "", arg_p or "", name, os.sep))) name = name.split(",")[0] # Update the test params for each test TestInputSingleton.input.test_params = params TestInputSingleton.input.test_params.update(runtime_test_params) TestInputSingleton.input.test_params["case_number"] = case_number TestInputSingleton.input.test_params["logs_folder"] = logs_folder print("Test Input params:") print((TestInputSingleton.input.test_params)) if "get-coredumps" in TestInputSingleton.input.test_params: if TestInputSingleton.input.param("get-coredumps", True): clear_old_core_dumps(TestInputSingleton.input, logs_folder) if case_number == 1: before_suite_name = "%s.%s" % (name[:name.rfind('.')], BEFORE_SUITE) try: print(("Run before suite setup for %s" % name)) suite = unittest.TestLoader().loadTestsFromName( before_suite_name) print(("-->before_suite_name:{},suite: {}".format( before_suite_name, suite))) result = unittest.TextTestRunner(verbosity=2).run(suite) print(("-->result: {}".format(result))) if "get-coredumps" in TestInputSingleton.input.test_params: if TestInputSingleton.input.param("get-coredumps", True): if get_core_dumps(TestInputSingleton.input, logs_folder): result = unittest.TextTestRunner( verbosity=2)._makeResult() result.errors = [ (name, "Failing test : new core dump(s) " "were found and collected." " Check testrunner logs folder.") ] log.info( "FAIL: New core dump(s) was found and collected" ) except AttributeError as ex: traceback.print_exc() pass try: suite = unittest.TestLoader().loadTestsFromName(name) except AttributeError as e: print(("Test {0} was not found: {1}".format(name, e))) result = unittest.TextTestRunner(verbosity=2)._makeResult() result.errors = [(name, str(e))] except SyntaxError as e: print(("SyntaxError in {0}: {1}".format(name, e))) result = unittest.TextTestRunner(verbosity=2)._makeResult() result.errors = [(name, str(e))] else: test_timeout = TestInputSingleton.input.param("test_timeout", None) t = StoppableThreadWithResult( target=unittest.TextTestRunner(verbosity=2).run, name="test_thread", args=(suite)) t.start() result = t.join(timeout=test_timeout) if "get-coredumps" in TestInputSingleton.input.test_params: if TestInputSingleton.input.param("get-coredumps", True): if get_core_dumps(TestInputSingleton.input, logs_folder): result = unittest.TextTestRunner( verbosity=2)._makeResult() result.errors = [(name, "Failing test : new core dump(s) " "were found and collected." " Check testrunner logs folder.")] log.info( "FAIL: New core dump(s) was found and collected") if not result: for t in threading.enumerate(): if t != threading.current_thread(): t._Thread__stop() result = unittest.TextTestRunner(verbosity=2)._makeResult() case_number += 1000 print("========TEST WAS STOPPED DUE TO TIMEOUT=========") result.errors = [(name, "Test was stopped due to timeout")] time_taken = time.time() - start_time # Concat params to test name # To make tests more readable params = '' if TestInputSingleton.input.test_params: for key, value in list( TestInputSingleton.input.test_params.items()): if key and value: params += "," + str(key) + ":" + str(value) if result.failures or result.errors: # Immediately get the server logs, if # the test has failed or has errors if "get-logs" in TestInputSingleton.input.test_params: get_server_logs(TestInputSingleton.input, logs_folder) if "get-logs-cluster-run" in TestInputSingleton.input.test_params: if TestInputSingleton.input.param("get-logs-cluster-run", True): # Generate path to ns_server directory ns_server_path = os.path.normpath(abs_path + os.sep + os.pardir + os.sep + "ns_server") get_logs_cluster_run(TestInputSingleton.input, logs_folder, ns_server_path) if "get-cbcollect-info" in TestInputSingleton.input.test_params: if TestInputSingleton.input.param("get-cbcollect-info", True): get_cbcollect_info(TestInputSingleton.input, logs_folder) if "get-couch-dbinfo" in TestInputSingleton.input.test_params and \ TestInputSingleton.input.param("get-couch-dbinfo", True): get_couch_dbinfo(TestInputSingleton.input, logs_folder) errors = [] for failure in result.failures: test_case, failure_string = failure errors.append(failure_string) break for error in result.errors: test_case, error_string = error errors.append(error_string) break xunit.add_test(name=name, status='fail', time=time_taken, errorType='membase.error', errorMessage=str(errors), params=params) results.append({"result": "fail", "name": name}) else: xunit.add_test(name=name, time=time_taken, params=params) results.append({ "result": "pass", "name": name, "time": time_taken }) xunit.write("{0}{2}report-{1}".format(os.path.dirname(logs_folder), str_time, os.sep)) xunit.print_summary() print(("testrunner logs, diags and results are available under {0}". format(logs_folder))) case_number += 1 if (result.failures or result.errors) and \ TestInputSingleton.input.param("stop-on-failure", False): print("test fails, all of the following tests will be skipped!!!") break print("\n*** Tests executed count: {}\n".format(test_exec_count)) if test_exec_count > 0: after_suite_name = "%s.%s" % (name[:name.rfind('.')], AFTER_SUITE) try: print(("Run after suite setup for %s" % name)) suite = unittest.TestLoader().loadTestsFromName(after_suite_name) result = unittest.TextTestRunner(verbosity=2).run(suite) except AttributeError as ex: pass if "makefile" in TestInputSingleton.input.test_params: # print out fail for those tests which failed and do sys.exit() error code fail_count = 0 for result in results: if result["result"] == "fail": print((result["name"], " fail ")) fail_count += 1 else: print((result["name"], " pass")) if fail_count > 0: sys.exit(1) print("During the test, Remote Connections: %s, Disconnections: %s" % (RemoteMachineShellConnection.connections, RemoteMachineShellConnection.disconnections)) if TestInputSingleton.input.param("get-delays", False): sd.stop_measure_sched_delay() sd.fetch_logs() # terminate any non main thread - these were causing hangs for t in threading.enumerate(): if t.name != 'MainThread' and t.is_alive(): print(('Thread', t, 'was not properly terminated, will be terminated now.')) if hasattr(t, 'shutdown'): print("Shutting down the thread...") t.shutdown(True) else: print("Stopping the thread...") try: t._stop() except Exception as e: pass return results, xunit, "{0}{2}report-{1}".format( os.path.dirname(logs_folder), str_time, os.sep)
from gui.gui import * import pmuThreads import pyshark from ptpSniffer import ptpSniffer, ptpPacketData from threading import Thread from time import sleep from datetime import datetime from hanging_threads import start_monitoring monitoring_thread = start_monitoring() class PMUrun(Thread): def __init__(self, pmuid, pmuip, port, buffsize, setTS): self.pmu_id = pmuid self.pmu_ip = pmuip self.port = port self.buff_size = buffsize self.set_TS = setTS self.queue = queue Thread.__init__(self) self.daemon = True self.output = None # self.start() def run(self): print("Starting PMU " + str(self.pmu_id) + "\n") pmuThreads.pmuThread(self.pmu_id, self.pmu_ip, self.port, self.buff_size, self.set_TS) class PDCrun(Thread):
# noinspection PyUnresolvedReferences from mozprocess import processhandler # XXX We want to convince ourselves that the race condition that importing # hanging_threads.py works around, tracked by bug 1046873, is gone, and get # rid of this inclusion entirely before we hook our functional tests up to # Tbpl, so that we don't introduce an intermittent failure. import sys import os sys.path.append(os.path.dirname(__file__)) import hanging_threads from config import CONTENT_SERVER_PORT, CONTENT_SERVER_URL, LOOP_SERVER_PORT, LOOP_SERVER_URL, \ FIREFOX_PREFERENCES, TEST_SERVER, USE_LOCAL_STANDALONE hanging_threads.start_monitoring() WORKING_DIR = os.getcwd() CONTENT_SERVER_COMMAND = ["make", "runserver_nowatch"] CONTENT_SERVER_ENV = os.environ.copy() # Set PORT so that it does not interfere with any other # development server that might be running CONTENT_SERVER_ENV.update({"PORT": str(CONTENT_SERVER_PORT), "LOOP_SERVER_URL": LOOP_SERVER_URL}) ROOMS_WEB_APP_URL_BASE = CONTENT_SERVER_URL ROOMS_WEB_APP_URL = ROOMS_WEB_APP_URL_BASE + \ "/{token}"
from layout.annotator.annotation import Annotator, AnnotationQueueRest from layout.upload_annotation.upload_annotation import UploadAnnotator from layout.upload_annotation.upload_annotation import UploadAnnotator from layout.annotation_thread import layout_annotate_train_model, UploadAnnotationQueueRest, sample_pipe, model_pipe, \ upload_pipe, full_model_path from language.PredictionAlignment2Css import PredictionAlignment2Css from layout.Layout2ReadingOrder import Layout2ReadingOrder from language.transformer.ElmoDifference import ElmoDifference, ElmoDifferenceQueueRest, elmo_difference_pipe, \ elmo_difference_model_pipe, annotate_difference_elmo # from language.heuristic.heuristic_difference import HeurisiticalLogician from hanging_threads import start_monitoring monitoring_thread = start_monitoring(seconds_frozen=30, test_interval=1000) def get_all_routes(api): routes_list = [] def get_children(node): if len(node.children): for child_node in node.children: get_children(child_node) else: routes_list.append((node.uri_template, node.resource)) [get_children(node) for node in api._router._roots] return routes_list
# noinspection PyUnresolvedReferences from mozprocess import processhandler # XXX We want to convince ourselves that the race condition that importing # hanging_threads.py works around, tracked by bug 1046873, is gone, and get # rid of this inclusion entirely before we hook our functional tests up to # Tbpl, so that we don't introduce an intermittent failure. import sys import os sys.path.append(os.path.dirname(__file__)) import hanging_threads from config import CONTENT_SERVER_PORT, CONTENT_SERVER_URL, LOOP_SERVER_PORT, LOOP_SERVER_URL, \ FIREFOX_PREFERENCES, TEST_SERVER, USE_LOCAL_STANDALONE hanging_threads.start_monitoring() WORKING_DIR = os.getcwd() CONTENT_SERVER_COMMAND = ["make", "runserver_nowatch"] CONTENT_SERVER_ENV = os.environ.copy() # Set PORT so that it does not interfere with any other # development server that might be running CONTENT_SERVER_ENV.update({ "PORT": str(CONTENT_SERVER_PORT), "LOOP_SERVER_URL": LOOP_SERVER_URL }) ROOMS_WEB_APP_URL_BASE = CONTENT_SERVER_URL ROOMS_WEB_APP_URL = ROOMS_WEB_APP_URL_BASE + \
import argparse import asyncio import itertools import sys import time from typing import Iterator, List import aiohttp from mtgjson4 import mtg_builder, mtg_global, mtg_storage try: import hanging_threads THREAD_MONITOR = hanging_threads.start_monitoring() except ImportError: print( 'INFO: hanging_threads not installed - Thread Monitor is not running.') async def main(loop: asyncio.AbstractEventLoop, session: aiohttp.ClientSession, args: dict) -> None: """ Main method that starts the entire build process :param args: :param loop: :param session: :return: """ def get_next_batch_of_sets(queue: Iterator[List[str]]) -> List[List[str]]: """ To ensure better performance, we limit the number of sets built at a time
import time from hanging_threads import start_monitoring def sleep(t): time.sleep(t) if __name__ == '__main__': print("Starting the deadlocks monitoring") monitoring_thread = start_monitoring(seconds_frozen=1) print("Sleep 3 seconds in custom func") sleep(3) print("Sleep 3 seconds") time.sleep(3) print("Sleep 3 seconds") time.sleep(3) print("Stopping the deadlocks monitoring") # This may be useful when working in shell. monitoring_thread.stop() print("Sleep 3 seconds") time.sleep(3) print("Sleep 3 seconds") time.sleep(3) print("Exiting")