class SubProcessWrapper: cname = __name__ + '.SubProcessWrapper' def __init__(self, target, name=None): self.target = target self.running = False self.name = name if name else target.task_name() self.kill_event = Event() self.logger = logging.getLogger(self.cname) def run(self): self.logger.info("starting SubProcessTask: {}".format(self.target.task_name())) th = Thread(target=self.target, name=self.target.task_name()) th.start() signal.signal(signal.SIGINT, signal.SIG_IGN) self.kill_event.wait() self.logger.info("stopping SubProcessTask: {}".format(self.target.task_name())) self.target.stop() th.join() self.logger.info("Stopped SubProcessTask: {}".format(self.target.task_name())) def __call__(self): self.run() def get_kill_event(self): return self.kill_event
def execute_action(self, action): event = Event() queue = Queue() proc = Process( target=execute_action_proc, args=(self.execute, action, event, queue)) proc.start() # Send heartbeat. heartbeat_retry = 0 while not event.is_set(): event.wait(config.ACTIVITY_HEARTBEAT_INTERVAL) try: res = self.heartbeat(self.task_token) if res['cancelRequested']: proc.terminate() proc.join() return Result('cancelled', -1, '', '', '', -1) except Exception as err: if heartbeat_retry <= config.ACTIVITY_HEARTBEAT_MAX_RETRY: heartbeat_retry += 1 continue else: proc.terminate() proc.join() raise # Evaluate the result. result = queue.get_nowait() proc.join() return result
def test_play_and_record(self): """ Verifies that a Device and play back prerecorded events. """ device = evemu.Device(self.get_device_file()) devnode = device.devnode events_file = self.get_events_file() # device.record() calls evemu_record() and is thus missing the # description that the input file has with open(events_file) as e: indata = extract_events(strip_comments(e.readlines())) recording_started = Event() q = Queue() record_process = Process(target=record, args=(recording_started, devnode, q)) record_process.start() recording_started.wait(100) device.play(open(events_file)) outdata = strip_comments(q.get()) record_process.join() self.assertEquals(len(indata), len(outdata)) fuzz = re.compile("E: \d+\.\d+ (.*)") for i in range(len(indata)): lhs = fuzz.match(indata[i]) self.assertTrue(lhs) rhs = fuzz.match(outdata[i]) self.assertTrue(rhs) self.assertEquals(lhs.group(1), rhs.group(1))
class ChildChecker(threading.Thread): def __init__(self, killEvent): super(ChildChecker, self).__init__() self.killEvent = killEvent self.event = Event() self.process = Process(target=childsPlay, args=(self.event,)) def run(self): self.process.start() while not self.killEvent.is_set(): self.event.wait() print "Child checked, and is done playing" if raw_input("Do again? y/n:") == "y": self.event.clear() self.process = Process(target=endlessChildsPlay, args=(self.event,)) self.process.start() else: self.cleanChild() self.killEvent.set() def join(self): print "Joining child process" # Timeout on 5 seconds self.process.join(5) if self.process.is_alive(): print "Child did not join! Killing.." self.process.terminate() print "Joining ChildChecker thread" super(ChildChecker, self).join() def cleanChild(self): print "Cleaning up the child..."
def test_host_is_down(self): from gaw import interface_class, service_class, client_class server_start = Event() @interface_class(service_name='ServiceInterfaceStyle') class Interface(object): def ping(self): pass def server(): @service_class class Service(Interface): def ping(self): return True GawServer(ip='0.0.0.0', port=4000).add(Service).run(lambda: server_start.set()) def client(): @client_class(ip='localhost', port=4000, retries=0) # do not retry class Client(Interface): pass self.assertRaises(Exception, Client().ping) p = Process(target=server) p.start() server_start.wait() p.terminate() p.join() client()
def pipelineDaemon(pipeline, returnEvent, options=None, programName=None): """Launches Pyro server and (if specified by options) pipeline executors""" #check for valid pipeline if pipeline.runnable.empty()==None: print "Pipeline has no runnable stages. Exiting..." sys.exit() if options.urifile==None: options.urifile = os.path.abspath(os.curdir + "/" + "uri") e = Event() process = Process(target=launchServer, args=(pipeline,options,e,)) process.start() e.wait() if options.num_exec != 0: processes = [Process(target=launchPipelineExecutor, args=(options,programName,)) for i in range(options.num_exec)] for p in processes: p.start() for p in processes: p.join() #Return to calling code if pipeline has no more runnable stages: #Event will be cleared once clients are unregistered. while e.is_set(): time.sleep(5) returnEvent.set()
class ServerProc(object): def __init__(self): self.proc = None self.daemon = None self.stop = Event() def start(self, init_func, config, paths, port): self.proc = Process(target=self.create_daemon, args=(init_func, config, paths, port)) self.proc.daemon = True self.proc.start() def create_daemon(self, init_func, config, paths, port): try: self.daemon = init_func(config, paths, port) except socket.error: logger.error("Socket error on port %s" % port) raise if self.daemon: self.daemon.start(block=False) try: self.stop.wait() except KeyboardInterrupt: pass def wait(self): self.stop.set() self.proc.join() def kill(self): self.stop.set() self.proc.terminate() self.proc.join()
class StoppableProcess(Process): exit = None sleep = None def __init__(self, sleep=1, *args, **kwargs): self.exit = Event() self.sleep = sleep super(StoppableProcess, self).__init__(*args, **kwargs) def _setup(self): pass def _teardown(self): pass def _ping(self): raise NotImplementedError def _should_exit(self): return self.exit.wait(0) def run(self): self._setup() while True: if self._ping() or self.exit.wait(self.sleep * 1.0): self._teardown() return def stop(self): self.exit.set() self.join(self.sleep) if self.is_alive(): self.terminate()
def test_multiple_concurrent_request_on_same_client(self): import time from multiprocessing.pool import ThreadPool send_msg = u'test ฟนำีฟนำีฟนำีฟนำี' server_start = Event() def on_message(msg): self.assertEqual(msg, send_msg) def server(): PostofficeServer(ip='0.0.0.0', port=4000, on_message=on_message, after_start_cb=lambda: server_start.set()) from multiprocessing import Process p = Process(target=server) try: p.start() server_start.wait() c = PostofficeClient(ip='localhost', port=4000) def client(ith): c.send(send_msg) pool = ThreadPool(100) pool.map(client, [i for i in range(1000)]) p.terminate() p.join() except Exception as e: # gracefully stop p.terminate() p.join() raise
def test_server_client(self): import time send_msg = u'test ฟนำีฟนำีฟนำีฟนำี' server_start = Event() def on_message(msg): self.assertEqual(msg, send_msg) return msg def server(): PostofficeServer(ip='0.0.0.0', port=4000, on_message=on_message, after_start_cb=lambda: server_start.set()) def client(): client = PostofficeClient(ip='localhost', port=4000) response = client.send(send_msg) self.assertEqual(response, send_msg) from multiprocessing import Process p = Process(target=server) try: p.start() server_start.wait() client() p.terminate() p.join() except Exception as e: # gracefully stop p.terminate() p.join() raise
def test_host_is_down(self): from multiprocessing import Event, Process server_start = Event() def a(path): return path def b(path, a, b): return a + b def c(path, a, b): return a * b def server(): server = JsonSocketServer(ip='0.0.0.0', port=4000, verbose=True) server.register_route(u'a ฟนำี', a) server.register_route('b', b) server.register_route('c', c) server.start(lambda: server_start.set()) def client(): client = JsonSocketClient(verbose=True) self.assertRaises(Exception, client.request, ip='localhost', port=4000, path=u'a ฟนำี', payload=dict(), retries=0) p = Process(target=server) p.start() server_start.wait() p.terminate() p.join() client()
def wait_for(self, key, value): d = Manager().dict() d[key] = None v = Manager().Value('s', ' ') e = Event() p_state = Process(target=self.state_refresher, args=(self, d, e)) p_input = Process(target=self.input_waiter, args=(self, v, e)) p_state.start() p_input.start() while v.value != 'exit' and dict(d.items())[key] != value: e.wait() e.clear() self.state = d['state'] self.current = d['current'] self.enemy = d['enemy'] self.battlefield = d['battlefield'] p_state.terminate() p_input.terminate() curses.endwin() p_state.join() p_input.join() return True if dict(d.items())[key] == value else False
def throughput(seconds=10,cocurrency=1): ''' seconds should be greater than or equal to 10 1000w pv = 115 rps ''' stop_flag=Event() processes=[] t=Timer(seconds,stop,args=[stop_flag]) q = Queue() for i in range(cocurrency): processes.append(Process(target=run,args=(q,stop_flag))) t.start() for p in processes: p.start() #print 'start waiting for workers:',len(processes) stop_flag.wait() for t in processes: t.join() total=err=cost=0 while not q.empty(): (req_counter,err_counter,time_cost)=q.get() total=total+req_counter err=err+err_counter cost=cost+time_cost cost=cost/total if total>0 else 0 return total,err,cost
class StoppableProcess(Process): """ Base class for Processes which require the ability to be stopped by a process-safe method call """ def __init__(self): self._should_stop = Event() self._should_stop.clear() super(StoppableProcess, self).__init__() def join(self, timeout=0): """ Joins the current process and forces it to stop after the timeout if necessary :param timeout: Timeout duration in seconds """ self._should_stop.wait(timeout) if not self.should_stop(): self.stop() super(StoppableProcess, self).join(0) def stop(self): self._should_stop.set() def should_stop(self): return self._should_stop.is_set() def __repr__(self): return "<%s(should_stop=%s)>" % ( self.__class__.__name__, self.should_stop())
class SharedFile(object): def __init__(self, filename): self.filename = filename self.fevent = Event() # self.state = Value('i', 0) self.fevent.set() def write(self, mode, data): # print("Write {}".format(inspect.stack()[1][3])) self.wait_freedom_and_lock() f = open(self.filename, mode) f.write(data) f.close self.unlock() def read(self): # print("Read {}".format(inspect.stack()[1][3])) self.wait_freedom_and_lock() f = open(self.filename, 'r') data = f.read() f.close self.unlock() return data def wait_freedom_and_lock(self): self.fevent.wait() self.fevent.clear() # return def unlock(self): self.fevent.set()
class TestProxyData(TestData): def setup(self): create_link('dummyX', 'dummy') t_url = 'unix://\0%s' % (uuid.uuid4()) p_url = 'unix://\0%s' % (uuid.uuid4()) self.connect = Event() self.release = Event() target = Process(target=_run_remote_uplink, args=(t_url, self.connect, self.release)) target.daemon = True target.start() self.connect.wait() self.connect.clear() proxy = Process(target=_run_remote_uplink, args=(p_url, self.connect, self.release)) proxy.daemon = True proxy.start() self.connect.wait() self.connect.clear() self.ip = IPRoute(do_connect=False) link, proxy = self.ip.connect(p_url) self.ip.register('bala', proxy) link, host = self.ip.connect(t_url, addr=proxy) service = self.ip.discover(self.ip.default_target, addr=host) self.ip.default_peer = host self.ip.default_dport = service self.dev = self.ip.link_lookup(ifname='dummyX')
def _start_async_server(self): self.server = LiveSyncSocketServer(port=self.liveport) server_started_event = Event() self.server_process = Process(target=self.server.start, args=(server_started_event,)) self.server_process.daemon = True self.server_process.start() server_started_event.wait(timeout=0.1) return server_started_event.is_set()
def start_server(self): #use a generic manager's shared dictionary to handle strings and ints manager = Manager() dictionary = manager.dict() dictionary["key"] = '' dictionary["finished chunks"] = 0 # update is an event intended to be set by server to let the UI know that the shared dictionary has been updated. update = Event() update.clear() # shutdown is linked to the server/client shared shutdown command, setting this should shutdown server and clients. shutdown = Event() shutdown.clear() # shared is just an array, each shared variable is added to allow passing all shared values using one variable. shared = [] shared.append(dictionary) shared.append(shutdown) shared.append(update) preset = 8 if self.settings[preset]["single"] == "False": network = True else: network = False self.server = Process(target=Server, args=(self.settings[preset], shared)) self.server.start() print "server pid: %i" % self.server.pid # this is a very simple example of how an update loop in the UI classes might work while not shutdown.is_set(): # update is an Event, this means that a process can use the .wait() command to block until it is .set() update.wait(timeout=.5) print "%i/%i chunks completed." % (dictionary["finished chunks"], dictionary["total chunks"]) print "Current word: %s" % dictionary["current word"] if preset > 8: print "%i hashes found." % dictionary["hashes found"] os.system('cls' if os.name == 'nt' else 'clear') if dictionary["key"] is not '': print "Printing key from start_server method: " + dictionary["key"] break #print "%d chunks completed of %i total." % (dictionary["finished chunks"], dictionary["total chunks"]) # after UI data has been updated, .clear() the update event so it will wait again in the next iteration update.clear() shutdown.set() time.sleep(1) #self.server.terminate() self.server.join() self.server.terminate() #os.kill(self.server.pid, signal.SIGKILL) self.server.join() manager.shutdown() if network: print self.server.pid
def test_multiple_service(self): from gaw import entrypoint server_start = Event() class A(object): name = 'A' @entrypoint def a(self): return 10 class B(object): name = 'B' @entrypoint def b(self): return 20 class Both(object): name = 'Both' @entrypoint def sum(self): client = GawClient(ip='localhost', port=4000) A = client.A B = client.B return A.a() + B.b() def server(): GawServer(ip='0.0.0.0', port=4000).add(A).add(B).add(Both).run(lambda: server_start.set()) def client(): conn = GawClient(ip='localhost', port=4000) a = conn.A b = conn.B both = conn.Both self.assertEqual(a.a(), 10) self.assertEqual(b.b(), 20) self.assertEqual(both.sum(), 30) p = Process(target=server) try: p.start() server_start.wait() client() p.terminate() p.join() except Exception as e: # gracefully stop p.terminate() p.join() raise
def _test_remote(self, url): connect = Event() release = Event() target = Process(target=_run_remote_uplink, args=(url, connect, release)) target.daemon = True target.start() connect.wait() ip = IPRoute(host=url) ip.release() release.set()
def main(): # Setup process pooling manager = Manager() # Make queue for processes to grab a thread number from # also used so that the last worker knows it's last. thread_number = manager.Queue() [thread_number.put(t) for t in range(getthreads())] # Set event for handling worker ready condition workers_ready = Event() workers_idle = Event() workers_idle.set() # Set event for handling daemon quit condition for stuff that doesnt exit clean daemon_exit = Event() # Set event for handling new file condition new_file = Event() # Setup workqueue queue = manager.Queue() print("Starting daemons") watcherd = Process(name="watcherd", target=watcher, daemon=True, args=(queue, new_file,)) watcherd.start() print("Daemons started.") print("Initializing workers...") workers = Pool(processes=thread_number.qsize(), initializer=initworker, initargs=(thread_number, workers_ready,)) workers_ready.wait() print("All workers ready") # Set signal handler on main process def exit_handler(signum, frame): cleanup(workers, daemon_exit, workers_idle) signal(SIGINT, exit_handler) signal(SIGTERM, exit_handler) # Main loop while not daemon_exit.is_set(): # wait until new file is on queue, forced check every 5 minutes. status = new_file.wait(timeout=300.0) if status: new_file.clear() workers_idle.clear() # Send to workers workers.map(compressors.dyncompress,[queue.get() for q in range(queue.qsize())]) workers_idle.set()
class EngineManager(ObjectsManager, Process): """Uses header values to handle directed requests and results.""" def __init__(self, *args, **kwargs): super(EngineManager, self).__init__(*args, **kwargs) self.manager = Manager() self.results = self.manager.dict() self.block = Event() def retrieve(self, jid=None, inputs=False, timeout=None, block=True): """Retrieve a job given a jid""" if jid is None: jid = self.cjid if block: self.block.wait() hdr, func, args, kwargs, rvalue = self.results[jid] self.block.clear() if inputs: return func, args, kwargs, rvalue else: return rvalue def ready(self, jid): """Check if a job id is ready.""" return jid in self.results def __getattr__(self, attr): """Call a method on the underlying threaded object""" def method(*args, **kwargs): """A threaded method""" jid = id((attr, args, kwargs)) self.cjid = jid self.hdr["jid"] = jid self.input.put((self.hdr, attr, args, kwargs)) return jid return method def start(self): """Start""" super(EngineManager, self).start() def run(self): """Run the subthread to move things off the output.""" running = True while running: hdr, func, args, kwargs, rvalue = self.output.get() if "stop" in hdr: running = False else: self.results[hdr["jid"]] = (hdr, func, args, kwargs, rvalue) self.block.set()
def processexecute(scenario_path, debug, nb_browser, launch_delay, keep_alive, random_time, headless): from multiprocessing import Process from multiprocessing import Event processes = [] fire_synchro_end = Event() for process_number in xrange(0, nb_browser): fire_instanciation_end = Event() process = Process(target=doinprocess, args=(scenario_path, debug, process_number, launch_delay, keep_alive, random_time, headless, fire_instanciation_end, fire_synchro_end,),name='scenario-' + process_number) fire_instanciation_end.wait() #todo add timeout fire_synchro_end.set(True) for process in processes: process.join()
def test_lock_multiproc(self): e = Event() @asyncio.coroutine def do_async_lock(): self.assertEqual(False, (yield from self.lock.coro_acquire(False))) self.assertEqual(True, (yield from self.lock.coro_acquire(timeout=4))) p = Process(target=do_lock_acquire, args=(self.lock, e)) p.start() e.wait() self.loop.run_until_complete(do_async_lock())
class TestRemoteData(TestData): def setup(self): create_link('dummyX', 'dummy') url = 'unix://\0%s' % (uuid.uuid4()) self.connect = Event() self.release = Event() target = Process(target=_run_remote_uplink, args=(url, self.connect, self.release)) target.daemon = True target.start() self.connect.wait() self.ip = IPRoute(host=url) self.dev = self.ip.link_lookup(ifname='dummyX')
class ServerProc(object): def __init__(self): self.proc = None self.daemon = None self.stop = Event() def start(self, init_func, host, port, paths, routes, bind_address, config, ssl_config, **kwargs): self.proc = Process(target=self.create_daemon, args=(init_func, host, port, paths, routes, bind_address, config, ssl_config), kwargs=kwargs) self.proc.daemon = True self.proc.start() def create_daemon(self, init_func, host, port, paths, routes, bind_address, config, ssl_config, **kwargs): try: self.daemon = init_func(host, port, paths, routes, bind_address, config, ssl_config, **kwargs) except socket.error: print("Socket error on port %s" % port, file=sys.stderr) raise except Exception: print(traceback.format_exc(), file=sys.stderr) raise if self.daemon: try: self.daemon.start(block=False) try: self.stop.wait() except KeyboardInterrupt: pass except Exception: print(traceback.format_exc(), file=sys.stderr) raise def wait(self): self.stop.set() self.proc.join() def kill(self): self.stop.set() self.proc.terminate() self.proc.join() def is_alive(self): return self.proc.is_alive()
def start_feedback(self, name, port): """Starts the given Feedback in a new process.""" self.logger.debug("Starting new Process...",) if self.currentProc: self.logger.warning("Trying to start feedback but another one is still running. Killing the old one now and proceed.") self.stop_feedback() ipcReady = Event() self.currentProc = FeedbackProcess(self.pluginController.availablePlugins[name], name, ipcReady, port) self.currentProc.start() # Wait until the network from the Process is ready, this is necessary # since spawning a new process under Windows is very slow. self.logger.debug("Waiting for IPC channel to become ready...") ipcReady.wait() self.logger.debug("IPC channel ready.") self.logger.debug("Done starting process.")
class ProcessWorker(Process, BaseWorker): def __init__(self, *args, **kwargs): Process.__init__(self) BaseWorker.__init__(self, *args, **kwargs) self.finish_event = Event() def start_work(self): self.start() def wait_work_finish(self): self.finish_event.wait() def run(self): self._internal_do_work() self.finish_event.set()
class ServerProc(object): def __init__(self): self.proc = None self.daemon = None self.stop = Event() def start(self, init_func, host, paths, port, bind_hostname, external_config, ssl_config): self.proc = Process(target=self.create_daemon, args=(init_func, host, paths, port, bind_hostname, external_config, ssl_config)) self.proc.daemon = True self.proc.start() def create_daemon(self, init_func, host, paths, port, bind_hostname, external_config, ssl_config): try: self.daemon = init_func(host, paths, port, bind_hostname, external_config, ssl_config) except socket.error: print >> sys.stderr, "Socket error on port %s" % port raise except: print >> sys.stderr, traceback.format_exc() raise if self.daemon: try: self.daemon.start(block=False) try: self.stop.wait() except KeyboardInterrupt: pass except: print >> sys.stderr, traceback.format_exc() raise def wait(self): self.stop.set() self.proc.join() def kill(self): self.stop.set() self.proc.terminate() self.proc.join() def is_alive(self): return self.proc.is_alive()
def test_get_put(self): q = aioprocessing.AioQueue() e = Event() val = 2 @asyncio.coroutine def queue_put(): yield from q.coro_put(val) p = Process(target=queue_get, args=(q, e)) p.start() self.loop.run_until_complete(queue_put()) e.wait() out = q.get() p.join() self.assertEqual(out, val)
def render_frame(data_q: Queue, finish_e: Event, renderer_provider): renderer = renderer_provider() while True: try: old_state, action, new_state, reward, done = data_q.get( timeout=0.1) renderer.plot(old_state, action, new_state, reward, done) renderer.render() except Empty: renderer.render() if finish_e.wait(0.1): break print("shut down online rendering !!!")
def test_deploy_package(self, os_mock): """ Tests asynchronous deploy-package process. """ package_callback_name = "package_callback" self.mock_config[package_callback_name] = package_callback_name self.mock_package_registar.self.package_exists = Mock(return_value=False) self.mock_repository.get_package.return_value = 'abcd' class MockDeploymentManager(DeploymentManager): def create_mocks(self): self._application_creator = Mock() self._package_parser = Mock() deployment_manager = self._initialize_deployment_manager(MockDeploymentManager) deployment_manager.create_mocks() deployment_manager.rest_client = Mock() # mock status interface: on_complete = Event() # holds the result of the test, false if test has not finished yet: test_result = [None] # listen for callbacks from the deployer: on_rest_callback = self._create_rest_callback_verifier(expected_statuses=[PackageDeploymentState.DEPLOYING, PackageDeploymentState.DEPLOYED], on_complete=on_complete, test_result=test_result) deployment_manager.rest_client.post = on_rest_callback self.mock_application_registar.set_application_status(self.test_app_name, ApplicationState.STARTED) # launch the asynch test deployment_manager.deploy_package(self.test_package_name) # wait for test to finsish on_complete.wait(5) self.assertIsNotNone(test_result[0], "async task completed") info = test_result[0] # check that error was reported: self.assertEquals(info.get("data")[0]["state"], PackageDeploymentState.DEPLOYED) self.assertFalse("Error deploying" in info.get("data")[0]["information"])
def start_feedback(self, name, port): """Starts the given Feedback in a new process. :param name: Feedback :type name: str :param port: Parallel Port """ self.logger.debug("Starting new Process...", ) if self.currentProc: self.logger.warning( "Trying to start feedback but another one is still running. Killing the old one now and proceed." ) self.stop_feedback() ipcReady = Event() self.currentProc = FeedbackProcess( self.pluginController.availablePlugins[name], name, ipcReady, port) self.currentProc.start() # Wait until the network from the Process is ready, this is necessary # since spawning a new process under Windows is very slow. self.logger.debug("Waiting for IPC channel to become ready...") ipcReady.wait() self.logger.debug("IPC channel ready.") self.logger.debug("Done starting process.")
def receive(self): def _receive(dispatcher: Dispatcher, initialized: Event) -> None: for k, v in self._performers.items(): event = Event() dispatcher._running_performers[k] = v.run(event) event.wait( 3) # TODO: Do we want to configure this polling interval? initialized.set() while self._state == DispatcherState.ds_running: time.sleep( 5 ) # yield to avoid spinning, between checking for changes to state if self._state == DispatcherState.ds_awaiting: initialized = Event() self._supervisor = Thread(target=_receive, args=(self, initialized)) initialized.wait( 5 ) # TODO: Should this be number of performs and configured with related? self._state = DispatcherState.ds_running self._supervisor.start()
def Trigger(tc): stop_ev = Event() err_ev = Event() fin_ev = Event() thr = Process(target=ahs_loop, args=(tc.RF, fin_ev, stop_ev, err_ev,)) thr.start() err = False try: for _iter in range(tc.iterators.count): if err_ev.is_set() is True: raise RuntimeError('AHS error') api.Logger.info('Run: %d' % _iter) api.Logger.info("Issuing server restart") ret = api.IpmiNodes([tc.node_name], ipmiMethod='cycle', useNcsi=True) if ret != api.types.status.SUCCESS: api.Logger.info("server restart failed") return api.types.status.FAILURE api.Logger.info("server restart done") tc.test_node.WaitForHost() if err_ev.is_set() is True: raise RuntimeError('AHS error') except: api.Logger.error(traceback.format_exc()) err = True finally: stop_ev.set() api.Logger.info("Waiting for AHS loop to complete") fin_ev.wait(timeout=10) thr.terminate() thr.join() if err: return api.types.status.FAILURE return api.types.status.SUCCESS
class GuiIF(): #用于向guiengine发射信号 def __init__(self, server_address=None, layout=layout): # Q_c2g:传递内容 # 1. 单字符串:'_q_u_i_t_' -> 结束标志 # 2. 列表:[stimulus setting, marker] -> 刺激设置,marker标志 self.Q_c2g = Queue() self.E_g2c = Event() self.layout = layout self.args = { 'layout': self.layout, 'Q_c2g': self.Q_c2g, 'E_g2c': self.E_g2c, 'server_address': server_address } def quit(self): self.Q_c2g.put('_q_u_i_t_') def wait(self): self.E_g2c.wait() def update(self, stimulus, marker): self.Q_c2g.put([stimulus, marker])
class StopWatch(Process): def __init__(self, hotkey=None): super().__init__(name='Stop Watch') self._stop_event = Event() self._stop_event.clear() self._waiter = Event() self._waiter.clear() self._elapsed_time = Value('Q', 0) self._hotkey = hotkey or 'space' super().start() # Waits till the process is started. self._stop_event.wait() def start(self): if self.is_alive: self._stop_event.set() else: raise RuntimeError() def join(self, *args, forceStop=False, **kwarks): zeroTimer = not self._waiter.is_set() if forceStop and self.is_alive(): self._waiter.set() super().join(*args, **kwarks) if zeroTimer: with self._elapsed_time.get_lock(): self._elapsed_time.value = 0 def getValue(self): return self._elapsed_time.value or None def run(self): s, e = None, None def hotkey_action(): if s is not None: self._waiter.set() remove = add_hotkey(self._hotkey, hotkey_action) # Inform the main process that this process is started self._stop_event.set() # Clears the stop event so that process halts till it is started again with start function. self._stop_event.clear() self._stop_event.wait() s = timer() self._waiter.wait() e = timer() with self._elapsed_time.get_lock(): self._elapsed_time.value = e - s remove_hotkey(remove)
class ProcessPeer(Peer, Process): def __init__( self, host: Tuple[str, int], name: str, role: str, cert: Tuple[str, str], program_hash: str, ns: str = None, loop_delay: int = 1, auto_register: bool = False, logger: "logging.Logger" = getLogger(__name__), ): super().__init__( host=host, name=name, role=role, cert=cert, program_hash=program_hash, ns=ns, auto_register=auto_register, logger=logger, ) self.loopDelay = loop_delay self.stopped = Event() self.started = Event() def is_start(self) -> bool: return self.started.is_set() def start(self) -> None: super().start() self.loop_start() self.started.set() def stop(self) -> None: self.loop_stop() self.stopped.set() self.started.clear() def run(self) -> None: while self.stopped.wait( self.loopDelay) is False or self.send_queue != {}: self.loop() self.loop_stop_post() sleep(2) self.logger.info("{} stopped.".format(self.server_info))
class TestTagPropagation(unittest.TestCase): def setUp(self): self._tmp_dir = TemporaryDirectory() self._config = initialize_config(self._tmp_dir) self.analysis_finished_event = Event() self.uid_of_key_file = '530bf2f1203b789bfe054d3118ebd29a04013c587efd22235b3b9677cee21c0e_2048' self._mongo_server = MongoMgr(config=self._config, auth=False) self.backend_interface = BackEndDbInterface(config=self._config) self._analysis_scheduler = AnalysisScheduler(config=self._config, pre_analysis=self.backend_interface.add_object, post_analysis=self.count_analysis_finished_event) self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.start_analysis_of_object) def count_analysis_finished_event(self, fw_object): self.backend_interface.add_analysis(fw_object) if fw_object.uid == self.uid_of_key_file and 'crypto_material' in fw_object.processed_analysis: sleep(1) self.analysis_finished_event.set() def tearDown(self): self._unpack_scheduler.shutdown() self._analysis_scheduler.shutdown() clean_test_database(self._config, get_database_names(self._config)) self._mongo_server.shutdown() self._tmp_dir.cleanup() gc.collect() def test_run_analysis_with_tag(self): test_fw = Firmware(file_path='{}/container/with_key.7z'.format(get_test_data_dir())) test_fw.release_date = '2017-01-01' test_fw.scheduled_analysis = ['crypto_material'] self._unpack_scheduler.add_task(test_fw) assert self.analysis_finished_event.wait(timeout=20) processed_fo = self.backend_interface.get_object(self.uid_of_key_file, analysis_filter=['crypto_material']) assert processed_fo.processed_analysis['crypto_material']['tags'], 'no tags set in analysis' processed_fw = self.backend_interface.get_object(test_fw.uid, analysis_filter=['crypto_material']) assert processed_fw.analysis_tags, 'tags not propagated properly' assert processed_fw.analysis_tags['crypto_material']['private_key_inside']
class SafeEvent(object): __thread_pool = SingletonThreadPool() def __init__(self): self._event = ProcessEvent() def is_set(self): return self._event.is_set() def set(self): if not BackgroundMonitor.is_subprocess_enabled() or BackgroundMonitor.is_subprocess_alive(): self._event.set() # SafeEvent.__thread_pool.get().apply_async(func=self._event.set, args=()) def clear(self): return self._event.clear() def wait(self, timeout=None): return self._event.wait(timeout=timeout)
def __enter__(self): ready = Event() def target(): async def callback_notify(): # Run periodically by the Uvicorn server. ready.set() uvicorn.run(self.app, callback_notify=callback_notify, **self.kwargs) self._process = Process(target=target) self._process.start() if not ready.wait(self.ready_timeout): # pragma: no cover raise TimeoutError( f"Live server not ready after {self.ready_timeout} seconds") return self
def registerhook(hookconf): """Start and register a hook :param hookconf: A dict with the config of the hook to register """ rtn = False hookcfg = dict(hookconf) hookclass = config.getHookType(hookcfg['hooktype']) hookqueue = Queue() hookname = hookcfg[config.HOOKNAME] hooklevel = hookcfg[config.LOGLEVEL] hookevent = Event() hookprocess = hookclass(hookconf, logobject.getLoggerHandle(hookname, hooklevel), hookqueue, hookevent) hookprocess.start() if hookevent.wait(2.0): registeredhooks.append((hookname, hookcfg, hookprocess, hookqueue)) rtn = True else: loghandle.critical("Failed to start hook process {}".format(hookname)) rtn = False return rtn
def lock_and_call(callback, path_to_lock): """ Grab a lock on path_to_lock from a foreign process then execute the callback. :param callable callback: object to call after acquiring the lock :param str path_to_lock: path to file or directory to lock """ # Reload certbot.util module to reset internal _LOCKS dictionary. reload_module(util) emit_event = Event() receive_event = Event() process = Process(target=_handle_lock, args=(emit_event, receive_event, path_to_lock)) process.start() # Wait confirmation that lock is acquired assert receive_event.wait(timeout=10), 'Timeout while waiting to acquire the lock.' # Execute the callback callback() # Trigger unlock from foreign process emit_event.set() # Wait for process termination process.join(timeout=10) assert process.exitcode == 0
def startProcess(self): if self._process: # This should almost never happen (but it does) # Make sure the previous process is killed before a new one is started self._logger.warn( "A previous process was still running, killing it") self._kill() onListeningEvent = Event() errorState = Value( 'b', False) #If True, it means the process had an error while starting self._listening = False self._process = Process( target=startPipelineProcess, args=(self._device, self._size, self._rotation, self._source, self._encoding, onListeningEvent, errorState, (self._parentConn, self._processConn), settings().getInt(['camera', 'debug-level']))) self._process.daemon = True self._process.start() if onListeningEvent.wait(20.0): if errorState.value: self._logger.error('Pipeline Failed to start.') self._kill() self._logger.debug('Pipeline Process killed.') else: self._logger.debug('Pipeline Process Started.') self._listening = True else: self._logger.debug( 'Timeout while waiting for pipeline process to start') self._kill()
class TestFileAddition(unittest.TestCase): @patch('unpacker.unpack.FS_Organizer', MockFSOrganizer) def setUp(self): self._tmp_dir = TemporaryDirectory() self._config = initialize_config(self._tmp_dir) self.elements_finished_analyzing = Value('i', 0) self.analysis_finished_event = Event() self.compare_finished_event = Event() self._mongo_server = MongoMgr(config=self._config, auth=False) self.backend_interface = BackEndDbInterface(config=self._config) self._analysis_scheduler = AnalysisScheduler( config=self._config, post_analysis=self.count_analysis_finished_event) self._unpack_scheduler = UnpackingScheduler( config=self._config, post_unpack=self._analysis_scheduler.add_task) self._compare_scheduler = CompareScheduler( config=self._config, callback=self.trigger_compare_finished_event) def count_analysis_finished_event(self, fw_object): self.backend_interface.add_analysis(fw_object) self.elements_finished_analyzing.value += 1 if self.elements_finished_analyzing.value == 4 * 2 * 2: # 2 container with 3 files each and 2 plugins self.analysis_finished_event.set() def trigger_compare_finished_event(self): self.compare_finished_event.set() def tearDown(self): self._compare_scheduler.shutdown() self._unpack_scheduler.shutdown() self._analysis_scheduler.shutdown() clean_test_database(self._config, get_database_names(self._config)) self._mongo_server.shutdown() self._tmp_dir.cleanup() gc.collect() def test_unpack_analyse_and_compare(self): test_fw_1 = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) test_fw_1.release_date = '2017-01-01' test_fw_2 = Firmware( file_path='{}/regression_one'.format(get_test_data_dir())) test_fw_2.release_date = '2017-01-01' self._unpack_scheduler.add_task(test_fw_1) self._unpack_scheduler.add_task(test_fw_2) self.analysis_finished_event.wait(timeout=20) compare_id = normalize_compare_id(';'.join( [fw.uid for fw in [test_fw_1, test_fw_2]])) self.assertIsNone( self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error') self.compare_finished_event.wait(timeout=10) with ConnectTo(CompareDbInterface, self._config) as sc: result = sc.get_compare_result(compare_id) self.assertEqual(result['plugins']['Software'], self._expected_result()['Software']) self.assertCountEqual( result['plugins']['File_Coverage']['files_in_common'], self._expected_result()['File_Coverage']['files_in_common']) @staticmethod def _expected_result(): return { 'File_Coverage': { 'files_in_common': { 'all': [], 'collapse': False } }, 'Software': { 'Compare Skipped': { 'all': 'Required analysis not present: [\'software_components\', \'software_components\']' } } }
class DataLogger(Process): def __init__(self, controlPipe, dataPipe, workerPipe): super(DataLogger, self).__init__() #threading.Thread.__init__(self) self.__logName = None # File to log to self.__logFrequency = 1 # Time per second to write to the log file self.__logHeadings = [] # List of headings for the log file. self.__running = False # Internal running flag self.__paused = True # Internal paused flag self.__logPath = './' # Path for logfile. Default to current directory self.deamon = True # Sets the process to daemon. Stops if the parent process stops self.name = 'Logger' # Sets the process name. Helps with debigging. self.__pipes = {} self.__pipes['APPLICATION'] = PipeWatcher( self, controlPipe, 'APP->LOGGER' ) # Communication pipe with the controlling application self.__pipes['DATA'] = PipeWatcher( self, dataPipe, 'COLLECTOR->LOGGER' ) # Comminication pipe with the Data Collector process self.__pipes['WORKER'] = PipeWatcher( self, workerPipe, 'WORKER->LOGGER' ) # Comminication pipe with the Data Collector process self.__data = dict() # Data dictionary to use to write log diles self.__refreshRequired = True # Flag to determine if the dictionary needs to be refreshed self.__refreshRequested = False # Flag to determine if the refresh request has been sent self.__logFormat = '%Y%m%d%H%M' # Log file name format self.__pauseLog = False # Flag to pause logging self.__pid = None # Pricess ID of Logging process self.__snapshot_ready = Event() logger.debug('Logging process initalised') def run(self): self.__running = True logger.info('Starting Logger process on PID {}'.format(self.pid)) for p in self.__pipes: self.__pipes[p].start() timer = time() while self.__running: try: if not self.__paused: if self.__logName is None: logger.debug('Logger name not set. Pausing') self.pause() #logger.debug('Running: {}, Paused: {}, Required: {}, Requested: {}'.format(self.__running, self.__paused, self.__refreshRequired, self.__refreshRequested)) if self.__refreshRequired: #logger.debug('Getting snapshot') self.__pipes['DATA'].send(Message('SNAPSHOT')) self.__snapshot_ready.wait() self.__snapshot_ready.clear() line = '' #logger.debug('Recording data') for l in self.__logHeadings: if l in self.__data: line += str(self.__data[l]['LOG']).strip() + ',' else: #logger.debug('{} is not in snapshot'.format(l)) line += '-,' with open(self.__logName + '.log', 'ab') as f: f.write(bytes(line[:len(line) - 1] + '\n', 'UTF-8')) self.__refreshRequired = True sleeptime = (1.0 / self.__logFrequency) - (time() - timer) if sleeptime < 0: logger.warning( 'Logger sleep time reached zero. Concider reducing log frequency' ) #self.logFrequency-=1 else: sleep(sleeptime) timer = time() except (KeyboardInterrupt, SystemExit): self.__running = False continue except: logger.critical( 'Unhandled exception occured in Logger process: ', exc_info=True, stack_info=True) continue logger.info('Logging process stopped') def stop(self, p=None): #Stop logging. Thread stops - This is final. Cannot be restarted self.__running = False def resume(self, p=None): #Resume Logging logger.info('Logging resumed') self.__paused = self.__pauseLog = False if self.__logName is None: self.__setName() def pause(self, p=None): #Pause logging, thread keeps running if not self.__paused: logger.info('Logging paused') self.__paused = True def frequency(self, p): self.__logFrequency = p['FREQUENCY'] def logpath(self, p): self.__logPath = p['PATH'] def logname(self, p): return Message('LOG_NAME', NAME=self.__logName) def snap_shot(self, p): self.__data = p['SNAPSHOT'] self.__refreshRequired = False self.__snapshot_ready.set() def __setName(self): if self.__logHeadings == []: logger.warning( 'No column headings have been set. No log file started.') else: self.__logName = (self.__logPath + datetime.now().strftime(self.__logFormat)) logger.info('Logging started - output: {}.log'.format( self.__logName)) line = '' for l in self.__logHeadings: line += l + ',' with open(self.__logName + '.log', 'wb') as f: # Clobber output file if it exists f.write(bytes(line[:len(line) - 1] + '\n', 'UTF-8')) # def headings(self, p): # #Set the log headings # self.__logHeadings = p['HEADINGS'] def save(self, p=None): #Compress the logfile if not self.__paused: self.pause() if self.__logName is None: return try: with open(self.__logName + '.log', 'rb') as f: with gzip.open(self.__logName + '.log.gz', 'wb') as z: shutil.copyfileobj(f, z) os.remove(self.__logName + '.log') self.__logName = None except: logger.error('Error compressing log file {}'.format( self.__logName), exc_info=True) self.__logName = None def discard(self, p=None): #Delete the logfile if self.__paused: self.__paused = True try: logger.debug('Discarding log file {}.log'.format(self.__logName)) os.remove(self.__logName + '.log') except: logger.warning('Could not delete log file {}.log'.format( self.__logname), exc_info=True) self.__logName = None def getstatus(self, p=None): d = dict() d['Running'] = self.__running d['Paused'] = self.__paused d['Log Name'] = self.__logName d['Frequency'] = self.__logFrequency d['Log Path'] = self.__logPath d['Headings'] = self.__logHeadings return Message('LOGSTATUS', STATUS=d) def add_headings(self, p): insert = False if 'INSERT' in p: insert = p['INSERT'] for a in range(len(p['HEADINGS'])): if p['HEADINGS'][a] not in self.__logHeadings: if insert: self.__logHeadings.insert(a, p['HEADINGS'][a]) else: self.__logHeadings.append(p['HEADINGS'][a]) logger.info('Log headings updated to {}'.format(self.__logHeadings)) def remove_headings(self, p): for h in p['HEADINGS']: if h in self.__logHeadings: self.__logHeadings.remove(h) logger.info('Log headings updated to {}'.format(self.__logHeadings))
class MultiProcessConsumer(Consumer): """ A consumer implementation that consumes partitions for a topic in parallel using multiple processes client: a connected KafkaClient group: a name for this consumer, used for offset storage and must be unique topic: the topic to consume auto_commit: default True. Whether or not to auto commit the offsets auto_commit_every_n: default 100. How many messages to consume before a commit auto_commit_every_t: default 5000. How much time (in milliseconds) to wait before commit num_procs: Number of processes to start for consuming messages. The available partitions will be divided among these processes partitions_per_proc: Number of partitions to be allocated per process (overrides num_procs) Auto commit details: If both auto_commit_every_n and auto_commit_every_t are set, they will reset one another when one is triggered. These triggers simply call the commit method on this class. A manual call to commit will also reset these triggers """ def __init__(self, client, group, topic, auto_commit=True, auto_commit_every_n=AUTO_COMMIT_MSG_COUNT, auto_commit_every_t=AUTO_COMMIT_INTERVAL, num_procs=1, partitions_per_proc=0): # Initiate the base consumer class super(MultiProcessConsumer, self).__init__(client, group, topic, partitions=None, auto_commit=auto_commit, auto_commit_every_n=auto_commit_every_n, auto_commit_every_t=auto_commit_every_t) # Variables for managing and controlling the data flow from # consumer child process to master self.queue = Queue(1024) # Child consumers dump messages into this self.start = Event() # Indicates the consumers to start fetch self.exit = Event() # Requests the consumers to shutdown self.pause = Event() # Requests the consumers to pause fetch self.size = Value('i', 0) # Indicator of number of messages to fetch partitions = self.offsets.keys() # If unspecified, start one consumer per partition # The logic below ensures that # * we do not cross the num_procs limit # * we have an even distribution of partitions among processes if not partitions_per_proc: partitions_per_proc = round(len(partitions) * 1.0 / num_procs) if partitions_per_proc < num_procs * 0.5: partitions_per_proc += 1 # The final set of chunks chunker = lambda *x: [] + list(x) chunks = map(chunker, *[iter(partitions)] * int(partitions_per_proc)) self.procs = [] for chunk in chunks: chunk = filter(lambda x: x is not None, chunk) proc = Process(target=self._consume, args=(chunk,)) proc.daemon = True proc.start() self.procs.append(proc) def _consume(self, partitions): """ A child process worker which consumes messages based on the notifications given by the controller process """ # Make the child processes open separate socket connections self.client.reinit() # We will start consumers without auto-commit. Auto-commit will be # done by the master controller process. consumer = SimpleConsumer(self.client, self.group, self.topic, partitions=partitions, auto_commit=False, auto_commit_every_n=None, auto_commit_every_t=None) # Ensure that the consumer provides the partition information consumer.provide_partition_info() while True: # Wait till the controller indicates us to start consumption self.start.wait() # If we are asked to quit, do so if self.exit.is_set(): break # Consume messages and add them to the queue. If the controller # indicates a specific number of messages, follow that advice count = 0 for partition, message in consumer: self.queue.put((partition, message)) count += 1 # We have reached the required size. The controller might have # more than what he needs. Wait for a while. # Without this logic, it is possible that we run into a big # loop consuming all available messages before the controller # can reset the 'start' event if count == self.size.value: self.pause.wait() break # In case we did not receive any message, give up the CPU for # a while before we try again if count == 0: time.sleep(0.1) consumer.stop() def stop(self): # Set exit and start off all waiting consumers self.exit.set() self.pause.set() self.start.set() for proc in self.procs: proc.join() proc.terminate() super(MultiProcessConsumer, self).stop() def __iter__(self): """ Iterator to consume the messages available on this consumer """ # Trigger the consumer procs to start off. # We will iterate till there are no more messages available self.size.value = 0 self.pause.set() while True: self.start.set() try: # We will block for a small while so that the consumers get # a chance to run and put some messages in the queue # TODO: This is a hack and will make the consumer block for # at least one second. Need to find a better way of doing this partition, message = self.queue.get(block=True, timeout=1) except Empty: break # Count, check and commit messages if necessary self.offsets[partition] = message.offset self.start.clear() yield message self.count_since_commit += 1 self._auto_commit() self.start.clear() def get_messages(self, count=1, block=True, timeout=10): """ Fetch the specified number of messages count: Indicates the maximum number of messages to be fetched block: If True, the API will block till some messages are fetched. timeout: If None, and block=True, the API will block infinitely. If >0, API will block for specified time (in seconds) """ messages = [] # Give a size hint to the consumers. Each consumer process will fetch # a maximum of "count" messages. This will fetch more messages than # necessary, but these will not be committed to kafka. Also, the extra # messages can be provided in subsequent runs self.size.value = count self.pause.clear() while count > 0: # Trigger consumption only if the queue is empty # By doing this, we will ensure that consumers do not # go into overdrive and keep consuming thousands of # messages when the user might need only a few if self.queue.empty(): self.start.set() try: partition, message = self.queue.get(block, timeout) except Empty: break messages.append(message) # Count, check and commit messages if necessary self.offsets[partition] = message.offset self.count_since_commit += 1 self._auto_commit() count -= 1 self.size.value = 0 self.start.clear() self.pause.set() return messages
from multiprocessing import Event e = Event() print(e.is_set()) # print('1111111') # e.wait() # # print('22222222') # e.wait() e.set() print('333333') print(e.is_set()) e.wait() print('444444') e.clear() print(e.is_set()) e.wait() print(e.is_set())
class ValkkaProcess(Process): """ Semantics: Frontend: the part of the forked process that keeps running in the current, user virtual memory space Backend : the part of the forked process that runs in its own virtual memory space (e.g. "in the background") This class has both backend and frontend methods: - Backend methods should only be called from backend. They are designated with "_". - Frontend methods should only be called from frontend To avoid confusion, backend methods are designated with "_", except for the "run()" method, that's always in the backend Frontend methods use a pipe to send a signal to backend that then handles the signal with a backend method having the same name (but with "_" in the end) Backend methods can, in a similar fashion, send signals to the frontend using a pipe. In frontend, a listening thread is needed. That thread can then call the handleSignal method that chooses the correct frontend method to call TODO: add the possibility to bind the process to a certain processor """ # incoming signals : from frontend to backend incoming_signal_defs = { # each key corresponds to a front- and backend methods "test_": { "test_int": int, "test_str": str }, "stop_": [] } # outgoing signals : from back to frontend. Don't use same names as for # incoming signals .. outgoing_signal_defs = { "test_o": { "test_int": int, "test_str": str }, } def __init__(self, name, affinity=-1, **kwargs): super().__init__() self.pre = self.__class__.__name__ + " : " + name + \ " : " # auxiliary string for debugging output self.name = name self.affinity = affinity self.signal_in = Event() self.signal_out = Event() # communications pipe. Frontend uses self.pipe, backend self.childpipe self.pipe, self.childpipe = Pipe() self.signal_in.clear() self.signal_out.clear() # print(self.pre, "init") def getPipe(self): """Returns communication pipe for front-end """ return self.pipe def preRun_(self): """After the fork, but before starting the process loop """ if (self.affinity > -1): os.system("taskset -p -c %d %d" % (self.affinity, os.getpid())) def postRun_(self): """Just before process exit """ print(self.pre, "post: bye!") def cycle_(self): # Do whatever your process should be doing, remember timeout every now # and then time.sleep(5) print(self.pre, "hello!") def startAsThread(self): from threading import Thread t = Thread(target=self.run) t.start() def run(self ): # No "_" in the name, but nevertheless, running in the backed """After the fork. Now the process starts running """ # print(self.pre," ==> run") self.preRun_() self.running = True while (self.running): self.cycle_() self.handleSignal_() self.postRun_() def handleSignal_(self): """Signals handling in the backend """ if (self.signal_in.is_set()): signal_dic = self.childpipe.recv() method_name = signal_dic.pop("name") method = getattr(self, method_name) method(**signal_dic) self.signal_in.clear() self.signal_out.set() def sendSignal(self, **kwargs ): # sendSignal(name="test",test_int=1,test_str="kokkelis") """Incoming signals: this is used by frontend methods to send signals to the backend """ try: name = kwargs.pop("name") except KeyError: raise (AttributeError("Signal name missing")) # a dictionary: {"parameter_name" : parameter_type} model = self.incoming_signal_defs[name] for key in kwargs: # raises error if user is using undefined signal model_type = model[key] parameter_type = kwargs[key].__class__ if (model_type == parameter_type): pass else: raise (AttributeError("Wrong type for parameter " + str(key))) kwargs["name"] = name self.pipe.send(kwargs) self.signal_out.clear() self.signal_in.set() # indicate that there is a signal self.signal_out.wait() # wait for the backend to clear the signal def handleSignal(self, signal_dic): """Signal handling in the frontend """ method_name = signal_dic.pop("name") method = getattr(self, method_name) method(**signal_dic) def sendSignal_(self, **kwargs): # sendSignal_(name="test_out",..) """Outgoing signals: signals from backend to frontend """ try: name = kwargs.pop("name") except KeyError: raise (AttributeError("Signal name missing")) # a dictionary: {"parameter_name" : parameter_type} model = self.outgoing_signal_defs[name] for key in kwargs: # raises error if user is using undefined signal try: model_type = model[key] except KeyError: print("your outgoing_signal_defs for", name, "is:", model) print("you requested key:", key) raise parameter_type = kwargs[key].__class__ if (model_type == parameter_type): pass else: raise (AttributeError("Wrong type for parameter " + str(key))) kwargs["name"] = name self.childpipe.send(kwargs) # *** backend methods corresponding to each incoming signals *** def stop_(self): self.running = False def test_(self, test_int=0, test_str="nada"): print(self.pre, "test_ signal received with", test_int, test_str) # ** frontend methods corresponding to each incoming signal: these communicate with the backend via pipes ** def stop(self): self.sendSignal(name="stop_") def test(self, **kwargs): dictionaryCheck(self.incoming_signal_defs["test_"], kwargs) kwargs["name"] = "test_" self.sendSignal(**kwargs) # ** frontend methods corresponding to each outgoing signal ** # typically, there is a QThread in the frontend-side reading the process pipe # the QThread reads kwargs dictionary from the pipe, say # {"name":"test_o", "test_str":"eka", "test_int":1} # And calls handleSignal(kwargs) def test_o(self, **kwargs): pass
class objrThread(threading.Thread): dl_detector = None def __init__(self, filePath, update_signal, finish_signal): super(objrThread, self).__init__() self.isWorking = False self.update_signal = update_signal self.finish_signal = finish_signal self.model_path = 'model_data/trained_weights_final.h5' # model path or trained weights path self.classes_path = 'model_data/newclothclass.txt' self.anchors_path = 'model_data/yolo_anchors.txt' classes_path = os.path.expanduser(self.classes_path) with open(classes_path) as f: class_names = f.readlines() self.class_names = [c.strip() for c in class_names] # hsv_tuples = [(x / len(self.class_names), 1., 1.) # for x in range(len(self.class_names))] # self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) # self.colors = list( # map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), # self.colors)) # np.random.seed(10101) # Fixed seed for consistent colors across runs. # np.random.shuffle(self.colors) # Shuffle colors to decorrelate adjacent classes. # np.random.seed(None) # Reset seed to default self.colors = [(0, 0, 255), (0, 255, 127), (173, 222, 255), (18, 153, 255), (214, 112, 218), (255, 153, 18), (255, 0, 0)] self.lastepochFlag = False self.isWorking = True if filePath == "": self.filePath = Manager().Value( c_char_p, "E:/YJF/hotelvideo/video17.MP4") # 共享字符串变量 else: self.filePath = Manager().Value(c_char_p, filePath) # 共享字符串变量 self.stopMainProcSignal = Event() self.input_queue = mpQueue() self.camereStartSignal = Event() self.camereNotPauseSignal = Event() self.camereStopSignal = Event() self.camereNewinputSignal = Event() self.camereInputendSignal = Event() self.detDoneSignal = Event() self.newRltSignal = Event() self.cameraProc = CameraProc( self.filePath, self.input_queue, self.camereStartSignal, self.camereNotPauseSignal, self.camereStopSignal, self.camereNewinputSignal, self.camereInputendSignal, self.detDoneSignal, self.newRltSignal) self.rlt_queue = mpQueue() self.detproc = DetectorProc(self.input_queue, self.rlt_queue, self.camereNewinputSignal, self.newRltSignal, self.detDoneSignal, self.model_path, self.classes_path, self.anchors_path) self.areaClass = ['toilet', 'sink', 'desktop'] self.errdict = {} self.corrList = [] self.corrDict = {} self.testTotalError = [] self.testTotalCorr = [] self.testDelError = [] self.testDelCorr = [] self.f = open('test.txt', 'a') # self.f = open('test.txt') def _run(self): while True: print('tun in sub_thread') time.sleep(1) def stopMainThread(self): self.isWorking = False self.cameraProc.terminate() self.detproc.terminate() self.newRltSignal.set() self.stopMainProcSignal.set() print('stop') def Pause(self): self.camereNotPauseSignal.clear() def breakPause(self): self.camereNotPauseSignal.set() def startCamera(self, filePath): self.filePath.value = filePath self.filename = filePath.split('/')[-1].split('.')[0] self.f = open('./rlt_txt/' + self.filename + '.txt', 'a') self.camereInputendSignal.clear() self.camereStopSignal.clear() self.camereNotPauseSignal.set() self.detDoneSignal.set() self.camereStartSignal.set() def stopCamera(self): self.camereStartSignal.clear() self.camereStopSignal.set() def setFilePath(self, filePath): self.filePath.value = filePath def label_match(self, arealabel, clothlabel): if (arealabel == 'sink' and clothlabel == 'lightblue') or \ (arealabel == 'toilet' and clothlabel == 'pink') or \ (arealabel == 'desktop' and clothlabel == 'orange'): return True else: return False def bbox_inter_area(self, box1, box2): # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4 box2 = box2.transpose() b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3] b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3] # Intersection area inter_area = np.maximum(np.minimum(b2_x2 , np.full(b2_x2.shape, b1_x2)) - np.maximum(b2_x1 , np.full(b2_x1.shape, b1_x1)), 0) * \ np.maximum(np.minimum(b2_y2 , np.full(b2_y2.shape, b1_y2)) - np.maximum(b2_y1 , np.full(b2_y1.shape, b1_y1)), 0) return inter_area def run(self): self.detproc.daemon = True self.cameraProc.daemon = True self.cameraProc.start() self.detproc.start() # showcount = 0 while self.isWorking: if self.camereInputendSignal.is_set(): self.camereInputendSignal.clear() self.finish_signal.emit() self.f.close() self.newRltSignal.wait() while not self.rlt_queue.empty(): # showcount += 1 out_boxes, out_scores, out_classes, image, frameIndex = self.rlt_queue.get( ) tmp = cv2.cvtColor(np.asarray(image), cv2.COLOR_BGR2RGB) image = Image.fromarray(tmp) # for i in range(out_boxes.shape[0]): # for j in range(4): # self.f.write(str(out_boxes[i][j])) # if not (i == out_boxes.shape[0]-1 and j==3): # self.f.write(',') # self.f.write(';') # for i in range(out_scores.shape[0]): # self.f.write(str(out_scores[i])) # if i <out_scores.shape[0]-1: # self.f.write(',') # self.f.write(';') # for i in range(out_classes.shape[0]): # self.f.write(str(out_classes[i])) # if i <out_classes.shape[0]-1: # self.f.write(',') # self.f.write('\n') # line = self.f.readline().split(';') # tmp = [] # for item in line[0].split(','): # tmp.append(float(item)) # boxs = np.array(tmp).reshape((-1, 4)) # # tmp =[] # for item in line[1].split(','): # tmp.append(float(item)) # scores = np.array(tmp) # # tmp = [] # for item in line[2].split(','): # tmp.append(int(item)) # classes = np.array(tmp) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 cloths_box = np.empty(shape=[0, 4]) areas_box = np.empty(shape=[0, 4]) cloths_label = [] areas_label = [] for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) if predicted_class in self.areaClass: areas_box = np.append(areas_box, [[left, top, right, bottom]], axis=0) areas_label.append(predicted_class) else: cloths_box = np.append(cloths_box, [[left, top, right, bottom]], axis=0) cloths_label.append(predicted_class) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw # tmp = cv2.cvtColor(np.asarray(image), cv2.COLOR_BGR2RGB) # res = cv2.resize(tmp,None, fx = 0.5, fy=0.5) # cv2.imshow('rlt', res) # if cv2.waitKey(1) & 0xFF == ord('q'): # break # image.show() ''' areas_box = np.append(areas_box, [[10, 10, 20, 20]], axis=0) areas_box = np.append(areas_box, [[50, 50, 100, 100]], axis=0) areas_label.append('clot') areas_label.append('clot') cloths_box = np.append(cloths_box, [[15, 15, 30, 30]], axis=0) cloths_box = np.append(cloths_box, [[15, 15, 60, 60]], axis=0) cloths_label.append('clot') cloths_label.append('clot1') ''' if len(areas_label) and len(cloths_label): for i, areas in enumerate(areas_box): inter_area = self.bbox_inter_area(areas, cloths_box) for clothIndex in np.nonzero(inter_area)[0]: if not self.label_match(areas_label[i], cloths_label[clothIndex]): errkey = '{}-{}'.format( areas_label[i], cloths_label[clothIndex]) if errkey in self.errdict.keys(): self.errdict[errkey][0] += 1 self.errdict[errkey][2] = frameIndex else: self.errdict[errkey] = [ 1, frameIndex, frameIndex, errkey ] else: if areas_label[i] in self.corrDict.keys(): self.corrDict[areas_label[i]][0] += 1 self.corrDict[ areas_label[i]][2] = frameIndex else: self.corrDict[areas_label[i]] = [ 1, frameIndex, frameIndex, areas_label[i] ] if areas_label[i] not in self.corrList: self.corrList.append(areas_label[i]) # print(self.errdict) notmatchList = [] keys = list(self.errdict.keys()) for key in keys: if frameIndex - self.errdict[key][2] > 60: if self.errdict[key][0] > 10: notmatchList.append(self.errdict[key]) # self.testTotalError[key] = self.errdict[key] self.testTotalError.append(self.errdict[key]) del (self.errdict[key]) else: # self.testDelError [key] = self.errdict[key] self.testDelError.append(self.errdict[key]) del (self.errdict[key]) matchList = [] corrKeys = list(self.corrDict.keys()) for key in corrKeys: if frameIndex - self.corrDict[key][2] > 60: if self.corrDict[key][0] > 10: # self.testTotalCorr[key] = self.corrDict[key] matchList.append(self.corrDict[key]) self.testTotalCorr.append(self.corrDict[key]) del (self.corrDict[key]) else: self.testDelCorr.append(self.corrDict[key]) del (self.corrDict[key]) self.update_signal.emit(np.asarray(image)) # print ('emit cout {}'.format(showcount)) # print(notmatchList) self.newRltSignal.clear() if self.stopMainProcSignal.is_set(): break
from multiprocessing import Event #创建事件对象 e = Event() print(e.is_set()) e.set() e.wait(5) print("*************") print(e.is_set()) e.clear() # 清除设置 print(e.is_set()) e.wait()
class Trigger(Process): """ Stytra uses :class:`Trigger <stytra.triggering.Trigger.__init__()>` objects to control the beginning of a stimulation protocol via an external event. In the most obvious case, the signal is sent by an acquisition device such as a microscope to synchronize data acquisition and stimulation. The trigger has a check_trigger function that is constantly called in a while loop in the run(). When :meth:`Trigger <stytra.triggering.Trigger.check_trigger( )>` returns True, the start_event is set. The Experiment class, if it has a trigger assigned, will wait until the start_event to be set before starting. The control in check_trigger() is defined in subclasses to reflect the condition that we want to control the beginning of the protocol. **Events** start_event: event that is set when check_trigger() returns True. It is used by stytra to control the beginning of the protocol; kill_event: can be set to kill the Trigger process; **Output Queues** queue_trigger_params: can be used to send to the Experiment data about the triggering event or device. For example, if triggering happens from a microscope via a ZMQ message, setting of the microscope can be sent in that message to be saved together with experiment metadata. """ def __init__(self): super().__init__() self.start_event = Event() self.t = datetime.datetime.now() self.kill_event = Event() self.queue_trigger_params = Queue() def check_trigger(self): """ Check condition required for triggering to happen. Implemented in subclasses. Returns ------- bool True if triggering condition is satisfied (e.g., message received); False otherwise. """ return False def run(self): """ In this process, we constantly invoke the check_trigger class to control if start_event has to be set. Once it has been set, we wait an arbitrary time (0.1 s now) and then we clear it to be set again. """ TIME_START_EVENT_ON = 0.1 while True: self.kill_event.wait(0.0001) if self.kill_event.is_set(): break if self.start_event.is_set(): # Keep the signal on for at least 0.1 s time.sleep(TIME_START_EVENT_ON) self.start_event.clear() if self.start_event.is_set(): print( "Trying to start when the start event is already set") if self.check_trigger(): print("Trigger signal received") self.start_event.set() self.t = datetime.datetime.now()
class TestAcceptanceCompareFirmwares(TestAcceptanceBase): def setUp(self): super().setUp() self.analysis_finished_event = Event() self.compare_finished_event = Event() self.elements_finished_analyzing = Value('i', 0) self.db_backend_service = BackEndDbInterface(config=self.config) self._start_backend(post_analysis=self._analysis_callback, compare_callback=self._compare_callback) time.sleep(2) # wait for systems to start def tearDown(self): self._stop_backend() self.db_backend_service.shutdown() super().tearDown() def _analysis_callback(self, fo): self.db_backend_service.add_object(fo) self.elements_finished_analyzing.value += 1 if self.elements_finished_analyzing.value == 4 * 2 * 2: # two firmware container with 3 included files each times two plugins self.analysis_finished_event.set() def _compare_callback(self): self.compare_finished_event.set() def _upload_firmware_get(self): rv = self.test_client.get('/upload') self.assertIn(b'<h2>Upload Firmware</h2>', rv.data, 'upload page not displayed correctly') def _upload_firmware_put(self, path, device_name, uid): testfile_path = os.path.join(get_test_data_dir(), path) with open(testfile_path, 'rb') as fp: data = { 'file': fp, 'device_name': device_name, 'device_part': 'full', 'device_class': 'test_class', 'version': '1.0', 'vendor': 'test_vendor', 'release_date': '01.01.1970', 'tags': '', 'analysis_systems': [] } rv = self.test_client.post('/upload', content_type='multipart/form-data', data=data, follow_redirects=True) self.assertIn(b'Upload Successful', rv.data, 'upload not successful') self.assertIn(uid.encode(), rv.data, 'uid not found on upload success page') def _add_firmwares_to_compare(self): rv = self.test_client.get('/analysis/{}'.format(self.test_fw_a.uid)) self.assertIn(self.test_fw_a.uid, rv.data.decode(), '') rv = self.test_client.get('/comparison/add/{}'.format( self.test_fw_a.uid), follow_redirects=True) self.assertIn('Firmwares Selected for Comparison', rv.data.decode()) rv = self.test_client.get('/analysis/{}'.format(self.test_fw_c.uid)) self.assertIn(self.test_fw_c.uid, rv.data.decode()) self.assertIn(self.test_fw_c.name, rv.data.decode()) rv = self.test_client.get('/comparison/add/{}'.format( self.test_fw_c.uid), follow_redirects=True) self.assertIn('Remove All', rv.data.decode()) def _start_compare(self): rv = self.test_client.get('/compare', follow_redirects=True) self.assertIn(b'Your compare task is in progress.', rv.data, 'compare wait page not displayed correctly') def _show_comparison_results(self): rv = self.test_client.get('/compare/{};{}'.format( self.test_fw_a.uid, self.test_fw_c.uid)) self.assertIn(self.test_fw_a.name.encode(), rv.data, 'test firmware a comparison not displayed correctly') self.assertIn(self.test_fw_c.name.encode(), rv.data, 'test firmware b comparison not displayed correctly') self.assertIn(b'File Coverage', rv.data, 'comparison page not displayed correctly') def _show_home_page(self): rv = self.test_client.get('/') self.assertIn(b'Latest Comparisons', rv.data, 'latest comparisons not displayed on "home"') def _show_compare_browse(self): rv = self.test_client.get('/database/browse_compare') self.assertIn(self.test_fw_a.name.encode(), rv.data, 'no compare result shown in browse') def _show_analysis_without_compare_list(self): rv = self.test_client.get('/analysis/{}'.format(self.test_fw_a.uid)) assert b'Show List of Known Comparisons' not in rv.data def _show_analysis_with_compare_list(self): rv = self.test_client.get('/analysis/{}'.format(self.test_fw_a.uid)) assert b'Show List of Known Comparisons' in rv.data def test_compare_firmwares(self): self._upload_firmware_get() for fw in [self.test_fw_a, self.test_fw_c]: self._upload_firmware_put(fw.path, fw.name, fw.uid) self.analysis_finished_event.wait(timeout=20) self._show_analysis_without_compare_list() self._add_firmwares_to_compare() self._start_compare() self.compare_finished_event.wait(timeout=20) self._show_comparison_results() self._show_home_page() self._show_compare_browse() self._show_analysis_with_compare_list()
class Core(Process): __SESSIONFILENAME = 'session.ini' __CONFIGFILENAME = 'config.ini' DEFAULT_LANGUAGE = 'english' DEFAULT_USERNAME = '******' DEFAULT_PASSWORD = '******' DEFAULT_STORAGEDIRNAME = 'downloads' DEFAULT_LOGDIRNAME = 'logs' DEFAULT_LOGFILENAME = 'log.txt' def _init_consolelogger(self): if self.config.get('log', 'color_console') and ismodule('colorlog'): fmt = "%(label)s %(levelname)-8s %(reset)s %(log_color)s%(asctime)s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" primary_colors = { 'DEBUG': "bold,cyan", 'WARNING': "bold,yellow", 'ERROR': "bold,red", 'CRITICAL': "bold,purple", } secondary_colors = { 'label': { 'DEBUG': "bold,white,bg_cyan", 'INFO': "bold,white,bg_green", 'WARNING': "bold,white,bg_yellow", 'ERROR': "bold,white,bg_red", 'CRITICAL': "bold,white,bg_purple", } } consoleform = colorlog.ColoredFormatter( fmt, datefmt, primary_colors, secondary_log_colors=secondary_colors) else: fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" consoleform = logging.Formatter(fmt, datefmt) consolehdlr = logging.StreamHandler(sys.stdout) consolehdlr.setFormatter(consoleform) self.log.addHandler(consolehdlr) def _init_syslogger(self): # try to mimic to normal syslog messages fmt = "%(asctime)s %(name)s: %(message)s" datefmt = "%b %e %H:%M:%S" syslogform = logging.Formatter(fmt, datefmt) syslogaddr = None syslog = self.config.get('log', 'syslog') if syslog == 'remote': syslog_host = self.config.get('log', 'syslog_host') syslog_port = self.config.get('log', 'syslog_port') syslogaddr = (syslog_host, syslog_port) else: syslog_folder = self.config.get('log', 'syslog_folder') if syslogaddr: syslogaddr = syslog_folder elif sys.platform == 'darwin': syslogaddr = '/var/run/syslog' elif os.name != 'nt': syslogaddr = '/dev/log' sysloghdlr = logging.handlers.SysLogHandler(syslogaddr) sysloghdlr.setFormatter(syslogform) self.log.addHandler(sysloghdlr) def _init_filelogger(self): fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" fileform = logging.Formatter(fmt, datefmt) logfile_folder = self.config.get('log', 'logfile_folder') if not logfile_folder: logfile_folder = self.DEFAULT_LOGDIRNAME makedirs(logfile_folder, exist_ok=True) logfile_name = self.config.get('log', 'logfile_name') if not logfile_name: logfile_name = self.DEFAULT_LOGFILENAME logfile = os.path.join(logfile_folder, logfile_name) if self.config.get('log', 'rotate'): logfile_size = self.config.get('log', 'logfile_size') << 10 max_logfiles = self.config.get('log', 'max_logfiles') filehdlr = logging.handlers.RotatingFileHandler( logfile, maxBytes=logfile_size, backupCount=max_logfiles, encoding=locale.getpreferredencoding(do_setlocale=False)) else: filehdlr = logging.FileHandler( logfile, encoding=locale.getpreferredencoding(do_setlocale=False)) filehdlr.setFormatter(fileform) self.log.addHandler(filehdlr) # TODO: Extend `logging.Logger` like `..plugin.Log` def _init_logger(self): level = logging.DEBUG if self.debug else logging.INFO # Init logger self.log = logging.getLogger() self.log.setLevel(level) # Set console handler self._init_consolelogger() # Set syslog handler if self.config.get('log', 'syslog') != 'no': self._init_syslogger() # Set file handler if self.config.get('log', 'logfile'): self._init_filelogger() def _setup_permissions(self): if os.name == 'nt': return None change_group = self.config.get('permission', 'change_group') change_user = self.config.get('permission', 'change_user') if change_group: try: group = self.config.get('permission', 'group') set_process_group(group) except Exception as e: self.log.error(self._("Unable to change gid"), str(e)) if change_user: try: user = self.config.get('permission', 'user') set_process_user(user) except Exception as e: self.log.error(self._("Unable to change uid"), str(e)) def set_language(self, lang): localedir = resource_filename(__package__, 'locale') lc = locale.locale_alias[lang.lower()].split('_', 1)[0] trans = get_translation('core', localedir, (lc, )) try: self._ = trans.ugettext except AttributeError: self._ = trans.gettext def _setup_language(self): self.log.debug("Loading language ...") lang = self.config.get('general', 'language') default = self.DEFAULT_LANGUAGE if not lang: code = locale.getlocale()[0] or locale.getdefaultlocale()[0] lang = default if code is None else code.lower().split('_', 1)[0] try: self.set_language(lang) except Exception as e: if lang == default: raise self.log.warning( self._("Unable to load `{0}` language, " "use default `{1}`").format(lang, default), str(e)) self.set_language(default) def _setup_debug(self): if self.__debug is None: debug_log = self.config.get('log', 'debug') verbose_log = self.config.get('log', 'verbose') self.__debug = 0 if not debug_log else 2 if verbose_log else 1 # def start_interface(self, webui=None, rpc=None): # if webui is None: # webui = self.__webui # if rpc is None: # rpc = self.__rpc # # TODO: Parse `remote` # if rpc or self.config.get('rpc', 'activated'): # self.log.debug("Activating RPC interface ...") # self.rem.start() # elif not webui: # webui = True # TODO: Parse remote host:port # if isinstance(webui, str): # host, port = map(str.strip, webui.rsplit(':', 1)) # webui = True # else: # host, port = (None, None) # kwgs = { # 'server': self.config.get('webui', 'server'), # 'host': host or self.config.get('webui', 'host'), # 'port': port or self.config.get('webui', 'port'), # 'key': self.config.get('ssl', 'key'), # 'cert': self.config.get('ssl', 'cert'), # 'ssl': self.config.get('ssl', 'activated') # } # if webui or self.config.get('webui', 'activated'): # from .thread.webserver import WebServer # self.webserver = WebServer(self) # self.webserver.start() # self.svm.add('webui', **kwgs) # self.svm.start() def _init_api(self): from .api import Api self.api = Api(self) def _init_database(self): from .database import DatabaseBackend from .datatype import Permission, Role # TODO: Move inside DatabaseBackend newdb = not os.path.isfile(DatabaseBackend.DB_FILE) self.db = DatabaseBackend(self) self.db.setup() if self.__restore or newdb: self.db.add_user(self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD, Role.Admin, Permission.All) if self.__restore: self.log.warning( self._("Restored default login credentials `admin|pyload`")) def _init_managers(self): from .manager import (AccountManager, AddonManager, EventManager, ExchangeManager, FileManager, InfoManager, PluginManager, TransferManager) self.scheduler = sched.scheduler(time.time, time.sleep) self.filemanager = self.files = FileManager(self) self.pluginmanager = self.pgm = PluginManager(self) self.exchangemanager = self.exm = ExchangeManager(self) self.eventmanager = self.evm = EventManager(self) self.accountmanager = self.acm = AccountManager(self) self.infomanager = self.iom = InfoManager(self) self.transfermanager = self.tsm = TransferManager(self) self.addonmanager = self.adm = AddonManager(self) # self.remotemanager = self.rem = RemoteManager(self) # self.servermanager = self.svm = ServerManager(self) self.db.manager = self.files # ugly? def _init_requests(self): self.request = self.req = RequestFactory(self) def _init_config(self): session = ConfigParser(self.__SESSIONFILENAME, session_defaults) flags = portalocker.LOCK_EX | portalocker.LOCK_NB portalocker.lock(session.fp, flags) profiledir = os.path.join(self.configdir, self.profile) psp = psutil.Process() session.set('current', 'id', time.time()) session.set('current', 'profile', 'path', profiledir) session.set('current', 'profile', 'pid', psp.pid) session.set('current', 'profile', 'ctime', psp.create_time()) self.config = ConfigParser(self.__CONFIGFILENAME, config_defaults) self.session = session def _init_cache(self): # Re-use cache tempdir = self.__tempdir if tempdir is None: tempdir = self.session.get('previous', 'cache', 'path') if tempdir is None or not os.path.isdir(tempdir): pydir = os.path.join(TMPDIR, __namespace__) makedirs(pydir, exist_ok=True) tempdir = tempfile.mkdtemp(dir=pydir) self.session.set('current', 'cache', 'path', tempdir) self.cachedir = tempdir # if tempdir not in sys.path: # sys.path.append(tempdir) def _register_signals(self): shutfn = lambda s, f: self.shutdown() quitfn = lambda s, f: self.terminate() try: if os.name == 'nt': # signal.signal(signal.CTRL_C_EVENT, shutfn) signal.signal(signal.CTRL_BREAK_EVENT, shutfn) else: signal.signal(signal.SIGTERM, shutfn) # signal.signal(signal.SIGINT, shutfn) signal.signal(signal.SIGQUIT, quitfn) # signal.signal(signal.SIGTSTP, lambda s, f: self.stop()) # signal.signal(signal.SIGCONT, lambda s, f: self.run()) except Exception: pass def __init__(self, profiledir=None, tempdir=None, debug=None, restore=None): self.__running = Event() self.__do_restart = False self.__do_shutdown = False self.__debug = debug if debug is None else int(debug) self.__restore = bool(restore) self.__tempdir = tempdir self._ = lambda x: x self._init_profile(profiledir) # if refresh: # cleanpy(PACKDIR) Process.__init__(self) @property def version(self): return __version__ @property def version_info(self): return __version_info__ @property def running(self): return self.__running.is_set() @property def debug(self): return self.__debug def _init_profile(self, profiledir): profiledir = fullpath(profiledir) os.chdir(profiledir) self.configdir, self.profile = os.path.split(profiledir) def _setup_process(self): try: set_process_name('pyLoad') except AttributeError: pass niceness = self.config.get('general', 'niceness') renice(niceness=niceness) ioniceness = int(self.config.get('general', 'ioniceness')) ionice(niceness=ioniceness) def _setup_storage(self): storage_folder = self.config.get('general', 'storage_folder') if not storage_folder: storage_folder = os.path.join(USERDIR, self.DEFAULT_STORAGEDIRNAME) self.log.debug("Storage: {0}".format(storage_folder)) makedirs(storage_folder, exist_ok=True) avail_space = format.size(availspace(storage_folder)) self.log.info( self._("Available storage space: {0}").format(avail_space)) def _workloop(self): self.__running.set() self.tsm.pause = False # NOTE: Recheck... while True: self.__running.wait() self.tsm.work() self.iom.work() self.exm.work() if self.__do_restart: raise Restart if self.__do_shutdown: raise Shutdown self.scheduler.run() def _start_plugins(self): # TODO: Move to accountmanager self.log.info(self._("Activating accounts ...")) self.acm.load_accounts() # self.scheduler.enter(0, 0, self.acm.load_accounts) self.adm.activate_addons() def _show_info(self): self.log.info(self._("Welcome to pyLoad v{0}").format(self.version)) self.log.info(self._("Profile: {0}").format(self.profile)) self.log.info(self._("Config directory: {0}").format(self.configdir)) self.log.debug("Cache directory: {0}".format(self.cachedir)) def run(self): self._init_config() self._init_cache() self._setup_debug() self._init_logger() try: self.log.debug("Running pyLoad ...") self._setup_language() self._setup_permissions() self._init_database() self._init_managers() self._init_requests() self._init_api() self._show_info() self._setup_storage() self._start_plugins() self._setup_process() self.log.info(self._("pyLoad is up and running")) self.evm.fire('pyload:started') # # some memory stats # from guppy import hpy # hp=hpy() # print(hp.heap()) # import objgraph # objgraph.show_most_common_types(limit=30) # import memdebug # memdebug.start(8002) # from meliae import scanner # scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json')) self._workloop() except Restart: self.restart() except Shutdown: self.shutdown() except (KeyboardInterrupt, SystemExit): self.shutdown() except Exception as e: self.log.critical(str(e)) self.terminate() raise else: self.shutdown() def _remove_loggers(self): for handler in self.log.handlers: with closing(handler) as hdlr: self.log.removeHandler(hdlr) def restart(self): self.stop() self.log.info(self._("Restarting pyLoad ...")) self.evm.fire('pyload:restarting') self.start() def _register_instance(self): profiledir = os.path.join(self.configdir, self.profile) if profiledir in _pmap: raise RuntimeError("A pyLoad instance using profile `{0}` " "is already running".format(profiledir)) _pmap[profiledir] = self def _unregister_instance(self): profiledir = os.path.join(self.configdir, self.profile) _pmap.pop(profiledir, None) def _close_session(self): id = self.session.get('previous', 'id') self.session[id] = self.session['previous'] self.session['previous'] = self.session['current'] self.session['current'].reset() self.session.close() def terminate(self): try: self.log.debug("Killing pyLoad ...") self._unregister_instance() self._close_session() finally: Process.terminate(self) def shutdown(self): try: self.stop() self.log.info(self._("Exiting pyLoad ...")) self.tsm.shutdown() self.db.shutdown() # NOTE: Why here? self.config.close() self._remove_loggers() # if cleanup: # self.log.info(self._("Deleting temp files ...")) # remove(self.tempdir, ignore_errors=True) finally: self.terminate() def start(self): if not self.is_alive(): self._register_instance() self._register_signals() Process.start(self) elif not self.running: self.log.info(self._("Starting pyLoad ...")) self.evm.fire('pyload:starting') self.__running.set() def stop(self): if not self.running: return None try: self.log.info(self._("Stopping pyLoad ...")) self.evm.fire('pyload:stopping') self.adm.deactivate_addons() self.api.stop_all_downloads() finally: self.files.sync_save() self.__running.clear() self.evm.fire('pyload:stopped')
class NefitCore(object): _accesskey_prefix = 'Ct7ZR03b_' _rrc_contact_prefix = 'rrccontact_' _rrc_gateway_prefix = 'rrcgateway_' _magic = bytearray.fromhex("58f18d70f667c9c79ef7de435bf0f9b1553bbb6e61816212ab80e5b0d351fbb1") serial_number = None access_key = None password = None jid = None _from = None _to = None client = None encryption = None event = None container = {} def __init__(self, serial_number, access_key, password, host="wa2-mz36-qrmzh6.bosch.de", sasl_mech="DIGEST-MD5"): """ :param serial_number: :param access_key: :param password: :param host: :param sasl_mech: """ serial_number = str(serial_number) self.serial_number = serial_number self.access_key = access_key self.password = password self.encryption = AESCipher(self._magic, access_key, password) identifier = serial_number + "@" + host self.jid = jid = self._from = self._rrc_contact_prefix + identifier self._to = self._rrc_gateway_prefix + identifier self.client = ClientXMPP(jid=jid, password=self._accesskey_prefix + access_key, sasl_mech=sasl_mech) self.client.add_event_handler("session_start", self.session_start) self.client.register_plugin('xep_0199') @staticmethod def set_verbose(): import logging logging.basicConfig(filename="debug.log", level=logging.DEBUG) def message(self, msg): if msg['type'] in ('chat', 'normal'): headers = msg['body'].split("\n")[:-1] body = msg['body'].split("\n")[-1:][0] if 'HTTP/1.0 400 Bad Request' in headers: return response = self.decrypt(body) if 'Content-Type: application/json' in headers: _LOGGER.debug("response='{}'".format(response)) response = response.strip() if len(response) > 1: response = json.loads(response.strip()) self.container[id(self.event)] = response self.event.set() def connect(self, block=False): self.client.connect() self.client.process(block=block) def session_start(self, event): self.client.send_presence() self.client.get_roster() def disconnect(self): self.client.disconnect() def get(self, uri, timeout=10): self.event = Event() self.client.add_event_handler("message", self.message) self.send("GET %s HTTP/1.1\rUser-Agent: NefitEasy\r\r" % uri) self.event.wait(timeout=timeout) self.client.del_event_handler("message", self.message) return self.container[id(self.event)] if id(self.event) in self.container.keys() else None def put(self, uri, data, timeout=10): data = data if isinstance(data, str) else json.dumps(data, separators=(',', ':')) encrypted_data = self.encrypt(data).decode("utf8") body = "\r".join([ 'PUT %s HTTP/1.1' % uri, 'Content-Type: application/json', 'Content-Length: %i' % len(encrypted_data), 'User-Agent: NefitEasy\r', encrypted_data ]) self.event = Event() self.client.add_event_handler("message", self.message) self.send(body) self.event.wait(timeout=timeout) self.client.del_event_handler("message", self.message) def send(self, body): # this horrible piece of code breaks xml syntax but does actually work... body = body.replace("\r", " \n") message = self.client.make_message(mto=self._to, mfrom=self._from, mbody=body) message['lang'] = None str_data = tostring(message.xml, xmlns=message.stream.default_ns, stream=message.stream, top_level=True) str_data = str_data.replace("&#13;", " ") return message.stream.send_raw(str_data) def encrypt(self, data): return self.encryption.encrypt(data) def decrypt(self, data): return self.encryption.decrypt(data) def get_display_code(self): return self.get('/system/appliance/displaycode') def get_status(self): return self.get('/ecus/rrc/uiStatus') def get_location(self): return self.get('/system/location/latitude'), self.get('/system/location/longitude') def get_outdoor(self): return self.get('/system/sensors/temperatures/outdoor_t1') def get_pressure(self): return self.get('/system/appliance/systemPressure') def get_program(self): return ( self.get('/ecus/rrc/userprogram/activeprogram'), self.get('/ecus/rrc/userprogram/program1'), self.get('/ecus/rrc/userprogram/program2'), ) def get_year_total(self): return self.get('/ecus/rrc/recordings/yearTotal') def set_temperature(self, temperature): self.put('/heatingCircuits/hc1/temperatureRoomManual', {'value': float(temperature)}) self.put('/heatingCircuits/hc1/manualTempOverride/status', {'value': 'on'}) self.put('/heatingCircuits/hc1/manualTempOverride/temperature', {'value': float(temperature)}) def get_actualSupplyTemperature(self): return self.get('/heatingCircuits/hc1/actualSupplyTemperature')
class Speedtest(object): def __init__(self): from multiprocessing import Event self.event = Event() self.has_stopped = False def speedtest_thread(self): if self.event.wait(600): return logging.info("Speedtest starting...You can't stop right now!") CTid = 0 speedtest_ct = speedtest.Speedtest() speedtest_ct.get_servers() servers_list = [] for _, servers in sorted(speedtest_ct.servers.items()): for server in servers: if server['country'].find('China') != -1 and server[ 'sponsor'].find('Telecom') != -1: servers_list.append(server) speedtest_ct.get_best_server(servers_list) results_ct = speedtest_ct.results CTPing = str(results_ct.server['latency']) + ' ms' speedtest_ct.download() CTDLSpeed = str(round( (results_ct.download / 1000 / 1000), 2)) + " Mbit/s" speedtest_ct.upload() CTUpSpeed = str(round( (results_ct.upload / 1000 / 1000), 2)) + " Mbit/s" CUid = 0 speedtest_cu = speedtest.Speedtest() speedtest_cu.get_servers() servers_list = [] for _, servers in sorted(speedtest_cu.servers.items()): for server in servers: if server['country'].find('China') != -1 and server[ 'sponsor'].find('Unicom') != -1: servers_list.append(server) speedtest_cu.get_best_server(servers_list) results_cu = speedtest_cu.results CUPing = str(results_cu.server['latency']) + ' ms' speedtest_cu.download() CUDLSpeed = str(round( (results_cu.download / 1000 / 1000), 2)) + " Mbit/s" speedtest_cu.upload() CUUpSpeed = str(round( (results_cu.upload / 1000 / 1000), 2)) + " Mbit/s" CMid = 0 speedtest_cm = speedtest.Speedtest() speedtest_cm.get_servers() servers_list = [] for _, servers in sorted(speedtest_cm.servers.items()): for server in servers: if server['country'].find('China') != -1 and server[ 'sponsor'].find('Mobile') != -1: servers_list.append(server) speedtest_cm.get_best_server(servers_list) results_cm = speedtest_cm.results CMPing = str(results_cm.server['latency']) + ' ms' speedtest_cm.download() CMDLSpeed = str(round( (results_cm.download / 1000 / 1000), 2)) + " Mbit/s" speedtest_cm.upload() CMUpSpeed = str(round( (results_cm.upload / 1000 / 1000), 2)) + " Mbit/s" if configloader.get_config().API_INTERFACE == 'modwebapi': webapi.postApi('func/speedtest', {'node_id': configloader.get_config().NODE_ID}, { 'data': [{ 'telecomping': CTPing, 'telecomeupload': CTUpSpeed, 'telecomedownload': CTDLSpeed, 'unicomping': CUPing, 'unicomupload': CUUpSpeed, 'unicomdownload': CUDLSpeed, 'cmccping': CMPing, 'cmccupload': CMUpSpeed, 'cmccdownload': CMDLSpeed }] }) else: import cymysql if configloader.get_config().MYSQL_SSL_ENABLE == 1: conn = cymysql.connect( host=configloader.get_config().MYSQL_HOST, port=configloader.get_config().MYSQL_PORT, user=configloader.get_config().MYSQL_USER, passwd=configloader.get_config().MYSQL_PASS, db=configloader.get_config().MYSQL_DB, charset='utf8', ssl={ 'ca': configloader.get_config().MYSQL_SSL_CA, 'cert': configloader.get_config().MYSQL_SSL_CERT, 'key': configloader.get_config().MYSQL_SSL_KEY }) else: conn = cymysql.connect( host=configloader.get_config().MYSQL_HOST, port=configloader.get_config().MYSQL_PORT, user=configloader.get_config().MYSQL_USER, passwd=configloader.get_config().MYSQL_PASS, db=configloader.get_config().MYSQL_DB, charset='utf8') conn.autocommit(True) cur = conn.cursor() cur.execute( "INSERT INTO `speedtest` (`id`, `nodeid`, `datetime`, `telecomping`, `telecomeupload`, `telecomedownload`, `unicomping`, `unicomupload`, `unicomdownload`, `cmccping`, `cmccupload`, `cmccdownload`) VALUES (NULL, '" + str(configloader.get_config().NODE_ID) + "', unix_timestamp(), '" + CTPing + "', '" + CTUpSpeed + "', '" + CTDLSpeed + "', '" + CUPing + "', '" + CUUpSpeed + "', '" + CUDLSpeed + "', '" + CMPing + "', '" + CMUpSpeed + "', '" + CMDLSpeed + "')") cur.close() conn.close() logging.info("Speedtest finished") @staticmethod def thread_db(obj): if configloader.get_config().SPEEDTEST == 0: return if configloader.get_config().API_INTERFACE == 'modwebapi': import webapi_utils global webapi webapi = webapi_utils.WebApi() global db_instance db_instance = obj() try: while True: try: db_instance.speedtest_thread() except Exception as e: import traceback trace = traceback.format_exc() logging.error(trace) #logging.warn('db thread except:%s' % e) if db_instance.event.wait(configloader.get_config().SPEEDTEST * 3600): break if db_instance.has_stopped: break except KeyboardInterrupt as e: pass db_instance = None @staticmethod def thread_db_stop(): global db_instance db_instance.has_stopped = True db_instance.event.set()
class Artifacts(object): _flush_frequency_sec = 300. # notice these two should match _save_format = '.csv.gz' _compression = 'gzip' # hashing constants _hash_block_size = 65536 _pd_artifact_type = 'data-audit-table' class _ProxyDictWrite(dict): """ Dictionary wrapper that updates an arguments instance on any item set in the dictionary """ def __init__(self, artifacts_manager, *args, **kwargs): super(Artifacts._ProxyDictWrite, self).__init__(*args, **kwargs) self._artifacts_manager = artifacts_manager # list of artifacts we should not upload (by name & weak-reference) self.artifact_metadata = {} # list of hash columns to calculate uniqueness for the artifacts self.artifact_hash_columns = {} def __setitem__(self, key, value): # check that value is of type pandas if pd and isinstance(value, pd.DataFrame): super(Artifacts._ProxyDictWrite, self).__setitem__(key, value) if self._artifacts_manager: self._artifacts_manager.flush() else: raise ValueError( 'Artifacts currently support pandas.DataFrame objects only' ) def unregister_artifact(self, name): self.artifact_metadata.pop(name, None) self.pop(name, None) def add_metadata(self, name, metadata): self.artifact_metadata[name] = deepcopy(metadata) def get_metadata(self, name): return self.artifact_metadata.get(name) def add_hash_columns(self, artifact_name, hash_columns): self.artifact_hash_columns[artifact_name] = hash_columns def get_hash_columns(self, artifact_name): return self.artifact_hash_columns.get(artifact_name) @property def registered_artifacts(self): return self._artifacts_container @property def summary(self): return self._summary def __init__(self, task): self._task = task # notice the double link, this important since the Artifact # dictionary needs to signal the Artifacts base on changes self._artifacts_container = self._ProxyDictWrite(self) self._last_artifacts_upload = {} self._unregister_request = set() self._thread = None self._flush_event = Event() self._exit_flag = False self._summary = '' self._temp_folder = [] self._task_artifact_list = [] self._task_edit_lock = RLock() self._storage_prefix = None def register_artifact(self, name, artifact, metadata=None, uniqueness_columns=True): """ :param str name: name of the artifacts. Notice! it will override previous artifacts if name already exists. :param pandas.DataFrame artifact: artifact object, supported artifacts object types: pandas.DataFrame :param dict metadata: dictionary of key value to store with the artifact (visible in the UI) :param list uniqueness_columns: list of columns for artifact uniqueness comparison criteria. The default value is True, which equals to all the columns (same as artifact.columns). """ # currently we support pandas.DataFrame (which we will upload as csv.gz) if name in self._artifacts_container: LoggerRoot.get_base_logger().info( 'Register artifact, overwriting existing artifact \"{}\"'. format(name)) self._artifacts_container.add_hash_columns( name, list(artifact.columns if uniqueness_columns is True else uniqueness_columns)) self._artifacts_container[name] = artifact if metadata: self._artifacts_container.add_metadata(name, metadata) def unregister_artifact(self, name): # Remove artifact from the watch list self._unregister_request.add(name) self.flush() def upload_artifact(self, name, artifact_object=None, metadata=None, delete_after_upload=False): if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning( 'Artifacts not supported by your TRAINS-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_container: raise ValueError( "Artifact by the name of {} is already registered, use register_artifact" .format(name)) artifact_type_data = tasks.ArtifactTypeData() override_filename_in_uri = None override_filename_ext_in_uri = None uri = None if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = str(artifact_object.__repr__()) override_filename_ext_in_uri = '.npz' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = str(artifact_object.__repr__()) override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = desc[1:desc.find(' at ')] override_filename_ext_in_uri = '.png' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.save(local_filename) delete_after_upload = True elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' preview = json.dumps(artifact_object, sort_keys=True, indent=4) override_filename_ext_in_uri = '.json' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.write(fd, bytes(preview.encode())) os.close(fd) artifact_type_data.preview = preview delete_after_upload = True elif isinstance(artifact_object, six.string_types) and urlparse( artifact_object).scheme in remote_driver_schemes: # we should not upload this, just register local_filename = None uri = artifact_object artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] elif isinstance(artifact_object, six.string_types + (Path, )): # check if single file artifact_object = Path(artifact_object) artifact_object.expanduser().absolute() try: create_zip_file = not artifact_object.is_file() except Exception: # Hack for windows pathlib2 bug, is_file isn't valid. create_zip_file = True else: # We assume that this is not Windows os if artifact_object.is_dir(): # change to wildcard artifact_object /= '*' if create_zip_file: folder = Path('').joinpath(*artifact_object.parts[:-1]) if not folder.is_dir() or not folder.parts: raise ValueError( "Artifact file/folder '{}' could not be found".format( artifact_object.as_posix())) wildcard = artifact_object.parts[-1] files = list(Path(folder).rglob(wildcard)) override_filename_ext_in_uri = '.zip' override_filename_in_uri = folder.parts[ -1] + override_filename_ext_in_uri fd, zip_file = mkstemp( prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri) try: artifact_type_data.content_type = 'application/zip' artifact_type_data.preview = 'Archive content {}:\n'.format( artifact_object.as_posix()) with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(files): if filename.is_file(): relative_file_name = filename.relative_to( folder).as_posix() artifact_type_data.preview += '{} - {}\n'.format( relative_file_name, humanfriendly.format_size( filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning( 'Exception {}\nFailed zipping artifact folder {}'. format(folder, e)) return None finally: os.close(fd) artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object delete_after_upload = True else: if not artifact_object.is_file(): raise ValueError( "Artifact file '{}' could not be found".format( artifact_object.as_posix())) override_filename_in_uri = artifact_object.parts[-1] artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object else: raise ValueError("Artifact type {} not supported".format( type(artifact_object))) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError( "Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break if not local_filename: file_size = None file_hash = None else: # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning( 'Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = self.sha256sum(local_filename.as_posix()) file_size = local_filename.stat().st_size uri = self._upload_local_file( local_filename, name, delete_after_upload=delete_after_upload, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) timestamp = int(time()) artifact = tasks.Artifact( key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts with self._task_edit_lock: self._task_artifact_list.append(artifact) self._task.set_artifacts(self._task_artifact_list) return True def flush(self): # start the thread if it hasn't already: self._start() # flush the current state of all artifacts self._flush_event.set() def stop(self, wait=True): # stop the daemon thread and quit # wait until thread exists self._exit_flag = True self._flush_event.set() if wait: if self._thread: self._thread.join() # remove all temp folders for f in self._temp_folder: try: Path(f).rmdir() except Exception: pass def _start(self): """ Start daemon thread if any artifacts are registered and thread is not up yet """ if not self._thread and self._artifacts_container: # start the daemon thread self._flush_event.clear() self._thread = Thread(target=self._daemon) self._thread.daemon = True self._thread.start() def _daemon(self): while not self._exit_flag: self._flush_event.wait(self._flush_frequency_sec) self._flush_event.clear() artifact_keys = list(self._artifacts_container.keys()) for name in artifact_keys: try: self._upload_data_audit_artifacts(name) except Exception as e: LoggerRoot.get_base_logger().warning(str(e)) # create summary self._summary = self._get_statistics() def _upload_data_audit_artifacts(self, name): logger = self._task.get_logger() pd_artifact = self._artifacts_container.get(name) pd_metadata = self._artifacts_container.get_metadata(name) # remove from artifacts watch list if name in self._unregister_request: try: self._unregister_request.remove(name) except KeyError: pass self._artifacts_container.unregister_artifact(name) if pd_artifact is None: return override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) local_csv = Path(local_csv) pd_artifact.to_csv(local_csv.as_posix(), index=False, compression=self._compression) current_sha2, file_sha2 = self.sha256sum(local_csv.as_posix(), skip_header=32) if name in self._last_artifacts_upload: previous_sha2 = self._last_artifacts_upload[name] if previous_sha2 == current_sha2: # nothing to do, we can skip the upload try: local_csv.unlink() except Exception: pass return self._last_artifacts_upload[name] = current_sha2 # If old trains-server, upload as debug image if not Session.check_min_api_version('2.3'): logger.report_image(title='artifacts', series=name, local_path=local_csv.as_posix(), delete_after_upload=True, iteration=self._task.get_last_iteration(), max_image_history=2) return # Find our artifact artifact = None for an_artifact in self._task_artifact_list: if an_artifact.key == name: artifact = an_artifact break file_size = local_csv.stat().st_size # upload file uri = self._upload_local_file( local_csv, name, delete_after_upload=True, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) # update task artifacts with self._task_edit_lock: if not artifact: artifact = tasks.Artifact(key=name, type=self._pd_artifact_type) self._task_artifact_list.append(artifact) artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.data_hash = current_sha2 artifact_type_data.content_type = "text/csv" artifact_type_data.preview = str( pd_artifact.__repr__()) + '\n\n' + self._get_statistics( {name: pd_artifact}) artifact.type_data = artifact_type_data artifact.uri = uri artifact.content_size = file_size artifact.hash = file_sha2 artifact.timestamp = int(time()) artifact.display_data = [ (str(k), str(v)) for k, v in pd_metadata.items() ] if pd_metadata else None self._task.set_artifacts(self._task_artifact_list) def _upload_local_file(self, local_file, name, delete_after_upload=False, override_filename=None, override_filename_ext=None): """ Upload local file and return uri of the uploaded file (uploading in the background) """ upload_uri = self._task.output_uri or self._task.get_logger( ).get_default_upload_destination() if not isinstance(local_file, Path): local_file = Path(local_file) ev = UploadEvent( metric='artifacts', variant=name, image_data=None, upload_uri=upload_uri, local_image_path=local_file.as_posix(), delete_after_upload=delete_after_upload, override_filename=override_filename, override_filename_ext=override_filename_ext, override_storage_key_prefix=self._get_storage_uri_prefix()) _, uri = ev.get_target_full_upload_uri(upload_uri) # send for upload self._task.reporter._report(ev) return uri def _get_statistics(self, artifacts_dict=None): summary = '' artifacts_dict = artifacts_dict or self._artifacts_container thread_pool = ThreadPool() try: # build hash row sets artifacts_summary = [] for a_name, a_df in artifacts_dict.items(): hash_cols = self._artifacts_container.get_hash_columns(a_name) if not pd or not isinstance(a_df, pd.DataFrame): continue if hash_cols is True: hash_col_drop = [] else: hash_cols = set(hash_cols) missing_cols = hash_cols.difference(a_df.columns) if missing_cols == hash_cols: LoggerRoot.get_base_logger().warning( 'Uniqueness columns {} not found in artifact {}. ' 'Skipping uniqueness check for artifact.'.format( list(missing_cols), a_name)) continue elif missing_cols: # missing_cols must be a subset of hash_cols hash_cols.difference_update(missing_cols) LoggerRoot.get_base_logger().warning( 'Uniqueness columns {} not found in artifact {}. Using {}.' .format(list(missing_cols), a_name, list(hash_cols))) hash_col_drop = [ col for col in a_df.columns if col not in hash_cols ] a_unique_hash = set() def hash_row(r): a_unique_hash.add(hash(bytes(r))) a_shape = a_df.shape # parallelize a_hash_cols = a_df.drop(columns=hash_col_drop) thread_pool.map(hash_row, a_hash_cols.values) # add result artifacts_summary.append(( a_name, a_shape, a_unique_hash, )) # build intersection summary for i, (name, shape, unique_hash) in enumerate(artifacts_summary): summary += '[{name}]: shape={shape}, {unique} unique rows, {percentage:.1f}% uniqueness\n'.format( name=name, shape=shape, unique=len(unique_hash), percentage=100 * len(unique_hash) / float(shape[0])) for name2, shape2, unique_hash2 in artifacts_summary[i + 1:]: intersection = len(unique_hash & unique_hash2) summary += '\tIntersection with [{name2}] {intersection} rows: {percentage:.1f}%\n'.format( name2=name2, intersection=intersection, percentage=100 * intersection / float(len(unique_hash2))) except Exception as e: LoggerRoot.get_base_logger().warning(str(e)) finally: thread_pool.close() thread_pool.terminate() return summary def _get_temp_folder(self, force_new=False): if force_new or not self._temp_folder: new_temp = mkdtemp(prefix='artifacts_') self._temp_folder.append(new_temp) return new_temp return self._temp_folder[0] def _get_storage_uri_prefix(self): if not self._storage_prefix: self._storage_prefix = self._task._get_output_destination_suffix() return self._storage_prefix @staticmethod def sha256sum(filename, skip_header=0): # create sha2 of the file, notice we skip the header of the file (32 bytes) # because sometimes that is the only change h = hashlib.sha256() file_hash = hashlib.sha256() b = bytearray(Artifacts._hash_block_size) mv = memoryview(b) try: with open(filename, 'rb', buffering=0) as f: # skip header if skip_header: file_hash.update(f.read(skip_header)) for n in iter(lambda: f.readinto(mv), 0): h.update(mv[:n]) if skip_header: file_hash.update(mv[:n]) except Exception as e: LoggerRoot.get_base_logger().warning(str(e)) return None, None return h.hexdigest(), file_hash.hexdigest() if skip_header else None
class EventsDevice(Process): def __init__(self, log_dir): super(EventsDevice, self).__init__() self._ready = Event() self._pub_port = Value('d', 0) self._sub_port = Value('d', 0) self.event_log_base_dir = os.path.join(log_dir, 'events_log') makedirs(self.event_log_base_dir) self.raw_events_filename = os.path.join(self.event_log_base_dir, 'raw_events.log') def run(self): # pylint: disable=no-member; disable `no-member' messages because of zmq. context = frontend = backend = None try: context = zmq.Context(1) # Socket facing clients. frontend = context.socket(zmq.SUB) self._pub_port.value = frontend.bind_to_random_port("tcp://*") frontend.setsockopt(zmq.SUBSCRIBE, b"") frontend.setsockopt(zmq.LINGER, 0) # Socket facing services. backend = context.socket(zmq.PUB) self._sub_port.value = backend.bind_to_random_port("tcp://*") backend.setsockopt(zmq.LINGER, 0) self._ready.set() LOGGER.info("EventsDevice listen on pub_port=%d, sub_port=%d", self._pub_port.value, self._sub_port.value) zmq.proxy(frontend, backend) except Exception: # pylint: disable=broad-except LOGGER.exception("zmq device failed") except (KeyboardInterrupt, SystemExit) as ex: LOGGER.debug("EventsDevice was halted by %s", ex.__class__.__name__) finally: if frontend: frontend.close() if backend: backend.close() if context: context.term() @staticmethod @timeout(timeout=120) def wait_till_event_loop_is_working(number_of_events): """ It waits 120 seconds till row of {number_of_events} events is delivered with no loss """ for _ in range(number_of_events): try: StartupTestEvent().publish(guaranteed=True) except TimeoutError: raise RuntimeError("Event loop is not working properly") @property def sub_port(self): if self._ready.wait(timeout=EVENTS_DEVICE_START_TIMEOUT): return self._sub_port raise RuntimeError("EventsDevice is not ready to send events.") @property def pub_port(self): if self._ready.wait(timeout=EVENTS_DEVICE_START_TIMEOUT): return self._pub_port raise RuntimeError("EventsDevice is not ready to receive events.") def get_client_socket(self, filter_type=b''): context = zmq.Context() socket = context.socket(zmq.SUB) # pylint: disable=no-member socket.connect("tcp://localhost:%d" % self.sub_port.value) socket.setsockopt(zmq.SUBSCRIBE, filter_type) # pylint: disable=no-member return socket def subscribe_events(self, filter_type=b'', stop_event=None): # pylint: disable=too-many-nested-blocks,too-many-branches LOGGER.info("subscribe to server with port %d", self.sub_port.value) socket = self.get_client_socket(filter_type) filters = dict() try: while stop_event is None or not stop_event.isSet(): if socket.poll(timeout=1): obj = socket.recv_pyobj() # remove filter objects when log event timestamp on the # specific node is bigger the time filter was canceled if isinstance(obj, DatabaseLogEvent): for filter_key, filter_obj in list(filters.items()): if filter_obj.expire_time and filter_obj.expire_time < obj.timestamp: del filters[filter_key] obj_filtered = any( [f.eval_filter(obj) for f in filters.values()]) if isinstance(obj, DbEventsFilter): if not obj.clear_filter: filters[obj.id] = obj else: object_filter = filters.get(obj.id, None) if object_filter is None: filters[obj.id] = obj else: filters[obj.id].expire_time = obj.expire_time if not obj.expire_time: del filters[obj.id] obj_filtered = obj_filtered or isinstance(obj, SystemEvent) if not obj_filtered: yield obj.__class__.__name__, obj except (KeyboardInterrupt, SystemExit) as ex: LOGGER.debug("%s - subscribe_events was halted by %s", current_process().name, ex.__class__.__name__) socket.close() def publish_event(self, event): context = zmq.Context() socket = context.socket(zmq.PUB) # pylint: disable=no-member socket.connect("tcp://localhost:%d" % self.pub_port.value) time.sleep(0.01) socket.send_pyobj(event) with open(self.raw_events_filename, 'a+') as log_file: log_file.write(event.to_json() + '\n') socket.close() return True @retrying(n=3, sleep_time=0, allowed_exceptions=TimeoutError) def publish_event_guaranteed(self, event): client_socket = self.get_client_socket() self.publish_event(event) end_time = time.time() + 2 while time.time() < end_time: # This iteration cycle needed to make sure that it does not stop on getting very first event, # which could be event not it is looking for, but something that is generated in other Thread try: if not client_socket.poll(timeout=1): continue received_event = client_socket.recv_pyobj(flags=1) if event == received_event: return True except zmq.ZMQError: continue raise TimeoutError(f"Event {str(self)} was not delivered")
d['2'] = 2 d[0.25] = None l.reverse() ev.set() #wake up main if __name__ == '__main__': event = Event() #flag is false lock = RLock() #Data can be stored in a shared memory map using Value or Array num = Value( 'd', 0.0) #typecode from array module, eg int(i), double(d), long(l), arr = Array('i', range(10)) p1 = Process(target=sharedf, args=(num, arr, event)) p1.start() event.wait() #or call event.is_set() with lock: print(num.value) print(arr[:]) #Manager #controls a server process which holds Python objects #and allows other processes to manipulate them using proxies #supports list, dict, Namespace, #Lock, RLock, Semaphore, BoundedSemaphore, Condition, Event, Queue, #Value and Array manager = Manager() event2 = manager.Event() d = manager.dict() l = manager.list(range(10)) p2 = Process(target=managerf, args=(d, l, event2))
class WriteBack(object): _changed = False _process = None chunk_size = 100 def __init__(self, db, storage): self._db = db self._storage = storage def close(self): if self._process: self._stop.set() self._event.set() self._process.join() def changed(self): self._changed = True def committed(self): if self._changed: self._changed = False if self._process: self._event.set() else: if FORK: from multiprocessing import Process, Event else: from threading import Thread as Process, Event self._event = Event() self._idle = Event() self._stop = Event() self._np = 1 + self._db._getMaxPartition() self._db = cPickle.dumps(self._db, 2) self._process = Process(target=self._run) self._process.daemon = True self._process.start() @property def wait(self): # For unit tests. return self._idle.wait def _run(self): util.setproctitle('neostorage: write back') self._db = cPickle.loads(self._db) try: @self._db.autoReconnect def _(): # Unfortunately, copyTransactionsFrom does not abort in case # of failure, so we have to reopen. zodb = storageFromString(self._storage) try: self.min_tid = util.add64(zodb.lastTransaction(), 1) zodb.copyTransactionsFrom(self) finally: zodb.close() finally: self._idle.set() self._db.close() def iterator(self): db = self._db np = self._np offset_list = xrange(np) while 1: with db: # Check the partition table at the beginning of every # transaction. Once the import is finished and at least one # cell is replicated, it is possible that some of this node # get outdated. In this case, wait for the next PT change. if np == len(db._readable_set): while 1: tid_list = [] max_tid = MAX_TID for offset in offset_list: x = db.getReplicationTIDList( self.min_tid, max_tid, self.chunk_size, offset) tid_list += x if len(x) == self.chunk_size: max_tid = x[-1] if not tid_list: break tid_list.sort() for tid in tid_list: if self._stop.is_set(): return yield TransactionRecord(db, tid) if tid == max_tid: break else: self.min_tid = util.add64(tid, 1) break self.min_tid = util.add64(tid, 1) if not self._event.is_set(): self._idle.set() self._event.wait() self._idle.clear() self._event.clear() if self._stop.is_set(): break