def test_one_activity_heartbeat_cancel_raise(self): # test heartbeat activity raising cancel exception class OneActivityHeartbeatCancelRaiseWorkflow(WorkflowDefinition): def __init__(self, workflow_execution): super(OneActivityHeartbeatCancelRaiseWorkflow, self).__init__(workflow_execution) self.activities_client = BunchOfActivities() @execute(version='1.1', execution_start_to_close_timeout=60) def execute(self): activity_future = self.activities_client.heartbeating_activity(10) yield workflow_time.sleep(2) yield activity_future.cancel() yield activity_future return_(False) wf = OneActivityHeartbeatCancelRaiseWorkflow wf_worker, act_worker = self.get_workers(wf) self.start_workflow(wf) act_worker_thread = Thread(target=act_worker.run_once) act_worker_thread.start() wf_worker.run_once() # schedule first two activities time.sleep(1) # ensure activity started wf_worker.run_once() # cancel decision wf_worker.run_once() # raise cancel act_worker_thread.join() time.sleep(1) hist = self.get_workflow_execution_history() self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCanceled') self.assertEqual(hist[-1]['workflowExecutionCanceledEventAttributes']['details'], 'Cancel was requested during activity heartbeat') # hist differs depending on whether the heartbeat activity started or not self.assertEqual(len(hist), 17)
def getSaveSymbols(self): ''' get and save data ''' counter = 0 rounds = 0 while counter < len(self.symbols): size = len(self.symbols) - counter if BATCH_SIZE < size: size = BATCH_SIZE symbols = self.symbols[counter: counter + size] threads = [] for symbol in symbols: thread = Thread(name = symbol, target = self.__getSaveOneSymbol, args = [symbol]) thread.daemon = True thread.start() threads.append(thread) for thread in threads: thread.join(THREAD_TIMEOUT) # no need to block, because thread should complete at last #can't start another thread to do commit because for sqlLite, only object for the same thread can be commited if 0 == rounds % 3: self.outputDAM.commit() counter += size rounds += 1
def checkTimeOutPut(args): global currCommandProcess global stde global stdo stde = None stdo = None def executeCommand(): global currCommandProcess global stdo global stde try: stdo, stde = currCommandProcess.communicate() printLog('stdout:\n'+str(stdo)) printLog('stderr:\n'+str(stde)) except: printLog("ERROR: UNKNOWN Exception - +checkWinTimeOutPut()::executeCommand()") currCommandProcess = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE) thread = Thread(target=executeCommand) thread.start() thread.join(TIMOUT_VAL) #wait for the thread to complete if thread.is_alive(): printLog('ERROR: Killing the process - terminating thread because it is taking too much of time to execute') currCommandProcess.kill() printLog('ERROR: Timed out exception') raise errorHandler.ApplicationException(__file__, errorHandler.TIME_OUT) if stdo == "" or stdo==None: errCode = currCommandProcess.poll() printLog('ERROR: @@@@@Raising Called processor exception') raise subprocess.CalledProcessError(errCode, args, output=stde) return stdo
def test_cancel_workflow_with_activity_cascade(self): class SelfCancellingWorkflowWithCascade(WorkflowDefinition): def __init__(self, workflow_execution): super(SelfCancellingWorkflowWithCascade, self).__init__(workflow_execution) self.activities_client = BunchOfActivities() @execute(version='1.1', execution_start_to_close_timeout=60) def execute(self): self.activities_client.heartbeating_activity(5) yield workflow_time.sleep(1) self.cancel() return_(True) wf = SelfCancellingWorkflowWithCascade wf_worker, act_worker = self.get_workers(wf) self.start_workflow(wf) act_worker_thread = Thread(target=act_worker.run_once) act_worker_thread.start() wf_worker.run_once() # start activity wf_worker.run_once() # cancel workflow and the heartbeat activity act_worker_thread.join() time.sleep(1) hist = self.get_workflow_execution_history() self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCanceled') self.assertEqual(hist[-2]['eventType'], 'ActivityTaskCancelRequested') # hist differs depending on whether the heartbeat activity started or not self.assertEqual(len(hist), 13)
class TestStatsdLoggingDelegation(unittest.TestCase): def setUp(self): self.port = 9177 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.bind(('localhost', self.port)) self.queue = Queue() self.reader_thread = Thread(target=self.statsd_reader) self.reader_thread.setDaemon(1) self.reader_thread.start() def tearDown(self): # The "no-op when disabled" test doesn't set up a real logger, so # create one here so we can tell the reader thread to stop. if not getattr(self, 'logger', None): self.logger = utils.get_logger({ 'log_statsd_host': 'localhost', 'log_statsd_port': str(self.port), }, 'some-name') self.logger.increment('STOP') self.reader_thread.join(timeout=4) self.sock.close() del self.logger time.sleep(0.15) # avoid occasional "Address already in use"? def statsd_reader(self): while True: try: payload = self.sock.recv(4096) if payload and 'STOP' in payload: return 42 self.queue.put(payload) except Exception, e: sys.stderr.write('statsd_reader thread: %r' % (e,)) break
def run_command(self, cmd, input_data=None): """ Run a command in a child process , passing it any input data specified. :param cmd: The command to run. :param input_data: If specified, this must be a byte string containing data to be sent to the child process. :return: A tuple consisting of the subprocess' exit code, a list of lines read from the subprocess' ``stdout``, and a list of lines read from the subprocess' ``stderr``. """ kwargs = { 'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE, } if input_data is not None: kwargs['stdin'] = subprocess.PIPE stdout = [] stderr = [] p = subprocess.Popen(cmd, **kwargs) # We don't use communicate() here because we may need to # get clever with interacting with the command t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout)) t1.start() t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr)) t2.start() if input_data is not None: p.stdin.write(input_data) p.stdin.close() p.wait() t1.join() t2.join() return p.returncode, stdout, stderr
def run(**kwargs): thread_queue = [] for provider in list_providers("openstack"): mgmt_sys = cfme_data['management_systems'][provider] rhos_credentials = credentials[mgmt_sys['credentials']] default_host_creds = credentials['host_default'] username = rhos_credentials['username'] password = rhos_credentials['password'] auth_url = mgmt_sys['auth_url'] rhosip = mgmt_sys['ipaddress'] sshname = default_host_creds['username'] sshpass = default_host_creds['password'] if not net.is_pingable(rhosip): continue if not net.net_check(ports.SSH, rhosip): print("SSH connection to {}:{} failed, port unavailable".format( provider, ports.SSH)) continue thread = Thread(target=upload_template, args=(rhosip, sshname, sshpass, username, password, auth_url, provider, kwargs.get('image_url'), kwargs.get('template_name'))) thread.daemon = True thread_queue.append(thread) thread.start() for thread in thread_queue: thread.join()
def clear_lines(n): """ Clear the space before using it """ thread0 = Thread(name='Clear Lines', target=clear, args=(n,)) thread0.start() thread0.join()
def test_request_retries_configurable(self): # We guess at some ports that will be unused by Riak or # anything else. client = self.create_client(http_port=DUMMY_HTTP_PORT, pb_port=DUMMY_PB_PORT) # Change the retry count client.retries = 10 self.assertEqual(10, client.retries) # The retry count should be a thread local retries = Queue() def _target(): retries.put(client.retries) retries.join() th = Thread(target=_target) th.start() self.assertEqual(3, retries.get(block=True)) retries.task_done() th.join() # Modify the retries in a with statement with client.retry_count(5): self.assertEqual(5, client.retries) self.assertRaises(IOError, client.ping)
def query(id1, id2): logging.debug("query starts") # global start_time # start_time = time() # conn = httplib.HTTPSConnection(hostname) expr1 = "Or(Id="+id1+",Composite(AA.AuId=" +id1+ "))" expr2 = "Or(Id="+id2+",Composite(AA.AuId=" +id2+ "))" #这两次请求好像比较慢一些 thread1 = Thread(target=send_request, args=[expr1]) thread2 = Thread(target=send_request, args=[expr2]) thread1.start() thread2.start() thread1.join() thread2.join() logging.debug("first 2 requests finished") # print "time: ", time() - start_time data1 = request_data[expr1] data2 = request_data[expr2] len1 = len(data1) len2 = len(data2) # 这里假定每个作者肯定写一篇以上的文章咯~ mr if(len1 <= 1): if(len2 <= 1): return query1(data1, data2, id1, id2) else: return query2(data1, data2, id1, id2) elif(len2 <= 1): return query3(data1, data2, id1, id2) else: return query4(data1, data2, id1, id2)
def main(self): global QUEUE QUEUE = TaskQueue(self.config) indexers = self.buildIndexers() for indexer in indexers: QUEUE.put(indexer) #start stat printing if self.statManager != None: timer = Timer() timer.scheduleAtFixedRate(StatLoggerTask(self.statManager, indexers), 0, self.config.statLoggingFrequency*60*1000) #start worker threads workers = [] for i in range(self.config.numThreads): t = Thread (target=ISWorker(self.config, self.statManager)) workers.append(t) t.setDaemon(1) t.start() for t in workers: t.join() log('Done!')
def cleanup_chunk(days, project_id, model, dtfield, order_by, num_shards, shard_ids): import pickle from threading import Thread model = pickle.loads(model) shard_ids = [int(s) for s in shard_ids.split(",")] task = create_deletion_task( days, project_id, model, dtfield, order_by) click.echo("days: %s, project_id: %s, model: %s, dtfield: %s, order_by: %s, shard_ids:%s" % (days, project_id, model, dtfield, order_by, shard_ids)) threads = [] for shard_id in shard_ids: t = Thread( target=( lambda shard_id=shard_id: _chunk_until_complete( task, num_shards=num_shards, shard_id=shard_id) ) ) t.start() threads.append(t) for t in threads: t.join()
class PseudoSocketServer(Server, PseudoSide): def __init__(self, file_path): if os.path.exists(file_path): os.remove(file_path) self._file_path = file_path self._listening = False self._async_listener = Thread(target=self._listen_task) def _listen_task(self): print("[server] Connects by {!s}".format(self._file_path)) open(self._file_path, 'a').close() self._listening = True while self._listening: received = self.read_and_remove(self._file_path, "server") if received: print("[server] Server got message") answer = self.HELLO_FORMAT.format(received) self.append_to_file(self._file_path, answer, "server") time.sleep(0.25) def listen(self): self._async_listener.start() def close(self): print("[server] Shutdowns socket") self._listening = False if self._async_listener.is_alive(): print("[server] Joins background task") self._async_listener.join()
def test_get_routes_multithreads(self): """Test case that ensures routes register only once even if multiple threads call get routes.""" self._settings_facade.get = Mock( return_value=[ "fantastico.routing_engine.tests.test_router.TestLoader", "fantastico.routing_engine.tests.test_router.TestLoader2", ] ) threads = [] def get_routes_async(router): router.register_routes() for i in range(0, 10): thread = Thread(target=get_routes_async, name="GetRoutesThread-%s" % i, kwargs={"router": self._router}) threads.append(thread) thread.start() for thread in threads: thread.join(100) routes = self._router.register_routes() self.assertIsNotNone(routes) self.assertEqual( "fantastico.routing_engine.tests.test_router.Controller.do_stuff", routes.get("/index.html")["http_verbs"]["GET"], ) self.assertEqual( "fantastico.routing_engine.tests.test_router.Controller.do_stuff2", routes.get("/index2.html")["http_verbs"]["GET"], )
class AlarmExecutor: def __init__(self): self.queue = JoinableQueue(10) self.running = False self.t = Thread(target=self._run, name="AlarmExecutor") def _run(self): while self.running: try: alarm = self.queue.get(block=True, timeout=1) alarm.execute() logging.debug("Alarm executed") self.queue.task_done() except Queue.Empty: continue def start(self): logging.debug("Starting alarm executor") self.running = True self.t.start() def stop(self): if self.running: logging.debug("Stoppping alarm executor") self.running = False self.t.join() else: logging.debug("Attempted to stop alarm executor when it is not running")
def test_root_crt_rotate_cluster(self): rest = RestConnection(self.master) x509main(self.master).setup_master() x509main().setup_cluster_nodes_ssl(self.servers) rest.create_bucket(bucket='default', ramQuotaMB=100) self.sleep(30) servers_in = self.servers[1:] self.cluster.rebalance(self.servers, servers_in, []) for server in self.servers: result = self._sdk_connection(host_ip=server.ip) self.assertTrue(result,"Can create a ssl connection with correct certificate") result,cb = self._sdk_connection(host_ip=self.master.ip) create_docs = Thread(name='create_docs', target=self.createBulkDocuments, args=(cb,)) create_docs.start() x509main(self.master)._delete_inbox_folder() x509main(self.master)._generate_cert(self.servers,root_cn="CB\ Authority") x509main(self.master).setup_master() x509main().setup_cluster_nodes_ssl(self.servers,reload_cert=True) create_docs.join() for server in self.servers: result = self._sdk_connection(host_ip=server.ip) self.assertTrue(result,"Can create a ssl connection with correct certificate")
def test_get_loaders_multithreads(self): """Test case that ensures loaders are loaded by one single thread even if multiple threads request loaders list.""" self._settings_facade.get = Mock( return_value=[ "fantastico.routing_engine.tests.test_router.TestLoader", "fantastico.routing_engine.tests.test_router.TestLoader2", "fantastico.routing_engine.tests.test_router.TestLoader3", ] ) threads = [] def get_loaders_async(router): router.get_loaders() for i in range(0, 10): thread = Thread(target=get_loaders_async, name="GetLoadersThread-%s" % i, kwargs={"router": self._router}) threads.append(thread) thread.start() for thread in threads: thread.join(100) loaders = self._router.get_loaders() self.assertIsNotNone(loaders) self.assertEqual(3, len(loaders)) self.assertIsInstance(loaders[0], TestLoader) self.assertIsInstance(loaders[1], TestLoader2) self.assertIsInstance(loaders[2], TestLoader3)
class SimpleServer(object): """Helper class which starts a simple server listening on localhost at the specified port """ def __init__(self): self.port = find_available_port() self.handler = SimpleHTTPServer.SimpleHTTPRequestHandler self.httpd = SocketServer.TCPServer(("", self.port), self.handler) self.close_signal = threading.Event() self.server_started = False def start(self, delay_sec=0.0): """Run the server after specified delay""" def run(): self.close_signal.wait(delay_sec) if not self.close_signal.is_set(): self.server_started = True self.httpd.serve_forever() self.background_thread = Thread(target=run) self.background_thread.start() def stop(self): self.close_signal.set() if self.server_started: self.httpd.shutdown() self.background_thread.join(timeout=.5) if self.background_thread.is_alive(): raise Exception("SimpleServer failed to stop quickly")
def test_with(): abstractdisplay.RANDOMIZE_DISPLAY_NR = False ls = [0, 0] def f1(): ls[0] = Display() # d1.start() def f2(): ls[1] = Display() # d2.start() t1 = Thread(target=f1) t2 = Thread(target=f2) t1.start() t2.start() t1.join() t2.join() # print ls dv1 = ls[0].new_display_var dv2 = ls[1].new_display_var # print dv1 # print dv2 ok_(dv1 != dv2)
def test_exception_propagation(self): def apprunner(configkey): app = flask.Flask(__name__) @app.route('/') def index(): 1/0 c = app.test_client() if config_key is not None: app.config[config_key] = True try: resp = c.get('/') except Exception: pass else: self.fail('expected exception') else: self.assert_equal(c.get('/').status_code, 500) # we have to run this test in an isolated thread because if the # debug flag is set to true and an exception happens the context is # not torn down. This causes other tests that run after this fail # when they expect no exception on the stack. for config_key in 'TESTING', 'PROPAGATE_EXCEPTIONS', 'DEBUG', None: t = Thread(target=apprunner, args=(config_key,)) t.start() t.join()
def Command(*cmd, **kwargs): """Enables to run subprocess commands in a different thread with TIMEOUT option! Based on jcollado's solution: http://stackoverflow.com/questions/1191374/subprocess-with-timeout/4825933#4825933 and https://gist.github.com/1306188 """ if kwargs.has_key("timeout"): timeout = kwargs["timeout"] del kwargs["timeout"] else: timeout = None process = [] def target(process,out,*cmd,**k): process.append(subprocess.Popen(cmd,stdout=subprocess.PIPE,**k)) out.put(process[0].communicate()[0]) outQueue = Queue() args = [process,outQueue] args.extend(cmd) thread = Thread(target=target, args=args, kwargs=kwargs) thread.start() thread.join(timeout) if thread.is_alive(): process[0].terminate() thread.join() raise Empty return outQueue.get()
def test_get_spatial_during_x_min_load_y_working_set_multiple_design_docs( self): num_docs = self.helper.input.param("num-docs", 10000) num_design_docs = self.helper.input.param("num-design-docs", 10) duration = self.helper.input.param("load-time", 1) self.log.info("description : will create {0} docs per design doc and " "{1} design docs that will be queried while the data " "is loaded for {2} minutes" .format(num_docs, num_design_docs, duration)) name = "dev_test_spatial_test_{0}_docs_{1}_design_docs_{2}_mins_load"\ .format(num_docs, num_design_docs, duration) view_test_threads = [] for i in range(0, num_design_docs): prefix = str(uuid.uuid4())[:7] design_name = "{0}-{1}-{2}".format(name, i, prefix) thread_result = [] t = Thread( target=SpatialViewTests._test_multiple_design_docs_thread_wrapper, name="Insert documents and query multiple design docs in parallel", args=(self, num_docs, duration, design_name, prefix, thread_result)) t.start() view_test_threads.append((t, thread_result)) for (t, failures) in view_test_threads: t.join() for (t, failures) in view_test_threads: if len(failures) > 0: self.fail("view thread failed : {0}".format(failures[0]))
def runHook(hookFunc, *args): class Hook(object): def __init__(self): self._stop = True def ended(self): self._stop = False def stop(self): return self._stop hookName = hookFunc.__name__ # Don't run hooks for action unitTests. if hookName == 'preHook': if args[0] == 'unitTests': return False hook = Hook() args += (hook,) thread = Thread(name=hookName, target=hookFunc, args=args, daemon=True) thread.start() thread.join(10) # TODO: get timeout from rascal. return hook.stop()
def test_bad_negotiation(self): portnum = find_available_port() servsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) servsock.bind(('localhost', portnum)) pc = c3.Connection( SocketFactory( (socket.AF_INET, socket.SOCK_STREAM), ('localhost', portnum) ), {} ) exc = None servsock.listen(1) def client_thread(): pc.connect() client = Thread(target = client_thread) try: client.start() c, addr = servsock.accept() try: c.recv(1024) finally: c.close() time.sleep(0.25) client.join() servsock.close() self.assertEqual(pc.xact.fatal, True) self.assertEqual(pc.xact.__class__, x3.Negotiation) self.assertEqual(pc.xact.error_message.__class__, e3.ClientError) self.assertEqual(pc.xact.error_message[b'C'], '08006') finally: servsock.close() if pc.socket is not None: pc.socket.close()
def api_call_threaded(self, requests): """ Interface for a multi-threaded API call, to speed up the request/response cycle when calling multiple endpoints when rendering a template. """ # Threads / responses threads = [] # Process each request for key,attr in requests.iteritems(): # Get any request data data = None if (len(attr) == 2) else attr[2] # Create the thread, append, and start thread = Thread(target=self._api_call_threaded_worker, args=[key, attr[0], attr[1], data]) threads.append(thread) thread.start() # Wait for the API calls to complete for thread in threads: thread.join() self.log.info('API_CALL_THREADED: %s' % str(self._response)) # Return the response object return self._response
def power_on(vm): """Powers on virtual machine and answers any input messages that might appear. """ # vm.powerOn() power_thd = Thread(target = vm.powerOn) power_thd.start() power_thd.join()
def test_SSL_failure(self): portnum = find_available_port() servsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) with servsock: servsock.bind(('localhost', portnum)) pc = c3.Connection( SocketFactory( (socket.AF_INET, socket.SOCK_STREAM), ('localhost', portnum) ), {} ) exc = None servsock.listen(1) def client_thread(): pc.connect(ssl = True) client = Thread(target = client_thread) try: client.start() c, addr = servsock.accept() with c: c.send(b'S') c.sendall(b'0000000000000000000000') c.recv(1024) c.close() client.join() finally: if pc.socket is not None: pc.socket.close() self.assertEqual(pc.xact.fatal, True) self.assertEqual(pc.xact.__class__, x3.Negotiation) self.assertEqual(pc.xact.error_message.__class__, e3.ClientError) self.assertTrue(hasattr(pc.xact, 'exception'))
def test_term_thread(self): """ctx.term should not crash active threads (#139)""" ctx = self.Context() evt = Event() evt.clear() def block(): s = ctx.socket(zmq.REP) s.bind_to_random_port('tcp://127.0.0.1') evt.set() try: s.recv() except zmq.ZMQError as e: self.assertEqual(e.errno, zmq.ETERM) return finally: s.close() self.fail("recv should have been interrupted with ETERM") t = Thread(target=block) t.start() evt.wait(1) self.assertTrue(evt.is_set(), "sync event never fired") time.sleep(0.01) ctx.term() t.join(timeout=1) self.assertFalse(t.is_alive(), "term should have interrupted s.recv()")
def handle_noargs(self, **options): verbose = int(options['verbosity']) send_all = True if verbose: print 'Starting sending newsletters...' activate(settings.LANGUAGE_CODE) senders = SMTPServer.objects.all() workers = [] for sender in senders: worker = SMTPMailer(sender, verbose=verbose) thread = Thread(target=functools.partial(worker.run, send_all), name=sender.name) workers.append((worker, thread)) handler = term_handler(workers) for s in [signal.SIGTERM, signal.SIGINT]: signal.signal(s, handler) # first close current connection signals.request_finished.send(sender=self.__class__) for worker, thread in workers: thread.start() signal.pause() # wait for sigterm for worker, thread in workers: if thread.is_alive(): thread.join() sys.exit(0)
def test_spatial_view_on_x_docs_y_design_docs(self): num_docs = self.helper.input.param("num-docs", 10000) num_design_docs = self.helper.input.param("num-design-docs", 21) self.log.info("description : will create {0} docs per design doc and " "{1} design docs that will be queried") name = "dev_test_spatial_test_{0}_docs_y_design_docs"\ .format(num_docs, num_design_docs) prefix = str(uuid.uuid4())[:7] design_names = ["{0}-{1}-{2}".format(name, i, prefix) \ for i in range(0, num_design_docs)] view_test_threads = [] for design_name in design_names: thread_result = [] t = Thread( target=SpatialViewTests._test_spatial_view_thread_wrapper, name="Insert documents and query in parallel", args=(self, num_docs, design_name, thread_result)) t.start() view_test_threads.append((t, thread_result)) for (t, failures) in view_test_threads: t.join() for (t, failures) in view_test_threads: if len(failures) > 0: self.fail("view thread failed : {0}".format(failures[0]))
def video_feed(): clientVideoSocketUniv = socket(family=AF_INET, type=SOCK_STREAM) clientVideoSocketUniv.connect((HOST, PORT_UNIV)) wvs = WebcamVideoStream(0).start() clientAudioSocket = socket(family=AF_INET, type=SOCK_STREAM) clientAudioSocket.connect((HOST, PORT_AUDIO)) PORTNUMBER = clientVideoSocketUniv.recv(4).decode() clientVideoSocket1 = socket(family=AF_INET, type=SOCK_STREAM) clientVideoSocket1.connect((HOST, int(PORTNUMBER))) ports[PORTNUMBER] = True for portnos in sorted(ports.keys()): if ports[portnos] == False: clientVideoSocket2 = socket(family=AF_INET, type=SOCK_STREAM) clientVideoSocket2.connect((HOST, int(portnos))) ports[portnos] = True RecieveFrameThread1 = Thread(target=RecieveFrame, args=(clientVideoSocket2, )).start() print(portnos, ' - Connected !') break for portnos in sorted(ports.keys()): if ports[portnos] == False: clientVideoSocket3 = socket(family=AF_INET, type=SOCK_STREAM) clientVideoSocket3.connect((HOST, int(portnos))) ports[portnos] = True RecieveFrameThread2 = Thread(target=RecieveFrame, args=(clientVideoSocket3, )).start() print(portnos, ' - Connected !') break for portnos in sorted(ports.keys()): if ports[portnos] == False: clientVideoSocket4 = socket(family=AF_INET, type=SOCK_STREAM) clientVideoSocket4.connect((HOST, int(portnos))) ports[portnos] = True RecieveFrameThread3 = Thread(target=RecieveFrame, args=(clientVideoSocket4, )).start() print(portnos, ' - Connected !') break audio = pyaudio.PyAudio() stream = audio.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, output=True, frames_per_buffer=CHUNK) closethread = Thread(target=initialize, args=(clientVideoSocketUniv, )) closethread.start() SendAudioThread = Thread(target=SendAudio, args=( clientAudioSocket, stream, )) RecieveAudioThread = Thread(target=RecieveAudio, args=( clientAudioSocket, stream, )) # DisplayThread = Thread(target=display) ############# TRY ###################### # RecieveAudioThread.start() # DisplayThread.start() # SendAudioThread.start() SendFrameThread = Thread(target=SendFrame, args=(clientVideoSocket1, wvs)).start() return Response(gen(wvs), mimetype='multipart/x-mixed-replace; boundary=frame') # SendAudioThread.join() closethread.join()
class CommonEtcdSynchronizer(object): PAUSE_BEFORE_RETRY_ON_EXCEPTION = 30 PAUSE_BEFORE_RETRY_ON_MISSING_KEY = 5 TIMEOUT_ON_WATCH = 5 def __init__(self, plugin, ip, etcd_ip=None): self._plugin = plugin self._ip = ip cxn_ip = etcd_ip or ip self._client = etcd.Client(cxn_ip, 4000) self._index = None self._last_value = None # Set the terminate flag and the abort read flag to false initially # The terminate flag controls whether the synchronizer as a whole # should terminate, the abort flag ensures that any synchronizer # threads controlled by a futures is shut down fully self._terminate_flag = False self._abort_read = False self.thread = Thread(target=self.main_wrapper, name=self.thread_name()) def start_thread(self): self.thread.daemon = True self.thread.start() def terminate(self): self._terminate_flag = True self.thread.join() def pause(self): sleep(self.PAUSE_BEFORE_RETRY_ON_EXCEPTION) def main_wrapper(self): # pragma: no cover # This function should be the entry point when we start an # EtcdSynchronizer thread. We use it to catch exceptions in main and # restart the whole process; if we didn't do this the thread would be # dead and we'd never notice. try: self.main() except Exception: # Log the exception and send a SIGTERM to this process. If the # process needs to do anything before shutting down, it will have a # handler for catching the SIGTERM. _log.error(traceback.format_exc()) os.kill(os.getpid(), signal.SIGTERM) def main(self): pass def default_value(self): return None def is_running(self): return True def thread_name(self): return self._plugin.__class__.__name__ # Read the state of the cluster from etcd (optionally waiting for a changed # state). Returns None if nothing could be read. def read_from_etcd(self, wait=True, timeout=None): result = None wait_index = None try: result = self._client.read(self.key(), quorum=True, timeout=timeout) wait_index = result.etcd_index + 1 if wait: # If the cluster view hasn't changed since we last saw it, then # wait for it to change before doing anything else. _log.info("Read value {} from etcd, " "comparing to last value {}".format( utils.safely_encode(result.value), utils.safely_encode(self._last_value))) if result.value == self._last_value: _log.info( "Watching for changes with {}".format(wait_index)) while not self._terminate_flag and not self._abort_read and self.is_running( ): _log.debug("Started a new watch") try: result = self._client.read( self.key(), timeout=self.TIMEOUT_ON_WATCH, waitIndex=wait_index, wait=True, recursive=False) break except etcd.EtcdException as e: if "Read timed out" in e.message: # Timeouts after TIMEOUT_ON_WATCH seconds are expected, so # ignore them - unless we're terminating, we'll # stay in the while loop and try again pass else: raise _log.debug("Finished watching") # Return if we're terminating. if self._terminate_flag: return self.tuple_from_result(result) except etcd.EtcdKeyError: _log.info("Key {} doesn't exist in etcd yet".format(self.key())) # Use any value on disk first, but the default value if not found try: f = open(self._plugin.file(), 'r') value = f.read() # pragma: no cover except: value = self.default_value() # Attempt to create new key in etcd. try: # The prevExist set to False will fail the write if it finds a # key already in etcd. This stops us overwriting a manually # uploaded file with the default template. self._client.write(self.key(), value, prevExist=False) return (value, None) except: # pragma: no cover _log.debug("Failed to create new key in the etcd store") # Sleep briefly to avoid hammering a non-existent key sleep(self.PAUSE_BEFORE_RETRY_ON_MISSING_KEY) # Return 'None' so that plugins do not write config to disk # that does not exist in the etcd store, leaving us out of sync return (None, None) except Exception as e: # Catch-all error handler (for invalid requests, timeouts, etc - # start over. _log.error("{} caught {!r} when trying to read with index {}" " - pause before retry".format(self._ip, e, wait_index)) # Sleep briefly to avoid hammering a failed server self.pause() # The main loop (which reads from etcd in a loop) should call this # function again after we return, causing the read to be retried. return self.tuple_from_result(result) def tuple_from_result(self, result): if result is None: return (None, None) elif self._abort_read is True: return (self._last_value, self._index) else: return (result.value, result.modifiedIndex) # Calls read_from_etcd, and updates internal state to track the previously # seen value. # # The difference is: # - calling read_from_etcd twice will return the same value # - calling update_from_etcd twice will block on the second call until the # value changes # # Only the main thread should call update_from_etcd to avoid race conditions # or missed reads. def update_from_etcd(self): self._last_value, self._index = self.read_from_etcd(wait=True) return self._last_value # Use this class instead of the class futures.ThreadPoolExecutor to log any exceptions # that occur inside 'background-started' threads class ThreadPoolExecutorWithExceptionHandler(futures.ThreadPoolExecutor): def submit(self, func): future = super( CommonEtcdSynchronizer.ThreadPoolExecutorWithExceptionHandler, self).submit(func) future.add_done_callback(self.log_exception) return future def log_exception(self, future): try: future.result() except Exception as e: #pragma: no cover _log.exception("%s: %s", type(e).__name__, e.__str__())
class EmailNotification(object): """ Radiosonde Email Notification Class. Accepts telemetry dictionaries from a decoder, and sends an email on newly detected sondes. Incoming telemetry is processed via a queue, so this object should be thread safe. """ # We require the following fields to be present in the input telemetry dict. REQUIRED_FIELDS = ['id', 'lat', 'lon', 'alt', 'type', 'freq'] def __init__(self, smtp_server='localhost', mail_from=None, mail_to=None, mail_subject=None): """ Init a new E-Mail Notification Thread """ self.smtp_server = smtp_server self.mail_from = mail_from self.mail_to = mail_to self.mail_subject = mail_subject # Dictionary to track sonde IDs self.sondes = {} # Input Queue. self.input_queue = Queue() # Start queue processing thread. self.input_processing_running = True self.input_thread = Thread(target=self.process_queue) self.input_thread.start() self.log_info("Started E-Mail Notifier Thread") def add(self, telemetry): """ Add a telemetery dictionary to the input queue. """ # Check the telemetry dictionary contains the required fields. for _field in self.REQUIRED_FIELDS: if _field not in telemetry: self.log_error("JSON object missing required field %s" % _field) return # Add it to the queue if we are running. if self.input_processing_running: self.input_queue.put(telemetry) else: self.log_error("Processing not running, discarding.") def process_queue(self): """ Process packets from the input queue. """ while self.input_processing_running: # Process everything in the queue. while self.input_queue.qsize() > 0: try: _telem = self.input_queue.get_nowait() self.process_telemetry(_telem) except Exception as e: self.log_error("Error processing telemetry dict - %s" % str(e)) # Sleep while waiting for some new data. time.sleep(0.5) def process_telemetry(self, telemetry): """ Process a new telemmetry dict, and send an e-mail if it is a new sonde. """ _id = telemetry['id'] if _id not in self.sondes: try: # This is a new sonde. Send the email. msg = 'Sonde launch detected:\n' msg += '\n' msg += 'Callsign: %s\n' % _id msg += 'Type: %s\n' % telemetry['type'] msg += 'Frequency: %s\n' % telemetry['freq'] msg += 'Position: %.5f,%.5f\n' % (telemetry['lat'], telemetry['lon']) msg += 'Altitude: %dm\n' % round(telemetry['alt']) msg += '\n' #msg += 'https://tracker.habhub.org/#!qm=All&q=RS_%s\n' % _id msg += 'https://sondehub.org/%s\n' % _id msg = MIMEText(msg, 'plain', 'UTF-8') # Construct subject _subject = self.mail_subject _subject = _subject.replace('<id>', telemetry['id']) _subject = _subject.replace('<type>', telemetry['type']) _subject = _subject.replace('<freq>', telemetry['freq']) logging.debug("Email - Subject: %s" % _subject) msg['Subject'] = _subject msg['From'] = self.mail_from msg['To'] = self.mail_to msg["Date"] = formatdate() s = smtplib.SMTP(self.smtp_server) s.sendmail(msg['From'], msg['To'], msg.as_string()) s.quit() self.log_info("E-mail sent.") except Exception as e: self.log_error("Error sending E-mail - %s" % str(e)) self.sondes[_id] = {'last_time': time.time()} def close(self): """ Close input processing thread. """ self.log_debug("Waiting for processing thread to close...") self.input_processing_running = False if self.input_thread is not None: self.input_thread.join() def running(self): """ Check if the logging thread is running. Returns: bool: True if the logging thread is running. """ return self.input_processing_running def log_debug(self, line): """ Helper function to log a debug message with a descriptive heading. Args: line (str): Message to be logged. """ logging.debug("E-Mail - %s" % line) def log_info(self, line): """ Helper function to log an informational message with a descriptive heading. Args: line (str): Message to be logged. """ logging.info("E-Mail - %s" % line) def log_error(self, line): """ Helper function to log an error message with a descriptive heading. Args: line (str): Message to be logged. """ logging.error("E-Mail - %s" % line)
class WebSocketConnection: def __init__(self, delegate, token_func, baseurl): self.delegate = delegate self.token_func = token_func self.baseurl = baseurl self.stop = False self.thread = None self.didstop = True self.last_ping = None self.send_queue = None def _get_url(self): token = self.token_func() while token is None: time.sleep(1) # <-- safe to block since it's called from a dedicated thread who only proceeds when we have the token token = self.token_func() return self.baseurl + '/' + token def start(self): assert(self.thread is None) self.stop = False self.didstop = False self.send_queue = queue.Queue() def _go(): self._connect() log.info("Starting Connection to AutoAuto Server...") self.thread = Thread(target=_go) self.thread.deamon = True self.thread.start() def _connect(self): ws = None send_thread = None def _send_from_queue(): try: while True: msg = self.send_queue.get() if msg is False: break ws.send(msg) if LOG_ALL_MESSAGES: log.info("> {}".format(msg)) except Exception as e: log.info("Send thread terminated with Exception: {}".format(e)) output = io.StringIO() traceback.print_exc(file=output) log.info(output.getvalue()) try: read_timeout_opt = (socket.SOL_SOCKET, socket.SO_RCVTIMEO, struct.pack('LL', SOCKET_READ_TIMEOUT_SECONDS, 0)) tcp_nodelay_opt = (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) url = self._get_url() ws = create_connection(url, sockopt=[read_timeout_opt, tcp_nodelay_opt]) self.onOpen(url) send_thread = Thread(target=_send_from_queue) send_thread.start() while not self.stop: msg = ws.recv() if msg != '': try: msg = json.loads(msg) except: log.info("failed to parse JSON: {}".format(msg)) raise self.onMessage(msg) except Exception as e: self.onError(e) if send_thread: self.send_queue.put(False) send_thread.join() send_thread = None if ws: ws.close() ws = None self.onClose() self.didstop = True def send(self, msg): if isinstance(msg, str): self.send_queue.put(msg) elif isinstance(msg, dict): self.send_queue.put(json.dumps(msg)) else: log.info("Trying to send unknown type: " + str(type(msg))) def send_ping(self): now = time.time() if self.last_ping is None or now - self.last_ping > PING_INTERVAL_SECONDS: self.send({'ping': True}) self.last_ping = now def close(self): self.stop = True if self.thread: self.thread.join() self.thread = None def onOpen(self, url): log.info("Connected: {}...".format(url[:44])) if hasattr(self.delegate, 'onOpen'): try: self.delegate.onOpen() except Exception as e: log.info("delegate's onOpen callback threw error: {}".format(e)) output = io.StringIO() traceback.print_exc(file=output) log.info(output.getvalue()) def onClose(self): log.info("Closing...") if hasattr(self.delegate, 'onClose'): try: self.delegate.onClose() except Exception as e: log.info("delegate's onClose callback threw error: {}".format(e)) output = io.StringIO() traceback.print_exc(file=output) log.info(output.getvalue()) def onMessage(self, msg): if LOG_ALL_MESSAGES: log.info("< {}".format(msg)) if 'whothere' in msg: self.send({'me': True}) return if 'ping' in msg: self.send({'pong': True}) return if hasattr(self.delegate, 'onMessage'): self.delegate.onMessage(msg) def onError(self, e): log.info("onError callback got error: {}".format(e)) output = io.StringIO() traceback.print_exc(file=output) log.info(output.getvalue()) if hasattr(self.delegate, 'onError'): try: self.delegate.onError(e) except Exception as e: log.info("delegate's onError callback threw error: {}".format(e)) output = io.StringIO() traceback.print_exc(file=output) log.error(output.getvalue())
class ServiceManager( object ): """ Manages the scheduling of services. """ def __init__(self, jobStore, toilState): logger.debug("Initializing service manager") self.jobStore = jobStore self.toilState = toilState self.jobGraphsWithServicesBeingStarted = set() self._terminate = Event() # This is used to terminate the thread associated # with the service manager self._jobGraphsWithServicesToStart = Queue() # This is the input queue of # jobGraphs that have services that need to be started self._jobGraphsWithServicesThatHaveStarted = Queue() # This is the output queue # of jobGraphs that have services that are already started self._serviceJobGraphsToStart = Queue() # This is the queue of services for the # batch system to start self.jobsIssuedToServiceManager = 0 # The number of jobs the service manager # is scheduling # Start a thread that starts the services of jobGraphs in the # jobsWithServicesToStart input queue and puts the jobGraphs whose services # are running on the jobGraphssWithServicesThatHaveStarted output queue self._serviceStarter = Thread(target=self._startServices, args=(self._jobGraphsWithServicesToStart, self._jobGraphsWithServicesThatHaveStarted, self._serviceJobGraphsToStart, self._terminate, self.jobStore)) def start(self): """ Start the service scheduling thread. """ self._serviceStarter.start() def scheduleServices(self, jobGraph): """ Schedule the services of a job asynchronously. When the job's services are running the jobGraph for the job will be returned by toil.leader.ServiceManager.getJobGraphsWhoseServicesAreRunning. :param toil.jobGraph.JobGraph jobGraph: wrapper of job with services to schedule. """ # Add jobGraph to set being processed by the service manager self.jobGraphsWithServicesBeingStarted.add(jobGraph) # Add number of jobs managed by ServiceManager self.jobsIssuedToServiceManager += sum(map(len, jobGraph.services)) + 1 # The plus one accounts for the root job # Asynchronously schedule the services self._jobGraphsWithServicesToStart.put(jobGraph) def getJobGraphWhoseServicesAreRunning(self, maxWait): """ :param float maxWait: Time in seconds to wait to get a jobGraph before returning :return: a jobGraph added to scheduleServices whose services are running, or None if no such job is available. :rtype: JobGraph """ try: jobGraph = self._jobGraphsWithServicesThatHaveStarted.get(timeout=maxWait) self.jobGraphsWithServicesBeingStarted.remove(jobGraph) assert self.jobsIssuedToServiceManager >= 0 self.jobsIssuedToServiceManager -= 1 return jobGraph except Empty: return None def getServiceJobsToStart(self, maxWait): """ :param float maxWait: Time in seconds to wait to get a job before returning. :return: a tuple of (serviceJobStoreID, memory, cores, disk, ..) representing a service job to start. :rtype: toil.job.ServiceJobNode """ try: serviceJob = self._serviceJobGraphsToStart.get(timeout=maxWait) assert self.jobsIssuedToServiceManager >= 0 self.jobsIssuedToServiceManager -= 1 return serviceJob except Empty: return None def killServices(self, services, error=False): """ :param dict services: Maps service jobStoreIDs to the communication flags for the service """ for serviceJobStoreID in services: serviceJob = services[serviceJobStoreID] if error: self.jobStore.deleteFile(serviceJob.errorJobStoreID) self.jobStore.deleteFile(serviceJob.terminateJobStoreID) def isActive(self, serviceJobNode): """ Returns true if the service job has not been told to terminate. :rtype: boolean """ return self.jobStore.fileExists(serviceJobNode.terminateJobStoreID) def isRunning(self, serviceJobNode): """ Returns true if the service job has started and is active :rtype: boolean """ return (not self.jobStore.fileExists(serviceJobNode.startJobStoreID)) and self.isActive(serviceJobNode) def check(self): """ Check on the service manager thread. :raise RuntimeError: If the underlying thread has quit. """ if not self._serviceStarter.is_alive(): raise RuntimeError("Service manager has quit") def shutdown(self): """ Cleanly terminate worker threads starting and killing services. Will block until all services are started and blocked. """ logger.debug('Waiting for service manager thread to finish ...') startTime = time.time() self._terminate.set() self._serviceStarter.join() # Kill any services still running to avoid deadlock for services in list(self.toilState.servicesIssued.values()): self.killServices(services, error=True) logger.debug('... finished shutting down the service manager. Took %s seconds', time.time() - startTime) @staticmethod def _startServices(jobGraphsWithServicesToStart, jobGraphsWithServicesThatHaveStarted, serviceJobsToStart, terminate, jobStore): """ Thread used to schedule services. """ servicesThatAreStarting = set() servicesRemainingToStartForJob = {} serviceToJobGraph = {} while True: with throttle(1.0): if terminate.is_set(): logger.debug('Received signal to quit starting services.') break try: jobGraph = jobGraphsWithServicesToStart.get_nowait() if len(jobGraph.services) > 1: # Have to fall back to the old blocking behavior to # ensure entire service "groups" are issued as a whole. blockUntilServiceGroupIsStarted(jobGraph, jobGraphsWithServicesThatHaveStarted, serviceJobsToStart, terminate, jobStore) continue # Found a new job that needs to schedule its services. for serviceJob in jobGraph.services[0]: serviceToJobGraph[serviceJob] = jobGraph servicesRemainingToStartForJob[jobGraph] = len(jobGraph.services[0]) # Issue the service jobs all at once. for serviceJob in jobGraph.services[0]: logger.debug("Service manager is starting service job: %s, start ID: %s", serviceJob, serviceJob.startJobStoreID) serviceJobsToStart.put(serviceJob) # We should now start to monitor these services to see if # they've started yet. servicesThatAreStarting.update(jobGraph.services[0]) except Empty: # No new jobs that need services scheduled. pass for serviceJob in list(servicesThatAreStarting): if not jobStore.fileExists(serviceJob.startJobStoreID): # Service has started! servicesThatAreStarting.remove(serviceJob) parentJob = serviceToJobGraph[serviceJob] servicesRemainingToStartForJob[parentJob] -= 1 assert servicesRemainingToStartForJob[parentJob] >= 0 del serviceToJobGraph[serviceJob] # Find if any jobGraphs have had *all* their services started. jobGraphsToRemove = set() for jobGraph, remainingServices in servicesRemainingToStartForJob.items(): if remainingServices == 0: jobGraphsWithServicesThatHaveStarted.put(jobGraph) jobGraphsToRemove.add(jobGraph) for jobGraph in jobGraphsToRemove: del servicesRemainingToStartForJob[jobGraph]
class DrEngine(object): """数据记录引擎""" settingFileName = 'DR_setting.json' settingFilePath = load_json_path(settingFileName, __file__) #---------------------------------------------------------------------- def __init__(self, mainEngine, eventEngine): """Constructor""" self.mainEngine = mainEngine self.eventEngine = eventEngine # 当前日期 self.today = todayDate() # 主力合约代码映射字典,key为具体的合约代码(如IF1604),value为主力合约代码(如IF0000) self.activeSymbolDict = {} # Tick对象字典 self.tickSymbolSet = set() # K线合成器字典 self.bmDict = {} # 配置字典 self.settingDict = OrderedDict() # 负责执行数据库插入的单独线程相关 self.active = False # 工作状态 self.queue = Queue() # 队列 self.thread = Thread(target=self.run) # 线程 # 载入设置,订阅行情 self.loadSetting() # 启动数据插入线程 self.start() # 注册事件监听 self.registerEvent() #---------------------------------------------------------------------- def loadSetting(self): """加载配置""" with open(self.settingFilePath) as f: drSetting = json.load(f) # 如果working设为False则不启动行情记录功能 working = drSetting['working'] if not working: return # Tick记录配置 if 'tick' in drSetting: l = drSetting['tick'] for setting in l: symbol = setting[0] gateway = setting[1] vtSymbol = symbol req = VtSubscribeReq() req.symbol = setting[0] # 针对LTS和IB接口,订阅行情需要交易所代码 if len(setting) >= 3: req.exchange = setting[2] vtSymbol = '.'.join([symbol, req.exchange]) # 针对IB接口,订阅行情需要货币和产品类型 if len(setting) >= 5: req.currency = setting[3] req.productClass = setting[4] self.mainEngine.subscribe(req, gateway) # tick = VtTickData() # 该tick实例可以用于缓存部分数据(目前未使用) #self.tickDict[vtSymbol] = tick self.tickSymbolSet.add(vtSymbol) # 保存到配置字典中 if vtSymbol not in self.settingDict: d = { 'symbol': symbol, 'gateway': gateway, 'tick': True } self.settingDict[vtSymbol] = d else: d = self.settingDict[vtSymbol] d['tick'] = True # 分钟线记录配置 if 'bar' in drSetting: l = drSetting['bar'] for setting in l: symbol = setting[0] gateway = setting[1] vtSymbol = symbol req = VtSubscribeReq() req.symbol = symbol if len(setting) >= 3: req.exchange = setting[2] vtSymbol = '.'.join([symbol, req.exchange]) if len(setting) >= 5: req.currency = setting[3] req.productClass = setting[4] self.mainEngine.subscribe(req, gateway) # 保存到配置字典中 if vtSymbol not in self.settingDict: d = {'symbol': symbol, 'gateway': gateway, 'bar': True} self.settingDict[vtSymbol] = d else: d = self.settingDict[vtSymbol] d['bar'] = True # 创建BarManager对象 self.bmDict[vtSymbol] = BarManager(self.onBar) # 主力合约记录配置 if 'active' in drSetting: d = drSetting['active'] self.activeSymbolDict = { vtSymbol: activeSymbol for activeSymbol, vtSymbol in d.items() } #---------------------------------------------------------------------- def getSetting(self): """获取配置""" return self.settingDict, self.activeSymbolDict #---------------------------------------------------------------------- def procecssTickEvent(self, event): """处理行情事件""" tick = event.dict_['data'] vtSymbol = tick.vtSymbol # 生成datetime对象 if not tick.datetime: tick.datetime = datetime.strptime(' '.join([tick.date, tick.time]), '%Y%m%d %H:%M:%S.%f') self.onTick(tick) bm = self.bmDict.get(vtSymbol, None) if bm: bm.updateTick(tick) #---------------------------------------------------------------------- def onTick(self, tick): """Tick更新""" vtSymbol = tick.vtSymbol if vtSymbol in self.tickSymbolSet: self.insertData(TICK_DB_NAME, vtSymbol, tick) if vtSymbol in self.activeSymbolDict: activeSymbol = self.activeSymbolDict[vtSymbol] self.insertData(TICK_DB_NAME, activeSymbol, tick) self.writeDrLog( data_recorder_text.TICK_LOGGING_MESSAGE.format( symbol=tick.vtSymbol, time=tick.time, last=tick.lastPrice, bid=tick.bidPrice1, ask=tick.askPrice1)) #---------------------------------------------------------------------- def onBar(self, bar): """分钟线更新""" vtSymbol = bar.vtSymbol self.insertData(MINUTE_DB_NAME, vtSymbol, bar) if vtSymbol in self.activeSymbolDict: activeSymbol = self.activeSymbolDict[vtSymbol] self.insertData(MINUTE_DB_NAME, activeSymbol, bar) self.writeDrLog( data_recorder_text.BAR_LOGGING_MESSAGE.format(symbol=bar.vtSymbol, time=bar.time, open=bar.open, high=bar.high, low=bar.low, close=bar.close)) #---------------------------------------------------------------------- def registerEvent(self): """注册事件监听""" self.eventEngine.register(EVENT_TICK, self.procecssTickEvent) #---------------------------------------------------------------------- def insertData(self, dbName, collectionName, data): """插入数据到数据库(这里的data可以是VtTickData或者VtBarData)""" self.queue.put((dbName, collectionName, data.__dict__)) #---------------------------------------------------------------------- def run(self): """运行插入线程""" while self.active: try: dbName, collectionName, d = self.queue.get(block=True, timeout=1) flt = {'datetime': d['datetime']} self.mainEngine.dbUpdate(dbName, collectionName, d, flt, True) except Empty: pass #---------------------------------------------------------------------- def start(self): """启动""" self.active = True self.thread.start() #---------------------------------------------------------------------- def stop(self): """退出""" if self.active: self.active = False self.thread.join() #---------------------------------------------------------------------- def writeDrLog(self, content): """快速发出日志事件""" log = VtLogData() log.logContent = content event = Event(type_=EVENT_DATARECORDER_LOG) event.dict_['data'] = log self.eventEngine.put(event)
def ServerMedia(): # HOST = input("Enter Host IP\n") HOST = '172.16.84.167' PORT_AUDIO = 10000 PORT1 = 4000 PORT2 = 5000 PORT3 = 6000 PORT4 = 7000 PORT_UNIV = 8000 lnF = 640 * 480 * 3 CHUNK = 1024 BufferSize = 4096 quitUsers = {} addressesAudio = {} addresses = {} USERS = {'4000': [], '5000': [], '6000': [], '7000': []} ports = { '10000': True, '8000': True, '4000': False, '5000': False, '6000': False, '7000': False } def accept(port, server1, server2, server3, server4): client1, addr1 = server1.accept() client2, addr2 = server2.accept() client3, addr3 = server3.accept() client4, addr4 = server4.accept() i = 0 if port == '4000': PORTS = ['4000', '5000', '6000', '7000'] print("1st User detected") elif port == '5000': PORTS = ['5000', '4000', '6000', '7000'] print("2nd User detected") elif port == '6000': PORTS = ['6000', '4000', '5000', '7000'] print("3rd User detected") elif port == '7000': PORTS = ['7000', '4000', '5000', '6000'] print("4th User detected") clients = [client1, client2, client3, client4] for PORT in PORTS: if PORT != port: USERS[PORT].append(clients[i]) i += 1 return tuple(clients), PORTS def ConnectionsUniv(): while True: client, addr = serverUniv.accept() addresses[client] = addr quitUsers[addr[0]] = False print("{} is connected!!".format(addr)) for port in sorted(ports.keys()): if ports[port] == False: client.sendall(port.encode()) ports[port] = True if port == '4000': clients, PORTS = accept(port, server1, server2, server3, server4) Thread(target=ClientConnectionVideo, args=( port, client, clients, PORTS, )).start() if port == '5000': clients, PORTS = accept(port, server2, server1, server3, server4) Thread(target=ClientConnectionVideo, args=( port, client, clients, PORTS, )).start() if port == '6000': clients, PORTS = accept(port, server3, server1, server2, server4) Thread(target=ClientConnectionVideo, args=( port, client, clients, PORTS, )).start() if port == '7000': clients, PORTS = accept(port, server4, server1, server2, server3) Thread(target=ClientConnectionVideo, args=( port, client, clients, PORTS, )).start() break def ConnectionsSound(): while True: clientAudio, addr = serverAudio.accept() print("{} is connected!!".format(addr)) addressesAudio[clientAudio] = addr[0] Thread(target=ClientConnectionSound, args=(clientAudio, )).start() def ClientConnectionVideo(port, client, clients, PORTS): try: while True: if len(addresses) > 1: databytes = b'' client1 = clients[0] lengthbuf = recvall(port, client1, 4) databytes += lengthbuf length, = struct.unpack('!I', lengthbuf) STATUS = recvall(port, client1, 6) databytes += STATUS STATUS = STATUS.decode() databytes += recvall(port, client1, length - 6) broadcastVideo(port, databytes) if STATUS == "INTIVE": i = 0 quitUsers[addresses[client][0]] = True del addresses[client] ports[port] = False for PORT in PORTS: if PORT != port: # clients[i].shutdown(1) # clients[i].close() USERS[PORT].remove(clients[i]) i += 1 # client1.shutdown(1) # client1.close() # client.shutdown(1) # client.close() break except: ports[port] = False def ClientConnectionSound(clientAudio): try: while True: if quitUsers[addressesAudio[clientAudio]] == False: data = clientAudio.recv(BufferSize) broadcastSound(clientAudio, data) else: quitUsers[addressesAudio[clientAudio]] = False del addressesAudio[clientAudio] break except: pass def recvall(port, client1, BufferSize): databytes = b'' i = 0 while i != BufferSize: to_read = BufferSize - i if to_read > (1000 * CHUNK): databytes += client1.recv(1000 * CHUNK) i = len(databytes) else: databytes += client1.recv(to_read) i = len(databytes) return databytes def broadcastVideoFrame(client, data_to_be_sent, port): try: client.sendall(data_to_be_sent) except: # ports[port] = False # USERS[port] = [] pass def broadcastVideo(port, data_to_be_sent): # threads = [] for client in USERS[port]: frameThread = Thread(target=broadcastVideoFrame, args=( client, data_to_be_sent, port, )) frameThread.start() # threads.append(frameThread) frameThread.join() # for thread in threads: # thread.join() def broadcastSound(clientSocket, data_to_be_sent): temp = addressesAudio.copy() for clientAudio in temp: if clientAudio != clientSocket: try: clientAudio.sendall(data_to_be_sent) except: pass serverAudio = socket(family=AF_INET, type=SOCK_STREAM) server1 = socket(family=AF_INET, type=SOCK_STREAM) server2 = socket(family=AF_INET, type=SOCK_STREAM) server3 = socket(family=AF_INET, type=SOCK_STREAM) server4 = socket(family=AF_INET, type=SOCK_STREAM) serverUniv = socket(family=AF_INET, type=SOCK_STREAM) try: serverAudio.bind((HOST, PORT_AUDIO)) except OSError: print("Server Audio is Busy") try: serverUniv.bind((HOST, PORT_UNIV)) except OSError: print("Server Univ Busy") try: server1.bind((HOST, PORT1)) except OSError: print("Server1 Busy") try: server2.bind((HOST, PORT2)) except OSError: print("Server2 Busy") try: server3.bind((HOST, PORT3)) except OSError: print("Server3 Busy") try: server4.bind((HOST, PORT4)) except OSError: print("Server4 Busy") serverAudio.listen(4) AcceptThreadAudio = Thread(target=ConnectionsSound) AcceptThreadAudio.start() serverUniv.listen(4) server1.listen(4) server2.listen(4) server3.listen(4) server4.listen(4) print("Waiting for connection..") AcceptThreadUniv = Thread(target=ConnectionsUniv) AcceptThreadUniv.start() AcceptThreadUniv.join() serverUniv.close()
def cc(data): # print(f'cc 输入 {data}') return_data = ac.get_answer(data) print(f"cc 输入 {data} : 返回值 {return_data}") if __name__ == "__main__": ac = action() t1 = Thread(target=cc, args=[['t1']]) t2 = Thread(target=cc, args=[['t2']]) t3 = Thread(target=cc, args=[['t3']]) t4 = Thread(target=cc, args=[['t4']]) t5 = Thread(target=cc, args=[['t5']]) import time a = time.time() t1.start() print('t1 启动') t2.start() print('t2 启动') t3.start() print('t3 启动') t4.start() print('t4 启动') t5.start() print('t5 启动') t1.join() t2.join() t3.join() t4.join() t5.join() b = time.time() print(f"{b-a} 秒")
class GameClient: ASSETS_ROOT = os.path.join('assets') LOGO_FILE = os.path.join(ASSETS_ROOT, 'logo.png') BACKGROUND = (66, 200, 244) def __init__(self, host, port): self.assets = { 'player1': pygame.image.load( os.path.join(GameClient.ASSETS_ROOT, 'bomberman1.png')), 'player2': pygame.image.load( os.path.join(GameClient.ASSETS_ROOT, 'bomberman3.png')), 'explosion': pygame.image.load( os.path.join(GameClient.ASSETS_ROOT, 'explosion.png')), 'obstacle': pygame.image.load( os.path.join(GameClient.ASSETS_ROOT, 'obstacle.png')), 'wall': pygame.image.load(os.path.join(GameClient.ASSETS_ROOT, 'wall.png')), 'bomb': pygame.image.load(os.path.join(GameClient.ASSETS_ROOT, 'bomb.png')), 'life1': pygame.image.load(os.path.join(GameClient.ASSETS_ROOT, 'life1.png')), 'life2': pygame.image.load(os.path.join(GameClient.ASSETS_ROOT, 'life2.png')), } self.host = host self.port = port self.server = None self.height = 0 self.width = 0 self.font = 0 self.screen = None self.player = None self.packet_receiver = None self.quit = False self.clock = pygame.time.Clock() self.raw_map = [] self.rows = 0 self.cols = 0 self.objects = [] self.map_ready = False self.bomb_cd = 0 self.expired_invulnerabilities = [] self.game_started = False self.game_over = None def connect(self): self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.connect((self.host, self.port)) data_size = Decoder.decode_int(self.server.recv(4)) resolution_string = Decoder.decode_string(self.server.recv(data_size)) self.height, self.width = resolution_string.split('x') self.height = int(self.height) self.width = int(self.width) self.rows = int(self.height / 50) self.cols = int(self.width / 50) for i in range(self.rows): self.raw_map.append([0 for _ in range(self.cols)]) pygame.init() pygame.font.init() pygame.display.set_icon(pygame.image.load(GameClient.LOGO_FILE)) pygame.display.set_caption("Bomberman") self.font = pygame.font.SysFont('Arial', 40, bold=True) self.screen = pygame.display.set_mode((self.width, self.height)) for asset in self.assets.keys(): self.assets[asset] = pygame.transform.scale( self.assets[asset], (50, 50)) self.assets['explosion'] = pygame.transform.scale( self.assets['explosion'], (150, 150)) self.packet_receiver = Thread(target=self.packets) self.packet_receiver.start() self.update() def packets(self): while not self.quit: try: opcode = Decoder.decode_int(self.server.recv(4)) if opcode == SMSG_INIT_DATA: for i in range(self.rows): for j in range(self.cols): self.raw_map[i][j] = Decoder.decode_int( self.server.recv(4)) self.map_ready = True if opcode == SMSG_INIT_PLAYERS: nr_of_players = Decoder.decode_int(self.server.recv(4)) for k in range(nr_of_players): guid = Decoder.decode_int(self.server.recv(4)) is_me = Decoder.decode_int(self.server.recv(4)) pl = Player(guid) pl.position = start_positions[pl.guid] self.objects.append(pl) if is_me == 1: self.player = pl self.game_started = True if opcode == SMSG_UPDATE_POSITION: player_guid = Decoder.decode_int(self.server.recv(4)) pos_x = Decoder.decode_int(self.server.recv(4)) pos_y = Decoder.decode_int(self.server.recv(4)) for obj in self.objects: if obj.guid == player_guid: obj.position = (pos_x, pos_y) if opcode == SMSG_PUT_BOMB_ACK: bomb_guid = Decoder.decode_int(self.server.recv(4)) pos_x = Decoder.decode_int(self.server.recv(4)) pos_y = Decoder.decode_int(self.server.recv(4)) b = Bomb(bomb_guid, (pos_x, pos_y)) self.objects.append(b) if opcode == SMSG_DETONATE_BOMB: bomb_guid = Decoder.decode_int(self.server.recv(4)) hit_units = Decoder.decode_int(self.server.recv(4)) for i in range(hit_units): guid = Decoder.decode_int(self.server.recv(4)) for ob in self.objects: if ob.guid == guid: ob.healths -= 1 ob.invulnerability_timer = 1 dead_players = [] for ob in self.objects: if isinstance(ob, Player): if ob.healths <= 0: dead_players.append(ob.guid) if len(dead_players) != 0: if len(dead_players) > 1: self.game_over = -1 else: self.game_over = 1 if dead_players[0] == 2 else 2 hit_walls = Decoder.decode_int(self.server.recv(4)) for i in range(hit_walls): x = Decoder.decode_int(self.server.recv(4)) y = Decoder.decode_int(self.server.recv(4)) self.raw_map[x][y] = GameEngine.EMPTY bomb = None for obj in self.objects: if obj.guid == bomb_guid: bomb = obj break if bomb is not None: self.objects.append( Explosion( (bomb.position[0] - 50, bomb.position[1] - 50), 100)) self.objects.remove(bomb) if opcode == SMSG_REMOVE_INVULNERABILITY: guid = Decoder.decode_int(self.server.recv(4)) for ob in self.objects: if ob.guid == guid: self.expired_invulnerabilities.append(ob) except: self.quit = True def update(self): while not self.quit: try: if self.player is None: self.screen.fill(GameClient.BACKGROUND) wait_surface = self.font.render( 'Waiting for other players...', False, (255, 255, 255)) self.screen.blit(wait_surface, (130, 250)) pygame.display.update() self.clock.tick(60) for event in pygame.event.get(): if event.type == pygame.QUIT: self.quit = True continue if self.game_over is not None: self.screen.fill(GameClient.BACKGROUND) if self.game_over != -1: wait_surface = self.font.render( 'Player {0} has won!'.format(self.game_over), False, (255, 255, 255)) self.screen.blit( self.assets['player' + str(self.game_over)], (300, 280)) self.screen.blit(wait_surface, (180, 170)) else: wait_surface = self.font.render( 'Draw!', False, (255, 255, 255)) self.screen.blit(wait_surface, (300, 250)) pygame.display.update() self.clock.tick(60) for event in pygame.event.get(): if event.type == pygame.QUIT: self.quit = True continue if self.bomb_cd > 0: self.bomb_cd -= 5 for ob in self.expired_invulnerabilities: ob.invulnerability_timer = 0 self.expired_invulnerabilities.clear() crt_flags = self.player.movement_flags for event in pygame.event.get(): if event.type == pygame.QUIT: self.quit = True elif event.type == pygame.KEYDOWN: if event.key == KEY_UP: self.player.movement_flags |= MOVEMENT_FLAG_UP if event.key == KEY_DOWN: self.player.movement_flags |= MOVEMENT_FLAG_DOWN if event.key == KEY_RIGHT: self.player.movement_flags |= MOVEMENT_FLAG_RIGHT if event.key == KEY_LEFT: self.player.movement_flags |= MOVEMENT_FLAG_LEFT elif event.type == pygame.KEYUP: if event.key == KEY_UP: self.player.movement_flags &= ~MOVEMENT_FLAG_UP if event.key == KEY_DOWN: self.player.movement_flags &= ~MOVEMENT_FLAG_DOWN if event.key == KEY_RIGHT: self.player.movement_flags &= ~MOVEMENT_FLAG_RIGHT if event.key == KEY_LEFT: self.player.movement_flags &= ~MOVEMENT_FLAG_LEFT if event.key == pygame.K_SPACE and self.bomb_cd <= 0: self.bomb_cd = 300 self.server.sendall( Encoder.encode_int(CMSG_PUT_BOMB)) self.server.sendall( Encoder.encode_int(self.player.position[0])) self.server.sendall( Encoder.encode_int(self.player.position[1])) self.screen.fill(GameClient.BACKGROUND) if self.map_ready: for i in range(self.rows): for j in range(self.cols): if self.raw_map[i][j] == GameEngine.WALL: self.screen.blit(self.assets['wall'], (j * 50, i * 50)) if self.raw_map[i][j] == GameEngine.OBSTACLE: self.screen.blit(self.assets['obstacle'], (j * 50, i * 50)) remove_objects = [] for obj in self.objects: if isinstance(obj, Player): if obj.invulnerability_timer > 0: obj.invulnerability_timer += 1 if obj.invulnerability_timer % 11 == 0 or obj.invulnerability_timer % 12 == 0 or \ obj.invulnerability_timer % 13 == 0 or obj.invulnerability_timer % 14 == 0: pass else: self.screen.blit( self.assets['player' + str(obj.guid)], obj.position) else: self.screen.blit( self.assets['player' + str(obj.guid)], obj.position) if obj.guid == 1: for i in range(obj.healths): self.screen.blit(self.assets['life1'], (50 * i, 0)) else: for i in range(obj.healths): self.screen.blit(self.assets['life2'], ((12 - i) * 50, 0)) if isinstance(obj, Bomb): obj.timer += 17 multiplier = cos( 3.14 * obj.timer / 400).real / 8 + 7 / 8 scaled_bomb = pygame.transform.scale( self.assets['bomb'], (int(50 * multiplier), int(50 * multiplier))) self.screen.blit( scaled_bomb, (obj.position[0] + 25 * (1 - multiplier), obj.position[1] + 25 * (1 - multiplier))) if isinstance(obj, Explosion): self.screen.blit(self.assets['explosion'], obj.position) obj.duration -= 3 if obj.duration <= 0: remove_objects.append(obj) for obj in remove_objects: self.objects.remove(obj) pygame.display.update() self.clock.tick(60) # Movement flags changed, send update to the server if crt_flags != self.player.movement_flags: self.server.sendall(Encoder.encode_int(CMSG_MOVE_UPDATE)) self.server.sendall(Encoder.encode_int(self.player.guid)) self.server.sendall( Encoder.encode_int(self.player.movement_flags)) except: self.quit = True self.close() def close(self): if self.packet_receiver: self.packet_receiver.join(1) self.server.close() pygame.quit() quit()
class OKEXWebsocket(object): """ Websocket API 创建Websocket client对象后,需要调用Start的方法去启动workder和ping线程 1. Worker线程会自动重连. 2. 使用stop方法去停止断开和销毁websocket client, 3. 四个回调方法.. * on_open * on_close * on_msg * on_error start()方法调用后,ping线程每隔60秒回自动调用一次。 """ def __init__(self, host=None, ping_interval=20, key=None, secret=None, _symbol=None, db_exchange=None, password=None): """Constructor""" self.key = key self.secret = secret self.password = password self.host = host self.ping_interval = ping_interval self.symbol = _symbol self.db_exchange = db_exchange self._ws_lock = Lock() self._ws = None self._worker_thread = None self._ping_thread = None self._active = False # 开启启动websocket的开关。 # debug需要.. self._last_sent_text = None self._last_received_text = None def start(self): """ 启动客户端,客户端连接成功后,会调用 on_open这个方法 on_open 方法调用后,才可以向服务器发送消息的方法. """ self._active = True self._worker_thread = Thread(target=self._run) self._worker_thread.start() self._ping_thread = Thread(target=self._run_ping) self._ping_thread.start() def stop(self): """ 停止客户端. """ self._active = False self._disconnect() def join(self): """ Wait till all threads finish. This function cannot be called from worker thread or callback function. """ self._ping_thread.join() self._worker_thread.join() def send_msg(self, msg: dict): """ 向服务器发送数据. 如果你想发送非json数据,可以重写该方法. """ text = json.dumps(msg) self._record_last_sent_text(text) return self._send_text(text) def _send_text(self, text: str): """ 发送文本数据到服务器. """ ws = self._ws if ws: ws.send(text, opcode=websocket.ABNF.OPCODE_TEXT) def _ensure_connection(self): """""" triggered = False with self._ws_lock: if self._ws is None: self._ws = websocket.create_connection(self.host) triggered = True if triggered: self.on_open() def _disconnect(self): """ """ triggered = False with self._ws_lock: if self._ws: ws: websocket.WebSocket = self._ws self._ws = None triggered = True if triggered: ws.close() self.on_close() def _run(self): try: while self._active: try: self._ensure_connection() ws = self._ws if ws: text = ws.recv() # ws object is closed when recv function is blocking if not text: self._disconnect() continue self._record_last_received_text(text) self.on_msg(text) # ws is closed before recv function is called # For socket.error, see Issue #1608 except (websocket.WebSocketConnectionClosedException, socket.error): self._disconnect() # other internal exception raised in on_msg except: # noqa et, ev, tb = sys.exc_info() self.on_error(et, ev, tb) self._disconnect() # except: # noqa et, ev, tb = sys.exc_info() self.on_error(et, ev, tb) self._disconnect() def _run_ping(self): while self._active: try: self._ping() except: # noqa et, ev, tb = sys.exc_info() self.on_error(et, ev, tb) sleep(1) for i in range(self.ping_interval): if not self._active: break sleep(1) def _ping(self): ws = self._ws if ws: ws.send("ping", websocket.ABNF.OPCODE_PING) def on_open(self): """on open """ print("on open") key = self.key passphrase = self.password secret = self.secret timestamp = str(time.time()) msg = timestamp + "GET" + "/users/self/verify" sign = hmac.new(secret.encode('utf-8'), msg.encode('utf-8'), digestmod=hashlib.sha256).digest() sign = base64.b64encode(sign).decode('utf-8') data = {"op": "login", "args": [key, passphrase, timestamp, sign]} # data = {"op": "subscribe", "args": ["swap/depth5:BTC-USD-SWAP"]} self.send_msg(data) def on_close(self): """ on close websocket """ def on_msg(self, data: bytes): """call when the msg arrive.""" decompress = zlib.decompressobj(-zlib.MAX_WBITS # see above ) msg = json.loads(decompress.decompress(data)) print(msg) if "event" in msg: if msg['event'] == 'login' and msg['success']: # print("登录成功。。") self.subscribe_topic() if 'table' in msg and msg['table'] == 'swap/order': client = pymongo.MongoClient('localhost', port=27017) # 连接数据库 db = client[self.db_exchange] col = db[self.symbol] if "data" in msg: data = msg["data"] for i in range(len(data)): if 'state' in data[i]: if data[i]['state'] == '2' or data[i]['state'] == '-1': pprint.pprint(data[i]) myquery = {"order_id": data[i]['order_id']} mydoc = col.find(myquery) if mydoc: if 'state' in data[i]: newvalues = { "$set": { "order_status": data[i]['state'] } } col.update_one(myquery, newvalues) elif "type" in i: newvalues = { "$set": { "order_side": data[i]['type'] } } col.update_one(myquery, newvalues) elif "price" in i: newvalues = { "$set": { "order_price": data[i]['price'] } } col.update_one(myquery, newvalues) def subscribe_topic(self): if self.symbol == 'BTC/USD': _orders_symbol = 'BTC-USD-SWAP' elif self.symbol == 'ETH/USD': _orders_symbol = 'ETH-USD-SWAP' elif self.symbol == 'BTC/USDT': _orders_symbol = 'BTC-USDT-SWAP' elif self.symbol == 'ETH/USDT': _orders_symbol = 'ETH-USDT-SWAP' else: _orders_symbol = self.symbol self.send_msg({ "op": "subscribe", "args": ["swap/order:" + _orders_symbol] }) def on_error(self, exception_type: type, exception_value: Exception, tb): """ Callback when exception raised. """ sys.stderr.write( self.exception_detail(exception_type, exception_value, tb)) return sys.excepthook(exception_type, exception_value, tb) def exception_detail(self, exception_type: type, exception_value: Exception, tb): """ Print detailed exception information. """ text = "[{}]: Unhandled WebSocket Error:{}\n".format( datetime.now().isoformat(), exception_type) text += "LastSentText:\n{}\n".format(self._last_sent_text) text += "LastReceivedText:\n{}\n".format(self._last_received_text) text += "Exception trace: \n" text += "".join( traceback.format_exception(exception_type, exception_value, tb)) return text def _record_last_sent_text(self, text: str): """ Record last sent text for debug purpose. """ self._last_sent_text = text[:1000] def _record_last_received_text(self, text: str): """ Record last received text for debug purpose. """ self._last_received_text = text[:1000]
class WorkManager(object): def __init__(self, comp_name, queue, work_dict): """ :type comp_name: str :type queue: zoom.agent.entities.unique_queue.UniqueQueue :type work_dict: dict """ self._operate = ThreadSafeObject(True) self._thread = Thread(target=self._run, name='work_manager', args=(self._operate, queue, work_dict)) self._thread.daemon = True self._log = logging.getLogger('sent.{0}.wm'.format(comp_name)) def start(self): self._log.info('starting work manager') self._thread.start() def stop(self): self._log.info('Stopping work manager.') self._operate.set_value(False) self._thread.join() self._log.info('Stopped work manager.') def _run(self, operate, queue, work_dict): """ :type operate: zoom.agent.entities.thread_safe_object.ThreadSafeObject :type queue: zoom.agent.entities.unique_queue.UniqueQueue :type work_dict: dict """ while operate == True: if queue: # if queue is not empty self._log.info('Current Task Queue:\n{0}' .format(pprint.pformat(list(queue)))) task = queue[0] # grab task, but keep it in the queue if task.func is None: func_to_run = work_dict.get(task.name, None) else: func_to_run = task.func if func_to_run is not None: self._log.info('Found work "{0}" in queue.' .format(task.name)) t = ThreadWithReturn(target=func_to_run, name=task.name, args=task.args, kwargs=task.kwargs) t.start() if task.block: task.result = t.join() else: self._log.warning('Cannot do "{0}", it is not a valid ' 'action.'.format(task.name)) try: queue.remove(task) except ValueError: self._log.debug('Item no longer exists in the queue: {0}' .format(task)) else: time.sleep(1) self._log.info('Done listening for work.') return
class Stores(OrderedDict): CHECK_INTERVAL = 24 * 60 * 60 def builtins_loaded(self): self.last_check_time = 0 self.version_map = {} self.cached_version_map = {} self.name_rmap = {} for key, val in self.iteritems(): prefix, name = val.__module__.rpartition('.')[0::2] if prefix == 'calibre.gui2.store.stores' and name.endswith( '_plugin'): module = sys.modules[val.__module__] sv = getattr(module, 'store_version', None) if sv is not None: name = name.rpartition('_')[0] self.version_map[name] = sv self.name_rmap[name] = key self.cache_file = JSONConfig('store/plugin_cache') self.load_cache() def load_cache(self): # Load plugins from on disk cache remove = set() pat = re.compile(r'^store_version\s*=\s*(\d+)', re.M) for name, src in self.cache_file.iteritems(): try: key = self.name_rmap[name] except KeyError: # Plugin has been disabled m = pat.search(src[:512]) if m is not None: try: self.cached_version_map[name] = int(m.group(1)) except (TypeError, ValueError): pass continue try: obj, ver = self.load_object(src, key) except VersionMismatch as e: self.cached_version_map[name] = e.ver continue except: import traceback prints('Failed to load cached store:', name) traceback.print_exc() else: if not self.replace_plugin(ver, name, obj, 'cached'): # Builtin plugin is newer than cached remove.add(name) if remove: with self.cache_file: for name in remove: del self.cache_file[name] def check_for_updates(self): if hasattr(self, 'update_thread') and self.update_thread.is_alive(): return if time.time() - self.last_check_time < self.CHECK_INTERVAL: return self.last_check_time = time.time() try: self.update_thread.start() except (RuntimeError, AttributeError): self.update_thread = Thread(target=self.do_update) self.update_thread.start() def join(self, timeout=None): hasattr(self, 'update_thread') and self.update_thread.join(timeout) def download_updates(self): ver_map = { name: max(ver, self.cached_version_map.get(name, -1)) for name, ver in self.version_map.iteritems() } try: updates = download_updates(ver_map) except: import traceback traceback.print_exc() else: for name, code in updates: yield name, code def do_update(self): replacements = {} for name, src in self.download_updates(): try: key = self.name_rmap[name] except KeyError: # Plugin has been disabled replacements[name] = src continue try: obj, ver = self.load_object(src, key) except VersionMismatch as e: self.cached_version_map[name] = e.ver replacements[name] = src continue except: import traceback prints('Failed to load downloaded store:', name) traceback.print_exc() else: if self.replace_plugin(ver, name, obj, 'downloaded'): replacements[name] = src if replacements: with self.cache_file: for name, src in replacements.iteritems(): self.cache_file[name] = src def replace_plugin(self, ver, name, obj, source): if ver > self.version_map[name]: if DEBUG: prints('Loaded', source, 'store plugin for:', self.name_rmap[name], 'at version:', ver) self[self.name_rmap[name]] = obj self.version_map[name] = ver return True return False def load_object(self, src, key): namespace = {} builtin = self[key] exec(src, namespace) ver = namespace['store_version'] cls = None for x in namespace.itervalues(): if (isinstance(x, type) and issubclass(x, StorePlugin) and x is not StorePlugin): cls = x break if cls is None: raise ValueError('No store plugin found') if cls.minimum_calibre_version > numeric_version: raise VersionMismatch(ver) return cls(builtin.gui, builtin.name, config=builtin.config, base_plugin=builtin.base_plugin), ver
class OEFConnection(Connection): """The OEFConnection connects the to the mailbox.""" def __init__(self, public_key: str, oef_addr: str, oef_port: int = 10000): """ Initialize. :param public_key: the public key of the agent. :param oef_addr: the OEF IP address. :param oef_port: the OEF port. """ super().__init__() core = AsyncioCore(logger=logger) self._core = core # type: AsyncioCore self.channel = OEFChannel(public_key, oef_addr, oef_port, core=core, in_queue=self.in_queue) self._stopped = True self._connected = False self.out_thread = None # type: Optional[Thread] @property def is_established(self) -> bool: """Get the connection status.""" return self._connected def _fetch(self) -> None: """ Fetch the messages from the outqueue and send them. :return: None """ while self._connected: try: msg = self.out_queue.get(block=True, timeout=1.0) self.send(msg) except Empty: pass def connect(self) -> None: """ Connect to the channel. :return: None :raises ConnectionError if the connection to the OEF fails. """ if self._stopped and not self._connected: self._stopped = False self._core.run_threaded() try: if not self.channel.connect(): raise ConnectionError("Cannot connect to OEFChannel.") self._connected = True self.out_thread = Thread(target=self._fetch) self.out_thread.start() except ConnectionError as e: self._core.stop() raise e def disconnect(self) -> None: """ Disconnect from the channel. :return: None """ assert self.out_thread is not None, "Call connect before disconnect." if not self._stopped and self._connected: self._connected = False self.out_thread.join() self.out_thread = None self.channel.disconnect() self._core.stop() self._stopped = True def send(self, envelope: Envelope): """ Send messages. :return: None """ if self._connected: self.channel.send(envelope) @classmethod def from_config( cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': """ Get the OEF connection from the connection configuration. :param public_key: the public key of the agent. :param connection_configuration: the connection configuration object. :return: the connection object """ oef_addr = cast(str, connection_configuration.config.get("addr")) oef_port = cast(int, connection_configuration.config.get("port")) return OEFConnection(public_key, oef_addr, oef_port)
def getDataFromCompany_mul(self, c_name): res_data = { 'c_p': { 'nodes': [], 'edges': [], }, 'c_s': { 'nodes': [], 'edges': [], }, 'remain_node': [], 'status': '' } c_nodes = self.getCompany(c_name) if c_nodes is None: res_data['status'] = 'error' return res_data c_node_f = c_nodes[0] for c_node in c_nodes[1:]: res_data['remain_node'].append( get_node_temp(c_node.identity, c_node['c_name'], dict(c_node), 1, 1)) cp = None def get_serve_data(c_node): data = { 'nodes': [], 'edges': [], } node_list = [c_node.identity] c_serves = self.getServeFromCompany(c_node) for serve in c_serves: time_s = dict(serve[1])['s_end_time'] if dict( serve[1]).get('s_end_time') else FuTime() if checkTime(time_s): data['edges'].append( get_edges_temp(1, serve[0].identity, serve[2].identity, dict(serve[1]))) data['nodes'].append( get_node_temp(serve[0].identity, serve[0]['p_name'], dict(serve[0]), 2, 2)) node_list.append(serve[0].identity) p_node = serve[0] p_serves = self.getServeFromPerson(p_node) for s in p_serves: time_s = dict(s[1])['s_end_time'] if dict( s[1]).get('s_end_time') else FuTime() if checkTime(time_s): data['edges'].append( get_edges_temp(1, s[0].identity, s[2].identity, dict(s[1]))) if s[2].identity not in node_list: data['nodes'].append( get_node_temp(s[2].identity, s[2]['c_name'], dict(s[2]), 1, 3)) node_list.append(s[2].identity) else: data['edges'].append( get_edges_temp(2, s[0].identity, s[2].identity, dict(s[1]))) if s[2].identity not in node_list: data['nodes'].append( get_node_temp(s[2].identity, s[2]['c_name'], dict(s[2]), 1, 3)) node_list.append(s[2].identity) res_data['c_p'] = data cs = None def get_hold_data(c_node): data = { 'nodes': [], 'edges': [], } node_list = [c_node.identity] c_holds = self.getHoldFromCompany(c_node) for hold in c_holds: data['nodes'].append( get_node_temp(hold[0].identity, hold[0]['s_name'], dict(hold[0]), 3, 2)) data['edges'].append( get_edges_temp(2, hold[0].identity, hold[2].identity, dict(hold[1]))) res_data['c_s'] = data t1 = Thread(target=get_serve_data, args=(c_node_f, )) t2 = Thread(target=get_hold_data, args=(c_node_f, )) t1.start() t2.start() t1.join() t2.join() res_data['c_p']['nodes'].append( get_node_temp(c_node_f.identity, c_node_f['c_name'], dict(c_node_f), 1, 1)) res_data['c_s']['nodes'].append( get_node_temp(c_node_f.identity, c_node_f['c_name'], dict(c_node_f), 1, 1)) res_data['status'] = 'success' return res_data
class USBTransport(object): '''Implement USB transport.''' def __init__(self, dev=None): '''Instantiate the first available PTP device over USB''' logger.debug('Init') self.__setup_constructors() # If no device is specified, find all devices claiming to be Cameras # and get the USB endpoints for the first one that works. cameras = find_usb_cameras() devs = [dev] if (dev is not None) else cameras for dev in devs: if self.__setup_device(dev): break else: message = 'No USB PTP device found.' logger.error(message) raise PTPError(message) if self.__dev.is_kernel_driver_active(self.__intf.bInterfaceNumber): try: self.__dev.detach_kernel_driver(self.__intf.bInterfaceNumber) usb.util.claim_interface(self.__dev, self.__intf) except usb.core.USBError: message = ( 'Could not detach kernel driver. ' 'Maybe the camera is mounted?' ) logger.error(message) raise PTPError(message) logger.debug('Claiming {}'.format(repr(dev))) usb.util.claim_interface(self.__dev, self.__intf) self.__event_queue = Queue() self.__event_shutdown = Event() # Locks for different end points. self.__inep_lock= RLock() self.__intep_lock= RLock() self.__outep_lock= RLock() self.__event_proc = Thread(name='EvtPolling', target=self.__poll_events) self.__event_proc.daemon = False atexit.register(self._shutdown) self.__event_proc.start() def _shutdown(self): logger.debug('Shutdown request') self.__event_shutdown.set() # Free USB resource on shutdown. # Only join a running thread. if self.__event_proc.is_alive(): self.__event_proc.join(2) logger.debug('Release {}'.format(repr(self.__dev))) usb.util.release_interface(self.__dev, self.__intf) # Helper methods. # --------------------- def __setup_device(self, dev): '''Get endpoints for a device. True on success.''' self.__inep = None self.__outep = None self.__intep = None self.__cfg = None self.__dev = None self.__intf = None # Attempt to find the USB in, out and interrupt endpoints for a PTP # interface. for cfg in dev: for intf in cfg: if intf.bInterfaceClass == PTP_USB_CLASS: for ep in intf: ep_type = endpoint_type(ep.bmAttributes) ep_dir = endpoint_direction(ep.bEndpointAddress) if ep_type == ENDPOINT_TYPE_BULK: if ep_dir == ENDPOINT_IN: self.__inep = ep elif ep_dir == ENDPOINT_OUT: self.__outep = ep elif ((ep_type == ENDPOINT_TYPE_INTR) and (ep_dir == ENDPOINT_IN)): self.__intep = ep if not (self.__inep and self.__outep and self.__intep): self.__inep = None self.__outep = None self.__intep = None else: logger.debug('Found {}'.format(repr(self.__inep))) logger.debug('Found {}'.format(repr(self.__outep))) logger.debug('Found {}'.format(repr(self.__intep))) self.__cfg = cfg self.__dev = dev self.__intf = intf return True return False def __setup_constructors(self): '''Set endianness and create transport-specific constructors.''' # Set endianness of constructors before using them. self._set_endian('little') self.__Length = Int32ul self.__Type = Enum( Int16ul, default=Pass, Undefined=0x0000, Command=0x0001, Data=0x0002, Response=0x0003, Event=0x0004, ) # This is just a convenience constructor to get the size of a header. self.__Code = Int16ul self.__Header = Struct( 'Length' / self.__Length, 'Type' / self.__Type, 'Code' / self.__Code, 'TransactionID' / self._TransactionID, ) # These are the actual constructors for parsing and building. self.__CommandHeader = Struct( 'Length' / self.__Length, 'Type' / self.__Type, 'OperationCode' / self._OperationCode, 'TransactionID' / self._TransactionID, ) self.__ResponseHeader = Struct( 'Length' / self.__Length, 'Type' / self.__Type, 'ResponseCode' / self._ResponseCode, 'TransactionID' / self._TransactionID, ) self.__EventHeader = Struct( 'Length' / self.__Length, 'Type' / self.__Type, 'EventCode' / self._EventCode, 'TransactionID' / self._TransactionID, ) # Apparently nobody uses the SessionID field. Even though it is # specified in ISO15740:2013(E), no device respects it and the session # number is implicit over USB. self.__Param = Range(0, 5, self._Parameter) self.__FullParam = Struct(Array(5, self._Parameter)) self.__FullEventParam = Struct(Array(3, self._Parameter)) self.__CommandTransactionBase = Struct( Embedded(self.__CommandHeader), 'Payload' / Bytes( lambda ctx, h=self.__Header: ctx.Length - h.sizeof() ) ) self.__CommandTransaction = ExprAdapter( self.__CommandTransactionBase, encoder=lambda obj, ctx, h=self.__Header: Container( Length=len(obj.Payload) + h.sizeof(), **obj ), decoder=lambda obj, ctx: obj, ) self.__ResponseTransactionBase = Struct( Embedded(self.__ResponseHeader), 'Payload' / Bytes( lambda ctx, h=self.__Header: ctx.Length - h.sizeof()) ) self.__ResponseTransaction = ExprAdapter( self.__ResponseTransactionBase, encoder=lambda obj, ctx, h=self.__Header: Container( Length=len(obj.Payload) + h.sizeof(), **obj ), decoder=lambda obj, ctx: obj, ) self.__EventTransactionBase = Struct( Embedded(self.__EventHeader), 'Payload' / Bytes( lambda ctx, h=self.__Header: ctx.Length - h.sizeof()), ) self.__EventTransaction = ExprAdapter( self.__EventTransactionBase, encoder=lambda obj, ctx, h=self.__Header: Container( Length=len(obj.Payload) + h.sizeof(), **obj ), decoder=lambda obj, ctx: obj, ) def __parse_response(self, usbdata): '''Helper method for parsing USB data.''' # Build up container with all PTP info. usbdata = bytearray(usbdata) transaction = self.__ResponseTransaction.parse(usbdata) response = Container( SessionID=self.session_id, TransactionID=transaction.TransactionID, ) if transaction.Type == 'Response': response['ResponseCode'] = transaction.ResponseCode response['Parameter'] = self.__Param.parse(transaction.Payload) elif transaction.Type == 'Event': event = self.__EventHeader.parse( usbdata[0:self.__Header.sizeof()] ) response['EventCode'] = event.EventCode response['Parameter'] = self.__Param.parse(transaction.Payload) else: command = self.__CommandHeader.parse( usbdata[0:self.__Header.sizeof()] ) response['OperationCode'] = command.OperationCode response['Data'] = transaction.Payload return response def __recv(self, event=False, wait=False, raw=False): '''Helper method for receiving data.''' # TODO: clear stalls automatically ep = self.__intep if event else self.__inep lock = self.__intep_lock if event else self.__inep_lock with lock: try: usbdata = ep.read( ep.wMaxPacketSize, timeout=0 if wait else 5 ) except usb.core.USBError as e: # Ignore timeout or busy device once. if e.errno == 110 or e.errno == 16: if event: return None else: usbdata = ep.read( ep.wMaxPacketSize, timeout=5000 ) else: raise e header = self.__ResponseHeader.parse( bytearray(usbdata[0:self.__Header.sizeof()]) ) if header.Type not in ['Response', 'Data', 'Event']: raise PTPError( 'Unexpected USB transfer type.' 'Expected Response, Event or Data but received {}' .format(header.Type) ) while len(usbdata) < header.Length: usbdata += ep.read( ep.wMaxPacketSize, timeout=5000 ) if raw: return usbdata else: return self.__parse_response(usbdata) def __send(self, ptp_container, event=False): '''Helper method for sending data.''' ep = self.__intep if event else self.__outep lock = self.__intep_lock if event else self.__outep_lock transaction = self.__CommandTransaction.build(ptp_container) with lock: try: ep.write(transaction, timeout=1) except usb.core.USBError as e: # Ignore timeout or busy device once. if e.errno == 110 or e.errno == 16: ep.write(transaction, timeout=5000) def __send_request(self, ptp_container): '''Send PTP request without checking answer.''' # Don't modify original container to keep abstraction barrier. ptp = Container(**ptp_container) # Don't send unused parameters try: while not ptp.Parameter[-1]: ptp.Parameter.pop() if len(ptp.Parameter) == 0: break except IndexError: # The Parameter list is already empty. pass # Send request ptp['Type'] = 'Command' ptp['Payload'] = self.__Param.build(ptp.Parameter) self.__send(ptp) def __send_data(self, ptp_container, data): '''Send data without checking answer.''' # Don't modify original container to keep abstraction barrier. ptp = Container(**ptp_container) # Send data ptp['Type'] = 'Data' ptp['Payload'] = data self.__send(ptp) # Actual implementation # --------------------- def send(self, ptp_container, data): '''Transfer operation with dataphase from initiator to responder''' logger.debug('SEND {}{}'.format( ptp_container.OperationCode, ' ' + str(list(map(hex, ptp_container.Parameter))) if ptp_container.Parameter else '', )) self.__send_request(ptp_container) self.__send_data(ptp_container, data) # Get response and sneak in implicit SessionID and missing parameters. response = self.__recv() logger.debug('SEND {} {}{}'.format( ptp_container.OperationCode, response.ResponseCode, ' ' + str(list(map(hex, response.Parameter))) if ptp_container.Parameter else '', )) return response def recv(self, ptp_container): '''Transfer operation with dataphase from responder to initiator.''' logger.debug('RECV {}{}'.format( ptp_container.OperationCode, ' ' + str(list(map(hex, ptp_container.Parameter))) if ptp_container.Parameter else '', )) self.__send_request(ptp_container) dataphase = self.__recv() if hasattr(dataphase, 'Data'): response = self.__recv() if ( (ptp_container.OperationCode != dataphase.OperationCode) or (ptp_container.TransactionID != dataphase.TransactionID) or (ptp_container.SessionID != dataphase.SessionID) or (dataphase.TransactionID != response.TransactionID) or (dataphase.SessionID != response.SessionID) ): raise PTPError( 'Dataphase does not match with requested operation.' ) response['Data'] = dataphase.Data else: response = dataphase logger.debug('RECV {} {}{}'.format( ptp_container.OperationCode, response.ResponseCode, ' ' + str(list(map(hex, response.Parameter))) if response.Parameter else '', )) return response def mesg(self, ptp_container): '''Transfer operation without dataphase.''' logger.debug('MESG {}{}'.format( ptp_container.OperationCode, ' ' + str(list(map(hex, ptp_container.Parameter))) if ptp_container.Parameter else '', )) self.__send_request(ptp_container) # Get response and sneak in implicit SessionID and missing parameters # for FullResponse. response = self.__recv() logger.debug('MESG {} {}{}'.format( ptp_container.OperationCode, response.ResponseCode, ' ' + str(list(map(hex, response.Parameter))) if response.Parameter else '', )) return response def event(self, wait=False): '''Check event. If `wait` this function is blocking. Otherwise it may return None. ''' evt = None usbdata = None timeout = None if wait else 0.001 if not self.__event_queue.empty(): usbdata = self.__event_queue.get(block=not wait, timeout=timeout) if usbdata is not None: evt = self.__parse_response(usbdata) return evt def __poll_events(self): '''Poll events, adding them to a queue.''' while not self.__event_shutdown.is_set() and _main_thread_alive(): evt = self.__recv(event=True, wait=False, raw=True) if evt is not None: logger.debug('Event queued') self.__event_queue.put(evt)
def retrieve_fastas_by_acc(acc_dict, db_dir, local_fasta): # Function downloads set of records from Genbank according to accessions passed to it. # Downloaded FASTA file will be placed in 'db_dir' directory and named 'local_seq_set.fasta' # :param acc_dict: dictionary comntaining accession data of hits; # :type acc_dict: dict<str: tuple<str, str, int>>; # :param db_dir: path to directory in which downloaded FASTA file will be placed; # :type db_dir: str; # :param local_fasta: path to file with reference sequences to be included in database; # :type local_fasta: str; # Path to file with current chunk (see below "100 accession numbers...") tmp_fasta = os.path.join(db_dir, "tmp.fasta") accessions = tuple(set(acc_dict.keys())) if len(accessions) == 0: # just in case return # end if # 100 accession numbers in order not to make too long URL # Download genomes by chunks of 100 sequences. max_accnum = 100 i = 0 accnum = len(accessions) while i < accnum: curr_accessions = accessions[i:i + max_accnum] # slice chunk accs_del_comma = ','.join( curr_accessions) # accessions must be separated by comma in url # E-utilities provide a possibility to download records from Genbank by accessions. retrieve_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?\ db=nuccore&id={}&rettype=fasta&retmode=text".format(accs_del_comma) log_info("Retrieve URL: `{}`".format(retrieve_url)) # GNU wget utility is safer, but there can be presence of absence of it :) wget_util = "wget" util_found = False for d in os.environ["PATH"].split(os.pathsep): if os.path.isdir(d) and wget_util in os.listdir(d): util_found = True break # end if # end for print() printlog_info("{} - Downloading {} reference sequences...".format( getwt(), len(curr_accessions))) if util_found: # If we have wget -- just use it wget_cmd = 'wget --no-check-certificate "{}" -O {}'.format( retrieve_url, tmp_fasta) pipe = sp_Popen(wget_cmd, shell=True) pipe.communicate() if pipe.returncode != 0: printlog_error_time( "Error occured while downloading reference sequences") platf_depend_exit(pipe.returncode) # end if else: # If there are no wget -- we will download sequences with Python disposal stop_wait = Event( ) # a flag variable that will signal waiter-function to stop executing def download_waiter(stop_wait): """ Function waits untill 'local_fasta' file is downloaded. It prints size of downloaded data to console during downloading. This function just waits -- it won't bring you the menu :). """ # Wait untill downloading starts while not os.path.exists(tmp_fasta): if not stop_wait.is_set(): return # end if sleep(1) # end while MB_size = 1024**2 # we will divide by it to get megabytes while stop_wait.is_set(): # Get size of downloaded data fsize = round(os.path.getsize(tmp_fasta) / MB_size, 1) # get megabytes printn("\r{} - {} MB downloaded ".format(getwt(), fsize)) sleep(1) # instant updates are not necessary # end while # Print total size of downloaded file (it can be deleted by this time) try: fsize = round(os.path.getsize(tmp_fasta) / MB_size, 1) except OSError: # We can pass this ecxeption -- we do delete this file if downloading crushes # And this function just waits :) pass # end try printlog_info("\r{} - {} MB downloaded ".format( getwt(), fsize)) # end def download_waiter error = True while error: try: waiter = Thread(target=download_waiter, args=(stop_wait, )) # create thread stop_wait.set() # raise the flag waiter.start() # start waiting urllib.request.urlretrieve( retrieve_url, tmp_fasta) # retrieve FASTA file except OSError as err: printlog_error_time( "Error occured while downloading fasta file.") printlog_error(str(err)) printlog_error( "`barapost-local.py` will try again in 30 seconds") if os.path.exists(tmp_fasta): os.unlink(tmp_fasta) # end if sleep(30) else: error = False finally: stop_wait.clear() # lower the flag waiter.join( ) # main thread will wait until waiter function ends it's work # end try # end while # end if printlog_info_time("Downloading is completed") # Write chunk to result fasta file with open(tmp_fasta, 'r') as infile, open(local_fasta, 'a') as outfile: outfile.write(infile.read()) # end with # Remove temp chunk file os.unlink(tmp_fasta) i += max_accnum # go to next chunk
tello.streamon() frame_read = tello.get_frame_read() def videoRecorder(): # create a VideoWrite object, recoring to ./video.avi height, width, _ = frame_read.frame.shape video = cv2.VideoWriter('video.avi', cv2.VideoWriter_fourcc(*'XVID'), 30, (width, height)) while keepRecording: video.write(frame_read.frame) time.sleep(1 / 30) video.release() # we need to run the recorder in a seperate thread, otherwise blocking options # would prevent frames from getting added to the video recorder = Thread(target=videoRecorder) recorder.start() tello.takeoff() tello.move_up(100) tello.rotate_counter_clockwise(360) tello.land() keepRecording = False recorder.join() tello.streamoff()
myvar = 10000 def thread_task2(): global myvar for _ in range(20): myvar += 1 print("子线程+1后 mymar=", myvar) time.sleep(1) print("子线程任务结束") t = Thread(target=thread_task2) t.start() def main_task(): global myvar for _ in range(10): myvar -= 2 print("主线程-1后 mymar=", myvar) time.sleep(1) print("主线程任务结束") main_task() t.join() print("变量的值是:", myvar)
t_env = Thread(target=enviar_missatge, args=(dades, usuari)) # t_env.start() def connectar(s): while True: try: conn, addr = s.accept() #Acceptar la conexió nomUsuari = conn.recv(1024) #Assignar nom usuari if nomUsuari.find("\n") != -1: nomUsuari = nomUsuari[:(len(nomUsuari) - 1)] + ":" print(nomUsuari) usuari = (conn, nomUsuari) #Assignar conexió client amb el seu nom llista_usuaris.append(usuari) #Afegim client a llista_clients t_reb = Thread(target=rebre_missatge, args=(usuari, )) t_reb.start() except: print("Ha fallat la connexió.") t_con = Thread(target=connectar, args=(s, )) t_con.start() t_con.join() time.sleep(1) s.close()
print("开始存钱") balance = self.balance time.sleep(5) self.balance = balance + num print("存入%s 成功" % num) # 解锁 self.lock.release() def draw(self, num): # 加锁 self.lock.acquire() print("开始存钱") balance = self.balance time.sleep(5) self.balance = balance - num print("取出%s 成功" % num) # 解锁 self.lock.release() acount1 = Account(10000, "小李") t1 = Thread(target=acount1.save, args=(1000,)) t2 = Thread(target=acount1.draw, args=(500,)) t1.start() t2.start() t1.join() t2.join() print("余额:%s" % acount1.balance)
class DroneTracker: major_ver, minor_ver, subminor_ver = (cv2.__version__).split('.') def __init__(self, videoFile): """ Initializes the tracker with the video file path :param videoFile: the path to the video to analyze """ self.current_frame = 0 self.total_frames = 0 self.data_points = [] self.videoFile = videoFile self.frame_queue = queue.Queue(maxsize=100) self.done_reading = False self.read_video_thread = Thread(target=self.read_video, args=()) tracker_types = [ 'BOOSTING', 'MIL', 'KCF', 'TLD', 'MEDIANFLOW', 'GOTURN', 'MOSSE', 'CSRT' ] self.tracker_type = tracker_types[2] # here the tracker type in the above line is instantiated if self.tracker_type == 'BOOSTING': self.tracker = cv2.TrackerBoosting_create() elif self.tracker_type == 'MIL': self.tracker = cv2.TrackerMIL_create() elif self.tracker_type == 'KCF': self.tracker = cv2.TrackerKCF_create() elif self.tracker_type == 'TLD': self.tracker = cv2.TrackerTLD_create() elif self.tracker_type == 'MEDIANFLOW': self.tracker = cv2.TrackerMedianFlow_create() elif self.tracker_type == 'GOTURN': self.tracker = cv2.TrackerGOTURN_create() elif self.tracker_type == 'MOSSE': self.tracker = cv2.TrackerMOSSE_create() elif self.tracker_type == "CSRT": self.tracker = cv2.TrackerCSRT_create() def rescale_frame(self, frame, percent=50): """ Resizes a frame as a percentage of the original frame size :param frame: the frame to be resized :param percent: the percent value the frame needs to be rescaled to """ width = int(frame.shape[1] * percent / 100) height = int(frame.shape[0] * percent / 100) dim = (width, height) return cv2.resize(frame, dim, interpolation=cv2.INTER_AREA) def resize_bbox(self, bbox: tuple, factor=2) -> tuple: """ Resizes the bounding box for translating it to the full size video In order to be able to see enough of the footage on screen to draw the box around the drone, the video frame must be resized, so the drawn bounding box must be translated back into the coordinate system the full size video uses For example, if the 4k footage is shrunk by 50% (to 1080p), the scale factor here must be 2 so the coordinates chosen in the 1080p frame will match up with the actual drone coordinates in the 4k frame :param bbox: bounding box of selected drone, which is (x, y, box_width, box_height) :param factor: the factor by which to scale the bounding box :return: tuple """ x1 = bbox[0] * factor y1 = bbox[1] * factor width = bbox[2] * factor height = bbox[3] * factor return (x1, y1, width, height) def is_light_on(self, frame): """ Takes in a video frame and returns the frame at which the light turns on """ # HSV range for white light white_lower = np.array([0, 0, 20]) white_upper = np.array([5, 2, 255]) frame_count = 0 # Capture frame-by-frame hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) mask = cv2.inRange(hsv, white_lower, white_upper) count = cv2.countNonZero(mask) # light is on if count > 100000 and count < 180000: return True else: return False def read_video(self) -> None: """ This function is to be threaded, and its purpose is to read in the video file all at once to improve performance """ video = cv2.VideoCapture(self.videoFile) # If unable to open the video file (probably wrong path) if not video.isOpened(): print("Could not open video") sys.exit() # Read first frame. ok, frame = video.read() # If unable to get a frame (probably bad format) if not ok: print('Cannot read video file') """ here i should delete the lock files """ sys.exit() self.total_frames += 1 self.frame_queue.put(frame) # Read frames until the end of the video is reached (or until it hits corruption) while True: ok, frame = video.read() # If not ok, which is normally when the end of the file has been read, break from loop if not ok: break # Add the frame to the respective frame queue self.frame_queue.put(frame, block=True) self.total_frames += 1 # Signal that the thread is done reading the video file self.done_reading = True def trackDrone(self): # Start video thread self.read_video_thread.start() # Reset the counter so the time from when the sync light is detected is the same for both videos self.current_frame = 0 frame = self.frame_queue.get(block=True) # Here I resize the frame to 40% of its size, so the entire frame can be displayed onto the screen # The 4k footage is too large for my laptop screen, so this scales it down resizedFrame = self.rescale_frame(frame, 30) # Ask the user to draw a box around the drone bbox = cv2.selectROI("Select Drone", resizedFrame, False) # Reposition the bounding box from 40% resolution to 100% resolution bbox = self.resize_bbox(bbox, 3.33) cv2.destroyWindow("Select Drone") # Initialize tracker with first frame and bounding box ok = self.tracker.init(frame, bbox) # This while loop will run until the footage is finished, processing all of the drone footage while True: # Check if queue is empty but the read thread for the video is not done if self.frame_queue.empty() and not self.done_reading: # If so, sleep for a second for the producer to catch up and try again time.sleep(1) print("slept for 1 second") continue # Check if queue is empty and the read thread for video 1 is done if self.frame_queue.empty() and self.done_reading: # If so, destroy the window and break out of loop cv2.destroyWindow("Tracking") print("done tracking this drone at " + str(tm) + " seconds") break # Get frame from the frame queue frame = self.frame_queue.get(block=True) # Start timer timer = cv2.getTickCount() # Update tracker ok, bbox = self.tracker.update(frame) # Calculate Frames per second (FPS) fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer) tm = self.current_frame / 30 # Draw bounding box if ok: # Tracking success p1 = (int(bbox[0]), int(bbox[1])) p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])) if self.current_frame % 15 == 0: x_coord = bbox[0] + bbox[2] / 2 y_coord = bbox[1] + bbox[3] / 2 print("drone coordinate at time " + str(tm) + ": [" + str(x_coord) + ", " + str(y_coord) + "] (" + str((x_coord / 3860) * 15) + ", " + str((y_coord / 2160) * 10) + ")") self.data_points.append((x_coord, y_coord, tm)) cv2.rectangle(frame, p1, p2, (255, 0, 0), 2, 1) else: # Tracking failure cv2.putText(frame, "Tracking failure detected", (100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2) skip = 5 # This will loop until the drone is reselected, and creates a new tracker that will # continue to track the drone while True: # Rescale the frame so the user can see the entire frame to reselect the drone resizedFrame = self.rescale_frame(frame, 30) # Ask the user to draw a box around the drone bbox = cv2.selectROI("Reselect Drone", resizedFrame, False) # Reposition the bounding box from 40% resolution to 100% resolution bbox = self.resize_bbox(bbox, 3.33) # Close the ROI selection window cv2.destroyWindow("Reselect Drone") # This will skip 1 second of video if the drone is out of frame, # which is detected if the user presses escape on the selectROI function if bbox[0] == 0.0 and bbox[1] == 0.0 and bbox[ 2] == 0.0 and bbox[3] == 0.0: print("skipping " + str(skip) + " frames") for i in range(1, skip): if self.current_frame % 15 == 0: tm = self.current_frame / 30 self.data_points.append((None, None, tm)) # If the video has totally been processed, exit the function # Kind of a hacky way to do it, but I need to break out of 3 loops here # so returning is easier than functionalizing the small parts of this giant function if self.frame_queue.empty() and self.done_reading: self.read_video_thread.join() return self.data_points frame = self.frame_queue.get(block=True) self.current_frame += 1 if skip == 5: skip = 15 elif skip == 15: skip = 30 elif skip == 30: skip = 60 elif skip == 60: skip = 90 # else: # skip += 30 # If the user selected a bounding box, create a new tracker # and continue to track the drone else: # Creates a new KCF tracker (this is due to the update function not working # the same in python as it does in C++: in C++, we could simply update the # tracker with the new bounding box instead of creating a new one, but this # is a known limitation in python's implementation of OpenCV and this is my workaround) self.tracker = cv2.TrackerKCF_create() # Initialize the tracker with the newly selected bounding box ok = self.tracker.init(frame, bbox) break # Display tracker type on frame cv2.putText(frame, self.tracker_type + " Tracker", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2) # Display FPS on frame cv2.putText(frame, "FPS : " + str(int(fps)), (100, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2) resizedFrame = self.rescale_frame(frame, 20) # Display result cv2.imshow("Tracking", resizedFrame) self.current_frame += 1 # Exit if ESC pressed k = cv2.waitKey(1) & 0xff if k == 27: break # Allows the user to reset the bounding box in the case that the tracker # begins to track a different object or isn't tracking the drone accurately # enough elif "r" == chr(k): # Rescale the frame so the user can see the entire frame to reselect the drone resizedFrame = self.rescale_frame(frame, 40) # Ask the user to draw a box around the drone bbox = cv2.selectROI("Reselect Drone", resizedFrame, False) # Reposition the bounding box from 40% resolution to 100% resolution bbox = self.resize_bbox(bbox, 2.5) # Close the ROI selection window cv2.destroyWindow("Reselect Drone") # Creates a new KCF tracker (this is due to the update function not working # the same in python as it does in C++: in C++, we could simply update the # tracker with the new bounding box instead of creating a new one, but this # is a known limitation in python's implementation of OpenCV and this is my workaround) self.tracker = cv2.TrackerKCF_create() # Initialize the tracker with the newly selected bounding box ok = self.tracker.init(frame, bbox) self.read_video_thread.join() return self.data_points def get_data_points(self, data_points_1, data_points_2) -> list: """ Returns a list of tuples that represent the flight path of the drone :return: list of tuples of coordinates and time values that represent the flight path of the drones in the format [(time, x_coord, y_coord, z_coord)] """ points = [] shortest_len = min(len(data_points_1), len(data_points_2)) # Cycle through all of the data points that can be represented with the extracted coordinates # from both videos for i in range(0, shortest_len - 1): tup = ( data_points_1[i][2], # time value (data_points_1[i][0] / 3840) * 15, # x coordinate scaled to 15 meters (data_points_2[i][0] / 3840) * 15, # y coordinate scaled to 15 meters (((data_points_1[i][1] + data_points_2[i][1]) / 2) / 2160) * 10 ) # z coordinate, # scaled to 10 meters points.append(tup) return points
def send(self, data): thr = Thread(target=self._send_on_thread, args=(data, )) thr.start() thr.join()
class MetricWelder: _ID = 'metric' _ONE = Decimal('1.0') _HALF = Decimal('0.5') _ZERO = Decimal('0.0') def __init__(self, context): self.__context = context self.__logger = context.get_logger(self) self.__thread = Thread(daemon=False, target=self._execute) def run(self): self.__thread.start() def _join(self): self.__thread.join() def _execute(self): self.__logger.info('Processing : %s', self._ID) threads = [ Thread(target=self._wrap, args=(self.process_metric, 30)), Thread(target=self._wrap, args=(self.purge_metric, 3600)), ] for t in threads: t.start() for t in threads: t.join() self.__logger.info('Terminated.') def _wrap(self, func, interval): while not self.__context.is_closed(): try: func() except BaseException as e: self.__logger.warn('%s - %s : %s', func.__name__, type(e), e.args) sleep(interval) def process_metric(self, *, default_time=None, default_count=3): base_time = default_time if default_time is not None else self.__context.get_now( ) count = int( self.__context.get_property(self._ID, 'timestamp', default_count)) timestamps = [ base_time.replace(second=0, microsecond=0) - timedelta(minutes=i) for i in range(0, count) ] self.__logger.debug('Metrics : %s', [t.strftime('%Y-%m-%d %H:%M') for t in timestamps]) threads = [] for timestamp in timestamps: prices = self.process_ticker(timestamp) threads.append( Thread(target=self.process_balance, args=(timestamp, prices))) threads.append( Thread(target=self.process_position, args=(timestamp, prices))) threads.append( Thread(target=self.process_transaction_trade, args=(timestamp, prices))) threads.append( Thread(target=self.process_transaction_volume, args=(timestamp, prices))) for t in threads: t.start() for t in threads: t.join() def calculate_evaluation(self, evaluation, prices): if evaluation is None: return None price = self._ONE if evaluation.ev_ticker_site is not None and evaluation.ev_ticker_code is not None: codes = prices.get(evaluation.ev_ticker_site) p = codes.get( evaluation.ev_ticker_code) if codes is not None else None if p is None or p == self._ZERO: return None price = price * p if evaluation.ev_convert_site is not None and evaluation.ev_convert_code is not None: codes = prices.get(evaluation.ev_convert_site) p = codes.get( evaluation.ev_convert_code) if codes is not None else None if p is None or p == self._ZERO: return None price = price * p return price def process_ticker(self, timestamp): prices = None try: values = self.__context.fetch_tickers(timestamp, include_expired=True) prices = defaultdict(lambda: dict()) for dto in values if values is not None else []: ticker = dto.ticker ask = ticker.tk_ask if ticker.tk_ask != self._ZERO else None bid = ticker.tk_bid if ticker.tk_bid != self._ZERO else None ltp = ticker.tk_ltp if ticker.tk_ltp != self._ZERO else None price = None if ask is not None and bid is not None: price = (ask + bid) * self._HALF if price is None: price = ask if price is None: price = bid if price is None: price = ltp prices[ticker.tk_site][ticker.tk_code] = price metrics = [] threshold_minutes = self.__context.get_property( self._ID, 'ticker_threshold', 3) threshold_cutoff = timestamp - timedelta( minutes=int(threshold_minutes)) for dto in values if values is not None else []: if dto.ticker.tk_time.replace( tzinfo=threshold_cutoff.tzinfo) < threshold_cutoff: continue price = prices[dto.ticker.tk_site][dto.ticker.tk_code] if price is None or price == self._ZERO or dto.product is None: continue expiry = dto.product.pr_expr if expiry is not None and expiry.astimezone( timestamp.tzinfo) < timestamp: continue rate = self.calculate_evaluation(dto.fund, prices) if rate is None: continue metric = Metric() metric.mc_type = 'ticker' metric.mc_name = dto.product.pr_disp metric.mc_time = timestamp metric.mc_amnt = price * rate metrics.append(metric) self.__context.save_metrics(metrics) except BaseException as e: self.__logger.warn('Ticker : %s : %s', type(e), e.args) return prices def process_balance(self, timestamp, prices): try: metrics = [] values = self.__context.fetch_balances(timestamp) for dto in values if values is not None else []: amount = dto.balance.bc_amnt rate = self.calculate_evaluation(dto.evaluation, prices) if dto.account is None or amount is None or rate is None: continue metric = Metric() metric.mc_type = 'balance' metric.mc_name = dto.account.ac_disp metric.mc_time = timestamp metric.mc_amnt = amount * rate metrics.append(metric) self.__context.save_metrics(metrics) except BaseException as e: self.__logger.warn('Balance : %s : %s', type(e), e.args) def process_position(self, timestamp, prices): try: metrics = [] values = self.__context.fetch_positions(timestamp) for dto in values if values is not None else []: amount = dto.position.ps_fund rate = self.calculate_evaluation(dto.fund, prices) if dto.product is None or amount is None or rate is None: continue metric = Metric() metric.mc_type = 'position@upl' metric.mc_name = dto.product.pr_disp metric.mc_time = timestamp metric.mc_amnt = amount * rate metrics.append(metric) for dto in values if values is not None else []: amount = dto.position.ps_inst rate = self.calculate_evaluation(dto.inst, prices) if dto.product is None or amount is None or rate is None: continue metric = Metric() metric.mc_type = 'position@qty' metric.mc_name = dto.product.pr_disp metric.mc_time = timestamp metric.mc_amnt = amount * rate metrics.append(metric) self.__context.save_metrics(metrics) except BaseException as e: self.__logger.warn('Position : %s : %s', type(e), e.args) def process_transaction_trade(self, timestamp, prices): try: metrics = [] offset = timedelta(minutes=int( self.__context.get_property(self._ID, 'offset', 9 * 60))) t = timestamp + offset windows = { 'DAY': t.replace(microsecond=0, second=0, minute=0, hour=0) - offset, 'MTD': t.replace(microsecond=0, second=0, minute=0, hour=0, day=1) - offset, 'YTD': t.replace( microsecond=0, second=0, minute=0, hour=0, day=1, month=1) - offset, } for key, val in windows.items(): values = self.__context.fetch_transactions(val, timestamp) for dto in values if values is not None else []: inst_qty = dto.tx_net_inst fund_qty = dto.tx_net_fund inst_rate = self.calculate_evaluation(dto.ev_inst, prices) fund_rate = self.calculate_evaluation(dto.ev_fund, prices) if dto.product is None \ or inst_qty is None or fund_qty is None \ or inst_rate is None or fund_rate is None: continue metric = Metric() metric.mc_type = 'trade@' + key metric.mc_name = dto.product.pr_disp metric.mc_time = timestamp metric.mc_amnt = (inst_qty * inst_rate) + (fund_qty * fund_rate) metrics.append(metric) self.__context.save_metrics(metrics) except BaseException as e: self.__logger.warn('Transaction (trade) : %s : %s', type(e), e.args) def process_transaction_volume(self, timestamp, prices): try: metrics = [] windows = { '12H': timestamp - timedelta(hours=12), '01D': timestamp - timedelta(hours=24), '30D': timestamp - timedelta(days=30), } for key, val in windows.items(): values = self.__context.fetch_transactions(val, timestamp) for dto in values if values is not None else []: amount = dto.tx_grs_fund rate = self.calculate_evaluation(dto.ev_fund, prices) if dto.product is None or amount is None or rate is None: continue metric = Metric() metric.mc_type = 'volume@' + key metric.mc_name = dto.product.pr_disp metric.mc_time = timestamp metric.mc_amnt = amount * rate metrics.append(metric) self.__context.save_metrics(metrics) except BaseException as e: self.__logger.warn('Transaction (volume) : %s : %s', type(e), e.args) def purge_metric( self, *, intervals=( # [0] Older than 5+ years, delete all. (1984, tuple()), # [1] Older than 1+ month, hourly interval. (42, tuple([0])), # [2] Older than 1 week, 30 minutes interval. (7, tuple(i * 30 for i in range(0, 2))), # [3] Older than 2 days, 15 minutes interval. (2, tuple(i * 15 for i in range(0, 4))), # [4] Older than 1 day, 5 minutes interval. (1, tuple(i * 5 for i in range(0, 12))), )): now = self.__context.get_now() for idx, entry in enumerate(intervals): days = self.__context.get_property(self._ID, 'purge_%s' % idx, entry[0]) if days is None: continue cutoff = now - timedelta(days=max(int(days), 1)) count = self.__context.delete_metrics(cutoff, exclude_minutes=entry[1]) self.__logger.debug('Purged [%s] cutoff=%s count=%s', idx, cutoff, count)
class Search: """ This class contains all search logic. """ search_thread = None search_thread_stop = None def __init__(self): self.ui = cozy.ui.main_view.CozyUI() self.builder = Gtk.Builder.new_from_resource( "/de/geigi/cozy/search_popover.ui") self.popover = self.builder.get_object("search_popover") self.book_label = self.builder.get_object("book_label") self.track_label = self.builder.get_object("track_label") self.author_label = self.builder.get_object("author_label") self.reader_label = self.builder.get_object("reader_label") self.reader_box = self.builder.get_object("reader_result_box") self.author_box = self.builder.get_object("author_result_box") self.book_box = self.builder.get_object("book_result_box") self.track_box = self.builder.get_object("track_result_box") self.entry = self.builder.get_object("search_entry") self.scroller = self.builder.get_object("search_scroller") self.book_separator = self.builder.get_object("book_separator") self.author_separator = self.builder.get_object("author_separator") self.reader_separator = self.builder.get_object("reader_separator") self.stack = self.builder.get_object("search_stack") self.entry.connect("search-changed", self.__on_search_changed) if Gtk.get_minor_version() > 20: self.scroller.set_max_content_width(400) self.scroller.set_max_content_height(600) self.scroller.set_propagate_natural_height(True) self.scroller.set_propagate_natural_width(True) self.search_thread = Thread(target=self.search, name="SearchThread") self.search_thread_stop = threading.Event() def get_popover(self): """ Get the search popover :return: Search popover """ return self.popover def search(self, user_search): """ Perform a search with the current search entry """ # we need the main context to call methods in the main thread after the search is finished main_context = GLib.MainContext.default() books = search_books(user_search) if self.search_thread_stop.is_set(): return main_context.invoke_full(GLib.PRIORITY_DEFAULT, self.__on_book_search_finished, books) authors = search_authors(user_search) if self.search_thread_stop.is_set(): return main_context.invoke_full(GLib.PRIORITY_DEFAULT, self.__on_author_search_finished, authors) readers = search_readers(user_search) if self.search_thread_stop.is_set(): return main_context.invoke_full(GLib.PRIORITY_DEFAULT, self.__on_reader_search_finished, readers) if readers.count() < 1 and authors.count() < 1 and books.count() < 1: main_context.invoke_full(GLib.PRIORITY_DEFAULT, self.stack.set_visible_child_name, "nothing") def close(self, object=None): """ Close the search popover specific to the used gtk version. """ if Gtk.get_minor_version() < 22: self.popover.hide() else: self.popover.popdown() def __on_search_changed(self, sender): """ Reset the search if running and start a new async search. """ self.search_thread_stop.set() #self.throbber.set_visible(True) #self.throbber.start() # we want to avoid flickering of the search box size # as we remove widgets and add them again # so we get the current search popup size and set it as # the preferred size until the search is finished # this helps only a bit, the widgets are still flickering self.popover.set_size_request(self.popover.get_allocated_width(), self.popover.get_allocated_height()) # hide nothing found self.stack.set_visible_child_name("main") # First clear the boxes tools.remove_all_children(self.book_box) tools.remove_all_children(self.author_box) tools.remove_all_children(self.reader_box) # Hide all the labels & separators self.book_label.set_visible(False) self.author_label.set_visible(False) self.reader_label.set_visible(False) self.book_separator.set_visible(False) self.author_separator.set_visible(False) self.reader_separator.set_visible(False) self.track_label.set_visible(False) user_search = self.entry.get_text() if user_search: if self.search_thread.is_alive(): self.search_thread.join(timeout=0.2) self.search_thread_stop.clear() self.search_thread = Thread(target=self.search, args=(user_search, )) self.search_thread.start() else: #self.throbber.stop() #self.throbber.set_visible(False) self.stack.set_visible_child_name("start") self.popover.set_size_request(-1, -1) def __on_book_search_finished(self, books): """ This gets called after the book search is finished. It adds all the results to the gui. :param books: Result peewee query containing the books """ if len(books) > 0: self.stack.set_visible_child_name("main") self.book_label.set_visible(True) self.book_separator.set_visible(True) for book in books: if self.search_thread_stop.is_set(): return self.book_box.add( BookSearchResult(book, self.ui.jump_to_book, self.ui.window.get_scale_factor())) def __on_author_search_finished(self, authors): """ This gets called after the author search is finished. It adds all the results to the gui. :param authors: Result peewee query containing the authors """ if len(authors) > 0: self.stack.set_visible_child_name("main") self.author_label.set_visible(True) self.author_separator.set_visible(True) for author in authors: if self.search_thread_stop.is_set(): return self.author_box.add( ArtistSearchResult(self.ui.jump_to_author, author, True)) def __on_reader_search_finished(self, readers): """ This gets called after the reader search is finished. It adds all the results to the gui. It also resets the gui to a state before the search. :param readers: Result peewee query containing the readers """ if len(readers) > 0: self.stack.set_visible_child_name("main") self.reader_label.set_visible(True) self.reader_separator.set_visible(True) for reader in readers: if self.search_thread_stop.is_set(): return self.reader_box.add( ArtistSearchResult(self.ui.jump_to_reader, reader, False)) # the reader search is the last that finishes # so we stop the throbber and reset the prefered height & width #self.throbber.stop() #self.throbber.set_visible(False) self.popover.set_size_request(-1, -1)
def Download(self, blocking=True): """ Start The download! Blocking is given by the blocking param # default blocking is True self.Download(blocking=True) """ def cheak_if_exist(): # this function is only defines to used recursively if isfile(self.location + self.file_name): if getsize(self.location + self.file_name) == self.size: self.done = self.size self.completed = True raise FileExistsError("Same File Probably Exists") else: LOG()(1, "File with Same \ File Name Exists Changing File \ Name to :{}".format('New ' + self.file_name)) self.file_name = "New " + self.file_name cheak_if_exist() else: pass cheak_if_exist() self.update_lock = Lock() self.block = blocking if self.downloadable and self.downloading: raise ValueError('Allready Downloading, self.downloading = True') def _Download(): try: logging.debug('self.block@_Download@{}: {}'.format( self, self.block)) if self.downloadable and (not self.downloading): self.downloading = True logging.info('Starting Download!') self.stoped = False mother_thread = E_Thread(_download, Max_thread=8) if (not self.pauseable) or (self.size is None): self.no_of_parts = 1 Part_length = 0 else: Part_length = self.size // self.no_of_parts logging.debug('Part_length@_Download@{}: {}'.format( self, Part_length)) for _ in range(self.no_of_parts - 1): mother_thread.start(self, (_ * Part_length, (_ + 1) * Part_length), _) mother_thread.start( self, ((self.no_of_parts - 1) * Part_length, self.size), self.no_of_parts - 1) mother_thread.join() self.completed = True self.downloading = False _writer(self) else: pass except Exception as e: logging.critical('HarvesterEngine._Download@{}[{}]:\ {}'.format(self, e.__traceback__.tb_lineno, e)) self.downloading = False self.DownloadError = e print(e) mother_thread.close() main_thread = Thread(target=_Download, daemon=True) main_thread.start() if self.block: try: main_thread.join() except KeyboardInterrupt: self.Pause()
def forward(): global leg_formation if (leg_formation == 1): #send leg1 to lateral and leg3 to parallel (lift) t1 = Thread(target=leg1_p2l) t3 = Thread(target=leg3_l2p) #send leg2 to lateral, and leg4 to parallel (drag) t2 = Thread(target=leg2, args=(back_lateral, footdown, pincer_down)) t4 = Thread(target=leg4, args=(front_parallel, footdown, pincer_down)) t1.start() t3.start() t2.start() t4.start() t1.join() t3.join() t2.join() t4.join() time.sleep(step_delay) #now right side legs are parallel and left side legs are lateral if (leg_formation == 2): #send leg2 to parallel and leg4 to lateral (lift) t2 = Thread(target=leg2_l2p) t4 = Thread(target=leg4_p2l) # sending leg3 to lateral, and leg1 to parallel t3 = Thread(target=leg3, args=(back_lateral, footdown, pincer_down)) t1 = Thread(target=leg1, args=(front_parallel, footdown, pincer_down)) t3.start() t1.start() t2.start() t4.start() t2.join() t4.join() t3.join() t1.join() time.sleep(step_delay) #now left side legs are parallel and right side legs are lateral if (leg_formation == 1): leg_formation = 2 elif (leg_formation == 2): leg_formation = 1
from threading import Thread COUNT = 50000000 # 50 M def countdown(counter): """ function decrements counter by one each time, until counter becomes zero Args: counter (int): counter value """ while counter > 0: counter -= 1 if __name__ == "__main__": THREAD1 = Thread(target=countdown, args=(COUNT // 2,)) THREAD2 = Thread(target=countdown, args=(COUNT // 2,)) START = time.time() THREAD1.start() print(THREAD1.isAlive()) THREAD2.start() THREAD1.join() print(THREAD1.isAlive()) print(THREAD2.isAlive()) THREAD2.join() END = time.time() print("Time taken in seconds -", END - START)
class DecisionMaker(WithLogger): """This class implements the decision maker.""" def __init__( self, decision_maker_handler: DecisionMakerHandler, ) -> None: """ Initialize the decision maker. :param agent_name: the agent name :param decision_maker_handler: the decision maker handler """ WithLogger.__init__(self, logger=decision_maker_handler.logger) self._agent_name = decision_maker_handler.identity.name self._queue_access_code = uuid4().hex self._message_in_queue = ProtectedQueue( self._queue_access_code) # type: ProtectedQueue self._decision_maker_handler = decision_maker_handler self._thread = None # type: Optional[Thread] self._lock = threading.Lock() self._message_out_queue = decision_maker_handler.message_out_queue self._stopped = True @property def message_in_queue(self) -> ProtectedQueue: """Get (in) queue.""" return self._message_in_queue @property def message_out_queue(self) -> AsyncFriendlyQueue: """Get (out) queue.""" return self._message_out_queue @property def decision_maker_handler(self) -> DecisionMakerHandler: """Get the decision maker handler.""" return self._decision_maker_handler def start(self) -> None: """Start the decision maker.""" with self._lock: if not self._stopped: # pragma: no cover self.logger.debug( "[{}]: Decision maker already started.".format( self._agent_name)) return self._stopped = False self._thread = Thread(target=self.execute, name=self.__class__.__name__) self._thread.start() def stop(self) -> None: """Stop the decision maker.""" with self._lock: self._stopped = True self.message_in_queue.put(None) if self._thread is not None: self._thread.join() self.logger.debug("[{}]: Decision Maker stopped.".format( self._agent_name)) self._thread = None def execute(self) -> None: """ Execute the decision maker. Performs the following while not stopped: - gets internal messages from the in queue and calls handle() on them :return: None """ while not self._stopped: message = self.message_in_queue.protected_get( self._queue_access_code, block=True) # type: Optional[Message] if message is None: self.logger.debug( "[{}]: Received empty message. Quitting the processing loop..." .format(self._agent_name)) continue self.handle(message) def handle(self, message: Message) -> None: """ Handle an internal message from the skills. :param message: the internal message :return: None """ self.decision_maker_handler.handle(message)
client.close() del clients[client] broadcast(bytes("%s left" % name, "utf8")) break def broadcast(msg, prefix=""): # prefix is for name identification. """Broadcasts a message to all the clients.""" for sock in clients: sock.send(bytes(prefix, "utf8")+msg) clients = {} addresses = {} HOST = '' PORT = 33001 BUFSIZ = 1024 ADDR = (HOST, PORT) SERVER = socket(AF_INET, SOCK_STREAM) SERVER.bind(ADDR) if __name__ == "__main__": SERVER.listen(5) print("Waiting for Connections..") ACCEPT_THREAD = Thread(target=accept_incoming_connections) ACCEPT_THREAD.start() ACCEPT_THREAD.join() SERVER.close()
thread = Thread(target=error_thread, args=( model, outf, )) fetchers = [] for i in range(1, 9): ft = Thread(target=fetch_thread, args=(i, )) ft.start() fetchers.append(ft) try: main(thread, log) print(fin) except KeyboardInterrupt: for ft in fetchers: ft.join() thread.join() if log: outf.close() print("outf closed by interrupt") exit() if log: print("issued log halt") for ft in fetchers: ft.join() thread.join() time.sleep(10) outf.close() print("outf closed by END")