def do_start_proxy(): log("do_start_proxy()") message_queue = MQueue() try: ioe = client_proto.wait_for_io_threads_exit(5+self._socket_timeout) if not ioe: log.error("Error: some network IO threads have failed to terminate") return client_conn.set_active(True) process = ProxyInstanceProcess(uid, gid, env_options, session_options, self._socket_dir, self.video_encoders, self.csc_modules, client_conn, disp_desc, client_state, cipher, encryption_key, server_conn, c, message_queue) log("starting %s from pid=%s", process, os.getpid()) self.processes[process] = (display, message_queue) process.start() log("process started") popen = process._popen assert popen #when this process dies, run reap to update our list of proxy processes: self.child_reaper.add_process(popen, "xpra-proxy-%s" % display, "xpra-proxy-instance", True, True, self.reap) finally: #now we can close our handle on the connection: client_conn.close() server_conn.close() message_queue.put("socket-handover-complete")
def do_start_proxy(): log("do_start_proxy()") try: #stop IO in proxy: #(it may take up to _socket_timeout until the thread exits) client_conn.set_active(False) ioe = client_proto.wait_for_io_threads_exit( 0.1 + self._socket_timeout) if not ioe: log.error("IO threads have failed to terminate!") return #now we can go back to using blocking sockets: self.set_socket_timeout(client_conn, None) client_conn.set_active(True) assert uid != 0 and gid != 0 message_queue = MQueue() process = ProxyInstanceProcess( uid, gid, env_options, session_options, self._socket_dir, self.video_encoders, self.csc_modules, client_conn, client_state, cipher, encryption_key, server_conn, c, message_queue) log("starting %s from pid=%s", process, os.getpid()) process.start() log("process started") #FIXME: remove processes that have terminated self.processes[process] = (display, message_queue) finally: #now we can close our handle on the connection: client_conn.close() server_conn.close()
def evolve_p_s(settings: EnhanceModelSettings, optimal_cost, cum_cost, ev_set:EvolveSettings, secondaries=None): cons = [] returnq = MQueue() new_set = EnhanceModelSettings(item_store=GearItemStore()) new_set.set_state_json(settings.get_state_json()) if secondaries is not None: new_set[new_set.P_FAIL_STACKER_SECONDARY] = secondaries new_set[settings.P_R_FAIL_STACKERS] = [] new_set[settings.P_FAIL_STACKERS] = [] new_set[settings.P_R_STACKER_SECONDARY] = [] new_set[settings.P_ENHANCE_ME] = [] new_set[settings.P_R_ENHANCE_ME] = [] new_set[settings.P_R_FOR_PROFIT] = [] new_set[settings.P_R_FOR_PROFIT] = [] new_set[settings.P_ALTS] = [] new_set[settings.P_NADERR_BAND] = [] new_set[settings.P_VALKS] = {} new_primaries = [None]*len(optimal_cost) num_proc = ev_set.num_procs for i in range(num_proc): cons.append(MPipe(False)) procs = [] for i in range(num_proc): procs.append(Process(target=evolve_multi_process_landing, args=(cons[i-1][0], cons[i][1], returnq, new_set.get_state_json(), new_primaries, optimal_cost, cum_cost, ev_set))) return returnq, procs
def start_proxy_process(): log("start_proxy_process()") message_queue = MQueue() client_conn = None try: #no other packets should be arriving until the proxy instance responds to the initial hello packet def unexpected_packet(packet): if packet: log.warn("Warning: received an unexpected packet") log.warn(" from the proxy connection %s:", client_proto) log.warn(" %s", repr_ellipsized(packet)) client_proto.close() client_conn = client_proto.steal_connection(unexpected_packet) client_state = client_proto.save_state() log("start_proxy_process(..) client connection=%s", client_conn) log("start_proxy_process(..) client state=%s", client_state) ioe = client_proto.wait_for_io_threads_exit( 5 + self._socket_timeout) if not ioe: log.error( "Error: some network IO threads have failed to terminate" ) client_proto.close() return client_conn.set_active(True) from xpra.server.proxy.proxy_instance_process import ProxyInstanceProcess process = ProxyInstanceProcess( uid, gid, env_options, session_options, self._socket_dir, self.video_encoders, self.pings, client_conn, disp_desc, client_state, cipher, encryption_key, server_conn, c, message_queue) log("starting %s from pid=%s", process, os.getpid()) self.instances[process] = (True, display, message_queue) process.start() log("ProxyInstanceProcess started") popen = process._popen assert popen #when this process dies, run reap to update our list of proxy instances: self.child_reaper.add_process(popen, "xpra-proxy-%s" % display, "xpra-proxy-instance", True, True, self.reap) except Exception as e: log("start_proxy_process() failed", exc_info=True) log.error("Error starting proxy instance process:") log.error(" %s", e) message_queue.put("error: %s" % e) message_queue.put("stop") finally: #now we can close our handle on the connection: log("handover complete: closing connection from proxy server") if client_conn: client_conn.close() server_conn.close() log("sending socket-handover-complete") message_queue.put("socket-handover-complete")
def __init__(self, queue=None): super(PygameThread, self).__init__() self.window = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT)) self.clock = pygame.time.Clock() self.lock = Lock() self._queue = queue if not queue: self._queue = MQueue(maxsize=2) self.sprites = [] self.spriteGroup = pygame.sprite.Group()
def __init__(self, proc_count: Optional[int] = None, address: Optional[bytes] = None, id_generator: Optional[Callable[[], Iterator[Union[ str, int]]]] = lambda: count()): self.log = logger.new() self.job_data = MinerJob() self.signal = Value('B') self.queue = MQueue() self.proc_count = proc_count self.job = {} self.miners = [] self.loop = None self.address = address self._iter_id = id_generator and id_generator() or None
def __init__(self, cls, *args, **kwargs): super(New_Process_Actor, self).__init__() self.cls = cls self.args = list(args) self.kwargs = kwargs self.mqueue = MQueue() self.mevent = MEvent() if 'input_channel' not in kwargs: kwargs['input_channel'] = self.args[0] chan = kwargs['input_channel'] kwargs['input_channel'] = self.mqueue print 'chan: ', chan self.c2p = Channel2Process(chan, self.mevent, self.mqueue) self.c2p.start()
def do_start_proxy(): log("do_start_proxy()") message_queue = MQueue() try: ioe = client_proto.wait_for_io_threads_exit(5+self._socket_timeout) if not ioe: log.error("some network IO threads have failed to terminate!") return client_conn.set_active(True) assert uid!=0 and gid!=0 process = ProxyInstanceProcess(uid, gid, env_options, session_options, self._socket_dir, self.video_encoders, self.csc_modules, client_conn, client_state, cipher, encryption_key, server_conn, c, message_queue) log("starting %s from pid=%s", process, os.getpid()) self.processes[process] = (display, message_queue) process.start() log("process started") finally: #now we can close our handle on the connection: client_conn.close() server_conn.close() message_queue.put("socket-handover-complete")
def __init__(self, max_size): super().__init__(max_size) self.queue = MQueue(maxsize=max_size)
import requests from lxml import etree from queue import Queue as TQueue from multiprocessing import Process from multiprocessing import JoinableQueue as MQueue import urllib.parse as urlparse from io import BytesIO import threading import time import re import multiprocessing import os url_queue = MQueue() html_queue = MQueue() class HTMLProcessor(Process): def __init__(self, inputQue, outputQue, host, release, proc_id, **kwargs): super(HTMLProcessor, self).__init__() self.inputQue = inputQue self.outputQue = outputQue self.host = host self.release = release self.kwargs = kwargs self.parser = etree.HTMLParser() self.base = None self.proc_id = 'Process%s' % proc_id self.filename = 'proc%s_log.txt' % proc_id
frame = mqframes.get_nowait() suit.blackout() for pixels in frame: strip.setPixelColor(pixels[0], pixels[1]) strip.show() if __name__ == '__main__': s = Suit() connr, conns = Pipe() strip = s.strip strip.begin() mqtrigger = MQueue(maxsize=3) mqanimation = MQueue() mqframes = MQueue() pa = Process(target=getFreqInfo, args=(s, mqtrigger)) pb = Process(target=buildAnimations, args=(mqtrigger, mqanimation, connr)) p2 = Process(target=playAnimations, args=(s, mqanimation, mqframes)) pc = Process(target=ampCheck, args=(conns, )) pf = Process(target=playFrames, args=(s, mqframes)) pa.start() pb.start() p2.start() pc.start() pf.start() pa.join() pb.join() p2.join()
# queue.Queue – Locking Semantics for Parallel Computing q = Queue() q.put('eat') q.put('sleep') q.put('code') print(q) print(q.get()) print(q.get()) print(q.get()) try: print(q.get_nowait()) except Exception as e: print(e) try: print(q.get()) except Exception as e: print(e) # multiprocessing.Queue – Shared Job Queues q = MQueue() q.put('eat') q.put('sleep') q.put('code') print(q) print(q.get()) print(q.get()) print(q.get()) print(q.get())
def main(*args, **kwargs): global key global thread global DownloadThread keyword = args key = keyword[0] mod = kwargs['mod'] thread = int(kwargs['thread']) DownloadThread = int(kwargs['downloadthread']) manage = Manager() manage2 = ManagerCookie() # cookiepool = utils.CookiePool('tianyanchaCookie') # lock = Lock() # listinfolock = MLock() # downlock = MLock() cookiepool = manage2.CookiePool('tianyanchaCookie') listinfolock = manage.Lock() rlk_list = {} for i in cookiepool.getWeightData()['tianyanchaCookieDict_Spider']: rlk_list[i] = RLock() logger.info( "[*] thead %s | DownloadThead %s | DataThread %s" % (kwargs['thread'], kwargs['downloadthread'], kwargs['datathread'])) logger.info("[*] 正在进行基线检测") if settings.RD.exists('tianyanchaDownCookie'): getAllPages(search_url.format(page=1, key=key), cookiepool=cookiepool, user='******') else: get_cookie(mod, user='******', browsermod=False) for k in ['Spider', 'List']: if cookiepool.exists(k): getAllPages(search_url.format(page=1, key=key), cookiepool=cookiepool, user=k) else: get_cookie(mod, browsermod=False) thList = [] logger.info("[*] 开始收集信息") listQue = MQueue(1000) filterQue = MQueue(10) ThreadFlag = Value("d", 1) # for i in range(thread): # i = threading.Thread(target= # , args=('Spider', listQue, rlk_list, cookiepool, ThreadFlag)) # i.start() # thList.append(i) p = Process(target=SpiderURL, args=(filterQue, )) p.start() thList.append(p) time.sleep(2) logger.info("[*] 开始爬取信息") for i in range(DownloadThread): x = Process(target=SpiderThreadPages, args=('List', filterQue, listQue, listinfolock, cookiepool, ThreadFlag)) x.start() thList.append(x) for i in thList: i.join() logger.info('Waiting for all subprocesses done...')
def runner(event_stopper): try: with open('config.json', 'r') as f: config = json.load(f) except Exception as err: logger.exception(err) event_stopper.set() return # start another process to render the graphical representation # of what the camera sees queue = MQueue(5) video_process = Process(target=video_worker, args=(queue, event_stopper)) video_process.start() try: # initialize devices pixy = pixy2.Pixy2I2C(address=0x54) gopigo3 = EasyGoPiGo3() logger.info('initialized pixy2 and gopigo3') # turn on the built-in LED for better visibility pixy.set_lamp(1) logger.info('turn on pixy2 lamp') # set it to follow white lanes pixy.set_mode(pixy2.LINE_MODE_WHITE_LINE) logger.info('set pixy2 line mode on white') # get the number of frames per second fps = pixy.get_fps() period = 1 / fps logger.info('pixy2 running at {} fps'.format(fps)) # camera resolution width, height = pixy.get_resolution() logger.info('pixy2 camera width={} height={}'.format(width, height)) # keep previous measurements for smoothing out the data previous_angle = 0 previous_magnitude = 0 previous_ema_angle = 0 previous_ema_magnitude = 0 frame_index = 0 alfa = config['alfa'] yotta = config['yotta'] # magnitude importance logger.info('alfa (exp. moving average factor) = {:3.2f}'.format(alfa)) logger.info('yotta (magnitude importance) = {:3.2f}'.format(yotta)) # the 2 PIDs c1 = PID(Kp=config['pid1']['Kp'], Ki=config['pid1']['Ki'], Kd=config['pid1']['Kd'], previous_error=0.0, integral_area=0.0) c2 = PID(Kp=config['pid2']['Kp'], Ki=config['pid2']['Ki'], Kd=config['pid2']['Kd'], previous_error=0.0, integral_area=0.0) pid_switch_point = config['pid-switch-point'] logger.info('PID 1 = {}'.format(str(c1))) logger.info('PID 2 = {}'.format(str(c2))) logger.info('PID switchpoint = {}'.format(pid_switch_point)) set_point = 0 max_motor_speed = config['max-motor-speed'] logger.info('max motor speed = {}'.format(max_motor_speed)) while not event_stopper.is_set(): # track each step's time start = time() # get the main features from the pixy2 features = pixy.get_main_features(features=1) frame = None if features: if 'vectors' in features: frame = features['vectors'] else: frame = [] frame_index += 1 else: gopigo3.stop() sleep(period) continue # calculate the heading of the lane and detect how long the lane is lanes, win_size, roi = detect_lanes_from_vector_frame( frame, height, width) angle, magnitude = calculate_direction_and_magnitude( lanes, win_size, previous_angle, previous_magnitude) # run the determined heading and magnitude through an exponential moving average function # this has the effect of smoothing the results and of ignoring the short term noise ema_angle = exponential_moving_average(angle, previous_ema_angle, alfa) ema_magnitude = exponential_moving_average(magnitude, previous_ema_magnitude, alfa) # save the measurements for the next iteration previous_angle = angle previous_magnitude = magnitude previous_ema_angle = ema_angle previous_ema_magnitude = ema_magnitude angle = ema_angle magnitude = ema_magnitude # push the processed information to the graphical renderer (on a separate process) if not queue.full(): queue.put({ 'frame_idx': frame_index, 'frame': frame, 'resolution': (height, width), 'fps': fps, 'angle': angle, 'magnitude': magnitude, 'lanes': lanes, 'win_size': win_size }) # run the pid controller error = angle - set_point c1.integral_area += error c2.integral_area += error pid1 = True if magnitude < pid_switch_point: c = c1 c2.integral_area = 0.0 else: c = c2 c1.integral_area = 0.0 pid1 = False # calculate the correction correction = c.Kp * error + c.Ki * c.integral_area + c.Kd * ( error - c.previous_error) c1.previous_error = error c2.previous_error = error # determine motor speeds left_motor_speed = int(( (1 - yotta) + yotta * magnitude / 100) * max_motor_speed + correction) right_motor_speed = int(( (1 - yotta) + yotta * magnitude / 100) * max_motor_speed - correction) logger.debug( 'using PID {} | angle: {:3d} magnitude: {:3d} | left speed: {:3d} right speed: {:3d}' .format(1 if pid1 else 0, int(angle), int(magnitude), left_motor_speed, right_motor_speed)) # actuate the motors gopigo3.set_motor_dps(gopigo3.MOTOR_LEFT, dps=left_motor_speed) gopigo3.set_motor_dps(gopigo3.MOTOR_RIGHT, dps=right_motor_speed) # make it run at a specific loop frequency # equal to camera's fps end = time() diff = end - start remaining_time = period - diff if remaining_time > 0: sleep(remaining_time) # and turn the built-in LED off when the program ends # and stop the gopigo3 pixy.set_lamp(0) gopigo3.stop() except Exception as e: event_stopper.set() logger.exception(e) finally: video_process.join()