def __init__(self, options): super(Feeder, self).__init__() self.q = queues.Queue(WORKER_COUNT) kwargs = dict(vars(options)) self.builder_gen = find_builders(**kwargs) self.running = True self.stats = collections.defaultdict(int)
def subscribe(source, observer): """ :type observer: Observer :type source: ObservableSource """ assert isinstance( source, ObservableSource ), "source must be ObservableSource but %s" % type(source) assert isinstance( observer, Observer), "observer must be Observer but %s" % type(observer) queue = queues.Queue() bs = BlockingObserver(queue) observer.onSubscribe(bs) source.subscribe(bs) while not bs.isDisposed(): v = None try: v = queue.get(timeout=1) except queues.Empty: continue if v is None: bs.dispose() observer.onComplete() break if bs.isDisposed( ) or v == BlockingObserver.TERMINATED or acceptFull(v, observer): break
def __init__(self, stock_name_list): count = 0 self.stock_queues = queues.Queue(len(stock_name_list), ctx=multiprocessing) for stock_name in stock_name_list: self.stock_queues.put(stock_name) self.workers = []
def multi_process_simulate(self, stock_codes, judge_time, out_dir_name, target_function=None): """ 多进程进行模拟计算,充分发挥多核性能 :param target_function: 目标调用函数 :param out_dir_name: :param stock_codes: 所有的股票编码 :param judge_time: 判定时机的函数 :return: """ cpu_num = multiprocessing.cpu_count() each_cpu_load = len(stock_codes) // cpu_num queue = queues.Queue(20, ctx=multiprocessing) if target_function is None: raise Exception('没有传入模拟函数!!!') final_sum_dict = {} for item in self.sum_field: final_sum_dict[item] = 0 for item in self.max_field: final_sum_dict[item] = 0 for item in self.min_field: final_sum_dict[item] = 0 for i in range(cpu_num): temp_stock_code = [] for j in range(i * each_cpu_load, (i + 1) * each_cpu_load): temp_stock_code.append(stock_codes[j]) if i == cpu_num - 1: for j in range(cpu_num * each_cpu_load, len(stock_codes)): temp_stock_code.append(stock_codes[j]) process_i = multiprocessing.Process(target=target_function, args=(temp_stock_code, judge_time, queue, out_dir_name)) process_i.start() process_i.join() for i in range(cpu_num): temp_dict = queue.get() for item in self.sum_field: final_sum_dict[item] = final_sum_dict[item] + temp_dict[item] for item in self.max_field: final_sum_dict[item] = temp_dict[item] \ if final_sum_dict[item] < temp_dict[item] \ else final_sum_dict[item] for item in self.min_field: final_sum_dict[item] = temp_dict[item] \ if final_sum_dict[item] is None or final_sum_dict[item] > temp_dict[item] \ else final_sum_dict[item] all_field = self.sum_field all_field.extend(self.max_field) all_field.extend(self.min_field) sum_frame = pandas.DataFrame(columns=tuple(all_field)) sum_frame = sum_frame.append(final_sum_dict, ignore_index=True) FileOutput.csv_output(None, sum_frame, "final_sum_info", spe_dir_name=out_dir_name)
def __init__(self, actual, delayError, bufferSize): """ :type actual: Observer """ super(ObservableObserveOn.ObserveOnObserver, self).__init__() self.actual = actual self.delayError = delayError self.bufferSize = bufferSize self.s = None self.queue = queues.Queue(maxsize=self.bufferSize) self.done = False self.error = None self.canceled = False
def __init__(self): print 'Initializing IBDataManager' self._connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) self._channel = self._connection.channel() self._channel.queue_declare(queue='contractDetails') self._channel.queue_declare(queue='requests') self._channel.queue_declare(queue='tickPrice') self._channel.basic_consume(self._on_response, queue='contractDetails', no_ack=True) self._channel.basic_consume(self._on_response, queue='tickPrice', no_ack=True) self._responses = queues.Queue() self._req_id = 0 self._requests = {} self._market_data = {} self._dirty = False
def Queue(*args, **kwargs): return queues.Queue(*args, ctx=mp.get_context('spawn'), **kwargs)
# print('end', li) # 方式一 queues from multiprocessing import Process from multiprocessing import queues import multiprocessing def foo(i, arg): arg.put(i) print('say hi', i, arg.qsize()) if __name__ == '__main__': li = queues.Queue(20, ctx=multiprocessing) for i in range(10): p = Process(target=foo, args=( i, li, )) p.start() # 方式二 Array数据 # from multiprocessing import Process # from multiprocessing import Array # # # def foo(i, arg): # arg[i] = i + 100
img = Image.open(imgpath1).resize((subimgsize,subimgsize)) elif(os.path.exists(imgpath2)): img = Image.open(imgpath2).resize((subimgsize,subimgsize)) else: pass imgqueue.put((img,averrgb(img))) time.sleep(0.01) print('done') #time.sleep(25) os._exit(0) #main begin if __name__ == '__main__': #rgbqueue = queues.Queue(5000,ctx=multiprocessing) imgqueue = queues.Queue(5000,ctx=multiprocessing) ts=time.time() n = 0 CPUS = os.cpu_count() averthread_num = int(img_num/CPUS) #读取图片 for i in range(0,CPUS): p.append(Process(target = loadimg,args = (i*averthread_num+1,(i+1)*averthread_num,imgqueue,))) for i in range(0,CPUS): p[i].start() while (len(imglist)!=img_num):#生产者消费者模型 while not imgqueue.empty(): imglist.append(imgqueue.get())
def Pipe(maxsize=0): return queues.Queue(maxsize, ctx=mp.get_context())
"/usr/local/workerman-thrift/workerman-thrift/ToWeb.thrift", module_name="gui_client_thrift") #endpoint deamon thrift rpc endpoint_service = thriftpy.load( "/home/wangdl/nodedeamon/endpoint_service.thrift", module_name="endpoint_service_thrift") if len(sys.argv) < 2: print("Usage:[work]") sys.exit() else: work = sys.argv[1] import multiprocessing from multiprocessing import queues e_que = queues.Queue(5, ctx=multiprocessing) def get_host_ip(): try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('8.8.8.8', 80)) ip = s.getsockname()[0] finally: s.close() return ip ip = get_host_ip() port = 0
def __init__(self): super(Logger, self).__init__() self.q = queues.Queue() self.stats = collections.defaultdict(int) self.running = True
# -*- coding:utf-8 -*- # @Author: wzy # @Time: 2021/5/15 # import os from multiprocessing import queues, Lock from flask import Flask, session, request, copy_current_request_context from flask_socketio import SocketIO, emit, join_room, leave_room, close_room, rooms, disconnect app = Flask(import_name=__name__) app.config['SECRET_KEY'] = os.environ.get("SECRET_KEY", "PyChromeExt") socket_io = SocketIO(app, cors_allowed_origins=os.environ.get( "CORS_ALLOWED_ORIGINS", "*")) cmd_channel = queues.Queue() thread = None thread_lock = Lock() @socket_io.event def ping(): emit('pong') def background_cmd_thread(): # 不断向客户端发送命令 count = 0 while True: socket_io.sleep(10) count += 1
def open(self): if not self._opened: self._events_queue = mqueues.Queue() self._controller_process = multiprocessing.Process(target=self._controller, args=(self._events_queue, )) self._controller_process.start() self._opened = True
def __init__(self): super(BlockingMultiObserver, self).__init__(count=1) self.d = None self.value = queues.Queue() self.error = queues.Queue()
def __init__(self, parent, callback=None): wx.Panel.__init__(self, parent, -1) print('mkswifi loaded') sys.setrecursionlimit(1000000) self.callback = callback self.sceneview = self.GetParent().GetParent().GetParent().GetParent().GetParent().scene self.port = 8080 self._socket = None self._address = None self._isConnect = False self._isPrinting = False self._isPause = False self._isSending = False self._ischanging = False self._printing_filename = "" self._printing_progress = 0 self._printing_time = 0 self._sdFileList = False self.sdFiles = [] self._ishotBedpreHeat = False self._isextruder1preHeat = False self._isextruder2preHeat = False self._ishotBedR = False self._isextruder1R = False self._isextruder2R = False self.extruderCount = int(profile.getMachineSetting('extruder_amount')) self.bagSizer = wx.GridBagSizer(0, 0) # ConnectToPrinter self.connectTitle = wx.StaticText(self, -1, _("Connect printer")) self.connectTitle.SetFont(wx.Font(wx.SystemSettings.GetFont(wx.SYS_ANSI_VAR_FONT).GetPointSize(), wx.FONTFAMILY_DEFAULT, wx.NORMAL, wx.FONTWEIGHT_BOLD)) self.ipLabel = wx.lib.stattext.GenStaticText(self, -1, _("IP Address")) # self.label.Bind(wx.EVT_ENTER_WINDOW, self.OnMouseEnter) # self.label.Bind(wx.EVT_LEAVE_WINDOW, self.OnMouseExit) ip = profile.getMachineSetting('ip_address') self.ipCtrl = wx.TextCtrl(self, -1, ip) self.ipCtrl.Bind(wx.EVT_TEXT, self.OnIpAddressChange) # self.ctrl.Bind(wx.EVT_ENTER_WINDOW, self.OnMouseEnter) # self.ctrl.Bind(wx.EVT_LEAVE_WINDOW, self.OnMouseExit) self.connectBtn = wx.Button(self, -1, _("Connect")) self.connectBtn.Bind(wx.EVT_BUTTON, lambda e: self.controlPrinter()) # self.connectBtn.Bind(wx.EVT_BUTTON, lambda e: self.sendFile()) # self.unConnectBtn = wx.Button(self, -1, _("UNConnect")) # self.unConnectBtn.Bind(wx.EVT_BUTTON, lambda e: self.controlPrinter()) self.sendJobBtn = wx.Button(self, -1, _("Send Job")) self.sendJobBtn.Bind(wx.EVT_BUTTON, lambda e: self.sendFile()) self.bagSizer.Add(self.connectTitle, (0, 0), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(wx.StaticLine(self), (1, 0), (1, 4), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(self.ipLabel, (2, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(self.ipCtrl, (2, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.EXPAND, border=10) self.bagSizer.Add(self.connectBtn, (2, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT , border=10) # self.bagSizer.Add(self.unConnectBtn, (3, 1), flag=wx.ALL, border=5) self.bagSizer.Add(self.sendJobBtn, (3, 0), (1, 3), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP |wx.EXPAND, border=10) # ControlPrinter self.controlTitle = wx.StaticText(self, -1, _("Control printer")) self.controlTitle.SetFont(wx.Font(wx.SystemSettings.GetFont(wx.SYS_ANSI_VAR_FONT).GetPointSize(), wx.FONTFAMILY_DEFAULT, wx.NORMAL, wx.FONTWEIGHT_BOLD)) self.hotBedLabel = wx.lib.stattext.GenStaticText(self, -1, _("Hot Bed")) self.hotBedCurrentLabel = wx.lib.stattext.GenStaticText(self, -1, _("Current temp (deg C)")) self.hotBedCurrenttext = wx.StaticText(self, -1, _("0")) self.hotBedTargetLabel = wx.lib.stattext.GenStaticText(self, -1, _("Target temp (deg C)")) self.hotBedTargetCtrl = wx.TextCtrl(self, -1, "0") self.hotBedTargetCtrl.Bind(wx.EVT_SET_FOCUS, self.setFocusBed) self.hotBedTargetCtrl.Bind(wx.EVT_KILL_FOCUS, self.killFocusBed) self.hotBedpreHeatBtn = wx.Button(self, -1, _("PreHeat")) self.hotBedpreHeatBtn.Bind(wx.EVT_BUTTON, lambda e: self.hotBedpreHeat()) self.bagSizer.Add(self.controlTitle, (4, 0), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(wx.StaticLine(self), (5, 0), (1, 4), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(self.hotBedLabel, (6, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(self.hotBedCurrentLabel, (7, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.hotBedCurrenttext, (7, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.hotBedTargetLabel, (8, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.hotBedTargetCtrl, (8, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.hotBedpreHeatBtn, (8, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP, border=10) # self.hotBedTargetCtrl.SetEditable(False) self.extrLabel1 = wx.lib.stattext.GenStaticText(self, -1, _("Extruder 1")) self.extrCurrentLabel1 = wx.lib.stattext.GenStaticText(self, -1, _("Current temp (deg C)")) self.extrCurrenttext1 = wx.StaticText(self, -1, _("0")) self.extrTargetLabel1 = wx.lib.stattext.GenStaticText(self, -1, _("Target temp (deg C)")) self.extrTargetCtrl1 = wx.TextCtrl(self, -1, "0") self.extrTargetCtrl1.Bind(wx.EVT_SET_FOCUS, self.setFocusE1) self.extrTargetCtrl1.Bind(wx.EVT_KILL_FOCUS, self.killFocusE1) self.preHeatBtn1 = wx.Button(self, -1, _("PreHeat")) self.preHeatBtn1.Bind(wx.EVT_BUTTON, lambda e: self.extruder1PreHeat()) self.bagSizer.Add(self.extrLabel1, (9, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(self.extrCurrentLabel1, (10, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.extrCurrenttext1, (10, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.extrTargetLabel1, (11, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.extrTargetCtrl1, (11, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.preHeatBtn1, (11, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP, border=10) tag = 0 if self.extruderCount > 1: tag = 3 self.extrLabel2 = wx.lib.stattext.GenStaticText(self, -1, _("Extruder 2")) self.extrCurrentLabel2 = wx.lib.stattext.GenStaticText(self, -1, _("Current temp (deg C)")) self.extrCurrenttext2 = wx.StaticText(self, -1, _("0")) self.extrTargetLabel2 = wx.lib.stattext.GenStaticText(self, -1, _("Target temp (deg C)")) self.extrTargetCtrl2 = wx.TextCtrl(self, -1, "0") self.extrTargetCtrl2.Bind(wx.EVT_SET_FOCUS, self.setFocusE2) self.extrTargetCtrl2.Bind(wx.EVT_KILL_FOCUS, self.killFocusE2) self.preHeatBtn2 = wx.Button(self, -1, _("PreHeat")) self.preHeatBtn2.Bind(wx.EVT_BUTTON, lambda e: self.extruder2PreHeat()) self.bagSizer.Add(self.extrLabel2, (12, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10) self.bagSizer.Add(self.extrCurrentLabel2, (13, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.extrCurrenttext2, (13, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.extrTargetLabel2, (14, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.extrTargetCtrl2, (14, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.preHeatBtn2, (14, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP, border=10) self.printingNameLabel = wx.lib.stattext.GenStaticText(self, -1, _("Printing Job")) self.printingName = wx.StaticText(self, -1, "------") self.printingTimeLabel = wx.lib.stattext.GenStaticText(self, -1, _("Printing Time")) self.printingTime = wx.StaticText(self, -1, "00:00:00") self.totalTimeLabel = wx.lib.stattext.GenStaticText(self, -1, _("Print progress")) self.totalTime = wx.StaticText(self, -1, "0%") self.bagSizer.Add(self.printingNameLabel, (12 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.printingName, (12 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.printingTimeLabel, (13 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.printingTime, (13 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.totalTimeLabel, (14 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.totalTime, (14 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.coolDownBtn = wx.Button(self, -1, _("Cool Down")) self.coolDownBtn.Bind(wx.EVT_BUTTON, lambda e: self.coolDown()) self.pauseBtn = wx.Button(self, -1, _("Pause")) # Resume self.pauseBtn.Bind(wx.EVT_BUTTON, lambda e: self.pauseOrResume()) self.stopBtn = wx.Button(self, -1, _("Stop")) self.stopBtn.Bind(wx.EVT_BUTTON, lambda e: self.stopPrint()) self.bagSizer.Add(self.coolDownBtn, (15 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.pauseBtn, (15 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.stopBtn, (15 + tag, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.delSDFileBtn = wx.Button(self, -1, _("Delete SD File")) self.delSDFileBtn.Bind(wx.EVT_BUTTON, lambda e: self.delSDFile()) self.printSDFileBtn = wx.Button(self, -1, _("Print SD File")) self.printSDFileBtn.Bind(wx.EVT_BUTTON, lambda e: self.printSDFile()) self.sendFileBtn = wx.Button(self, -1, _("Send File")) self.sendFileBtn.Bind(wx.EVT_BUTTON, lambda e: self.showSelectFile()) self.bagSizer.Add(self.delSDFileBtn, (16 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.printSDFileBtn, (16 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.bagSizer.Add(self.sendFileBtn, (16 + tag, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10) self.SetSizer(self.bagSizer) self._update_timer = None self._recv_timer = None self._command_queue = queues.Queue() self.updateConnectUI() self.uploadProcessDlg = None
data = store_q.get() if data == 'end': print('存储进程接受通知然后结束!') output.ouput_end(output.filepath) return output.store_data(data) else: time.sleep(0.1) pass if __name__ == '__main__': #初始化4个队列 url_q = queues.Queue() result_q = queues.Queue() store_q = queues.Queue() conn_q = queues.Queue() #创建分布式管理器 node = NodeManager() manager = node.start_Manager(url_q, result_q) #创建URL管理进程、 数据提取进程和数据存储进程 url_manager_proc = process(target=node.url_manager_proc, args=( url_q, conn_q, 'http://baike.baidu.com/view/284853.htm', )) result_solve_proc = process(target=node.result_solve_proc, args=(
data = storeQueue.get() if data == "END": print "存储进程结束" data_save.output_data() return data_save.store_data(data) else: time.sleep(0.1) except BaseException, e: time.sleep(0.1) if __name__ == '__main__': # url管理器把url给爬虫节点的通道 urlQueue = queues.Queue() #print urlQueue # 爬虫节点把数据返回解析通道 resultQueue = queues.Queue() # 数据解析通道 parseQueue = queues.Queue() # 存储数据通道 storeQueue = queues.Queue() server = SpiderServer() server_manager = server.start_server(urlQueue, resultQueue) url_manager_proc = Process( target=server.url_manager_proc, args=('http://e.pingan.com/pa18shoplife/category/list.jsp', urlQueue, parseQueue)) #print urlQueue result_proc = Process(target=server.result_proc,
def p_write(q): print('%s start write:') % os.getpid() cont = ['A', 'B', 'C', 'D'] for i in cont: print("%s put %s" % (os.getpid(), i)) q.put(i) time.sleep(random.random() * 3) def p_read(q): print('%s start read:') % os.getpid() while True: cont = q.get(True) print('%s get %s' % (os.getpid(), cont)) if __name__ == "__main__": print('parent %s start work.' % os.getpid()) q = queues.Queue() write = Process(target=p_write, args=(q, )) read = Process(target=p_read, args=(q, )) write.start() read.start() write.join() read.terminate() print('game over')
@software:PyCharm @file:my_process.py @time(UTC+8):16/9/18-14:28 ''' import time from multiprocessing import Process from multiprocessing import queues import multiprocessing def func(i, q): time.sleep(1) q.put(i) # print(q.qsize()) # 这里报错,不能使用qsize() 这个方法,搞笑 q = queues.Queue(20, ctx=multiprocessing) # use queues.Queue() share data with sub process for i in range(10): p = Process(target=func, args=(i, q, )) # p.daemon = True p.start() p.join() # wait all sub process end for i in range(10): print(q.get()) print('end of main process ---------')