Ejemplo n.º 1
0
 def __init__(self, options):
     super(Feeder, self).__init__()
     self.q = queues.Queue(WORKER_COUNT)
     kwargs = dict(vars(options))
     self.builder_gen = find_builders(**kwargs)
     self.running = True
     self.stats = collections.defaultdict(int)
Ejemplo n.º 2
0
    def subscribe(source, observer):
        """

        :type observer: Observer
        :type source: ObservableSource
        """
        assert isinstance(
            source, ObservableSource
        ), "source must be ObservableSource but %s" % type(source)
        assert isinstance(
            observer,
            Observer), "observer must be Observer but %s" % type(observer)

        queue = queues.Queue()

        bs = BlockingObserver(queue)
        observer.onSubscribe(bs)

        source.subscribe(bs)

        while not bs.isDisposed():
            v = None
            try:
                v = queue.get(timeout=1)
            except queues.Empty:
                continue

            if v is None:
                bs.dispose()
                observer.onComplete()
                break

            if bs.isDisposed(
            ) or v == BlockingObserver.TERMINATED or acceptFull(v, observer):
                break
Ejemplo n.º 3
0
 def __init__(self, stock_name_list):
     count = 0
     self.stock_queues = queues.Queue(len(stock_name_list),
                                      ctx=multiprocessing)
     for stock_name in stock_name_list:
         self.stock_queues.put(stock_name)
     self.workers = []
Ejemplo n.º 4
0
    def multi_process_simulate(self, stock_codes, judge_time, out_dir_name, target_function=None):
        """
        多进程进行模拟计算,充分发挥多核性能
        :param target_function: 目标调用函数
        :param out_dir_name:
        :param stock_codes: 所有的股票编码
        :param judge_time: 判定时机的函数
        :return:
        """
        cpu_num = multiprocessing.cpu_count()
        each_cpu_load = len(stock_codes) // cpu_num
        queue = queues.Queue(20, ctx=multiprocessing)

        if target_function is None:
            raise Exception('没有传入模拟函数!!!')

        final_sum_dict = {}
        for item in self.sum_field:
            final_sum_dict[item] = 0
        for item in self.max_field:
            final_sum_dict[item] = 0
        for item in self.min_field:
            final_sum_dict[item] = 0
        for i in range(cpu_num):
            temp_stock_code = []
            for j in range(i * each_cpu_load, (i + 1) * each_cpu_load):
                temp_stock_code.append(stock_codes[j])

            if i == cpu_num - 1:
                for j in range(cpu_num * each_cpu_load, len(stock_codes)):
                    temp_stock_code.append(stock_codes[j])
            process_i = multiprocessing.Process(target=target_function, args=(temp_stock_code, judge_time, queue,
                                                                                       out_dir_name))
            process_i.start()
            process_i.join()
        for i in range(cpu_num):
            temp_dict = queue.get()
            for item in self.sum_field:
                final_sum_dict[item] = final_sum_dict[item] + temp_dict[item]
            for item in self.max_field:
                final_sum_dict[item] = temp_dict[item] \
                    if final_sum_dict[item] < temp_dict[item] \
                    else final_sum_dict[item]
            for item in self.min_field:
                final_sum_dict[item] = temp_dict[item] \
                    if final_sum_dict[item] is None or final_sum_dict[item] > temp_dict[item] \
                    else final_sum_dict[item]
        all_field = self.sum_field
        all_field.extend(self.max_field)
        all_field.extend(self.min_field)
        sum_frame = pandas.DataFrame(columns=tuple(all_field))
        sum_frame = sum_frame.append(final_sum_dict, ignore_index=True)
        FileOutput.csv_output(None, sum_frame, "final_sum_info", spe_dir_name=out_dir_name)
Ejemplo n.º 5
0
        def __init__(self, actual, delayError, bufferSize):
            """

            :type actual: Observer
            """
            super(ObservableObserveOn.ObserveOnObserver, self).__init__()

            self.actual = actual
            self.delayError = delayError
            self.bufferSize = bufferSize
            self.s = None
            self.queue = queues.Queue(maxsize=self.bufferSize)
            self.done = False
            self.error = None
            self.canceled = False
Ejemplo n.º 6
0
 def __init__(self):
     print 'Initializing IBDataManager'
     self._connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
     self._channel = self._connection.channel()
     self._channel.queue_declare(queue='contractDetails')
     self._channel.queue_declare(queue='requests')
     self._channel.queue_declare(queue='tickPrice')
     
     self._channel.basic_consume(self._on_response, queue='contractDetails', no_ack=True)
     self._channel.basic_consume(self._on_response, queue='tickPrice', no_ack=True)
     
     self._responses = queues.Queue()
     
     self._req_id = 0
     
     self._requests = {}
     
     self._market_data = {}
     
     self._dirty = False
Ejemplo n.º 7
0
 def Queue(*args, **kwargs):
     return queues.Queue(*args, ctx=mp.get_context('spawn'), **kwargs)
Ejemplo n.º 8
0
#     print('end', li)

# 方式一 queues

from multiprocessing import Process
from multiprocessing import queues
import multiprocessing


def foo(i, arg):
    arg.put(i)
    print('say hi', i, arg.qsize())


if __name__ == '__main__':
    li = queues.Queue(20, ctx=multiprocessing)
    for i in range(10):
        p = Process(target=foo, args=(
            i,
            li,
        ))
        p.start()

# 方式二 Array数据

# from multiprocessing import Process
# from multiprocessing import Array
#
#
# def foo(i, arg):
#     arg[i] = i + 100
Ejemplo n.º 9
0
			img = Image.open(imgpath1).resize((subimgsize,subimgsize))
		elif(os.path.exists(imgpath2)):
			img = Image.open(imgpath2).resize((subimgsize,subimgsize))
		else:
			pass
		imgqueue.put((img,averrgb(img)))
		time.sleep(0.01)
	print('done')
	#time.sleep(25)
	os._exit(0)


#main begin
if __name__ == '__main__':
	#rgbqueue = queues.Queue(5000,ctx=multiprocessing)
	imgqueue = queues.Queue(5000,ctx=multiprocessing)
	
	ts=time.time()
	n = 0
	
	CPUS = os.cpu_count()
	averthread_num = int(img_num/CPUS)
	#读取图片
	for i in range(0,CPUS):
		p.append(Process(target = loadimg,args = (i*averthread_num+1,(i+1)*averthread_num,imgqueue,)))
	for i in range(0,CPUS):
		p[i].start()

	while (len(imglist)!=img_num):#生产者消费者模型
		while not imgqueue.empty():
			imglist.append(imgqueue.get())
Ejemplo n.º 10
0
def Pipe(maxsize=0):
    return queues.Queue(maxsize, ctx=mp.get_context())
Ejemplo n.º 11
0
    "/usr/local/workerman-thrift/workerman-thrift/ToWeb.thrift",
    module_name="gui_client_thrift")
#endpoint deamon  thrift rpc
endpoint_service = thriftpy.load(
    "/home/wangdl/nodedeamon/endpoint_service.thrift",
    module_name="endpoint_service_thrift")

if len(sys.argv) < 2:
    print("Usage:[work]")
    sys.exit()
else:
    work = sys.argv[1]

import multiprocessing
from multiprocessing import queues
e_que = queues.Queue(5, ctx=multiprocessing)


def get_host_ip():
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 80))
        ip = s.getsockname()[0]
    finally:
        s.close()

    return ip


ip = get_host_ip()
port = 0
Ejemplo n.º 12
0
 def __init__(self):
     super(Logger, self).__init__()
     self.q = queues.Queue()
     self.stats = collections.defaultdict(int)
     self.running = True
Ejemplo n.º 13
0
# -*- coding:utf-8 -*-
# @Author: wzy
# @Time: 2021/5/15
#
import os
from multiprocessing import queues, Lock

from flask import Flask, session, request, copy_current_request_context
from flask_socketio import SocketIO, emit, join_room, leave_room, close_room, rooms, disconnect

app = Flask(import_name=__name__)
app.config['SECRET_KEY'] = os.environ.get("SECRET_KEY", "PyChromeExt")
socket_io = SocketIO(app,
                     cors_allowed_origins=os.environ.get(
                         "CORS_ALLOWED_ORIGINS", "*"))
cmd_channel = queues.Queue()
thread = None
thread_lock = Lock()


@socket_io.event
def ping():
    emit('pong')


def background_cmd_thread():
    # 不断向客户端发送命令
    count = 0
    while True:
        socket_io.sleep(10)
        count += 1
Ejemplo n.º 14
0
 def open(self):
     if not self._opened:
         self._events_queue = mqueues.Queue()
         self._controller_process = multiprocessing.Process(target=self._controller, args=(self._events_queue, ))
         self._controller_process.start()
         self._opened = True
Ejemplo n.º 15
0
 def __init__(self):
     super(BlockingMultiObserver, self).__init__(count=1)
     self.d = None
     self.value = queues.Queue()
     self.error = queues.Queue()
Ejemplo n.º 16
0
    def __init__(self, parent, callback=None):
        wx.Panel.__init__(self, parent, -1)
        print('mkswifi loaded')
        sys.setrecursionlimit(1000000)
        self.callback = callback

        self.sceneview = self.GetParent().GetParent().GetParent().GetParent().GetParent().scene

        self.port = 8080
        self._socket = None
        self._address = None
        self._isConnect = False
        self._isPrinting = False
        self._isPause = False
        self._isSending = False
        self._ischanging = False
        self._printing_filename = ""
        self._printing_progress = 0
        self._printing_time = 0
        self._sdFileList = False
        self.sdFiles = []
        self._ishotBedpreHeat = False
        self._isextruder1preHeat = False
        self._isextruder2preHeat = False
        self._ishotBedR = False
        self._isextruder1R = False
        self._isextruder2R = False

        self.extruderCount = int(profile.getMachineSetting('extruder_amount'))
        self.bagSizer = wx.GridBagSizer(0, 0)
        # ConnectToPrinter
        self.connectTitle = wx.StaticText(self, -1, _("Connect printer"))
        self.connectTitle.SetFont(wx.Font(wx.SystemSettings.GetFont(wx.SYS_ANSI_VAR_FONT).GetPointSize(), wx.FONTFAMILY_DEFAULT, wx.NORMAL, wx.FONTWEIGHT_BOLD))
        self.ipLabel = wx.lib.stattext.GenStaticText(self, -1, _("IP Address"))
        # self.label.Bind(wx.EVT_ENTER_WINDOW, self.OnMouseEnter)
        # self.label.Bind(wx.EVT_LEAVE_WINDOW, self.OnMouseExit)
        ip = profile.getMachineSetting('ip_address')
        self.ipCtrl = wx.TextCtrl(self, -1, ip)
        self.ipCtrl.Bind(wx.EVT_TEXT, self.OnIpAddressChange)
        # self.ctrl.Bind(wx.EVT_ENTER_WINDOW, self.OnMouseEnter)
        # self.ctrl.Bind(wx.EVT_LEAVE_WINDOW, self.OnMouseExit)
        self.connectBtn = wx.Button(self, -1, _("Connect"))
        self.connectBtn.Bind(wx.EVT_BUTTON, lambda e: self.controlPrinter())
        # self.connectBtn.Bind(wx.EVT_BUTTON, lambda e: self.sendFile())
        # self.unConnectBtn = wx.Button(self, -1, _("UNConnect"))
        # self.unConnectBtn.Bind(wx.EVT_BUTTON, lambda e: self.controlPrinter())
        self.sendJobBtn = wx.Button(self, -1, _("Send Job"))
        self.sendJobBtn.Bind(wx.EVT_BUTTON, lambda e: self.sendFile())
        self.bagSizer.Add(self.connectTitle, (0, 0), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10)
        self.bagSizer.Add(wx.StaticLine(self), (1, 0), (1, 4), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10)
        self.bagSizer.Add(self.ipLabel, (2, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10)
        self.bagSizer.Add(self.ipCtrl, (2, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.EXPAND, border=10)
        self.bagSizer.Add(self.connectBtn, (2, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT , border=10)
        # self.bagSizer.Add(self.unConnectBtn, (3, 1), flag=wx.ALL, border=5)
        self.bagSizer.Add(self.sendJobBtn, (3, 0), (1, 3), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP |wx.EXPAND, border=10)

        # ControlPrinter
        self.controlTitle = wx.StaticText(self, -1, _("Control printer"))
        self.controlTitle.SetFont(wx.Font(wx.SystemSettings.GetFont(wx.SYS_ANSI_VAR_FONT).GetPointSize(), wx.FONTFAMILY_DEFAULT, wx.NORMAL, wx.FONTWEIGHT_BOLD))
        self.hotBedLabel = wx.lib.stattext.GenStaticText(self, -1, _("Hot Bed"))
        self.hotBedCurrentLabel = wx.lib.stattext.GenStaticText(self, -1, _("Current temp (deg C)"))
        self.hotBedCurrenttext = wx.StaticText(self, -1, _("0"))
        self.hotBedTargetLabel = wx.lib.stattext.GenStaticText(self, -1, _("Target temp (deg C)"))
        self.hotBedTargetCtrl = wx.TextCtrl(self, -1, "0")
        self.hotBedTargetCtrl.Bind(wx.EVT_SET_FOCUS, self.setFocusBed)
        self.hotBedTargetCtrl.Bind(wx.EVT_KILL_FOCUS, self.killFocusBed)
        self.hotBedpreHeatBtn = wx.Button(self, -1, _("PreHeat"))
        self.hotBedpreHeatBtn.Bind(wx.EVT_BUTTON, lambda e: self.hotBedpreHeat())
        self.bagSizer.Add(self.controlTitle, (4, 0), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10)
        self.bagSizer.Add(wx.StaticLine(self), (5, 0), (1, 4), flag=wx.EXPAND | wx.TOP | wx.LEFT, border=10)
        self.bagSizer.Add(self.hotBedLabel, (6, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10)
        self.bagSizer.Add(self.hotBedCurrentLabel, (7, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.hotBedCurrenttext, (7, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.hotBedTargetLabel, (8, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.hotBedTargetCtrl, (8, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.hotBedpreHeatBtn, (8, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP, border=10)

        # self.hotBedTargetCtrl.SetEditable(False)

        self.extrLabel1 = wx.lib.stattext.GenStaticText(self, -1, _("Extruder 1"))
        self.extrCurrentLabel1 = wx.lib.stattext.GenStaticText(self, -1, _("Current temp (deg C)"))
        self.extrCurrenttext1 = wx.StaticText(self, -1, _("0"))
        self.extrTargetLabel1 = wx.lib.stattext.GenStaticText(self, -1, _("Target temp (deg C)"))
        self.extrTargetCtrl1 = wx.TextCtrl(self, -1, "0")
        self.extrTargetCtrl1.Bind(wx.EVT_SET_FOCUS, self.setFocusE1)
        self.extrTargetCtrl1.Bind(wx.EVT_KILL_FOCUS, self.killFocusE1)
        self.preHeatBtn1 = wx.Button(self, -1, _("PreHeat"))
        self.preHeatBtn1.Bind(wx.EVT_BUTTON, lambda e: self.extruder1PreHeat())
        self.bagSizer.Add(self.extrLabel1, (9, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10)
        self.bagSizer.Add(self.extrCurrentLabel1, (10, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.extrCurrenttext1, (10, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.extrTargetLabel1, (11, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.extrTargetCtrl1, (11, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.preHeatBtn1, (11, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP, border=10)
        tag = 0
        if self.extruderCount > 1:
            tag = 3
            self.extrLabel2 = wx.lib.stattext.GenStaticText(self, -1, _("Extruder 2"))
            self.extrCurrentLabel2 = wx.lib.stattext.GenStaticText(self, -1, _("Current temp (deg C)"))
            self.extrCurrenttext2 = wx.StaticText(self, -1, _("0"))
            self.extrTargetLabel2 = wx.lib.stattext.GenStaticText(self, -1, _("Target temp (deg C)"))
            self.extrTargetCtrl2 = wx.TextCtrl(self, -1, "0")
            self.extrTargetCtrl2.Bind(wx.EVT_SET_FOCUS, self.setFocusE2)
            self.extrTargetCtrl2.Bind(wx.EVT_KILL_FOCUS, self.killFocusE2)
            self.preHeatBtn2 = wx.Button(self, -1, _("PreHeat"))
            self.preHeatBtn2.Bind(wx.EVT_BUTTON, lambda e: self.extruder2PreHeat())
            self.bagSizer.Add(self.extrLabel2, (12, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.LEFT, border=10)
            self.bagSizer.Add(self.extrCurrentLabel2, (13, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                              border=10)
            self.bagSizer.Add(self.extrCurrenttext2, (13, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                              border=10)
            self.bagSizer.Add(self.extrTargetLabel2, (14, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                              border=10)
            self.bagSizer.Add(self.extrTargetCtrl2, (14, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                              border=10)
            self.bagSizer.Add(self.preHeatBtn2, (14, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP, border=10)

        self.printingNameLabel = wx.lib.stattext.GenStaticText(self, -1, _("Printing Job"))
        self.printingName = wx.StaticText(self, -1, "------")
        self.printingTimeLabel = wx.lib.stattext.GenStaticText(self, -1, _("Printing Time"))
        self.printingTime = wx.StaticText(self, -1, "00:00:00")
        self.totalTimeLabel = wx.lib.stattext.GenStaticText(self, -1, _("Print progress"))
        self.totalTime = wx.StaticText(self, -1, "0%")
        self.bagSizer.Add(self.printingNameLabel, (12 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)
        self.bagSizer.Add(self.printingName, (12 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)
        self.bagSizer.Add(self.printingTimeLabel, (13 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)
        self.bagSizer.Add(self.printingTime, (13 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)
        self.bagSizer.Add(self.totalTimeLabel, (14 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)
        self.bagSizer.Add(self.totalTime, (14 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)

        self.coolDownBtn = wx.Button(self, -1, _("Cool Down"))
        self.coolDownBtn.Bind(wx.EVT_BUTTON, lambda e: self.coolDown())
        self.pauseBtn = wx.Button(self, -1, _("Pause"))  # Resume
        self.pauseBtn.Bind(wx.EVT_BUTTON, lambda e: self.pauseOrResume())
        self.stopBtn = wx.Button(self, -1, _("Stop"))
        self.stopBtn.Bind(wx.EVT_BUTTON, lambda e: self.stopPrint())
        self.bagSizer.Add(self.coolDownBtn, (15 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)
        self.bagSizer.Add(self.pauseBtn, (15 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)
        self.bagSizer.Add(self.stopBtn, (15 + tag, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT,
                          border=10)

        self.delSDFileBtn = wx.Button(self, -1, _("Delete SD File"))
        self.delSDFileBtn.Bind(wx.EVT_BUTTON, lambda e: self.delSDFile())
        self.printSDFileBtn = wx.Button(self, -1, _("Print SD File"))
        self.printSDFileBtn.Bind(wx.EVT_BUTTON, lambda e: self.printSDFile())
        self.sendFileBtn = wx.Button(self, -1, _("Send File"))
        self.sendFileBtn.Bind(wx.EVT_BUTTON, lambda e: self.showSelectFile())
        self.bagSizer.Add(self.delSDFileBtn, (16 + tag, 0), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.printSDFileBtn, (16 + tag, 1), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)
        self.bagSizer.Add(self.sendFileBtn, (16 + tag, 2), flag=wx.ALIGN_CENTER_VERTICAL | wx.TOP | wx.RIGHT, border=10)

        self.SetSizer(self.bagSizer)
        self._update_timer = None
        self._recv_timer = None
        self._command_queue = queues.Queue()

        self.updateConnectUI()
        self.uploadProcessDlg = None
Ejemplo n.º 17
0
                data = store_q.get()
                if data == 'end':
                    print('存储进程接受通知然后结束!')
                    output.ouput_end(output.filepath)

                    return
                output.store_data(data)
            else:
                time.sleep(0.1)
        pass


if __name__ == '__main__':
    #初始化4个队列

    url_q = queues.Queue()
    result_q = queues.Queue()
    store_q = queues.Queue()
    conn_q = queues.Queue()
    #创建分布式管理器
    node = NodeManager()
    manager = node.start_Manager(url_q, result_q)
    #创建URL管理进程、 数据提取进程和数据存储进程
    url_manager_proc = process(target=node.url_manager_proc,
                               args=(
                                   url_q,
                                   conn_q,
                                   'http://baike.baidu.com/view/284853.htm',
                               ))
    result_solve_proc = process(target=node.result_solve_proc,
                                args=(
Ejemplo n.º 18
0
                    data = storeQueue.get()
                    if data == "END":
                        print "存储进程结束"
                        data_save.output_data()

                        return
                    data_save.store_data(data)
                else:
                    time.sleep(0.1)
            except BaseException, e:
                time.sleep(0.1)


if __name__ == '__main__':
    # url管理器把url给爬虫节点的通道
    urlQueue = queues.Queue()
    #print urlQueue
    # 爬虫节点把数据返回解析通道
    resultQueue = queues.Queue()
    # 数据解析通道
    parseQueue = queues.Queue()
    # 存储数据通道
    storeQueue = queues.Queue()
    server = SpiderServer()
    server_manager = server.start_server(urlQueue, resultQueue)
    url_manager_proc = Process(
        target=server.url_manager_proc,
        args=('http://e.pingan.com/pa18shoplife/category/list.jsp', urlQueue,
              parseQueue))
    #print urlQueue
    result_proc = Process(target=server.result_proc,
Ejemplo n.º 19
0
def p_write(q):
    print('%s start write:') % os.getpid()
    cont = ['A', 'B', 'C', 'D']
    for i in cont:
        print("%s put %s" % (os.getpid(), i))
        q.put(i)
        time.sleep(random.random() * 3)


def p_read(q):
    print('%s start read:') % os.getpid()
    while True:
        cont = q.get(True)
        print('%s get %s' % (os.getpid(), cont))


if __name__ == "__main__":
    print('parent %s start work.' % os.getpid())

    q = queues.Queue()
    write = Process(target=p_write, args=(q, ))
    read = Process(target=p_read, args=(q, ))

    write.start()
    read.start()

    write.join()
    read.terminate()
    print('game over')
Ejemplo n.º 20
0
@software:PyCharm
@file:my_process.py
@time(UTC+8):16/9/18-14:28
'''

import time
from multiprocessing import Process
from multiprocessing import queues
import multiprocessing


def func(i, q):
    time.sleep(1)
    q.put(i)
    # print(q.qsize())  # 这里报错,不能使用qsize() 这个方法,搞笑


q = queues.Queue(20, ctx=multiprocessing)  # use queues.Queue() share data with sub process


for i in range(10):
    p = Process(target=func, args=(i, q, ))
    # p.daemon = True
    p.start()


p.join()  # wait all sub process end
for i in range(10):
    print(q.get())
print('end of main process ---------')