-
Notifications
You must be signed in to change notification settings - Fork 0
/
BroadFirst.py
81 lines (72 loc) · 2.65 KB
/
BroadFirst.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Created on 2014年1月20日
@author: peng
@contact: myme5261314@gmail.com
'''
from multiprocessing import Process,Queue
import multiprocessing
from BroadFirstConfig import BroadFirstConfig
from BroadFirstDownloader import BroadFirstDownloader
import time
import logging
class BroadFirstGenerator(Process):
def __init__(self, config_path):
Process.__init__(self)
self.config = BroadFirstConfig(config_path)
self.data = Queue()
self.x = self.config.x
self.y = self.config.y
self.stop_produce = False
self.stop_consume = False
self.total = self.config.getTotalTileNum()
self.count = 0
def run(self):
init = True
while True and not (self.stop_produce):
dl = self.data.qsize()
if dl <= self.config.MAX_QUEUE/2 and not self.stop_produce:
for _ in range(self.config.MAX_QUEUE/4):
if not self.getNextXY():
self.stop_produce = True
break
else:
self.data.put((self.x, self.y, self.config.min_z))
if init:
init = False
num_consumers = multiprocessing.cpu_count() * 2
# num_consumers = 2
consumers = [BroadFirstDownloader(self.data, self.config) for _ in range(num_consumers)]
for consumer in consumers:
consumer.start()
if self.data.empty():
break
time.sleep(60)
def getNextXY(self):
# logger = multiprocessing.get_logger()
if self.config.debug:
if self.count >= self.config.debugTryTimes:
self.x = None
self.y = None
return False
self.count = self.count + 1
if self.count % 1 == 0 and self.x != None and self.y != None:
# process = self.config.getProcess(self.x, self.y, self.data.qsize())
# print '%s: Proceed:%f/100' % (time.ctime(), process)
# logger.debug('%s: Proceed:%f/100' % (time.ctime(), process))
self.config.updateState(self.x, self.y)
if self.y < self.config.max_y:
if self.x < self.config.max_x:
self.x = self.x + 1
else:
self.x = self.min_x
self.y = self.y + 1
return True
else:
self.x = None
self.y = None
return False
if __name__ == '__main__':
r = BroadFirstGenerator('wuhan.cfg')
r.start()