def main():
ts = time()
start = input("Please input the start value: ")
stop = input("Please input the stop value: ")
# set the number of threads
threads = input("Please input the download numbers every piece: ")
k = stop-start
# acquire the download links
links = [l for l in get_link(start,stop)]
# set the download storage directory
down_dir = setup_dir()
queue = Queue()
# judge download numbers if greater than threads or not
# if K< = threads ,set the k for threads,else set the threads for the number of thread
if k <= threads:
for x in range(k-1):
print queue.qsize()
worker = DownloadWorker(queue)
worker.setDaemon(True)
worker.start()
else:
for x in range(threads):
worker = DownloadWorker(queue)
worker.setDaemon(True)
worker.start()
# traverse the links and put the link to queue
for link in links:
queue.put((down_dir,link))
# the new queue joining
queue.join()
print 'Took {}'.format(time()-ts)
print "The finished time:" + ctime()
def record_on_motion(savedir='.', fourcc=cv2.cv.CV_FOURCC(*"XVID"), fps=30, size=(640,480), color=True):
vidbuf = Queue()
output = cv2.VideoWriter()
bufsize = 60
recording = False
for i in range(0, bufsize):
vidbuf.put(capture_from(camera))
while True:
frame = capture_from(camera)
if frame.has_motion:
recording = True
if not output.isOpened():
filename = savedir + '/' + str(datetime.now()) + ".avi"
output.open(filename, fourcc, fps, size, color)
while 0 < vidbuf.qsize():
output.write(vidbuf.get().image)
output.write(frame.image)
else: #if not frame.has_motion:
vidbuf.put(capture_from(camera))
if bufsize < vidbuf.qsize():
vidbuf.get(True)
if recording == True:
while 0 < vidbuf.qsize():
output.write(vidbuf.get().image)
recording = False
break
class MemorySubscriber:
def __init__(self):
''' Initializes the empty queue for a particular subscriber. '''
self.messages = Queue()
def getNext(self):
''' Returns the next message available in the queue. Returns None if queue is empty. '''
if self.messages.qsize() == 0:
return None
else:
return self.messages.get(block=False, timeout=None)
def getAll(self):
''' Get all the messages available in the queue, appends them in the 'item' list and returns the list. '''
items = []
maxItemsToRetreive = self.messages.qsize()
for numOfItemsRetrieved in range(0, maxItemsToRetreive):
try:
if numOfItemsRetrieved == maxItemsToRetreive:
break
items.append(self.messages.get_nowait())
except Empty, e:
break
return items
class MongoDBSubscriber:
def __init__(self, collection, topics, timestamp):
self.messages = Queue()
self.collection = collection
self.topics = list(topics)
self.timestamp = timestamp
def getNext(self):
self.cursor = self.collection.find({'topic': {'$in': self.topics}, 'timestamp': {'$gte': self.timestamp}})
for data in self.cursor:
self.messages.put_nowait(data['message'])
self.timestamp = datetime.now()
if self.messages.qsize() == 0:
return None
else:
return self.messages.get(block=False, timeout=None)
def getAll(self):
self.cursor = self.collection.find({'topic': {'$in': self.topics}, 'timestamp': {'$gte': self.timestamp}})
for data in self.cursor:
self.messages.put_nowait(data['message'])
self.timestamp = datetime.now()
items = []
maxItemsToRetreive = self.messages.qsize()
for numOfItemsRetrieved in range(0, maxItemsToRetreive):
try:
if numOfItemsRetrieved == maxItemsToRetreive:
break
items.append(self.messages.get_nowait())
except Empty, e:
break
return items
def levelOrder(self, root):
"""
:type root: TreeNode
:rtype: List[List[int]]
"""
rel = []
if root is None:
return rel
from Queue import Queue
level = Queue()
level.put(root)
while level.qsize() > 0:
t = []
level_num = level.qsize()
for i in range(level_num):
node = level.get()
t.append(node.val)
if node.left:
level.put(node.left)
if node.right:
level.put(node.right)
rel.append(t)
return rel
class BambooHandler(SocketHandler):
def __init__(self, host, port, prefix=''):
SocketHandler.__init__(self, host, port)
self.closeOnError = 1
self.prefix = prefix.format(hostname=socket.getfqdn())
self.queue = Queue()
self.thread = Thread(target=self.run)
self.thread.setDaemon(True)
self.thread.start()
def makePickle(self, record):
"""
Use JSON.
"""
#ei = record.exc_info
#if ei:
# dummy = self.format(record) # just to get traceback text into record.exc_text
# record.exc_info = None # to avoid Unpickleable error
s = '%s%s:%i:%s\n' % (self.prefix, record.name, int(record.created), self.format(record))
#if ei:
# record.exc_info = ei # for next handler
return s
def send(self, s):
if self.queue.qsize() > 200:
sys.stderr.write('Dropping bamboo log: Queue is full (%i)\n' %
self.queue.qsize())
return
self.queue.put(s)
def run(self):
print 'Start logger thread'
while True:
s = self.queue.get()
SocketHandler.send(self, s)
def get_tenant_quota_usage_per_region(self, project_id):
# Return quota usage dict with keys as region name & values as usages.
# Calculates the usage from each region concurrently using threads.
# Retrieve regions for the project
region_lists = sdk.OpenStackDriver().get_all_regions_for_project(
project_id)
usage_queue = Queue()
regions_usage_dict = collections.defaultdict(dict)
regions_thread_list = []
for current_region in region_lists:
thread = threading.Thread(target=self.read_quota_usage,
args=(project_id, current_region,
usage_queue))
regions_thread_list.append(thread)
thread.start()
# Wait for all the threads to finish reading usages
for current_thread in regions_thread_list:
current_thread.join()
# Check If all the regions usages are read
if len(region_lists) == usage_queue.qsize():
for i in range(usage_queue.qsize()):
# Read Queue
current_region_data = usage_queue.get()
regions_usage_dict.update(current_region_data)
return regions_usage_dict
class UploadController:
def __init__(self, fail_record, directory):
# 初始化上传队列
self.upload_list = Queue()
# 初始化线程List
self.threads = []
# 继承失败记录
self.fail_record = fail_record
# 初始化上传队列大小
self.total = self.upload_list.qsize()
self.working_directory = directory
# 增加新的线程
def add_thread(self):
index = len(self.threads)
thread = UploadingThread(number=index, direcotry= self.working_directory, q=self.upload_list, fail_record=self.fail_record)
self.threads.append(thread)
# 增加path所对应文件到队列
def add_item(self, path):
self.upload_list.put(path)
self.total = self.upload_list.qsize()
# 返回队列大小
def get_total_number(self):
return self.total
def cleanall(self):
self.upload_list = Queue()
# print self.upload_list.qsize()
def refresh_threads(self):
index = len(self.threads)
for i in range(index):
self.add_thread()
class CQueueObject:
def __init__(self,sQueueName):
self.sQueueName = sQueueName
self.queue = Queue() #(maxsize = 1000) #处理队列，默认容量无限制
def GetQueueName(self):
return self.sQueueName
def GetQueueSize(self):
return self.queue.qsize()
def PutToQueue(self, oObject):
dictObj = {}
dictObj['object'] = oObject
dictObj['ins_time'] = GetCurrentTime()
self.queue.put(dictObj, block=True)
return self.queue.qsize()
def GetFmQueue(self, sThreadName):
while True:
try:
dictObj = self.queue.get(timeout=0.1) #timeout 0.1s
self.queue.task_done()
return dictObj
except Empty: #如果 Queue 是空，就会出异常
PrintAndSleep(0.1,'%s.GetFmQueue' % sThreadName,False) #避免过多日志
return None
class Fetcher:
def __init__(self, threads=5):
self.__thread_cnt = threads
self.__threads = []
self.__lock = Lock()
self.q_req = Queue()
self.q_resp = Queue()
self.__running = 0
for i in range(self.__thread_cnt):
new_thread = threading.Thread(target=self.__task)
new_thread.setDaemon(True)
new_thread.start()
self.__threads.append(new_thread)
def __task(self):
while True:
req = self.q_req.get()
print "Req: %s" % (req)
with self.__lock:
self.__running += 1
content = urllib2.urlopen(req).read()
self.q_resp.put((req, content))
with self.__lock:
self.__running -= 1
def push(self, req):
self.q_req.put(req)
def pop(self):
return self.q_resp.get()
def task_left(self):
return self.__running + self.q_resp.qsize() + self.q_req.qsize()
class StreamReaderThread(object):
def __init__(self, stream):
self._queue = Queue()
self._thread = None
self._stream = stream
def run(self):
self._thread = threading.Thread(target=self._enqueue_output,
args=(self._stream,))
self._thread.daemon = True
self._thread.start()
def _enqueue_output(self, out):
for line in iter(out.readline, b''):
self._queue.put(line)
def pop(self):
result = ""
try:
myqueuerng = xrange(self._queue.qsize())
except NameError: # py3
myqueuerng = range(self._queue.qsize())
for _ in myqueuerng:
try:
result += encoding.console_decode(self._queue.get_nowait(),
encoding.OUTPUT_ENCODING if IS_WINDOWS
else 'UTF-8')
except Empty:
pass
return result
class DlinkManager(object):
def __init__(self, targets, cmd='cat /var/passwd', thread_count=10):
self.targets = targets
self.thread_count = thread_count
self.cmd = cmd
def run(self):
self.queue = Queue()
for i in range(self.thread_count):
t = DlinkThread(self.queue, self.cmd)
t.setDaemon(True)
t.start()
logging.info('Filling queue with %i items' % len(self.targets))
for target in self.targets:
self.queue.put((target['ip'], target['port']))
init_size = self.queue.qsize()
current_count = 0
while not self.queue.empty():
q = init_size - self.queue.qsize()
#stdout.write("\r%i/%i checked. Check speed %i per sec " % (q, init_size, q - current_count))
#stdout.flush()
current_count = q
sleep(1)
return
class Stack:
# initialize your data structure here.
def __init__(self):
self.queue = Queue()
# @param x, an integer
# @return nothing
def push(self, x):
self.queue.put(x)
# @return nothing
def pop(self):
_tmp, _size = Queue(), self.queue.qsize()
for i in range(_size):
_ = self.queue.get()
if i < _size - 1:
_tmp.put(_)
self.queue = _tmp
# @return an integer
def top(self):
_tmp = Queue()
for i in range(self.queue.qsize()):
_ = self.queue.get()
_tmp.put(_)
self.queue = _tmp
return _
# @return an boolean
def empty(self):
return self.queue.empty()
def main_tool(reader,run,check,writer,th_ct=3,min_qsize=3000,log=print,**kwargs):
inq = Queue()
outq = Queue()
threads = thread_tool(run, check, inq, outq, th_ct, **kwargs)
closed, put_ct = reader(inq=inq, **kwargs)
get_ct = 0
while True:
#input
if not closed:
if inq.qsize() < min_qsize:
closed, put_ct = reader(inq=inq, **kwargs)
if closed:
if inq.qsize() == 0 and put_ct == get_ct:
exit_threads(threads)
if threads_all_died(threads):
break
#status
sleep_rand(2, 2)
status = 'put:{} get:{} inq:{} outq:{}'.format(
put_ct, get_ct, inq.qsize(), outq.qsize())
log(status)
#output
while True:
try:
out = outq.get_nowait()
get_ct = writer(out, inq=inq, **kwargs)
except Empty:
break
class FetchIO:
def __init__(self, process):
self.lock = Lock()
self.q_req = Queue(10)
self.q_ans = Queue(10)
for i in range(10):
t = Thread(target=self.threadget)
t.setDaemon(True)
t.start()
self.running = 0
self.process = process
def __del__(self):
self.q_rep.join()
self.q_ans.join()
def taskleft(self):
return self.q_req.qsize()+self.q_ans.qsize()+self.running
def push(self, req):
self.q_req.put(req, True, None)
print "push"
def pop(self):
self.q_ans.get()
def threadget(self):
while True:
req = self.q_req.get()
with self.lock:
self.running += 1
out = self.process(req)
self.q_ans.put(out, True, None)
with self.lock:
self.running -= 1
self.q_req.task_done()
class Stack:
# initialize your data structure here.
def __init__(self):
self.q = Queue()
# @param x, an integer
# @return nothing
def push(self, x):
self.q.put(x)
# @return nothing
def pop(self):
p = Queue()
while self.q.empty() is not True:
if self.q.qsize() == 1:
break
p.put(self.q.get())
self.q = p
# @return an integer
def top(self):
p = Queue()
while self.q.empty() is not True:
if self.q.qsize() == 1:
tmp = self.q.get()
p.put(tmp)
self.q = p
return tmp
p.put(self.q.get())
# @return an boolean
def empty(self):
return self.q.empty()
def bfs_expensive_impl(g, start):
"""
Constraint: граф не взвешенный?
http://cs.stackexchange.com/questions/4973/does-a-weighted-breadth-first-search-have-memory-when-moving-to-the-next-verte
Dijkstra's algorithm for weighted?
"""
assert start
assert g
# Finding
graph_store = Vertex.recode_graph(g)
start = graph_store[start]
# Mark
start.explored = True
Q = Queue()
Q.put(start)
assert Q.qsize() == 1
while not Q.empty():
size = Q.qsize()
v = Q.get()
print v.self # data extracting
assert Q.qsize() == size - 1
for w in v.ends:
if not w.explored:
w.explored = True
# mark patch
w.distance = v.distance + 1
Q.put(w)
assert Q.empty()
class UrlResolverManager():
def __init__(self, cacheFilename, tweetResolverListener):
self.__queue = Queue(maxsize=50)
self.__workers = []
self.__tweetResolverListener = tweetResolverListener
self.__resolverCache = UrlResolverCache(cacheFilename)
for i in range(0,30):
self.__workers.append(UrlResolverWorker(self, self.__queue, self.__resolverCache, i))
self.__lastReportedQueueSize = None
def start(self):
self.__resolverCache.start()
for worker in self.__workers:
worker.start()
def stop(self):
self.__resolverCache.stop()
for worker in self.__workers:
worker.stop()
logger.info("Urls in queue: " + str(self.__queue.qsize()))
while not self.__queue.empty():
self.__queue.get_nowait()
def pauseWorkers(self):
for worker in self.__workers:
worker.pauseJob()
def continueWorkers(self):
logger.debug("Continue workers job")
for worker in self.__workers:
worker.continueJob()
def cacheHitRate(self):
return self.__resolverCache.hitRate()
def addUrlToQueue(self, url):
if url.getState() is not None:
logger.info(u"Url already in queue:" + unicode(url))
return
url.setState("pending")
self.__queue.put(url, timeout=3)
s = self.__queue.qsize()
if s % 10 == 0 and s > 10 and self.__lastReportedQueueSize != s:
self.__lastReportedQueueSize = s
logger.warning("Queue size is too big: " + unicode(s))
def afterResolveUrl(self, url):
if url.getState() != "pending":
raise ValueError(unicode(url))
url.setState("finished")
if not url.isError():
url.getText()
self.__notifyUrlResolved(url)
def __notifyUrlResolved(self, url):
tweet = url.tweet()
if tweet.isResolved():
self.__tweetResolverListener.tweetResolved(tweet)
class WatchLocal():
def __init__(self, pid, path ):
self.pid = pid
self.path = path
self.queue = Queue()
event_handler = EventHandler(self.queue)
self.observer = Observer()
self.observer.schedule(event_handler, path, recursive=True)
# Ensure we do NOT use kqueue on MacOSX.
if sys.platform.startswith('darwin'):
for ermitter in self.observer._emitters:
if ermitter.__class__.__name__ != 'FSEventsEmitter':
raise AssertionError, 'Found unsupported observer on MacOSX: ' + ermitter.__class__.__name__
def get_changes(self):
files = []
for i in range(self.queue.qsize()):
item = self.queue.get()
if item not in files:
files.append(item)
self.queue.task_done()
return files
def start(self):
notify('GitBox Start', 'monitoring %s' % self.path )
self.observer.start()
qsize = self.queue.qsize()
lastchange = time.time()
while True:
if qsize != self.queue.qsize():
lastchange = time.time()
if time.time()-lastchange >= 10 and self.queue.qsize() > 0:
files = self.get_changes()
notify('GitBox Upload', 'send new and changed files in %' )
cmd = 'git add .'
process = subprocess.Popen(cmd.split(' '), cwd=self.path)
process.communicate()
cmd = 'git commit -am auto-commit'
process = subprocess.Popen(cmd.split(' '), cwd=self.path)
process.communicate()
cmd = 'git push'
process = subprocess.Popen(cmd.split(' '), cwd=self.path)
process.communicate()
qsize = self.queue.qsize()
time.sleep(1)
def stop(self):
self.observer.stop()
self.observer.join()
class fetcher:
def __init__(self,threads):
self.lock = Lock() #线程锁
self.q_req = Queue() #任务队列
self.q_ans = Queue() #完成队列
self.threads = threads
self.running = 0
for i in range(threads):
t = Thread(target=self.threadget,args=(i , ))
t.setDaemon(True)
t.start()
def __del__(self): #解构时需等待两个队列完成
time.sleep(0.5)
self.q_req.join()
self.q_ans.join()
def taskleft(self):
sys.stderr.write("req:%d\tans:%d\trunning:%d\n" % (self.q_req.qsize(),self.q_ans.qsize(),self.running))
#return self.q_req.qsize()+self.q_ans.qsize()+self.running
def push(self,req):
self.q_req.put(req)
def pop(self):
data = self.q_ans.get()
self.q_ans.task_done() #ans finish
return data
def haveResult(self):
return self.q_ans.qsize()
def threadget(self,myId):
opener = httplib2.Http(cache=None,timeout = 3) #复用
while True:
req = self.q_req.get()
self.lock.acquire() #要保证该操作的原子性，进入critical area
self.running += 1
self.lock.release()
ans = self.get(myId,req,opener)
self.q_ans.put((req,ans))
self.lock.acquire() #要保证该操作的原子性，进入critical area
self.running -= 1
self.lock.release()
self.q_req.task_done() #req finish
time.sleep(1)
def get(self,myId,req,opener,retries=3):
try:
(resp_headers , data) = opener.request(req , "GET")
except Exception , what:
if retries>0:
time.sleep(3)
return self.get(myId,req,opener,retries-1)
else:
sys.stderr.write("myId:%d:\t%s:GET FAIL\tREQ:%s\t%s\n"% (myId,datetime.datetime.now(),req,what))
return ''
return data
def main():
"""
Define script parameters
"""
LOGLEVEL = logging.DEBUG
ITEMCOUNT = 100000
# Set up logging
logformatter = logging.Formatter('%(asctime)s - %(process)d'
' - %(levelname)s - %(message)s')
loghandler = logging.StreamHandler(stderr)
loghandler.setFormatter(logformatter)
loghandler.setLevel(LOGLEVEL)
scriptlogger = logging.getLogger('cassbench')
scriptlogger.addHandler(loghandler)
scriptlogger.setLevel(LOGLEVEL)
scriptlogger.debug('Script logging enabled')
# Set up variables needed by processes
iq = Queue()
# Call function to create, start, and wait for enqueue processes
# enqueue(scriptlogger, ITEMCOUNT, iq)
plist = []
scriptlogger.debug('Spawning enqueue processes')
for i in xrange(MAXPROCS):
p = multiprocessing.Process(target=EnqueuePeople, args=(scriptlogger,
ITEMCOUNT,
iq,),
name='EeQueue-%d' % i)
# t.daemon = True
p.start()
scriptlogger.debug('Done spawning enqueue processes')
while len(multiprocessing.active_children()) > 0:
scriptlogger.debug('Waiting for %d enqueue processes'
% len(multiprocessing.active_children()))
try:
scriptlogger.info('Queue: %d items.' % iq.qsize())
sleep(1)
except KeyboardInterrupt:
scriptlogger.critical('Caught ctrl-c. Stopping worker processes.')
for p in multiprocessing.active_children():
p.terminate()
while len(multiprocessing.active_children()) > 0:
scriptlogger.info('Waiting for %d remaining processes to die.'
% len(multiprocessing.active_children()))
sleep(1)
break
scriptlogger.debug('Done waiting for enqueue processes')
# Call function to create, start, and wait for dequeue processes
dequeue(scriptlogger, iq)
scriptlogger.info('Queue: %d items.' % iq.qsize())
def ncd_loop(doInit, dlThreadNum):
ndutil.setTimezone()
#read config
cnfManager = CnfManager()
cnfManager.load('./ndc.cnf')
cnfData = cnfManager.getCnfData()
#check dirs
ndutil.enableDir(cnfData['dirWorking'])
ndutil.enableDir(cnfData['dirStore'])
#ndlcom
logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort'])
logger.logger('Initiating')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
if doInit:
dbManager.create_table()
os.system('rm -f %s/*' % cnfData['dirWorking'])
os.system('rm -f %s/*' % cnfData['dirStore'])
#logger.logger('Customizing Spiders')
#spiderGenerator = SpiderGenerator('template', 'spider/spiders')
#for spider in cnfData['spiders']:
# spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage'])
rpQueue = Queue()
pdQueue = Queue()
dpQueue = Queue()
pdLock = threading.Lock()
rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor')
rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum)
rpcMonitor.setDownloadTotal(pdQueue.qsize())
rpcMonitor.setPdQueueSize(pdQueue.qsize())
botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler')
receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver')
preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor')
downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader')
processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor')
logger.logger('Starting Threads')
rpcMonitor.start()
botScheduler.start()
receiver.start()
preProcessor.start()
downloader.start()
processor.start()
processor.join()
downloader.join()
preProcessor.join()
receiver.join()
botScheduler.join()
rpcMonitor.join()
def main_CommentDetail(fileName=dirCheck.dirGen('D:\spider\jd\InnerPageProductDetail_2015-08-26 10_53_41.csv'),
columnNo=2, threadCount=50):
global queue_sku_commentDetail, queue_skuPageUrl_commentDetail, queue_commentDetail_result
queue_sku_commentDetail = Queue(0)
queue_skuPageUrl_commentDetail = Queue(0)
queue_commentDetail_result = Queue(0)
# 参数说明;fileName为需打开的文件绝对路径加文件名；columnNo为特定字段在第几列（第一列为0）；threadCount为开启线程数，默认开启50个线程
# sku信息put至queue备用
with open(fileName, 'r') as f:
reader = csv.reader(f)
i = 1
for row in reader:
if i > 1:
queue_sku_commentDetail.put(row[columnNo])
i += 1
# 评论信息抓取
CommentDetail_thread = []
for i in range(threadCount):
CommentDetail_thread.append(CommentDetail())
for item in CommentDetail_thread:
item.start()
time.sleep(0.01)
while True:
if queue_commentDetail_result.qsize() > 200000:
resultForCommentDetail = []
for i in range(200000):
resultForCommentDetail.append(queue_commentDetail_result.get())
title = ['productSku', 'userId', 'userGuid', 'content', 'createTime', 'referenceId',
'referenceTime', 'replyCount', 'score', 'userLevelId', 'userProvince',
'productColor', 'userLevelName', 'userClientShow', 'isMobile', 'urlFrom'
]
# 数据写入csv文件
writer = MyCsv.Write_Csv(path=dirCheck.dirGen('d:/spider/jd/commentDetail'), name='commentDetail',
title=title, result=resultForCommentDetail)
writer.add_title_data()
if queue_skuPageUrl_commentDetail.qsize() == 0:
break
# for item in CommentDetail_thread:
# item.join()
# 评论信息持久化
resultForCommentDetail = []
for i in range(queue_commentDetail_result.qsize()):
resultForCommentDetail.append(queue_commentDetail_result.get())
title = ['productSku', 'userId', 'userGuid', 'content', 'createTime', 'referenceId',
'referenceTime', 'replyCount', 'score', 'userLevelId', 'userProvince',
'productColor', 'userLevelName', 'userClientShow', 'isMobile'
]
# 数据写入csv文件
writer = MyCsv.Write_Csv(path=dirCheck.dirGen('d:/spider/jd/commentDetail'), name='commentDetail',
title=title, result=resultForCommentDetail)
writer.add_title_data()
def max_queue_size():
"""We are not suffering from python limitations! - made for reassurance"""
lst = list()
for i in range(1, 1000000):
lst.append(i)
print len(lst)
que = Queue()
for i in range(1, 1000000):
que.put(i)
print que.qsize()
class SCFifo(object):
def __init__(self, depth):
"""
Single Clock FIFO using rdy/valid.
Parameters:
-----------
depth: int
maximum size of FIFO.
Returns:
--------
None
"""
self.depth_m1 = depth - 1
self.queue = Queue(maxsize=depth)
def generate(self, i_clk,
o_wrrdy, i_wrvalid, i_wrdata,
i_rdrdy, o_rdvalid, o_rddata,
o_fullness):
"""
Generate instance.
Ports:
------
i_clk: Signal(bool)
access clock
o_wrrdy: Signal(bool)
FIFO ready to accept data from source on next cycle
i_rdrdy: Signal(bool)
Sink ready to accept data from FIFO on next cycle
i_wrdata, o_rddata: Signal(any)
i_wrvalid, o_rdvalid: Signal(bool)
signify that applicable data lines can be sampled
o_fullness: Signal(int)
number of elements in FIFO
"""
@always(i_clk.posedge)
def wr_access():
o_wrrdy.next = (self.queue.qsize() < self.depth_m1)
if i_wrvalid and o_wrrdy:
self.queue.put_nowait(i_wrdata.val)
o_fullness.next = self.queue.qsize()
@always(i_clk.posedge)
def rd_access():
if i_rdrdy and (not self.queue.empty()):
o_rddata.next = self.queue.get_nowait()
o_fullness.next = self.queue.qsize()
o_rdvalid.next = True
else:
o_rdvalid.next = False
return instances()
def Dir(path=u'/'):
app_url=GetAppUrl()
if path=='/':
BaseUrl=app_url+u'v1.0/me/drive/root/children?expand=thumbnails'
items.remove()
queue=Queue()
# queue.put(dict(url=BaseUrl,grandid=grandid,parent=parent,trytime=1))
g=GetItemThread(queue)
g.GetItem(BaseUrl)
queue=g.queue
if queue.qsize()==0:
return
tasks=[]
for i in range(min(5,queue.qsize())):
t=GetItemThread(queue)
t.start()
tasks.append(t)
for t in tasks:
t.join()
RemoveRepeatFile()
else:
grandid=0
parent=''
if path.endswith('/'):
path=path[:-1]
if not path.startswith('/'):
path='/'+path
if items.find_one({'grandid':0,'type':'folder'}):
parent_id=0
for idx,p in enumerate(path[1:].split('/')):
if parent_id==0:
parent_id=items.find_one({'name':p,'grandid':idx})['id']
else:
parent_id=items.find_one({'name':p,'grandid':idx,'parent':parent_id})['id']
items.delete_many({'parent':parent_id})
grandid=idx+1
parent=parent_id
path=urllib.quote(path)
BaseUrl=app_url+u'v1.0/me/drive/root:{}:/children?expand=thumbnails'.format(path)
queue=Queue()
# queue.put(dict(url=BaseUrl,grandid=grandid,parent=parent,trytime=1))
g=GetItemThread(queue)
g.GetItem(BaseUrl,grandid,parent,1)
queue=g.queue
if queue.qsize()==0:
return
tasks=[]
for i in range(min(5,queue.qsize())):
t=GetItemThread(queue)
t.start()
tasks.append(t)
for t in tasks:
t.join()
RemoveRepeatFile()
class EventListener(Thread):
"""Event thread to handle event messages from Photoshop"""
def __init__(self, passwd, host=None, interval=None, *args, **kwargs):
super(EventListener, self).__init__(*args, **kwargs)
self._connection = Connection()
self._connection.connect(passwd, host)
self._sub = Queue()
self._unsub = Queue()
self._ids = {}
self._interval = interval
self._stop = Event()
def subscribe(self, eventName, callback, args=()):
self._sub.put({'eventName': eventName, 'func': callback, 'args': args})
def unsubscribe(self, eventName, callback):
self._unsub.put({'eventName': eventName, 'func': callback})
def stop(self):
self._stop.set()
self._connection.close()
def run(self):
while not self._stop.is_set():
## -- Add any subs
while self._sub.qsize():
item = self._sub.get()
LOGGER.debug('Subscribing %s', item)
msg = self._connection.send(SUBSCRIBE % item['eventName'], True)
if msg is not None:
self._ids[msg.id] = item
## -- Remove subs
while self._unsub.qsize():
item = self._unsub.get()
LOGGER.debug('Unsubscribing %s', item)
for id_, sub in self._ids.iteritems():
if sub['eventName'] == item['eventName'] and \
sub['func'] == item['func']:
del self._ids[id_]
break
## -- Receive bytes from PS
message = self._connection.recv()
if message is None:
continue
obj = self._ids.get(message.id)
if obj:
obj['func'](message, *obj['args'])
if self._interval:
self._stop.wait(self._interval)
class TestBaseEventModule(TestBase):
"""
Tests if a sample task_function is applied correctly to some event
queue.
"""
def setUp(self):
_settings['settings'] = ConfigObj(
{'General':
{'Log Directory': 'logs'},
'TestLogger': {'General': {'Log Subdirectory': 'testlog',
'Log Filename': 'testlog.txt',
}
}
}
)
_cmdoptions['cmdoptions'] = {}
self.q = Queue(0)
def test_quick_ten_times(self):
# Fill the queue with the events from which it pops off events.
for i in range(10):
self.q.put('test %d' % i)
# The loggername is where the logs go.
loggername = 'TestLogger'
fsbec = FirstStageBaseEventClass('test', self.q, loggername)
self.assertEqual(10, self.q.qsize())
fsbec.start()
self.assertGreaterEqual(10, self.q.qsize())
# Debe terminar muuuuy rápido
time.sleep(0.0007)
fsbec.cancel()
self.assertTrue(self.q.empty())
def test_quick_eighteen_times_slow(self):
# Fill the queue with the events from which it pops off events.
for i in range(10):
self.q.put('test %d' % i)
# The loggername is where the logs go.
loggername = 'TestLogger'
fsbec = FirstStageBaseEventClass('test', self.q, loggername)
self.assertEqual(10, self.q.qsize())
fsbec.start()
time.sleep(0.00001)
for i in range(5):
self.q.put('test %d' % i)
time.sleep(0.00001)
self.assertGreaterEqual(15, self.q.qsize())
time.sleep(0.00005)
for i in range(3):
self.q.put('test %d' % i)
time.sleep(0.001)
fsbec.cancel()
self.assertTrue(self.q.empty())
class ThreadPool(object): def __init__(self, threadNum): self.pool = [] #线程池 self.threadNum = threadNum #线程数 self.lock = Lock() #线程锁 self.running = 0 #正在run的线程数 self.taskQueue = Queue() #任务队列 self.resultQueue = Queue() #结果队列 def startThreads(self): for i in range(self.threadNum): self.pool.append(Worker(self)) def stopThreads(self): for thread in self.pool: thread.stop() thread.join() del self.pool[:] def putTask(self, func, *args, **kargs): self.taskQueue.put((func, args, kargs)) def getTask(self, *args, **kargs): task = self.taskQueue.get(*args, **kargs) return task def taskJoin(self, *args, **kargs): self.taskQueue.join() def taskDone(self, *args, **kargs): self.taskQueue.task_done() def putTaskResult(self, *args): self.resultQueue.put(args) def getTaskResult(self, *args, **kargs): return self.resultQueue.get(*args, **kargs) def increaseRunsNum(self): self.lock.acquire() #锁住该变量,保证操作的原子性 self.running += 1 #正在运行的线程数加1 self.lock.release() def decreaseRunsNum(self): self.lock.acquire() self.running -= 1 self.lock.release() def getTaskLeft(self): #线程池的所有任务包括： #taskQueue中未被下载的任务, resultQueue中完成了但是还没被取出的任务, 正在运行的任务 #因此任务总数为三者之和 return self.taskQueue.qsize()+self.resultQueue.qsize()+self.running
class ThreadPoolSubsystem(Subsystem):
NAME = "threadpool"
DEFAULT_MAX_WORKERS = 10
ID_GENERATOR = itertools.count()
def _init(self):
self._workers = {}
self._tasks = Queue()
self._max_workers = self.DEFAULT_MAX_WORKERS
def get_num_of_workers(self):
return len(self._workers)
def set_max_workers(self, count):
self._max_workers = count
if count < len(self._workers):
for _ in range(len(self._workers) - count):
self._tasks.put(None)
def _spawn_workers(self):
if self._tasks.qsize() <= len(self._workers):
return
if len(self._workers) >= self._max_workers:
return
needed = min(self._tasks.qsize() - len(self._workers), self._max_workers)
for _ in range(needed):
tid = self.ID_GENERATOR.next()
thd = threading.Thread(name = "pooled-thread %d" % (tid,), target = self._worker_thread, args = (tid,))
thd.setDaemon(True)
self._workers[tid] = thd
thd.start()
def _worker_thread(self, id):
try:
while True:
task = self._tasks.get()
if not task:
break
dfr, func, args, kwargs = task
try:
res = func(*args, **kwargs)
except Exception as ex:
dfr.throw(ex)
else:
dfr.set(res)
self.reactor._wakeup()
finally:
self._workers.pop(id, None)
def call(self, func, *args, **kwargs):
dfr = ReactorDeferred(self.reactor)
self._tasks.put((dfr, func, args, kwargs))
self._spawn_workers()
return dfr
class proxy():
def __init__(self, browser_type='chrome', ua='', log_dir=''):
self.browser_type = browser_type
self.logger = logging.getLogger("PORXY")
if not log_dir:
log_dir, filename = os.path.split(os.path.abspath(sys.argv[0]))
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.service_log_path = log_dir + '/ghostdriver.log'
self.ua = ua if ua else "user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
mongo_db = MongoClient('192.168.60.65', 10010).anti_ban
TJ_IP_MAPS = {
'10.10.10.21': '125.39.72.11',
'10.10.10.22': '125.39.72.118',
# '10.10.10.23': '125.39.72.225',
'10.10.10.23': '60.28.11.194',
'10.10.10.24': '125.39.0.146',
'10.10.10.25': '60.28.110.66',
'10.10.10.26': '125.39.149.48',
'10.10.10.27': '111.161.24.2'
}
self.proxy_list = []
for item in mongo_db.tj_proxy.find():
ip = item['ip']
proxy_host = TJ_IP_MAPS[ip[ip.find('://') + 3:ip.rfind(':')]]
proxy_port = ip[ip.rfind(':') + 1:]
proxy_username, proxy_password = item['user_pass'].split(':')
source_ip = item['source_ip']
self.proxy_list.append({
'proxy_host': proxy_host,
'proxy_port': int(proxy_port),
'proxy_username': proxy_username,
'proxy_password': proxy_password,
'source_ip': source_ip,
'type': 2
})
random.shuffle(self.proxy_list)
def init_proxy_queue(self):
from Queue import Queue
self.proxy_queue = Queue()
tj_ips = {}
for i in self.proxy_list:
ip_head = i['source_ip'].split('.')[0]
if ip_head not in tj_ips:
tj_ips[ip_head] = []
tj_ips[ip_head].append(i)
short_length = 99999
for k in tj_ips:
if short_length > len(tj_ips[k]):
short_length = len(tj_ips[k])
for _ in range(short_length):
for k in tj_ips:
self.proxy_queue.put(tj_ips[k].pop())
# mongo_db = MongoClient('192.168.60.65', 10010).proxy
# for item in mongo_db.ip_tbl.find({"status": 1}).sort([("update_time", -1)]):
# proxy_host = item['ip']
# proxy_port = item['port']
# source_ip = item['ip']
# proxy_type = item['type']
# self.proxy_queue.put({'proxy_host': proxy_host,
# 'proxy_port': int(proxy_port),
# 'source_ip': source_ip,
# 'type': proxy_type})
logging.info('proxy initialization complete %d',
self.proxy_queue.qsize())
def browser_quit(self, browser):
try:
if browser:
pid = browser.service.process.pid
browser.quit()
os.kill(pid, 9)
except:
pass
finally:
browser = None
def get_chrome_driver_with_proxy(self,
proxy_info,
scheme='http',
plugin_path=None):
"""Proxy Auth Extension
args:
proxy_host (str): domain or ip address, ie proxy.domain.com
proxy_port (int): port
proxy_username (str): auth username
proxy_password (str): auth password
kwargs:
scheme (str): proxy scheme, default http
plugin_path (str): absolute path of the extension
return str -> plugin_path
"""
import string
import zipfile
if plugin_path is None:
plugin_path = 'c://chrome_proxyauth_plugin.zip'
manifest_json = """
{
"version": "1.0.0",
"manifest_version": 2,
"name": "Chrome Proxy",
"permissions": [
"proxy",
"tabs",
"unlimitedStorage",
"storage",
"<all_urls>",
"webRequest",
"webRequestBlocking"
],
"background": {
"scripts": ["background.js"]
},
"minimum_chrome_version":"22.0.0"
}
"""
background_js = string.Template("""
var config = {
mode: "fixed_servers",
rules: {
singleProxy: {
scheme: "${scheme}",
host: "${host}",
port: parseInt(${port})
},
bypassList: ["foobar.com"]
}
};
chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
function callbackFn(details) {
return {
authCredentials: {
username: "******",
password: "******"
}
};
}
chrome.webRequest.onAuthRequired.addListener(
callbackFn,
{urls: ["<all_urls>"]},
['blocking']
);
""").substitute(
host=proxy_info['proxy_host'],
port=proxy_info['proxy_port'],
username=proxy_info.get('proxy_username', ''),
password=proxy_info.get('proxy_password', ''),
scheme=scheme,
)
with zipfile.ZipFile(plugin_path, 'w') as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js", background_js)
opts = Options()
opts.add_argument("--start-maximized")
# opts.add_argument("--headless")
# opts.add_argument("--disable-gpu")
opts.add_argument(self.ua)
if 'proxy_username' in proxy_info and 'proxy_password' in proxy_info:
opts.add_extension(plugin_path)
else:
opts.add_argument(
'--proxy-server=%s://%s:%d' %
(scheme, proxy_info['proxy_host'], proxy_info['proxy_port']))
chrome_driver = os.path.abspath("./chromedriver.exe")
browser = Chrome(chrome_driver,
service_log_path=self.service_log_path,
chrome_options=opts)
self.logger.info('set proxy %s:%s source ip: %s type: %d',
proxy_info['proxy_host'],
proxy_info.get('proxy_username', ''),
proxy_info['source_ip'], proxy_info['type'])
return browser
def get_phantomjs_driver_with_proxy(self, proxy_info, scheme='http'):
webdriver.DesiredCapabilities.PHANTOMJS[
'phantomjs.page.settings.userAgent'] = self.ua
webdriver.DesiredCapabilities.PHANTOMJS[
'phantomjs.page.settings.resourceTimeout'] = '500000'
if 'proxy_username' in proxy_info and 'proxy_password' in proxy_info:
service_args = [
'--proxy=%s:%s' %
(proxy_info['proxy_host'], proxy_info['proxy_port']),
'--proxy-auth=%s:%s' %
(proxy_info['proxy_username'], proxy_info['proxy_password']),
'--proxy-type=%s' % (scheme),
'--ignore-ssl-errors=true',
'--ssl-protocol=any',
]
else:
service_args = [
'--proxy=%s:%s' %
(proxy_info['proxy_host'], proxy_info['proxy_port']),
'--proxy-type=%s' % (scheme),
'--ignore-ssl-errors=true',
'--ssl-protocol=any',
]
browser = PhantomJS(service_args=service_args)
browser.set_page_load_timeout('10')
browser.implicitly_wait('10')
self.logger.info('set proxy %s:%s source ip: %s type: %d',
proxy_info['proxy_host'],
proxy_info.get('proxy_username'),
proxy_info['source_ip'], proxy_info['type'])
return browser
def get_new_webdriver_with_proxy(self, browser=None):
if browser:
self.browser_quit(browser)
proxy_info = choice(self.proxy_list)
if self.browser_type.lower() == 'chrome':
return self.get_chrome_driver_with_proxy(proxy_info)
elif self.browser_type.lower() == 'phantomjs':
return self.get_phantomjs_driver_with_proxy(proxy_info)
else:
self.logger.error('invalid browser type!')
return None
def get_test_proxy_webdriver(self, browser=None):
if browser:
self.browser_quit(browser)
proxy_info = self.proxy_queue.get()
if self.browser_type.lower() == 'chrome':
return self.get_chrome_driver_with_proxy(proxy_info)
elif self.browser_type.lower() == 'phantomjs':
return self.get_phantomjs_driver_with_proxy(proxy_info)
else:
self.logger.error('invalid browser type!')
return None
def train(self, sentences, total_words=None, word_count=0, chunksize=100):
"""
Update the model's neural weights from a sequence of sentences (can be a once-only generator stream).
Each sentence must be a list of utf8 strings.
"""
logger.info(
"training model with %i workers on %i vocabulary and %i features" %
(self.workers, len(self.vocab), self.layer1_size))
if not self.vocab:
raise RuntimeError(
"you must first build vocabulary before training the model")
start, next_report = time.time(), [1.0]
word_count, total_words = [
word_count
], total_words or sum(v.count for v in self.vocab.itervalues())
jobs = Queue(
maxsize=2 * self.workers
) # buffer ahead only a limited number of jobs.. this is the reason we can't simply use ThreadPool :(
lock = threading.Lock(
) # for shared state (=number of words trained so far, log reports...)
def worker_train():
"""Train the model, lifting lists of sentences from the jobs queue."""
work = matutils.zeros_aligned(
self.layer1_size,
dtype=REAL) # each thread must have its own work memory
while True:
job = jobs.get()
if job is None: # data finished, exit
break
# update the learning rate before every job
alpha = max(
self.min_alpha,
self.alpha * (1 - 1.0 * word_count[0] / total_words))
# how many words did we train on? out-of-vocabulary (unknown) words do not count
job_words = sum(
train_sentence(self, sentence, alpha, work)
for sentence in job)
with lock:
word_count[0] += job_words
elapsed = time.time() - start
if elapsed >= next_report[0]:
logger.info(
"PROGRESS: at %.2f%% words, alpha %.05f, %.0f words/s"
% (100.0 * word_count[0] / total_words, alpha,
word_count[0] / elapsed if elapsed else 0.0))
next_report[
0] = elapsed + 1.0 # don't flood the log, wait at least a second between progress reports
workers = [
threading.Thread(target=worker_train) for _ in xrange(self.workers)
]
for thread in workers:
thread.daemon = True # make interrupting the process with ctrl+c easier
thread.start()
# convert input strings to Vocab objects (or None for OOV words), and start filling the jobs queue
no_oov = ([self.vocab.get(word, None) for word in sentence]
for sentence in sentences)
for job_no, job in enumerate(utils.grouper(no_oov, chunksize)):
logger.debug("putting job #%i in the queue, qsize=%i" %
(job_no, jobs.qsize()))
jobs.put(job)
logger.info(
"reached the end of input; waiting to finish %i outstanding jobs" %
jobs.qsize())
for _ in xrange(self.workers):
jobs.put(
None
) # give the workers heads up that they can finish -- no more work!
for thread in workers:
thread.join()
elapsed = time.time() - start
logger.info("training on %i words took %.1fs, %.0f words/s" %
(word_count[0], elapsed,
word_count[0] / elapsed if elapsed else 0.0))
return word_count[0]
class TwitterListener(StreamListener):
"""A listener class for handling streaming Twitter data."""
def __init__(self, callback, logs_to_cloud):
self.logs_to_cloud = logs_to_cloud
self.logs = Logs(name="twitter-listener", to_cloud=self.logs_to_cloud)
self.callback = callback
self.error_status = None
self.start_queue()
def start_queue(self):
"""Creates a queue and starts the worker threads."""
self.queue = Queue()
self.stop_event = Event()
self.logs.debug("Starting %s worker threads." % NUM_THREADS)
self.workers = []
for worker_id in range(NUM_THREADS):
worker = Thread(target=self.process_queue, args=[worker_id])
worker.daemon = True
worker.start()
self.workers.append(worker)
def stop_queue(self):
"""Shuts down the worker threads."""
if not self.workers:
self.logs.warn("No worker threads to stop.")
return
self.stop_event.set()
for worker in self.workers:
# Terminate the thread immediately.
worker.join(0)
def process_queue(self, worker_id):
"""Continuously processes tasks on the queue."""
# Create a new logs instance (with its own httplib2 instance) so that
# there is a separate one for each thread.
logs = Logs("twitter-listener-worker-%s" % worker_id,
to_cloud=self.logs_to_cloud)
logs.debug("Started worker thread: %s" % worker_id)
while not self.stop_event.is_set():
# The main loop doesn't catch and report exceptions from background
# threads, so do that here.
try:
size = self.queue.qsize()
logs.debug("Processing queue of size: %s" % size)
data = self.queue.get(block=True)
self.handle_data(logs, data)
self.queue.task_done()
except BaseException as exception:
logs.catch(exception)
logs.debug("Stopped worker thread: %s" % worker_id)
def on_error(self, status):
"""Handles any API errors."""
self.logs.error("Twitter error: %s" % status)
self.error_status = status
self.stop_queue()
return False
def get_error_status(self):
"""Returns the API error status, if there was one."""
return self.error_status
def on_data(self, data):
"""Puts a task to process the new data on the queue."""
# Stop streaming if requested.
if self.stop_event.is_set():
return False
# Put the task on the queue and keep streaming.
self.queue.put(data)
return True
def handle_data(self, logs, data):
"""Sanity-checks and extracts the data before sending it to the
callback.
"""
# Decode the JSON response.
try:
tweet = loads(data)
except ValueError:
logs.error("Failed to decode JSON data: %s" % data)
return
# Do a basic check on the response format we expect.
if ("user" not in tweet or "id_str" not in tweet["user"]
or "screen_name" not in tweet["user"]):
logs.error("Malformed tweet: %s" % tweet)
return
# We're only interested in tweets from Mr. Trump himself, so skip the
# rest.
user_id_str = tweet["user"]["id_str"]
screen_name = tweet["user"]["screen_name"]
if user_id_str != TRUMP_USER_ID:
logs.debug("Skipping tweet from user: %s (%s)" %
(screen_name, user_id_str))
return
logs.info("Examining tweet: %s" % tweet)
# Call the callback.
self.callback(tweet)
class APRSUploader(object):
'''
Queued APRS Telemetry Uploader class
This performs uploads to the Habitat servers, and also handles generation of flight documents.
Incoming telemetry packets are fed into queue, which is checked regularly.
If a new callsign is sighted, a payload document is created in the Habitat DB.
The telemetry data is then converted into a UKHAS-compatible format, before being added to queue to be
uploaded as network speed permits.
If an upload attempt times out, the packet is discarded.
If the queue fills up (probably indicating no network connection, and a fast packet downlink rate),
it is immediately emptied, to avoid upload of out-of-date packets.
Note that this uploader object is intended to handle telemetry from multiple sondes
'''
# We require the following fields to be present in the incoming telemetry dictionary data
REQUIRED_FIELDS = [
'frame', 'id', 'datetime', 'lat', 'lon', 'alt', 'temp', 'type', 'freq',
'freq_float', 'datetime_dt'
]
def __init__(
self,
aprs_callsign='N0CALL',
aprs_passcode="00000",
object_name_override=None,
object_comment="RadioSonde",
position_report=False,
aprsis_host='rotate.aprs2.net',
aprsis_port=14580,
station_beacon=False,
station_beacon_rate=30,
station_beacon_position=(0.0, 0.0, 0.0),
station_beacon_comment="radiosonde_auto_rx SondeGate v<version>",
station_beacon_icon="/r",
synchronous_upload_time=30,
callsign_validity_threshold=5,
upload_queue_size=16,
upload_timeout=10,
inhibit=False):
""" Initialise an APRS Uploader object.
Args:
aprs_callsign (str): Callsign of the uploader, used when logging into APRS-IS.
aprs_passcode (tuple): Optional - a tuple consisting of (lat, lon, alt), which if populated,
is used to plot the listener's position on the Habitat map, both when this class is initialised, and
when a new sonde ID is observed.
object_name_override (str): Override the object name in the uploaded sentence with this value.
WARNING: This will horribly break the aprs.fi map if multiple sondes are uploaded simultaneously under the same callsign.
USE WITH CAUTION!!!
object_comment (str): A comment to go with the object. Various fields will be replaced with telmetry data.
position_report (bool): If True, upload positions as APRS position reports, otherwise, upload as an Object.
aprsis_host (str): APRS-IS Server to upload packets to.
aprsis_port (int): APRS-IS TCP port number.
station_beacon (bool): Enable beaconing of station position.
station_beacon_rate (int): Time delay between beacon uploads (minutes)
station_beacon_position (tuple): (lat, lon, alt), in decimal degrees, of the station position.
station_beacon_comment (str): Comment field for the station beacon. <version> will be replaced with the current auto_rx version.
station_beacon_icon (str): The APRS icon to be used, as the two characters (symbol table, symbol index), as per http://www.aprs.org/symbols.html
synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0
This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously,
and also acts as decimation on the number of sentences uploaded to APRS-IS.
callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5
upload_queue_size (int): Maximum number of sentences to keep in the upload queue. If the queue is filled,
it will be emptied (discarding the queue contents).
upload_timeout (int): Timeout (Seconds) when performing uploads to APRS-IS. Default: 10 seconds.
inhibit (bool): Inhibit all uploads. Mainly intended for debugging.
"""
self.aprs_callsign = aprs_callsign
self.aprs_passcode = aprs_passcode
self.object_comment = object_comment
self.position_report = position_report
self.aprsis_host = aprsis_host
self.aprsis_port = aprsis_port
self.upload_timeout = upload_timeout
self.upload_queue_size = upload_queue_size
self.synchronous_upload_time = synchronous_upload_time
self.callsign_validity_threshold = callsign_validity_threshold
self.inhibit = inhibit
self.station_beacon = {
'enabled': station_beacon,
'position': station_beacon_position,
'rate': station_beacon_rate,
'comment': station_beacon_comment,
'icon': station_beacon_icon
}
if object_name_override is None:
self.object_name_override = "<id>"
else:
self.object_name_override = object_name_override
# Our two Queues - one to hold sentences to be upload, the other to temporarily hold
# input telemetry dictionaries before they are converted and processed.
self.aprs_upload_queue = Queue(upload_queue_size)
self.input_queue = Queue()
# Dictionary where we store sorted telemetry data for upload when required.
# Elements will be named after payload IDs, and will contain:
# 'count' (int): Number of times this callsign has been observed. Uploads will only occur when
# this number rises above callsign_validity_threshold.
# 'data' (Queue): A queue of telemetry sentences to be uploaded. When the upload timer fires,
# this queue will be dumped, and the most recent telemetry uploaded.
self.observed_payloads = {}
# Record of when we last uploaded a user station position to Habitat.
self.last_user_position_upload = 0
# Start the uploader thread.
self.upload_thread_running = True
self.upload_thread = Thread(target=self.aprs_upload_thread)
self.upload_thread.start()
# Start the input queue processing thread.
self.input_processing_running = True
self.input_thread = Thread(target=self.process_queue)
self.input_thread.start()
self.timer_thread_running = True
self.timer_thread = Thread(target=self.upload_timer)
self.timer_thread.start()
self.log_info("APRS Uploader Started.")
def aprsis_upload(self, source, packet, igate=False):
""" Upload a packet to APRS-IS
Args:
source (str): Callsign of the packet source.
packet (str): APRS packet to upload.
igate (boolean): If True, iGate the packet into APRS-IS
(i.e. use the original source call, but add SONDEGATE and our callsign to the path.)
"""
# If we are inhibited, just return immediately.
if self.inhibit:
self.log_info("Upload Inhibited: %s" % packet)
return True
# Generate APRS packet
if igate:
# If we are emulating an IGATE, then we need to add in a path, a q-construct, and our own callsign.
# We have the TOCALL field 'APRARX' allocated by Bob WB4APR, so we can now use this to indicate
# that these packets have arrived via radiosonde_auto_rx!
_packet = '%s>APRARX,SONDEGATE,TCPIP,qAR,%s:%s\n' % (
source, self.aprs_callsign, packet)
else:
# Otherwise, we are probably just placing an object, usually sourced by our own callsign
_packet = '%s>APRS:%s\n' % (source, packet)
# create socket & connect to server
_s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
_s.settimeout(self.upload_timeout)
try:
_s.connect((self.aprsis_host, self.aprsis_port))
# Send logon string
_logon = 'user %s pass %s vers VK5QI-AutoRX \n' % (
self.aprs_callsign, self.aprs_passcode)
_s.send(_logon.encode('ascii'))
# send packet
_s.send(_packet.encode('ascii'))
# close socket
_s.shutdown(0)
_s.close()
self.log_info("Uploaded to APRS-IS: %s" % _packet)
return True
except Exception as e:
self.log_error("Upload to APRS-IS Failed - %s" % str(e))
return False
def beacon_station_position(self):
''' Send a station position beacon into APRS-IS '''
if self.station_beacon['enabled']:
if (self.station_beacon['position'][0]
== 0.0) and (self.station_beacon['position'][1] == 0.0):
self.log_error(
"Station position is 0,0, not uploading position beacon.")
self.last_user_position_upload = time.time()
return
# Generate the station position packet
# Note - this is now generated as an APRS position report, for radiosondy.info compatability.
_packet = generate_station_object(
self.aprs_callsign,
self.station_beacon['position'][0],
self.station_beacon['position'][1],
self.station_beacon['comment'],
self.station_beacon['icon'],
position_report=True)
# Send the packet as an iGated packet.
self.aprsis_upload(self.aprs_callsign, _packet, igate=True)
self.last_user_position_upload = time.time()
def update_station_position(self, lat, lon, alt):
""" Update the internal station position record. Used when determining the station position by GPSD """
self.station_beacon['position'] = (lat, lon, alt)
def aprs_upload_thread(self):
''' Handle uploading of packets to Habitat '''
self.log_debug("Started APRS Uploader Thread.")
while self.upload_thread_running:
if self.aprs_upload_queue.qsize() > 0:
# If the queue is completely full, jump to the most recent telemetry sentence.
if self.aprs_upload_queue.qsize() == self.upload_queue_size:
while not self.aprs_upload_queue.empty():
_telem = self.aprs_upload_queue.get()
self.log_warning(
"Uploader queue was full - possible connectivity issue."
)
else:
# Otherwise, get the first item in the queue.
_telem = self.aprs_upload_queue.get()
# Convert to a packet.
try:
(_packet, _call) = telemetry_to_aprs_position(
_telem,
object_name=self.object_name_override,
aprs_comment=self.object_comment,
position_report=self.position_report)
except Exception as e:
self.log_error(
"Error converting telemetry to APRS packet - %s" %
str(e))
_packet = None
# Attempt to upload it.
if _packet is not None:
# If we are uploading position reports, the source call is the generated callsign
# usually based on the sonde serial number, and we iGate the position report.
# Otherwise, we upload APRS Objects, sourced by our own callsign, but still iGated via us.
if self.position_report:
self.aprsis_upload(_call, _packet, igate=True)
else:
self.aprsis_upload(self.aprs_callsign,
_packet,
igate=True)
else:
# Wait for a short time before checking the queue again.
time.sleep(0.1)
self.log_debug("Stopped APRS Uploader Thread.")
def upload_timer(self):
""" Add packets to the habitat upload queue if it is time for us to upload. """
while self.timer_thread_running:
if int(time.time()) % self.synchronous_upload_time == 0:
# Time to upload!
for _id in self.observed_payloads.keys():
# If no data, continue...
if self.observed_payloads[_id]['data'].empty():
continue
else:
# Otherwise, dump the queue and keep the latest telemetry.
while not self.observed_payloads[_id]['data'].empty():
_telem = self.observed_payloads[_id]['data'].get()
# Attept to add it to the habitat uploader queue.
try:
self.aprs_upload_queue.put_nowait(_telem)
except Exception as e:
self.log_error(
"Error adding sentence to queue: %s" % str(e))
# Sleep a second so we don't hit the synchronous upload time again.
time.sleep(1)
else:
# Not yet time to upload, wait for a bit.
time.sleep(0.1)
def process_queue(self):
""" Process packets from the input queue.
This thread handles packets from the input queue (provided by the decoders)
Packets are sorted by ID, and a dictionary entry is created.
"""
while self.input_processing_running:
# Process everything in the queue.
while self.input_queue.qsize() > 0:
# Grab latest telem dictionary.
_telem = self.input_queue.get_nowait()
_id = _telem['id']
if _id not in self.observed_payloads:
# We haven't seen this ID before, so create a new dictionary entry for it.
self.observed_payloads[_id] = {'count': 1, 'data': Queue()}
self.log_debug(
"New Payload %s. Not observed enough to allow upload."
% _id)
# However, we don't yet add anything to the queue for this payload...
else:
# We have seen this payload before!
# Increment the 'seen' counter.
self.observed_payloads[_id]['count'] += 1
# If we have seen this particular ID enough times, add the data to the ID's queue.
if self.observed_payloads[_id][
'count'] >= self.callsign_validity_threshold:
# Add the telemetry to the queue
self.observed_payloads[_id]['data'].put(_telem)
else:
self.log_debug(
"Payload ID %s not observed enough to allow upload."
% _id)
if (time.time() - self.last_user_position_upload
) > self.station_beacon['rate'] * 60:
self.beacon_station_position()
time.sleep(0.1)
def add(self, telemetry):
""" Add a dictionary of telemetry to the input queue.
Args:
telemetry (dict): Telemetry dictionary to add to the input queue.
"""
# Discard any telemetry which is indicated to be encrypted.
if 'encrypted' in telemetry:
return
# Check the telemetry dictionary contains the required fields.
for _field in self.REQUIRED_FIELDS:
if _field not in telemetry:
self.log_error("JSON object missing required field %s" %
_field)
return
# Add it to the queue if we are running.
if self.input_processing_running:
self.input_queue.put(telemetry)
else:
self.log_error("Processing not running, discarding.")
def close(self):
''' Shutdown uploader and processing threads. '''
self.log_debug("Waiting for threads to close...")
self.input_processing_running = False
self.timer_thread_running = False
self.upload_thread_running = False
# Wait for all threads to close.
if self.upload_thread is not None:
self.upload_thread.join()
if self.timer_thread is not None:
self.timer_thread.join()
if self.input_thread is not None:
self.input_thread.join()
def log_debug(self, line):
""" Helper function to log a debug message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.debug("APRS-IS - %s" % line)
def log_info(self, line):
""" Helper function to log an informational message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.info("APRS-IS - %s" % line)
def log_error(self, line):
""" Helper function to log an error message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.error("APRS-IS - %s" % line)
def log_warning(self, line):
""" Helper function to log a warning message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.warning("APRS-IS - %s" % line)
except Exception, e:
print "error:%s" % e
except KeyboardInterrupt:
print "You pressed Ctrl+c"
sys.exit()
iq.task_done()
file = raw_input("Enter the file to check the services:")
f = open(file, 'r')
#place ip into ip_queue
for eachline in f:
ip_queue.put(eachline)
f.close()
#spawn pool of scan_port threads
if ip_queue.qsize() < num_threads:
num_threads = ip_queue.qsize()
for i in range(num_threads):
worker = Thread(target=scan_port, args=(ip_queue, 5))
worker.setDaemon(True)
worker.start()
print "Main Thread waiting"
ip_queue.join()
print "Done"
class ExportBuildTest(object):
"""Object to encapsulate logic for progen build testing"""
def __init__(self, tests, parser, options):
"""
Initialize an instance of class ProgenBuildTest
Args:
tests: array of TestCase instances
"""
self.total = len(tests)
self.parser = parser
self.options = options
self.counter = 0
self.successes = []
self.failures = []
self.skips = []
self.tests = [ExportBuildTest.test_case(test) for test in tests]
self.build_queue = Queue()
@staticmethod
def test_case(case):
TestCase = namedtuple('TestCase', case.keys())
return TestCase(**case)
def handle_log(self, log):
try:
with open(log, 'r') as in_log:
print in_log.read()
sys.stdout.flush()
log_name = join(EXPORT_DIR, dirname(log) + "_log.txt")
if exists(log_name):
# delete it if so
remove(log_name)
rename(log, log_name)
except IOError:
pass
def batch_tests(self, clean=False):
"""Performs all exports of self.tests
Peroform_exports will fill self.build_queue.
This function will empty self.build_queue and call the test's
IDE's build function."""
do_queue(Reader, self.perform_exports, self.tests)
self.counter = 0
self.total = self.build_queue.qsize()
while not self.build_queue.empty():
build = self.build_queue.get()
self.counter += 1
exporter = build[0]
test_case = build[1]
self.display_counter(
"Building test case %s::%s\t%s" %
(test_case.mcu, test_case.ide, test_case.name))
cwd = os.getcwd()
os.chdir(exporter.export_dir)
res = EXPORTERS[exporter.NAME.lower()].build(exporter.project_name,
cleanup=False)
os.chdir(cwd)
if res:
self.failures.append(
"%s::%s\t%s" %
(test_case.mcu, test_case.ide, test_case.name))
else:
self.successes.append(
"%s::%s\t%s" %
(test_case.mcu, test_case.ide, test_case.name))
self.handle_log(exporter.generated_files[-1])
if clean:
rmtree(exporter.export_dir)
def display_counter(self, message):
with print_lock:
sys.stdout.write(
"{}/{} {}".format(self.counter, self.total, message) + "\n")
sys.stdout.flush()
def perform_exports(self, test_case):
"""
Generate the project file for test_case and fill self.build_queue
Args:
test_case: object of type TestCase
"""
sys.stdout.flush()
self.counter += 1
name_str = ('%s_%s_%s') % (test_case.mcu, test_case.ide,
test_case.name)
self.display_counter("Exporting test case %s::%s\t%s" %
(test_case.mcu, test_case.ide, test_case.name))
exporter, toolchain = get_exporter_toolchain(test_case.ide)
if test_case.mcu not in exporter.TARGETS:
self.skips.append("%s::%s\t%s" %
(test_case.mcu, test_case.ide, test_case.name))
return
profile = extract_profile(self.parser, self.options, toolchain)
exporter = export(test_case.mcu,
test_case.ide,
project_id=test_case.id,
zip_proj=None,
clean=True,
src=test_case.src,
export_path=join(EXPORT_DIR, name_str),
silent=True,
build_profile=profile)
exporter.generated_files.append(
join(EXPORT_DIR, name_str, test_case.log))
self.build_queue.put((exporter, test_case))
class Fetcher:
def __init__(self, threads):
self.lock = Lock() #线程锁
self.q_req = Queue() #任务队列
self.q_ans = Queue() #完成队列
self.threads = threads
for i in range(threads):
t = Thread(target=self.threadget) #括号中的是每次线程要执行的任务
t.setDaemon(True) #设置子线程是否随主线程一起结束，必须在start()
#之前调用。默认为False
t.start() #启动线程
self.running = 0 #设置运行中的线程个数
def __del__(self): #解构时需等待两个队列完成
time.sleep(0.5)
self.q_req.join() #Queue等待队列为空后再执行其他操作
self.q_ans.join()
#返回还在运行线程的个数，为0时表示全部运行完毕
def taskleft(self):
return self.q_req.qsize() + self.q_ans.qsize() + self.running
def push(self, req):
self.q_req.put(req)
def pop(self):
return self.q_ans.get()
#线程执行的任务，根据req来区分
def threadget(self):
while True:
line = self.q_req.get()
word = line.strip()
'''
Lock.lock()操作，使用with可以不用显示调用acquire和release，
这里锁住线程，使得self.running加1表示运行中的线程加1，
如此做防止其他线程修改该值，造成混乱。
with下的语句结束后自动解锁。
'''
with self.lock:
self.running += 1
'''构造请求头，header请自行修改'''
headers = [
"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding:gzip, deflate, sdch",
"Accept-Language:zh-CN,zh;q=0.8,en;q=0.6",
"Cache-Control:max-age=0", "Connection:keep-alive",
"Cookie:BAIDUID=4B0DC2F54860625BA83681F98C507951:SL=0:NR=10:FG=1; BIDUPSID=4B0DC2F54860625BA83681F98C507951; PSTM=1439274940; BDUSS=05jM09FaU5vakhpMmk0bi1USExiTlBVckdaWGJvZGg3MEc0bWljc35GQlpPdkpWQVFBQUFBJCQAAAAAAAAAAAEAAAAJkstJv7TXvMTj1NnM-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFmtylVZrcpVS; BDSFRCVID=49AsJeCCxG3FME5lFVm78qZlaTKqMyh8xgII3J; H_BDCLCKID_SF=JJ4O_C-5tCv8fjrzhJbM-J3H-UnLq508067Z0lOnMp05sl6HjtvP34rWehQz2PonW56uXRPyMn3zODO_e4bK-TrLDGuttx5; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a01898970688; BDRCVFR[ltbVPlNi2ac]=mbxnW11j9Dfmh7GuZR8mvqV; BD_HOME=1; BD_UPN=123253; sug=3; sugstore=1; ORIGIN=0; bdime=0; H_PS_645EC=4337UU5N7NfqMN6E1HAyzTniY68nK7axTtmIEncpHKSgv8eqcIQ1Y9O6drzulgAwpkOr; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; BD_CK_SAM=1; BDSVRTM=100; H_PS_PSSID=1468_7477_13245_12826_10212_12868_16799_16426_16514_15864_12103_13932_14924_16866; ispeed_lsm=10",
"Host:www.baidu.com",
"RA-Sid:7739A016-20140918-030243-3adabf-48f828",
"RA-Ver:2.10.4",
"User-Agent:%s" % getUA()
]
bdjd_str = ','.join(bdjd_list)
#newip = ip()
bdjd = getBDJD(bdjd_str)
url = 'http://www.baidu.com/s?wd=%s' % urllib.quote_plus(word)
html = is_index(url, headers)
'''xpath方法'''
tree = etree.HTML(html)
div = tree.xpath('//div[@class="result c-container "]')
for line in div:
line_html = h.unescape(etree.tostring(line)).encode('utf-8')
data = []
data.append(word)
data.append(line_html)
writer = csv.writer(csvfile, dialect='excel')
writer.writerow(data)
print '>> 已抓取：%s，返回%s条结果' % (word, len(div))
'''beautiful方法'''
# soup = bs(html)
# '''提取百度1-10名的块级元素'''
# b_tags = soup.find_all('div', {'class': 'result c-container '})
# if len(b_tags) == 0:
# #print 'html_title：%s，ip:%s' % (search(r'<title[^>]*?>([\s\S]*?)</title>',html),newip)
# continue
# '''将百度1-10名块级元素的代码下载至serp_html.csv，之后在计算首页词数、展现次数、排名质量分等数据需求'''
# for line in b_tags:
# newline = str(line)
# number = search(r'id="(\d+)"',newline)
# #urldiv = search(r'<span class="g">(.*?)</span>',newline) #获取源码中domain所在的<span>
# data = []
# data.append(word)
# data.append(newline)
# writer = csv.writer(csvfile,dialect='excel')
# writer.writerow(data)
# print '>> 已抓取：%s，返回%s条结果' % (word,len(b_tags))
time.sleep(2)
#self.q_ans.put((req,ans)) # 将完成的任务压入完成队列，在主程序中返回
self.q_ans.put(line)
with self.lock:
self.running -= 1
self.q_req.task_done() # 在完成一项工作之后，Queue.task_done()
# 函数向任务已经完成的队列发送一个信号
time.sleep(0.1) # don't spam
class SimulationHandler:
# TODO: Docs
"""
"""
def __init__(self):
self.file_name = ""
self.file_obj = None
self.repeat_enabled = False
self.thread_acquisition = ThreadHandler(self.acquire_routine,
self.close)
self.buffer_acquisition = Queue(1024 * 4)
@property
def data_waiting(self):
"""
The size of the acquisition buffer
"""
return self.buffer_acquisition.qsize()
def open(self):
"""
If it is not already open, it will open the file.
"""
if self.file_obj.closed:
self.file_obj = open(file=self.file_name, mode='r')
def close(self):
"""
If the serial port is open, this method will try to close it.
"""
if not self.serialPort.closed:
self.file_obj.close()
def start_acquisition(self):
"""
Opens the serial port and starts a thread for acquisition.
The read objects will be in the buffer_acquisition.
"""
self.open()
self.thread_acquisition.start()
def stop_acquisition(self):
"""
Let the thread for acquisition reaches its end and, when it finally happens,
closes the serial port.
"""
self.thread_acquisition.stop()
@staticmethod
def get_arduino_serial_port():
"""
Tries to found a serial port compatible.
If there is only one serial port available, return this one.
Otherwise it will verify the manufacturer of all serial ports
and compares with the manufacturer defined in the ArduinoConstants.
This method will return the first match.
If no one has found, it will return a empty string.
:return: Serial Port String
"""
serial_ports = serial_tools.comports()
if len(serial_ports) == 0:
return ""
if len(serial_ports) == 1:
return serial_ports[0].device
for serial_port_found in serial_ports:
if serial_port_found.manufacturer == ArduinoConstants.MANUFACTURER:
return serial_port_found.device
return ""
@staticmethod
def to_int16(msb_byte, lsb_byte):
"""
Concatenate two bytes(8 bits) into a word(16 bits).
It will shift the msb_byte by 8 places to the right,
then it will sum the result with the lsb_byte.
:param msb_byte: The most significant byte.
:param lsb_byte: The less significant byte.
:return: The word created by the two bytes.
"""
return c_short((msb_byte << 8) + lsb_byte).value
def acquire_routine(self):
"""
This routine is automatically called by the acquisition thread.
Do not call this by yourself.
Description
-----------
If the serial port is open and there is more than the size of one
packet in the buffer. It will:
- verify the starter byte;
- read the data in the packet;
- verify the end byte;
- Put the read packet in a buffer (Queue).
"""
if self.serialPort.isOpen():
if self.serialPort.inWaiting() >= ArduinoConstants.PACKET_SIZE:
_starter_byte = self.serialPort.read()
if chr(ord(_starter_byte)) == ArduinoConstants.PACKET_START:
if self.qnt_ch == 1:
_msb = self.serialPort.read()
_lsb = self.serialPort.read()
_msb = ord(_msb)
_lsb = ord(_lsb)
_value_to_put = ArduinoHandler.to_int16(_msb, _lsb)
else:
_value_to_put = []
for n in range(self.qnt_ch):
_msb = self.serialPort.read()
_lsb = self.serialPort.read()
_msb = ord(_msb)
_lsb = ord(_lsb)
_value_to_put.append(
ArduinoHandler.to_int16(_msb, _lsb))
_end_byte = self.serialPort.read()
if chr(ord(_end_byte)) == ArduinoConstants.PACKET_END:
self.buffer_acquisition.put(_value_to_put)
def __str__(self):
return "ArduinoHandlerObject" +\
"\n\tSerialPort: " + str(self.serial_tools_obj.device) +\
"\n\tDescription: " + str(self.serial_tools_obj.description) +\
"\n\tOpen: " + str(self.serialPort.isOpen()) +\
"\n\tAcquiring: " + str(self.thread_acquisition.isRunning) +\
"\n\tInWaiting: " + str(self.serialPort.inWaiting() if self.serialPort.isOpen() else 'Closed') +\
"\n\tBufferAcq: " + str(self.buffer_acquisition.qsize())
def get_buffers_status(self, separator):
"""
Returns a string like:
Serial: 4/1024 - Acq: 1/1024
:param separator: Separates the strings, example ' - ', ' | ', '\n'
:return: A string containing the status of all the buffers involved in the acquisition
"""
return "Serial: %4d" % (self.serialPort.inWaiting()/4 if self.serialPort.isOpen() else 0) + '/' + str(4096/4) +\
separator + "Acq: %4d" % (self.buffer_acquisition.qsize()) + '/' + str(self.buffer_acquisition.maxsize)
class Fetcher:
def __init__(self,threads):
self.lock = Lock() #线程锁
self.q_req = Queue() #任务队列
self.q_ans = Queue() #完成队列
self.threads = threads
for i in range(threads):
t = Thread(target=self.threadget) #括号中的是每次线程要执行的任务
t.setDaemon(True) #设置子线程是否随主线程一起结束，必须在start()
#之前调用。默认为False
t.start() #启动线程
self.running = 0 #设置运行中的线程个数
def __del__(self): #解构时需等待两个队列完成
time.sleep(0.5)
self.q_req.join() #Queue等待队列为空后再执行其他操作
self.q_ans.join()
#返回还在运行线程的个数，为0时表示全部运行完毕
def taskleft(self):
return self.q_req.qsize()+self.q_ans.qsize()+self.running
def push(self,req):
self.q_req.put(req)
def pop(self):
return self.q_ans.get()
#线程执行的任务，根据req来区分
def threadget(self):
while True:
line = self.q_req.get()
word = line.strip()
'''
Lock.lock()操作，使用with可以不用显示调用acquire和release，
这里锁住线程，使得self.running加1表示运行中的线程加1，
如此做防止其他线程修改该值，造成混乱。
with下的语句结束后自动解锁。
'''
with self.lock:
self.running += 1
headers = [
"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding:gzip, deflate, sdch",
"Accept-Language:zh-CN,zh;q=0.8,en;q=0.6",
"Cache-Control:max-age=0",
"Connection:keep-alive",
#Cookie:QiHooGUID=127CC37B17FFC7B0958109BD32F1DE9D.1433901388314; stc_haosou_home=3273bbb969b3; __guid=238775686.4313411313784545000.1433901387665.2856; __sid=238775686.4313411313784545000.1433901387665.2856.1433901402749; tso_Anoyid=11143390142911824467; erules=p2-4%7Cp1-24%7Ckd-10%7Cp4-17%7Cecl-3%7Cecr-3; _S=dfooajg5qkao23otci70ifm8b7; __huid=10oAmPm07eL8NvDJxbC43U1oYtU2Jpci0OiG6aawNnnoc%3D; count=24; test_cookie_enable=null
"Host:www.haosou.com",
"RA-Sid:7739A016-20140918-030243-3adabf-48f828",
"RA-Ver:2.10.4",
"Referer:http://www.haosou.com/",
"User-Agent:%s" % getUA()
]
newip = ip()
url = 'http://www.haosou.com/s?ie=utf-8&shb=1&src=360sou_newhome&q=%s' % urllib.quote(word)
html = is_index(url,headers,newip)
if '您的电脑或所在局域网络对本站有异常访问' in html:
print '当前IP：%s，需要输入验证码' % newip
continue
soup = bs(html)
b_tags = soup.find_all('li', {'class': 'res-list'})
for line in b_tags:
data = search(r'<h3 class="res-title ">([\s\S]*?)</h3>',str(line).replace('\n',''))
lading = search(r'href="([^"]*?")',data)
rank = search(r'"pos":(\d+),',data)
title = re.sub(r'<[^>]*?>','',search(r'<a[^>]*?>(.*?)</a>',data))
data = []
data.append(word)
data.append(lading)
data.append(rank)
data.append(title)
data.append(newip)
data.append(current_date)
writer = csv.writer(csvfile,dialect='excel')
writer.writerow(data)
print "以获取：%s" % word
#self.q_ans.put((req,ans)) # 将完成的任务压入完成队列，在主程序中返回
self.q_ans.put(line)
with self.lock:
self.running -= 1
self.q_req.task_done() # 在完成一项工作之后，Queue.task_done()
# 函数向任务已经完成的队列发送一个信号
time.sleep(0.1) # don't spam
class QueryManager(object):
"""
Implements handling multiple threads used to speed up serverqueries
"""
def __init__(self):
"""
Constructor -
It starts with some basic initialisations and spawns a coordinator
thread which creates more threads to perform the master server query
and also the status updates for the servers.
"""
self.serverqueue = Queue()
self.messageque = Queue()
self.pulsemessageque = Queue()
self.threadcount = 0
self.servercount = 0
self.processedserver = 0
self.filterdcount = 0
self.gui_lock = None
self.geo_lock = None
coord = Thread(target=self.coordinator)
coord.daemon = True
coord.start()
dbname = Globals.geoip_dir+ '/GeoIP.dat'
self.pygeoip = GeoIP(dbname, pygeoip.const.MMAP_CACHE)
self.abort = False
def start_serverlist_refresh(self, liststore, tab):
"""
Refreshes the Serverlist of a tab
@param liststore - the liststore which contains the servers to be
refreshed
@param tab - the tab requesting the refresh
"""
self.tab = tab
self.filter = None
iter = liststore.iter_children(None)
while iter:
server = liststore.get_value(iter, 8)
self.serverqueue.put(server)
iter = liststore.iter_next(iter)
self.servercount = self.serverqueue.qsize()
gobject.idle_add(tab.clearServerList)
self.messageque.put('serverlist_loaded')
def startMasterServerQueryThread(self, filter, tab):
"""
Starts the masterserver query.
@param filter - filter to apply
@param tab - tab requesting the serverlist
"""
self.tab = tab
self.filter = filter
tab.clearServerList()
#this message will cause the coordinator to start querying the master
#server
self.messageque.put('start_master_server_query')
def startRecentServersLoadingThread(self, tab):
"""
Starts loading the recent servers list
@param tab - tab rquesting the recent servers
"""
fm = FileManager()
self.tab = tab
self.filter = None
serverdict = fm.getRecentServers()
for key in serverdict:
self.serverqueue.put(serverdict[key])
self.servercount = len(serverdict)
#notify the coordinator thread, that the serverlist is loaded
self.messageque.put('serverlist_loaded')
def startFavoritesLoadingThread(self, tab):
"""
Starts loading the favorites
@param tab - the tab requesting the favoriteslist
"""
fm = FileManager()
self.tab = tab
self.filter = None
serverlist = fm.getFavorites().values()
for server in serverlist:
self.serverqueue.put(server)
self.servercount = len(serverlist)
#notify the coordinator thread, that the serverlist is loaded
self.messageque.put('serverlist_loaded')
def lookup_server(self, server, tab):
"""
Starts the lookup of a certain server.
@param server - the server to be looked up
@param tab - the requesting tab
"""
self.tab = tab
self.filter = None
self.serverqueue.put(server)
self.servercount = 1
self.messageque.put('serverlist_loaded')
def coordinator(self):
"""
Method that runs as coordinator thread.
Spawning additional threads based on string messages in the messagueue
Messages accepted:
start_master_server_query - is the start signal. will cause the coordinator
to spawn two new threads. The first is to pulse
the progressbar on self.tab every 0.1 seconds.
The second thread performs the master server
query, when this thread finishes puts the
message serverlist_loaded into the
messagequeue indicating that the query
was succesfull and the serverqueue is
filled with servers
serverlist_loaded - spawns 10 worker threads that will perform the
get_status request of the servers in the
serverqueue
finished - if the status of all servers has been retreived the last
thread puts this message in the messagueue. calls the
serverlist_loading_finished method on self.tab and terminates
the coordinator thread
"""
# main thread loop
Log.log.debug('Thread:Coordinator started...')
while True:
try:
message = self.messageque.get()
if message == 'start_master_server_query':
Log.log.info('Thread:Coordinator - start_master_server_' \
+'query signal received')
# spawn the pulse progressbar thread
pt = Thread(target=self.pulse_progressbar_thread)
pt.setDaemon(True)
pt.start()
#spawns the master server query thread
pt = Thread(target=self.master_server_query_thread)
pt.setDaemon(True)
pt.start()
elif message == 'serverlist_loaded':
Log.log.info('Thread:Coordinator - received serverlist' \
+'_loaded signal. Queuesize is ' \
+ str(self.serverqueue.qsize()))
#stop the pulsing of the progressbar
self.pulsemessageque.put('stop_pulse')
#start 10 worker threads retreiving the status of
#the servers in the serverqueue
for i in range(10):
name = 'Worker_' + str(i+1)
t = Thread(target=self.get_server_status_thread, name=name)
t.setDaemon(True)
t.start()
elif message == 'finished':
#finish tasks :)
Log.log.info('Thread:Coordinator - received the ' \
+ 'finished signal')
self.gui_lock = threading.RLock()
with self.gui_lock:
gobject.idle_add(self.tab.serverlist_loading_finished)
break
elif message == 'all_aborted':
#all_aborted tasks :)
Log.log.info('Thread:Coordinator - received the ' \
+ 'all_aborted signal')
self.gui_lock = threading.RLock()
with self.gui_lock:
gobject.idle_add(self.set_progressbar_aborted)
gobject.idle_add(self.tab.serverlist_loading_finished)
break
except Empty:
True
def master_server_query_thread(self):
"""
This method is running as a thread to retreive a list of servers
from the master server.
"""
query = Q3ServerQuery()
empty = self.filter.show_empty
full = self.filter.show_full
#query the urban terror master server
serverlist = query.getServerList('master.urbanterror.info'
,27900
,empty
,full)
#put all servers in the serverqueue
for server in serverlist:
self.serverqueue.put(server)
self.servercount = len(serverlist)
#notify the coordinator thread, that the serverlist is loaded
self.messageque.put('serverlist_loaded')
def get_server_status_thread(self):
"""
This method will run as worker thread to retreive the status of
the servers in the serverqueue
"""
#increment thread count.
#the counter will be decreased on exit and compared to 0
#so the last thread can notify the coordinator that all threads finished
#their work
self.gui_lock = threading.RLock()
with self.gui_lock:
self.threadcount+=1
Log.log.debug('Thread:' + threading.current_thread().name + \
' started')
# main thread loop
while True:
try:
self.gui_lock = threading.RLock()
with self.gui_lock:
if self.abort:
self.threadcount -= 1
Log.log.info('Thread:' + threading.current_thread().name + \
' exiting due to abort signal')
if self.threadcount == 0: #last thread reached
Log.log.info('Thread:' + threading.current_thread().name + \
' notifying the coordinator thread that all threads ' \
+ 'was aborted')
self.messageque.put('all_aborted')
break
server = self.serverqueue.get(False)
#perform the statusrequest
query = Q3ServerQuery()
server = query.getServerStatus(server)
#add the server to the gui
self.gui_lock = threading.RLock()
with self.gui_lock:
self.set_location(server)
self.processedserver+=1
gobject.idle_add(self.set_progressbar_fraction)
if None == self.filter or \
self.filter.does_filter_match_server(server):
gobject.idle_add(self.tab.addServer, server)
else:
self.filterdcount+=1 # server is not added but filterd
except Empty:
#no more threads in the queue break thread execution
self.gui_lock = threading.RLock()
with self.gui_lock:
self.threadcount -= 1
Log.log.debug('Thread:' + threading.current_thread().name + \
' finishes working and exiting')
if self.threadcount == 0: #last thread reached
Log.log.info('Thread:' + threading.current_thread().name + \
' notifying the coordinator thread that the queue ' \
+ 'processing is finished')
self.messageque.put('finished')
break
def pulse_progressbar_thread(self):
"""
This method runs as a background thread that pulse the progressbar
of self.tab every 0.1 seconds
"""
while True:
try:
message = self.pulsemessageque.get(True, 0.1)
if message == 'stop_pulse':
break
except Empty:
self.gui_lock = threading.RLock()
with self.gui_lock:
gobject.idle_add(self.pulse_progressbar)
def set_progressbar_fraction(self):
"""
Sets the progressbar fraction. Uses the total servercount and the
processed servercount values to calculate the fraction
"""
if not self.abort:
fraction = float(self.processedserver) / float(self.servercount)
bartext = None
if 1.0 == fraction:
bartext = 'finished getting server status - displaying ' \
+ str((self.servercount-self.filterdcount)) + \
' servers (' + str(self.filterdcount) + ' filtered)'
self.tab.statusbar.progressbar.set_fraction(0.0)
else:
bartext = 'fetching server status (' + str(self.processedserver) + \
' / ' + str(self.servercount) + ') - ' + \
str(self.filterdcount) + ' servers filtered'
self.tab.statusbar.progressbar.set_fraction(fraction)
self.tab.statusbar.progressbar.set_text(bartext)
def pulse_progressbar(self):
"""
Pulse the progressbar, called by the thread using gobject.idle_add
"""
self.tab.statusbar.progressbar.set_text('fetching serverlist from master server')
self.tab.statusbar.progressbar.pulse()
def set_progressbar_aborted(self):
"""
Sets the text of the progressbar to the aborted message and resets fraction
"""
self.tab.statusbar.progressbar.set_text('task aborted')
self.tab.statusbar.progressbar.set_fraction(0.0)
def abort_current_task(self):
"""
Stops the processing of the queue by setting a abort flag.
"""
self.gui_lock = threading.RLock()
with self.gui_lock:
self.abort = True
def set_location(self, server):
"""
Determine location of a server based on the ip adress of the server
and set it at the server object
Extra threading lock used because there was some strange effects
without it.
@param - the server object
"""
self.geo_lock = threading.RLock()
with self.geo_lock:
#location = country(server.getHost())
location = self.pygeoip.country_code_by_addr(server.getHost())
locname = self.pygeoip.country_name_by_addr(server.getHost())
server.set_location(location)
server.set_location_name(locname)
'''
Queue
'''
from Queue import Queue
if __name__ == '__main__':
q = Queue()
q.put(1)
q.put(2)
q.put(3)
print(q.qsize())
print(q.get())
print(q.get())
print(q.get())
print(q.get())
class TaskQueue(JsonSerializable):
def __init__(self, cache, queue_uuid, trigger_time, sync_group=True):
self.create_time = get_time.current_ymd_hms()
self.sync_group = sync_group
self.trigger_time = trigger_time
self.expire_time = get_time.calc_expire_time(self.create_time,
self.trigger_time)
self.destroy_time = get_time.calc_destroy_time(self.create_time,
self.trigger_time)
self.queue_status = QueueStatus.Initiating
self.run_all = False
self.queue_uuid = queue_uuid
self.todo_task_queue = None
self.task_list = []
self.task_result_list = []
self.__exclude__.append('todo_task_queue')
self._condition = threading.Condition(threading.RLock())
self._cache = cache
self._cache[queue_uuid] = self
@staticmethod
def from_dict(cache, dict_data):
queue = TaskQueue(cache,
dict_data['queue_uuid'],
trigger_time=dict_data['trigger_time'],
sync_group=dict_data['sync_group'])
queue.queue_status = QueueStatus(dict_data['queue_status'])
queue.run_all = dict_data['run_all']
for task_define in dict_data['task_list']:
task = Task.from_dict(task_define)
queue.task_list.append(task)
for result_define in dict_data['task_result_list']:
result = TaskResult.from_dict(result_define)
queue.task_result_list.append(result)
queue.make_todo_task_queue(force=True)
return queue
@property
def IsSync(self):
"""
查看任务队列是否为同步队列
"""
return self.sync_group
@property
@locker
def IsRunAll(self):
return self.run_all
@property
def UUID(self):
"""
返回任务队列的UUID
"""
return self.queue_uuid
@property
@locker
def Status(self):
"""
返回任务队列的状态
"""
return self.queue_status
@locker
def wait(self, timeout=None):
if self.queue_status.Blocking:
self._condition.wait(timeout)
@locker
def append(self, task):
"""
添加任务至任务列表，该操作不更新待做任务队列
如需更新待做任务队列，在完成该操作后调用 make_todo_task_queue
"""
self._ready = False
if self.queue_status == QueueStatus.Initiating:
logging.info('Task added: {}'.format(task.to_dict()))
self.task_list.append(task)
else:
raise QueueError(u'队列未在初始化状态')
@locker
def extend(self, task_list):
"""
扩展任务列表，该操作不更新待做任务队列
如需更新待做任务队列，在完成该操作后调用 make_todo_task_queue
"""
self._ready = False
if self.queue_status == QueueStatus.Initiating:
self.task_list.extend(task_list)
else:
raise QueueError(u'队列未在初始化状态')
@locker
@dumper
def make_todo_task_queue(self, force=False):
"""
初始化待做任务队列
"""
if self.queue_status not in (QueueStatus.Normal,
QueueStatus.Initiating) and not force:
logging.warning('Invalid queue status.')
return False
self.todo_task_queue = Queue()
for task in self.task_list[len(self.task_result_list):]:
self.todo_task_queue.put(task)
if not force and self.todo_task_queue.empty():
logging.warning('Queue[{}] todo queue empty in status[{}]'.format(
self.queue_uuid, self.queue_status.name))
return False
elif not force:
self.queue_status = QueueStatus.Normal
return True
@locker
def peek(self, task_uuid=None):
"""
查看下一项待做任务且不从待做任务队列中取出任务
"""
if self.queue_status in (QueueStatus.Empty, QueueStatus.Done):
return self.queue_status
task = self.task_list[len(self.task_list) -
self.todo_task_queue.qsize()]
if task_uuid and task_uuid != task.task_uuid:
return None
else:
return task
@locker
@dumper
def get(self):
"""
从待做队列中取出任务
"""
if self.todo_task_queue.empty():
return None
if self.queue_status != QueueStatus.Normal:
return self.queue_status
if not self.todo_task_queue.empty(
) and self.queue_status == QueueStatus.Normal:
task = self.todo_task_queue.get()
self.queue_status = QueueStatus.JobIssued
self._condition.notifyAll()
return task
@locker
def put(self, data, idx=0):
"""
向任务队列添加单个任务或任务列表
该函数将直接更新待做任务队列
"""
if isinstance(data, Task):
if idx:
if idx >= len(self.task_result_list):
self.task_list = self.task_list[:idx] + [
data
] + self.task_list[idx:]
else:
raise Exception('Index in done task list')
else:
self.append(data)
elif isinstance(data, list) or isinstance(data, tuple):
if idx:
if idx >= len(self.task_result_list):
self.task_list = self.task_list[:idx] + list(
data) + self.task_list[idx:]
else:
raise Exception('Index in done task list')
else:
self.extend(data)
else:
raise Exception('Invalid data input')
return self.make_todo_task_queue()
@locker
@dumper
def set_run_all(self):
self.run_all = True
@locker
@dumper
def skip_failed(self, session=None):
"""
跳过队列中的失败任务
"""
if self.queue_status != QueueStatus.JobFailed:
return False
last_result = self.task_result_list[-1]
last_result.status_code = TaskStatus.Skipped
last_result.session = session
if self.todo_task_queue.empty():
self.queue_status = QueueStatus.Done
else:
self.queue_status = QueueStatus.Normal
self._condition.notifyAll()
# TaskQueueManager._notify_outside(last_result)
return True
@locker
@dumper
def resume_failed(self):
"""
将失败任务恢复至待做队列
"""
if self.queue_status == QueueStatus.JobFailed:
self.task_result_list.pop()
self.queue_status = QueueStatus.Normal
self.make_todo_task_queue(force=True)
return True
else:
return False
@locker
@dumper
def skip_task(self, session=None):
"""
跳过下一项任务
"""
if self.todo_task_queue.empty():
return False
next_task = self.get()
if isinstance(next_task, Task):
result = TaskResult(next_task.queue_uuid, next_task.task_uuid,
TaskStatus.Skipped,
MSG_DICT[TaskStatus.Skipped], session)
self.task_result_list.append(result)
# TaskQueueManager._notify_outside(result)
if self.todo_task_queue.empty():
self.queue_status = QueueStatus.Done
else:
self.queue_status = QueueStatus.Normal
return True
else:
logging.warning(next_task)
raise Exception(u'任务定义不正确')
@locker
@dumper
def update_status_by_result(self, task_result):
"""
根据任务结果更新队列状态
"""
current_task = self.task_list[len(self.task_list) -
self.todo_task_queue.qsize() - 1]
if task_result.task_uuid != current_task.task_uuid:
logging.warning(u'任务结果与当前执行任务不匹配！')
logging.warning(u'当前任务：{}'.format(
json.dumps(current_task.to_dict())))
logging.warning(u'任务结果：{}'.format(json.dumps(
task_result.to_dict())))
return False
switch = {
TaskStatus.UnKnown:
lambda: QueueStatus.NotRecoverable,
TaskStatus.Dispatched:
lambda: QueueStatus.JobDispatched,
TaskStatus.InitFailed:
lambda: QueueStatus.NotRecoverable,
TaskStatus.Runnable:
lambda: QueueStatus.JobRunning,
TaskStatus.TriggerTimeWaiting:
lambda: QueueStatus.JobWaiting,
TaskStatus.WorkerWaiting:
lambda: QueueStatus.JobWaiting,
TaskStatus.TimeRangeExcept:
lambda: QueueStatus.JobFailed,
TaskStatus.Running:
lambda: QueueStatus.JobRunning
if self.sync_group else QueueStatus.Normal,
TaskStatus.Success:
lambda: QueueStatus.Done
if self.todo_task_queue.empty() else QueueStatus.Normal,
TaskStatus.Failed:
lambda: QueueStatus.JobFailed,
TaskStatus.Timeout:
lambda: QueueStatus.JobFailed,
TaskStatus.Terminated:
lambda: QueueStatus.JobFailed,
TaskStatus.Skipped:
lambda: QueueStatus.Done
if self.todo_task_queue.empty() else QueueStatus.Normal
}
try:
self.queue_status = switch[task_result.status_code]()
except KeyError:
logging.warning('Invalid status[{}] in result: {}'.format(
task_result.task_status, task_result.to_dict()))
return False
else:
if self.queue_status == QueueStatus.JobFailed:
self.run_all = False
if self.task_result_list:
last_result = self.task_result_list[-1]
if last_result.task_uuid == task_result.task_uuid:
self.task_result_list.pop()
self.task_result_list.append(task_result)
logging.info('Queue status[{}] updated by result: {}'.format(
self.queue_status, task_result.to_dict()))
if not self.queue_status.Blocking:
self._condition.notifyAll()
return True
@locker
@dumper
def update_task_define(self, task):
"""
更新已有任务的任务定义
"""
if task.task_uuid not in map(lambda x: x.task_uuid, self.task_list):
return None
elif task.task_uuid in map(lambda x: x.task_uuid,
self.task_result_list):
if task.task_uuid == self.task_result_list[-1].task_uuid:
if self.task_result_list[-1].status_code in (
TaskStatus.Failed, TaskStatus.Skipped,
TaskStatus.Terminated, TaskStatus.Timeout,
TaskStatus.InitFailed, TaskStatus.TimeRangeExcept):
self.task_result_list.pop()
else:
return self.task_result_list[-1].status_code
else:
return None
new_list = self.task_list[:len(self.task_result_list)]
for old_task in self.task_list[len(self.task_result_list):]:
if task.task_uuid == old_task.task_uuid:
new_list.append(task)
else:
new_list.append(old_task)
self.task_list = new_list
self.make_todo_task_queue()
return task
@locker
def make_snapshot(self, compatible=True):
if compatible:
old_snap = {
"create_time":
self.create_time,
"trigger_time":
self.trigger_time,
"expire_time":
self.expire_time,
"destroy_time":
self.destroy_time,
"group_block":
self.sync_group,
"controller_queue_status":
self.queue_status.value,
"controller_queue_uuid":
self.queue_uuid,
"task_list": [{
'earliest': task.task_earliest,
'latest': task.task_latest,
'task_uuid': task.task_uuid,
'detail': task.task_info
} for task in self.task_list],
"task_result_list": [None for x in self.task_list],
"task_status_list": [None for x in self.task_list]
}
for idx in xrange(len(self.task_result_list)):
result = self.task_result_list[idx]
old_snap['task_result_list'][idx] = {
'controller_queue_uuid':
self.queue_uuid,
'task_uuid':
result.task_uuid,
'run_all':
self.run_all,
'session':
result.session,
'task_result':
result.task_result.to_dict()
if result.task_result else None,
'task_status':
[result.status_code.value, result.status_code.name]
}
old_snap['task_status_list'][idx] = (result.status_code.value,
result.session)
return old_snap
else:
return self.to_dict()
@locker
def expire(self):
''' if self.queue_status.Blocking:
def wait_and_expire(queue):
queue.wait()
queue.expire()
tr = threading.Thread(
target=wait_and_expire, args=(self, ))
tr.setDaemon(True)
tr.start()
else:
self.queue_status = QueueStatus.Expired '''
if not (self.queue_status.Blocking
or self.queue_status == QueueStatus.Done):
self.queue_status = QueueStatus.Expired
@locker
def destroy(self):
''' if self.queue_status.Blocking:
def wait_and_destroy(queue):
queue.wait()
queue.destroy()
tr = threading.Thread(
target=wait_and_destroy, args=(self, ))
tr.setDaemon(True)
tr.start()
else:
del self._cache[self.queue_uuid] '''
if not self.queue_status.Blocking:
del self._cache[self.queue_uuid]
class RpcQueue(threading.Thread):
def __init__(self, jsonrpcBackend, size, poll=0.01):
threading.Thread.__init__(self)
self.jsonrpcBackend = jsonrpcBackend
self.size = size
self.queue = Queue(size)
self.poll = poll
self.stopped = False
self.jsonrpcs = {}
self.idle = threading.Event()
def add(self, jsonrpc):
logger.debug(u'Adding jsonrpc %s to queue (current queue size: %d)' %
(jsonrpc, self.queue.qsize()))
self.queue.put(jsonrpc, block=True)
logger.debug2(u'Added jsonrpc %s to queue' % jsonrpc)
def stop(self):
self.stopped = True
def run(self):
logger.debug(u"RpcQueue started")
self.idle.set()
while not self.stopped or not self.queue.empty():
self.idle.wait()
jsonrpcs = []
while not self.queue.empty():
self.idle.clear()
try:
jsonrpc = self.queue.get(block=False)
if jsonrpc:
logger.debug(u'Got jsonrpc %s from queue' % jsonrpc)
jsonrpcs.append(jsonrpc)
if len(jsonrpcs) >= self.size:
break
except Empty:
break
if jsonrpcs:
self.process(jsonrpcs=jsonrpcs)
time.sleep(self.poll)
logger.debug(u"RpcQueue stopped (empty: %s, stopped: %s)" %
(self.queue.empty(), self.stopped))
def process(self, jsonrpcs):
self.jsonrpcs = {}
for jsonrpc in forceList(jsonrpcs):
self.jsonrpcs[jsonrpc.id] = jsonrpc
if not self.jsonrpcs:
return
logger.info("Executing bunched jsonrpcs: %s" % self.jsonrpcs)
isExit = False
try:
retry = False
baseUrl = None
rpc = []
for jsonrpc in self.jsonrpcs.values():
isExit = jsonrpc.method in ('backend_exit', 'exit')
if jsonrpc.retry:
retry = True
if not baseUrl:
baseUrl = jsonrpc.baseUrl
elif baseUrl != jsonrpc.baseUrl:
raise OpsiRpcError(
u"Can't execute jsonrpcs with different base urls at once: (%s != %s)"
% (baseUrl, jsonrpc.baseUrl))
rpc.append(jsonrpc.getRpc())
rpc = json.dumps(rpc)
logger.debug2(u"jsonrpc: %s" % rpc)
response = self.jsonrpcBackend._request(baseUrl=baseUrl,
data=rpc,
retry=retry)
logger.debug(u"Got response from host %s" %
self.jsonrpcBackend._host)
try:
response = forceList(json.loads(response))
except Exception as error:
raise OpsiRpcError(u"Failed to json decode response %s: %s" %
(response, error))
for resp in response:
try:
responseId = resp['id']
except KeyError as error:
raise KeyError(u"Failed to get id from: %s (%s): %s" %
(resp, response, error))
try:
jsonrpc = self.jsonrpcs[responseId]
except KeyError as error:
raise KeyError(u"Failed to get jsonrpc with id %s: %s" %
(responseId, error))
try:
jsonrpc.processResult(resp)
except Exception as error:
raise RuntimeError(
u"Failed to process response %s with jsonrpc %s: %s" %
(resp, jsonrpc, error))
except Exception as error:
if not isExit:
logger.logException(error)
for jsonrpc in self.jsonrpcs.values():
jsonrpc.error = error
jsonrpc._gotResult()
self.jsonrpcs = {}
self.idle.set()
except Exception, e:
print "error:%s" % e
except KeyboardInterrupt:
print "You pressed Ctrl+c"
sys.exit()
port.task_done()
file = raw_input("Enter the file to check the services:")
f = open(file, 'r')
#place ip into ip_queue
for eachline in f:
ip = eachline.strip('\n')
for port in range(1, 1025):
port_queue.put(eachline)
if port_queue.qsize() < num_threads:
num_threads = port_queue.qsize()
for i in range(num_threads):
worker = Thread(target=scan_port, args=(ip, port_queue, 5))
worker.setDaemon(True)
worker.start()
f.close()
print "Main Thread waiting"
port_queue.join()
print "Done"
ret.append(''.join(tmp))
return ret
fin = open("A-small-attempt0.in")
T = int(fin.readline())
fout = open('A_output.txt', 'w')
for t in range(1, T + 1):
S, K = fin.readline().strip().split(' ')
K = int(K)
target = '+' * len(S)
open = set()
open.add(S)
Q = Queue()
Q.put((S, 0))
res = "Case #%s: %s\n" % (t, "IMPOSSIBLE")
flag = False
while Q.qsize() > 0:
now, deep = Q.get()
if now == target:
res = "Case #%s: %s\n" % (t, deep)
flag = True
break
for nc in con(now, K):
if nc in open:
continue
open.add(nc)
Q.put((nc, deep + 1))
fout.write(res)
fin.close()
fout.close()
class SSDVUploader(object):
"""
Queued SSDV Imagery Uploader Class
Based on the Queued habitat uploader class from auto_rx.
"""
SSDV_URL = "http://ssdv.habhub.org/api/v0/packets"
def __init__(self,
uploader_callsign="N0CALL",
enable_file_watch=True,
watch_directory="./rx_images/",
file_mask="*.bin",
watch_time=5,
queue_size=8192,
upload_block_size=256,
upload_timeout=20,
upload_retries=3,
upload_anyway=10):
"""
Initialise a SSDV Uploader Object
Args:
upload_retries (int): How many times to retry an upload on a timeout before discarding.
"""
self.uploader_callsign = uploader_callsign
self.upload_block_size = upload_block_size
self.upload_timeout = upload_timeout
self.upload_retries = upload_retries
self.upload_anyway = upload_anyway
self.watch_time = watch_time
# Generate search mask.
self.search_mask = os.path.join(watch_directory, file_mask)
# Set up Queue
self.upload_queue = Queue(queue_size)
# Count of uploaded packets.
self.upload_count = 0
# Count of discarded packets due to upload failures.
self.discard_count = 0
# Start uploader and file watcher threads.
self.uploader_running = True
self.uploader_thread = Thread(target=self.uploader_loop)
self.uploader_thread.start()
if enable_file_watch:
self.file_watch_thread = Thread(target=self.file_watch_loop)
self.file_watch_thread.start()
def ssdv_encode_packet(self, packet):
''' Convert a packet to a suitable JSON blob. '''
_packet_dict = {
"type": "packet",
"packet": b64encode(packet).decode(
'ascii'
), # Note - b64encode accepts bytes objects under Python 3, and strings under Python 2.
"encoding": "base64",
# Because .isoformat() doesnt give us the right format... (boo)
"received":
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
"receiver": self.uploader_callsign,
}
return _packet_dict
def ssdv_upload_single(self, packet):
_packet_dict = self.ssdv_encode_packet(packet, callsign)
_attempts = 1
while _attempts <= self.upload_retries:
try:
_r = requests.post(self.SSDV_URL,
json=_packet_dict,
timeout=self.upload_timeout)
return True
except requests.exceptions.Timeout:
# Timeout! We can re-try.
_attempts += 1
continue
except Exception as e:
logging.error("Uploader - Error when uploading: %s" % str(e))
break
return False
def ssdv_upload_multiple(self, count):
# Sanity check that there are enough packet in the queue to upload.
if count > self.upload_queue.qsize():
count = self.upload_queue.qsize()
_encoded_array = []
for i in range(count):
_encoded_array.append(
self.ssdv_encode_packet(self.upload_queue.get()))
_packet_dict = {"type": "packets", "packets": _encoded_array}
_attempts = 1
while _attempts <= self.upload_retries:
try:
_r = requests.post(self.SSDV_URL,
json=_packet_dict,
timeout=self.upload_timeout)
logging.debug("Uploader - Successfuly uploaded %d packets." %
count)
return True
except requests.exceptions.Timeout:
# Timeout! We can re-try.
_attempts += 1
logging.debug("Uploader - Upload Timeout (attempt %d/%d)." %
(_attempts, self.upload_retries))
continue
except Exception as e:
logging.error("Uploader - Error when uploading: %s" % str(e))
return False
logging.error("Uploader - Upload timed out after %d attempts." %
_attempts)
return False
def uploader_loop(self):
logging.info("Uploader - Started uploader thread.")
_last_upload_time = time.time()
while self.uploader_running:
if self.upload_queue.qsize() >= self.upload_block_size:
if self.ssdv_upload_multiple(self.upload_block_size):
# Upload successful!
self.upload_count += self.upload_block_size
else:
# The upload has failed,
self.discard_count += self.upload_block_size
_last_upload_time = time.time()
elif (self.upload_queue.qsize() > 0) and (
(time.time() - _last_upload_time) > self.upload_anyway):
# We have some packets in the queue, and haven't uploaded in a while. Upload them.
_packet_count = self.upload_queue.qsize()
if self.ssdv_upload_multiple(_packet_count):
# Upload successful!
self.upload_count += _packet_count
else:
# The upload has failed,
self.discard_count += _packet_count
_last_upload_time = time.time()
time.sleep(1)
logging.info("Uploader - Closed uploader thread.")
def file_watch_loop(self):
logging.info("Directory Watcher - Started Directory Watcher Thread.")
_rx_images = glob.glob(self.search_mask)
_rx_images.sort()
while self.uploader_running:
# Wait a few seconds before checking for new files.
time.sleep(self.watch_time)
# Check for new files.
_folder_check = glob.glob(self.search_mask)
if len(_folder_check) == 0:
# No files in directory, continue.
continue
# Sort list. Image filenames are timestamps, so the last element in the array will be the latest image.
_folder_check.sort()
# Determine which images are new
_folder_check = set(_folder_check)
_new_images = [x for x in _folder_check if x not in _rx_images]
_new_images.sort()
for _image in _new_images:
# New file! Wait a short amount of time in case the file is still being written out.
time.sleep(0.5)
# Add it to the queue!
try:
self.add_file(_image)
except Exception as e:
logging.error(
"Directory Watcher - Error when adding image: %s" %
str(e))
# Update the list of uploaded images
_rx_images.append(_image)
time.sleep(1)
logging.info("Directory Watcher - Closed Directory Watch Thread.")
def get_queue_size(self):
""" Return the packets remaining in the queue """
return self.upload_queue.qsize()
def get_upload_count(self):
""" Return the total number of packets uploaded so far """
return self.upload_count
def get_discard_count(self):
""" Return the total number of packets uploaded so far """
return self.discard_count
def add_packet(self, data):
""" Add a single packet to the uploader queue.
If the queue is full, the packet will be immediately discarded.
Under Python 2, this function should be passed strings.
Under Python 3, it should be passed bytes objects.
"""
if len(data) == 256:
try:
self.upload_queue.put_nowait(data)
return True
except:
# Queue was full.
self.discard_count += 1
if self.discard_count % 256 == 0:
logging.warning(
"Upload Queue Full - Packets are being dropped.")
return False
def add_file(self, filename):
""" Attempt to add a file to the upload queue """
_file_size = os.path.getsize(filename)
if _file_size % 256 != 0:
logging.error(
"Directory Watcher - %s size (%d) not a multiple of 256, likely not a SSDV file."
% (filename, _file_size))
return False
_packet_count = _file_size // 256
logging.info("Directory Watcher - New file %s contains %d packets." %
(filename, _packet_count))
_packets_added = 0
_f = open(filename, 'rb')
for _i in range(_packet_count):
_packet = _f.read(256)
if self.add_packet(_packet):
_packets_added += 1
_f.close()
logging.info("Directory Watcher - Added %d packets to queue." %
_packets_added)
def close(self):
""" Stop uploader thread. """
logging.info("Shutting down threads.")
self.uploader_running = False
def spiderPlanet(only_if_new = False):
""" Spider (fetch) an entire planet """
log = planet.logger
global index
index = True
timeout = config.feed_timeout()
try:
socket.setdefaulttimeout(float(timeout))
log.info("Socket timeout set to %d seconds", timeout)
except:
try:
import timeoutsocket
timeoutsocket.setDefaultSocketTimeout(float(timeout))
log.info("Socket timeout set to %d seconds", timeout)
except:
log.warning("Timeout set to invalid value '%s', skipping", timeout)
from Queue import Queue
from threading import Thread
fetch_queue = Queue()
parse_queue = Queue()
threads = {}
http_cache = config.http_cache_directory()
# Should this be done in config?
if http_cache and not os.path.exists(http_cache):
os.makedirs(http_cache)
if int(config.spider_threads()):
# Start all the worker threads
for i in range(int(config.spider_threads())):
threads[i] = Thread(target=httpThread,
args=(i,fetch_queue, parse_queue, log))
threads[i].start()
else:
log.info("Building work queue")
# Load the fetch and parse work queues
for uri in config.subscriptions():
# read cached feed info
sources = config.cache_sources_directory()
feed_source = filename(sources, uri)
feed_info = feedparser.parse(feed_source)
if feed_info.feed and only_if_new:
log.info("Feed %s already in cache", uri)
continue
if feed_info.feed.get('planet_http_status',None) == '410':
log.info("Feed %s gone", uri)
continue
if threads and _is_http_uri(uri):
fetch_queue.put(item=(uri, feed_info))
else:
parse_queue.put(item=(uri, feed_info, uri))
# Mark the end of the fetch queue
for thread in threads.keys():
fetch_queue.put(item=(None, None))
# Process the results as they arrive
feeds_seen = {}
while fetch_queue.qsize() or parse_queue.qsize() or threads:
while parse_queue.qsize():
(uri, feed_info, feed) = parse_queue.get(False)
try:
if not hasattr(feed,'headers') or int(feed.headers.status)<300:
options = {}
if hasattr(feed_info,'feed'):
options['etag'] = \
feed_info.feed.get('planet_http_etag',None)
try:
modified=time.strptime(
feed_info.feed.get('planet_http_last_modified',
None))
except:
pass
data = feedparser.parse(feed, **options)
else:
data = feedparser.FeedParserDict({'version': None,
'headers': feed.headers, 'entries': [], 'feed': {},
'href': feed.url, 'bozo': 0,
'status': int(feed.headers.status)})
# duplicate feed?
id = data.feed.get('id', None)
if not id: id = feed_info.feed.get('id', None)
href=uri
if data.has_key('href'): href=data.href
duplicate = None
if id and id in feeds_seen:
duplicate = id
elif href and href in feeds_seen:
duplicate = href
if duplicate:
feed_info.feed['planet_message'] = \
'duplicate subscription: ' + feeds_seen[duplicate]
log.warn('Duplicate subscription: %s and %s' %
(uri, feeds_seen[duplicate]))
if href: feed_info.feed['planet_http_location'] = href
if id: feeds_seen[id] = uri
if href: feeds_seen[href] = uri
# complete processing for the feed
writeCache(uri, feed_info, data)
except Exception, e:
import sys, traceback
type, value, tb = sys.exc_info()
log.error('Error processing %s', uri)
for line in (traceback.format_exception_only(type, value) +
traceback.format_tb(tb)):
log.error(line.rstrip())
time.sleep(0.1)
for index in threads.keys():
if not threads[index].isAlive():
del threads[index]
if not threads:
log.info("Finished threaded part of processing.")
timestamp = time.time()
dircount = 0
filemeta = []
for root, dirs, files in walk(ROOTDIR_LOCAL):
dircount += 1
totaldirs += 1
if os.path.basename(root) in EXCLUDED_DIRS:
del dirs[:]
del files[:]
continue
for f in files:
q_spider.put(os.path.join(root, f))
q_spider.join()
while q_spider_meta.qsize() > 0:
item = q_spider_meta.get()
filemeta.append(item)
q_spider_meta.task_done()
mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime = os.lstat(
root)
root = root.replace(ROOTDIR_LOCAL, ROOTDIR_REMOTE)
packet.append(
((root, (mode, ino, dev, nlink, uid, gid, size, atime,
mtime, ctime)), dirs[:], filemeta[:]))
if len(packet) >= BATCH_SIZE:
q.put(pickle.dumps(packet))
del packet[:]
class HabitatUploader(object):
'''
Queued Habitat Telemetry Uploader class
This performs uploads to the Habitat servers, and also handles generation of flight documents.
Incoming telemetry packets are fed into queue, which is checked regularly.
If a new callsign is sighted, a payload document is created in the Habitat DB.
The telemetry data is then converted into a UKHAS-compatible format, before being added to queue to be
uploaded as network speed permits.
If an upload attempt times out, the packet is discarded.
If the queue fills up (probably indicating no network connection, and a fast packet downlink rate),
it is immediately emptied, to avoid upload of out-of-date packets.
Note that this uploader object is intended to handle telemetry from multiple sondes
'''
# We require the following fields to be present in the incoming telemetry dictionary data
REQUIRED_FIELDS = [
'frame', 'id', 'datetime', 'lat', 'lon', 'alt', 'temp', 'type', 'freq',
'freq_float', 'datetime_dt'
]
def __init__(self,
user_callsign='N0CALL',
station_position=(0.0, 0.0, 0.0),
user_antenna="",
payload_callsign_override=None,
synchronous_upload_time=30,
callsign_validity_threshold=5,
upload_queue_size=16,
upload_timeout=10,
upload_retries=5,
upload_retry_interval=0.25,
user_position_update_rate=6,
inhibit=False,
url="http://habitat.sondehub.org/"):
""" Initialise a Habitat Uploader object.
Args:
user_callsign (str): Callsign of the uploader.
station_position (tuple): Optional - a tuple consisting of (lat, lon, alt), which if populated,
is used to plot the listener's position on the Habitat map, both when this class is initialised, and
when a new sonde ID is observed.
payload_callsign_override (str): Override the payload callsign in the uploaded sentence with this value.
WARNING: This will horribly break the tracker map if multiple sondes are uploaded under the same callsign.
USE WITH CAUTION!!!
synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0
This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously,
and also acts as decimation on the number of sentences uploaded to Habitat.
callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5
upload_queue_size (int): Maximum umber of sentences to keep in the upload queue. If the queue is filled,
it will be emptied (discarding the queue contents).
upload_timeout (int): Timeout (Seconds) when performing uploads to Habitat. Default: 10 seconds.
upload_retries (int): Retry an upload up to this many times. Default: 5
upload_retry_interval (int): Time interval between upload retries. Default: 0.25 seconds.
user_position_update_rate (int): Time interval between automatic station position updates, hours.
Set to 6 hours by default, updating any more often than this is not really useful.
inhibit (bool): Inhibit all uploads. Mainly intended for debugging.
"""
self.user_callsign = user_callsign
self.station_position = station_position
self.user_antenna = user_antenna
self.payload_callsign_override = payload_callsign_override
self.upload_timeout = upload_timeout
self.upload_retries = upload_retries
self.upload_retry_interval = upload_retry_interval
self.upload_queue_size = upload_queue_size
self.synchronous_upload_time = synchronous_upload_time
self.callsign_validity_threshold = callsign_validity_threshold
self.inhibit = inhibit
self.user_position_update_rate = user_position_update_rate
# set the habitat upload url
global url_habitat_uuids, url_habitat_db, habitat_url
url_habitat_uuids = url + "_uuids?count=%d"
url_habitat_db = url + "habitat/"
habitat_url = url
# Our two Queues - one to hold sentences to be upload, the other to temporarily hold
# input telemetry dictionaries before they are converted and processed.
self.habitat_upload_queue = Queue(upload_queue_size)
self.input_queue = Queue()
# Dictionary where we store sorted telemetry data for upload when required.
# Elements will be named after payload IDs, and will contain:
# 'count' (int): Number of times this callsign has been observed. Uploads will only occur when
# this number rises above callsign_validity_threshold.
# 'data' (Queue): A queue of telemetry sentences to be uploaded. When the upload timer fires,
# this queue will be dumped, and the most recent telemetry uploaded.
# 'habitat_document' (bool): Indicates if a habitat document has been created for this payload ID.
# 'listener_updated' (bool): Indicates if the listener position has been updated for the start of this ID's flight.
self.observed_payloads = {}
# Record of when we last uploaded a user station position to Habitat.
self.last_user_position_upload = 0
# Lock for dealing with telemetry uploads.
self.upload_lock = Lock()
# Start the uploader thread.
self.upload_thread_running = True
self.upload_thread = Thread(target=self.habitat_upload_thread)
self.upload_thread.start()
# Start the input queue processing thread.
self.input_processing_running = True
self.input_thread = Thread(target=self.process_queue)
self.input_thread.start()
self.timer_thread_running = True
self.timer_thread = Thread(target=self.upload_timer)
self.timer_thread.start()
def user_position_upload(self):
""" Upload the the station position to Habitat. """
if self.station_position == None:
self.last_user_position_upload = time.time()
return False
if (self.station_position[0] != 0.0) or (self.station_position[1] !=
0.0):
_success = uploadListenerPosition(self.user_callsign,
self.station_position[0],
self.station_position[1],
version=auto_rx_version,
antenna=self.user_antenna)
self.last_user_position_upload = time.time()
return _success
else:
return False
def habitat_upload(self, sentence):
''' Upload a UKHAS-standard telemetry sentence to Habitat
Args:
sentence (str): The UKHAS-standard telemetry sentence to upload.
'''
if self.inhibit:
self.log_info("Upload inhibited.")
return
# Generate payload to be uploaded
_sentence_b64 = b64encode(sentence.encode(
'ascii')) # Encode to ASCII to be able to perform B64 encoding...
_date = datetime.datetime.utcnow().isoformat("T") + "Z"
_user_call = self.user_callsign
_data = {
"type": "payload_telemetry",
"data": {
"_raw": _sentence_b64.decode(
'ascii'
) # ... but decode back to a string to enable JSON serialisation.
},
"receivers": {
_user_call: {
"time_created": _date,
"time_uploaded": _date,
},
},
}
# The URL to upload to.
_url = habitat_url + "habitat/_design/payload_telemetry/_update/add_listener/%s" % sha256(
_sentence_b64).hexdigest()
# Delay for a random amount of time between 0 and upload_retry_interval*2 seconds.
time.sleep(random.random() * self.upload_retry_interval * 2.0)
_retries = 0
# When uploading, we have three possible outcomes:
# - Can't connect. No point immediately re-trying in this situation.
# - The packet is uploaded successfuly (201 / 403)
# - There is a upload conflict on the Habitat DB end (409). We can retry and it might work.
while _retries < self.upload_retries:
# Run the request.
try:
headers = {"User-Agent": "autorx-" + auto_rx_version}
_req = requests.put(_url,
data=json.dumps(_data),
timeout=self.upload_timeout,
headers=headers)
except Exception as e:
self.log_error("Upload Failed: %s" % str(e))
break
if _req.status_code == 201 or _req.status_code == 403:
# 201 = Success, 403 = Success, sentence has already seen by others.
self.log_info("Uploaded sentence to Habitat successfully: %s" %
sentence.strip())
_upload_success = True
break
elif _req.status_code == 409:
# 409 = Upload conflict (server busy). Sleep for a moment, then retry.
self.log_debug("Upload conflict.. retrying.")
time.sleep(random.random() * self.upload_retry_interval)
_retries += 1
else:
self.log_error(
"Error uploading to Habitat. Status Code: %d %s." %
(_req.status_code, _req.text))
break
if _retries == self.upload_retries:
self.log_error("Upload conflict not resolved with %d retries." %
self.upload_retries)
return
def habitat_upload_thread(self):
''' Handle uploading of packets to Habitat '''
self.log_debug("Started Habitat Uploader Thread.")
while self.upload_thread_running:
if self.habitat_upload_queue.qsize() > 0:
# If the queue is completely full, jump to the most recent telemetry sentence.
if self.habitat_upload_queue.qsize() == self.upload_queue_size:
while not self.habitat_upload_queue.empty():
sentence = self.habitat_upload_queue.get()
self.log_warning(
"Uploader queue was full - possible connectivity issue."
)
else:
# Otherwise, get the first item in the queue.
sentence = self.habitat_upload_queue.get()
# Attempt to upload it.
self.habitat_upload(sentence)
else:
# Wait for a short time before checking the queue again.
time.sleep(0.1)
self.log_debug("Stopped Habitat Uploader Thread.")
def handle_telem_dict(self, telem, immediate=False):
# Try and convert it to a UKHAS sentence
try:
_sentence = sonde_telemetry_to_sentence(telem)
except Exception as e:
self.log_error("Error converting telemetry to sentence - %s" %
str(e))
return
_callsign = "RS_" + telem['id']
# Wait for the upload_lock to be available, to ensure we don't end up with
# race conditions resulting in multiple payload docs being created.
self.upload_lock.acquire()
# Create a habitat document if one does not already exist:
if not self.observed_payloads[telem['id']]['habitat_document']:
# Check if there has already been telemetry from this ID observed on Habhub
_document_exists = check_callsign(_callsign)
# If so, we don't need to create a new document
if _document_exists:
self.observed_payloads[telem['id']]['habitat_document'] = True
else:
# Otherwise, we attempt to create a new document.
if self.inhibit:
# If we have an upload inhibit, don't create a payload doc.
_created = True
else:
_created = initPayloadDoc(
_callsign,
description="Meteorology Radiosonde",
frequency=telem['freq_float'])
if _created:
self.observed_payloads[
telem['id']]['habitat_document'] = True
else:
self.log_error("Error creating payload document!")
self.upload_lock.release()
return
if immediate:
self.log_info(
"Performing immediate upload for first telemetry sentence of %s."
% telem['id'])
self.habitat_upload(_sentence)
else:
# Attept to add it to the habitat uploader queue.
try:
self.habitat_upload_queue.put_nowait(_sentence)
except Exception as e:
self.log_error("Error adding sentence to queue: %s" % str(e))
self.upload_lock.release()
def upload_timer(self):
""" Add packets to the habitat upload queue if it is time for us to upload. """
while self.timer_thread_running:
if int(time.time()) % self.synchronous_upload_time == 0:
# Time to upload!
for _id in self.observed_payloads.keys():
# If no data, continue...
if self.observed_payloads[_id]['data'].empty():
continue
else:
# Otherwise, dump the queue and keep the latest telemetry.
while not self.observed_payloads[_id]['data'].empty():
_telem = self.observed_payloads[_id]['data'].get()
self.handle_telem_dict(_telem)
# Sleep a second so we don't hit the synchronous upload time again.
time.sleep(1)
else:
# Not yet time to upload, wait for a bit.
time.sleep(0.1)
def process_queue(self):
""" Process packets from the input queue.
This thread handles packets from the input queue (provided by the decoders)
Packets are sorted by ID, and a dictionary entry is created.
"""
while self.input_processing_running:
# Process everything in the queue.
while self.input_queue.qsize() > 0:
# Grab latest telem dictionary.
_telem = self.input_queue.get_nowait()
_id = _telem['id']
if _id not in self.observed_payloads:
# We haven't seen this ID before, so create a new dictionary entry for it.
self.observed_payloads[_id] = {
'count': 1,
'data': Queue(),
'habitat_document': False,
'listener_updated': False
}
self.log_debug(
"New Payload %s. Not observed enough to allow upload."
% _id)
# However, we don't yet add anything to the queue for this payload...
else:
# We have seen this payload before!
# Increment the 'seen' counter.
self.observed_payloads[_id]['count'] += 1
# If we have seen this particular ID enough times, add the data to the ID's queue.
if self.observed_payloads[_id][
'count'] >= self.callsign_validity_threshold:
# If this is the first time we have observed this payload, update the listener position.
if (self.observed_payloads[_id]['listener_updated']
== False) and (self.station_position
is not None):
self.observed_payloads[_id][
'listener_updated'] = self.user_position_upload(
)
# Because receiving balloon telemetry appears to be a competition, immediately upload the
# first valid position received.
self.handle_telem_dict(_telem, immediate=True)
else:
# Otherwise, add the telemetry to the upload queue
self.observed_payloads[_id]['data'].put(_telem)
else:
self.log_debug(
"Payload ID %s not observed enough to allow upload."
% _id)
# If we haven't uploaded our station position recently, re-upload it.
if (time.time() - self.last_user_position_upload
) > self.user_position_update_rate * 3600:
self.user_position_upload()
time.sleep(0.1)
def add(self, telemetry):
""" Add a dictionary of telemetry to the input queue.
Args:
telemetry (dict): Telemetry dictionary to add to the input queue.
"""
# Discard any telemetry which is indicated to be encrypted.
if 'encrypted' in telemetry:
if telemetry['encrypted'] == True:
return
# Check the telemetry dictionary contains the required fields.
for _field in self.REQUIRED_FIELDS:
if _field not in telemetry:
self.log_error("JSON object missing required field %s" %
_field)
return
# Add it to the queue if we are running.
if self.input_processing_running:
self.input_queue.put(telemetry)
else:
self.log_error("Processing not running, discarding.")
def update_station_position(self, lat, lon, alt):
""" Update the internal station position record. Used when determining the station position by GPSD """
self.station_position = (lat, lon, alt)
def close(self):
''' Shutdown uploader and processing threads. '''
self.log_debug("Waiting for threads to close...")
self.input_processing_running = False
self.timer_thread_running = False
self.upload_thread_running = False
# Wait for all threads to close.
if self.upload_thread is not None:
self.upload_thread.join()
if self.timer_thread is not None:
self.timer_thread.join()
if self.input_thread is not None:
self.input_thread.join()
def log_debug(self, line):
""" Helper function to log a debug message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.debug("Habitat - %s" % line)
def log_info(self, line):
""" Helper function to log an informational message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.info("Habitat - %s" % line)
def log_error(self, line):
""" Helper function to log an error message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.error("Habitat - %s" % line)
def log_warning(self, line):
""" Helper function to log a warning message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.warning("Habitat - %s" % line)
def train(self, sentences, total_words=None, word_count=0, chunksize=100):
"""
Update the model's neural weights from a sequence of sentences (can be a once-only generator stream).
Each sentence must be a list of utf8 strings.
更新词向量权重
每输入一个句子，表示一次迭代更新。
`sentences` 输入的用于训练的句子集合
`total_words` 总词频和
`word_count` 词种类数
`chunksize` 多线程的时候，每个线程一次性处理或被分配到的句子数
"""
logger.info("training model with %i workers on %i vocabulary and %i features" % (self.workers, len(self.vocab), self.layer1_size))
if not self.vocab:
raise RuntimeError("you must first build vocabulary before training the model")
self.neg_labels = []
if self.negative >= 1:
# 负样本标签
self.neg_labels = np.zeros(self.negative + 1)
self.neg_labels[0] = 1.
# 用来记录时间的起始与打log的判定时间，
# `next_report`存到list是因为需要在多个线程内作为类对象被访问，`word_count`同理
start, next_report = time.time(), [1.0]
word_count, total_words = [word_count], total_words or sum(v.count for v in self.vocab.itervalues())
# 考虑缓冲区
jobs = Queue(maxsize=2 * self.workers)
# 因为有共享变量（如`word_count`），所以加锁
lock = threading.Lock()
def worker_train():
"""Train the model, lifting lists of sentences from the jobs queue."""
# 多线程配给的memory
work = zeros_aligned(self.layer1_size, dtype=REAL)
while True:
# 一个job就是一个chunksize条句子的数据集
job = jobs.get()
if job is None: # 数据读完，退出
break
# 在开始训练之前先减小训练速率
alpha = max(self.min_alpha, self.alpha * (1 - 1.0 * word_count[0] / total_words / self.epochs))
# 训练参数；统计当前job训练的词数，OOV的词不算
if self.sg >= 1:
func = train_sentence_sg
else:
func = train_sentence_cbow
job_words, total_loss = np.sum([func(self, sentence, alpha, work) for sentence in job], axis=0)
total_loss /= len(job)
# 请求锁，用来更新`word_count`
with lock:
word_count[0] += job_words
elapsed = time.time() - start
# 防止一直打log
if elapsed >= next_report[0]:
logger.info("PROGRESS: at %.2f%% words, alpha %.05f, %.0f words/s, loss %.08f" %
(100.0 * word_count[0] / total_words / self.epochs, alpha, word_count[0] / elapsed if elapsed else 0.0, total_loss))
# 可以让log保持一秒及一秒以上打一次
next_report[0] = elapsed + 1.0
workers = [threading.Thread(target=worker_train) for _ in xrange(self.workers)]
for thread in workers:
thread.daemon = True # 可以更便捷的用ctrl+c中断程序
thread.start()
# 把输入的string变成Vocab类，对于OOV则用None表示，并把数据拆分成多个job，存到queue里
no_oov = ([self.vocab.get(word, None) for word in sentence] for sentence in sentences)
for job_no, job in enumerate(grouper(no_oov, chunksize)):
logger.debug("putting job #%i in the queue, qsize=%i" % (job_no, jobs.qsize()))
jobs.put(job)
logger.info("reached the end of input; waiting to finish %i outstanding jobs" % jobs.qsize())
# 再补充`self.workers`个jobs，用来告知线程数据读取完成
for _ in xrange(self.workers):
jobs.put(None)
for thread in workers:
thread.join()
elapsed = time.time() - start
logger.info("training on %i words took %.1fs, %.0f words/s" %
(word_count[0], elapsed, word_count[0] / elapsed if elapsed else 0.0))
return word_count[0]
class Fetcher:
def __init__(self, threads):
self.lock = Lock() #线程锁
self.q_req = Queue() #任务队列
self.q_ans = Queue() #完成队列
self.threads = threads
for i in range(threads):
t = Thread(target=self.threadget) #括号中的是每次线程要执行的任务
t.setDaemon(True) #设置子线程是否随主线程一起结束，必须在start()
#之前调用。默认为False
t.start() #启动线程
self.running = 0 #设置运行中的线程个数
def __del__(self): #解构时需等待两个队列完成
time.sleep(0.5)
self.q_req.join() #Queue等待队列为空后再执行其他操作
self.q_ans.join()
#返回还在运行线程的个数，为0时表示全部运行完毕
def taskleft(self):
return self.q_req.qsize() + self.q_ans.qsize() + self.running
def push(self, req):
self.q_req.put(req)
def pop(self):
return self.q_ans.get()
#线程执行的任务，根据req来区分
def threadget(self):
while True:
line = self.q_req.get()
word = line.strip()
'''
Lock.lock()操作，使用with可以不用显示调用acquire和release，
这里锁住线程，使得self.running加1表示运行中的线程加1，
如此做防止其他线程修改该值，造成混乱。
with下的语句结束后自动解锁。
'''
with self.lock:
self.running += 1
'''构造请求头，header请自行修改'''
headers = [
"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding:gzip, deflate, sdch",
"Accept-Language:zh-CN,zh;q=0.8,en;q=0.6",
"Cache-Control:no-cache",
"Connection:keep-alive",
"Host:%s" % word,
"Pragma:no-cache",
"Upgrade-Insecure-Requests:1",
"User-Agent:%s" % getUA(),
]
code = is_index(word, headers)
if '错误信息' in str(code):
print word
csvfile.write('%s\n' % word)
#self.q_ans.put((req,ans)) # 将完成的任务压入完成队列，在主程序中返回
self.q_ans.put(line)
with self.lock:
self.running -= 1
self.q_req.task_done() # 在完成一项工作之后，Queue.task_done()
# 函数向任务已经完成的队列发送一个信号
time.sleep(0.1) # don't spam
except:
pass
else:
try:
results = res.json()
except:
print results
pass
else:
if res.status_code != 200:
print "error occurred: %s" % results["error"]
sys.exit(1)
else:
result_iter = iter(results["results"])
for result in result_iter:
queue.put("http://%s" % result["ip"])
def test():
for i in range(thread_num):
t = testTarget()
t.start()
if __name__ == '__main__':
getIp(cur_page)
if not queue.empty():
test()
while queue.qsize() > 0:
if cur_page <= PAGES:
getIp(cur_page)
cur_page += 1
time.sleep(0.1)
class QueMail(Thread):
instance = None
def init(self,
smtp_host,
smtp_login,
smtp_pswd,
smtp_port=25,
queue_size=100):
self._queue = Queue(queue_size)
log.info(
"Initializing QueMail with queue size %i. Using SMTP server: %s:%i."
% (queue_size, smtp_host, smtp_port))
self.smtp_host = smtp_host
self.smtp_login = smtp_login
self.smtp_password = smtp_pswd
self.smtp_port = smtp_port
def __init__(self):
Thread.__init__(self)
self._do_quit = False
self.setName("QueMail")
self.smtp_host = None
self.smtp_login = None
self.smtp_password = None
self.smtp_port = None
self.check_interval = 5 # the number of seconds to check the queue
def end(self):
'''
Waits until all emails will be sent and after that stops thread
'''
log.info("Stopping QueMail thread...")
self._do_quit = True
self.join()
log.info("Stopped.")
def run(self):
while not self._do_quit:
if not self._queue.empty():
log.debug(u"Connecting to SMTP server: %s:%i" %
(self.smtp_host, self.smtp_port))
smtp = None
try:
smtp = smtplib.SMTP()
smtp.connect(self.smtp_host, self.smtp_port)
smtp.login(self.smtp_login, self.smtp_password)
while not self._queue.empty():
t = time.time()
eml = self._queue.get()
log.info(u"Sending (qs=%i): %s" %
(self._queue.qsize(), eml))
try:
msg = eml.as_rfc_message()
content = msg.as_string()
log.debug(u"with content: %s" % content)
smtp.sendmail(eml.adr_from, eml.adr_to, content)
log.warning(
u"Sent (qs=%i,t=%f): %s" %
(self._queue.qsize(), time.time() - t, eml))
except Exception as e:
log.error(
u"Exception occured while sending email: %s" %
eml)
log.exception(e)
# FIXME not good idea: when exception occured, add email at end of queue
self._queue.put(eml, False)
sleep(1)
except Exception as e:
log.exception(e)
finally:
if smtp:
smtp.quit()
sleep(self.check_interval)
def send(self, eml):
self._queue.put(eml, True, 5)
log.debug(u'Accepted (qs=%i): %s' % (self._queue.qsize(), eml))
@classmethod
def get_instance(cls):
if not cls.instance:
cls.instance = QueMail()
return cls.instance
class Dispatcher(object):
"""
Dispatcher object that communicates and coordinates individual workers.
There should never be more than one dispatcher running at any one time.
"""
def __init__(self, maxsize=0):
"""
Note that the constructor does not fully initialize the dispatcher;
use the `initialize()` function to populate it with workers etc.
"""
self.maxsize = maxsize
self.workers = {}
self.callback = None # a pyro proxy to this object (unknown at init time, but will be set later)
def initialize(self, **model_params):
"""
`model_params` are parameters used to initialize individual workers (gets
handed all the way down to worker.initialize()).
"""
self.jobs = Queue(maxsize=self.maxsize)
self.lock_update = threading.Lock()
self._jobsdone = 0
self._jobsreceived = 0
# locate all available workers and store their proxies, for subsequent RMI calls
self.workers = {}
with utils.getNS() as ns:
self.callback = Pyro4.Proxy(
'PYRONAME:gensim.lsi_dispatcher') # = self
for name, uri in iteritems(ns.list(prefix='gensim.lsi_worker')):
try:
worker = Pyro4.Proxy(uri)
workerid = len(self.workers)
# make time consuming methods work asynchronously
logger.info("registering worker #%i from %s" %
(workerid, uri))
worker.initialize(workerid,
dispatcher=self.callback,
**model_params)
self.workers[workerid] = worker
except Pyro4.errors.PyroError:
logger.exception(
"unresponsive worker at %s, deleting it from the name server"
% uri)
ns.remove(name)
if not self.workers:
raise RuntimeError(
'no workers found; run some lsi_worker scripts on your machines first!'
)
def getworkers(self):
"""
Return pyro URIs of all registered workers.
"""
return [worker._pyroUri for worker in itervalues(self.workers)]
def getjob(self, worker_id):
logger.info("worker #%i requesting a new job" % worker_id)
job = self.jobs.get(block=True, timeout=1)
logger.info("worker #%i got a new job (%i left)" %
(worker_id, self.jobs.qsize()))
return job
def putjob(self, job):
self._jobsreceived += 1
self.jobs.put(job, block=True, timeout=HUGE_TIMEOUT)
logger.info("added a new job (len(queue)=%i items)" %
self.jobs.qsize())
def getstate(self):
"""
Merge projections from across all workers and return the final projection.
"""
logger.info("end of input, assigning all remaining jobs")
logger.debug("jobs done: %s, jobs received: %s" %
(self._jobsdone, self._jobsreceived))
while self._jobsdone < self._jobsreceived:
time.sleep(0.5) # check every half a second
# TODO: merge in parallel, so that we're done in `log_2(workers)` merges,
# and not `workers - 1` merges!
# but merging only takes place once, after all input data has been processed,
# so the overall effect would be small... compared to the amount of coding :-)
logger.info("merging states from %i workers" % len(self.workers))
workers = list(self.workers.items())
result = workers[0][1].getstate()
for workerid, worker in workers[1:]:
logger.info("pulling state from worker %s" % workerid)
result.merge(worker.getstate())
logger.info("sending out merged projection")
return result
def reset(self):
"""
Initialize all workers for a new decomposition.
"""
for workerid, worker in iteritems(self.workers):
logger.info("resetting worker %s" % workerid)
worker.reset()
worker.requestjob()
self._jobsdone = 0
self._jobsreceived = 0
@Pyro4.oneway
@utils.synchronous('lock_update')
def jobdone(self, workerid):
"""
A worker has finished its job. Log this event and then asynchronously
transfer control back to the worker.
In this way, control flow basically oscillates between dispatcher.jobdone()
worker.requestjob().
"""
self._jobsdone += 1
logger.info("worker #%s finished job #%i" % (workerid, self._jobsdone))
worker = self.workers[workerid]
worker.requestjob(
) # tell the worker to ask for another job, asynchronously (one-way)
def jobsdone(self):
"""Wrap self._jobsdone, needed for remote access through proxies"""
return self._jobsdone
@Pyro4.oneway
def exit(self):
"""
Terminate all registered workers and then the dispatcher.
"""
for workerid, worker in iteritems(self.workers):
logger.info("terminating worker %s" % workerid)
worker.exit()
logger.info("terminating dispatcher")
os._exit(
0) # exit the whole process (not just this thread ala sys.exit())
class CommandDispatcher(object):
#this class contains a queue where request
def __init__(self, vbaware, verbose=False):
#have a queue , in case of not my vbucket error
#let's reinitialize the config/memcached socket connections ?
self.queue = Queue(10000)
self.status = "initialized"
self.vbaware = vbaware
self.reconfig_callback = self.vbaware.reconfig_vbucket_map
self.start_connection_callback = self.vbaware.start_vbucket_connection
self.restart_connection_callback =\
self.vbaware.restart_vbucket_connection
self.verbose = verbose
self.log = logger("CommandDispatcher")
self._dispatcher_stopped_event = Event()
def put(self, item):
try:
self.queue.put(item, False)
except Full:
#TODO: add a better error message here
raise Exception("queue is full")
def shutdown(self):
if self.status != "shutdown":
self.status = "shutdown"
if self.verbose:
self.log.info("dispatcher shutdown command received")
self._dispatcher_stopped_event.wait(2)
def reconfig_completed(self):
self.status = "ok"
def dispatch(self):
while (self.status != "shutdown" or (self.status == "shutdown" and
self.queue.qsize() > 0)):
#wait if its reconfiguring the vbucket-map
if self.status == "vbucketmap-configuration":
continue
try:
item = self.queue.get(block=True, timeout=5)
if item:
try:
self.do(item)
# do will only raise not_my_vbucket_exception,
# EOF and socket.error
except MemcachedError, ex:
# if we get a not_my_vbucket then requeue item
# with fast forward map vbucket
self.log.error(ex)
if 'vbucket' in ex:
self.reconfig_callback(ex.vbucket)
self.start_connection_callback(ex.vbucket)
else:
raise Empty
item["fastforward"] = True
self.queue.put(item)
except EOFError, ex:
# we go an EOF error, restart the connection
self.log.error(ex)
if 'vbucket' in ex:
self.restart_connection_callback(ex.vbucket)
else:
raise Empty
self.queue.put(item)
except socket.error, ex:
# we got a socket error, restart the connection
self.log.error(ex)
if 'vbucket' in ex:
self.restart_connection_callback(ex.vbucket)
else:
raise Empty
self.queue.put(item)
class NFQ:
##
# @param in_size number of encoded features
# @param out_size = number of possible actions
def __init__(self,
in_size,
out_size,
learning_rate=0.75,
model_path=None,
weights_path=None):
assert in_size > 1
assert type(in_size) is int
assert out_size > 1
assert type(out_size) is int
self.in_size = in_size
self.out_size = out_size
self.gamma = learning_rate
if model_path is None:
self.model = Sequential()
self.model.add(Dense(64, input_dim=in_size, init='lecun_uniform'))
self.model.add(Activation('relu'))
# self.model.add(BatchNormalization())
# self.model.add(Dropout(0.2))
self.model.add(Dense(40, init='lecun_uniform'))
self.model.add(Activation('relu'))
# self.model.add(Dropout(0.2))
self.model.add(Dense(out_size, init='lecun_uniform'))
self.model.add(Activation('linear'))
else:
assert weights_path is not None
self.model = model_from_json(open(model_path).read())
self.model.load_weights(weights_path)
self.model.compile(
loss='mse', # maybe binary_crossentrpy?
optimizer='rmsprop')
self.transitions = Queue(25000)
##
# Calculate target Q-fun values and train a NN on it
def train(self):
queue_size = self.transitions.qsize()
np_data = list(self.transitions.queue)
np_data = np.array(np_data)
r = np.random.randint(queue_size - 1, size=3000)
np_data = np_data[r, :]
# trim the input data of neural net because it also contains
# unwanted state' and reward information => need it only for target Q
in_data = np.delete(np_data, np.s_[self.in_size::], 1)
out_data = self.get_training_data(np_data)
# stop_cb = EarlyStopping(monitor='val_loss', patience=0, verbose=0, mode='auto')
print 'Learning...'
hist = self.model.fit(in_data,
out_data,
nb_epoch=500,
batch_size=256,
verbose=0,
validation_split=0.4)
print('Loss: from ', hist.history['loss'][0], ' to ',
hist.history['loss'][-1])
# callbacks=[stop_cb]
##
# Predicts next action, which is the one maximizing the Q function in current state
def predict_action(self, state):
q = self.model.predict(state)
return np.argmax(q)
##
# Process training data into correct format (transitions)
def get_training_data(self, data):
out_data = list()
for row in data:
reward = row[-1]
selected_action = row[self.in_size]
next_state = row[self.in_size + 1:-1]
next_state = next_state.reshape(1, next_state.size)
predicted_Q = self.model.predict(next_state)
maxQ = np.max(predicted_Q)
minQ = np.min(predicted_Q)
out = np.zeros((self.out_size, ))
if reward >= 1:
out[int(selected_action)] = 1
elif reward < 0:
out[int(selected_action)] = -1
else:
out[int(selected_action)] = reward + self.gamma * maxQ
for i in xrange(self.out_size):
if i != int(selected_action):
out[i] = minQ
out_data.append(out)
return np.array(out_data)
##
# Add transition to training dataset
# transition has form [st, a, st+1, r]
def add_transition(self, transition):
# transition must be a numpy array
assert type(transition) is np.ndarray
if self.transitions.full():
self.transitions.get()
self.transitions.put(transition)
##
# Save NN to external file
# @param model_path File containing NN architecture
# @param weights_path File containing NN weights
def save_net(self, model_path=None, weights_path=None):
if model_path is None:
model_path = 'model.json'
if weights_path is None:
weights_path = 'weights.h5'
# remove old file if exists
try:
os.remove(weights_path)
except OSError:
pass
try:
self.model.save_weights(weights_path)
open(model_path, 'w').write(self.model.to_json())
except Exception as e:
print('Saving a model failed')
print(type(e))
print(e)
class EventEngine:
def __init__(self,account):
"""初始化事件引擎"""
self.__account = account
# 事件队列
self.__queue = Queue()
# 事件引擎开关
self.__active = False
# 事件处理线程
self.__thread = Thread(target = self.__run)
self.__handlers = {}
#----------------------------------------------------------------------
def __run(self):
"""引擎运行"""
while self.__active == True:
try:
event = self.__queue.get(block = True, timeout = 5) # 获取事件的阻塞时间设为5秒
self.__process(event)
except Empty:
pass
#----------------------------------------------------------------------
def __process(self, event):
"""处理事件"""
# 检查是否存在对该事件进行监听的处理函数
if event.type_ in self.__handlers:
#若存在，则按顺序将事件传递给处理函数执行
[handler(event) for handler in self.__handlers[event.type_]]
def start(self):
"""引擎启动"""
# 将引擎设为启动
self.__active = True
# 启动事件处理线程
self.__thread.start()
# 启动计时器，计时器事件间隔默认设定为1秒
# self.__timer.start(100)
#----------------------------------------------------------------------
def stop(self):
"""停止引擎"""
# 将引擎设为停止
self.__active = False
# 等待事件处理线程退出
self.__thread.join()
#----------------------------------------------------------------------
def register(self, type_, handler):
"""注册事件处理函数监听"""
# 尝试获取该事件类型对应的处理函数列表，若无则创建
try:
handlerList = self.__handlers[type_]
except KeyError:
handlerList = []
self.__handlers[type_] = handlerList
# 若要注册的处理器不在该事件的处理器列表中，则注册该事件
if handler not in handlerList:
handlerList.append(handler)
#----------------------------------------------------------------------
def unregister(self, type_, handler):
"""注销事件处理函数监听"""
# 尝试获取该事件类型对应的处理函数列表，若无则忽略该次注销请求
try:
handlerList = self.__handlers[type_]
# 如果该函数存在于列表中，则移除
if handler in handlerList:
handlerList.remove(handler)
# 如果函数列表为空，则从引擎中移除该事件类型
if not handlerList:
del self.__handlers[type_]
except KeyError:
pass
#----------------------------------------------------------------------
def put(self, event):
"""向事件队列中存入事件"""
event.dict_['_type_'] = event.type_
event.dict_['_qsize_'] = self.__queue.qsize()
event.dict_['_account_'] = self.__account.get('userid','NONE_USERID')
self.__queue.put(event)
class WorkerPool(object):
"""TODO:"""
halt_command = 'halt command'
def __init__(self, numWorkers=16, items=None, daemonize=False, logger=gplog.get_default_logger()):
if numWorkers <= 0:
raise Exception("WorkerPool(): numWorkers should be greater than 0.")
self.workers = []
self.should_stop = False
self.work_queue = Queue()
self.completed_queue = Queue()
self._assigned = 0
self.daemonize = daemonize
self.logger = logger
if items is not None:
for item in items:
self.addCommand(item)
for i in range(0, numWorkers):
w = Worker("worker%d" % i, self)
self.workers.append(w)
w.start()
self.numWorkers = numWorkers
###
def getNumWorkers(self):
return self.numWorkers
def getNextWorkItem(self):
return self.work_queue.get(block=True)
def addFinishedWorkItem(self, command):
self.completed_queue.put(command)
self.work_queue.task_done()
def markTaskDone(self):
self.work_queue.task_done()
def addCommand(self, cmd):
self.logger.debug("Adding cmd to work_queue: %s" % cmd.cmdStr)
self.work_queue.put(cmd)
self._assigned += 1
def _join_work_queue_with_timeout(self, timeout):
"""
Queue.join() unfortunately doesn't take a timeout (see
https://bugs.python.org/issue9634). Fake it here, with a solution
inspired by notes on that bug report.
XXX This solution uses undocumented Queue internals (though they are not
underscore-prefixed...).
"""
done_condition = self.work_queue.all_tasks_done
done_condition.acquire()
try:
while self.work_queue.unfinished_tasks:
if (timeout <= 0):
# Timed out.
return False
start_time = time.time()
done_condition.wait(timeout)
timeout -= (time.time() - start_time)
finally:
done_condition.release()
return True
def join(self, timeout=None):
"""
Waits (up to an optional timeout) for the worker queue to be fully
completed, and returns True if the pool is now done with its work.
A None timeout indicates that join() should wait forever; the return
value is always True in this case. Zero and negative timeouts indicate
that join() will query the queue status and return immediately, whether
the queue is done or not.
"""
if timeout is None:
self.work_queue.join()
return True
return self._join_work_queue_with_timeout(timeout)
def joinWorkers(self):
for w in self.workers:
w.join()
def _pop_completed(self):
"""
Pops an item off the completed queue and decrements the assigned count.
If the queue is empty, throws Queue.Empty.
"""
item = self.completed_queue.get(False)
self._assigned -= 1
return item
def getCompletedItems(self):
completed_list = []
try:
while True:
item = self._pop_completed() # will throw Empty
if item is not None:
completed_list.append(item)
except Empty:
return completed_list
def check_results(self):
""" goes through all items in the completed_queue and throws an exception at the
first one that didn't execute successfully
throws ExecutionError
"""
try:
while True:
item = self._pop_completed() # will throw Empty
if not item.get_results().wasSuccessful():
raise ExecutionError("Error Executing Command: ", item)
except Empty:
return
def empty_completed_items(self):
while not self.completed_queue.empty():
self._pop_completed()
def isDone(self):
# TODO: not sure that qsize() is safe
return (self.assigned == self.completed_queue.qsize())
@property
def assigned(self):
"""
A read-only count of the number of commands that have been added to the
pool. This count is only decremented when items are removed from the
completed queue via getCompletedItems(), empty_completed_items(), or
check_results().
"""
return self._assigned
@property
def completed(self):
"""
A read-only count of the items in the completed queue. Will be reset to
zero after a call to empty_completed_items() or getCompletedItems().
"""
return self.completed_queue.qsize()
def haltWork(self):
self.logger.debug("WorkerPool haltWork()")
self.should_stop = True
for w in self.workers:
w.haltWork()
self.work_queue.put(self.halt_command)
class Server():
def __init__(self):
## Socket Initialization
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Opening the socket
self.sock.bind((TCP_IP, TCP_PORT))
self.sock.listen(MAX_CLIENTS)
self.connect_status = False # Connection not yet established.
## Controller tracking variables
# The controller pose just before sending move signal to baxter.
self.baxter_previous_controller_pose = {'left':{'position':None, 'orientation':None}, 'right':{'position':None, 'orientation':None}}
# reference pose is the controller pose when the trigger is clicked
self.left_reference_controller_pose = {'left':{'position':None, 'orientation':None}, 'right':{'position':None, 'orientation':None}}
self.right_reference_controller_pose = {'left':{'position':None, 'orientation':None}, 'right':{'position':None, 'orientation':None}}
# Current controller pose
self.controller_pose = {'left':{'position':None, 'orientation':None}, 'right':{'position':None, 'orientation':None}}
# Current controller status
self.controller_status = {'left':{'pad_click':False, 'pad_press':False, 'trig_click':False, 'trig_press':False}, \
'right':{'pad_click':False, 'pad_press':False, 'trig_click':False, 'trig_press':False}}
# Previous controller pose
self.previous_controller_pose = {'left':{'position':None, 'orientation':None}, 'right':{'position':None, 'orientation':None}}
# Previous controller status
self.previous_controller_status = {'left':{'pad_click':False, 'pad_press':False, 'trig_click':False, 'trig_press':False}, \
'right':{'pad_click':False, 'pad_press':False, 'trig_click':False, 'trig_press':False}}
## Baxter Initialization: Left and right are separately initialized.
rospy.init_node('baxter_controller')
self.bax_left = Baxter('left')
self.bax_right = Baxter('right')
# Queue of dpositions for left and right arms separately.
self.left_queue_dposition = Queue(maxsize=MAX_QUEUE_SIZE)
self.right_queue_dposition = Queue(maxsize=MAX_QUEUE_SIZE)
# Current baxter dposition
self.baxter_dposition = {'left':[0.0,0.0,0.0], 'right':[0.0,0.0,0.0]}
## Miscellaneous
self.count = 0
## Waiting for establishing the connection between server and client
self.wait_for_connection()
def wait_for_connection(self):
self.client, self.addr = (self.sock.accept())
data = self.client.recv(BUFFER_SIZE)
if data == INITIAL_MESSAGE:
print 'Received a handshake'
self.connect_status = True
self.client.send(str(True)) # Send a success note
def parse_data(self, data):
# 0,1,2 - left x,y,z
# 3,4,5 - left yaw, pitch, roll
# 6,7,8 - flagPress, flagTouch, serial_no
# 9, 10, 11 - right x, y, z
# 12, 13, 14 - right yaw, pitch, roll
# 15, 16, 17 - flagPress, flagTouch, serial_no
data = map( float, data.split(' ') )
# Extracting left controller data
left_position = data[0:3]; left_orientation = data[3:6]; left_state = data[6:9]
left_data = {'position':left_position, 'orientation':left_orientation, 'state':left_state}
# Extracting right controller data
right_position = data[9:12]; right_orientation = data[12:15]; right_state = data[15:18]
right_data = {'position':right_position, 'orientation':right_orientation, 'state':right_state}
return left_data, right_data
def update_controller_info(self, data):
left_data, right_data = data
# Update left controller pose
self.controller_pose['left']['position'] = left_data['position']
self.controller_pose['left']['orientation'] = left_data['orientation']
# Update right controller pose
self.controller_pose['right']['position'] = right_data['position']
self.controller_pose['right']['orientation'] = right_data['orientation']
# 0 for CLICK and 1 for PRESS
# Update left controller status
if int(left_data['state'][0]) == TOUCHPAD_ID: self.controller_status['left']['pad_click'] = True
else: self.controller_status['left']['pad_click'] = False
if int(left_data['state'][1]) == TOUCHPAD_ID: self.controller_status['left']['pad_press'] = True
else: self.controller_status['left']['pad_press'] = False
if int(left_data['state'][0]) == TRIGGER_ID: self.controller_status['left']['trig_click'] = True
else: self.controller_status['left']['trig_click'] = False
if int(left_data['state'][1]) == TRIGGER_ID: self.controller_status['left']['trig_press'] = True
else: self.controller_status['left']['trig_press'] = False
# Update both the controller status
if int(left_data['state'][0]) == TOUCHPAD_ID+TRIGGER_ID:
self.controller_status['left']['pad_click'] = True
self.controller_status['left']['trig_click'] = True
if int(left_data['state'][1]) == TOUCHPAD_ID+TRIGGER_ID:
self.controller_status['left']['pad_press'] = True
self.controller_status['left']['trig_press'] = True
# Update right controller status
if int(right_data['state'][0]) == TOUCHPAD_ID: self.controller_status['right']['pad_click'] = True
else: self.controller_status['right']['pad_click'] = False
if int(right_data['state'][1]) == TOUCHPAD_ID: self.controller_status['right']['pad_press'] = True
else: self.controller_status['right']['pad_press'] = False
if int(right_data['state'][0]) == TRIGGER_ID: self.controller_status['right']['trig_click'] = True
else: self.controller_status['right']['trig_click'] = False
if int(right_data['state'][1]) == TRIGGER_ID: self.controller_status['right']['trig_press'] = True
else: self.controller_status['right']['trig_press'] = False
# Update both the controller status
if int(right_data['state'][0]) == TOUCHPAD_ID+TRIGGER_ID:
self.controller_status['right']['pad_click'] = True
self.controller_status['right']['trig_click'] = True
if int(right_data['state'][1]) == TOUCHPAD_ID+TRIGGER_ID:
self.controller_status['right']['pad_press'] = True
self.controller_status['right']['trig_press'] = True
def generate_baxter_data(self):
if self.controller_status['left']['trig_click']:
self.baxter_dposition['left'] = np.array(self.controller_pose['left']['position']) - \
np.array(self.previous_controller_pose['left']['position'])
self.baxter_dposition['left'] = np.dot(R_left, self.baxter_dposition['left'].reshape(3,1)).flatten().copy().tolist()
if self.controller_status['right']['trig_click']:
self.baxter_dposition['right'] = np.array(self.controller_pose['right']['position']) - \
np.array(self.previous_controller_pose['right']['position'])
self.baxter_dposition['right'] = np.dot(R_right, self.baxter_dposition['right'].reshape(3,1)).flatten().tolist()
def print_data(self):
print 'Data Received: Yayy'
print 'Left Controller: '
print 'Position: ', self.controller_pose['left']['position'], 'Orientation: ', self.controller_pose['left']['orientation'], 'Status: ', self.controller_status['left']
# print 'Right Controller: '
# print 'Position: ', self.controller_pose['right']['position'], 'Orientation: ', self.controller_pose['right']['orientation'], 'Status: ', self.controller_status['right']
def run_left_controller(self):
START_COUND_IDX = 1
while self.connect_status and (not rospy.is_shutdown()):
try:
# Receive VIVE Controller information
data = self.client.recv(BUFFER_SIZE)
self.client.send(str(True)) # Send a success note
# Parse and update controller information
self.update_controller_info(self.parse_data(data))
# self.print_data() # Prints current controller data for both controllers.
#### For first reading from vive: Update previous controller pose and continue
if self.count == 0: # Initially previous and current poses are the same
self.previous_controller_pose = deepcopy(self.controller_pose)
self.previous_controller_status = deepcopy(self.controller_status)
self.count = START_COUND_IDX
continue
### --- From here it is left controller specific. --- ###
#### From second VIVE reading onwards
# Continue if previous and current states of trigger click are False
if (not self.previous_controller_status['left']['trig_click']) and (not self.controller_status['left']['trig_click']):
self.previous_controller_status = deepcopy(self.controller_status)
self.previous_controller_pose = deepcopy(self.controller_pose)
continue
print self.previous_controller_status['left']['trig_click'], self.controller_status['left']['trig_click']
# Transition from False to True is marked as initial point for motion: Left Controller
# Consider that point as a reference
if (not self.previous_controller_status['left']['trig_click']) and self.controller_status['left']['trig_click']:
self.left_reference_controller_pose = deepcopy(self.controller_pose['left'])
self.baxter_previous_controller_pose = deepcopy(self.controller_pose) # It stores for both controllers.
self.previous_controller_status = deepcopy(self.controller_status)
self.previous_controller_pose = deepcopy(self.controller_pose)
self.count = START_COUND_IDX
print 'left_reference_controller_pose', self.left_reference_controller_pose
continue
# trigger is clicked for both previous and current states.
if self.previous_controller_status['left']['trig_click'] and self.controller_status['left']['trig_click']:
self.generate_baxter_data()
# print 'baxter_dposition[left]: ', self.baxter_dposition['left']
if np.mean(np.abs(np.array(self.baxter_dposition['left']))) > 1e-3:
print 'baxter_dposition[left]: ', self.baxter_dposition['left']
self.left_queue_dposition.put(deepcopy(self.baxter_dposition['left']))
if self.previous_controller_status['left']['pad_click'] != self.controller_status['left']['pad_click']:
self.bax_left.toggle_gripper_status()
self.count += 1
self.previous_controller_status = deepcopy(self.controller_status)
self.previous_controller_pose = deepcopy(self.controller_pose)
except Exception as exp:
print exp
print 'Connection Closed'
self.connect_status = False
self.client.close()
self.connect_status = False
def increment_baxter_left_beta(self): ### Updating prevous position in here.
# Emptying half of the queue at once. Hence Baxter moves quickly as distance is longer.
while self.connect_status and (not rospy.is_shutdown()):
sum_dposition = np.zeros(3)
try:
if self.left_queue_dposition.qsize() < 1:
print 1/0 # Intentionally creating the exception
for _ in range(self.left_queue_dposition.qsize()/1):
sum_dposition += np.array(self.left_queue_dposition.get(block=False))
self.bax_left.increment_ee_position(sum_dposition.tolist())
except Exception as exp:
pass
# print 'Queue is empty ... ^^'
# time.sleep(0.001)
### Updating the previous controller pose ###
self.baxter_previous_controller_pose = deepcopy(self.controller_pose)
def increment_baxter_left(self): ### Updating prevous position in here.
# Emptying the queue one at a time. It is slower.
while self.connect_status and (not rospy.is_shutdown()):
try:
print 'queue_size: ', self.left_queue_dposition.qsize()
self.bax_left.increment_ee_position(self.left_queue_dposition.get(block=False))
except Exception as exp:
pass
# print 'Queue is empty ... ^^'
# time.sleep(0.001)
### Updating the previous controller pose ###
self.baxter_previous_controller_pose = deepcopy(self.controller_pose)
def increment_baxter_right(self):
while self.connect_status and (not rospy.is_shutdown()) and (not self.right_queue_dposition.empty()):
### Updating the previous controller pose ###
self.bax_right.increment_ee_position(self.right_queue_dposition.get())
class RegisteredMetric(RegisteredEntity):
def __init__(self, ref_metric, ref_dcc, reg_entity_id):
super(RegisteredMetric, self).__init__(ref_entity=ref_metric,
ref_dcc=ref_dcc,
reg_entity_id=reg_entity_id)
self.flag_alive = False
self._next_run_time = None
self.current_aggregation_size = 0
# -------------------------------------------------------------------
# Elements in this queue are (ts, v) pairs.
#
self.values = Queue()
def start_collecting(self):
self.flag_alive = True
# TODO: Add a check to ensure that start_collecting for a metric is
# called only once by the client code
metric_handler.initialize()
self._next_run_time = getUTCmillis() + (self.ref_entity.interval *
1000)
metric_handler.event_ds.put_and_notify(self)
def stop_collecting(self):
self.flag_alive = False
log.debug("Metric %s is marked for deletion" %
str(self.ref_entity.name))
def add_collected_data(self, collected_data):
if isinstance(collected_data, list):
for data_sample in collected_data:
self.values.put(data_sample)
return len(collected_data)
elif isinstance(collected_data, tuple):
self.values.put(collected_data)
return 1
else:
self.values.put((getUTCmillis(), collected_data))
return 1
def get_next_run_time(self):
return self._next_run_time
def set_next_run_time(self):
self._next_run_time = self._next_run_time + \
(self.ref_entity.interval * 1000)
log.debug("Set next run time to:" + str(self._next_run_time))
def is_ready_to_send(self):
log.debug("self.current_aggregation_size:" +
str(self.current_aggregation_size))
log.debug("self.aggregation_size:" +
str(self.ref_entity.aggregation_size))
return self.current_aggregation_size >= self.ref_entity.aggregation_size
def collect(self):
log.debug("Collecting values for the resource {0} ".format(
self.ref_entity.name))
self.args_required = len(
inspect.getargspec(self.ref_entity.sampling_function)[0])
if self.args_required is not 0:
self.collected_data = self.ref_entity.sampling_function(1)
else:
self.collected_data = self.ref_entity.sampling_function()
log.info("{0} Sample Value: {1}".format(self.ref_entity.name,
self.collected_data))
log.debug("Size of the queue {0}".format(self.values.qsize()))
no_of_values_added = self.add_collected_data(self.collected_data)
self.current_aggregation_size = self.current_aggregation_size + no_of_values_added
def reset_aggregation_size(self):
self.current_aggregation_size = 0
def send_data(self):
log.info("Publishing values for the resource {0} ".format(
self.ref_entity.name))
if not self.values:
# No values measured since last report_data
return True
self.ref_dcc.publish(self)
def __str__(self, *args, **kwargs):
return str(self.ref_entity.name) + ":" + str(self._next_run_time)
def __cmp__(self, other):
if other is None:
return -1
if not isinstance(other, RegisteredMetric):
return -1
return cmp(self._next_run_time, other._next_run_time)
