def creat(item: Item):
"""
通过该接口可以创建一个抽奖规则
"""
myredis = MyRedis()
rq = RedisQueue(item.luckname)
print("ok")
if rq.qsize():
return {"ret": 500, "msg": "该抽奖已经存在，请删除后重试"}
result = {"ret": 0, "item": item}
awardlist = item.award
lucklist = []
luckdict = {}
for ward in awardlist:
luckdict[ward.name] = ward.count
for i in range(ward.count):
lucklist.append(ward.name)
othercount = item.total - len(lucklist)
if othercount:
luckdict[item.other] = othercount
others = [item.other] * othercount
lucklist = lucklist + others
random.shuffle(lucklist)
print(lucklist)
for luck in lucklist:
rq.put(luck)
myredis.hmset(item.luckname, luckdict)
result = {'ret': 0, 'msg': "succses"}
return result
def luck(id: int, luckname: str):
"""
开始抽奖
"""
rd = RedisQueue(luckname)
myredis = MyRedis()
winner = luckname + "_winner"
if myredis.hexists(winner, id):
return {"ret": 0, "msg": "您已经抽过了，不能再抽了"}
award = rd.get_nowait()
if award:
myredis.hset(winner, id, award)
myredis.hincrby(luckname, award, -1)
result = {
"ret": 0,
'data': {
"flag": 1,
"msg": "恭喜你中奖了",
"award": award
}
}
else:
result = {
"ret": 0,
'data': {
"flag": 0,
"msg": "奖抽完了",
}
}
return result
def __init__(self, event_kill, name, concurrency, ip, max_time=-1):
self.queue_name = name
self.concurrency = concurrency
self.max_time = max_time
self.event_kill = event_kill
self.conn = RedisQueue(Redis(host=ip), self.queue_name)
super(Worker, self).__init__()
def __init__(self, bluetuuid="94f39d29-7d6d-437d-973b-fba39e49d4ee"):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
self.devID = b''
self.uuid = bluetuuid
self.thread_read_flg = False
self.thread_read = None
self.runallflg = False
self.allthread = None
def __init__(self):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',host='127.0.0.1',psw='123456',db=1)
self.influxdbclass=PInfluxdbClass(host='127.0.0.1', port=8086, username='******', password='******', database = 'RFIDreportdb')
self.tagstoRedis=DataObjectJSRedisClass(host='127.0.0.1',psw='123456',port=6379,indexkey='EPC')
self.devID = b''
self.thread_read_flg=False
self.thread_read=None
self.runallflg=False
self.allthread=None
self.clientID=""
self.sendEmqClient=EMQSendClass(host='192.168.1.178',port=1883,username='******',pwd='shuiJing1',client_id='')
def __init__(self,
bluetuuid="94f39d29-7d6d-437d-973b-fba39e49d4ee",
name="FireFly-3288C",
port=6):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
self.devID = b''
self.uuid = bluetuuid
self.bluetoothname = name
self.bluetoothport = port
self.thread_read_flg = False
self.thread_read = None
self.runallflg = False
self.allthread = None
self.allerr = False
def _run_script_background(self, command, username, region):
p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
time.sleep(0.1) # forget why i sleep here, if not sure, don't modify this
rq = RedisQueue('script_serviceDeploy_%s_%s' % (region, username))
rq.delete_queue() # todo: delete key only
stdout, stderr = p.communicate()
ret_code = p.poll()
if not ret_code:
script_result = stdout
logger.info('run command: %s success, output:\n%s' % (command, stdout))
rq.put('*finished*')
self.add_script_exec_log(username, self.script_name, self.script_path, True, script_result)
else:
script_result = stdout + stderr
logger.error('run command: %s failed, output:\n%s' % (command, script_result))
errout = [err for err in stderr.splitlines() if err]
errout = '\n'.join(errout)
rq.put('*failed*%s' % errout)
self.add_script_exec_log(username, self.script_name, self.script_path, False, script_result)
class TestBasicQueueFunctions(unittest.TestCase):
def setUp(self):
self.rq = RedisQueue("testqueue", "testworker01", db=10)
self.r = Redis(db=10)
self.r.delete("testqueue")
self.r.delete(self.rq.workeritem)
def test_add_to_queue(self):
self.rq.push("testitem")
self.assertEqual(len(self.rq), 1)
self.assertEqual(self.rq.pop(), "testitem")
def test_multiple_additions(self):
results = map(self.rq.push, xrange(10))
self.assertEquals(results, [True] * 10)
def test_temp_workerqueue(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
def test_workerqueue_complete(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_complete()
item = self.rq.pop()
self.assertEqual(item, "nextitem")
self.assertEqual(len(self.rq), 0)
def test_requeue_on_fail(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_failed()
self.assertEqual(len(self.rq), 2)
def test_requeue_to_end_on_fail(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_failed()
self.assertEqual(len(self.rq), 2)
item = self.rq.pop()
self.assertEqual(item, "nextitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_complete()
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 0)
def setUp(self):
self.rq = RedisQueue("testqueue", "testworker01", db=10)
self.r = Redis(db=10)
self.r.delete("testqueue")
self.r.delete(self.rq.workeritem)
class Spider():
base_url = 'https://weixin.sogou.com/weixin'
keyword = 'Python'
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
'Cache-Control':
'max-age=0',
'Connection':
'keep-alive',
'Cookie':
'SUV=00EE712F77F8F1285C9A141F36334105; ABTEST=0|1562225137|v1; IPLOC=CN4403; SUID=1C013C3A4A42910A000000005D1DA9F1; SUID=1C013C3A5218910A000000005D1DA9F2; weixinIndexVisited=1; sct=1; SNUID=43615C5A5F65ECB5BB7AE454600977ED; ppinf=5|1562234904|1563444504|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo3OnN0aWNrZXJ8Y3J0OjEwOjE1NjIyMzQ5MDR8cmVmbmljazo3OnN0aWNrZXJ8dXNlcmlkOjQ0Om85dDJsdU9oeDFXdktJNm1GLV9JSk1PZ241NHdAd2VpeGluLnNvaHUuY29tfA; pprdig=j3pXVh43ek0qmlyN738VY_0Lno7YTfzPO1-52g0fRZ3SW93xnLY_uCVNTjYNKkADx0ySyXYgvrWojM6Lb8vFNzELWfGt6IoZ-G7QiYNH83yeiXZ2j4y1fexHB--636SyKIU_My2i1UdrvvNOQnSWUC-iVWJd04vyVl0ZQCpbHpo; sgid=12-41039275-AV0d0BgzrgksahCjtjianUTM; ppmdig=1562306345000000b94d15df908fdedf755f385f9678951a; JSESSIONID=aaasgD9OOLOWpcz5edkRw',
'DNT':
'1',
'Host':
'weixin.sougou.com',
'Upgrade-Insecure-Requests':
'1',
'User-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
}
session = Session()
queue = RedisQueue()
mysql = MySQL()
def get_proxy(self):
"""
从代理池中获取代理
:return:
"""
try:
response = requests.get(PROXY_POOL_URL)
if response.status_code == 200:
print('Get Proxy', response.text)
return response.text
return None
except requests.ConnectionError:
return None
def start(self):
"""
初始化工作
:return:
"""
# 全局更新Headers
self.session.headers.update(self.headers)
start_url = self.base_url + '?' + urlencode({
'query': self.keyword,
'type': 2
})
weixin_request = WeixinRequest(url=start_url,
callback=self.parse_index,
need_proxy=True)
# 调度第一个请求
self.queue.add(weixin_request)
def parse_index(self, response):
"""
解析索引页
:param response: 响应
:return: 新的响应
"""
doc = pq(response)
items = doc('.new-box .news-list li .txt-box h3 a').items()
for item in items:
url = item.attr('href')
weixin_request = WeixinRequest(url=url, callback=self.parse_detail)
yield weixin_request
next = doc('#sogou_next').attr('href')
if next:
url = self.base_url + str(next)
weixin_request = WeixinRequest(url=url,
callback=self.parse_index,
need_proxy=True)
yield weixin_request
def parse_detail(self, response):
"""
解析详情页
:param response: 响应
:return: 公众号文章
"""
doc = pq(response.text)
data = {
'title':
doc('.rich_media_title').text(),
'content':
doc('.rich_media_content').text(),
'data':
doc('#post-data').text(),
'nickname':
doc('#js_profile_qrcode > div > strong').text(),
'wechat':
doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
}
yield data
def request(self, weixin_request):
"""
执行请求
:param weixin_request: 请求
:return: 响应
"""
try:
if weixin_request.need_proxy:
propxy = self.get_proxy()
if propxy:
proxies = {
'http': 'http://' + propxy,
'https': 'https://' + propxy,
}
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timeout,
allow_redirects=False,
proxies=proxies)
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timout,
allow_redirects=False)
except (ConnectionError, ReadTimeout) as e:
print(e.args)
return False
def error(self, weixin_request):
"""
错误处理
:param weixin_request: 请求
:return:
"""
weixin_request.fail_time = weixin_request.fail_time + 1
print('Request fail', weixin_request.fail_time, 'Times',
weixin_request.url)
if weixin_request.fail_time < MAX_FAILED_TIME:
self.queue.add(weixin_request)
def schedule(self):
"""
调度请求
:return:
"""
while not self.queue.empty():
weixin_request = self.queue.pop()
callback = weixin_request.callback
print('Schedule', weixin_request.url)
response = self.request(weixin_request)
if response and response.status_code in VALID_STATUSES:
results = list(callback(response))
if results:
for result in results:
print('New Result', result)
if isinstance(result, WeixinRequest):
self.queue.add(result)
if isinstance(result, dict):
self.mysql.insert('articles', result)
else:
self.error(weixin_request)
else:
self.error(weixin_request)
def run(self):
"""
入口
:return:
"""
self.start()
self.schedule()
class Spider():
base_url = 'http://weixin.sogou.com/weixin'
keyword = 'NBA'
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4,zh-TW;q=0.2,mt;q=0.2',
'Cache-Control':
'max-age=0',
'Connection':
'keep-alive',
'Cookie':
'CXID=300E866F4542ED22AE87B0EAAC93DE49; SUID=DE19F93A4B238B0A5B45763600018391; ABTEST=7|1535183484|v1; IPLOC=CN4401; weixinIndexVisited=1; SUV=002D1DEC1B2F3AEE5B810A7E9992A029; sct=1; JSESSIONID=aaalrUYv_EZgp9x02tBvw; PHPSESSID=1l812rviclinat3jkguqd8h4k5; SUIR=3DE9FCC8D3D1A7E910541485D4DA05AF; SNUID=C10A1F2B30355B0E7FDD6C8430137DAF; seccodeRight=success; successCount=1|Sat, 25 Aug 2018 08:36:18 GMT',
'Host':
'weixin.sogou.com',
'Upgrade-Insecure-Requests':
'1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
session = Session()
queue = RedisQueue()
mysql = MySQL()
def get_proxy(self):
"""
从代理池获取代理
:return:
"""
try:
response = requests.get(PROXY_POOL_URL)
if response.status_code == 200:
print('Get Proxy', response.text)
return response.text
return None
except requests.ConnectionError:
return None
def start(self):
"""
初始化工作
"""
# 全局更新Headers
self.session.headers.update(self.headers)
start_url = self.base_url + '?' + urlencode({
'query': self.keyword,
'type': 2
})
weixin_request = WeixinRequest(url=start_url,
callback=self.parse_index,
need_proxy=True)
# print(start_url) # http://weixin.sogou.com/weixin?query=NBA&type=2
# 调度第一个请求( 网页 )
self.queue.add(weixin_request)
# callback 函数
def parse_index(self, response):
"""
解析索引页
:param response: 响应
:return: 新的响应
"""
doc = pq(response.text)
# 拿到item中的title
items = doc('.news-box .news-list li .txt-box h3 a').items()
for item in items:
# title 是a 标签
url = item.attr('href')
weixin_request = WeixinRequest(url=url, callback=self.parse_detail)
yield weixin_request
next = doc('#sogou_next').attr('href')
if next:
url = self.base_url + str(next)
weixin_request = WeixinRequest(url=url,
callback=self.parse_index,
need_proxy=True)
yield weixin_request
# 请求的回调
def parse_detail(self, response):
"""
解析详情页
:param response: 响应
:return: 微信公众号文章
"""
doc = pq(response.text)
data = {
'title':
doc('.rich_media_title').text(),
'content':
doc('.rich_media_content').text(),
'date':
doc('#post-date').text(),
'nickname':
doc('#js_profile_qrcode > div > strong').text(),
'wechat':
doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
}
yield data
# 发起网络请求
def request(self, weixin_request):
"""
执行请求
:param weixin_request: 请求
:return: 响应
"""
# try:
if weixin_request.need_proxy:
proxy = self.get_proxy()
if proxy:
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy
}
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timeout,
allow_redirects=False,
proxies=proxies)
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timeout,
allow_redirects=False)
# except (ConnectionError, ReadTimeout) as e:
# print(e.args)
# return False
def error(self, weixin_request):
"""
错误处理
:param weixin_request: 请求
:return:
"""
weixin_request.fail_time = weixin_request.fail_time + 1
print('Request Failed', weixin_request.fail_time, 'Times',
weixin_request.url)
if weixin_request.fail_time < MAX_FAILED_TIME:
self.queue.add(weixin_request)
# 调度请求
def schedule(self):
"""
调度请求
:return:
"""
while not self.queue.empty():
weixin_request = self.queue.pop()
# callback有两种情况
callback = weixin_request.callback
print('Schedule', weixin_request.url)
# 发起网络请求（每次都会修改代理）
response = self.request(weixin_request)
print('response ===: ', response)
if response and response.status_code in VALID_STATUSES:
# https://mp.weixin.qq.com/s?src=11×tamp=1535185884&ver=1081&signature=es5CzC0VNUqdPKGjOndCHZF9*9lta4Z4SCyS9V0IzD9aHF9ruvINszh-1xqfZ4YmVqDIl9KxQacmv3KmJyXz*Aby5hVGKgaWbPt6buCTNxZeuokt7OhireVn*BJy*89Y&new=1
# 1.results 是给生成器函数：里面有很多的 weixin_request , 获取每一个item 的detail
# 2.results 是一个字典
results = list(callback(response))
if results:
for result in results:
print('New Result', type(result))
if isinstance(result, WeixinRequest):
# 把请求添加多请求队列
self.queue.add(result)
if isinstance(result, dict):
self.mysql.insert('articles', result)
else:
self.error(weixin_request)
else:
self.error(weixin_request)
# 只执行一次
break
def run(self):
"""
入口
:return:
"""
self.start()
self.schedule()
import RFIDSYSSqliteClass
import socket
import json
import threading
# import psycopg2
import datetime
#import Com4GModularClass
from redisqueue import RedisQueue
from bluetooth import *
import threading
usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
devID = b''
def GetRfidDevID():
global devID
if devID == b'':
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
if conn.ping():
print('conn ok')
ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_number = sys.argv[1]
worker_section = "worker_pirus2"
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "pirus_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
try:
plugin_name = c.get(worker_section, "repository_plugin")
plugin_module = __import__(plugin_name)
components = plugin_name.split('.')
for comp in components[1:]:
plugin_module = getattr(plugin_module, comp)
except ImportError, e:
logger.error("Coundn't import module: '%s' - %s" % (c.get(worker_section, "repository_plugin"), e))
sys.exit(2)
if c.has_option(worker_section, 'pauseonfail'):
try:
def test_live_queue():
live_queue = RedisQueue("test_queue", MockTask, namespace="pytest")
assert live_queue.connected is False
task = MockTask()
task2 = MockTask()
task.uri = 'thisIsUnique'
task2.uri = 'thisIsUnique'
assert live_queue.connect(host=live_host, port=live_port,
password=live_pass) is True
assert live_queue.connected is True
live_queue.clear()
assert live_queue.qsize == 0
live_queue.put(task)
assert live_queue.qsize == 1
live_queue.put(task2)
assert live_queue.qsize == 2
new_task = live_queue.get()
assert isinstance(new_task, MockTask)
assert new_task.uid == task.uid
assert new_task.uri == 'thisIsUnique'
live_queue.clear()
task.unique = True
task2.unique = True
assert task.unique_hash() == task2.unique_hash()
live_queue.put(task)
with pytest.raises(TaskAlreadyInQueueException):
live_queue.put(task2)
assert live_queue.qsize == 1
live_queue.clear()
# test getting and putting the same task into the queue
assert live_queue.qsize == 0
live_queue.put(task)
my_task = live_queue.get()
live_queue.put(my_task)
assert live_queue.qsize == 1
from uuid import uuid4
import hashlib
if __name__ == "__main__":
c = Config()
redis_section = "redis"
store_section = "ofs"
worker_section = "worker_%s" % PROCESS
worker_number = sys.argv[1]
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "%s_%s" % (PROCESS, worker_number),
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
r = Redis(db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port"))
ofs_module = c.get(store_section, "ofs_client_module")
ofs_root = c.get(store_section, "ofs_root")
ofs_class = c.get(store_section, "ofs_class")
try:
OFS_mod = __import__(ofs_module)
OFS_impl = getattr(OFS_mod, ofs_class)
except ImportError:
print "Module %s could not be imported" % ofs_module_root
sys.exit(2)
def handle(self):
print(self.client_address)
sqstr = '{0}:{1}'.format(self.client_address[0],
self.client_address[1])
print(sqstr)
q = RedisQueue(name=sqstr)
#q.redisidrun=q.isrun()
#print ('running')
while True:
try:
data = self.request.recv(1024)
if (data == b''):
try:
conn = psycopg2.connect(database="materiel",
user="******",
password="******",
host="58.214.232.165",
port="5432")
cur = conn.cursor()
print('lin postgresql OK')
except:
print('InsertToPostgresql Connect Fail')
break
#ProssData(q)
while True:
if q.isrun() == 0:
break
qdata = q.get_nowait()
if qdata == None:
try:
conn.commit()
conn.close()
print('InsertToPostgresql close')
break
except:
print('InsertToPostgresql commit Fail')
break
#break
print(qdata)
if b'DEVID' in qdata:
devid, value = qdata.split(b':')
print('did:{0}'.format(devid))
lisoo = value.split(b';')
print('did:{0}-epc:{1}'.format(lisoo[0], lisoo[1]))
sqlselectstr = (
"INSERT INTO tb_epc_record (device_id,epc,report_time) VALUES ('{0}','{1}',NOW()::timestamp)"
).format(lisoo[0].decode(encoding='utf-8'),
lisoo[1].decode(encoding='utf-8'))
print(sqlselectstr)
cur.execute(sqlselectstr)
break #continue
#print (data)
q.put(data)
#if data == b'exit':
# print ('exit')
# self.server.shutdown()
# self.request.close()
# break
except Exception as e:
self.server.shutdown()
self.request.close()
break
def start():
round_robin = 0
while True:
try:
manager_queue = RedisQueue('manager', host=settings.REDIS_HOST, port=settings.REDIS_PORT)
# get next message - first find an available queue
keys = RedisKeys(pattern='rendertaskqueue:*', host=settings.REDIS_HOST, port=settings.REDIS_PORT).keys
if keys is not None and len(keys) > 0:
round_robin = BlendClusterClient.next_queue(round_robin, keys)
queue_name = keys[round_robin] # just grab the first queue name
print 'round robin: ' + queue_name
queue = RedisQueue('namechangedbelow', host=settings.REDIS_HOST, port=settings.REDIS_PORT)
queue.key = queue_name # override key name (queue name) here as we have the qualified name
print 'getting render task message'
render_task_json = queue.get()
print render_task_json
render_task_obj = RenderTaskMessage.json_to_object(render_task_json)
hg_url = render_task_obj.hg_url
blend_file = render_task_obj.blend_file
# notify manager we are working
msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id,
'state': 1}) # set working state
manager_queue.put(msg)
print 'Set task to WORKING state: ' + str(render_task_obj.id)
print '-------------------------'
print 'WORKING'
print '-------------------------'
# so some work
print 'About to prepare hg'
hg = HgUtils(url=hg_url)
print 'about to clone or pull hg'
hg.clone_or_pull()
full_blend_filename = hg.local_project_path + '/' + render_task_obj.blend_file
short_blend_filename = ntpath.basename(full_blend_filename).split('.')[0]
render_dir = hg.local_project_path + '/renders/' + short_blend_filename + '/'\
+ render_task_obj.scene_name + '/'
print 'Rendering in dir: ' + render_dir
if not os.path.exists(render_dir):
os.makedirs(render_dir)
hg.push_all_changes()
blender = Blender(blend_file=full_blend_filename, scene_name=render_task_obj.scene_name,
start_frame=render_task_obj.frame, end_frame=render_task_obj.frame,
output_dir=render_dir)
blender.run()
# before pushing changes update local repo
hg.get_latest_changes()
hg.push_all_changes()
print 'done hg'
msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id,
'state': 4}) # set success state
manager_queue.put(msg)
print '-------------------------'
print 'SUCCESSFUL'
print '-------------------------'
else:
print('No messages available right now, will try again in 5 seconds...')
time.sleep(5) # wait 5 secs before trying again
except Exception as e:
print('Error raised: ' + e.message)
try:
msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id,
'state': 3}) # set working state failed
manager_queue.put(msg)
print '-------------------------'
print 'FAILED'
print '-------------------------'
except:
print('Trying to respond with a failure status but could not!')
print('Will try again in 5 seconds...')
time.sleep(5) # wait 5 secs before trying again
class SendDataToDestinationBLC(object):
def __init__(self, bluetuuid="94f39d29-7d6d-437d-973b-fba39e49d4ee"):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
self.devID = b''
self.uuid = bluetuuid
self.thread_read_flg = False
self.thread_read = None
self.runallflg = False
self.allthread = None
self.allerr = False
def GetRfidDevID(self):
if self.devID == b'':
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
if conn.ping():
print('conn ok')
else:
print('shawanyi')
except: #
return self.devID
#conn.set('RFIDDevID', devID)
if conn.exists('RFIDDevID'):
pass
else:
print(self.devID)
return self.devID
ret = conn.get('RFIDDevID')
if ret != self.devID:
self.devID = ret
print('devID', self.devID)
return self.devID
def GetSysBluetoothCMD(self):
cmd = 0
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
except:
print('GetSysBluetoothCMD ERR')
return 0
ret = conn.get('RFIDSysBluetoothCMD')
if ret == None:
return 0
if ret == b'START':
cmd = 1
elif ret == b'STOP':
cmd = 2
elif ret == b'POWERUP':
cmd = 3
elif ret == b'POWERDOWN':
cmd = 4
elif b'POWER:' in ret:
p, var = ret.split(b':')
cmd = int(var)
else:
cmd = 0
return cmd
def SetSysBluetoothCMD(self, cmd=0):
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
except:
print('GetSysBluetoothCMD ERR')
return False
strcmd = ''
#ret=conn.get('RFIDSysBluetoothCMD')
if cmd == 1: #b'START':
strcmd = 'START'
elif cmd == 2: #b'STOP':
strcmd = 'STOP'
elif cmd == 3: #b'POWERUP':
strcmd = 'POWERUP'
elif cmd == 4: #b'POWERDOWN':
strcmd = 'POWERDOWN'
elif cmd == 5:
strcmd = 'RUN'
elif cmd >= 10 and cmd <= 30:
strcmd = 'POWER:' + str(cmd)
else:
strcmd = 'START'
conn.set('RFIDSysBluetoothCMD', strcmd)
return True
def RecvThreading(self, s):
#global thread_read_flg
while self.thread_read_flg:
try:
data = s.recv(1024)
except:
print('Bluetooth sock recv err')
self.allerr = True
break
if len(data) == 0:
continue #break
print("received [%s]" % data)
if b'START' in data:
self.SetSysBluetoothCMD(cmd=1)
elif b'STOP' in data:
self.SetSysBluetoothCMD(cmd=2)
elif b'POWERUP' in data:
self.SetSysBluetoothCMD(cmd=3)
elif b'POWERDOWN' in data:
self.SetSysBluetoothCMD(cmd=4)
elif b'POWER:' in data:
p, var = data.split(b':')
self.SetSysBluetoothCMD(cmd=int(var))
else:
pass
def StopRecvThread(self):
self.thread_read_flg = False
self.thread_read.join()
self.thread_read = None
self.SetSysBluetoothCMD(cmd=2)
def StopAll(self):
self.runallflg = False
self.StopRecvThread()
self.allthread = None
def EPCSendRoServerForBluetooth(self):
#global devID
#global usedtosendEPCQueue
#global thread_read_flg
#global thread_read
print('EPCSendRoServerForBluetooth')
#uuid = "94f39d29-7d6d-437d-973b-fba39e49d4ee"
uuid = self.uuid
try:
service_matches = find_service(uuid=uuid, address=None)
if len(service_matches) == 0:
print("couldn't find the SampleServer service =(")
return
first_match = service_matches[0]
port = first_match["port"]
name = first_match["name"]
host = first_match["host"]
print("connecting to \"%s\" on %s" % (name, host))
# Create the client socket
sock = BluetoothSocket(RFCOMM)
sock.connect((host, port))
self.thread_read_flg = True
self.thread_read = threading.Thread(target=self.RecvThreading,
args=(sock, ))
self.thread_read.setDaemon(True)
self.thread_read.start()
print("connected. type stuff")
except:
print("Bluetooth connect err")
return
try:
s_conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
if s_conn.ping():
print('EPCSendRoServerForBluetooth ok')
else:
print('shawanyi')
except: #
print('EPCSendRoServerForBluetooth err')
s_conn = None
return False
s_conn.set('RFIDSysIsStart', 'isok')
#self.SetSysBluetoothCMD(cmd=1)
while self.devID == b'':
retdevid = self.GetRfidDevID()
time.sleep(2)
self.allerr = False
self.SetSysBluetoothCMD(cmd=2)
try:
while True:
if self.allerr == True:
break
msg = self.usedtosendEPCQueue.get_nowait()
#print('msg:',msg)
if msg == None:
continue
varepc = msg.split(b':')
sdevs = b'DEVID:' + self.devID + b';' + varepc[1] + b'+'
print('queueSend:', sdevs)
sock.send(sdevs)
except:
print('Bluetooth sock err')
self.StopRecvThread()
#self.thread_read_flg=False
#self.thread_read.join()
#self.thread_read=None
#self.SetSysBluetoothCMD(cmd=2)
sock.close()
self.SetSysBluetoothCMD(cmd=2)
print('bluetooth stop -------------------------')
def EPCSendRoServerForBluetoothRUN(self):
if self.runallflg == True:
self.StopAll()
self.runallflg = True
while self.runallflg:
self.EPCSendRoServerForBluetooth()
def RunAll_Bluetooth(self):
if self.allthread != None:
return False
self.allthread = threading.Thread(
target=self.EPCSendRoServerForBluetoothRUN)
self.allthread.setDaemon(True)
self.allthread.start()
return True
def delect(name: str):
rq = RedisQueue(name)
if rq.qsize:
rq.lpop(name)
return {'ret': 0, 'msg': "删除成功"}
class Worker(Process):
def __init__(self, event_kill, name, concurrency, ip, max_time=-1):
self.queue_name = name
self.concurrency = concurrency
self.max_time = max_time
self.event_kill = event_kill
self.conn = RedisQueue(Redis(host=ip), self.queue_name)
super(Worker, self).__init__()
def safe_worker(self, func, return_dict, apply_max_time, body):
try:
return_dict['result'] = call(func, apply_max_time,
*body['args'], **body['kwargs'])
return_dict['success'] = True
except:
return_dict['result'] = traceback.format_exc()
return_dict['success'] = False
logging.error(return_dict['result'])
def safe_call(self, func, apply_max_time, body):
# os.setpgrp() # kill non propagate
if 'gevent' not in sys.modules:
return_dict = Manager().dict()
p = Process(target=self.safe_worker, args=(func, return_dict,
apply_max_time, body))
p.start()
p.join()
else:
return_dict = {}
self.safe_worker(func, return_dict, apply_max_time, body)
return return_dict
def callback(self, body):
logging.info("execute %s" % body['event'])
_id = body['args'][0]
if settings.ROCKETTM_CALLBACK:
send('results', {'_id': _id, 'status': 'processing'})
if not body['event'] in tasks.subs:
if settings.ROCKETTM_CALLBACK:
send('results', {'_id': _id,
'result': 'task not defined',
'status': 'finished',
'success': False})
return False
result = []
for func, max_time2 in tasks.subs[body['event']]:
logging.info("exec func: %s, timeout: %s" % (func, max_time2))
if max_time2 != -1:
apply_max_time = max_time2
else:
apply_max_time = self.max_time
result.append(dict(self.safe_call(func, apply_max_time, body)))
success = not any(r['success'] is False for r in result)
send('results', {'_id': _id, 'status': 'finished',
'success': success, 'result': result})
return True
def run(self):
while not self.event_kill.is_set():
try:
task = self.conn.get(timeout=20)
if task:
self.callback(task)
except:
logging.error(traceback.format_exc())
logging.error("connection loss, try reconnect")
time.sleep(5)
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os, sys
import time
import json
import redis
from redisqueue import RedisQueue
from redishash import RedisHash
def process(job):
if job["action"]=="process_file":
# { "jobid", "path", "name", "changed", "filesize" }
# ... do stuff ...
os.remove(job["path"])
# return fake entry
return { "status": "OK", "metadata": { "foo": 47 } }
else:
raise runtime_error("unknown job action '%s'" % job["action"])
if __name__=='__main__':
config= json.load(open("config.json"))
processq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-process-queue"])
resulthash= RedisHash(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-result-hash"])
resulthash.clear()
while True:
job= json.loads(processq.get())
print("processing job %s..." % job["jobid"])
result= process(job)
resulthash[job["jobid"]]= result
document['text'].append(unicode(o).encode("utf-8"))
print document
return document
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_section = "worker_solr"
worker_number = sys.argv[1]
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "solr_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
rdfdb_config = Config("%s/production.ini" % DB_ROOT)
granary_root = rdfdb_config.get("app:main", "granary.store", 0, {'here':DB_ROOT})
g = Granary(granary_root)
solr = SolrConnection(c.get(worker_section, "solrurl"))
idletime = 2
while(True):
line = rq.pop()
if line:
try:
import sys
from time import sleep
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_number = sys.argv[1]
worker_section = sys.argv[2]
if len(sys.argv) == 4:
if "redis_%s" % sys.argv[3] in c.sections():
redis_section = "redis_%s" % sys.argv[3]
rq = RedisQueue(c.get(worker_section, "listento"), "logger_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
with open(c.get(worker_section, "logfile"), "a+") as logfile:
while(True):
line = rq.pop()
if line:
try:
if line.endswith("\n"):
logfile.write(line)
rq.task_complete()
else:
logfile.writelines((line, "\n"))
rq.task_complete()
except Exception,e:
from redisqueue import RedisQueue
from redishash import RedisHash
def sqlconn(config):
conn= MySQLdb.connect( read_default_file=os.path.expanduser('~/replica.my.cnf'), host='tools.labsdb', use_unicode=True, cursorclass=MySQLdb.cursors.DictCursor )
cursor= conn.cursor()
return conn,cursor
def writeresumefile(config, job):
dir= config["download-dir"]
tmpfile= os.path.join(dir, "resume.tmp")
resumefile= os.path.join(dir, "resume.json")
with open(tmpfile, "w") as f:
f.write(json.dumps(job))
os.rename(tmpfile, resumefile)
if __name__=='__main__':
config= json.load(open("config.json"))
joborderq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-job-order-queue"])
resulthash= RedisHash(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-result-hash"])
conn,cursor= sqlconn(config)
while True:
nextjob= json.loads(joborderq.get())
while not nextjob["jobid"] in resulthash:
time.sleep(1.0)
result= resulthash[nextjob["jobid"]]
print("got result: %s" % json.dumps(result))
# xxx write result to db
writeresumefile(config, nextjob)
from redis import Redis
from redisqueue import RedisQueue
from easysnmp import Session, EasySNMPError
from worker import Pool
# loading .env file
load_dotenv()
# Init redis connection
memo = Redis(host=os.getenv('REDIS_IP'),
port=os.getenv('REDIS_PORT'),
db=os.getenv('REDIS_DB'))
# init redis queue
queue = RedisQueue(connection=memo, name=os.getenv('QUEUE'))
# init logging config
logging.basicConfig(filename=os.getenv('LOGGING'),
format="%(asctime)s - %(levelname)s - %(message)s",
level=getattr(logging,
os.getenv('LOG_LEVEL').upper()))
class ServiceExit(Exception):
"""
Custom exception which is used to trigger the clean exit
of all running threads and the main program.
"""
pass
import sys
from time import sleep
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_section = "worker_broker"
worker_number = sys.argv[1]
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "broker_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
if c.has_option(worker_section, "fanout_status_queue"):
# keep a queue of messages to deliver for a given push'd item
# better resumeability at the cost of more redis operations
topushq = RedisQueue(c.get(worker_section, "fanout_status_queue"), "fanout_broker_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
fanout_queues = [x.strip() for x in c.get(worker_section, "fanout").split(",") if x]
if c.has_option(worker_section, "idletime"):
try:
idletime = float(c.get(worker_section, "idletime"))
class SendDataToDestinationBLServer(object):
def __init__(self,
bluetuuid="94f39d29-7d6d-437d-973b-fba39e49d4ee",
name="FireFly-3288C",
port=6):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
self.devID = b''
self.uuid = bluetuuid
self.bluetoothname = name
self.bluetoothport = port
self.thread_read_flg = False
self.thread_read = None
self.runallflg = False
self.allthread = None
self.allerr = False
def GetRfidDevID(self):
if self.devID == b'':
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
if conn.ping():
print('conn ok')
else:
print('shawanyi')
except: #
return self.devID
#conn.set('RFIDDevID', devID)
if conn.exists('RFIDDevID'):
pass
else:
print(self.devID)
return self.devID
ret = conn.get('RFIDDevID')
if ret != self.devID:
self.devID = ret
print('devID', self.devID)
return self.devID
def GetSysBluetoothCMD(self):
cmd = 0
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
except:
print('GetSysBluetoothCMD ERR')
return 0
ret = conn.get('RFIDSysBluetoothCMD')
if ret == None:
return 0
if ret == b'START':
cmd = 1
elif ret == b'STOP':
cmd = 2
elif ret == b'POWERUP':
cmd = 3
elif ret == b'POWERDOWN':
cmd = 4
elif b'POWER:' in ret:
p, var = ret.split(b':')
cmd = int(var)
else:
cmd = 0
return cmd
def SetSysBluetoothCMD(self, cmd=0):
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
except:
print('GetSysBluetoothCMD ERR')
return False
strcmd = ''
#ret=conn.get('RFIDSysBluetoothCMD')
if cmd == 1: #b'START':
strcmd = 'START'
elif cmd == 2: #b'STOP':
strcmd = 'STOP'
elif cmd == 3: #b'POWERUP':
strcmd = 'POWERUP'
elif cmd == 4: #b'POWERDOWN':
strcmd = 'POWERDOWN'
elif cmd == 5:
strcmd = 'RUN'
elif cmd >= 10 and cmd <= 30:
strcmd = 'POWER:' + str(cmd)
else:
strcmd = 'START'
conn.set('RFIDSysBluetoothCMD', strcmd)
return True
def RecvThreading(self, s):
#global thread_read_flg
while self.thread_read_flg:
try:
data = s.recv(1024)
except:
print('Bluetooth sock recv err')
self.allerr = True
break
if len(data) == 0:
continue #break
print("received [%s]" % data)
if b'START' in data:
self.SetSysBluetoothCMD(cmd=1)
elif b'STOP' in data:
self.SetSysBluetoothCMD(cmd=2)
elif b'POWERUP' in data:
self.SetSysBluetoothCMD(cmd=3)
elif b'POWERDOWN' in data:
self.SetSysBluetoothCMD(cmd=4)
elif b'POWER:' in data:
p, var = data.split(b':')
self.SetSysBluetoothCMD(cmd=int(var))
else:
pass
def StopRecvThread(self):
self.thread_read_flg = False
self.thread_read.join()
self.thread_read = None
self.SetSysBluetoothCMD(cmd=2)
def StopAll(self):
self.runallflg = False
self.StopRecvThread()
self.allthread = None
def EPCSendRoServerForBluetooth(self):
print('EPCSendRoServerForBluetooth --- server bluetooth')
#uuid = "94f39d29-7d6d-437d-973b-fba39e49d4ee"
uuid = self.uuid
server_sock = BluetoothSocket(RFCOMM)
#server_sock.bind(("",self.bluetoothport))
server_sock.bind(("", PORT_ANY))
server_sock.listen(1)
port = server_sock.getsockname()[1]
advertise_service(
server_sock,
self.bluetoothname,
service_id=uuid,
service_classes=[uuid, SERIAL_PORT_CLASS],
profiles=[SERIAL_PORT_PROFILE],
)
print("Waiting for connection on RFCOMM channel %d" % port)
client_sock, client_info = server_sock.accept()
print("Accepted connection from ", client_info)
self.thread_read_flg = True
self.thread_read = threading.Thread(target=self.RecvThreading,
args=(client_sock, ))
self.thread_read.setDaemon(True)
self.thread_read.start()
print("connected. type stuff")
try:
s_conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
if s_conn.ping():
print('EPCSendRoServerForBluetooth ok')
else:
print('shawanyi')
except: #
print('EPCSendRoServerForBluetooth err')
s_conn = None
return False
s_conn.set('RFIDSysIsStart', 'isok')
#self.SetSysBluetoothCMD(cmd=1)
while self.devID == b'':
retdevid = self.GetRfidDevID()
time.sleep(2)
self.allerr = False
self.SetSysBluetoothCMD(cmd=2)
try:
while True:
if self.allerr == True:
break
msg = self.usedtosendEPCQueue.get_nowait()
#print('msg:',msg)
if msg == None:
continue
varepc = msg.split(b':')
sdevs = b'DEVID:' + self.devID + b';' + varepc[1] + b'+'
print('queueSend:', sdevs)
client_sock.send(sdevs)
except:
print('Bluetooth sock err')
self.StopRecvThread()
#self.thread_read_flg=False
#self.thread_read.join()
#self.thread_read=None
print('bluetooth exit thread-------------------------')
#self.SetSysBluetoothCMD(cmd=2)
client_sock.close()
server_sock.close()
self.SetSysBluetoothCMD(cmd=2)
print('bluetooth stop -------------------------')
def EPCSendRoClientForBluetoothRUN(self):
if self.runallflg == True:
self.StopAll()
self.runallflg = True
while self.runallflg:
self.EPCSendRoServerForBluetooth()
def RunAll_Bluetooth(self):
if self.allthread != None:
return False
self.allthread = threading.Thread(
target=self.EPCSendRoClientForBluetoothRUN)
self.allthread.setDaemon(True)
self.allthread.start()
return True
class TestBasicQueueFunctions(unittest.TestCase):
def setUp(self):
self.rq = RedisQueue("testqueue", "testworker01", db=10)
self.r = Redis(db=10)
self.r.delete("testqueue")
self.r.delete(self.rq.workeritem)
def test_add_to_queue(self):
self.rq.push("testitem")
self.assertEqual(len(self.rq), 1)
self.assertEqual(self.rq.pop(), "testitem")
def test_multiple_additions(self):
results = map(self.rq.push, xrange(10))
self.assertEquals(results, [True]*10)
def test_temp_workerqueue(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
def test_workerqueue_complete(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_complete()
item = self.rq.pop()
self.assertEqual(item, "nextitem")
self.assertEqual(len(self.rq), 0)
def test_requeue_on_fail(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_failed()
self.assertEqual(len(self.rq), 2)
def test_requeue_to_end_on_fail(self):
self.rq.push("testitem")
self.rq.push("nextitem")
# worker can pop item, but until it declares complete/fail,
# subsequent pops will return that same item
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_failed()
self.assertEqual(len(self.rq), 2)
item = self.rq.pop()
self.assertEqual(item, "nextitem")
self.assertEqual(len(self.rq), 1)
self.rq.task_complete()
item = self.rq.pop()
self.assertEqual(item, "testitem")
self.assertEqual(len(self.rq), 0)
class SendDataToDestinationBLC(object):
def __init__(self):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',host='127.0.0.1',psw='123456',db=1)
self.influxdbclass=PInfluxdbClass(host='127.0.0.1', port=8086, username='******', password='******', database = 'RFIDreportdb')
self.tagstoRedis=DataObjectJSRedisClass(host='127.0.0.1',psw='123456',port=6379,indexkey='EPC')
self.devID = b''
self.thread_read_flg=False
self.thread_read=None
self.runallflg=False
self.allthread=None
self.clientID=""
self.sendEmqClient=EMQSendClass(host='192.168.1.178',port=1883,username='******',pwd='shuiJing1',client_id='')
def GetRfidDevID(self):
if self.devID == b'':
try:
conn = redis.Redis(host='127.0.0.1', password='******', port=6379, db=1)
if conn.ping():
print('conn ok')
else:
print('shawanyi')
except: #
return self.devID
#conn.set('RFIDDevID', devID)
if conn.exists('RFIDDevID'):
pass
else:
print(self.devID)
return self.devID
ret=conn.get('RFIDDevID')
if ret!=self.devID:
self.devID=ret
print('devID',self.devID)
return self.devID
def GetSysBluetoothCMD(self):
cmd=0
try:
conn = redis.Redis(host='127.0.0.1', password='******', port=6379, db=1)
except:
print('GetSysBluetoothCMD ERR')
return 0
ret=conn.get('RFIDSysBluetoothCMD')
if ret==None:
return 0
if ret==b'START':
cmd=1
elif ret==b'STOP':
cmd=2
elif ret==b'POWERUP':
cmd=3
elif ret==b'POWERDOWN':
cmd=4
elif b'POWER:' in ret:
p,var = ret.split(b':')
cmd=int(var)
else:
cmd=0
return cmd
def SetSysBluetoothCMD(self,cmd=0):
try:
conn = redis.Redis(host='127.0.0.1', password='******', port=6379, db=1)
except:
print('GetSysBluetoothCMD ERR')
return False
strcmd=''
#ret=conn.get('RFIDSysBluetoothCMD')
if cmd==1:#b'START':
strcmd='START'
elif cmd==2:#b'STOP':
strcmd='STOP'
elif cmd==3:#b'POWERUP':
strcmd='POWERUP'
elif cmd==4:#b'POWERDOWN':
strcmd='POWERDOWN'
elif cmd==5:
strcmd='RUN'
elif cmd>=10 and cmd<=30:
strcmd='POWER:'+str(cmd)
else:
strcmd='START'
conn.set('RFIDSysBluetoothCMD',strcmd)
return True
def RecvThreading(self,s):
pass
def StopRecvThread(self):
self.thread_read_flg=False
self.thread_read.join()
self.thread_read=None
self.SetSysBluetoothCMD(cmd=2)
def StopAll(self):
self.runallflg=False
self.StopRecvThread()
self.allthread=None
def listtodict(self,data):
d={}
for v in data:
key,value=v.split(':')
d[key]=value
#key,value=v.split(b':')
#d[key.decode(encoding='utf-8')]=value.decode(encoding='utf-8')
d['devID']=self.devID
return d
def UseReportdata(self,data):
udict=self.MakeDataToDictMode(data)
print(udict)
if len(udict) <= 0:
return False
# self.influxdbclass.SaveDataToInfluxdb(measurement='RFIDreporttabl',tags={"NUMS":udict['NUMS']},fields=udict)
# print('Get:',self.influxdbclass.GetDataFrominfluxdb(measurement='RFIDreporttabl'))
for v in udict['tags']:
ret = self.tagstoRedis.AddObject(data=v)
print(ret)
if ret[1]==1:
print('GetObjectObjectlist:',self.tagstoRedis.GetObjectObjectlist())#获得对象hash name 列表
print('GetObjectindexKeylist:',self.tagstoRedis.GetObjectindexKeylist())#获得索引键列表
print('GetObjectKVlist:',self.tagstoRedis.GetObjectKVlist())#获得索引hash里 索引key+对象id列表
#发送数据
if self.clientID == "":
self.clientID=self.tagstoRedis.GetSelfUUID
print ('Get client uuid:',self.clientID)
self.sendEmqClient.PublishOnceS(topic='RFIDEPCTAG',payload=str(v),qos=1,retain=False, will=None, auth={'username':'******', 'password':'******'}, tls=None)
else:
print('GetObjectContentByName:',self.tagstoRedis.GetObjectContentByName(name=ret[0]))
nums=self.tagstoRedis.GetObjectSize()
if nums>50:
self.tagstoRedis.DeleteObjectAll()
print('It s new day')
return True
def MakeDataToDictMode(self,data):
retdict={}
print(data)
print(type(data))
strdata=data.decode(encoding='utf-8')
ldata=strdata.split(',')
nums, value = ldata[2].split(':')
#nums, value = data[2].split(b':')
if 'NUMS' == nums:
getnums = int(value, base=16)
if getnums <= 0:
print('UseReportdata is zero')
return retdict
retdict['NUMS']=getnums
tags=[]
epcdata=ldata[3:]
el=len(epcdata)
gl=int(el/getnums)
for i in range(getnums):
l=epcdata[i*gl:i*gl+gl]
d=self.listtodict(l)
tags.append(d)
retdict['tags']=tags
else:
print('nothing to do')
self.influxdbclass.SaveDataToInfluxdb(measurement='RFIDreporttabl',tags={"NUMS":retdict['NUMS']},fields={"reportRfid":strdata})
#print('Get:',self.influxdbclass.GetDataFrominfluxdb(measurement='RFIDreporttabl'))
return retdict
def EPCSendRoServerForBluetooth(self):
try:
s_conn = redis.Redis(host='127.0.0.1', password='******', port=6379, db=1)
if s_conn.ping():
print('EPCSendRoServerForBluetooth ok')
else:
print('shawanyi')
except: #
print('EPCSendRoServerForBluetooth err')
s_conn = None
return False
s_conn.set('RFIDSysIsStart','isok')
self.SetSysBluetoothCMD(cmd=1)
while self.devID==b'':
retdevid=self.GetRfidDevID()
time.sleep(2)
while True:
msg = self.usedtosendEPCQueue.get_nowait()
#print('msg:',msg)
if msg==None:
continue
self.UseReportdata(msg)
# try:
# while True:
# msg = self.usedtosendEPCQueue.get_nowait()
# #print('msg:',msg)
# if msg==None:
# continue
# self.UseReportdata(msg)
# #varepc = msg.split(b':')
# #sdevs = b'DEVID:' + self.devID + b';' + varepc[1] + b'+'
# #print('queueSend:',sdevs)
# #sock.send(sdevs)
# except:
# print('Bluetooth sock err')
self.StopRecvThread()
#self.thread_read_flg=False
#self.thread_read.join()
#self.thread_read=None
#self.SetSysBluetoothCMD(cmd=2)
sock.close()
def EPCSendRoServerForBluetoothRUN(self):
if self.runallflg==True:
self.StopAll()
self.runallflg=True
while self.runallflg:
self.EPCSendRoServerForBluetooth()
def RunAll_Bluetooth(self):
if self.allthread!=None:
return False
self.allthread=threading.Thread(target=self.EPCSendRoServerForBluetoothRUN)
self.allthread.setDaemon(True)
self.allthread.start()
return True
def setUp(self):
self.rq = RedisQueue("testqueue", "testworker01", db=10)
self.r = Redis(db=10)
self.r.delete("testqueue")
self.r.delete(self.rq.workeritem)
class Spider():
base_url = 'http://weixin.sogou.com/weixin'
keyword = 'NBA'
headers = {
'Accept': 'text / html, application / xhtml + xml, application / xml;q = 0.9, image / webp, image / apng, * / *;q = 0.8',
'Accept - Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Host': 'mp.weixin.qq.com',
'Referer': 'http://weixin.sogou.com/weixin?oq=&query=NBA&_sug_type_=1&sut=0&lkt=0%2C0%2C0&s_from=input&ri=5&_sug_=n&type=2&sst0=1530794048607&page=1&ie=utf8&p=40040108&dp=1&w=01015002&dr=1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
session = Session()
queue = RedisQueue()
mysql = MySQL()
def get_proxy(self):
try:
response = requests.get(PROXY_POOL_URL)
if response.status_code == 200:
print('get proxy', response.text)
return response.text
return None
except requests.ConnectionError:
return None
def parse_index(self, response):
'''
解析索引页
:return:
'''
doc = pq(response.text)
items = doc('.news-list > li > div.txt-box > h3 > a').items()
for item in items:
url = item.attr('href')
weixin_request = WeixinRequest(url=url, callback=self.parse_detail)
yield weixin_request
next = doc('#sogou_next').attr('href')
if next:
url = self.base_url + str(next)
weixin_request = WeixinRequest(url=url, callback=self.parse_index, need_proxy=False)
yield weixin_request
def parse_detail(self, response):
'''
解析详情页
:param response:
:return:
'''
doc = pq(response.text)
data = {
'title': doc('#activity-name').text(),
'content': doc('.rich_media_content').text(),
'date': doc('#publish_time').text(),
'nickname': doc('#js_profile_qrcode > div > strong').text(),
'wechat': doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
}
yield data
def schedule(self):
while not self.queue.empty():
weixin_request = self.queue.pop()
callback = weixin_request.callback
print('schedule', weixin_request.url)
response = self.request(weixin_request)
time.sleep(1)
if response and response.status_code in VALID_STATUS:
results = list(callback(response))
if results:
for result in results:
print('New result', result)
if isinstance(result, WeixinRequest):
self.queue.add(result)
if isinstance(result, dict):
self.mysql.insert('articles', result)
else:
self.error(weixin_request)
else:
self.error(weixin_request)
def error(self, weixin_request):
weixin_request.fail_time = weixin_request.fail_time + 1
print('Request failed', weixin_request.fail_time, 'times', weixin_request.url)
if weixin_request.fail_time < MAX_FAILED_TIME:
self.queue.add(weixin_request)
def request(self, weixin_request):
try:
if weixin_request.need_proxy:
proxy = self.get_proxy()
if proxy:
proxies = {
'http': 'http://' + proxy,
'https': 'http://' + proxy
}
return self.session.send(weixin_request.prepare(), timeout=weixin_request.timeout, proxies=proxies)
return self.session.send(weixin_request.prepare(), timeout=weixin_request.timeout)
except (requests.ConnectionError, requests.ReadTimeout) as e:
print(e.args)
return False
def start(self):
self.session.headers.update(self.headers)
start_url = self.base_url + '?' + urlencode({'query': self.keyword, 'type': 2})
weixin_request = WeixinRequest(url=start_url, callback=self.parse_index, need_proxy=True)
self.queue.add(weixin_request)
def run(self):
self.start()
self.schedule()
def makedatadir(config):
try:
os.makedirs(config["download-dir"])
except OSError as ex:
if ex[0]==17: # directory already exists
pass
else:
raise
def makeprocessqentry(config, jobid, path, imgname, timestamp, size):
return { "action": "process_file", "jobid": jobid, "path": path, "name": imgname, "changed": timestamp, "filesize": size }
if __name__=='__main__':
config= json.load(open("config.json"))
makedatadir(config)
downloadq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-download-queue"])
processq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-process-queue"])
session= requests.Session()
processq.clear()
while True:
row= json.loads(downloadq.get())
print "%s => " % row['name'].encode('utf-8'),
r= session.get(row['url'])
print(r.status_code)
if r.status_code!=200:
raise RuntimeError("requests.get(%s) returned %s" % (row['url'], r.status_code))
outputpath= os.path.join(os.path.expanduser(config["download-dir"]), row["name"])
with open(outputpath, "w") as f:
f.write(r.content)
def RFIDGetTagStart():
global issysset
global e
global pool
global redisconnpool
global GetTagProsseFlg
global epcsendthreadalive
sl = 1
if not SysIniJson(): # if not SysIni():
print('SysIni Erro')
return
sleeptimes = 0
if e.RunAllStart():
pass
else:
print('RFIDGetTagStart ERR Return')
return
pool = redis.ConnectionPool(host='127.0.0.1',
password='******',
port=6379,
db=1)
# q = RedisQueue(name='RfidControl',host='127.0.0.1',psw='123456')
try:
redisconnpool = redis.Redis(connection_pool=pool)
if redisconnpool.ping():
pass
except:
e.CloseSerialCOM()
return
q = RedisQueue(name='RfidControl', pool=pool)
if not q.istrue:
e.CloseSerialCOM()
print('CloseSerialCOM 0')
return
# while True:
# msg = q.get_nowait()
# print(msg)
# if msg != None:
# if msg==b'Rfidstop':
# issysset=199
# q.put_extend('RfidControlAnswer','RfidstopOK')
# break
# elif msg==b'Rfidstart':
# issysset=0;
# q.put_extend('RfidControlAnswer','RfidstartOK')
# break
# time.sleep(2)
# while True:
# SysCommand1(q)
# time.sleep(2)
epcsendthreadalive = 1
thread_sendepc = threading.Thread(target=EPCSendToServerThread,
name='EPCSendToServerThread')
thread_sendepc.setDaemon(True)
thread_sendepc.start()
while issysset < 200:
# msg = redis_sub.parse_response()
msg = q.get_nowait()
if msg != None:
print(msg)
if msg == b'Rfidstop':
issysset = 199
q.put_extend('RfidControlAnswer', 'RfidstopOK')
elif msg == b'Rfidstart':
issysset = 0
q.put_extend('RfidControlAnswer', 'RfidstartOK')
# SysCommand1(q)
# SysCommand(q=q,flg=1)
if issysset != 199:
# EPCProsseRedis()
EPCProsseRedis1()
if issysset == 0:
sendcmd = RfidCMD.IsAppoBootLoad()
print('IsAppoBootLoad:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 1:
sendcmd = RfidCMD.BtoApp()
print('BtoApp:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 2:
sendcmd = RfidCMD.GetDNum()
# sendcmd=RfidCMD.ApptoB()
print('GetDNum:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 6:
GetTagProsseFlg += 1
if GetTagProsseFlg < int(sysdict['ts']):
time.sleep(sl)
continue
sendcmd = RfidCMD.GetTagMultiple()
GetTagProsseFlg = 0
print('GetTagMultiple:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
sleeptimes = 0
issysset = 106
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 7:
sendcmd = RfidCMD.GetTagMultipleCommand()
print('GetTagMultiple:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
sleeptimes = 0
issysset = 107
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 106:
sleeptimes += 1
if sleeptimes > 10:
issysset = 6
loggererro.debug('106Sleep')
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 107:
sleeptimes += 1
if sleeptimes > 10:
issysset = 7
loggererro.debug('107Sleep')
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 8:
sendcmd = RfidCMD.SubSetGetTagData()
print('GetTagMultiple:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 9:
# if usedAnt==1:
# sendcmd=RfidCMD.SetAntEnable(ant1=usedAnt,ant2=0,ant3=0,ant4=0)
# elif usedAnt==2:
# sendcmd=RfidCMD.SetAntEnable(ant1=0,ant2=usedAnt,ant3=0,ant4=0)
# elif usedAnt==3:
# sendcmd=RfidCMD.SetAntEnable(ant1=0,ant2=0,ant3=usedAnt,ant4=0)
# elif usedAnt==4:
# sendcmd=RfidCMD.SetAntEnable(ant1=0,ant2=0,ant3=0,ant4=usedAnt)
# else:
# sendcmd=RfidCMD.SetAntEnable(ant1=1,ant2=0,ant3=0,ant4=0)
sendcmd = RfidCMD.SetAntEnable(ant1=int(sysdict['a1']),
ant2=int(sysdict['a2']),
ant3=int(sysdict['a3']),
ant4=int(sysdict['a4']))
print('SetAntEnable:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 10:
# if usedAnt==1:
# sendcmd=RfidCMD.SetAntPower(ant1=usedAnt,ant2=0,ant3=0,ant4=0,rpower=usedPower)
# elif usedAnt==2:
# sendcmd=RfidCMD.SetAntPower(ant1=0,ant2=usedAnt,ant3=0,ant4=0,rpower=usedPower)
# elif usedAnt==3:
# sendcmd=RfidCMD.SetAntPower(ant1=0,ant2=0,ant3=usedAnt,ant4=0,rpower=usedPower)
# elif usedAnt==4:
# sendcmd=RfidCMD.SetAntPower(ant1=0,ant2=0,ant3=0,ant4=usedAnt,rpower=usedPower)
# else:
# sendcmd=RfidCMD.SetAntPower()
sendcmd = RfidCMD.SetAntPower(ant1=int(sysdict['a1']),
ant2=int(sysdict['a2']),
ant3=int(sysdict['a3']),
ant4=int(sysdict['a4']),
rpower=int(sysdict['aw']))
print('SetAntPower:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
else:
time.sleep(sl) # time.sleep(2)
col['resume']= resume
col['url']= getCommonsUrl(col['img_name'])
yield col
count+= 1
if limit and count>=limit:
return
def makeorderqentry(jobid, resume):
return { "jobid": jobid, "resume": resume }
def makedownloadqentry(config, jobid, name,url,timestamp,size):
return { "jobid": jobid, "name": name, "url": url, "changed": timestamp, "filesize": size }
if __name__=='__main__':
config= json.load(open("config.json"))
joborderq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-job-order-queue"])
downloadq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-download-queue"])
joborderq.clear()
downloadq.clear()
# xxxx remove stale files?
jobid= 1
for row in commonsfiles(sortkey='img_sha1', limit=50):
# wait for queue to shrink
# xxx todo: it would be nicer to have a blocking version of this, instead of polling every second
while joborderq.qsize()>=config["redis-max-queued-jobs"]:
time.sleep(1)
joborderq.put(json.dumps(makeorderqentry(jobid, row['resume'])))
downloadq.put(json.dumps(makedownloadqentry(config, jobid, row["img_name"], row["url"], row["img_timestamp"], row["img_size"])))
