def creat(item: Item): """ 通过该接口可以创建一个抽奖规则 """ myredis = MyRedis() rq = RedisQueue(item.luckname) print("ok") if rq.qsize(): return {"ret": 500, "msg": "该抽奖已经存在,请删除后重试"} result = {"ret": 0, "item": item} awardlist = item.award lucklist = [] luckdict = {} for ward in awardlist: luckdict[ward.name] = ward.count for i in range(ward.count): lucklist.append(ward.name) othercount = item.total - len(lucklist) if othercount: luckdict[item.other] = othercount others = [item.other] * othercount lucklist = lucklist + others random.shuffle(lucklist) print(lucklist) for luck in lucklist: rq.put(luck) myredis.hmset(item.luckname, luckdict) result = {'ret': 0, 'msg': "succses"} return result
def _run_script_background(self, command, username, region): p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(0.1) # forget why i sleep here, if not sure, don't modify this rq = RedisQueue('script_serviceDeploy_%s_%s' % (region, username)) rq.delete_queue() # todo: delete key only stdout, stderr = p.communicate() ret_code = p.poll() if not ret_code: script_result = stdout logger.info('run command: %s success, output:\n%s' % (command, stdout)) rq.put('*finished*') self.add_script_exec_log(username, self.script_name, self.script_path, True, script_result) else: script_result = stdout + stderr logger.error('run command: %s failed, output:\n%s' % (command, script_result)) errout = [err for err in stderr.splitlines() if err] errout = '\n'.join(errout) rq.put('*failed*%s' % errout) self.add_script_exec_log(username, self.script_name, self.script_path, False, script_result)
def start(): round_robin = 0 while True: try: manager_queue = RedisQueue('manager', host=settings.REDIS_HOST, port=settings.REDIS_PORT) # get next message - first find an available queue keys = RedisKeys(pattern='rendertaskqueue:*', host=settings.REDIS_HOST, port=settings.REDIS_PORT).keys if keys is not None and len(keys) > 0: round_robin = BlendClusterClient.next_queue(round_robin, keys) queue_name = keys[round_robin] # just grab the first queue name print 'round robin: ' + queue_name queue = RedisQueue('namechangedbelow', host=settings.REDIS_HOST, port=settings.REDIS_PORT) queue.key = queue_name # override key name (queue name) here as we have the qualified name print 'getting render task message' render_task_json = queue.get() print render_task_json render_task_obj = RenderTaskMessage.json_to_object(render_task_json) hg_url = render_task_obj.hg_url blend_file = render_task_obj.blend_file # notify manager we are working msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id, 'state': 1}) # set working state manager_queue.put(msg) print 'Set task to WORKING state: ' + str(render_task_obj.id) print '-------------------------' print 'WORKING' print '-------------------------' # so some work print 'About to prepare hg' hg = HgUtils(url=hg_url) print 'about to clone or pull hg' hg.clone_or_pull() full_blend_filename = hg.local_project_path + '/' + render_task_obj.blend_file short_blend_filename = ntpath.basename(full_blend_filename).split('.')[0] render_dir = hg.local_project_path + '/renders/' + short_blend_filename + '/'\ + render_task_obj.scene_name + '/' print 'Rendering in dir: ' + render_dir if not os.path.exists(render_dir): os.makedirs(render_dir) hg.push_all_changes() blender = Blender(blend_file=full_blend_filename, scene_name=render_task_obj.scene_name, start_frame=render_task_obj.frame, end_frame=render_task_obj.frame, output_dir=render_dir) blender.run() # before pushing changes update local repo hg.get_latest_changes() hg.push_all_changes() print 'done hg' msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id, 'state': 4}) # set success state manager_queue.put(msg) print '-------------------------' print 'SUCCESSFUL' print '-------------------------' else: print('No messages available right now, will try again in 5 seconds...') time.sleep(5) # wait 5 secs before trying again except Exception as e: print('Error raised: ' + e.message) try: msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id, 'state': 3}) # set working state failed manager_queue.put(msg) print '-------------------------' print 'FAILED' print '-------------------------' except: print('Trying to respond with a failure status but could not!') print('Will try again in 5 seconds...') time.sleep(5) # wait 5 secs before trying again
def handle(self): print(self.client_address) sqstr = '{0}:{1}'.format(self.client_address[0], self.client_address[1]) print(sqstr) q = RedisQueue(name=sqstr) #q.redisidrun=q.isrun() #print ('running') while True: try: data = self.request.recv(1024) if (data == b''): try: conn = psycopg2.connect(database="materiel", user="******", password="******", host="58.214.232.165", port="5432") cur = conn.cursor() print('lin postgresql OK') except: print('InsertToPostgresql Connect Fail') break #ProssData(q) while True: if q.isrun() == 0: break qdata = q.get_nowait() if qdata == None: try: conn.commit() conn.close() print('InsertToPostgresql close') break except: print('InsertToPostgresql commit Fail') break #break print(qdata) if b'DEVID' in qdata: devid, value = qdata.split(b':') print('did:{0}'.format(devid)) lisoo = value.split(b';') print('did:{0}-epc:{1}'.format(lisoo[0], lisoo[1])) sqlselectstr = ( "INSERT INTO tb_epc_record (device_id,epc,report_time) VALUES ('{0}','{1}',NOW()::timestamp)" ).format(lisoo[0].decode(encoding='utf-8'), lisoo[1].decode(encoding='utf-8')) print(sqlselectstr) cur.execute(sqlselectstr) break #continue #print (data) q.put(data) #if data == b'exit': # print ('exit') # self.server.shutdown() # self.request.close() # break except Exception as e: self.server.shutdown() self.request.close() break
if ex[0]==17: # directory already exists pass else: raise def makeprocessqentry(config, jobid, path, imgname, timestamp, size): return { "action": "process_file", "jobid": jobid, "path": path, "name": imgname, "changed": timestamp, "filesize": size } if __name__=='__main__': config= json.load(open("config.json")) makedatadir(config) downloadq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-download-queue"]) processq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-process-queue"]) session= requests.Session() processq.clear() while True: row= json.loads(downloadq.get()) print "%s => " % row['name'].encode('utf-8'), r= session.get(row['url']) print(r.status_code) if r.status_code!=200: raise RuntimeError("requests.get(%s) returned %s" % (row['url'], r.status_code)) outputpath= os.path.join(os.path.expanduser(config["download-dir"]), row["name"]) with open(outputpath, "w") as f: f.write(r.content) processq.put(json.dumps(makeprocessqentry(config, row["jobid"], outputpath, row["name"], row["changed"], row["filesize"])))
def test_live_queue(): live_queue = RedisQueue("test_queue", MockTask, namespace="pytest") assert live_queue.connected is False task = MockTask() task2 = MockTask() task.uri = 'thisIsUnique' task2.uri = 'thisIsUnique' assert live_queue.connect(host=live_host, port=live_port, password=live_pass) is True assert live_queue.connected is True live_queue.clear() assert live_queue.qsize == 0 live_queue.put(task) assert live_queue.qsize == 1 live_queue.put(task2) assert live_queue.qsize == 2 new_task = live_queue.get() assert isinstance(new_task, MockTask) assert new_task.uid == task.uid assert new_task.uri == 'thisIsUnique' live_queue.clear() task.unique = True task2.unique = True assert task.unique_hash() == task2.unique_hash() live_queue.put(task) with pytest.raises(TaskAlreadyInQueueException): live_queue.put(task2) assert live_queue.qsize == 1 live_queue.clear() # test getting and putting the same task into the queue assert live_queue.qsize == 0 live_queue.put(task) my_task = live_queue.get() live_queue.put(my_task) assert live_queue.qsize == 1
return def makeorderqentry(jobid, resume): return { "jobid": jobid, "resume": resume } def makedownloadqentry(config, jobid, name,url,timestamp,size): return { "jobid": jobid, "name": name, "url": url, "changed": timestamp, "filesize": size } if __name__=='__main__': config= json.load(open("config.json")) joborderq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-job-order-queue"]) downloadq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-download-queue"]) joborderq.clear() downloadq.clear() # xxxx remove stale files? jobid= 1 for row in commonsfiles(sortkey='img_sha1', limit=50): # wait for queue to shrink # xxx todo: it would be nicer to have a blocking version of this, instead of polling every second while joborderq.qsize()>=config["redis-max-queued-jobs"]: time.sleep(1) joborderq.put(json.dumps(makeorderqentry(jobid, row['resume']))) downloadq.put(json.dumps(makedownloadqentry(config, jobid, row["img_name"], row["url"], row["img_timestamp"], row["img_size"]))) print("pushed job %d (%s)..." % (jobid, row['img_name'])) jobid+= 1 print("filequeue task done, exiting.")