def job_runner(): redis = get_redis_connection() while True: job_definition = json.loads(redis.blpop("jobs")) run_job( job_definition["id"], job_definition["date_range"], job_definition["start_date"], job_definition["end_date"] )
def get_msg(self, queue="kohlrabi-tasks"): """ Get a new message off of the redis queue. This will automatically msgpack-decode the data, so don't use if you want to get raw values. :param queue: The queue to pick a message from. """ with (yield from self.redis_conn) as redis: assert isinstance(redis, aioredis.Redis) data = (yield from redis.blpop(queue))[1] return msgpack.unpackb(data, encoding='utf-8')
def get_message_response(self, msg): try: redis = self._get_redis() response = redis.blpop(get_redis_queue_name_by_message(msg), timeout=60) if response and len(response) > 1: response = Message.build(response[1]) else: response = None return response except Exception as e: self.logger.error( 'Error while processing response to {}: {}'.format( msg, str(e)))
def classify(url): try: response = requests.get(url, timeout=args.timeout) jpeg_base64 = resize_encode(response.content) dispatch = queue.send_message(MessageBody=json.dumps({ 'image': { 'data_uri': 'data:image/jpg;base64,{}'.format(jpeg_base64) } })) listname = '{}{}'.format(args.redis_prefix, dispatch['MessageId']) _, result = redis.blpop(listname) redis.delete(listname) except Exception as e: result = {'status': 'error', 'message': str(e)} return result
async def main(loop): global browser while True: try: print("[+] Handler has begun..") data = redis.blpop("query")[1].decode().split("/") if data: uuid, url = data[0], '/'.join(data[1:]) print(f"[*] Got a new request: {uuid} => {url}") if not browser: browser = await init_browser() _task = await asyncio.wait_for(render(uuid, url), timeout=5) except Exception as e: print(f"[!] {e}") finally: await asyncio.sleep(0.01)
while redis.lpop("musicaload") is not None: pass for ent in redis.lrange("musicaqueue", 0, -1): redis.rpush("musicaload", json.loads(ent.decode())["ytid"]) def gen_cmdline(ytid, for_title=False): return [YOUTUBE_DL, "--no-playlist", "--id", "--no-progress", "--format", "mp4"] + (["--get-title"] if for_title else []) + ["--", sanitize(ytid)] def get_title(ytid): return subprocess.check_output(gen_cmdline(ytid, for_title=True)) # "mplayer -fs" while True: _, to_load = redis.blpop("musicaload") try: to_load = to_load.decode() if redis.get("musicatitle." + to_load) is None or redis.get("musicatitle." + to_load).startswith(b"Could not load video "): redis.set("musicatitle." + to_load, get_title(to_load).strip()) if not os.path.exists(path_for(to_load)): if subprocess.call(gen_cmdline(to_load), cwd=DATA_DIR) != 0: redis.set("musicatitle." + to_load, ("Could not load video %s" % (to_load,)).encode()) continue subprocess.check_call(gen_cmdline(to_load), cwd=DATA_DIR) assert os.path.exists(path_for(to_load)) except: print("Failed to load.") traceback.print_exc()
def processMessage(data): print data in_path = data['result']['file_path'].encode(encoding='UTF-8') print "in_path:", in_path, type(in_path) url = "https://api.telegram.org/file/bot{}/{}".format(bot_token, in_path) print "getting image:", url wget.download(url, in_path) out_path = 'static/' + in_path #checkpoint_dir = 'checkpoints/udnie.ckpt' checkpoint_dir = random.choice(ckpt_files) ffwd_to_img(in_path, out_path, checkpoint_dir) print "done image" sendPhoto(data['chat_id'], "https://dl2.rohbot.cc/" + out_path) data = {'chat_id': 700442415, 'result': {'file_path': u'photos/file_18.jpg'}} print data #processMessage(data) while True: item = redis.blpop("queue:style") data = json.loads(item[1]) processMessage(data)
import os from subprocess import call, check_output import shutil import logging from config import TARGET_SECONDS, HOME_DIR, LOG_FILE import redis redis = redis.StrictRedis() logging.basicConfig(filename=LOG_FILE, level=logging.WARNING) os.chdir(HOME_DIR) while 1: id = redis.blpop('video')[1] logging.warning(os.environ['PATH']) logging.warning(os.environ['HOME']) # Get each frames time try: a = check_output(['identify', '-format', '"Frame %s: %Tcs\n"', 'uploads/' + id + '.gif']) except: logging.warning('identify error') redis.set(id, 'error') continue lengths = [] for l in a.split('\n'):
'language': language, 'status': 'doing' } res = observer.add_observer(data) if res['message'] is 'created': return res['id'] else: return None if __name__ == '__main__': chrome.run() while True: try: task_code, task_data = redis.blpop([REDIS_TOPIC_OBSERVER_URLS], 0) if task_data: urlData = task_data.decode("utf-8") urlObj = json.loads(urlData) if urlObj is not None: url = urlObj.get('url', None) if url is not None: language = urlObj['language'] id = register(url, language) if id is not None: detector.run(id, url, language) except Exception as ex: traceback.print_exc() print(ex)
import redis import json from time import sleep from random import randint if __name__ == '__main__': redis = redis.Redis(host='queue', port='6379', db='0') while True: mail = json.loads(redis.blpop('sender')[1]) print('Sending mail:', mail['subject'], '...') sleep(randint(10, 30)) print('mail', mail['subject'], 'sent with success!')
#!/usr/local/bin/python3 # -*- coding: utf-8 -*- import redis import biquke from crawler import Crawler redis = redis.Redis(connection_pool=Crawler.pool) while True: temp = redis.blpop("tq_book_https://www.biquke.com/")[1] print(temp) url0 = bytes.decode(temp) print("take url is :", url0) bs0 = biquke.Biquke1(url0) bs0.start() bs0.join()
def pross(): params = dict() params["logging_level"] = 3 params["output_resolution"] = "-1x-1" params["net_resolution"] = "-1x368" params["model_pose"] = "BODY_25" params["alpha_pose"] = 0.6 params["scale_gap"] = 0.3 params["scale_number"] = 1 params["render_threshold"] = 0.05 # If GPU version is built, and multiple GPUs are available, set the ID here params["num_gpu_start"] = 0 params["disable_blending"] = False # Ensure you point to the correct path where models are located params[ "default_model_folder"] = "/woody/software/source/openpose2019/models/" # Construct OpenPose object allocates GPU memory openpose = OpenPose(params) websocket_server = Websocket(9000, servers=[], new_client=new_client, client_left=client_left, message_received=message_received) websocket_server.start() while True: if websocket_server.servers is not None: break video_path = "/home/woody/tmp/openpose/test1.mp4" #video_path = "/woody/software/source/openpose/examples/media/video.avi" f_name = "test1.mp4" #video_path = "rtsp://172.16.3.26/test" #video_path = "/home/woody/tmp/openpose/video/4804_exit_overvie.mp4" video = cv2.VideoCapture() if not video.open(video_path): logger.info("can not open the video") exit(1) index = 1 count = 0 f_count = 1 imageArray = {} continuityArray = [] up_imageArray = [] images = [] start = time.time() while True: st = time.time() _, vlc_key = redis.blpop("vlcreader_key") logger.info("redis0>>>" + str(time.time() - st)) print(vlc_key.decode()) #_, frame = video.read() #if frame is None: # break frame_dump = redis.get(vlc_key.decode()) logger.info("redis1>>>" + str(time.time() - st)) frame = pickle.loads(frame_dump) logger.info("redis2>>>" + str(time.time() - st)) #cv2.imwrite("/home/woody/tmp/openpose/xxxx/" + str(index) + ".jpg", np.array(frame), [cv2.IMWRITE_JPEG_QUALITY, 90]) base64_data = frame2base64(frame) data_json = {} data_json["type"] = 1 data_json["image"] = "data:image/jpeg;base64," + base64_data.decode() websocket_server.servers[0].send_message_to_all( str(json.dumps(data_json))) if f_count % 1 == 0: st = time.time() output_image = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR) keypoints, scores = openpose.forward(output_image, False) logger.info("openpose>>>" + str(time.time() - st)) image = output_image continuity = False key = f_name[:-4] + "_" + str(index) for keypoint in keypoints: flag = False bule_flag, bule_totail = calcBuleRate( image, [keypoint[12][0], keypoint[12][1]], [keypoint[9][0], keypoint[9][1]], [keypoint[2][0], keypoint[2][1]], [keypoint[5][0], keypoint[5][1]]) if bule_flag == False: continue x1 = keypoint[2][0] y1 = keypoint[2][1] x2 = keypoint[9][0] y2 = keypoint[9][1] x3 = keypoint[10][0] y3 = keypoint[10][1] r_result, r_flag = calcHipAngle(x1, y1, x2, y2, x3, y3) x1 = keypoint[5][0] y1 = keypoint[5][1] x2 = keypoint[12][0] y2 = keypoint[12][1] x3 = keypoint[13][0] y3 = keypoint[13][1] l_result, l_flag = calcHipAngle(x1, y1, x2, y2, x3, y3) x1 = keypoint[9][0] y1 = keypoint[9][1] x2 = keypoint[10][0] y2 = keypoint[10][1] x3 = keypoint[11][0] y3 = keypoint[11][1] ra_result, ra_flag = calcKneeAngle(x1, y1, x2, y2, x3, y3) ra_len_result, ra_len_flag = calcLenRate( x1, y1, x2, y2, x3, y3) x1 = keypoint[12][0] y1 = keypoint[12][1] x2 = keypoint[13][0] y2 = keypoint[13][1] x3 = keypoint[14][0] y3 = keypoint[14][1] la_result, la_flag = calcKneeAngle(x1, y1, x2, y2, x3, y3) la_len_result, la_len_flag = calcLenRate( x1, y1, x2, y2, x3, y3) if ra_flag and la_flag: flag = True if ra_len_flag and la_len_flag: flag = True if (r_flag or l_flag) and (abs(r_result - l_result) <= 30) and (ra_result or la_result): flag = True #if la_result >= 170 or ra_result >= 170: # flag = False if (la_len_result >= 0.9 and la_len_result <= 1.09) or ( ra_len_result >= 0.9 and ra_len_result <= 1.09): flag = False if flag: continuityArray.append(keypoint[8]) continuity = True logger.info(key + " >>> sat_list >>>>>>>>>" + str(flag) + ">>>>>" + str(continuity)) if continuity: v_flag = False if len(up_imageArray) >= 1: if len(continuityArray) >= 1: for u_index in range(len(up_imageArray)): for c_index in range(len(continuityArray)): midHip2 = ( up_imageArray[u_index][0] - continuityArray[c_index][0]) * ( up_imageArray[u_index][0] - continuityArray[c_index][0]) + ( up_imageArray[u_index][1] - continuityArray[c_index][1]) * ( up_imageArray[u_index][1] - continuityArray[c_index][1]) midHip = cmath.sqrt(midHip2) f_midHip = cmath.sqrt( 0.09 * (up_imageArray[u_index][0] + up_imageArray[u_index][1]) * (up_imageArray[u_index][0] + up_imageArray[u_index][1])) if midHip.real < f_midHip.real: v_flag = True break #logger.info(str(key) + ">>>" + str(midHip.real) + "," + str(f_midHip.real)) if v_flag: break up_imageArray = continuityArray continuityArray = [] if count < 1: v_flag = True if v_flag: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/hh", str(key)) cv2.imwrite(save_path, image) if 'index' in imageArray.keys(): if imageArray['start'] >= count - E_TOTAIL: if imageArray['index'] == count - 1: imageArray['start'] = count imageArray['count'] = imageArray['count'] + 1 if imageArray['count'] >= APM_TOTAIL: if imageArray['apm'] == False: imageArray['apm'] = True save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/test", str(imageArray['key'])) cv2.imwrite(save_path, imageArray["image"]) base64_data = frame2base64( imageArray["image"]) data_json = {} data_json["type"] = 2 data_json[ "image"] = "data:image/jpeg;base64," + base64_data.decode( ) websocket_server.servers[ 0].send_message_to_all( str(json.dumps(data_json))) ws_data_list.append(data_json) image_data = {} image_data["key"] = key image_data["image"] = image images.append(image_data) for image_info in images: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/li", str(image_info['key'])) cv2.imwrite( save_path, image_info["image"]) images = [] imageArray['index'] = count if imageArray['apm'] == False: image_data = {} image_data["key"] = key image_data["image"] = image images.append(image_data) else: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/test", str(imageArray['key'])) if imageArray['apm'] == False: if imageArray['count'] >= APM_TOTAIL: imageArray['apm'] = True cv2.imwrite(save_path, imageArray["image"]) base64_data = frame2base64( imageArray["image"]) data_json = {} data_json["type"] = 2 data_json[ "image"] = "data:image/jpeg;base64," + base64_data.decode( ) websocket_server.servers[ 0].send_message_to_all( str(json.dumps(data_json))) ws_data_list.append(data_json) image_data = {} image_data["key"] = key image_data["image"] = image images.append(image_data) for image_info in images: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/li", str(image_info['key'])) cv2.imwrite(save_path, image_info["image"]) images = [] imageArray = {} images = [] else: imageArray['index'] = count imageArray['start'] = count imageArray['count'] = 0 imageArray['apm'] = False imageArray['image'] = output_image imageArray['key'] = key else: if 'index' in imageArray.keys(): imageArray['index'] = count save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/hh", str(key)) cv2.imwrite(save_path, image) else: if 'index' in imageArray.keys(): if imageArray['start'] < count - E_TOTAIL: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/test", str(imageArray['key'])) if imageArray['apm'] == False: if imageArray['count'] >= APM_TOTAIL: imageArray['apm'] = True cv2.imwrite(save_path, imageArray["image"]) base64_data = frame2base64(imageArray["image"]) data_json = {} data_json["type"] = 2 data_json[ "image"] = "data:image/jpeg;base64," + base64_data.decode( ) websocket_server.servers[ 0].send_message_to_all( str(json.dumps(data_json))) ws_data_list.append(data_json) for image_info in images: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/li", str(image_info['key'])) cv2.imwrite(save_path, image_info["image"]) images = [] imageArray = {} images = [] else: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/wu", str(key)) cv2.imwrite(save_path, image) up_imageArray = [] continuityArray = [] count = count + 1 logger.info("end ===============" + key) logger.info(str(count) + ",totail>>>" + str(time.time() - start)) index += 1 f_count = f_count + 1 if 'index' in imageArray.keys(): if imageArray['start'] < count - E_TOTAIL: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/test", str(imageArray['key'])) if imageArray['apm'] == False: if imageArray['count'] >= APM_TOTAIL: imageArray['apm'] = True cv2.imwrite(save_path, imageArray["image"]) base64_data = frame2base64(imageArray["image"]) data_json = {} data_json["type"] = 2 data_json[ "image"] = "data:image/jpeg;base64," + base64_data.decode( ) websocket_server.servers[0].send_message_to_all( str(json.dumps(data_json))) ws_data_list.append(data_json) for image_info in images: save_path = "{}/{:>03s}.jpg".format( "/home/woody/tmp/openpose/li", str(image_info['key'])) cv2.imwrite(save_path, image_info["image"]) images = [] imageArray = {} images = [] logger.info(">>>" + str(time.time() - start)) video.release() logger.info("Totally save {:d} pics".format(index - 1))
#!/usr/bin/python3 # -*- coding: utf-8 -*- import redis import biquke from crawler import Crawler b0 = biquke.Biquke0("https://www.biquke.com/") b0.start() redis = redis.Redis(connection_pool=Crawler.pool) while True : temp = redis.blpop(b0.bookpfx + b0.start_url)[1] print(temp) url0 = bytes.decode(temp) print("take url is :", url0) bs0 = biquke.Biquke1(url0) temp = redis.blpop(b0.bookpfx + b0.start_url)[1] url1 = bytes.decode(temp) print("take url is :", url1) bs1 = biquke.Biquke1(url1) bs0.start() bs1.start() bs0.join() bs1.join()
import pymongo import json,redis redis = redis.Redis(host="localhost",port=6379,db=0) clicent = pymongo.MongoClient() collection = clicent.qiushi.duanzi while True: # 这只是取一个记录 key, data = redis.blpop(["qiushi:items"]) print(key) d = json.loads(data) print(d) collection.insert_one(d)