def job_runner():
redis = get_redis_connection()
while True:
job_definition = json.loads(redis.blpop("jobs"))
run_job(
job_definition["id"],
job_definition["date_range"],
job_definition["start_date"],
job_definition["end_date"]
)
def get_msg(self, queue="kohlrabi-tasks"):
"""
Get a new message off of the redis queue.
This will automatically msgpack-decode the data, so don't use if you want to get raw values.
:param queue: The queue to pick a message from.
"""
with (yield from self.redis_conn) as redis:
assert isinstance(redis, aioredis.Redis)
data = (yield from redis.blpop(queue))[1]
return msgpack.unpackb(data, encoding='utf-8')
def get_msg(self, queue="kohlrabi-tasks"):
"""
Get a new message off of the redis queue.
This will automatically msgpack-decode the data, so don't use if you want to get raw values.
:param queue: The queue to pick a message from.
"""
with (yield from self.redis_conn) as redis:
assert isinstance(redis, aioredis.Redis)
data = (yield from redis.blpop(queue))[1]
return msgpack.unpackb(data, encoding='utf-8')
def get_message_response(self, msg):
try:
redis = self._get_redis()
response = redis.blpop(get_redis_queue_name_by_message(msg),
timeout=60)
if response and len(response) > 1:
response = Message.build(response[1])
else:
response = None
return response
except Exception as e:
self.logger.error(
'Error while processing response to {}: {}'.format(
msg, str(e)))
def classify(url):
try:
response = requests.get(url, timeout=args.timeout)
jpeg_base64 = resize_encode(response.content)
dispatch = queue.send_message(MessageBody=json.dumps({
'image': {
'data_uri': 'data:image/jpg;base64,{}'.format(jpeg_base64)
}
}))
listname = '{}{}'.format(args.redis_prefix, dispatch['MessageId'])
_, result = redis.blpop(listname)
redis.delete(listname)
except Exception as e:
result = {'status': 'error', 'message': str(e)}
return result
async def main(loop):
global browser
while True:
try:
print("[+] Handler has begun..")
data = redis.blpop("query")[1].decode().split("/")
if data:
uuid, url = data[0], '/'.join(data[1:])
print(f"[*] Got a new request: {uuid} => {url}")
if not browser:
browser = await init_browser()
_task = await asyncio.wait_for(render(uuid, url), timeout=5)
except Exception as e:
print(f"[!] {e}")
finally:
await asyncio.sleep(0.01)
while redis.lpop("musicaload") is not None:
pass
for ent in redis.lrange("musicaqueue", 0, -1):
redis.rpush("musicaload", json.loads(ent.decode())["ytid"])
def gen_cmdline(ytid, for_title=False):
return [YOUTUBE_DL, "--no-playlist", "--id", "--no-progress", "--format", "mp4"] + (["--get-title"] if for_title else []) + ["--", sanitize(ytid)]
def get_title(ytid):
return subprocess.check_output(gen_cmdline(ytid, for_title=True))
# "mplayer -fs"
while True:
_, to_load = redis.blpop("musicaload")
try:
to_load = to_load.decode()
if redis.get("musicatitle." + to_load) is None or redis.get("musicatitle." + to_load).startswith(b"Could not load video "):
redis.set("musicatitle." + to_load, get_title(to_load).strip())
if not os.path.exists(path_for(to_load)):
if subprocess.call(gen_cmdline(to_load), cwd=DATA_DIR) != 0:
redis.set("musicatitle." + to_load, ("Could not load video %s" % (to_load,)).encode())
continue
subprocess.check_call(gen_cmdline(to_load), cwd=DATA_DIR)
assert os.path.exists(path_for(to_load))
except:
print("Failed to load.")
traceback.print_exc()
def processMessage(data):
print data
in_path = data['result']['file_path'].encode(encoding='UTF-8')
print "in_path:", in_path, type(in_path)
url = "https://api.telegram.org/file/bot{}/{}".format(bot_token, in_path)
print "getting image:", url
wget.download(url, in_path)
out_path = 'static/' + in_path
#checkpoint_dir = 'checkpoints/udnie.ckpt'
checkpoint_dir = random.choice(ckpt_files)
ffwd_to_img(in_path, out_path, checkpoint_dir)
print "done image"
sendPhoto(data['chat_id'], "https://dl2.rohbot.cc/" + out_path)
data = {'chat_id': 700442415, 'result': {'file_path': u'photos/file_18.jpg'}}
print data
#processMessage(data)
while True:
item = redis.blpop("queue:style")
data = json.loads(item[1])
processMessage(data)
import os
from subprocess import call, check_output
import shutil
import logging
from config import TARGET_SECONDS, HOME_DIR, LOG_FILE
import redis
redis = redis.StrictRedis()
logging.basicConfig(filename=LOG_FILE, level=logging.WARNING)
os.chdir(HOME_DIR)
while 1:
id = redis.blpop('video')[1]
logging.warning(os.environ['PATH'])
logging.warning(os.environ['HOME'])
# Get each frames time
try:
a = check_output(['identify', '-format', '"Frame %s: %Tcs\n"', 'uploads/' + id + '.gif'])
except:
logging.warning('identify error')
redis.set(id, 'error')
continue
lengths = []
for l in a.split('\n'):
'language': language,
'status': 'doing'
}
res = observer.add_observer(data)
if res['message'] is 'created':
return res['id']
else:
return None
if __name__ == '__main__':
chrome.run()
while True:
try:
task_code, task_data = redis.blpop([REDIS_TOPIC_OBSERVER_URLS], 0)
if task_data:
urlData = task_data.decode("utf-8")
urlObj = json.loads(urlData)
if urlObj is not None:
url = urlObj.get('url', None)
if url is not None:
language = urlObj['language']
id = register(url, language)
if id is not None:
detector.run(id, url, language)
except Exception as ex:
traceback.print_exc()
print(ex)
import redis
import json
from time import sleep
from random import randint
if __name__ == '__main__':
redis = redis.Redis(host='queue', port='6379', db='0')
while True:
mail = json.loads(redis.blpop('sender')[1])
print('Sending mail:', mail['subject'], '...')
sleep(randint(10, 30))
print('mail', mail['subject'], 'sent with success!')
#!/usr/local/bin/python3
# -*- coding: utf-8 -*-
import redis
import biquke
from crawler import Crawler
redis = redis.Redis(connection_pool=Crawler.pool)
while True:
temp = redis.blpop("tq_book_https://www.biquke.com/")[1]
print(temp)
url0 = bytes.decode(temp)
print("take url is :", url0)
bs0 = biquke.Biquke1(url0)
bs0.start()
bs0.join()
def pross():
params = dict()
params["logging_level"] = 3
params["output_resolution"] = "-1x-1"
params["net_resolution"] = "-1x368"
params["model_pose"] = "BODY_25"
params["alpha_pose"] = 0.6
params["scale_gap"] = 0.3
params["scale_number"] = 1
params["render_threshold"] = 0.05
# If GPU version is built, and multiple GPUs are available, set the ID here
params["num_gpu_start"] = 0
params["disable_blending"] = False
# Ensure you point to the correct path where models are located
params[
"default_model_folder"] = "/woody/software/source/openpose2019/models/"
# Construct OpenPose object allocates GPU memory
openpose = OpenPose(params)
websocket_server = Websocket(9000,
servers=[],
new_client=new_client,
client_left=client_left,
message_received=message_received)
websocket_server.start()
while True:
if websocket_server.servers is not None:
break
video_path = "/home/woody/tmp/openpose/test1.mp4"
#video_path = "/woody/software/source/openpose/examples/media/video.avi"
f_name = "test1.mp4"
#video_path = "rtsp://172.16.3.26/test"
#video_path = "/home/woody/tmp/openpose/video/4804_exit_overvie.mp4"
video = cv2.VideoCapture()
if not video.open(video_path):
logger.info("can not open the video")
exit(1)
index = 1
count = 0
f_count = 1
imageArray = {}
continuityArray = []
up_imageArray = []
images = []
start = time.time()
while True:
st = time.time()
_, vlc_key = redis.blpop("vlcreader_key")
logger.info("redis0>>>" + str(time.time() - st))
print(vlc_key.decode())
#_, frame = video.read()
#if frame is None:
# break
frame_dump = redis.get(vlc_key.decode())
logger.info("redis1>>>" + str(time.time() - st))
frame = pickle.loads(frame_dump)
logger.info("redis2>>>" + str(time.time() - st))
#cv2.imwrite("/home/woody/tmp/openpose/xxxx/" + str(index) + ".jpg", np.array(frame), [cv2.IMWRITE_JPEG_QUALITY, 90])
base64_data = frame2base64(frame)
data_json = {}
data_json["type"] = 1
data_json["image"] = "data:image/jpeg;base64," + base64_data.decode()
websocket_server.servers[0].send_message_to_all(
str(json.dumps(data_json)))
if f_count % 1 == 0:
st = time.time()
output_image = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)
keypoints, scores = openpose.forward(output_image, False)
logger.info("openpose>>>" + str(time.time() - st))
image = output_image
continuity = False
key = f_name[:-4] + "_" + str(index)
for keypoint in keypoints:
flag = False
bule_flag, bule_totail = calcBuleRate(
image, [keypoint[12][0], keypoint[12][1]],
[keypoint[9][0], keypoint[9][1]],
[keypoint[2][0], keypoint[2][1]],
[keypoint[5][0], keypoint[5][1]])
if bule_flag == False:
continue
x1 = keypoint[2][0]
y1 = keypoint[2][1]
x2 = keypoint[9][0]
y2 = keypoint[9][1]
x3 = keypoint[10][0]
y3 = keypoint[10][1]
r_result, r_flag = calcHipAngle(x1, y1, x2, y2, x3, y3)
x1 = keypoint[5][0]
y1 = keypoint[5][1]
x2 = keypoint[12][0]
y2 = keypoint[12][1]
x3 = keypoint[13][0]
y3 = keypoint[13][1]
l_result, l_flag = calcHipAngle(x1, y1, x2, y2, x3, y3)
x1 = keypoint[9][0]
y1 = keypoint[9][1]
x2 = keypoint[10][0]
y2 = keypoint[10][1]
x3 = keypoint[11][0]
y3 = keypoint[11][1]
ra_result, ra_flag = calcKneeAngle(x1, y1, x2, y2, x3, y3)
ra_len_result, ra_len_flag = calcLenRate(
x1, y1, x2, y2, x3, y3)
x1 = keypoint[12][0]
y1 = keypoint[12][1]
x2 = keypoint[13][0]
y2 = keypoint[13][1]
x3 = keypoint[14][0]
y3 = keypoint[14][1]
la_result, la_flag = calcKneeAngle(x1, y1, x2, y2, x3, y3)
la_len_result, la_len_flag = calcLenRate(
x1, y1, x2, y2, x3, y3)
if ra_flag and la_flag:
flag = True
if ra_len_flag and la_len_flag:
flag = True
if (r_flag or l_flag) and (abs(r_result - l_result) <=
30) and (ra_result or la_result):
flag = True
#if la_result >= 170 or ra_result >= 170:
# flag = False
if (la_len_result >= 0.9 and la_len_result <= 1.09) or (
ra_len_result >= 0.9 and ra_len_result <= 1.09):
flag = False
if flag:
continuityArray.append(keypoint[8])
continuity = True
logger.info(key + " >>> sat_list >>>>>>>>>" + str(flag) +
">>>>>" + str(continuity))
if continuity:
v_flag = False
if len(up_imageArray) >= 1:
if len(continuityArray) >= 1:
for u_index in range(len(up_imageArray)):
for c_index in range(len(continuityArray)):
midHip2 = (
up_imageArray[u_index][0] -
continuityArray[c_index][0]) * (
up_imageArray[u_index][0] -
continuityArray[c_index][0]) + (
up_imageArray[u_index][1] -
continuityArray[c_index][1]) * (
up_imageArray[u_index][1] -
continuityArray[c_index][1])
midHip = cmath.sqrt(midHip2)
f_midHip = cmath.sqrt(
0.09 * (up_imageArray[u_index][0] +
up_imageArray[u_index][1]) *
(up_imageArray[u_index][0] +
up_imageArray[u_index][1]))
if midHip.real < f_midHip.real:
v_flag = True
break
#logger.info(str(key) + ">>>" + str(midHip.real) + "," + str(f_midHip.real))
if v_flag:
break
up_imageArray = continuityArray
continuityArray = []
if count < 1:
v_flag = True
if v_flag:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/hh", str(key))
cv2.imwrite(save_path, image)
if 'index' in imageArray.keys():
if imageArray['start'] >= count - E_TOTAIL:
if imageArray['index'] == count - 1:
imageArray['start'] = count
imageArray['count'] = imageArray['count'] + 1
if imageArray['count'] >= APM_TOTAIL:
if imageArray['apm'] == False:
imageArray['apm'] = True
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/test",
str(imageArray['key']))
cv2.imwrite(save_path,
imageArray["image"])
base64_data = frame2base64(
imageArray["image"])
data_json = {}
data_json["type"] = 2
data_json[
"image"] = "data:image/jpeg;base64," + base64_data.decode(
)
websocket_server.servers[
0].send_message_to_all(
str(json.dumps(data_json)))
ws_data_list.append(data_json)
image_data = {}
image_data["key"] = key
image_data["image"] = image
images.append(image_data)
for image_info in images:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/li",
str(image_info['key']))
cv2.imwrite(
save_path, image_info["image"])
images = []
imageArray['index'] = count
if imageArray['apm'] == False:
image_data = {}
image_data["key"] = key
image_data["image"] = image
images.append(image_data)
else:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/test",
str(imageArray['key']))
if imageArray['apm'] == False:
if imageArray['count'] >= APM_TOTAIL:
imageArray['apm'] = True
cv2.imwrite(save_path, imageArray["image"])
base64_data = frame2base64(
imageArray["image"])
data_json = {}
data_json["type"] = 2
data_json[
"image"] = "data:image/jpeg;base64," + base64_data.decode(
)
websocket_server.servers[
0].send_message_to_all(
str(json.dumps(data_json)))
ws_data_list.append(data_json)
image_data = {}
image_data["key"] = key
image_data["image"] = image
images.append(image_data)
for image_info in images:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/li",
str(image_info['key']))
cv2.imwrite(save_path,
image_info["image"])
images = []
imageArray = {}
images = []
else:
imageArray['index'] = count
imageArray['start'] = count
imageArray['count'] = 0
imageArray['apm'] = False
imageArray['image'] = output_image
imageArray['key'] = key
else:
if 'index' in imageArray.keys():
imageArray['index'] = count
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/hh", str(key))
cv2.imwrite(save_path, image)
else:
if 'index' in imageArray.keys():
if imageArray['start'] < count - E_TOTAIL:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/test",
str(imageArray['key']))
if imageArray['apm'] == False:
if imageArray['count'] >= APM_TOTAIL:
imageArray['apm'] = True
cv2.imwrite(save_path, imageArray["image"])
base64_data = frame2base64(imageArray["image"])
data_json = {}
data_json["type"] = 2
data_json[
"image"] = "data:image/jpeg;base64," + base64_data.decode(
)
websocket_server.servers[
0].send_message_to_all(
str(json.dumps(data_json)))
ws_data_list.append(data_json)
for image_info in images:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/li",
str(image_info['key']))
cv2.imwrite(save_path, image_info["image"])
images = []
imageArray = {}
images = []
else:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/wu", str(key))
cv2.imwrite(save_path, image)
up_imageArray = []
continuityArray = []
count = count + 1
logger.info("end ===============" + key)
logger.info(str(count) + ",totail>>>" + str(time.time() - start))
index += 1
f_count = f_count + 1
if 'index' in imageArray.keys():
if imageArray['start'] < count - E_TOTAIL:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/test", str(imageArray['key']))
if imageArray['apm'] == False:
if imageArray['count'] >= APM_TOTAIL:
imageArray['apm'] = True
cv2.imwrite(save_path, imageArray["image"])
base64_data = frame2base64(imageArray["image"])
data_json = {}
data_json["type"] = 2
data_json[
"image"] = "data:image/jpeg;base64," + base64_data.decode(
)
websocket_server.servers[0].send_message_to_all(
str(json.dumps(data_json)))
ws_data_list.append(data_json)
for image_info in images:
save_path = "{}/{:>03s}.jpg".format(
"/home/woody/tmp/openpose/li",
str(image_info['key']))
cv2.imwrite(save_path, image_info["image"])
images = []
imageArray = {}
images = []
logger.info(">>>" + str(time.time() - start))
video.release()
logger.info("Totally save {:d} pics".format(index - 1))
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import redis
import biquke
from crawler import Crawler
b0 = biquke.Biquke0("https://www.biquke.com/")
b0.start()
redis = redis.Redis(connection_pool=Crawler.pool)
while True :
temp = redis.blpop(b0.bookpfx + b0.start_url)[1]
print(temp)
url0 = bytes.decode(temp)
print("take url is :", url0)
bs0 = biquke.Biquke1(url0)
temp = redis.blpop(b0.bookpfx + b0.start_url)[1]
url1 = bytes.decode(temp)
print("take url is :", url1)
bs1 = biquke.Biquke1(url1)
bs0.start()
bs1.start()
bs0.join()
bs1.join()
import pymongo
import json,redis
redis = redis.Redis(host="localhost",port=6379,db=0)
clicent = pymongo.MongoClient()
collection = clicent.qiushi.duanzi
while True:
# 这只是取一个记录
key, data = redis.blpop(["qiushi:items"])
print(key)
d = json.loads(data)
print(d)
collection.insert_one(d)
