def checkRate(ip):
if redis.llen(ip) == 0:
print("First encounter")
i = 1
redis.lpush(ip,1)
redis.expire(ip,60)
else:
if redis.llen(ip) == 5:
return False
else:
redis.lpush(ip,1)
return True
def set_data(request):
"""
从前台界面发送过来的数据传给mqtt服务器，并且存入redis数据库
:param request:
:return:
"""
if request.method == 'POST':
for i in request.POST:
redis.rpush("set_value", i)
redis.rpush("set_value", request.POST[i])
print "set_value", i
print redis.llen('set_value')
return JsonResponse({'success': True})
def main():
print(f"Starting worker on queue: {queue}")
print(f"ES Index: {index}")
print(f"{queue} size:{redis.llen(queue)}")
processed = 0
skipped = 0
while True:
print("processed: {} // skipped: {} // {} size:{}".format(
processed, skipped, queue, redis.llen(queue)))
video_id = redis.lpop(queue)
if video_id is None:
continue
video_id = video_id.decode("utf-8")
try:
transcript = YouTubeTranscriptApi.get_transcript(video_id)
# [{'text': 'is that a million yeah oh this is a', 'start': 0.06, 'duration': 5.04}]
if not should_process(transcript, video_id=video_id):
skipped += 1
continue
processed += 1
write_to_es(transcript, video_id)
except KeyboardInterrupt:
print('SIGINT or CTRL-C detected. Exiting gracefully')
exit(0)
except Exception as ex:
print(f"Something went wrong. Keep going! Err -> {ex}")
def on_status(self, data):
global count
count += 1
if count % 100 == 0:
logging.debug('%s tweets gathered.' % str(count))
tweetID = data.id_str
tweet = {}
tweet['id'] = encode(data.id_str)
tweet['created_at'] = dumps(data.created_at, default=json_serial)
tweet['text'] = encode(data.text)
tweet['username'] = encode(data.user.screen_name)
tweet['url'] = encode(data.user.url)
tweet['location'] = encode(data.user.location)
json_tweet = json.dumps(tweet)
redis.lpush(args.query, json_tweet)
i = redis.llen(args.query)
if args.mode == 'f':
if i % 50 == 0:
os.system('python writer.py -q %s' % args.query)
if args.mode == 'd':
if i % 50 == 0:
os.system('python upload.py -q %s' % args.query)
def run(self):
from utils import topic
logger.info("TopicSpider thread start...")
_id = ""
try:
while self.id_manager.list_not_null():
logger.warning("Len {0}: {1}".format(
config.TOPIC_ID_SET,
redis.llen("list_" + config.TOPIC_ID_SET)))
_id = self.id_manager.get()
try:
topic.spider(_id)
except:
continue
self.exit_code = 0
except Exception as e:
# 之前的报错信息已被记录
logger.critical(
"Unexpected Exit TopicSpider: {0}, Message: {1}".format(
_id, e),
exc_info=True)
# send_mail("TopicSpider发生意料之外的错误，已退出线程")
finally:
logger.warning("TopicSpider finished")
if self.exit_code == 0:
topic.html_downloader.proxies.__exit__()
def doWelfareById(redis, uid, account, id):
"""
福利
"""
today = datetime.now().strftime("%Y-%m-%d")
if id == '2':
playerCoin = redis.hget(FORMAT_USER_TABLE % uid, 'gold')
playerCoin = playerCoin if playerCoin else 0
if int(playerCoin) >= SIGN_LINE:
return {'code': 1, 'msg': u'未达到低保线无法领取'}
key = WELFARE_USER_INSURANCE % (account, today)
if redis.llen(key) >= SIGN_MAX:
return {'code': 1, 'msg': u'已经领取了 {0} 次'.format(SIGN_MAX)}
redis.lpush(key, SIGN_COINNUM)
player_add_gold(redis, account, SIGN_COINNUM)
elif id == '1':
# 新手礼包
if redis.hget(GOLD_REWARD_NEW_PRESENT_HASH, account) == MESSION_STATUS_OVER:
return {'code': 1, 'msg': u'您已领取，无法再次领取'}
redis.hset(GOLD_REWARD_NEW_PRESENT_HASH, account, MESSION_STATUS_OVER)
elif id == '0':
# 每日首冲奖励
if not redis.sismember(DAILY_USER_GOLD2_SET % today, account):
return {'code': 1, 'msg': u'您今日还未完成首冲'}
elif redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account) == MESSION_STATUS_OVER:
return {'code': 1, 'msg': u'您已领取，无法再次领取'}
redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account, MESSION_STATUS_OVER)
return {'code': 0, 'msg': u'领取成功'}
def get_from_queue(name: str):
# удаляем и возвращаем первое значение в лист lpop
if redis.llen(name) > 0:
file = json.loads(redis.lpop(name))
return file
else:
return False
def get_stat(args, redis):
stat = {}
total_size = 0
for prio in str(args.priority_steps).split(','):
queue = make_queue_name_for_pri(args.celery_queue, prio)
queue_name = make_queue_name_for_pri_without_sep(
args.celery_queue, prio)
# size of queue
qlen = redis.llen(queue)
total_size += qlen
print('queue {0} size ->{1}'.format(queue_name, qlen))
metric = '{0}{1}.size'.format(args.dd_metric_prefix, queue_name)
statsd.gauge(metric, qlen)
# number of chunks to get
chunk_n = (qlen + args.chunk_size - 1) // args.chunk_size
for chunk in range(0, chunk_n):
for task in redis.lrange(queue, chunk * args.chunk_size,
chunk * args.chunk_size +
args.chunk_size):
payload = json.loads(task)
task = payload['headers']['task']
prio = payload['properties']['priority']
key = '{0}{1}'.format(task, prio)
if not key in stat:
stat[key] = 1
else:
stat[key] += 1
for task, count in stat.items():
metric = '{0}{1}'.format(args.dd_metric_prefix, task)
print('{0}->{1}'.format(task, count))
statsd.gauge(metric, count)
print('total size -> {0}'.format(total_size))
metric = '{0}total.size'.format(args.dd_metric_prefix)
statsd.gauge(metric, total_size)
def checkstatus():
print("Crawler\t\t\t\tQueued Items\t\t\tDic Items")
for item in checklist:
queued = item + ":requests"
queued_num = redis.zcard(queued)
cached = item + ":items"
cached_num = redis.llen(cached)
print("%s\t\t\t%d\t\t\t\t%d" % (item, queued_num, cached_num))
def reset_queue(redis, key, *fields):
"""
reset for queue-based keys
"""
while redis.llen(key) > 0:
redis.lpop(key)
d = {}
for f in fields:
d[f] = None
redis.rpush(key, str(json.dumps(d)))
def clean():
for item in checklist:
queued = item + ":requests"
queued_num = redis.zcard(queued)
if queued_num > 0:
redis.delete(queued)
cached = item + ":items"
cached_num = redis.llen(cached)
if cached_num > 0:
redis.delete(cached)
print("Clean all queued requests successfully!")
def post_to_facebook():
if redis.llen('articles') > 0:
article = Article.objects.get(
article_id=redis.lpop('articles').decode())
"""Post new articles to facebook"""
try:
status = graph_api.put_object("me",
"feed",
message=article.title,
link=article.url)
except facebook.GraphAPIError as er:
print("There is a problem ", str(er))
def disconnect():
lang = request.args.get('lang')
print("Disconnecting with lang: ", lang)
num_connected = redis.get("clients")
num_connected = int(num_connected.decode("utf-8"))
redis.set("clients", num_connected - 1)
print("PRINTING CLIENTS", redis.get("clients"))
redis.lrem("langs", 1, lang)
print("Current languages", redis.lrange("langs", 0, redis.llen("langs")))
return jsonify("HELLO")
def post_to_facebook():
"""Post new articles to facebook"""
for i in range(5):
if redis.llen('articles') > 0:
article = Article.objects.get(article_id = redis.lpop('articles'))
attachment = {"name":article.title , "link" :article.url , "description": article.description}
try:
status = api.put_wall_post(article.title, attachment )
except facebook.GraphAPIError as er:
print("There is a problem ", str(er))
def post_video_to_facebook():
"""Post new articles to facebook"""
for i in range(1):
if redis.llen('videos') > 0:
#get the first element
video = YoutubeVideo.objects.get(video_id = redis.lpop('videos'))
attachment = {"name":video.title , "link" :video.url , "description": video.description}
try:
status = api.put_wall_post(video.title, attachment )
except facebook.GraphAPIError as er:
print("There is a problem ", str(er))
def ta_orchestrator():
global market_list
redis = ExchangeDataHandler().getInstance().getRedisDb()
starttime = 0
diftime = 0
while True:
try:
if redis.llen("ta_markets") == 0:
if starttime > 0:
diftime = time.time() - starttime
print("completed ta cycle in {} seconds".format(diftime))
for market in market_list.split(" "):
redis.rpush("ta_markets", market)
starttime = time.time()
else:
time.sleep(5)
except Exception as ex:
print("push_ta_queue exception thrown: {}".format(ex))
queue_size = rospy.get_param("redis_qs", 5)
map_sub = rospy.Subscriber("/map", OccupancyGrid, map_cb)
bt.establish_reset(redis, redis_key, bt.reset_queue, None, "data", "id",
"Line_count", "width", "height")
bt.establish_pulse()
map_shift = [0, 0, 0]
map_rot = [0, 0, 0]
listener = tf.TransformListener()
tf_present = False
rate = rospy.Rate(5)
# main loop: get tf from odom to map
while not rospy.is_shutdown():
try:
map_shift, map_rot = listener.lookupTransform(
"odom", "map", rospy.Time(0)
) # gives us the tf from odom to map. values inverted for tf from map to odom.
map_rot = euler_from_quaternion(map_rot)
if not tf_present:
rospy.loginfo_once("MAP_BRIDGE: Odom to map tf found")
tf_present = True
except (tf.LookupException, tf.ConnectivityException,
tf.ExtrapolationException):
if tf_present:
rospy.loginfo_once("MAP_BRIDGE: no transform from odom to map")
tf_present = False
continue
# trim queue size
while redis.llen(redis_key) > queue_size:
redis.lpop(redis_key)
rate.sleep()
def draw_cards(self):
hand_length = redis.llen(self.key(':hand'))
for i in range(4 - hand_length):
card = redis.lpop('games:{}:deck'.format(self.gid))
if card:
redis.rpush(self.key(':hand'), card)
def deck_empty(self):
deck_length = int(redis.llen(self.key(':deck')))
return deck_length == 0
def re_map(msg_id, dbName):
start_id = int(msg_id) - 10
end_id = int(msg_id) + 10
sql = "select * from main.message WHERE type =3 and talker='9990246678@chatroom' and msgId >=" + str(
start_id) + " and msgId<=" + str(end_id) + ";"
data = query(dbName, sql)
for i in range(0, len(data)):
tailorImg(decodeDbName, data[i][0], data[i][1])
redis.lpush(has_done_key, msg_id)
if __name__ == "__main__":
uin = '-1763681588'
imei = '861795037451492'
fileName = 'reMap_EnMicroMsg_' + str(imei) + 'reMap.db'
decodeDbName = 'reMap_decrypted_' + fileName
# cpFile(uin, str(fileName))
#
# key = getKey(uin, imei)
#
# r = decrypt(key, fileName, decodeDbName)
list_len = redis.llen(redis_key)
for i in range(0, list_len):
msg_id = redis.rpop(redis_key)
print "但前处理" + str(msg_id)
re_map(msg_id, decodeDbName)
def __init__(self, redis, list_key):
self.idx = redis.llen(list_key)
self._r = redis
self._l = list_key
odom_sub = rospy.Subscriber("/odom", Odometry, odom_cb)
scan_sub = rospy.Subscriber("/scan", LaserScan, scan_cb)
cmd_pub = rospy.Publisher("/cmd_vel", Twist, queue_size=3)
bt.establish_reset(redis, redis_key)
bt.establish_pulse()
pose = None
flags = {"moving": False, "forward": False, "stalled": False, "stopped": False}
safe_thresh = 0.25
current_twist = Twist()
current_cmd = {}
next_cmd = {}
expiration_time = None
rate = rospy.Rate(10)
while not rospy.is_shutdown():
if redis.llen(redis_key) > 0 and not next_cmd:
next_cmd = json.loads(redis.lpop(redis_key)) # if next command slot is empty, then get a new commadn from redis
if not current_cmd and next_cmd: # if current command is empty and next command is not, then move the command to the current slot
current_cmd = next_cmd
next_cmd = {}
if "cmd" not in current_cmd.keys(): # check to make sure the json at least has a cmd key
rospy.logerr("[cmd_feedback invalid] cmd not given. No action available. Aborting.")
current_cmd = {}
else:
execute_command(current_cmd)
if current_cmd != None and next_cmd.get("cmd", "").strip().lower() == "stop": # if there is a current command, but the next command is a stop, then stop the current action
flags["stopped"] = True # set the stopped flag
expiration_time = rospy.get_time() #
next_cmd = {} # empty next command slot
if expiration_time:
if rospy.get_time() < expiration_time and not flags["stalled"]:
t = time.time()
detector = CNNClassificator(args['classlist'],
args['snapshot'], args['arch'],
args['mean'], args['gpu'])
tdiff = int(time.time() - t)
print 'model load time: %d:%02d' % (tdiff // 60,
tdiff % 60)
label, prob, imgdata = detect_image_label(
detector, ftp_h, fpath)
print('Label: {}, {}%'.format(label, int(prob * 100)))
output_name = REDIS_OUTPUT_PREFIX + label
if prob < 0.7:
queue_pfx = NO_LABEL_QUEUE
redis.rpush(queue_pfx, bname)
redis.ltrim(queue_pfx, max(0,
redis.llen(queue_pfx) - 100),
-1)
else:
last_rec = redis.lrange(output_name, -1, -1)
if last_rec:
last_rec = json.loads(last_rec[0])
if last_rec['ts'] < ts and last_rec['label'] != label:
msg = 'Changed at %s from %s to %s (diff=%d), %s' % (
dt.strftime('%d/%m %H:%M:%S'),
last_rec['label'], label, ts - last_rec['ts'],
bname)
print '*' * 10, msg, '*' * 10
send_email(
'*****@*****.**', '%s: %s' %
(dt.strftime('%H:%M:%S %d/%m'), label), msg,
[imgdata])
redis.set("counter", 1)
print(redis.get("counter")) # 1
redis.incr("counter")
print(redis.get("counter")) # 2
redis.decr("counter")
print(redis.get("counter")) #1
# List : possible to duplicate values
redis.rpush("members", "r1")
redis.rpush("members", "r2")
redis.lpush("members", "l1")
redis.lpush("members", "l2")
print(redis.lrange("members", 0, 0))
print(redis.lrange("members", 0, 1))
print(redis.lrange("members", 0, 2))
print(redis.llen("members"))
print(redis.lrange("members",0, redis.llen("members")-1))
print(redis.lindex("members",3))
print(redis.rpop("members"))
print(redis.lpop("members"))
print(redis.llen("members"))
print(redis.lrange("members",0, redis.llen("members")-1))
redis.delete("members")
#Sets : impossible to duplicate values
redis.sadd("members", "s1")
redis.sadd("members", "s1")
redis.sadd("members", "s2")
redis.sadd("members", "s3")
redis.sadd("members", "s4")
redis.sadd("members", "s5")
import xlrd
import xlsxwriter
import redis
redis = redis.StrictRedis(host='127.0.0.1', port=6379, decode_responses=True)
ExcelFile=xlrd.open_workbook(r'C:\Users\Administrator\Desktop\first_result.xlsx') #打开目标处理xlsx
sheet=ExcelFile.sheet_by_index(0) #定位到索引为0的工作表，即第一个工作表
cols=sheet.col_values(1) #取第3列的数据
# print(redis.llen('provider'))
for i in range(1,len(cols)):
redis.lpush('licenceCode1',cols[i])
print(redis.llen('licenceCode1'),cols[i])
# for i in range(redis.llen('licenceCode')):
# redis.lpop('licenceCode')
# print(redis.llen('licenceCode'))
# l = ['成都房江湖信息科技有限公司','成都昱超贸易有限公司','南宁市顺柏贸易有限公司','浙江力德节能科技有限公司','温岭市广源门窗幕墙工程有限公司','上海柏瑜实业有限公司','深圳市明源云客电子商务有限公司','广西金图工程咨询有限公司','辽宁同飞玺铭会计师事务所有限责任公司','重庆力瑞机械设备有限公司','云南卓立工程检测有限公司']
#
# for i in l :
# redis.lpush('provider_law',i)
import logging
import csv
import io
from util import encode
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
try:
redis = redis.StrictRedis(host='localhost', port=6379, db=0)
except:
print("Can't connect to Redis!")
parser = argparse.ArgumentParser()
parser.add_argument("-q",
"--queries",
dest="query",
help="Give names of queries stored in Redis.",
default='-')
args = parser.parse_args()
with io.open('data/%s.json' % args.query, 'a', encoding='utf-8') as outfile:
i = redis.llen(args.query)
logging.debug("Writing %s tweets to data/%s.json" % (i, args.query))
while i > 0:
data = json.loads(redis.lpop(args.query))
outfile.write(
unicode(
json.dumps(data, ensure_ascii=False, indent=4,
sort_keys=True)))
i = i - 1
# print 'set fish_table[%s]'%(key)
# pipe.hmset(key,update)
#
# goods_lists = redis.lrange(FISH_REWARD_ON_SHOP_LIST,0,-1)
# for goods in goods_lists:
# goods_type = redis.hget(FISH_REWARD_TABLE%(goods),'reward_type')
# pipe.lpush(FISH_REWARD_ON_SHOP_TYPE_LIST%(goods_type),goods)
# print 'set goods id index success....[%s]'%(goods_type)
# pipe.execute()
###############################################################
### exchange_table更新
### 新增1个字段 exchange_type
### 2018-01-03
###############################################################
total = redis.llen(FISH_EXCHANGE_LIST)
exchange_ids = redis.lrange(FISH_EXCHANGE_LIST, 0, -1)
exchange_id_keys = [
FISH_EXCHANGE_TABLE % (exchange_id) for exchange_id in exchange_ids
]
exchange_details = [
exchange_detail for exchange_detail in redis.mget(exchange_id_keys)
]
exchange_info = []
for exchange_detail in exchange_details:
exchange_detail = eval(exchange_detail)
exchange_detail['exchange_type'] = redis.hget(
FISH_REWARD_TABLE % (exchange_detail['exchange_reward_id']),
'reward_type')
pipe.set(FISH_EXCHANGE_TABLE % (exchange_detail['exchange_id']),
from psycopg2.extensions import AsIs
import datetime
try:
redis = redis.StrictRedis(host='localhost', port=6379, db=0)
except:
print "Can't connect to Redis!"
try:
conn = psycopg2.connect("dbname='postgres'")
except:
print "Can't connect to PSQL!"
subreddits = ['politics','worldnews','news']
date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
cur = conn.cursor()
for subreddit in subreddits:
i = redis.llen(subreddit)
while i > 0:
word = redis.lpop(subreddit)
cur.execute("INSERT INTO reddit.%s AS s (word, count, full_date, day) VALUES (%s, %s, %s, date_trunc('day', %s::date)) ON CONFLICT ON CONSTRAINT %s_word_day DO UPDATE SET count = s.count + 1",
(AsIs(subreddit), word, 1, date, date, AsIs(subreddit)))
i = i - 1
conn.commit()
cur.close()
conn.close()
parser.add_argument("-q",
"--queries",
dest="query",
help="Give names of queries stored in Redis.",
default='-')
args = parser.parse_args()
tablename = args.query
cur = conn.cursor()
cur.execute(
sql.SQL(
"CREATE TABLE IF NOT EXISTS twitter.{} (id BIGINT, createdate TIMESTAMP, body VARCHAR, username VARCHAR, url VARCHAR, location VARCHAR, address VARCHAR, incident_type VARCHAR)"
).format(sql.Identifier(tablename)))
i = redis.llen(tablename)
logging.debug("Writing %s tweets to database twitter.%s" % (i, tablename))
while i > 0:
data = json.loads(redis.lpop(tablename))
tweetBody = data['text']
at = 'at'
startBracket = '['
atIndex = tweetBody.find(at)
startBracketIndex = tweetBody.find(startBracket)
def status():
'''returns the status'''
count = redis.llen(trace_list)
last = redis.get(last_ip)
return "Online: %d traces in the db. Last sent IP is %s" % (count, last)
def get_welfare_info(redis, account):
today = datetime.now().strftime("%Y-%m-%d")
first = first_day_of_month()
last = date.today() - timedelta(days=1)
res = copy.deepcopy(WELFARE_CONFIG)
res['date'] = today
res['signed'] = []
res['unsinged'] = []
# 今天是否签到
if redis.exists(WELFARE_USER_SIGN % (account, today)):
res['issigned'] = 1
else:
res['issigned'] = 0
while first <= last:
if redis.exists(WELFARE_USER_SIGN % (account, first)):
res['signed'].append(first.strftime('%d'))
elif redis.sismember(WELFARE_USER_PATCH_SIGN % (account, first.strftime("%m")), first.strftime('%Y-%m-%d')):
res['signed'].append(first.strftime('%d'))
else:
res['unsinged'].append(first.strftime('%d'))
first += timedelta(1)
# 七天奖励
if not redis.exists(GOLD_WELFARE_SIGN_7DAYS):
res['rewardlist'][0]['status'] = MESSION_STATUS_NO
elif redis.hget(GOLD_WELFARE_SIGN_7DAYS, account) == MESSION_STATUS_OVER:
res['rewardlist'][0]['status'] = MESSION_STATUS_OVER
else:
res['rewardlist'][0]['status'] = MESSION_STATUS_OK
# 十五天奖励
if not redis.exists(GOLD_WELFARE_SIGN_15DAYS):
res['rewardlist'][0]['status'] = MESSION_STATUS_NO
elif redis.hget(GOLD_WELFARE_SIGN_15DAYS, account) == MESSION_STATUS_OVER:
res['rewardlist'][0]['status'] = MESSION_STATUS_OVER
else:
res['rewardlist'][0]['status'] = MESSION_STATUS_OK
# 月奖励
if not redis.exists(GOLD_WELFARE_SIGN_MONTH):
res['rewardlist'][0]['status'] = MESSION_STATUS_NO
elif redis.hget(GOLD_WELFARE_SIGN_MONTH, account) == MESSION_STATUS_OVER:
res['rewardlist'][0]['status'] = MESSION_STATUS_OVER
else:
res['rewardlist'][0]['status'] = MESSION_STATUS_OK
# 每日首冲奖励
if not redis.sismember(DAILY_USER_GOLD2_SET % today, account):
res['messionlist'][0]['status'] = MESSION_STATUS_NO
res['messionlist'][0]['parent_mode'] = CHECK_MALL
else:
if redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account) == MESSION_STATUS_OVER:
res['messionlist'][0]['status'] = MESSION_STATUS_OVER
else:
res['messionlist'][0]['status'] = MESSION_STATUS_OK
# 新手礼包
if redis.hget(GOLD_REWARD_NEW_PRESENT_HASH, account) == MESSION_STATUS_OVER:
res['messionlist'][1]['status'] = MESSION_STATUS_OVER
else:
res['messionlist'][1]['status'] = MESSION_STATUS_OK
# 破产补助
key = WELFARE_USER_INSURANCE % (account, today)
if not redis.exists(WELFARE_USER_INSURANCE % (account, today)):
res['messionlist'][2]['status'] = MESSION_STATUS_OK
else:
if redis.llen(key) >= SIGN_MAX:
res['messionlist'][2]['status'] = MESSION_STATUS_OVER
else:
res['messionlist'][2]['status'] = MESSION_STATUS_OK
return res
def count():
'''returns the number of traces currently stored'''
count = redis.llen(trace_list)
return jsonify({'count': count})
def connect():
language = request.args.get('lang')
# id parameter needed to avoid heroku caching request and waiting because
# it was exactly the same as previous request when languages are also the same
id = request.args.get('id')
# print("IN /CONNECT")
if id is None or language is None:
# print("Reconnection received")
langs = []
lang_arr = redis.lrange("langs", 0, redis.llen("langs"))
for lang in lang_arr:
if not isinstance(lang, str):
lang_key = lang.decode("utf-8")
else:
lang_key = lang
langs.append(lang_key)
# print("Language list currently contains: ", langs)
return jsonify(langs)
else:
print("IN ELSE")
print(language)
print(id)
try:
print("Connecting client with ID: " + id)
num_connected = redis.get("clients")
num_connected = int(num_connected.decode("utf-8"))
redis.set("clients", num_connected + 1)
print("PRINTING CLIENTS", redis.get("clients"))
# List logic
redis.lpush("langs", language)
# redis.sadd("languages", language)
print("Current languages", redis.smembers("languages"))
# langs = redis.smembers("languages")
while num_connected != 2:
num_connected = redis.get("clients")
num_connected = int(num_connected.decode("utf-8"))
time.sleep(.5)
print("waiting for other client in /connect. Currently have: ",
num_connected)
langs = []
lang_arr = redis.lrange("langs", 0, redis.llen("langs"))
for lang in lang_arr:
if not isinstance(lang, str):
lang_key = lang.decode("utf-8")
else:
lang_key = lang
langs.append(lang_key)
except Exception as err:
print("IN EXCEPT")
langs = []
lang_arr = redis.lrange("langs", 0, redis.llen("langs"))
for lang in lang_arr:
if not isinstance(lang, str):
lang_key = lang.decode("utf-8")
else:
lang_key = lang
langs.append(lang_key)
print("Language list currently contains: ", langs)
return jsonify(langs)
def delete_all():
'''Debug: clears the redis queue'''
res = redis.ltrim(trace_list, 0, 0)
res = redis.lpop(trace_list)
count = redis.llen(trace_list)
return jsonify({'count': count})
for mname in MODELLIST:
models[mname] = cPickle.loads(open(os.path.join(MODELPATH, mname + '.svc')).read())
# cPickle.dump(models[mname], open(os.path.join(MODELPATH, mname + '.svc.new'), 'w'), protocol=cPickle.HIGHEST_PROTOCOL)
tdiff = int(time.time() - t)
dbprint('models load time: %d:%02d' % (tdiff//60, tdiff%60))
msglist = []
labellist = []
dbprint('model names: %s' % models.keys())
for mname,model in models.items():
output_name = REDIS_OUTPUT_PREFIX + mname
dbprint('Start %s' % bname)
label,imgdata = detect_image_label(model, ftp_h, fpath)
if label == NO_LABEL:
queue_pfx = NO_LABEL_QUEUE_PREFIX + mname
redis.rpush(queue_pfx, bname)
redis.ltrim(queue_pfx, max(0, redis.llen(queue_pfx) - 100), -1)
elif label in ('open', 'close'):
redis.set('gate', json.dumps({'label': label, 'ts': time.time()}))
if label != NO_LABEL:
last_rec = redis.lrange(output_name, -1, -1)
if last_rec:
last_rec = json.loads(last_rec[0])
if last_rec['ts'] < ts and last_rec['label'] != label:
msg = '%s changed at %s from %s to %s (diff=%d), %s' % (mname, dt.strftime('%d/%m %H:%M:%S'), last_rec['label'], label, ts - last_rec['ts'], bname)
dbprint('%s %s' % (bname, msg))
msglist.append(msg)
labellist.append((mname, label))
else:
msg = 'Initial at %s %s' % (dt.strftime('%d/%m %H:%M:%S'), label)
dbprint('%s %s' % (bname, msg))
msglist.append(msg)
def GetEnableListLen(redis):
try:
return redis.llen("APO_ENABLE_LIST")
except Exception, e:
print "GET APO_ENABLE_LIST LENGTH FAILED"
traceback.print_exc()
lista = [
'arbor-collector',
'arbor-parser',
'archive',
'bruteforceblocker-collector',
'bruteforceblocker-parser',
'cymru-expert',
'deduplicator-expert',
'dragon-research-group-ssh-collector',
'dragon-research-group-ssh-parser',
'dragon-research-group-vnc-collector',
'dragon-research-group-vnc-parser',
'malware-domain-list-collector',
'malware-domain-list-parser',
'malwarepatrol-dans-guardian-collector',
'malwarepatrol-dans-guardian-parser',
'openbl-collector',
'openbl-parser',
'phishtank-collector',
'phishtank-parser',
'postgresql-output',
'sanitizer-expert',
'taxonomy-expert',
'vxvault-collector',
'vxvault-parser'
]
for bot in lista:
print "%s: %s" % (bot, redis.llen(bot+"-queue"))
from psycopg2 import *
import geocode
from geocode import geocode
try:
conn = psycopg2.connect("dbname='postgres'")
except:
print("Can't connect to PSQL!")
try:
redis = redis.StrictRedis(host='localhost', port=6379, db=0)
except:
print("Can't connect to Redis!")
cur = conn.cursor()
i = redis.llen('event')
while i > 0:
data = json.loads(redis.lpop('event'))
policeTweet = False
if str(data['username']) == 'pdxpolicelog':
policeTweet = True
# Check if the retweet indicator is in the text body. Do not process retweets.
isRetweet = False
if 'RT @pdxpolicelog: ' in str(data['text']):
isRetweet = True
if isRetweet != True:
