def checkRate(ip): if redis.llen(ip) == 0: print("First encounter") i = 1 redis.lpush(ip,1) redis.expire(ip,60) else: if redis.llen(ip) == 5: return False else: redis.lpush(ip,1) return True
def set_data(request): """ 从前台界面发送过来的数据传给mqtt服务器,并且存入redis数据库 :param request: :return: """ if request.method == 'POST': for i in request.POST: redis.rpush("set_value", i) redis.rpush("set_value", request.POST[i]) print "set_value", i print redis.llen('set_value') return JsonResponse({'success': True})
def main(): print(f"Starting worker on queue: {queue}") print(f"ES Index: {index}") print(f"{queue} size:{redis.llen(queue)}") processed = 0 skipped = 0 while True: print("processed: {} // skipped: {} // {} size:{}".format( processed, skipped, queue, redis.llen(queue))) video_id = redis.lpop(queue) if video_id is None: continue video_id = video_id.decode("utf-8") try: transcript = YouTubeTranscriptApi.get_transcript(video_id) # [{'text': 'is that a million yeah oh this is a', 'start': 0.06, 'duration': 5.04}] if not should_process(transcript, video_id=video_id): skipped += 1 continue processed += 1 write_to_es(transcript, video_id) except KeyboardInterrupt: print('SIGINT or CTRL-C detected. Exiting gracefully') exit(0) except Exception as ex: print(f"Something went wrong. Keep going! Err -> {ex}")
def on_status(self, data): global count count += 1 if count % 100 == 0: logging.debug('%s tweets gathered.' % str(count)) tweetID = data.id_str tweet = {} tweet['id'] = encode(data.id_str) tweet['created_at'] = dumps(data.created_at, default=json_serial) tweet['text'] = encode(data.text) tweet['username'] = encode(data.user.screen_name) tweet['url'] = encode(data.user.url) tweet['location'] = encode(data.user.location) json_tweet = json.dumps(tweet) redis.lpush(args.query, json_tweet) i = redis.llen(args.query) if args.mode == 'f': if i % 50 == 0: os.system('python writer.py -q %s' % args.query) if args.mode == 'd': if i % 50 == 0: os.system('python upload.py -q %s' % args.query)
def run(self): from utils import topic logger.info("TopicSpider thread start...") _id = "" try: while self.id_manager.list_not_null(): logger.warning("Len {0}: {1}".format( config.TOPIC_ID_SET, redis.llen("list_" + config.TOPIC_ID_SET))) _id = self.id_manager.get() try: topic.spider(_id) except: continue self.exit_code = 0 except Exception as e: # 之前的报错信息已被记录 logger.critical( "Unexpected Exit TopicSpider: {0}, Message: {1}".format( _id, e), exc_info=True) # send_mail("TopicSpider发生意料之外的错误,已退出线程") finally: logger.warning("TopicSpider finished") if self.exit_code == 0: topic.html_downloader.proxies.__exit__()
def doWelfareById(redis, uid, account, id): """ 福利 """ today = datetime.now().strftime("%Y-%m-%d") if id == '2': playerCoin = redis.hget(FORMAT_USER_TABLE % uid, 'gold') playerCoin = playerCoin if playerCoin else 0 if int(playerCoin) >= SIGN_LINE: return {'code': 1, 'msg': u'未达到低保线无法领取'} key = WELFARE_USER_INSURANCE % (account, today) if redis.llen(key) >= SIGN_MAX: return {'code': 1, 'msg': u'已经领取了 {0} 次'.format(SIGN_MAX)} redis.lpush(key, SIGN_COINNUM) player_add_gold(redis, account, SIGN_COINNUM) elif id == '1': # 新手礼包 if redis.hget(GOLD_REWARD_NEW_PRESENT_HASH, account) == MESSION_STATUS_OVER: return {'code': 1, 'msg': u'您已领取,无法再次领取'} redis.hset(GOLD_REWARD_NEW_PRESENT_HASH, account, MESSION_STATUS_OVER) elif id == '0': # 每日首冲奖励 if not redis.sismember(DAILY_USER_GOLD2_SET % today, account): return {'code': 1, 'msg': u'您今日还未完成首冲'} elif redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account) == MESSION_STATUS_OVER: return {'code': 1, 'msg': u'您已领取,无法再次领取'} redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account, MESSION_STATUS_OVER) return {'code': 0, 'msg': u'领取成功'}
def get_from_queue(name: str): # удаляем и возвращаем первое значение в лист lpop if redis.llen(name) > 0: file = json.loads(redis.lpop(name)) return file else: return False
def get_stat(args, redis): stat = {} total_size = 0 for prio in str(args.priority_steps).split(','): queue = make_queue_name_for_pri(args.celery_queue, prio) queue_name = make_queue_name_for_pri_without_sep( args.celery_queue, prio) # size of queue qlen = redis.llen(queue) total_size += qlen print('queue {0} size ->{1}'.format(queue_name, qlen)) metric = '{0}{1}.size'.format(args.dd_metric_prefix, queue_name) statsd.gauge(metric, qlen) # number of chunks to get chunk_n = (qlen + args.chunk_size - 1) // args.chunk_size for chunk in range(0, chunk_n): for task in redis.lrange(queue, chunk * args.chunk_size, chunk * args.chunk_size + args.chunk_size): payload = json.loads(task) task = payload['headers']['task'] prio = payload['properties']['priority'] key = '{0}{1}'.format(task, prio) if not key in stat: stat[key] = 1 else: stat[key] += 1 for task, count in stat.items(): metric = '{0}{1}'.format(args.dd_metric_prefix, task) print('{0}->{1}'.format(task, count)) statsd.gauge(metric, count) print('total size -> {0}'.format(total_size)) metric = '{0}total.size'.format(args.dd_metric_prefix) statsd.gauge(metric, total_size)
def checkstatus(): print("Crawler\t\t\t\tQueued Items\t\t\tDic Items") for item in checklist: queued = item + ":requests" queued_num = redis.zcard(queued) cached = item + ":items" cached_num = redis.llen(cached) print("%s\t\t\t%d\t\t\t\t%d" % (item, queued_num, cached_num))
def reset_queue(redis, key, *fields): """ reset for queue-based keys """ while redis.llen(key) > 0: redis.lpop(key) d = {} for f in fields: d[f] = None redis.rpush(key, str(json.dumps(d)))
def clean(): for item in checklist: queued = item + ":requests" queued_num = redis.zcard(queued) if queued_num > 0: redis.delete(queued) cached = item + ":items" cached_num = redis.llen(cached) if cached_num > 0: redis.delete(cached) print("Clean all queued requests successfully!")
def post_to_facebook(): if redis.llen('articles') > 0: article = Article.objects.get( article_id=redis.lpop('articles').decode()) """Post new articles to facebook""" try: status = graph_api.put_object("me", "feed", message=article.title, link=article.url) except facebook.GraphAPIError as er: print("There is a problem ", str(er))
def disconnect(): lang = request.args.get('lang') print("Disconnecting with lang: ", lang) num_connected = redis.get("clients") num_connected = int(num_connected.decode("utf-8")) redis.set("clients", num_connected - 1) print("PRINTING CLIENTS", redis.get("clients")) redis.lrem("langs", 1, lang) print("Current languages", redis.lrange("langs", 0, redis.llen("langs"))) return jsonify("HELLO")
def post_to_facebook(): """Post new articles to facebook""" for i in range(5): if redis.llen('articles') > 0: article = Article.objects.get(article_id = redis.lpop('articles')) attachment = {"name":article.title , "link" :article.url , "description": article.description} try: status = api.put_wall_post(article.title, attachment ) except facebook.GraphAPIError as er: print("There is a problem ", str(er))
def post_video_to_facebook(): """Post new articles to facebook""" for i in range(1): if redis.llen('videos') > 0: #get the first element video = YoutubeVideo.objects.get(video_id = redis.lpop('videos')) attachment = {"name":video.title , "link" :video.url , "description": video.description} try: status = api.put_wall_post(video.title, attachment ) except facebook.GraphAPIError as er: print("There is a problem ", str(er))
def ta_orchestrator(): global market_list redis = ExchangeDataHandler().getInstance().getRedisDb() starttime = 0 diftime = 0 while True: try: if redis.llen("ta_markets") == 0: if starttime > 0: diftime = time.time() - starttime print("completed ta cycle in {} seconds".format(diftime)) for market in market_list.split(" "): redis.rpush("ta_markets", market) starttime = time.time() else: time.sleep(5) except Exception as ex: print("push_ta_queue exception thrown: {}".format(ex))
queue_size = rospy.get_param("redis_qs", 5) map_sub = rospy.Subscriber("/map", OccupancyGrid, map_cb) bt.establish_reset(redis, redis_key, bt.reset_queue, None, "data", "id", "Line_count", "width", "height") bt.establish_pulse() map_shift = [0, 0, 0] map_rot = [0, 0, 0] listener = tf.TransformListener() tf_present = False rate = rospy.Rate(5) # main loop: get tf from odom to map while not rospy.is_shutdown(): try: map_shift, map_rot = listener.lookupTransform( "odom", "map", rospy.Time(0) ) # gives us the tf from odom to map. values inverted for tf from map to odom. map_rot = euler_from_quaternion(map_rot) if not tf_present: rospy.loginfo_once("MAP_BRIDGE: Odom to map tf found") tf_present = True except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException): if tf_present: rospy.loginfo_once("MAP_BRIDGE: no transform from odom to map") tf_present = False continue # trim queue size while redis.llen(redis_key) > queue_size: redis.lpop(redis_key) rate.sleep()
def draw_cards(self): hand_length = redis.llen(self.key(':hand')) for i in range(4 - hand_length): card = redis.lpop('games:{}:deck'.format(self.gid)) if card: redis.rpush(self.key(':hand'), card)
def deck_empty(self): deck_length = int(redis.llen(self.key(':deck'))) return deck_length == 0
def re_map(msg_id, dbName): start_id = int(msg_id) - 10 end_id = int(msg_id) + 10 sql = "select * from main.message WHERE type =3 and talker='9990246678@chatroom' and msgId >=" + str( start_id) + " and msgId<=" + str(end_id) + ";" data = query(dbName, sql) for i in range(0, len(data)): tailorImg(decodeDbName, data[i][0], data[i][1]) redis.lpush(has_done_key, msg_id) if __name__ == "__main__": uin = '-1763681588' imei = '861795037451492' fileName = 'reMap_EnMicroMsg_' + str(imei) + 'reMap.db' decodeDbName = 'reMap_decrypted_' + fileName # cpFile(uin, str(fileName)) # # key = getKey(uin, imei) # # r = decrypt(key, fileName, decodeDbName) list_len = redis.llen(redis_key) for i in range(0, list_len): msg_id = redis.rpop(redis_key) print "但前处理" + str(msg_id) re_map(msg_id, decodeDbName)
def __init__(self, redis, list_key): self.idx = redis.llen(list_key) self._r = redis self._l = list_key
odom_sub = rospy.Subscriber("/odom", Odometry, odom_cb) scan_sub = rospy.Subscriber("/scan", LaserScan, scan_cb) cmd_pub = rospy.Publisher("/cmd_vel", Twist, queue_size=3) bt.establish_reset(redis, redis_key) bt.establish_pulse() pose = None flags = {"moving": False, "forward": False, "stalled": False, "stopped": False} safe_thresh = 0.25 current_twist = Twist() current_cmd = {} next_cmd = {} expiration_time = None rate = rospy.Rate(10) while not rospy.is_shutdown(): if redis.llen(redis_key) > 0 and not next_cmd: next_cmd = json.loads(redis.lpop(redis_key)) # if next command slot is empty, then get a new commadn from redis if not current_cmd and next_cmd: # if current command is empty and next command is not, then move the command to the current slot current_cmd = next_cmd next_cmd = {} if "cmd" not in current_cmd.keys(): # check to make sure the json at least has a cmd key rospy.logerr("[cmd_feedback invalid] cmd not given. No action available. Aborting.") current_cmd = {} else: execute_command(current_cmd) if current_cmd != None and next_cmd.get("cmd", "").strip().lower() == "stop": # if there is a current command, but the next command is a stop, then stop the current action flags["stopped"] = True # set the stopped flag expiration_time = rospy.get_time() # next_cmd = {} # empty next command slot if expiration_time: if rospy.get_time() < expiration_time and not flags["stalled"]:
t = time.time() detector = CNNClassificator(args['classlist'], args['snapshot'], args['arch'], args['mean'], args['gpu']) tdiff = int(time.time() - t) print 'model load time: %d:%02d' % (tdiff // 60, tdiff % 60) label, prob, imgdata = detect_image_label( detector, ftp_h, fpath) print('Label: {}, {}%'.format(label, int(prob * 100))) output_name = REDIS_OUTPUT_PREFIX + label if prob < 0.7: queue_pfx = NO_LABEL_QUEUE redis.rpush(queue_pfx, bname) redis.ltrim(queue_pfx, max(0, redis.llen(queue_pfx) - 100), -1) else: last_rec = redis.lrange(output_name, -1, -1) if last_rec: last_rec = json.loads(last_rec[0]) if last_rec['ts'] < ts and last_rec['label'] != label: msg = 'Changed at %s from %s to %s (diff=%d), %s' % ( dt.strftime('%d/%m %H:%M:%S'), last_rec['label'], label, ts - last_rec['ts'], bname) print '*' * 10, msg, '*' * 10 send_email( '*****@*****.**', '%s: %s' % (dt.strftime('%H:%M:%S %d/%m'), label), msg, [imgdata])
redis.set("counter", 1) print(redis.get("counter")) # 1 redis.incr("counter") print(redis.get("counter")) # 2 redis.decr("counter") print(redis.get("counter")) #1 # List : possible to duplicate values redis.rpush("members", "r1") redis.rpush("members", "r2") redis.lpush("members", "l1") redis.lpush("members", "l2") print(redis.lrange("members", 0, 0)) print(redis.lrange("members", 0, 1)) print(redis.lrange("members", 0, 2)) print(redis.llen("members")) print(redis.lrange("members",0, redis.llen("members")-1)) print(redis.lindex("members",3)) print(redis.rpop("members")) print(redis.lpop("members")) print(redis.llen("members")) print(redis.lrange("members",0, redis.llen("members")-1)) redis.delete("members") #Sets : impossible to duplicate values redis.sadd("members", "s1") redis.sadd("members", "s1") redis.sadd("members", "s2") redis.sadd("members", "s3") redis.sadd("members", "s4") redis.sadd("members", "s5")
import xlrd import xlsxwriter import redis redis = redis.StrictRedis(host='127.0.0.1', port=6379, decode_responses=True) ExcelFile=xlrd.open_workbook(r'C:\Users\Administrator\Desktop\first_result.xlsx') #打开目标处理xlsx sheet=ExcelFile.sheet_by_index(0) #定位到索引为0的工作表,即第一个工作表 cols=sheet.col_values(1) #取第3列的数据 # print(redis.llen('provider')) for i in range(1,len(cols)): redis.lpush('licenceCode1',cols[i]) print(redis.llen('licenceCode1'),cols[i]) # for i in range(redis.llen('licenceCode')): # redis.lpop('licenceCode') # print(redis.llen('licenceCode')) # l = ['成都房江湖信息科技有限公司','成都昱超贸易有限公司','南宁市顺柏贸易有限公司','浙江力德节能科技有限公司','温岭市广源门窗幕墙工程有限公司','上海柏瑜实业有限公司','深圳市明源云客电子商务有限公司','广西金图工程咨询有限公司','辽宁同飞玺铭会计师事务所有限责任公司','重庆力瑞机械设备有限公司','云南卓立工程检测有限公司'] # # for i in l : # redis.lpush('provider_law',i)
import logging import csv import io from util import encode logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG) try: redis = redis.StrictRedis(host='localhost', port=6379, db=0) except: print("Can't connect to Redis!") parser = argparse.ArgumentParser() parser.add_argument("-q", "--queries", dest="query", help="Give names of queries stored in Redis.", default='-') args = parser.parse_args() with io.open('data/%s.json' % args.query, 'a', encoding='utf-8') as outfile: i = redis.llen(args.query) logging.debug("Writing %s tweets to data/%s.json" % (i, args.query)) while i > 0: data = json.loads(redis.lpop(args.query)) outfile.write( unicode( json.dumps(data, ensure_ascii=False, indent=4, sort_keys=True))) i = i - 1
# print 'set fish_table[%s]'%(key) # pipe.hmset(key,update) # # goods_lists = redis.lrange(FISH_REWARD_ON_SHOP_LIST,0,-1) # for goods in goods_lists: # goods_type = redis.hget(FISH_REWARD_TABLE%(goods),'reward_type') # pipe.lpush(FISH_REWARD_ON_SHOP_TYPE_LIST%(goods_type),goods) # print 'set goods id index success....[%s]'%(goods_type) # pipe.execute() ############################################################### ### exchange_table更新 ### 新增1个字段 exchange_type ### 2018-01-03 ############################################################### total = redis.llen(FISH_EXCHANGE_LIST) exchange_ids = redis.lrange(FISH_EXCHANGE_LIST, 0, -1) exchange_id_keys = [ FISH_EXCHANGE_TABLE % (exchange_id) for exchange_id in exchange_ids ] exchange_details = [ exchange_detail for exchange_detail in redis.mget(exchange_id_keys) ] exchange_info = [] for exchange_detail in exchange_details: exchange_detail = eval(exchange_detail) exchange_detail['exchange_type'] = redis.hget( FISH_REWARD_TABLE % (exchange_detail['exchange_reward_id']), 'reward_type') pipe.set(FISH_EXCHANGE_TABLE % (exchange_detail['exchange_id']),
from psycopg2.extensions import AsIs import datetime try: redis = redis.StrictRedis(host='localhost', port=6379, db=0) except: print "Can't connect to Redis!" try: conn = psycopg2.connect("dbname='postgres'") except: print "Can't connect to PSQL!" subreddits = ['politics','worldnews','news'] date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') cur = conn.cursor() for subreddit in subreddits: i = redis.llen(subreddit) while i > 0: word = redis.lpop(subreddit) cur.execute("INSERT INTO reddit.%s AS s (word, count, full_date, day) VALUES (%s, %s, %s, date_trunc('day', %s::date)) ON CONFLICT ON CONSTRAINT %s_word_day DO UPDATE SET count = s.count + 1", (AsIs(subreddit), word, 1, date, date, AsIs(subreddit))) i = i - 1 conn.commit() cur.close() conn.close()
parser.add_argument("-q", "--queries", dest="query", help="Give names of queries stored in Redis.", default='-') args = parser.parse_args() tablename = args.query cur = conn.cursor() cur.execute( sql.SQL( "CREATE TABLE IF NOT EXISTS twitter.{} (id BIGINT, createdate TIMESTAMP, body VARCHAR, username VARCHAR, url VARCHAR, location VARCHAR, address VARCHAR, incident_type VARCHAR)" ).format(sql.Identifier(tablename))) i = redis.llen(tablename) logging.debug("Writing %s tweets to database twitter.%s" % (i, tablename)) while i > 0: data = json.loads(redis.lpop(tablename)) tweetBody = data['text'] at = 'at' startBracket = '[' atIndex = tweetBody.find(at) startBracketIndex = tweetBody.find(startBracket)
def status(): '''returns the status''' count = redis.llen(trace_list) last = redis.get(last_ip) return "Online: %d traces in the db. Last sent IP is %s" % (count, last)
def get_welfare_info(redis, account): today = datetime.now().strftime("%Y-%m-%d") first = first_day_of_month() last = date.today() - timedelta(days=1) res = copy.deepcopy(WELFARE_CONFIG) res['date'] = today res['signed'] = [] res['unsinged'] = [] # 今天是否签到 if redis.exists(WELFARE_USER_SIGN % (account, today)): res['issigned'] = 1 else: res['issigned'] = 0 while first <= last: if redis.exists(WELFARE_USER_SIGN % (account, first)): res['signed'].append(first.strftime('%d')) elif redis.sismember(WELFARE_USER_PATCH_SIGN % (account, first.strftime("%m")), first.strftime('%Y-%m-%d')): res['signed'].append(first.strftime('%d')) else: res['unsinged'].append(first.strftime('%d')) first += timedelta(1) # 七天奖励 if not redis.exists(GOLD_WELFARE_SIGN_7DAYS): res['rewardlist'][0]['status'] = MESSION_STATUS_NO elif redis.hget(GOLD_WELFARE_SIGN_7DAYS, account) == MESSION_STATUS_OVER: res['rewardlist'][0]['status'] = MESSION_STATUS_OVER else: res['rewardlist'][0]['status'] = MESSION_STATUS_OK # 十五天奖励 if not redis.exists(GOLD_WELFARE_SIGN_15DAYS): res['rewardlist'][0]['status'] = MESSION_STATUS_NO elif redis.hget(GOLD_WELFARE_SIGN_15DAYS, account) == MESSION_STATUS_OVER: res['rewardlist'][0]['status'] = MESSION_STATUS_OVER else: res['rewardlist'][0]['status'] = MESSION_STATUS_OK # 月奖励 if not redis.exists(GOLD_WELFARE_SIGN_MONTH): res['rewardlist'][0]['status'] = MESSION_STATUS_NO elif redis.hget(GOLD_WELFARE_SIGN_MONTH, account) == MESSION_STATUS_OVER: res['rewardlist'][0]['status'] = MESSION_STATUS_OVER else: res['rewardlist'][0]['status'] = MESSION_STATUS_OK # 每日首冲奖励 if not redis.sismember(DAILY_USER_GOLD2_SET % today, account): res['messionlist'][0]['status'] = MESSION_STATUS_NO res['messionlist'][0]['parent_mode'] = CHECK_MALL else: if redis.hget(GOLD_REWARD_DAY_BUY_GOLD_HASH % today, account) == MESSION_STATUS_OVER: res['messionlist'][0]['status'] = MESSION_STATUS_OVER else: res['messionlist'][0]['status'] = MESSION_STATUS_OK # 新手礼包 if redis.hget(GOLD_REWARD_NEW_PRESENT_HASH, account) == MESSION_STATUS_OVER: res['messionlist'][1]['status'] = MESSION_STATUS_OVER else: res['messionlist'][1]['status'] = MESSION_STATUS_OK # 破产补助 key = WELFARE_USER_INSURANCE % (account, today) if not redis.exists(WELFARE_USER_INSURANCE % (account, today)): res['messionlist'][2]['status'] = MESSION_STATUS_OK else: if redis.llen(key) >= SIGN_MAX: res['messionlist'][2]['status'] = MESSION_STATUS_OVER else: res['messionlist'][2]['status'] = MESSION_STATUS_OK return res
def count(): '''returns the number of traces currently stored''' count = redis.llen(trace_list) return jsonify({'count': count})
def connect(): language = request.args.get('lang') # id parameter needed to avoid heroku caching request and waiting because # it was exactly the same as previous request when languages are also the same id = request.args.get('id') # print("IN /CONNECT") if id is None or language is None: # print("Reconnection received") langs = [] lang_arr = redis.lrange("langs", 0, redis.llen("langs")) for lang in lang_arr: if not isinstance(lang, str): lang_key = lang.decode("utf-8") else: lang_key = lang langs.append(lang_key) # print("Language list currently contains: ", langs) return jsonify(langs) else: print("IN ELSE") print(language) print(id) try: print("Connecting client with ID: " + id) num_connected = redis.get("clients") num_connected = int(num_connected.decode("utf-8")) redis.set("clients", num_connected + 1) print("PRINTING CLIENTS", redis.get("clients")) # List logic redis.lpush("langs", language) # redis.sadd("languages", language) print("Current languages", redis.smembers("languages")) # langs = redis.smembers("languages") while num_connected != 2: num_connected = redis.get("clients") num_connected = int(num_connected.decode("utf-8")) time.sleep(.5) print("waiting for other client in /connect. Currently have: ", num_connected) langs = [] lang_arr = redis.lrange("langs", 0, redis.llen("langs")) for lang in lang_arr: if not isinstance(lang, str): lang_key = lang.decode("utf-8") else: lang_key = lang langs.append(lang_key) except Exception as err: print("IN EXCEPT") langs = [] lang_arr = redis.lrange("langs", 0, redis.llen("langs")) for lang in lang_arr: if not isinstance(lang, str): lang_key = lang.decode("utf-8") else: lang_key = lang langs.append(lang_key) print("Language list currently contains: ", langs) return jsonify(langs)
def delete_all(): '''Debug: clears the redis queue''' res = redis.ltrim(trace_list, 0, 0) res = redis.lpop(trace_list) count = redis.llen(trace_list) return jsonify({'count': count})
for mname in MODELLIST: models[mname] = cPickle.loads(open(os.path.join(MODELPATH, mname + '.svc')).read()) # cPickle.dump(models[mname], open(os.path.join(MODELPATH, mname + '.svc.new'), 'w'), protocol=cPickle.HIGHEST_PROTOCOL) tdiff = int(time.time() - t) dbprint('models load time: %d:%02d' % (tdiff//60, tdiff%60)) msglist = [] labellist = [] dbprint('model names: %s' % models.keys()) for mname,model in models.items(): output_name = REDIS_OUTPUT_PREFIX + mname dbprint('Start %s' % bname) label,imgdata = detect_image_label(model, ftp_h, fpath) if label == NO_LABEL: queue_pfx = NO_LABEL_QUEUE_PREFIX + mname redis.rpush(queue_pfx, bname) redis.ltrim(queue_pfx, max(0, redis.llen(queue_pfx) - 100), -1) elif label in ('open', 'close'): redis.set('gate', json.dumps({'label': label, 'ts': time.time()})) if label != NO_LABEL: last_rec = redis.lrange(output_name, -1, -1) if last_rec: last_rec = json.loads(last_rec[0]) if last_rec['ts'] < ts and last_rec['label'] != label: msg = '%s changed at %s from %s to %s (diff=%d), %s' % (mname, dt.strftime('%d/%m %H:%M:%S'), last_rec['label'], label, ts - last_rec['ts'], bname) dbprint('%s %s' % (bname, msg)) msglist.append(msg) labellist.append((mname, label)) else: msg = 'Initial at %s %s' % (dt.strftime('%d/%m %H:%M:%S'), label) dbprint('%s %s' % (bname, msg)) msglist.append(msg)
def GetEnableListLen(redis): try: return redis.llen("APO_ENABLE_LIST") except Exception, e: print "GET APO_ENABLE_LIST LENGTH FAILED" traceback.print_exc()
lista = [ 'arbor-collector', 'arbor-parser', 'archive', 'bruteforceblocker-collector', 'bruteforceblocker-parser', 'cymru-expert', 'deduplicator-expert', 'dragon-research-group-ssh-collector', 'dragon-research-group-ssh-parser', 'dragon-research-group-vnc-collector', 'dragon-research-group-vnc-parser', 'malware-domain-list-collector', 'malware-domain-list-parser', 'malwarepatrol-dans-guardian-collector', 'malwarepatrol-dans-guardian-parser', 'openbl-collector', 'openbl-parser', 'phishtank-collector', 'phishtank-parser', 'postgresql-output', 'sanitizer-expert', 'taxonomy-expert', 'vxvault-collector', 'vxvault-parser' ] for bot in lista: print "%s: %s" % (bot, redis.llen(bot+"-queue"))
from psycopg2 import * import geocode from geocode import geocode try: conn = psycopg2.connect("dbname='postgres'") except: print("Can't connect to PSQL!") try: redis = redis.StrictRedis(host='localhost', port=6379, db=0) except: print("Can't connect to Redis!") cur = conn.cursor() i = redis.llen('event') while i > 0: data = json.loads(redis.lpop('event')) policeTweet = False if str(data['username']) == 'pdxpolicelog': policeTweet = True # Check if the retweet indicator is in the text body. Do not process retweets. isRetweet = False if 'RT @pdxpolicelog: ' in str(data['text']): isRetweet = True if isRetweet != True: