def main(): config = ConfigParser.ConfigParser() if len(config.read(['/home/pi/lol/conf.ini'])): api_key = config.get("user", "api_key") redis_host = config.get("database", "redis_host") redis_port = config.get("database", "redis_port") redis = setRedisConn(host=redis_host, port=redis_port) last_match = redis.lindex("last100", 0) if last_match is None: # for brand-new database match_id = getLastMatch(region, seed_pid="4460427", api_key=api_key) else: match_id = increaseId(last_match) region = "kr" while True: # crawl the match data match_details = getMatchDetails(region=region, match_id=match_id, api_key=api_key) if match_details is not None: # insert match data into redis mid = match_details.pop("matchId") redis.hmset(mid, match_details) # update list of last-inserted redis.lpush("last100", mid) redis.ltrim("last100", 0, 99) match_id = increaseId(match_id) time.sleep(1) # to avoid api overshooting (max 500 queries per 10 min) else: print "File conf.ini not found. Program aborted." exit(1)
def episode_callback(episode): # Compute total reward. rewards = [t[2] for t in episode] episode_reward = np.sum(rewards) #if np.random.random_sample() < 0.05: print('### episode_reward: {}.'.format(episode_reward)) redis.lpush(REWARDS_KEY, struct.pack('d', episode_reward)) redis.ltrim(REWARDS_KEY, 0, 20)
def on_msg(client, userdata, message): print('message received %s, searching for %s' % (str(message.payload.decode('utf-8')), sensor_key)) data = json.loads(message.payload.decode('utf-8')) data['value'] = data[sensor_key] data['when'] = str(datetime.now()) redis.lpush(sys.argv[2], json.dumps(data)) redis.ltrim(sys.argv[2], 0, 50)
def chat_message(g): if g.get('msg') and current_user.is_authenticated: message = { 'user': current_user.name, 'msg': escape_html(g.get('msg')[:250]) } redis.lpush('chathistory', json.dumps(message)) redis.ltrim('chathistory', 0, 20) socketio.emit('msg', message, namespace='/snt', room='chat')
def run_apps_updater(redis, client, sentry_sdk): while True: pipe = redis.pipeline() pipe.get('current_change') pipe.lrange('apps-queue', 0, Consts.REDIS_BATCH_SIZE - 1) pipe.llen('apps-queue') pipe_response = pipe.execute() changenumber = pipe_response[0] app_to_packages = list_of_jsons_to_json(pipe_response[1]) app_ids = app_to_packages.keys() apps_queue_size = pipe_response[2] print('{} Fetched {} apps from queue... ({} apps yet in queue)'.format( Consts.LOG_PREFIX, len(app_ids), apps_queue_size)) if not app_ids: time.sleep(5) continue while True: try: # run new-releases updater every n minutes schedule.run_pending() while True: data = client.get_product_info(apps=app_ids, timeout=300) if data is not None: break else: print( '{} Data is None, retrying in 3 seconds...'.format( Consts.LOG_PREFIX)) time.sleep(3) client = connect_to_steam() for app in data.get('apps', {}).values(): if is_valid_app(app): handle_app(app, app_to_packages) break except AttributeError as e: print( '{} Didn\'t get any apps, retrying in 5 seconds...'.format( Consts.LOG_PREFIX)) sentry_sdk.capture_exception(e) time.sleep(5) redis.ltrim('apps-queue', Consts.REDIS_BATCH_SIZE, -1) print('{} Batch completed, retrying in 3 seconds...'.format( Consts.LOG_PREFIX)) time.sleep(3)
def main(): top_story_ids = requests.get( "{}/topstories.json".format(API_PREFIX)).json() pool = Pool(50) rust_stories = list( filter(lambda story: "Rust" in story.get("title", ""), pool.imap(fetch_story, top_story_ids)))[:MAX_COUNT] stories_length = len(rust_stories) if stories_length < MAX_COUNT: existed_story_ids = set(map(int, redis.lrange(REDIS_KEY, 0, -1))) existed_story_ids -= set(item["id"] for item in rust_stories) rust_stories.extend( pool.imap(fetch_story, list(existed_story_ids)[:MAX_COUNT - stories_length])) redis.lpush(REDIS_KEY, *[item["id"] for item in rust_stories]) redis.ltrim(REDIS_KEY, 0, MAX_COUNT - 1) render(rust_stories)
def enqueue(): data = json.loads(request.data.decode()) if 'input_url' not in data: response = { 'error': "The Youtube URL to download must be provided as 'input_url'", } logger.warn("Rejecting /api/enqueue request missing 'input_url'") return json.dumps(response), 400 # bad request clean_url = util.validate_url(data['input_url']) if clean_url is None: response = { 'error': "I'm sorry, that doesn't really look like a Youtube URL. :-(", 'info': "Please try again using a link starting with 'https://www.youtube.com'.", } logger.warn("Rejecting /api/enqueue request for %s" % data['input_url']) return json.dumps(response), 403 # forbidden logger.info("Accepting /api/enqueue request for %s" % clean_url) job = rqueue.enqueue_call( func=util.download, args=(clean_url, ), result_ttl=900 # 15 minutes ) job_id = job.get_id() redis.lpush(joblist, job_id) redis.ltrim(joblist, 0, 9) job_details = { 'job_id': job_id, 'request_url': clean_url, 'submitted': time.time(), 'page_title': '...', # just a placeholder to keep it pretty } redis.hmset(jobkey(job_id), job_details) redis.expire(jobkey(job_id), 86400) # 24 hours response = { 'job_id': job_id, } return json.dumps(response), 201 # created
def run_packages_updater(redis): while True: pipe = redis.pipeline() pipe.get('current_change') pipe.lrange('packages-queue', 0, Consts.REDIS_BATCH_SIZE - 1) pipe.llen('packages-queue') pipe_response = pipe.execute() changenumber = pipe_response[0] package_ids = [int(num) for num in pipe_response[1]] packages_queue_size = pipe_response[2] print( '{} Fetched {} packages from queue... ({} packages yet in queue)'. format(Consts.LOG_PREFIX, len(package_ids), packages_queue_size)) if not package_ids: time.sleep(5) continue while True: client = connect_to_steam() try: data = client.get_product_info(packages=package_ids, timeout=300) for package in data.get('packages', {}).values(): if is_valid_package(package): handle_package(package, redis) break except AttributeError: print('{} Didn\'t get any products, retrying in 5 seconds...'. format(Consts.LOG_PREFIX)) time.sleep(5) redis.ltrim('packages-queue', Consts.REDIS_BATCH_SIZE, -1) print('{} Batch completed, retry in 10 seconds...'.format( Consts.LOG_PREFIX)) time.sleep(10)
def enqueue(): data = json.loads(request.data.decode()) if 'input_url' not in data: response = { 'error': "The Youtube URL to download must be provided as 'input_url'", } logger.warn("Rejecting /api/enqueue request missing 'input_url'") return json.dumps(response), 400 # bad request clean_url = util.validate_url(data['input_url']) if clean_url is None: response = { 'error': "I'm sorry, that doesn't really look like a Youtube URL. :-(", 'info': "Please try again using a link starting with 'https://www.youtube.com'.", } logger.warn("Rejecting /api/enqueue request for %s" % data['input_url']) return json.dumps(response), 403 # forbidden logger.info("Accepting /api/enqueue request for %s" % clean_url) job = rqueue.enqueue_call( func=util.download, args=(clean_url,), result_ttl=900 # 15 minutes ) job_id = job.get_id() redis.lpush(joblist, job_id) redis.ltrim(joblist, 0, 9) job_details = { 'job_id': job_id, 'request_url': clean_url, 'submitted': time.time(), 'page_title': '...', # just a placeholder to keep it pretty } redis.hmset(jobkey(job_id), job_details) redis.expire(jobkey(job_id), 86400) # 24 hours response = { 'job_id': job_id, } return json.dumps(response), 201 # created
def access_domain(domain, port): redis = _get_redis() if redis: redis.lpush('last-domains', '%s:%d' % (domain, port)) redis.ltrim('last-domains', 0, 10) redis.setex('%s:%d' % (domain, port), '', 60 * 10)
for mname in MODELLIST: models[mname] = cPickle.loads(open(os.path.join(MODELPATH, mname + '.svc')).read()) # cPickle.dump(models[mname], open(os.path.join(MODELPATH, mname + '.svc.new'), 'w'), protocol=cPickle.HIGHEST_PROTOCOL) tdiff = int(time.time() - t) dbprint('models load time: %d:%02d' % (tdiff//60, tdiff%60)) msglist = [] labellist = [] dbprint('model names: %s' % models.keys()) for mname,model in models.items(): output_name = REDIS_OUTPUT_PREFIX + mname dbprint('Start %s' % bname) label,imgdata = detect_image_label(model, ftp_h, fpath) if label == NO_LABEL: queue_pfx = NO_LABEL_QUEUE_PREFIX + mname redis.rpush(queue_pfx, bname) redis.ltrim(queue_pfx, max(0, redis.llen(queue_pfx) - 100), -1) elif label in ('open', 'close'): redis.set('gate', json.dumps({'label': label, 'ts': time.time()})) if label != NO_LABEL: last_rec = redis.lrange(output_name, -1, -1) if last_rec: last_rec = json.loads(last_rec[0]) if last_rec['ts'] < ts and last_rec['label'] != label: msg = '%s changed at %s from %s to %s (diff=%d), %s' % (mname, dt.strftime('%d/%m %H:%M:%S'), last_rec['label'], label, ts - last_rec['ts'], bname) dbprint('%s %s' % (bname, msg)) msglist.append(msg) labellist.append((mname, label)) else: msg = 'Initial at %s %s' % (dt.strftime('%d/%m %H:%M:%S'), label) dbprint('%s %s' % (bname, msg)) msglist.append(msg)
def delete_all(): '''Debug: clears the redis queue''' res = redis.ltrim(trace_list, 0, 0) res = redis.lpop(trace_list) count = redis.llen(trace_list) return jsonify({'count': count})
def _send_item_with_size(key, data, size): redis.lpush(key, data) redis.ltrim(key, 0, size - 1)
#!/usr/bin/python """ Logging Daemon (c) Christopher Goes 2015 """ import zmq import redis from auvlog import config redis = redis.StrictRedis() context = zmq.Context() socketS = context.socket(zmq.SUB) socketS.setsockopt(zmq.SUBSCRIBE, b'') socketS.bind("tcp://*:{0}".format(config.CLIENT_PORT)) socketP = context.socket(zmq.PUB) socketP.bind("tcp://*:{0}".format(config.SERVER_PORT)) print("sockets bound!", socketP, socketS) while True: res = socketS.recv() redis.lpush(config.KEY, res) redis.ltrim(config.KEY, 0, int(1e5)) socketP.send(res)
channel = settings.channel stream_host = settings.stream_host def bad (term): bad = len(term) <= 1 bad = bad or re.search("\D", term) is None bad = bad or re.search(settings.exclude_regex, term) is not None return bad if __name__ == '__main__': stream = redis.StrictRedis(host=stream_host, port=6379, db=0) redis = redis.StrictRedis(host="localhost", port=6379, db=0) ps = stream.pubsub() ps.subscribe([channel]) print "Subcribed to ", channel for item in ps.listen(): if item['type'] == 'message': i = item['data'].strip() # print "Received ", item['data'] if not bad(i): items = redis.lrange(channel, 0, -1) if not i in items: print "Added ", i redis.ltrim(channel, 0, 20) redis.lpush(channel, i) else : print "Ignored", i
def store_most_recently_added_certificates(cert): redis = _get_redis() if redis: redis.lpush('last-certificates', cert.get_pem()) redis.ltrim('last-certificates', 0, 10)
def delete_list(redis, key): while redis.llen(key) > 0: redis.ltrim(key, 0, -99)