def main():
config = ConfigParser.ConfigParser()
if len(config.read(['/home/pi/lol/conf.ini'])):
api_key = config.get("user", "api_key")
redis_host = config.get("database", "redis_host")
redis_port = config.get("database", "redis_port")
redis = setRedisConn(host=redis_host, port=redis_port)
last_match = redis.lindex("last100", 0)
if last_match is None: # for brand-new database
match_id = getLastMatch(region, seed_pid="4460427", api_key=api_key)
else:
match_id = increaseId(last_match)
region = "kr"
while True:
# crawl the match data
match_details = getMatchDetails(region=region, match_id=match_id, api_key=api_key)
if match_details is not None:
# insert match data into redis
mid = match_details.pop("matchId")
redis.hmset(mid, match_details)
# update list of last-inserted
redis.lpush("last100", mid)
redis.ltrim("last100", 0, 99)
match_id = increaseId(match_id)
time.sleep(1) # to avoid api overshooting (max 500 queries per 10 min)
else:
print "File conf.ini not found. Program aborted."
exit(1)
def episode_callback(episode):
# Compute total reward.
rewards = [t[2] for t in episode]
episode_reward = np.sum(rewards)
#if np.random.random_sample() < 0.05:
print('### episode_reward: {}.'.format(episode_reward))
redis.lpush(REWARDS_KEY, struct.pack('d', episode_reward))
redis.ltrim(REWARDS_KEY, 0, 20)
def on_msg(client, userdata, message):
print('message received %s, searching for %s' %
(str(message.payload.decode('utf-8')), sensor_key))
data = json.loads(message.payload.decode('utf-8'))
data['value'] = data[sensor_key]
data['when'] = str(datetime.now())
redis.lpush(sys.argv[2], json.dumps(data))
redis.ltrim(sys.argv[2], 0, 50)
def chat_message(g):
if g.get('msg') and current_user.is_authenticated:
message = {
'user': current_user.name,
'msg': escape_html(g.get('msg')[:250])
}
redis.lpush('chathistory', json.dumps(message))
redis.ltrim('chathistory', 0, 20)
socketio.emit('msg', message, namespace='/snt', room='chat')
def run_apps_updater(redis, client, sentry_sdk):
while True:
pipe = redis.pipeline()
pipe.get('current_change')
pipe.lrange('apps-queue', 0, Consts.REDIS_BATCH_SIZE - 1)
pipe.llen('apps-queue')
pipe_response = pipe.execute()
changenumber = pipe_response[0]
app_to_packages = list_of_jsons_to_json(pipe_response[1])
app_ids = app_to_packages.keys()
apps_queue_size = pipe_response[2]
print('{} Fetched {} apps from queue... ({} apps yet in queue)'.format(
Consts.LOG_PREFIX, len(app_ids), apps_queue_size))
if not app_ids:
time.sleep(5)
continue
while True:
try:
# run new-releases updater every n minutes
schedule.run_pending()
while True:
data = client.get_product_info(apps=app_ids, timeout=300)
if data is not None:
break
else:
print(
'{} Data is None, retrying in 3 seconds...'.format(
Consts.LOG_PREFIX))
time.sleep(3)
client = connect_to_steam()
for app in data.get('apps', {}).values():
if is_valid_app(app):
handle_app(app, app_to_packages)
break
except AttributeError as e:
print(
'{} Didn\'t get any apps, retrying in 5 seconds...'.format(
Consts.LOG_PREFIX))
sentry_sdk.capture_exception(e)
time.sleep(5)
redis.ltrim('apps-queue', Consts.REDIS_BATCH_SIZE, -1)
print('{} Batch completed, retrying in 3 seconds...'.format(
Consts.LOG_PREFIX))
time.sleep(3)
def main():
top_story_ids = requests.get(
"{}/topstories.json".format(API_PREFIX)).json()
pool = Pool(50)
rust_stories = list(
filter(lambda story: "Rust" in story.get("title", ""),
pool.imap(fetch_story, top_story_ids)))[:MAX_COUNT]
stories_length = len(rust_stories)
if stories_length < MAX_COUNT:
existed_story_ids = set(map(int, redis.lrange(REDIS_KEY, 0, -1)))
existed_story_ids -= set(item["id"] for item in rust_stories)
rust_stories.extend(
pool.imap(fetch_story,
list(existed_story_ids)[:MAX_COUNT - stories_length]))
redis.lpush(REDIS_KEY, *[item["id"] for item in rust_stories])
redis.ltrim(REDIS_KEY, 0, MAX_COUNT - 1)
render(rust_stories)
def main():
top_story_ids = requests.get(
"{}/topstories.json".format(API_PREFIX)).json()
pool = Pool(50)
rust_stories = list(
filter(lambda story: "Rust" in story.get("title", ""),
pool.imap(fetch_story, top_story_ids)))[:MAX_COUNT]
stories_length = len(rust_stories)
if stories_length < MAX_COUNT:
existed_story_ids = set(map(int, redis.lrange(REDIS_KEY, 0, -1)))
existed_story_ids -= set(item["id"] for item in rust_stories)
rust_stories.extend(
pool.imap(fetch_story,
list(existed_story_ids)[:MAX_COUNT - stories_length]))
redis.lpush(REDIS_KEY, *[item["id"] for item in rust_stories])
redis.ltrim(REDIS_KEY, 0, MAX_COUNT - 1)
render(rust_stories)
def enqueue():
data = json.loads(request.data.decode())
if 'input_url' not in data:
response = {
'error':
"The Youtube URL to download must be provided as 'input_url'",
}
logger.warn("Rejecting /api/enqueue request missing 'input_url'")
return json.dumps(response), 400 # bad request
clean_url = util.validate_url(data['input_url'])
if clean_url is None:
response = {
'error':
"I'm sorry, that doesn't really look like a Youtube URL. :-(",
'info':
"Please try again using a link starting with 'https://www.youtube.com'.",
}
logger.warn("Rejecting /api/enqueue request for %s" %
data['input_url'])
return json.dumps(response), 403 # forbidden
logger.info("Accepting /api/enqueue request for %s" % clean_url)
job = rqueue.enqueue_call(
func=util.download,
args=(clean_url, ),
result_ttl=900 # 15 minutes
)
job_id = job.get_id()
redis.lpush(joblist, job_id)
redis.ltrim(joblist, 0, 9)
job_details = {
'job_id': job_id,
'request_url': clean_url,
'submitted': time.time(),
'page_title': '...', # just a placeholder to keep it pretty
}
redis.hmset(jobkey(job_id), job_details)
redis.expire(jobkey(job_id), 86400) # 24 hours
response = {
'job_id': job_id,
}
return json.dumps(response), 201 # created
def run_packages_updater(redis):
while True:
pipe = redis.pipeline()
pipe.get('current_change')
pipe.lrange('packages-queue', 0, Consts.REDIS_BATCH_SIZE - 1)
pipe.llen('packages-queue')
pipe_response = pipe.execute()
changenumber = pipe_response[0]
package_ids = [int(num) for num in pipe_response[1]]
packages_queue_size = pipe_response[2]
print(
'{} Fetched {} packages from queue... ({} packages yet in queue)'.
format(Consts.LOG_PREFIX, len(package_ids), packages_queue_size))
if not package_ids:
time.sleep(5)
continue
while True:
client = connect_to_steam()
try:
data = client.get_product_info(packages=package_ids,
timeout=300)
for package in data.get('packages', {}).values():
if is_valid_package(package):
handle_package(package, redis)
break
except AttributeError:
print('{} Didn\'t get any products, retrying in 5 seconds...'.
format(Consts.LOG_PREFIX))
time.sleep(5)
redis.ltrim('packages-queue', Consts.REDIS_BATCH_SIZE, -1)
print('{} Batch completed, retry in 10 seconds...'.format(
Consts.LOG_PREFIX))
time.sleep(10)
def enqueue():
data = json.loads(request.data.decode())
if 'input_url' not in data:
response = {
'error': "The Youtube URL to download must be provided as 'input_url'",
}
logger.warn("Rejecting /api/enqueue request missing 'input_url'")
return json.dumps(response), 400 # bad request
clean_url = util.validate_url(data['input_url'])
if clean_url is None:
response = {
'error': "I'm sorry, that doesn't really look like a Youtube URL. :-(",
'info': "Please try again using a link starting with 'https://www.youtube.com'.",
}
logger.warn("Rejecting /api/enqueue request for %s" % data['input_url'])
return json.dumps(response), 403 # forbidden
logger.info("Accepting /api/enqueue request for %s" % clean_url)
job = rqueue.enqueue_call(
func=util.download,
args=(clean_url,),
result_ttl=900 # 15 minutes
)
job_id = job.get_id()
redis.lpush(joblist, job_id)
redis.ltrim(joblist, 0, 9)
job_details = {
'job_id': job_id,
'request_url': clean_url,
'submitted': time.time(),
'page_title': '...', # just a placeholder to keep it pretty
}
redis.hmset(jobkey(job_id), job_details)
redis.expire(jobkey(job_id), 86400) # 24 hours
response = {
'job_id': job_id,
}
return json.dumps(response), 201 # created
def access_domain(domain, port):
redis = _get_redis()
if redis:
redis.lpush('last-domains', '%s:%d' % (domain, port))
redis.ltrim('last-domains', 0, 10)
redis.setex('%s:%d' % (domain, port), '', 60 * 10)
for mname in MODELLIST:
models[mname] = cPickle.loads(open(os.path.join(MODELPATH, mname + '.svc')).read())
# cPickle.dump(models[mname], open(os.path.join(MODELPATH, mname + '.svc.new'), 'w'), protocol=cPickle.HIGHEST_PROTOCOL)
tdiff = int(time.time() - t)
dbprint('models load time: %d:%02d' % (tdiff//60, tdiff%60))
msglist = []
labellist = []
dbprint('model names: %s' % models.keys())
for mname,model in models.items():
output_name = REDIS_OUTPUT_PREFIX + mname
dbprint('Start %s' % bname)
label,imgdata = detect_image_label(model, ftp_h, fpath)
if label == NO_LABEL:
queue_pfx = NO_LABEL_QUEUE_PREFIX + mname
redis.rpush(queue_pfx, bname)
redis.ltrim(queue_pfx, max(0, redis.llen(queue_pfx) - 100), -1)
elif label in ('open', 'close'):
redis.set('gate', json.dumps({'label': label, 'ts': time.time()}))
if label != NO_LABEL:
last_rec = redis.lrange(output_name, -1, -1)
if last_rec:
last_rec = json.loads(last_rec[0])
if last_rec['ts'] < ts and last_rec['label'] != label:
msg = '%s changed at %s from %s to %s (diff=%d), %s' % (mname, dt.strftime('%d/%m %H:%M:%S'), last_rec['label'], label, ts - last_rec['ts'], bname)
dbprint('%s %s' % (bname, msg))
msglist.append(msg)
labellist.append((mname, label))
else:
msg = 'Initial at %s %s' % (dt.strftime('%d/%m %H:%M:%S'), label)
dbprint('%s %s' % (bname, msg))
msglist.append(msg)
def delete_all():
'''Debug: clears the redis queue'''
res = redis.ltrim(trace_list, 0, 0)
res = redis.lpop(trace_list)
count = redis.llen(trace_list)
return jsonify({'count': count})
def _send_item_with_size(key, data, size):
redis.lpush(key, data)
redis.ltrim(key, 0, size - 1)
#!/usr/bin/python
"""
Logging Daemon
(c) Christopher Goes 2015
"""
import zmq
import redis
from auvlog import config
redis = redis.StrictRedis()
context = zmq.Context()
socketS = context.socket(zmq.SUB)
socketS.setsockopt(zmq.SUBSCRIBE, b'')
socketS.bind("tcp://*:{0}".format(config.CLIENT_PORT))
socketP = context.socket(zmq.PUB)
socketP.bind("tcp://*:{0}".format(config.SERVER_PORT))
print("sockets bound!", socketP, socketS)
while True:
res = socketS.recv()
redis.lpush(config.KEY, res)
redis.ltrim(config.KEY, 0, int(1e5))
socketP.send(res)
channel = settings.channel
stream_host = settings.stream_host
def bad (term):
bad = len(term) <= 1
bad = bad or re.search("\D", term) is None
bad = bad or re.search(settings.exclude_regex, term) is not None
return bad
if __name__ == '__main__':
stream = redis.StrictRedis(host=stream_host, port=6379, db=0)
redis = redis.StrictRedis(host="localhost", port=6379, db=0)
ps = stream.pubsub()
ps.subscribe([channel])
print "Subcribed to ", channel
for item in ps.listen():
if item['type'] == 'message':
i = item['data'].strip()
# print "Received ", item['data']
if not bad(i):
items = redis.lrange(channel, 0, -1)
if not i in items:
print "Added ", i
redis.ltrim(channel, 0, 20)
redis.lpush(channel, i)
else :
print "Ignored", i
def store_most_recently_added_certificates(cert):
redis = _get_redis()
if redis:
redis.lpush('last-certificates', cert.get_pem())
redis.ltrim('last-certificates', 0, 10)
def delete_list(redis, key):
while redis.llen(key) > 0:
redis.ltrim(key, 0, -99)
def _send_item_with_size(key, data, size):
redis.lpush(key, data)
redis.ltrim(key, 0, size - 1)
