def predict(): logger.info('Use history weight: {}'.format(WEIGHT)) logger.info("Initting connection to redis") con = get_redis_client(config) for model in MODELS: if model not in ["tv", "movie", "cartoon", ]: continue data, ids = load_data(model, static=True) logger.info('Init sim handler') s = Sim(data, ids, weight=WEIGHT[model], static_=True) count = 0 try: for cover_id, result in s.process(): count += 1 if DEBUG: logger.info('{} {}'.format(cover_id, result)) continue con.set(KEY_PATTERN.format(cover_id), json.dumps(result)) con.expire(KEY_PATTERN.format(cover_id), 2592000) except Exception, e: logger.error('catched error :{}, processed num: {}, model: {}'.format(e, count, model)) traceback.print_exc() raise Exception('Error') logger.info('Num of {} result : {}'.format(model, count)) print('Num of {} result : {}'.format(model, count))
def index_orders(redis_url, order_queue, unindex=False): redisClient = util.get_redis_client(redis_url) while True: with util.get_queue_message(order_queue, redisClient) as message: if not unindex: try: record_order(message, util.Locker(redisClient)) except Exception: logger.exception("Error recording message") else: try: delete_order(message, util.Locker(redisClient)) except Exception: logger.exception("Error deleting record")
def populate_blockhash(redis_url, topic_name): global blockhash r = util.get_redis_client(redis_url) # Get block number initially. The pubsub channel will give us block hashes, # but from a caching perspective they should all be unique, which is the # important part. try: blockhash = r.get("topic://%s::blocknumber" % topic_name).decode("utf8") except Exception: pass p = r.pubsub() p.subscribe(topic_name) for message in p.listen(): if message.get("type") == "message": try: blockhash = json.loads(message["data"]) except KeyError: pass
def __init__(self, tid): self._redis_key = redis_prefix + str(tid) self._redis = get_redis_client() self._data = self.load() or {} self._destoried = False
# If we're doing this repeatedly, we wait until the queue is # cleared to load it up again. time.sleep(length_check_frequency) remaining_time = (start_time + delay) - time.time() if remaining_time > 0: # If the queue clears faster than the round delay, wait until the # time is up to load the queue again. time.sleep(remaining_time) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("redis_url") parser.add_argument("publish_queue") parser.add_argument("--round-delay", "-d", type=int, default=60) parser.add_argument("--length-check-freq", "-f", type=int, default=0.1) parser.add_argument("--recur", "-r", action="store_true", default=False) parser.add_argument("--queued-max", "-m", type=int, default=1000) parser.add_argument("--log-level", "-l", default="info") args = parser.parse_args() logging.basicConfig(level=getattr(logging, args.log_level.upper())) redisClient = util.get_redis_client(args.redis_url) if args.recur: main(redisClient, args.publish_queue, args.round_delay, args.queued_max, args.length_check_freq) else: queue_orders(redisClient, args.publish_queue, args.queued_max, args.length_check_freq)
# encoding:utf8 __author__ = 'brianyang' import sys import time import random import json import sys sys.path.append('/home/q/blog') from util import get_redis_client redis_client = get_redis_client() reload(sys) sys.setdefaultencoding('utf8') import requests from bs4 import BeautifulSoup url = 'http://xjh.haitou.cc/bj/page-%d' scrapy_size = 30 result = [] id_dict = {} def parse_result(soup, result): trs = soup.find_all('tr', attrs={'data-key': True}) for tr in trs: id_ = tr['data-key'] title_dom = tr.find('td', class_='cxxt-title') cancel = title_dom.find('span', class_='badge badge-cancel')