def run(self, params): host = params.get('h', '').strip() port = params.get('p', 80) hkey = '%s:%d' % (host, port) if _check_t(params): abort_current_job() connections.redis.sadd(CONF_DATA_ALL_KEY, hkey) tmp = connections.redis.hget(CONF_DATA_RANK_KEY, hkey) now_num = int(tmp) if tmp else 0 if not now_num: connections.redis.hincrby(CONF_DATA_RANK_KEY, hkey, 1) test = CONF_CHECK_PROXY_FUNC(host, port) test and log.info('CHECK OK proxy:%s, num:%d' % (hkey, now_num)) if test: if now_num <= 0: now_num = 1 if connections.redis.sismember( CONF_DATA_OK_KEY, hkey) else 10 connections.redis.hset(CONF_DATA_RANK_KEY, hkey, now_num) elif 0 < now_num < 20: connections.redis.hincrby(CONF_DATA_RANK_KEY, hkey, 1) now_num += 1 else: if now_num >= -10: connections.redis.hincrby(CONF_DATA_RANK_KEY, hkey, -1) now_num -= 1 if now_num > 0: connections.redis.sadd(CONF_DATA_OK_KEY, hkey) else: connections.redis.srem(CONF_DATA_OK_KEY, hkey) now_num <= -10 and connections.redis.srem(CONF_DATA_ALL_KEY, hkey) return {'proxy': hkey, 'num': now_num, 'test': test}
def run(self, params): log.info("Will abort this task") connections.mongodb_jobs.tests_inserts.insert(params) try: raise InAbortException except InAbortException: abort_current_job() raise Exception("Should not be reached")
def run(self, params): filename = params.get('f', '').strip() if _check_t(params): abort_current_job() timer_num = 3 timer_seq = CONF_CHECK_INTERVAL gql = pyfile.load_str(filename).strip() if not gql: abort_current_job() proxy_list, gret = run_gdom_page( gql, get_proxy=lambda: connections.redis.srandmember(CONF_DATA_OK_KEY)) proxy_list and log.info('FETCH OK filename:%s, num:%d' % (filename, len(proxy_list))) if gret.errors: log.error('FETCH ERROR filename:%s, errors:%s' % (filename, gret.errors)) if not proxy_list: log.error('FETCH EMPTY filename:%s, gret:%r' % (filename, gret)) abort_current_job() timestamp = int(time.time()) task_map = {} for proxy_str in proxy_list: host = proxy_str.split(':', 1)[0] port = int(proxy_str.split(':', 1)[1]) for t_idx in range(timer_num): next_tick = timestamp + pyutils.crc32_mod( proxy_str, timer_seq) + t_idx * timer_seq rawparam = '%s#%d#%d#%d' % (host, port, timer_seq, int(next_tick / timer_seq)) task_map.setdefault(rawparam, next_tick) queue_raw_jobs('check_proxy_timed_set', task_map) return { 'file': filename, 'num': len(proxy_list), 'proxy_list': proxy_list }
def run(self, params): abort_current_job()