def run(self, params):
host = params.get('h', '').strip()
port = params.get('p', 80)
hkey = '%s:%d' % (host, port)
if _check_t(params):
abort_current_job()
connections.redis.sadd(CONF_DATA_ALL_KEY, hkey)
tmp = connections.redis.hget(CONF_DATA_RANK_KEY, hkey)
now_num = int(tmp) if tmp else 0
if not now_num:
connections.redis.hincrby(CONF_DATA_RANK_KEY, hkey, 1)
test = CONF_CHECK_PROXY_FUNC(host, port)
test and log.info('CHECK OK proxy:%s, num:%d' % (hkey, now_num))
if test:
if now_num <= 0:
now_num = 1 if connections.redis.sismember(
CONF_DATA_OK_KEY, hkey) else 10
connections.redis.hset(CONF_DATA_RANK_KEY, hkey, now_num)
elif 0 < now_num < 20:
connections.redis.hincrby(CONF_DATA_RANK_KEY, hkey, 1)
now_num += 1
else:
if now_num >= -10:
connections.redis.hincrby(CONF_DATA_RANK_KEY, hkey, -1)
now_num -= 1
if now_num > 0:
connections.redis.sadd(CONF_DATA_OK_KEY, hkey)
else:
connections.redis.srem(CONF_DATA_OK_KEY, hkey)
now_num <= -10 and connections.redis.srem(CONF_DATA_ALL_KEY, hkey)
return {'proxy': hkey, 'num': now_num, 'test': test}
def run(self, params):
log.info("Will abort this task")
connections.mongodb_jobs.tests_inserts.insert(params)
try:
raise InAbortException
except InAbortException:
abort_current_job()
raise Exception("Should not be reached")
def run(self, params):
filename = params.get('f', '').strip()
if _check_t(params):
abort_current_job()
timer_num = 3
timer_seq = CONF_CHECK_INTERVAL
gql = pyfile.load_str(filename).strip()
if not gql:
abort_current_job()
proxy_list, gret = run_gdom_page(
gql,
get_proxy=lambda: connections.redis.srandmember(CONF_DATA_OK_KEY))
proxy_list and log.info('FETCH OK filename:%s, num:%d' %
(filename, len(proxy_list)))
if gret.errors:
log.error('FETCH ERROR filename:%s, errors:%s' %
(filename, gret.errors))
if not proxy_list:
log.error('FETCH EMPTY filename:%s, gret:%r' % (filename, gret))
abort_current_job()
timestamp = int(time.time())
task_map = {}
for proxy_str in proxy_list:
host = proxy_str.split(':', 1)[0]
port = int(proxy_str.split(':', 1)[1])
for t_idx in range(timer_num):
next_tick = timestamp + pyutils.crc32_mod(
proxy_str, timer_seq) + t_idx * timer_seq
rawparam = '%s#%d#%d#%d' % (host, port, timer_seq,
int(next_tick / timer_seq))
task_map.setdefault(rawparam, next_tick)
queue_raw_jobs('check_proxy_timed_set', task_map)
return {
'file': filename,
'num': len(proxy_list),
'proxy_list': proxy_list
}
def run(self, params):
abort_current_job()
def run(self, params):
abort_current_job()