def __init__(self, jobs, proxy='proxy_r'):
self.ac = AccountManager()
self.pm = ProxyManager()
self.pq = ProxyQueue()
self.pq.load(proxy)
self.pm.load(proxy)
self.jobs = jobs
self.threads = []
self.running = False
def __init__(self, jobs, thcnt=2, proxy='proxy_s', times=3):
self.ac = AccountManager()
self.pq = ProxyManager()
self.pq.load(proxy)
self.jobs = jobs
self.threads = []
self.running = False
self.thread_cnt = thcnt
self.times = times
self.job_queue = Queue.Queue()
self.done_job = 0
self.job_lock = threading.RLock()
def run_status(self, out='status.csv'):
rs = []
am = AccountManager()
for name, short in provinces.items():
r = [
name,
str(self.get_sch_size(short)),
str(self.get_spec_size(short)),
str(self.get_detail_size(short)), '0', '0',
str(self.get_remain_time(name, am))
]
rs.append('-'.join(r))
with open(out, 'w') as f:
for r in rs:
f.write(r + '\n')
# 'detail.parser': True, 'thcnt': 4}
#
# ]
# jobs = [
# {
# 'accounts': [
# {'username': '******', 'proxy': '101.200.178.46:3128', 'password': '******', 'ua': ua[2]},
# ],
# 'prefix': 'tj', 'sleep': 1.0, 'school': False, 'sch.parser': True, 'kldms': ['5', '1'], 'bkccs': ['2', '1'],
# 'spec.parser': True, 'spec': True, 'detail': False, 'name': 'tianjin', 'detail.recover': True,
# 'spec.recover': True, 'sch.recover': False, 'score': 750,
# 'detail.parser': False, 'thcnt': 4}
#
# ]
proxies = []
am = AccountManager()
with open('proxy_r') as f:
for l in f:
proxies.append(l.strip())
proxies.append(None)
idx = [22]
count = len(idx)
name = '青海'
ac = {
'accounts': []
, 'name': name
, 'prefix': provinces[name]
}
acsa = am.get(name, 8)
i = 0
acs = []
# accounts = [
# {'username': '******', 'password': '******', 'prefix': 'sc', 'sleep': 1.0, 'sch.seeds': 'school.res.sc',
# 'proxy': None, 'kldms': ['6', '2'], 'school': False, 'sch.parser': False,
# 'spec.parser': False, 'spec': False, 'detail': True, 'name': u'四川', 'ua': ua[5], 'detail.recover': True,
# 'detail.parser': False, 'detail.seeds': 'detail.seeds.sc.3', 'job_tag': '.3', 'thcnt': 4,
# 'bkccs': ['1', '2']}
# ]
# for ac in jobs[0]['accounts']:
# print BaseGkChsiFsxSpider.pay_time(ac, jobs[0]['prefix'], ac['card'])
# print BaseGkChsiFsxSpider.get_remain_time(ac, jobs[0]['prefix'])
cm = CardManager()
ac = AccountManager()
# pay_time('510', '广东', ac, cm, 7)
# pay_time('510', '河南', ac, cm, 8)
# pay_time('510', '山东', ac, cm, 6)
# pay_time('310', '山东', ac, cm, 1)
# pay_time('510', '安徽', ac, cm, 6)
# pay_time('150', '安徽', ac, cm, 1)
# pay_time('150', '江苏', ac, cm, 1)
# pay_time('310', '江苏', ac, cm, 1)
# pay_time('510', '江苏', ac, cm, 4)
# pay_time('510', '湖南', ac, cm, 4)
# pay_time('310', '湖南', ac, cm, 1)
# pay_time('150', '湖南', ac, cm, 1)
# pay_time('510', '河北', ac, cm, 5)
# pay_time('510', '江西', ac, cm, 4)
# pay_time('310', '江西', ac, cm, 1)
# 'detail.parser': True, 'thcnt': 4}
#
# ]
# jobs = [
# {
# 'accounts': [
# {'username': '******', 'proxy': '101.200.178.46:3128', 'password': '******', 'ua': ua[2]},
# ],
# 'prefix': 'tj', 'sleep': 1.0, 'school': False, 'sch.parser': True, 'kldms': ['5', '1'], 'bkccs': ['2', '1'],
# 'spec.parser': True, 'spec': True, 'detail': False, 'name': 'tianjin', 'detail.recover': True,
# 'spec.recover': True, 'sch.recover': False, 'score': 750,
# 'detail.parser': False, 'thcnt': 4}
#
# ]
proxies = []
am = AccountManager()
with open('proxy_r') as f:
for l in f:
proxies.append(l.strip())
idx = [1, 2, 3, 4, 5, 6, 7]
count = len(idx)
name = '广东'
ac = {'accounts': [], 'name': name, 'prefix': provinces[name]}
acs = am.get(name, count)
i = 0
for a in acs:
a.proxy = proxies[idx[i]]
ac['accounts'].append(a.gen_run_param())
i += 1
run_fun(ac)
#!/usr/bin/env python
# -*- coding:utf8 -*-
from accounts import CardManager, AccountManager, pay_account
if __name__ == '__main__':
cm = CardManager()
ac = AccountManager()
name = '福建'
cm.save()
# for card in cm.getall():
# if pay_account(ac, card, name):
# print card
# break
# else:
# print 'card is used', card
args = parser.parse_args()
load_proxies(args)
set_args(args)
set_account_db_args(args)
install_thread_excepthook()
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
fail_on_forced_update(args)
queue = PriorityQueue()
dbqueue = Queue()
account_manager = AccountManager("gymwatcher", False, args, [], [], Queue(), {})
account_manager.initialize(args.accountcsv, ())
seen_gyms = {}
running = True
def find_top_n(gyms, all_scanned_gyms, n):
result = {}
for gym in gyms:
id_ = gym["gym_id"]
if id_ not in all_scanned_gyms:
result[id_] = gym
all_scanned_gyms[id_] = gym
if len(result) > n:
break
load_proxies(args)
if args.proxy and not args.proxy_skip_check:
fully_ok, ptc_banned_proxies, niantic_banned_proxies = check_proxies(
args, args.proxy)
args.proxy = fully_ok
args.ptc_banned_proxy = ptc_banned_proxies
args.niantic_banned_proxy = niantic_banned_proxies
def set_account_level_from_args(accounts):
if args.level and accounts:
for acc in accounts:
acc["level"] = args.level
monocle_accounts = AccountManager.load_accounts(args.accountcsv)
if not args.login:
set_account_level_from_args(monocle_accounts)
duration = datetime.timedelta(
hours=int(args.allocation_duration)) if args.allocation_duration else None
AccountManager.insert_accounts(monocle_accounts, args.system_id, duration,
args.force_system_id, args.skip_assigned)
account_manager = AccountManager(args.system_id,
False,
args, [], [],
Queue(), {},
replace_warned=False)
account_manager.initialize(args.accountcsv, [])
# 'detail.parser': True, 'thcnt': 4}
#
# ]
# jobs = [
# {
# 'accounts': [
# {'username': '******', 'proxy': '101.200.178.46:3128', 'password': '******', 'ua': ua[2]},
# ],
# 'prefix': 'tj', 'sleep': 1.0, 'school': False, 'sch.parser': True, 'kldms': ['5', '1'], 'bkccs': ['2', '1'],
# 'spec.parser': True, 'spec': True, 'detail': False, 'name': 'tianjin', 'detail.recover': True,
# 'spec.recover': True, 'sch.recover': False, 'score': 750,
# 'detail.parser': False, 'thcnt': 4}
#
# ]
proxies = []
am = AccountManager()
with open('proxy_r') as f:
for l in f:
proxies.append(l.strip())
idx = [16]
count = len(idx)
acsa = am.get('天津', 8)
acs = []
for a in acsa:
acs.insert(0, a)
name = '天津'
for a in acs:
a.proxy = '192.168.1.39:3428'
runner = FullJobRunner(
gen_jobs_params([a.gen_run_param()],
except OutOfAccounts:
log.info("Worker done")
pass
except GaveUp:
logging.error("Gave up worker {}".format(str(worker)))
except:
logging.exception("Outer worker catch block caught exception")
load_proxies(args)
set_args(args)
set_account_db_args(args)
account_manager = AccountManager(args.system_id,
args.use_account_db,
args, [], [],
Queue(), {},
replace_warned=False)
account_manager.loader = load_accounts_for_blindcheck
account_manager.initialize(args.accountcsv, ())
log.info("{} accounts to check".format(str(account_manager.size())))
num_threads = args.threads_per_proxy
if args.proxy and len(args.proxy) > 0:
num_threads = max(1, len(args.proxy)) * args.threads_per_proxy
split = []
for x in range(0, num_threads):
split.append([])
fully_ok, ptc_banned_proxies, niantic_banned_proxies = check_proxies(
args, args.proxy)
args.proxy = fully_ok
args.ptc_banned_proxy = ptc_banned_proxies
args.niantic_banned_proxy = niantic_banned_proxies
args.player_locale = {
'country': 'NO',
'language': 'no',
'timezone': 'Europe/Oslo'
}
set_account_db_args(args)
account_manager = AccountManager(args.system_id,
args.use_account_db,
args, [], [],
Queue(), {},
replace_warned=False)
# account_manager.initialize(args.accountcsv, ())
def find_accounts():
temp_ban_time = datetime.datetime.now() - datetime.timedelta(days=35)
warn_time = datetime.datetime.now() - datetime.timedelta(days=20)
perm_ban_time = datetime.datetime.now() - datetime.timedelta(days=60)
blind_time = datetime.datetime.now() - datetime.timedelta(days=30)
pool = ThreadPoolExecutor(
thread_count(args)) # for many urls, this should probably be
class JobManager():
def __init__(self, jobs, proxy='proxy_r'):
self.ac = AccountManager()
self.pm = ProxyManager()
self.pq = ProxyQueue()
self.pq.load(proxy)
self.pm.load(proxy)
self.jobs = jobs
self.threads = []
self.running = False
def init(self):
random.seed(int(time.time()))
for job in self.jobs:
for ac in self.ac.get(job['name'], 2):
ac.proxy = self.pm.get_good_proxy(7)
ac.user_agent = ua[random.randint(0, len(ua)) % len(ua)]
for job in self.jobs:
for ac in self.ac.get(job['name'], job['count']):
if ac.proxy is None:
ac.proxy = self.pm.get_good_proxy(1)
ac.user_agent = ua[random.randint(0, len(ua)) % len(ua)]
def run(self):
if self.running:
return
self.init()
for tid in range(len(self.jobs)):
t = threading.Thread(target=self.runner, args=(tid, ))
self.threads.append(t)
for t in self.threads:
t.start()
time.sleep(2)
for t in self.threads:
t.join()
self.ac.save()
def random_run(self):
if self.running:
return
for tid in range(len(self.jobs)):
t = threading.Thread(target=self.rand_runner, args=(tid, ))
t.start()
t.setDaemon(True)
time.sleep(1)
self.threads.append(t)
time.sleep(2)
for t in self.threads:
t.join()
self.ac.save()
def rand_runner(self, tid):
job = self.jobs[tid]
ac = {
'accounts': [],
'name': job['name'],
'prefix': provinces[job['name']]
}
acs = self.ac.get(job['name'], job['count'])
if len(acs) > 0:
for a in acs:
a.proxy = self.pq.get_good_proxy()
ac['accounts'].append(a.gen_run_param())
level = 0
print '%s start ' % ac['name']
while level < 3:
if ac['name'] != '海南':
level = smart_full_job(ac['accounts'],
ac['name'],
ac['prefix'],
level=level)
else:
level = smart_full_job(ac['accounts'],
ac['name'],
ac['prefix'],
900,
level=level)
for a in acs:
self.pq.release(a.proxy)
a.proxy = None
def runner(self, tid):
job = self.jobs[tid]
ac = {
'accounts': [],
'name': job['name'],
'prefix': provinces[job['name']]
}
for a in self.ac.get(job['name'], job['count']):
ac['accounts'].append(a.gen_run_param())
level = 0
print '%s start ' % ac['name']
while level < 3:
if ac['name'] != '海南':
level = smart_full_job(ac['accounts'],
ac['name'],
ac['prefix'],
level=level)
else:
level = smart_full_job(ac['accounts'],
ac['name'],
ac['prefix'],
900,
level=level)
class QueueJobManager():
def __init__(self, jobs, thcnt=2, proxy='proxy_s', times=3):
self.ac = AccountManager()
self.pq = ProxyManager()
self.pq.load(proxy)
self.jobs = jobs
self.threads = []
self.running = False
self.thread_cnt = thcnt
self.times = times
self.job_queue = Queue.Queue()
self.done_job = 0
self.job_lock = threading.RLock()
def distpatch(self):
for job in self.jobs:
ac = {
'accounts': []
, 'name': job['name']
, 'prefix': provinces[job['name']]
, 'score': 750
, 'times': 0
}
if job['name'] == '海南':
ac['score'] = 900
elif job['name'] == '上海':
ac['score'] = 600
elif job['name'] == '江苏':
ac['score'] = 500
acs = self.ac.get(job['name'], job['count'])
if len(acs) > 0:
for a in acs:
ac['accounts'].append(a.gen_run_param())
self.job_queue.put(ac)
def run(self):
if self.running:
return
self.running = True
self.distpatch()
for tid in range(self.thread_cnt):
proxy = self.pq.get_good_proxy()
if proxy is None:
break
t = threading.Thread(target=self.runner, args=(proxy,))
self.threads.append(t)
for t in self.threads:
t.start()
time.sleep(2)
for t in self.threads:
t.join()
self.ac.save()
self.threads = []
def runner(self, proxy):
while True:
ac = self.job_queue.get()
ac['proxy'] = proxy
times = ac.get('times', 0)
if times < self.times:
print '%s start %d' % (ac['name'], ac['times'])
recruit_jobs(ac)
ac['times'] += 1
ac['proxy'] = None
self.job_queue.put(ac)
else:
print '%s start check crawling' % ac['name']
level = 0
while level < 3:
level = recheck(ac, level=level)
with self.job_lock:
self.done_job += 1
if self.done_job >= len(self.jobs):
break
class JobManager():
def __init__(self, jobs, proxy='proxy_s'):
self.ac = AccountManager()
self.pm = ProxyManager()
self.pq = ProxyQueue()
self.pq.load(proxy)
self.pm.load(proxy)
self.jobs = jobs
self.threads = []
self.running = False
self.job_queue = Queue.Queue()
def init(self):
random.seed(int(time.time()))
for job in self.jobs:
job['proxy'] = self.pm.get_good_proxy(1)
for ac in self.ac.get(job['name'], job['count']):
ac.user_agent = ua[random.randint(0, len(ua)) % len(ua)]
def run(self):
if self.running:
return
self.init()
for tid in range(len(self.jobs)):
t = threading.Thread(target=self.runner, args=(tid,))
self.threads.append(t)
for t in self.threads:
t.start()
time.sleep(2)
for t in self.threads:
t.join()
self.ac.save()
def random_run(self):
if self.running:
return
for tid in range(len(self.jobs)):
t = threading.Thread(target=self.rand_runner, args=(tid,))
t.start()
time.sleep(1)
self.threads.append(t)
time.sleep(2)
for t in self.threads:
t.join()
self.ac.save()
def rand_runner(self, tid):
job = self.jobs[tid]
ac = {
'accounts': []
, 'name': job['name']
, 'prefix': provinces[job['name']]
, 'proxy': None
, 'score': 750
}
if job['name'] == '海南':
ac['score'] = 900
elif job['name'] == '上海':
ac['score'] = 600
elif job['name'] == '江苏':
ac['score'] = 500
acs = self.ac.get(job['name'], job['count'])
if len(acs) > 0:
for a in acs:
ac['accounts'].append(a.gen_run_param())
proxy = self.pq.get_good_proxy()
print '%s start ' % ac['name']
recruit_jobs(ac)
self.pq.release(proxy)
def runner(self, tid):
job = self.jobs[tid]
ac = {
'accounts': []
, 'name': job['name']
, 'prefix': provinces[job['name']]
, 'score': 750
, 'proxy': job['proxy']
}
if job['name'] == '海南':
ac['score'] = 900
elif job['name'] == '上海':
ac['score'] = 600
elif job['name'] == '江苏':
ac['score'] = 500
for a in self.ac.get(job['name'], job['count']):
ac['accounts'].append(a.gen_run_param())
print '%s start ' % ac['name']
recruit_jobs(ac)
