def __init__(self, jobs, proxy='proxy_r'): self.ac = AccountManager() self.pm = ProxyManager() self.pq = ProxyQueue() self.pq.load(proxy) self.pm.load(proxy) self.jobs = jobs self.threads = [] self.running = False
def __init__(self, jobs, thcnt=2, proxy='proxy_s', times=3): self.ac = AccountManager() self.pq = ProxyManager() self.pq.load(proxy) self.jobs = jobs self.threads = [] self.running = False self.thread_cnt = thcnt self.times = times self.job_queue = Queue.Queue() self.done_job = 0 self.job_lock = threading.RLock()
def run_status(self, out='status.csv'): rs = [] am = AccountManager() for name, short in provinces.items(): r = [ name, str(self.get_sch_size(short)), str(self.get_spec_size(short)), str(self.get_detail_size(short)), '0', '0', str(self.get_remain_time(name, am)) ] rs.append('-'.join(r)) with open(out, 'w') as f: for r in rs: f.write(r + '\n')
# 'detail.parser': True, 'thcnt': 4} # # ] # jobs = [ # { # 'accounts': [ # {'username': '******', 'proxy': '101.200.178.46:3128', 'password': '******', 'ua': ua[2]}, # ], # 'prefix': 'tj', 'sleep': 1.0, 'school': False, 'sch.parser': True, 'kldms': ['5', '1'], 'bkccs': ['2', '1'], # 'spec.parser': True, 'spec': True, 'detail': False, 'name': 'tianjin', 'detail.recover': True, # 'spec.recover': True, 'sch.recover': False, 'score': 750, # 'detail.parser': False, 'thcnt': 4} # # ] proxies = [] am = AccountManager() with open('proxy_r') as f: for l in f: proxies.append(l.strip()) proxies.append(None) idx = [22] count = len(idx) name = '青海' ac = { 'accounts': [] , 'name': name , 'prefix': provinces[name] } acsa = am.get(name, 8) i = 0 acs = []
# accounts = [ # {'username': '******', 'password': '******', 'prefix': 'sc', 'sleep': 1.0, 'sch.seeds': 'school.res.sc', # 'proxy': None, 'kldms': ['6', '2'], 'school': False, 'sch.parser': False, # 'spec.parser': False, 'spec': False, 'detail': True, 'name': u'四川', 'ua': ua[5], 'detail.recover': True, # 'detail.parser': False, 'detail.seeds': 'detail.seeds.sc.3', 'job_tag': '.3', 'thcnt': 4, # 'bkccs': ['1', '2']} # ] # for ac in jobs[0]['accounts']: # print BaseGkChsiFsxSpider.pay_time(ac, jobs[0]['prefix'], ac['card']) # print BaseGkChsiFsxSpider.get_remain_time(ac, jobs[0]['prefix']) cm = CardManager() ac = AccountManager() # pay_time('510', '广东', ac, cm, 7) # pay_time('510', '河南', ac, cm, 8) # pay_time('510', '山东', ac, cm, 6) # pay_time('310', '山东', ac, cm, 1) # pay_time('510', '安徽', ac, cm, 6) # pay_time('150', '安徽', ac, cm, 1) # pay_time('150', '江苏', ac, cm, 1) # pay_time('310', '江苏', ac, cm, 1) # pay_time('510', '江苏', ac, cm, 4) # pay_time('510', '湖南', ac, cm, 4) # pay_time('310', '湖南', ac, cm, 1) # pay_time('150', '湖南', ac, cm, 1) # pay_time('510', '河北', ac, cm, 5) # pay_time('510', '江西', ac, cm, 4) # pay_time('310', '江西', ac, cm, 1)
# 'detail.parser': True, 'thcnt': 4} # # ] # jobs = [ # { # 'accounts': [ # {'username': '******', 'proxy': '101.200.178.46:3128', 'password': '******', 'ua': ua[2]}, # ], # 'prefix': 'tj', 'sleep': 1.0, 'school': False, 'sch.parser': True, 'kldms': ['5', '1'], 'bkccs': ['2', '1'], # 'spec.parser': True, 'spec': True, 'detail': False, 'name': 'tianjin', 'detail.recover': True, # 'spec.recover': True, 'sch.recover': False, 'score': 750, # 'detail.parser': False, 'thcnt': 4} # # ] proxies = [] am = AccountManager() with open('proxy_r') as f: for l in f: proxies.append(l.strip()) idx = [1, 2, 3, 4, 5, 6, 7] count = len(idx) name = '广东' ac = {'accounts': [], 'name': name, 'prefix': provinces[name]} acs = am.get(name, count) i = 0 for a in acs: a.proxy = proxies[idx[i]] ac['accounts'].append(a.gen_run_param()) i += 1 run_fun(ac)
#!/usr/bin/env python # -*- coding:utf8 -*- from accounts import CardManager, AccountManager, pay_account if __name__ == '__main__': cm = CardManager() ac = AccountManager() name = '福建' cm.save() # for card in cm.getall(): # if pay_account(ac, card, name): # print card # break # else: # print 'card is used', card
args = parser.parse_args() load_proxies(args) set_args(args) set_account_db_args(args) install_thread_excepthook() sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) fail_on_forced_update(args) queue = PriorityQueue() dbqueue = Queue() account_manager = AccountManager("gymwatcher", False, args, [], [], Queue(), {}) account_manager.initialize(args.accountcsv, ()) seen_gyms = {} running = True def find_top_n(gyms, all_scanned_gyms, n): result = {} for gym in gyms: id_ = gym["gym_id"] if id_ not in all_scanned_gyms: result[id_] = gym all_scanned_gyms[id_] = gym if len(result) > n: break
load_proxies(args) if args.proxy and not args.proxy_skip_check: fully_ok, ptc_banned_proxies, niantic_banned_proxies = check_proxies( args, args.proxy) args.proxy = fully_ok args.ptc_banned_proxy = ptc_banned_proxies args.niantic_banned_proxy = niantic_banned_proxies def set_account_level_from_args(accounts): if args.level and accounts: for acc in accounts: acc["level"] = args.level monocle_accounts = AccountManager.load_accounts(args.accountcsv) if not args.login: set_account_level_from_args(monocle_accounts) duration = datetime.timedelta( hours=int(args.allocation_duration)) if args.allocation_duration else None AccountManager.insert_accounts(monocle_accounts, args.system_id, duration, args.force_system_id, args.skip_assigned) account_manager = AccountManager(args.system_id, False, args, [], [], Queue(), {}, replace_warned=False) account_manager.initialize(args.accountcsv, [])
# 'detail.parser': True, 'thcnt': 4} # # ] # jobs = [ # { # 'accounts': [ # {'username': '******', 'proxy': '101.200.178.46:3128', 'password': '******', 'ua': ua[2]}, # ], # 'prefix': 'tj', 'sleep': 1.0, 'school': False, 'sch.parser': True, 'kldms': ['5', '1'], 'bkccs': ['2', '1'], # 'spec.parser': True, 'spec': True, 'detail': False, 'name': 'tianjin', 'detail.recover': True, # 'spec.recover': True, 'sch.recover': False, 'score': 750, # 'detail.parser': False, 'thcnt': 4} # # ] proxies = [] am = AccountManager() with open('proxy_r') as f: for l in f: proxies.append(l.strip()) idx = [16] count = len(idx) acsa = am.get('天津', 8) acs = [] for a in acsa: acs.insert(0, a) name = '天津' for a in acs: a.proxy = '192.168.1.39:3428' runner = FullJobRunner( gen_jobs_params([a.gen_run_param()],
except OutOfAccounts: log.info("Worker done") pass except GaveUp: logging.error("Gave up worker {}".format(str(worker))) except: logging.exception("Outer worker catch block caught exception") load_proxies(args) set_args(args) set_account_db_args(args) account_manager = AccountManager(args.system_id, args.use_account_db, args, [], [], Queue(), {}, replace_warned=False) account_manager.loader = load_accounts_for_blindcheck account_manager.initialize(args.accountcsv, ()) log.info("{} accounts to check".format(str(account_manager.size()))) num_threads = args.threads_per_proxy if args.proxy and len(args.proxy) > 0: num_threads = max(1, len(args.proxy)) * args.threads_per_proxy split = [] for x in range(0, num_threads): split.append([])
fully_ok, ptc_banned_proxies, niantic_banned_proxies = check_proxies( args, args.proxy) args.proxy = fully_ok args.ptc_banned_proxy = ptc_banned_proxies args.niantic_banned_proxy = niantic_banned_proxies args.player_locale = { 'country': 'NO', 'language': 'no', 'timezone': 'Europe/Oslo' } set_account_db_args(args) account_manager = AccountManager(args.system_id, args.use_account_db, args, [], [], Queue(), {}, replace_warned=False) # account_manager.initialize(args.accountcsv, ()) def find_accounts(): temp_ban_time = datetime.datetime.now() - datetime.timedelta(days=35) warn_time = datetime.datetime.now() - datetime.timedelta(days=20) perm_ban_time = datetime.datetime.now() - datetime.timedelta(days=60) blind_time = datetime.datetime.now() - datetime.timedelta(days=30) pool = ThreadPoolExecutor( thread_count(args)) # for many urls, this should probably be
class JobManager(): def __init__(self, jobs, proxy='proxy_r'): self.ac = AccountManager() self.pm = ProxyManager() self.pq = ProxyQueue() self.pq.load(proxy) self.pm.load(proxy) self.jobs = jobs self.threads = [] self.running = False def init(self): random.seed(int(time.time())) for job in self.jobs: for ac in self.ac.get(job['name'], 2): ac.proxy = self.pm.get_good_proxy(7) ac.user_agent = ua[random.randint(0, len(ua)) % len(ua)] for job in self.jobs: for ac in self.ac.get(job['name'], job['count']): if ac.proxy is None: ac.proxy = self.pm.get_good_proxy(1) ac.user_agent = ua[random.randint(0, len(ua)) % len(ua)] def run(self): if self.running: return self.init() for tid in range(len(self.jobs)): t = threading.Thread(target=self.runner, args=(tid, )) self.threads.append(t) for t in self.threads: t.start() time.sleep(2) for t in self.threads: t.join() self.ac.save() def random_run(self): if self.running: return for tid in range(len(self.jobs)): t = threading.Thread(target=self.rand_runner, args=(tid, )) t.start() t.setDaemon(True) time.sleep(1) self.threads.append(t) time.sleep(2) for t in self.threads: t.join() self.ac.save() def rand_runner(self, tid): job = self.jobs[tid] ac = { 'accounts': [], 'name': job['name'], 'prefix': provinces[job['name']] } acs = self.ac.get(job['name'], job['count']) if len(acs) > 0: for a in acs: a.proxy = self.pq.get_good_proxy() ac['accounts'].append(a.gen_run_param()) level = 0 print '%s start ' % ac['name'] while level < 3: if ac['name'] != '海南': level = smart_full_job(ac['accounts'], ac['name'], ac['prefix'], level=level) else: level = smart_full_job(ac['accounts'], ac['name'], ac['prefix'], 900, level=level) for a in acs: self.pq.release(a.proxy) a.proxy = None def runner(self, tid): job = self.jobs[tid] ac = { 'accounts': [], 'name': job['name'], 'prefix': provinces[job['name']] } for a in self.ac.get(job['name'], job['count']): ac['accounts'].append(a.gen_run_param()) level = 0 print '%s start ' % ac['name'] while level < 3: if ac['name'] != '海南': level = smart_full_job(ac['accounts'], ac['name'], ac['prefix'], level=level) else: level = smart_full_job(ac['accounts'], ac['name'], ac['prefix'], 900, level=level)
class QueueJobManager(): def __init__(self, jobs, thcnt=2, proxy='proxy_s', times=3): self.ac = AccountManager() self.pq = ProxyManager() self.pq.load(proxy) self.jobs = jobs self.threads = [] self.running = False self.thread_cnt = thcnt self.times = times self.job_queue = Queue.Queue() self.done_job = 0 self.job_lock = threading.RLock() def distpatch(self): for job in self.jobs: ac = { 'accounts': [] , 'name': job['name'] , 'prefix': provinces[job['name']] , 'score': 750 , 'times': 0 } if job['name'] == '海南': ac['score'] = 900 elif job['name'] == '上海': ac['score'] = 600 elif job['name'] == '江苏': ac['score'] = 500 acs = self.ac.get(job['name'], job['count']) if len(acs) > 0: for a in acs: ac['accounts'].append(a.gen_run_param()) self.job_queue.put(ac) def run(self): if self.running: return self.running = True self.distpatch() for tid in range(self.thread_cnt): proxy = self.pq.get_good_proxy() if proxy is None: break t = threading.Thread(target=self.runner, args=(proxy,)) self.threads.append(t) for t in self.threads: t.start() time.sleep(2) for t in self.threads: t.join() self.ac.save() self.threads = [] def runner(self, proxy): while True: ac = self.job_queue.get() ac['proxy'] = proxy times = ac.get('times', 0) if times < self.times: print '%s start %d' % (ac['name'], ac['times']) recruit_jobs(ac) ac['times'] += 1 ac['proxy'] = None self.job_queue.put(ac) else: print '%s start check crawling' % ac['name'] level = 0 while level < 3: level = recheck(ac, level=level) with self.job_lock: self.done_job += 1 if self.done_job >= len(self.jobs): break
class JobManager(): def __init__(self, jobs, proxy='proxy_s'): self.ac = AccountManager() self.pm = ProxyManager() self.pq = ProxyQueue() self.pq.load(proxy) self.pm.load(proxy) self.jobs = jobs self.threads = [] self.running = False self.job_queue = Queue.Queue() def init(self): random.seed(int(time.time())) for job in self.jobs: job['proxy'] = self.pm.get_good_proxy(1) for ac in self.ac.get(job['name'], job['count']): ac.user_agent = ua[random.randint(0, len(ua)) % len(ua)] def run(self): if self.running: return self.init() for tid in range(len(self.jobs)): t = threading.Thread(target=self.runner, args=(tid,)) self.threads.append(t) for t in self.threads: t.start() time.sleep(2) for t in self.threads: t.join() self.ac.save() def random_run(self): if self.running: return for tid in range(len(self.jobs)): t = threading.Thread(target=self.rand_runner, args=(tid,)) t.start() time.sleep(1) self.threads.append(t) time.sleep(2) for t in self.threads: t.join() self.ac.save() def rand_runner(self, tid): job = self.jobs[tid] ac = { 'accounts': [] , 'name': job['name'] , 'prefix': provinces[job['name']] , 'proxy': None , 'score': 750 } if job['name'] == '海南': ac['score'] = 900 elif job['name'] == '上海': ac['score'] = 600 elif job['name'] == '江苏': ac['score'] = 500 acs = self.ac.get(job['name'], job['count']) if len(acs) > 0: for a in acs: ac['accounts'].append(a.gen_run_param()) proxy = self.pq.get_good_proxy() print '%s start ' % ac['name'] recruit_jobs(ac) self.pq.release(proxy) def runner(self, tid): job = self.jobs[tid] ac = { 'accounts': [] , 'name': job['name'] , 'prefix': provinces[job['name']] , 'score': 750 , 'proxy': job['proxy'] } if job['name'] == '海南': ac['score'] = 900 elif job['name'] == '上海': ac['score'] = 600 elif job['name'] == '江苏': ac['score'] = 500 for a in self.ac.get(job['name'], job['count']): ac['accounts'].append(a.gen_run_param()) print '%s start ' % ac['name'] recruit_jobs(ac)