def add_email_to_db(self, filename, state='0'): ct = 0 ft = 0 batch = None state = str(state) for line in self.readfile(filename): item = line.split('\t') if len(item) < 4: ft += 1 continue cid = item[0].strip() if len(cid) != 16: ft += 1 continue email = item[3].lower() if not email or email.find('@') == -1: ft += 1 continue ct += 1 #value = self.get(cid) if True:#value is None or (value.find('@') == -1 and ):: self.db.Put(cid, email) ct += 1 if ct % SYNC == 0: #self.db.Write(batch, sync = True) log.trace('filename %s sync new %d skip %d', filename, ct, ft) else: ft += 1 if ft % SYNC == 0: log.trace('filename %s skip new %d skip %d', filename, ct, ft) return ct
def get_new_account(self, handler): try: ret = self._new_account_iter.next() log.trace('get new account %s', ret) return ret except: return None
def get_accounts(self, begin=None, state=None, create_time=None, limit = 10000, **kwargs): log.trace('get_accounts %s ', begin) params = [] sql = "select * from accounts where " if begin is not None: sql += 'id > %d' % begin if state is not None: if state < 1: sql += ' and state = %d' % state else: sql += ' and state >= %d' % state else: sql += ' and state != -1' if create_time is not None: sql += ' and create_time <= %s ' params.append(create_time) for k, v in kwargs.items(): if v is None: continue sql += ' and %s = %%s ' % (k) params.append(v) sql += ' order by id limit %d' % limit log.debug("sql %s params %s", sql, params) return self.db.query(sql, *params)
def export(self, state='0', target_dir='.'): if not os.path.exists(target_dir): os.makedirs(target_dir) total = 0 ct = 0 fct = 0 f = None for k, v in self.db.RangeIter(): total += 1 if v.find(state) == -1: continue if f is None: filename = '%s_%d.txt' % (self.TARGET_BASE, fct) filename = os.path.join(target_dir, filename) f = file(filename, 'w') log.trace('open new file %s', f.name) ct += 1 if state == '@': f.write('%s\n' % v) else: f.write('%s\n' % k) if ct % 10000 == 0: log.trace('export %s num %d total %d', f.name, ct, total) if ct % self.EXPORT_LIMIT == 0: f = None fct += 1 return ct
def work(self, name, id=0): try: account, psw = self.accounts.get(), '846266' except textfile.NotDataException: mysignal.ALIVE = False raise WorkerFinishException return self.msn_ct += 1 msn_ct = self.msn_ct log.error('%d %s begin connect' % (msn_ct, account)) app = OutLook(account, psw) app.login() for i in range(3): emails = app.get_contacts() if emails is not None: self.total += len(emails) log.trace('%s %s found emails %s total %s', msn_ct, account, len(emails), self.total) log.trace('%s %s found emails %s', msn_ct, account, ','.join(emails)) f = self.files[id] f.write('\n'.join(emails)) f.write('\n') f.flush() break
def init_tos_data(self): os.chdir(self.data_dir) tos_files = glob.glob('tos_*.txt') if not tos_files: log.trace('not find tos files, need to un tgz') for tgzfile in glob.glob('tos_*.txt.tgz'): os.popen('tar xzvf %s' % tgzfile).read() log.info("init_tos_data untgz tos %s", tgzfile) tos_files = glob.glob('tos_*.txt') if not tos_files: log.error("not found any tos file") raise NotDataException f = random.choice(tos_files) cmd = "mv %s %s.txt" % ( f, self.cur_tos) os.popen(cmd).read() log.trace(cmd) try: os.unlink('%s.info' % (self.cur_tos)) except: pass os.chdir(self.cur_dir) self.tos = AccountNotMemery(self.cur_tos)
def work(self, name, id): t = datetime.datetime.now() newlogfile = "%s/%s.log" % (self.logpath, t.strftime("%m%d%H")) #cmd = '%s server.js %s >> %s ' % ( self.pjs, self.id, newlogfile) cmd = '%s %s %s ' % ( self.pjs, self.jsname, id) log.trace('run cmd `%s`', cmd) res = os.system(cmd)
def get_to(self): while mysignal.ALIVE: try: return self.tos.get() except NotDataException: mysignal.ALIVE = False log.trace("not any tos") time.sleep(2) continue
def del_file(self, filename): ct = 0 for line in self.readfile(filename): try: self.db.Delete(line) except: continue ct += 1 log.trace('filename %s del %d', filename, ct) return ct
def shortcut(url, *args, **kwargs): url = '%s?%s' % (url, time.time()) for i in range(3): s = _shortcut(url, *args, **kwargs) if s.startswith('http://goo.gl/'): log.trace("url %s get short %s", url, s) return s else: log.exception("google shortcut %s fail", url) time.sleep(3)
def add_account(self, handler, names=[], state=0, kwargs={}): if not isinstance(names, list): names = [names] for name in names: log.trace('add account %s state %s', name, state) self.db[name] = state for k, v in kwargs.iteritems(): log.trace('add account %s state %s', k, v) self.db[k] = v return True
def parse_names(self): self.get_siteconfig() names = self.options.names if not names or names == ['all']: self.names = self.site_config.all else: not_valid = [x for x in names if x not in self.site_config.all] if not_valid: log.trace('%s not valid site') sys.exit(-1) self.names = names self.names = set(self.names)
def get_iter(self, action=None, state=None, *args, **kwargs): last_id = self.get_action_num(action) log.trace('get_iter for action %s from %s', action, last_id) while True: had_item = False for row in self.get_accounts(last_id, state, *args, **kwargs): yield row last_id = row.id had_item = True if not had_item: self.update_action_num(action, 0); break
def load(): open_log("load account") open_debug() parser = OptionParser() parser.add_option("-i", "--conf", dest="conf", action="store", type="string") parser.add_option("-n", "--num", dest="num", action="store", type="int") parser.add_option("-t", "--target", dest="target", action="store", type="string") options, args = parser.parse_args(sys.argv[1:]) if not options.conf or not options.target: parser.print_help() sys.exit(-1) conf = DictIni(options.conf) num = options.num or 100000 if os.path.exists(options.target): log.error("%s had exitst", options.target) return client = MagicClient(conf.account_server[0], int(conf.account_server[1])) total = 0 f = file(options.target, 'w') log.trace("begin load data") while total < num: data = client.get_add_friends() if data: total += len(data) f.write('\n'.join([x.strip() for x in data])) log.trace("load account success %s", total) elif not data and not total: log.trace("load account %s fail") break f.close() log.trace("load accounts %s success %s", len(data), total)
def load_thread(self): log.debug("begin %s", self.func_name) if len(self.data) >= self.limit: log.debug("skip %s", self.func_name) time.sleep(3) return try: data = self.load_accounts() if data: log.trace("%s load %s", self.func_name, len(data)) self.data.extend(data) except: log.exception() pass
def load_accounts(self): while True: try: self.data = self.client.get_send_accounts() if not self.data: log.trace('load send_accounts not data') time.sleep(3) continue break except: log.exception("get_send_accounts fail") time.sleep(3) continue shuffle(self.data) #file('data/send_accounts%s.txt' % (datetime.now().strftime('%Y%M%d%H%m%S')), 'w').write('\n'.join(self.data)) if not self.data: raise Exception('Not Data Left')
def work_js(self, name, id=0): with self.lock: try: email = self.next.next() except StopIteration: raise WorkerFinishException cmd = '%s active.js %s' % (self.pjs, email) res = os.system(cmd) res = res / 256 log.trace('%s active %s %s', name, email, res) res = int(res) try: if res == 0: self.client.add_account(email, 1) elif res == 2: self.client.add_account(email, -1) except: time.sleep(2)
def load_accounts(self): while True: try: data = getattr(self.client, self.func_name)(self.limit) if not data: log.trace("load %s not data", self.func_name) time.sleep(3) continue break except: log.exception("%s fail", self.func_name) time.sleep(3) continue # file('data/%s%s.txt' % (self.func_name, datetime.now().strftime('%Y%M%d%H%m%S')), 'w').write('\n'.join(data)) random.shuffle(data) if not data: raise Exception("Not Data Left") return data
def _work(self, name, id): line = self.accounts.get_rnd() if line.find("----") != -1: account, psw = line.split("----", 1) else: account = line.split("\t")[0] psw = "846266" account = account.strip() psw = psw.strip() msn = MSN(self.wait_chl) try: ret = msn.connect(account, psw) except TimeoutException: log.trace("%s login timeout", account) return if ret: # log.trace("%s login success", account) self.account_success += 1 for i in range(self.conf.add_num): to_email = self.get_to() ret = msn.add_contact(to_email, 1, self.hello) if ret == 0: members = msn.get_allow_email() num = members and len(members) or 0 if not self.name and num: self.accounts.update_contact(account, num) self.total += 1 log.trace("%s add %s success friends %s", account, to_email, num) # else: # log.trace('%s add %s fail ret %s friends %s', account, to_email, ret, num) # break else: # log.trace('%s %s login fail', account, psw) self.accounts.set_fail(line) self.account_fail += 1
def add_file_to_db(self, filename, state='0'): ct = 0 ft = 0 batch = None state = str(state) for line in self.readfile(filename): if batch is None: batch = leveldb.WriteBatch() if True or self.get(line, None) is None: batch.Put(line, state) ct += 1 if ct % SYNC == 0: self.db.Write(batch, sync = True) batch = None log.trace('filename %s sync new %d skip %d', filename, ct, ft) else: ft += 1 if ft % SYNC == 0: log.trace('filename %s skip new %d skip %d', filename, ct, ft) return ct
def work_py(self, name, id=0): with self.lock: try: email = self.next.next() except StopIteration: raise WorkerFinishException app = OutLook(email, '846266') try: res = app.login() if res: app.into_profile() #app = OutLook(email, '846266') #app.login() #if app.proof(): log.trace('%s active success', email) self.client.add_account(email, 1) except ProofException: log.trace('%s active success', email) self.client.add_account(email, 1) except: time.sleep(2)
def run(self): name = osp.basename(self.dbpath) open_log(name, logging.INFO) open_debug() parser = OptionParser() parser.add_option("-a", "--action", dest="action", help="what action[add|del|export|addemail]") parser.add_option("-f", "--file", dest="filename", help="write report to add", metavar="FILE") parser.add_option("-s", "--state", dest="state", help="what state want to set", type="string") parser.add_option('-t', "--target_dir", dest='target_dir', help="which target dir to export") parser.add_option('-c', "--count", dest='count', action="store_true") options, args = parser.parse_args(sys.argv[1:]) if options.count: return self.count(options.state or '0') if options.action == 'export': print 'begin export' ct = self.export(options.state or '0', options.target_dir) return if not options.filename: parser.print_help() return if osp.isdir(options.filename): filenames = [osp.join(options.filename, x) for x in os.listdir(options.filename)] else: filenames = [options.filename] if options.action == 'add': total = 0 for filename in filenames: if filename.find('.svn') != -1: continue ct = self.add_file_to_db(filename, options.state or 0) total += ct log.trace('action %s fielname %s result new %d total %d', options.action, filename, ct, total) if options.action == 'addemail': total = 0 for filename in filenames: ct = self.add_email_to_db(filename, options.state or 0) total += ct log.trace('action %s fielname %s result new %d total %d', options.action, filename, ct, total) if options.action == 'del': total = 0 for filename in filenames: ct = self.del_file(filename) total += ct log.trace('action %s fielname %s result new %d total %d', options.action, filename, ct, total)