def add_email_to_db(self, filename, state='0'):
ct = 0
ft = 0
batch = None
state = str(state)
for line in self.readfile(filename):
item = line.split('\t')
if len(item) < 4:
ft += 1
continue
cid = item[0].strip()
if len(cid) != 16:
ft += 1
continue
email = item[3].lower()
if not email or email.find('@') == -1:
ft += 1
continue
ct += 1
#value = self.get(cid)
if True:#value is None or (value.find('@') == -1 and )::
self.db.Put(cid, email)
ct += 1
if ct % SYNC == 0:
#self.db.Write(batch, sync = True)
log.trace('filename %s sync new %d skip %d', filename, ct, ft)
else:
ft += 1
if ft % SYNC == 0:
log.trace('filename %s skip new %d skip %d', filename, ct, ft)
return ct
def get_new_account(self, handler):
try:
ret = self._new_account_iter.next()
log.trace('get new account %s', ret)
return ret
except:
return None
def get_accounts(self, begin=None, state=None, create_time=None, limit = 10000, **kwargs):
log.trace('get_accounts %s ', begin)
params = []
sql = "select * from accounts where "
if begin is not None:
sql += 'id > %d' % begin
if state is not None:
if state < 1:
sql += ' and state = %d' % state
else:
sql += ' and state >= %d' % state
else:
sql += ' and state != -1'
if create_time is not None:
sql += ' and create_time <= %s '
params.append(create_time)
for k, v in kwargs.items():
if v is None:
continue
sql += ' and %s = %%s ' % (k)
params.append(v)
sql += ' order by id limit %d' % limit
log.debug("sql %s params %s", sql, params)
return self.db.query(sql, *params)
def export(self, state='0', target_dir='.'):
if not os.path.exists(target_dir):
os.makedirs(target_dir)
total = 0
ct = 0
fct = 0
f = None
for k, v in self.db.RangeIter():
total += 1
if v.find(state) == -1:
continue
if f is None:
filename = '%s_%d.txt' % (self.TARGET_BASE, fct)
filename = os.path.join(target_dir, filename)
f = file(filename, 'w')
log.trace('open new file %s', f.name)
ct += 1
if state == '@':
f.write('%s\n' % v)
else:
f.write('%s\n' % k)
if ct % 10000 == 0:
log.trace('export %s num %d total %d', f.name, ct, total)
if ct % self.EXPORT_LIMIT == 0:
f = None
fct += 1
return ct
def work(self, name, id=0):
try:
account, psw = self.accounts.get(), '846266'
except textfile.NotDataException:
mysignal.ALIVE = False
raise WorkerFinishException
return
self.msn_ct += 1
msn_ct = self.msn_ct
log.error('%d %s begin connect' % (msn_ct, account))
app = OutLook(account, psw)
app.login()
for i in range(3):
emails = app.get_contacts()
if emails is not None:
self.total += len(emails)
log.trace('%s %s found emails %s total %s', msn_ct, account, len(emails), self.total)
log.trace('%s %s found emails %s', msn_ct, account, ','.join(emails))
f = self.files[id]
f.write('\n'.join(emails))
f.write('\n')
f.flush()
break
def init_tos_data(self):
os.chdir(self.data_dir)
tos_files = glob.glob('tos_*.txt')
if not tos_files:
log.trace('not find tos files, need to un tgz')
for tgzfile in glob.glob('tos_*.txt.tgz'):
os.popen('tar xzvf %s' % tgzfile).read()
log.info("init_tos_data untgz tos %s", tgzfile)
tos_files = glob.glob('tos_*.txt')
if not tos_files:
log.error("not found any tos file")
raise NotDataException
f = random.choice(tos_files)
cmd = "mv %s %s.txt" % ( f, self.cur_tos)
os.popen(cmd).read()
log.trace(cmd)
try:
os.unlink('%s.info' % (self.cur_tos))
except:
pass
os.chdir(self.cur_dir)
self.tos = AccountNotMemery(self.cur_tos)
def work(self, name, id):
t = datetime.datetime.now()
newlogfile = "%s/%s.log" % (self.logpath, t.strftime("%m%d%H"))
#cmd = '%s server.js %s >> %s ' % ( self.pjs, self.id, newlogfile)
cmd = '%s %s %s ' % ( self.pjs, self.jsname, id)
log.trace('run cmd `%s`', cmd)
res = os.system(cmd)
def get_to(self):
while mysignal.ALIVE:
try:
return self.tos.get()
except NotDataException:
mysignal.ALIVE = False
log.trace("not any tos")
time.sleep(2)
continue
def del_file(self, filename):
ct = 0
for line in self.readfile(filename):
try:
self.db.Delete(line)
except:
continue
ct += 1
log.trace('filename %s del %d', filename, ct)
return ct
def shortcut(url, *args, **kwargs):
url = '%s?%s' % (url, time.time())
for i in range(3):
s = _shortcut(url, *args, **kwargs)
if s.startswith('http://goo.gl/'):
log.trace("url %s get short %s", url, s)
return s
else:
log.exception("google shortcut %s fail", url)
time.sleep(3)
def add_account(self, handler, names=[], state=0, kwargs={}):
if not isinstance(names, list):
names = [names]
for name in names:
log.trace('add account %s state %s', name, state)
self.db[name] = state
for k, v in kwargs.iteritems():
log.trace('add account %s state %s', k, v)
self.db[k] = v
return True
def parse_names(self):
self.get_siteconfig()
names = self.options.names
if not names or names == ['all']:
self.names = self.site_config.all
else:
not_valid = [x for x in names if x not in self.site_config.all]
if not_valid:
log.trace('%s not valid site')
sys.exit(-1)
self.names = names
self.names = set(self.names)
def get_iter(self, action=None, state=None, *args, **kwargs):
last_id = self.get_action_num(action)
log.trace('get_iter for action %s from %s', action, last_id)
while True:
had_item = False
for row in self.get_accounts(last_id, state, *args, **kwargs):
yield row
last_id = row.id
had_item = True
if not had_item:
self.update_action_num(action, 0);
break
def load():
open_log("load account")
open_debug()
parser = OptionParser()
parser.add_option("-i", "--conf", dest="conf", action="store", type="string")
parser.add_option("-n", "--num", dest="num", action="store", type="int")
parser.add_option("-t", "--target", dest="target", action="store", type="string")
options, args = parser.parse_args(sys.argv[1:])
if not options.conf or not options.target:
parser.print_help()
sys.exit(-1)
conf = DictIni(options.conf)
num = options.num or 100000
if os.path.exists(options.target):
log.error("%s had exitst", options.target)
return
client = MagicClient(conf.account_server[0], int(conf.account_server[1]))
total = 0
f = file(options.target, 'w')
log.trace("begin load data")
while total < num:
data = client.get_add_friends()
if data:
total += len(data)
f.write('\n'.join([x.strip() for x in data]))
log.trace("load account success %s", total)
elif not data and not total:
log.trace("load account %s fail")
break
f.close()
log.trace("load accounts %s success %s", len(data), total)
def load_thread(self):
log.debug("begin %s", self.func_name)
if len(self.data) >= self.limit:
log.debug("skip %s", self.func_name)
time.sleep(3)
return
try:
data = self.load_accounts()
if data:
log.trace("%s load %s", self.func_name, len(data))
self.data.extend(data)
except:
log.exception()
pass
def load_accounts(self):
while True:
try:
self.data = self.client.get_send_accounts()
if not self.data:
log.trace('load send_accounts not data')
time.sleep(3)
continue
break
except:
log.exception("get_send_accounts fail")
time.sleep(3)
continue
shuffle(self.data)
#file('data/send_accounts%s.txt' % (datetime.now().strftime('%Y%M%d%H%m%S')), 'w').write('\n'.join(self.data))
if not self.data:
raise Exception('Not Data Left')
def work_js(self, name, id=0):
with self.lock:
try:
email = self.next.next()
except StopIteration:
raise WorkerFinishException
cmd = '%s active.js %s' % (self.pjs, email)
res = os.system(cmd)
res = res / 256
log.trace('%s active %s %s', name, email, res)
res = int(res)
try:
if res == 0:
self.client.add_account(email, 1)
elif res == 2:
self.client.add_account(email, -1)
except:
time.sleep(2)
def load_accounts(self):
while True:
try:
data = getattr(self.client, self.func_name)(self.limit)
if not data:
log.trace("load %s not data", self.func_name)
time.sleep(3)
continue
break
except:
log.exception("%s fail", self.func_name)
time.sleep(3)
continue
# file('data/%s%s.txt' % (self.func_name, datetime.now().strftime('%Y%M%d%H%m%S')), 'w').write('\n'.join(data))
random.shuffle(data)
if not data:
raise Exception("Not Data Left")
return data
def _work(self, name, id):
line = self.accounts.get_rnd()
if line.find("----") != -1:
account, psw = line.split("----", 1)
else:
account = line.split("\t")[0]
psw = "846266"
account = account.strip()
psw = psw.strip()
msn = MSN(self.wait_chl)
try:
ret = msn.connect(account, psw)
except TimeoutException:
log.trace("%s login timeout", account)
return
if ret:
# log.trace("%s login success", account)
self.account_success += 1
for i in range(self.conf.add_num):
to_email = self.get_to()
ret = msn.add_contact(to_email, 1, self.hello)
if ret == 0:
members = msn.get_allow_email()
num = members and len(members) or 0
if not self.name and num:
self.accounts.update_contact(account, num)
self.total += 1
log.trace("%s add %s success friends %s", account, to_email, num)
# else:
# log.trace('%s add %s fail ret %s friends %s', account, to_email, ret, num)
# break
else:
# log.trace('%s %s login fail', account, psw)
self.accounts.set_fail(line)
self.account_fail += 1
def add_file_to_db(self, filename, state='0'):
ct = 0
ft = 0
batch = None
state = str(state)
for line in self.readfile(filename):
if batch is None:
batch = leveldb.WriteBatch()
if True or self.get(line, None) is None:
batch.Put(line, state)
ct += 1
if ct % SYNC == 0:
self.db.Write(batch, sync = True)
batch = None
log.trace('filename %s sync new %d skip %d', filename, ct, ft)
else:
ft += 1
if ft % SYNC == 0:
log.trace('filename %s skip new %d skip %d', filename, ct, ft)
return ct
def work_py(self, name, id=0):
with self.lock:
try:
email = self.next.next()
except StopIteration:
raise WorkerFinishException
app = OutLook(email, '846266')
try:
res = app.login()
if res:
app.into_profile()
#app = OutLook(email, '846266')
#app.login()
#if app.proof():
log.trace('%s active success', email)
self.client.add_account(email, 1)
except ProofException:
log.trace('%s active success', email)
self.client.add_account(email, 1)
except:
time.sleep(2)
def run(self):
name = osp.basename(self.dbpath)
open_log(name, logging.INFO)
open_debug()
parser = OptionParser()
parser.add_option("-a", "--action", dest="action", help="what action[add|del|export|addemail]")
parser.add_option("-f", "--file", dest="filename",
help="write report to add", metavar="FILE")
parser.add_option("-s", "--state", dest="state", help="what state want to set", type="string")
parser.add_option('-t', "--target_dir", dest='target_dir', help="which target dir to export")
parser.add_option('-c', "--count", dest='count', action="store_true")
options, args = parser.parse_args(sys.argv[1:])
if options.count:
return self.count(options.state or '0')
if options.action == 'export':
print 'begin export'
ct = self.export(options.state or '0', options.target_dir)
return
if not options.filename:
parser.print_help()
return
if osp.isdir(options.filename):
filenames = [osp.join(options.filename, x) for x in os.listdir(options.filename)]
else:
filenames = [options.filename]
if options.action == 'add':
total = 0
for filename in filenames:
if filename.find('.svn') != -1:
continue
ct = self.add_file_to_db(filename, options.state or 0)
total += ct
log.trace('action %s fielname %s result new %d total %d', options.action, filename, ct, total)
if options.action == 'addemail':
total = 0
for filename in filenames:
ct = self.add_email_to_db(filename, options.state or 0)
total += ct
log.trace('action %s fielname %s result new %d total %d', options.action, filename, ct, total)
if options.action == 'del':
total = 0
for filename in filenames:
ct = self.del_file(filename)
total += ct
log.trace('action %s fielname %s result new %d total %d', options.action, filename, ct, total)
