def ban_account(self, flag=-1):
'''
:param flag: -1 means wrong password
:return:
'''
AccountParameters.ban(self.running_account, flag)
self.accounts.pop(self.account_index)
if self.account_index == len(self.accounts):
self.account_index -= 1
def get_profiles(self):
self.user_list = TargetUser.get_all_targets()
self.accounts = AccountParameters.get_account_parameters(DOMAIN3)
self.running_account = self.get_account()
print 'Account %s loaded!' % (self.running_account.account,)
self.account_timer = Timer()
for uid in self.user_list:
uid = str(uid)
print 'Getting UID: %s profile...' % (uid,)
profile = self.get_profile(uid)
if profile is None:
print 'No profile!!!'
continue
self.profile_list.append(profile)
if len(self.profile_list) > PROFILE_LIMIT:
User.dumps(self.profile_list)
self.profile_list = []
User.dumps(self.profile_list) # clean up the bottom
print 'All Done!'
def get_history_timelines(self):
self.running_mode = 3
self.accounts = AccountParameters.get_account_parameters(DOMAIN3)
self.running_account = self.get_account()
print 'Account %s loaded!' % (self.running_account.account,)
self.account_timer = Timer()
for domain in xrange(MIN_DM, MAX_DM+1):
self.targets = TargetUser.get_targets_by_domain(domain)
i = 0
for tgt in self.targets:
i += 1
print 'Getting history posts for %s - Num: %d / Domain: %d' % (tgt.uid, i, domain)
self.get_history_posts(tgt.uid)
self.reset_buffer()
print 'Job Done!'
def monitor(self):
self.running_mode = 1
self.targets = TargetUser.get_target_users()
self.accounts = AccountParameters.get_account_parameters(DOMAIN)
self.running_account = self.get_account()
print 'Account %s loaded!' % (self.running_account.account,)
self.account_timer = Timer()
while True:
for tgt in self.targets:
print 'monitor target: %s... - account %s - access count %d' % \
(tgt.uid, self.running_account.account, self.running_account.count)
timelines = self.fetch_timelines(tgt.uid)
for tmln in timelines:
duration = tmln.timestamp - tmln.created_at
days = duration.days
seconds = duration.seconds
if days != 0:
continue # this post is too early.
if seconds < LOOP_TIME:
self.buffer.append(tmln)
else:
continue
if seconds < TIME_WINDOW:
self.memory.append(tmln)
if len(self.buffer) > BUFFER_LIMIT:
self.reset_buffer()
elif len(self.profile_list) > PROFILE_BUFFER_LIMIT:
self.reset_buffer()
self.check_memory()
self.reset_buffer() # After one loop, write data to DB.
def reinforce(self):
"""
Monitor the alarm, once it is on, it will reinforce the main spider
If it finds the alarm is off, it will go back to his monitoring status
:return:
"""
self.running_mode = 2
self.accounts = AccountParameters.get_account_parameters(DOMAIN2)
while True:
alarm = Alarm.get_latest_alarm()
if alarm is None:
print 'Waiting...'
time.sleep(REST)
continue
self.running_account = self.get_account()
self.account_timer = Timer()
target_list = TargetUser.get_targets_by_alarm(alarm)
while True:
for tgt in target_list:
print 'Reinforce to monitor target: %s...' % (tgt.uid,)
timelines = self.fetch_timelines(tgt.uid)
self.buffer.extend(timelines)
if len(self.buffer) > BUFFER_LIMIT_MINI or len(self.profile_list) > PROFILE_BUFFER_LIMIT:
self.reset_buffer()
self.reset_buffer()
if self.is_alarm_on(alarm) is False:
print 'Alarm disabled!'
break
else:
continue
def reset_accounts(self):
"""
Reset all the accounts
:return:
"""
self.account_timer.reset()
day = self.account_management_timer.click().days
if day > 0:
self.account_management_timer.reset()
AccountParameters.reset403()
if len(self.accounts) < ACCOUNT_LIMIT:
if self.running_mode == 1:
AccountParameters.supplement_account(DOMAIN)
self.accounts = AccountParameters.get_account_parameters(DOMAIN)
elif self.running_mode == 2:
AccountParameters.supplement_account(DOMAIN2)
self.accounts = AccountParameters.get_account_parameters(DOMAIN2)
elif self.running_mode == 3:
AccountParameters.supplement_account(DOMAIN3)
self.accounts = AccountParameters.get_account_parameters(DOMAIN3)
return 0
for acc in self.accounts:
acc.count = 0
from Database import AccountParameters
from Object import AccountParameters as Account
if __name__ == '__main__':
with open('/Users/ZHU_Chenghao/Downloads/accounts_161002.txt',
'r') as reader:
ln_num = 1
accounts = []
for line in reader.readlines():
print ln_num
line = line.split('\t')
if len(line) != 2:
continue
account = line[0]
paras = line[1]
domain = 0
paras = paras.split('&')
i = paras[1].split('=')[-1]
s = paras[2].split('=')[-1]
gsid = paras[7].split('=')[-1]
accounts.append(
Account(account=account, domain=domain, i=i, s=s, gsid=gsid))
ln_num += 1
AccountParameters.add_accounts(accounts)