def get_friend_list(self, fid, requestToken, _rtk):
pn = 0
frinds_list = []
while True:
page_json = {"fid": fid, "pz": "24", "type": "WEB_FRIEND", "pn": pn}
data = {"p": json.dumps(page_json), "requestToken": requestToken, "_rtk": _rtk }
_, f, _ = self.http_post(self.good_friend_url, self.headers, data, self.cookie_dict)
api_data = json.loads(f)
try:
f_info = api_data.get('data').get('friends')
f_state = api_data.get('data').get('more')
except Exception as e:
Log.get_logger().exception(e)
break
if not f_info or not f_state:
break
for fri in f_info:
frinds_list.append(fri)
pn += 1
return frinds_list
def logAction(fs):
l = {}
if 'page' in fs.keys():
l = Log(fs).getAllLogs()
elif 'user_id' in fs.keys():
l = Log(fs).newLog()
returnJson(l)
def __init__(self, cookies):
try:
self.init_url = Config.ren_index_url
self.login_url = Config.login_url
self.login_token_url = Config.login_ajax_token_url
self.groups_data_url = Config.groups_data_url
self.good_friend_url = Config.good_friend_url
self.username = Config.username
self.password = Config.password
except Exception as e:
Log.get_logger().exception(e)
self.parse_cookie(cookies)
def http_post(self, url, headers, data=None, cookies=None, json=None):
try:
response = requests.post(url,
headers=headers,
cookies=cookies,
data=data)
except Exception as e:
Log.get_logger().exception(e)
#Log.get_logger().info("send http with POST method detected>>>>>>>>>>>>>>>>>>>>>>>>>")
#Log.get_logger().info("[ response code ]: %s " % response.status_code)
#Log.get_logger().info("[ response text ]: %s " % response.text.encode('unicode_escape').decode('utf-8'))
#Log.get_logger().info("[ response cookies ]: %s " % response.cookies)
#Log.get_logger().info("[ response headers ]: %s " % response.headers)
return response.status_code, response.json(
) if json else response.text, response.cookies
def download_fid(Cookie, rtk, _rtk):
spider = Spider(Cookie)
mgo = MongoDB()
for d in mgo.get_collection('renren_member').find():
fid = d.get('fid') or None
if fid:
target = spider.get_friend_list(fid, rtk, _rtk)
if target:
for t in target:
try:
t_fid = t.get('fid')
except Exception as e:
Log.get_logger().exception(e)
print("[Error] fid is empty!")
query_this = mgo.get_collection('renren_member').find_one({"fid": t_fid})
if not query_this:
mgo.get_collection('renren_member').insert_one(t)
print("%s insert ok!" % t)
else:
print("%s pass!" % t)
download_fid(Cookie, rtk, _rtk)
def log(user, post, session):
new_log = Log(user=user, post=post)
session.add(new_log)
session.commit()
return new_log
import sys
from app import Log
from app.res.language import load_language, LANGUAGES
from app.res.language.translate_language import TranslateLanguage
from tools.translate import *
# build translate language data.
Log.append('Build', 'Info', 'Building translate data now...')
load_language()
for lang_type in LANGUAGES.values():
if issubclass(lang_type, TranslateLanguage):
lang = lang_type()
if not lang._translated:
if lang._translate_to == 'cn_t':
translator = zhconv()
elif '--translate-baidu' in sys.argv:
translator = baidu_translate()
else:
translator = google_translate()
Log.append('Build', 'Translate',
'Using %s' % translator.__class__.__name__)
lang.translate(translator)
lang.save_current_translate()
Log.append('Build', 'Info', 'Build finish.')
def __init__(self, cookies):
super(Spider, self).__init__(cookies)
try:
self.headers = Config.chrome_headers
except Exception as e:
Log.get_logger().exception(e)
from app.res.const import Const
from app.res.language import load_language, LANGUAGES
from app.res.language.translate_language import TranslateLanguage
from tools.translate import *
from tools.utils.zip import zip_directory
datas = {}
def add_data(src, dest):
if os.path.exists(src):
datas[src] = dest
# build translate language data.
Log.append('Build', 'Info', 'Building translate data now...')
load_language()
for lang_type in LANGUAGES.values():
if issubclass(lang_type, TranslateLanguage):
lang = lang_type()
if not lang._translated:
if lang._translate_to == 'cn_t':
translator = zhconv()
elif '--translate-baidu' in sys.argv:
translator = baidu_translate()
else:
translator = google_translate()
Log.append('Build', 'Translate',
'Using %s' % translator.__class__.__name__)
lang.translate(translator)
try:
t_fid = t.get('fid')
except Exception as e:
Log.get_logger().exception(e)
print("[Error] fid is empty!")
query_this = mgo.get_collection('renren_member').find_one({"fid": t_fid})
if not query_this:
mgo.get_collection('renren_member').insert_one(t)
print("%s insert ok!" % t)
else:
print("%s pass!" % t)
download_fid(Cookie, rtk, _rtk)
if __name__ == '__main__':
Log.get_logger().info("----------------application begin to execute------------------")
parser = argparse.ArgumentParser()
parser.add_argument("-r", "--run", type=str, default='spider', choices=['spider'], help="please input init operation~!")
parser.add_argument("-p", "--operate", type=str, default='mainfriend', choices=['mainfriend', 'queryfriend', 'fire'], help="please input init operation~!")
parser.add_argument("-f", "--fid", type=str, default='227858937', help="please input member fid~!")
parser.add_argument("-t", "--rtk", type=str, default='-1716276427', help="please input request post token~!")
parser.add_argument("-k", "--_rtk", type=str, default='6574b18f', help="please input request post token~!")
parser.add_argument("-d", "--dg", type=str, default='fire', help="please input dg operate command~!")
args = parser.parse_args()
Cookie = "anonymid=jox4s669-z94iug; _r01_=1; [email protected]; ln_hurl=http://hdn.xnimg.cn/photos/hdn221/20141225/2040/main_OfDO_747d00029c29195a.jpg; jebe_key=06244583-718b-4363-9e4a-2ce0463f14ff%7Cf38044e0b216ea6d1642d82816df489c%7C1543165317107%7C1%7C1543165316373; __utmz=10481322.1543418285.1.1.utmcsr=renren.com|utmccn=(referral)|utmcmd=referral|utmcct=/223411359/profile; depovince=GW; _de=085C0D6D57D8483C5BF7EAB63017E04A; wp_fold=0; __utma=10481322.203082104.1543418285.1543418285.1544345645.2; _ga=GA1.2.2008285777.1543150874; _gid=GA1.2.1701398263.1544349788; jebecookies=04a25418-dab1-4e63-a670-62159647e5eb|||||; ick_login=8e1d318c-2d97-4e61-b557-a851dc538510; p=82c528b770f9356d5650c238e672e7ed7; first_login_flag=1; t=25423cb670a8526cbfe75ef6305329e07; societyguester=25423cb670a8526cbfe75ef6305329e07; id=102747867; xnsid=6d84261a; loginfrom=syshome; ch_id=10016; jebe_key=06244583-718b-4363-9e4a-2ce0463f14ff%7Cf38044e0b216ea6d1642d82816df489c%7C1543165317107%7C1%7C1544350852382"
Cookie = "anonymid=jox4s669-z94iug; _r01_=1; [email protected]; ln_hurl=http://hdn.xnimg.cn/photos/hdn221/20141225/2040/main_OfDO_747d00029c29195a.jpg; jebe_key=06244583-718b-4363-9e4a-2ce0463f14ff%7Cf38044e0b216ea6d1642d82816df489c%7C1543165317107%7C1%7C1543165316373; depovince=GW; _de=085C0D6D57D8483C5BF7EAB63017E04A; _ga=GA1.2.2008285777.1543150874; _gid=GA1.2.1701398263.1544349788; __utma=10481322.203082104.1543418285.1544345645.1544351771.3; __utmz=10481322.1544351771.3.2.utmcsr=renren.com|utmccn=(referral)|utmcmd=referral|utmcct=/145749760/profile; __utmb=10481322.1.10.1544351771; jebecookies=15db1112-05e6-4653-809d-d791f8b34f6b|||||; ick_login=d59c6497-d399-4cf2-9894-929952a37e1b; p=82c528b770f9356d5650c238e672e7ed7; first_login_flag=1; t=25423cb670a8526cbfe75ef6305329e07; societyguester=25423cb670a8526cbfe75ef6305329e07; id=102747867; xnsid=6d6baaff; loginfrom=syshome; ch_id=10016; jebe_key=06244583-718b-4363-9e4a-2ce0463f14ff%7Cf38044e0b216ea6d1642d82816df489c%7C1543165317107%7C1%7C1544352459489; wp_fold=0"
if args.run == "spider":
if args.operate == "mainfriend":