def generate_cookie(self, hotel_id: int):
''' generate cookie '''
# url = '{}hotel/{}.html'.format(HOTELS_URL, hotel_id)
# _, cookie = basic_req(url, 3, need_cookie=True)
cookie = {**self.login_cookie(), **self.generate_v1()}
print(self.encoder_cookie(cookie))
changeHeaders({'Cookie': self.encoder_cookie(cookie)})
def check_rank(self, av_id: int, times=0):
rank_list = self.rank_map[av_id] if av_id in self.rank_map else []
changeHeaders({'Referer': self.BASIC_AV_URL % av_id})
url = self.ARCHIVE_STAT_URL % av_id
json_req = proxy_req(url, 1)
if not self.have_error(json_req):
if (av_id not in self.av_id_list
and times < 3) or (av_id in self.av_id_list and times < 6):
self.check_rank(av_id, times + 1)
return
json_req = json_req['data']
need = [
'view', 'like', 'coin', 'favorite', 'reply', 'share', 'danmaku'
]
data = [json_req[index] for index in need]
if not self.check_view(av_id, data[0]):
if times < 3:
self.check_rank(av_id, times + 1)
return
if len(rank_list):
data = [time_str(), *data, *rank_list[:2], *rank_list[-2:]]
else:
data = [time_str(), *data]
with codecs.open('%s%d.csv' % (history_dir, av_id),
'a',
encoding='utf-8') as f:
f.write(','.join([str(index) for index in data]) + '\n')
if av_id in self.last_check and int(
time.time()) - self.last_check[av_id] > one_day:
self.del_map[av_id] = 1
del self.rank_map[av_id]
if av_id == self.basic_av_id:
clean_csv(av_id)
elif av_id not in self.last_check and int(
time.time()) > one_day + self.begin_timestamp:
self.del_map[av_id] = 1
del self.rank_map[av_id]
if av_id == self.basic_av_id:
clean_csv(av_id)
self.last_view[av_id] = data[1]
now_time = time.time()
echo(0, av_id, av_id == self.basic_av_id, av_id in self.public,
(now_time - self.public[av_id][0]) < 3.1 * one_day * 60,
self.public[av_id])
if av_id == self.basic_av_id and av_id in self.public and (
now_time - self.public[av_id][0]) < 3.1 * one_day * 60:
time_gap = (now_time - self.public[av_id][0]) / 60
echo(3, 'Time Gap:', round(time_gap / 10))
if round(time_gap / 10) in self.history_check_list and round(
time_gap / 10) not in self.history_check_finish:
self.history_rank(time_gap, data, av_id)
def check_type_req(self, av_id: int):
changeHeaders({'Referer': self.BASIC_AV_URL % av_id})
url = self.VIEW_URL % av_id
json_req = proxy_req(url, 1)
if json_req is None or 'data' not in json_req or 'tid' not in json_req[
'data']:
if can_retry(url):
self.check_type_req(av_id)
return
self.rank_type[av_id] = json_req['data']['tid'] == self.assign_tid
def public_data(self, av_id: int, times: int):
''' get public basic data '''
changeHeaders({'Referer': self.BASIC_AV_URL % av_id})
url = self.VIEW_URL % av_id
json_req = proxy_req(url, 1)
if json_req is None or not 'data' in json_req or not 'pubdate' in json_req[
'data']:
if times < 3:
self.public_data(av_id, times + 1)
return
data_time = json_req['data']['pubdate']
mid = json_req['data']['owner']['mid']
self.get_star_num(mid, 0)
self.public[av_id] = [data_time, mid]
def prepare_req(self,
hotel_id: int = 4889292,
city_id: int = 2,
startDate: str = time_str(-1, '%Y-%m-%d'),
depDate: str = time_str(int(time.time() + one_day),
'%Y-%m-%d')):
referer_url = HOTEL_DETAIL_URL % hotel_id
changeHeaders({'Referer': referer_url})
data = {
'city': city_id,
'checkin': startDate,
'cjeckout': depDate,
'defalutVal': None
}
return basic_req(AJAX_PROMOTION_URL, 11, data=data)
def check_rank_v2(self, av_id: int, times=0):
rank_list = self.rank_map[av_id] if av_id in self.rank_map else []
changeHeaders({'Referer': self.BASIC_AV_URL % av_id})
url = self.ARCHIVE_STAT_URL % av_id
json_req = proxy_req(url, 1)
if not self.have_error(json_req):
if times < 3:
self.check_rank_v2(av_id, times + 1)
return
json_req = json_req['data']
need = [
'view', 'like', 'coin', 'favorite', 'reply', 'share', 'danmaku'
]
data = [json_req[index] for index in need]
if len(rank_list):
data = [time_str(), *data, *rank_list[:2], *rank_list[-2:]]
else:
data = [time_str(), *data]
self.data_v2[av_id] = data
def generate_eleven(self, hotel_id: int):
################################################################
#
# [generate eleven] version 19.4.21(Test ✔️) write by gunjianpan
#
# 1. random generate 15 bit param `callback`;
# 2. use callback request OCEANBALL -> get origin js;
# 3. eval once -> (match array, and then chr() it) -> decoder js;
# 4. replace document and windows(you also can use execjs & jsdom);
# 5. warning you should replace `this` to some params,
# Otherwise, you will get `老板给小三买了包， 却没有给你钱买房`
# 6. final, return, and joint params;
#
################################################################
callback = self.generate_callback(15)
now_time = int(time.time() * 1000)
url = '{}?callback={}&_={}'.format(OCEANBALL_URL, callback, now_time)
referer_url = HOTEL_DETAIL_URL % hotel_id
changeHeaders({
'Referer': referer_url,
'if-modified-since': 'Thu, 01 Jan 1970 00:00:00 GMT'
})
oceanball_js, cookie = basic_req(url, 3, need_cookie=True)
print(cookie)
array = re.findall(r'\(\[(.*)\],', oceanball_js)[0].split(',')
array = [int(ii) for ii in array]
offset = int(re.findall(r'item-(\d*?)\)', oceanball_js)[0])
''' String.fromCharCode '''
oe = ''.join([chr(ii - offset) for ii in array])
''' replace window[callback] callback function '''
replace_str = re.findall(r'{}\(new.*\)\);'.format(callback), oe)[0]
eleven_params = re.findall(
r'{}\(new.*\+ (.*?) \+.*\)\);'.format(callback), oe)[0]
replaced_str = 'return {};'.format(eleven_params)
oe = oe.replace(replace_str, replaced_str)
oe = oe.replace('\'', '"').replace('\r', '')
oe = oe.replace(';!', 'let aaa = ', 1)
replace = '''
function(){let href="https://hotels.ctrip.com/hotel/%d.html";
a={"documentElement": {"attributes":{}}};
b={};
function c(){};
userAgent ="Chrome/73.0.3682.0";
geolocation = 1;
''' % hotel_id
''' replace document & windown & navigator '''
oe = oe.replace('document.body.innerHTML.length',
'888').replace('document.body.innerHTML', '""')
oe = oe.replace('document.createElement("div")', '{}')
oe = oe.replace('window.HTMLSpanElement',
'c').replace('document.createElement("span")', 'new c')
oe = oe.replace('window.location.href',
'href').replace('location.href', 'href')
oe = oe.replace('navigator.', '')
oe = oe.replace('new Image().', '').replace('new Image();', '')
oe = oe.replace('document.all', '0').replace('document.referrer', '""')
oe = oe.replace('this || ', '')
oe = oe.replace('window["document"]', 'a')
oe = oe.replace('document', 'a').replace('window', 'b')
oe = oe.replace('function(){', replace, 1)
''' eval script '''
eleven = js2py.eval_js(oe)
echo(1, 'eleven', eleven)
return eleven
def load_rank_index(self, index: int, day_index: int):
''' load rank '''
changeHeaders({'Referer': self.AV_URL})
url = self.RANKING_URL % (index, day_index)
text = basic_req(url, 3)
rank_str = re.findall('window.__INITIAL_STATE__=(.*?);', text)
if not len(rank_str):
if can_retry(url):
self.load_rank_index(index, day_index)
return False
rank_map = json.loads(rank_str[0])
rank_list = rank_map['rankList']
now_av_id = []
wait_check_public = []
rank_map = {}
for ii, rank in enumerate(rank_list):
av_id = int(rank['aid'])
need_params = [
'pts', 'author', 'mid', 'play', 'video_review', 'coins',
'duration', 'title'
]
temp_rank_list = [
ii, *[rank[ii] for ii in need_params], index, day_index
]
now_av_id.append(av_id)
if not self.check_type(av_id):
continue
self.check_rank_rose(av_id, temp_rank_list)
if self.add_av(av_id, ii, temp_rank_list[1]):
rank_map[av_id] = temp_rank_list
''' check assign av rank '''
for ii in self.assign_ids:
if not ii in self.public:
wait_check_public.append(ii)
if not ii in self.last_view and not ii in self.rank_map:
self.rank_map[ii] = []
have_assign = len([0 for ii in self.assign_ids if ii in now_av_id]) > 0
''' check tid type '''
threading_public = []
for ii in rank_map.keys():
work = threading.Thread(target=self.check_type_req, args=(ii, ))
threading_public.append(work)
for work in threading_public:
work.start()
for work in threading_public:
work.join()
for ii, jj in rank_map.items():
if self.check_type(ii) != True:
continue
if not ii in self.public:
wait_check_public.append(ii)
self.last_check[ii] = int(time.time())
self.rank_map[ii] = jj
''' load public basic data '''
threading_public = []
for ii in wait_check_public:
work = threading.Thread(target=self.public_data, args=(
ii,
0,
))
threading_public.append(work)
for work in threading_public:
work.start()
for work in threading_public:
work.join()
''' begin monitor '''
threading_list = []
for ii, jj in self.public.items():
if not ii in self.public_list and jj[0] + one_day > int(
time.time()):
work = threading.Thread(target=self.public_monitor,
args=(
ii,
0,
))
threading_list.append(work)
for work in threading_list:
work.start()
return have_assign
def generate_cookie(self, types: str = 'explore'):
''' generate bid '''
bid = "".join(random.sample(string.ascii_letters + string.digits, 11))
changeHeaders({"Cookie": "bid={}".format(bid),
'Referer': '{}{}'.format(self.BASIC_URL, types)})
