def parse_body(html, pageName, number):
doc = jq(html)('.c_content_overflow')
imgs = jq(doc('img'))
urls = []
for img in imgs:
urls.append(''.join(['http://qq.yh31.com', jq(img).attr('src')]))
return tools.saveImgsByNum(urls, pageName, number)
def get_one_page(page, mysql):
url = 'https://hz.fang.lianjia.com/loupan/xihu-xiacheng-jianggan-gongshu-shangcheng-binjiang-yuhang-xiaoshan/pg%s#xiaoshan' % page
browser.get(url)
html = browser.page_source
doc = jq(html)
print('now is in page %s' % page)
if page > 1:
WebDriverWait(browser, 10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, '.page-box .next')))
btn = browser.find_element_by_css_selector('.page-box .next')
if not btn:
print('no more page')
return
browser.execute_script(
'window.scrollTo(0, document.body.scrollHeight)')
btn.click()
WebDriverWait(browser, 5).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, '.page-box span.active')))
WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, '.resblock-list')))
items = doc.find('.resblock-list .resblock-desc-wrapper')
for item in items:
mysql.insert(parse_one_item(jq(item)))
def Crack(Resp=None, Content=None, Url=None):
Content = Resp.text if Resp else Content
Url = Resp.url if Resp else Url
JQData = jq(Content)("#challenge-form")
JSCode = jq(Content)("script").eq(0).text().replace(
"\n", "").split("setTimeout(function(){")[1].split(
"f.action += location.hash")[0].replace("a.value", "res").replace(
"t.length", "12").split(";")
KeyData = json.loads('{"' + JSCode[0].split(",")[-1].replace(
'":', '":"').replace("}", '"}').replace("=", '":') + "}")
Key = list(KeyData.keys())[0]
Key2 = list(KeyData[Key].keys())[0]
Key3 = f"{Key}.{Key2}".strip()
StartData = execjs.eval(JSCode[0].split(",")[-1])
print(f"keyCode_{StartData[Key2]} -- > {JSCode[0].split(',')[-1]}")
for i in JSCode[10:-3]:
Code = i.replace(Key3, "")
Method = Code[0]
Code = "{" + f'"{Key2}":{Code[2:]}' + "}"
StartData[Key2] = eval(
f"StartData[Key2]{Method}{execjs.eval(Code)[Key2]}")
print(f"keyCode_{StartData[Key2]} --> {Code}")
Params = {i.attr("name"): i.attr("value") for i in JQData('input').items()}
Params["jschl_answer"] = execjs.eval(JSCode[-3].replace(
Key3, str(StartData[Key2])))
time.sleep(4)
return f'{urljoin(Url, JQData.attr("action"))}?{"&".join([f"{k}={v}" for k,v in Params.items()])}'
def GetTask(self):
try:
resp = self.session.get(TargetUrl)
if self.resp_check(resp):
content = jq(resp.content)
self.task = [self.pkgtaskitem(items) for items in jq(resp.content)(
"body > div.box > div > div.fallsFlow > ul").items('li>a') if "神奇世界看看看" in jq(items).attr("title")]
self.logger.info(f"Get Task Success --> {self.task}")
except Exception as e:
self.logger.error(e)
def data_parser(self, resp, method=None):
if method == "getcookie":
content = resp.text
jqdata = jq(content)("#challenge-form")
path = jqdata.attr("action")
scripts = jq(content)("script").eq(0).text()
jscode = scripts.replace(
"\n", "").split("setTimeout(function(){")[1].split(
"f.action += location.hash")[0].replace(
"a.value", "res").replace("t.length", "12").split(";")
keydata = json.loads('{"' + jscode[0].split(",")[-1].replace(
'":', '":"').replace("}", '"}').replace("=", '":') + "}")
key = list(keydata.keys())[0]
key_2 = list(keydata[key].keys())[0]
key_3 = "{key}.{key_2}".format(key=key, key_2=key_2).strip()
start_data = execjs.eval(jscode[0].split(",")[-1])
self.loger.info("keycode_{0} -- > {1}".format(
start_data[key_2], jscode[0].split(',')[-1]))
for i in jscode[10:-3]:
code = i.replace(key_3, "")
method = code[0]
code = "{" + '"{0}":{1}'.format(key_2, code[2:]) + "}"
start_data[key_2] = eval("start_data[key_2]{0}{1}".format(
method,
execjs.eval(code)[key_2]))
self.loger.info("keycode_{0} -- > {1}".format(
start_data[key_2], code))
params = {
i.attr("name"): i.attr("value")
for i in jqdata('input').items()
}
params["jschl_answer"] = execjs.eval(jscode[-3].replace(
key_3, str(start_data[key_2])))
return '{0}?{1}'.format(
urljoin(self.main_url, path),
"&".join(["{0}={1}".format(k, v) for k, v in params.items()]))
else:
with open("test.html", "wb") as f:
f.write(resp.content)
for item in jq(resp.text)("#maindiv >table tr").items():
key = item("td:nth-child(1)").text()
self.datas[key] = {}
self.datas[key]['TxHASH'] = item("td:nth-child(1)").text()
self.datas[key]['Age'] = item("td:nth-child(2)").text()
self.datas[key]['From'] = item("td:nth-child(3)").text()
self.datas[key]['Other'] = item("td:nth-child(4)").text()
self.datas[key]['To'] = item("td:nth-child(5)").text()
self.datas[key]['Quantity'] = item("td:nth-child(5)").text()
def get_image():
isNext = 0
url = 'http://qq.yh31.com/zjbq/0551964.html'
pageName = []
pageName.append("jinguanzhang")
number = 1
while (isNext == 0):
html = tools.getHTML(url)
doc = jq(html)("#pe100_page_contentpage")
next = jq(doc("a:last")).attr('href')
nextUrl = ''.join(['http://qq.yh31.com', next])
if (nextUrl == url):
isNext = 1
number = parse_body(html, pageName[0], number)
url = nextUrl
def checkIP():
try:
resp = requests.get('https://ipinfo.info/html/my_ip_address.php',
proxies={'https': 'socks5://127.0.0.1:9150'})
loger.info(jq(resp.text)("#Text10 > p > span > b").text())
except Exception as e:
loger.error(e)
def FirstFetch(self):
targets = DarkNetWebSites.select().where(
DarkNetWebSites.ismaster == self.usemaster)
if not targets:
return
target = targets[0]
try:
self.warn(f'[{target.domain}]Getting PHPSESSID')
resp = self.session.get(f'http://{target.domain}')
target.ismaster = True
target.title = jq(resp.text)('title').text()
self.usemaster = True
self.master = target
self.domain = target.domain
user = DarkNet_User.select().where(
DarkNet_User.useful == True).order_by(fn.Rand()).limit(1)
if not bool(user):
self.Reg()
else:
self.usr = user[0].user
self.pwd = user[0].pwd
if random.choice([1, 0]): # 佛系注册堆积账号池
self.Reg()
return True
except KeyboardInterrupt:
pass
except requests.Timeout:
target.alive = False
target.ismaster = False
self.usemaster = False
except Exception as e:
raise
finally:
target.save()
def getData(type: str):
url = "https://proxy.mimvp.com/freeopen.php?proxy=" + type
proxy = getProxy()
with requests.get(url=url,
proxies=proxy,
headers=header.getHeader(
"https://proxy.mimvp.com/freesecret.php")) as rsp:
trs = jq(
rsp.text)("table.mimvp-tbl.free-proxylist-tbl>tbody>tr").items()
for tr in trs:
tds = tr("td")
ip = tds.eq(1).text()
if ip == "": continue
portSrc = "https://proxy.mimvp.com/" + tds.eq(2)("img").attr("src")
with requests.get(portSrc,
proxies=proxy,
headers=header.getHeader(url)) as rsp:
port = pytesseract.image_to_string(Image.open(
io.BytesIO(rsp.content)),
lang="num")
proto = tds.eq(3).text()
if proto.index("HTTP") > -1:
proto = "HTTP"
elif proto.index("HTTPS") > -1:
proto = "HTTPS"
else:
continue
addr = {"ip": ip, "port": port, "proto": proto}
pool.addUnremove(addr)
return
def parse_body(html):
movies = []
doc = jq(html)('.x-item')
for item in doc:
item = jq(item)
name = parse_words(item, '.title')
magnet = jq(item('.title')[1]).attr('href')
detail = jq(item('.tail')[0]).text()
#print '名称 ：',name
#print '详细 ：',detail
#print '磁力链接: ',magnet
#print '--------------------------------------------------------------'
movie = []
movie.append(name)
movie.append(detail)
movie.append(magnet)
movies.append(movie)
return movies
def parse_cilisou(html):
movies = []
doc = jq(html)('.search-item')
for item in doc:
item = jq(item)
name = parse_words(item, '.item-title')
magnet = jq(item('a')[1]).attr('href')
detail = jq(item('span')).text()
# print '名称 ：',name
# print '详细 ：',detail
# print '磁力链接: ',magnet
# print '--------------------------------------------------------------'
movie = []
movie.append(name)
movie.append(detail)
movie.append(magnet)
movies.append(movie)
return movies
def get_online_url(title):
print('title:%s' % title)
browser.get('%s/search?q=%s' % (ONLINE_WEBSITE_PREFIX, title))
WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, '.movie-detail')))
html = browser.page_source
doc = jq(html)
movie_details = list(doc.find('.movie-detail').items())[:3]
infos = []
for movie_detail in movie_details:
strong = movie_detail.find('.text-danger')
if strong:
url = strong.parent().attr('href')
url_title = strong.parent().text()
browser.get('%s%s' % (ONLINE_WEBSITE_PREFIX, url))
WebDriverWait(browser, 10).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, '.movie-info')))
html = browser.page_source
print(html)
doc = jq(html)
play_list = doc.find('.play-list')
if len(list(play_list)):
play_urls = ''
for play in play_list.find('li').items():
url = ONLINE_WEBSITE_PREFIX + play.find('a').attr('href')
play_urls += url + '\n'
info = {
'title': title,
'url_info': {
'url_title': url_title,
'url': play_urls
}
}
infos.append(info)
else:
print('no online resource')
if len(infos):
print(infos)
mysql.update_online_url(title, str(infos))
else:
mysql.update_online_url(title, 'null')
def GetTaskItem(self, url):
try:
resp = self.session.get(url)
if self.resp_check(resp):
res = jq(resp.content)(".m_r > script").text().split(
'"img" :')[1].split("};")[0]
res = eval(res.replace(" ", ""))
self.logger.info(f"Get Item Success --> {url} {len(res)}")
return res
except Exception as e:
self.logger.error(e)
def data_parser(self, resp):
content = resp.text
jqdata = jq(content)("#challenge-form")
path = jqdata.attr("action")
scripts = jq(content)("script").eq(0).text()
jscode = scripts.replace("\n",
"").split("setTimeout(function(){")[1].split(
"f.action += location.hash")[0].replace(
"a.value",
"res").replace("t.length",
"12").split(";")
keydata = json.loads('{"' + jscode[0].split(",")[-1].replace(
'":', '":"').replace("}", '"}').replace("=", '":') + "}")
key = list(keydata.keys())[0]
key_2 = list(keydata[key].keys())[0]
key_3 = "{key}.{key_2}".format(key=key, key_2=key_2).strip()
start_data = execjs.eval(jscode[0].split(",")[-1])
logger.debug("keycode_{0} -- > {1}".format(start_data[key_2],
jscode[0].split(',')[-1]))
for i in jscode[10:-3]:
code = i.replace(key_3, "")
method = code[0]
code = "{" + '"{0}":{1}'.format(key_2, code[2:]) + "}"
start_data[key_2] = eval("start_data[key_2]{0}{1}".format(
method,
execjs.eval(code)[key_2]))
logger.debug("keycode_{0} -- > {1}".format(start_data[key_2],
code))
params = {
i.attr("name"): i.attr("value")
for i in jqdata('input').items()
}
params["jschl_answer"] = execjs.eval(jscode[-3].replace(
key_3, str(start_data[key_2])))
return '{0}?{1}'.format(
urljoin(resp.url, path),
"&".join(["{0}={1}".format(k, v) for k, v in params.items()]))
def get_top_250(page_num):
url = 'https://movie.douban.com/top250'
browser.get(url)
html = browser.page_source
doc = jq(html)
print("now is in page %s" % page_num)
if page_num > 1:
next_btn = list(
browser.find_elements_by_css_selector('.paginator a'))[page_num -
1]
browser.execute_script(
'window.scrollTo(0, document.body.scrollHeight)')
next_btn.click()
html = browser.page_source
doc = jq(html)
WebDriverWait(browser, 10).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, '#wrapper #content .grid_view')))
ol = doc.find('.grid_view')
for item in get_items_in_one_page(ol):
movies.append(item)
def mainPart(son):
cont = son(".cont")
source = son(".source")
title = cont("p").eq(0).text().strip()
preface = None
dynasty = source("a").eq(1).text()
author = source('a').eq(0).text()
poemContent = ""
for i in cont("#contson p"):
span = jq(i).find('span')
if span:
preface = filt(span.html())
else:
poemContent += filt(jq(i).html()) + "\n"
poemContent = poemContent.strip()
if not poemContent:
poemContent = filt(son("#contson").html())
return {
'title': title,
'content': poemContent,
'preface': preface,
'dynasty': dynasty,
'author': author
}
def __get_max_page_num(self):
browser.get(self._url % 1)
try:
WebDriverWait(browser, 5).until(
EC.presence_of_all_elements_located(
(By.CSS_SELECTOR, '.page-box a')))
html = browser.page_source
doc = jq(html)
page_btn = doc.find('.page-box a').items()
max_page_num = int(list(page_btn)[len(list(page_btn)) - 2].text())
distinct = doc.find('.position a.selected').text()
self._max_page_num = max_page_num
self._distinct = distinct
print('max page num:%d' % max_page_num)
except Exception as e:
print(e.args)
print('no page num')
self._max_page_num = 0
def GetTypeDatas(self, qeaid, name, page=1):
url = f"http://{self.domain}/pay/user_area.php?page_y1={page}&q_u_id=0&m_order=&q_ea_id={qeaid}&sid={self.sid}#page_y1"
self.warn(url)
resp = self.session.get(url)
resp.encoding = "utf8"
hasres = False
try:
self.CheckIfNeedLogin(resp)
self.SaveError(f"{qeaid}_{name}_{page}.html", resp)
self.info(len(resp.text))
jqdata = jq(resp.text)
for item in jqdata("table.m_area_a tr").items():
detailPath = item("div.length_400>a").attr("href")
if detailPath:
detailsURL = urljoin(resp.url, detailPath)
self.GetDetails(
detailsURL,
{
"lines":
FixNums(
item("td:nth-child(7)").text().replace(
"天", "")),
"hot":
FixNums(item("td:nth-child(8)").text()),
"title":
item("td:nth-child(5)").text(),
"area":
item("td:nth-child(3)").text(),
},
)
hasres = True
if page == 1:
maxpageStr = "".join(
jqdata(".page_b1:nth-last-child(1)").text().split())
return (FixNums(maxpageStr, to=1, error=1)
if maxpageStr and not self.justupdate else 1)
if hasres:
return True
except Exception as e:
self.error(f"GetTypeDatas: {e}")
self.SaveError("GetTypeDatas.html", resp)
raise
def GetTypeDatas(self, qeaid, name, page=1):
url = f"http://{self.domain}/pay/user_area.php?page_y1={page}&q_u_id=0&m_order=&q_ea_id={qeaid}&sid={self.sid}#page_y1"
self.warn(url)
resp = self.session.get(url)
resp.encoding = "utf8"
hasres = False
try:
self.CheckIfNeedLogin(resp)
self.SaveError(f'{qeaid}_{name}_{page}.html', resp)
self.info(len(resp.text))
jqdata = jq(resp.text)
for item in jqdata('table.m_area_a tr').items():
detailPath = item('div.length_400>a').attr('href')
if detailPath:
detailsURL = urljoin(resp.url, detailPath)
self.GetDetails(
detailsURL, {
'lines':
FixNums(
item('td:nth-child(7)').text().replace(
'天', '')),
'hot':
FixNums(item('td:nth-child(8)').text()),
'title':
item('td:nth-child(5)').text(),
'area':
item('td:nth-child(3)').text()
})
hasres = True
if page == 1:
maxpageStr = ''.join(
jqdata('.page_b1:nth-last-child(1)').text().split())
return FixNums(
maxpageStr, to=1,
error=1) if maxpageStr and not self.justupdate else 1
if hasres:
return True
except Exception as e:
self.error(f"GetTypeDatas: {e}")
self.SaveError('GetTypeDatas.html', resp)
raise
def get_index_page():
if not cookies:
do_login()
url = 'http://tgkb168.com/index_cctv.asp'
res = do_comm_request(url)
# 可能是登入超时或者登入失效
if int(res.headers['Content-Length']) < 250:
do_login()
res = do_comm_request(url)
return str(res.content, 'gb2312')
dom = jq(res.content, parser='html')
dom('.phone-top').remove()
dom('script').remove()
dom('#divkjt_bg').remove()
return dom.__str__()
def Login(self):
"""
### 再次尝试
1.因为网络问题重试
### 重新注册
2.因为账户被封重试
3.因为账户认证错误重试
"""
self.warn("Login...")
url = f"http://{self.domain}/ucp.php?mode=login"
data = {
"username": self.usr,
"password": self.pwd,
"login": "******",
"redirect": f"./index.php&sid={self.sid}",
}
resp = self.session.post(url, data=data, verify=False, timeout=120)
self.sid = "".join(re.findall("sid=(.*?)'", resp.text)[:1])
if self.usr not in resp.text and "暗网欢迎您" not in resp.text:
self.error("Auth faild")
self.SaveError("Autherror.html", resp)
if "已被封禁" in resp.text:
DarkNet_User.update({"useful": False}).where(
DarkNet_User.user == self.usr
).execute()
self.Reg()
raise ValueError
else:
self.report("Auth Success")
self.types = {
item(".index_list_title")
.attr("href")
.split("=")[1]
.split("&")[0]: item("tr:nth-child(1) > td")
.text()
.split()[0]
for item in jq(resp.text)(".ad_table_b").items()
}
self.report(self.types)
def FirstFetch(self):
targets = DarkNetWebSites.select().where(
DarkNetWebSites.ismaster == self.usemaster)
if not targets:
return
target = targets[0]
try:
self.warn(f"[{target.domain}]Getting PHPSESSID")
self.session.cookies.clear()
self.info(f"Already Cleaned Session Cookies.")
resp = self.session.get(f"http://{target.domain}")
resp = self.session.get(f"http://{target.domain}/index.php")
self.info(f"Current Cookie Nums: {len(self.session.cookies)}")
target.ismaster = True
target.title = jq(resp.text)("title").text()
self.usemaster = True
self.master = target
self.domain = target.domain
user = (DarkNet_User.select().where(
DarkNet_User.useful == True).order_by(fn.Rand()).limit(1))
if not bool(user):
self.Reg()
else:
self.usr = user[0].user
self.pwd = user[0].pwd
if random.choice([1, 0, 0]): # 佛系注册堆积账号池
self.Reg()
return True
except KeyboardInterrupt:
pass
except requests.Timeout:
target.alive = False
target.ismaster = False
self.usemaster = False
except Exception as e:
raise
finally:
target.save()
def Login(self):
"""
### 再次尝试
1.因为网络问题重试
### 重新注册
2.因为账户被封重试
3.因为账户认证错误重试
"""
self.warn('Login...')
url = f'http://{self.domain}/ucp.php?mode=login'
data = {
"username": self.usr,
"password": self.pwd,
"login": "******",
"redirect": f"./index.php&sid={self.sid}"
}
resp = self.session.post(url, data=data, verify=False, timeout=120)
self.sid = ''.join(re.findall("sid=(.*?)'", resp.text)[:1])
self.info(f"SID: {self.sid}")
if self.usr not in resp.text:
self.error('Auth faild')
self.SaveError('Autherror.html', resp)
if "已被封禁" in resp.text:
DarkNet_User.update({
"useful": False
}).where(DarkNet_User.user == self.usr).execute()
self.Reg()
raise ValueError
else:
self.report('Auth Success')
self.types = {
item('.index_list_title').attr('href').split('=')[1].split('&')
[0]: item('tr:nth-child(1) > td').text().split()[0]
for item in jq(resp.text)('.ad_table_b').items()
}
self.report(self.types)
def ParserSelect(text, url, types, city):
jqdata = jq(text)
payload = {}
try:
if types == 'xxx':
for item in jqdata('.xxx').items():
hid = item.text()
payload[hid] = {
"hid": hid,
}
addtotal()
success(f"{hid}")
elif types == 'xxx_details':
payload = {}
addupdate()
except Exception as e:
raise
return payload
import requests as request
import urllib.parse as url
from pyquery import PyQuery as jq
cookies = {}
html = request.get("https://www.baidu.com/").text
dom = jq(html)
print(dom('#lg'))
def do_login():
res = request.post(
'http://tgkb168.com/muserChkLoginc.asp?hx66=login',
allow_redirects=False,
data='username=%C1%F5%BA%E3&password=qwer7226361',
headers={'Content-Type': 'application/x-www-form-urlencoded'})
for i in res.cookies.items():
cookies[i[0]] = i[1]
for k, v in cookies.items():
print(k, v)
# do_login()
def do_comm_request(url):
return request.get(url, allow_redirects=False, cookies=cookies)
def get_index_page():
import requests
from pyquery import PyQuery as jq
import re
import json
ar = []
for i in range(1, 774):
href = "http://shige.laiyo.com/leixing_%dA%d.aspx"
url = href % (i, 1)
print("requesting", url)
resp = requests.get(url)
print("response over", resp.status_code)
resp.encoding = "utf8"
html = jq(resp.text)
category = html('.left .title h1 span').text()
poems = []
for j in range(1, 0xffffff):
url = href % (i, j)
print("requesting", url)
resp = requests.get(url)
print("response over", resp.status_code)
shige = re.findall("#shige\d+", resp.text)
if not shige:
break
for k in shige:
poems.append(int(re.search("\d+", k).group()))
d = {"name": category, "poems": poems}
ar.append(d)
json.dump(ar,
open("type.json", "w", encoding="utf8"),
indent=1,
def parse_words(doc, selector):
try:
return jq(doc(selector)[0]).text()
except:
return jq(doc(selector)).text()
def GetDetails(self, url, muti):
# time.sleep(2)
resp = self.session.get(url)
resp.encoding = "utf8"
self.CheckIfNeedLogin(resp)
jqdata = jq(resp.text)
jqdetail = jqdata(".v_table_1")
jqperson = jqdata(".v_table_2")
try:
uid = FixNums(jqperson("tr:nth-child(5) > td:nth-child(2)").text())
sid = FixNums(jqdetail("tr:nth-child(3) > td:nth-child(2)").text())
details = DarkNet_DataSale.select().where(
(DarkNet_DataSale.sid == sid))
person = DarkNet_Saler.select().where((DarkNet_Saler.uid == uid))
notice = DarkNet_Notice.select().where((DarkNet_Notice.sid == sid))
img = DarkNet_IMGS.select().where((DarkNet_IMGS.sid == sid))
personDatas = {
"salenums":
FixNums(jqperson("tr:nth-child(3) > td:nth-child(4)").text()),
"totalsales":
float(jqperson("tr:nth-child(5) > td:nth-child(4)").text()),
"totalbuys":
float(jqperson("tr:nth-child(7) > td:nth-child(4)").text()),
}
username = jqperson("tr:nth-child(3) > td:nth-child(2)").text()
if not person:
personDatas.update({
"uid":
uid,
"user":
username,
"regtime":
moment.date(
jqperson("tr:nth-child(7) > td:nth-child(2)").text()).
format("YYYY-MM-DD"),
})
person = DarkNet_Saler.create(**personDatas)
else:
DarkNet_Saler.update(personDatas).where(
(DarkNet_Saler.uid == uid)).execute()
person = person[0].uid
if not notice:
notice = DarkNet_Notice.create(**{"sid": sid})
else:
notice = notice[0].sid
detailImages = None
detailContent = " ".join(
jqdata(".postbody .content").text().split())
if not img:
urls = [_.attr("src") for _ in jqdata(".postbody img").items()]
img = DarkNet_IMGS.create(**{
"sid": sid,
"img": urls,
"detail": detailContent
})
detailImages = self.SavePics(urls, sid)
else:
img = img[0].sid
currentYear = moment.now().year
soldNum = FixNums(
jqdetail("tr:nth-child(7) > td:nth-child(4)").text(), to=99999)
toCurrentYearDateTime = moment.date(
f"{currentYear} " +
jqdetail("tr:nth-child(3) > td:nth-child(6)").text())
RealUpTimeJQ = jqdata(".author")
RealUpTimeJQ.remove("a")
RealUpTimeJQ.remove("span")
RealUpTime = moment.date(RealUpTimeJQ.text().replace(
"年", "").replace("月", "").replace("日", ""))
RealUpTime = RealUpTime if RealUpTime._date else toCurrentYearDateTime
detailsDatas = {
"lasttime":
moment.date(f"{currentYear} " +
jqdetail("tr:nth-child(7) > td:nth-child(6)").text(
)).format("YYYY-MM-DD HH:mm:ss"),
"priceBTC":
float(
jqdetail(
"tr:nth-child(3) > td:nth-child(4) > span").text()),
"priceUSDT":
float(
jqdetail("tr:nth-child(5) > td:nth-child(4)").text().split(
)[0]),
"lines":
muti["lines"],
"uptime":
RealUpTime.format("YYYY-MM-DD HH:mm:ss"),
"hot":
muti["hot"],
"types":
jqdetail("tr:nth-child(5) > td:nth-child(2)").text(),
"status":
jqdetail("tr:nth-child(7) > td:nth-child(2)").text(),
"oversell":
jqdetail("tr:nth-child(9) > td:nth-child(2)").text(),
"sold":
soldNum,
}
if not details:
detailsDatas.update({
"sid": sid,
"user": person,
"area": muti["area"],
"title": muti["title"],
"detailurl": url,
"img": img,
"notice": notice,
})
details = DarkNet_DataSale.create(**detailsDatas)
self.MakeMsg(details, detailContent, detailImages, sid,
username)
else:
self.warn(f'-{RealUpTime}- {muti["title"]}')
DarkNet_DataSale.update(detailsDatas).where(
(DarkNet_DataSale.sid == sid)).execute()
except Exception as e:
self.error(f"GetDetails {e}")
self.SaveError("error_264.html", resp)
raise
def Reg(self):
self.warn("Start Regging")
headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding":
"gzip, deflate",
"Accept-Language":
"zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control":
"no-cache",
"Connection":
"keep-alive",
"Content-Type":
"application/x-www-form-urlencoded",
"Origin":
f"http://{self.domain}",
"Pragma":
"no-cache",
"Referer":
f"http://{self.domain}/ucp.php?mode=register&sid={self.sid}",
"Upgrade-Insecure-Requests":
"1",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
}
step1resp = self.session.get(
f"http://{self.domain}/ucp.php?mode=register").text
step1 = jq(step1resp)
self.info(f'RegPage Confirmed. {"sid" in step1resp}')
token = step1('input[name="form_token"]').attr("value")
self.report(f"Got Token: {token}")
creation_time = step1('input[name="creation_time"]').attr("value")
self.report(f"Got Create_time: {creation_time}")
url = f"http://{self.domain}/ucp.php?mode=register"
step2resp = self.session.post(
url,
data={
"agreed": "===好的,我已明白,请跳转到下一页继续注册====",
"change_lang": "",
"creation_time": creation_time,
"form_token": token,
},
headers=headers,
)
self.SaveError("step2.html", step2resp)
step2 = jq(step2resp.text)
token = step2('input[name="form_token"]').attr("value")
creation_time = step2('input[name="creation_time"]').attr("value")
qa_answer = re.findall("请在右边框中输入： (.*?)：</label>", step2resp.text)[0]
self.report(f"Got answer: {qa_answer}")
qa_confirm_id = step2("#qa_confirm_id").attr("value")
self.usr = self.RandomKey(12)
self.pwd = self.RandomKey()
self.info(f"set Usr: {self.usr} ,Pwd: {self.pwd}")
data = {
"username":
self.usr,
"new_password":
self.pwd,
"password_confirm":
self.pwd,
"email":
"*****@*****.**",
"lang":
"zh_cmn_hans",
"tz_date":
"UTC+08:00+-+Asia/Brunei+-+" +
moment.now().format("DD+MM月+YYYY,+HH:mm"),
"tz":
"Asia/Hong_Kong",
"agreed":
"true",
"change_lang":
"0",
"qa_answer":
qa_answer,
"qa_confirm_id":
qa_confirm_id,
"submit":
" 用户名与密码已填好,+点此提交 ",
"creation_time":
creation_time,
"form_token":
token,
}
resp = self.session.post(url, data=data, headers=headers)
try:
assert "感谢注册" in resp.text
self.report("Reg success！")
DarkNet_User.create(**{"user": self.usr, "pwd": self.pwd})
except AssertionError:
self.error(jq(resp.text).text())
self.SaveError("reg.html", resp)
def part(son):
partName = son.find(".cont p").eq(0).text()
return {partName: filt(jq(son).find(".cont").html())}
