def cookies(self) -> RequestsCookieJar:
jar = RequestsCookieJar()
for name, cookie_dict in self['cookies'].items():
jar.set_cookie(
create_cookie(name, cookie_dict.pop('value'), **cookie_dict))
jar.clear_expired_cookies()
return jar
def cookies(self):
jar = RequestsCookieJar()
for name, cookie_dict in self['cookies'].items():
jar.set_cookie(create_cookie(
name, cookie_dict.pop('value'), **cookie_dict))
jar.clear_expired_cookies()
return jar
def generate_cookiejar(cookies):
jar = RequestsCookieJar()
for domain, content in cookies.items():
for line in content.split("\n"):
name, value = None, None
kwargs = {}
for i, record in enumerate(line.split(";")):
columns = record.split("=")
if len(columns) == 2:
key = columns[0].strip()
value_ = columns[1].strip()
if i == 0:
name = key
value = value_
continue
if key == "expires":
try:
dt = datetime.strptime(value_, "%A, %d-%b-%Y %H:%M:%S GMT").replace(tzinfo=pytz.UTC)
expire = unix_time_seconds(dt)
except ValueError:
try:
dt = datetime.strptime(value_, "%a, %d-%b-%Y %H:%M:%S GMT").replace(tzinfo=pytz.UTC)
expire = unix_time_seconds(dt)
except ValueError:
logger.warniing("Can't parse datetime [{}]".format(value_))
expire = unix_time_seconds() + 100
kwargs[key] = expire
continue
kwargs[key] = value_
continue
if len(columns) == 1:
value_ = columns[0].strip()
kwargs["secure"] = value_ == "secure"
jar.set_cookie(create_cookie(name, value, **kwargs))
return jar
def read_cookies(self, filename='cookies.txt'):
cookiesjar = RequestsCookieJar()
with open(filename, 'r') as f:
for c in f:
_c = c.strip().split('=')
cookiesjar.set_cookie(create_cookie(_c[0], _c[1], path=_c[2]))
return cookiesjar
def getHTTPClient(self):
if self._session:
return self._session
rcj = RequestsCookieJar()
for c in settings.cj:
rcj.set_cookie(c)
self._session = requests.Session()
self._session.cookies = rcj
return self._session
def json_post(url, data, cookies=None):
if cookies is None:
cookies = {}
if isinstance(cookies, RequestsCookieJar):
jar = cookies
else:
jar = RequestsCookieJar()
for k, v in cookies.items():
jar.set_cookie(k, v)
return treq.post(url,
json.dumps(data, cls=LiteAuthJsonEncoder).encode('ascii'),
headers={b'Content-Type': [b'application/json']},
cookies=jar,
timeout=LDAP_API_TIMEOUT)
def __merge_cookies(self, request_cookies):
if type(request_cookies) is RequestsCookieJar:
cookies = request_cookies
else:
cookies = RequestsCookieJar()
for name, value in request_cookies:
cookies.set(name, value)
if type(self.cookies) is RequestsCookieJar:
for cookie in iter(self.cookies):
cookies.set_cookie(cookie)
elif type(self.cookies) is dict:
for name, value in self.cookies.items():
cookies.set(name, value)
return cookies
def test_cookie_dict():
c = RetsHttpClient('login_url', 'username', 'password')
c._session = mock.MagicMock()
jar = RequestsCookieJar()
c1 = Cookie(
1,
'name1',
'value1',
80,
80,
'domain',
'domain_specified',
'domain_initial_dot',
'path',
'path_specified',
True,
True,
False,
'comment',
'comment_url',
'rest',
)
c2 = Cookie(
1,
'name2',
'value2',
80,
80,
'domain',
'domain_specified',
'domain_initial_dot',
'path',
'path_specified',
True,
True,
False,
'comment',
'comment_url',
'rest',
)
jar.set_cookie(c1)
jar.set_cookie(c2)
c._session.cookies = jar
assert c.cookie_dict == {'name1': 'value1', 'name2': 'value2'}
def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True, cookies_specified_kw=None):
""" 以下代码引用自requests库
具体参数说明参考：requests.cookies.cookiejar_from_dict。
"""
if not cookies_specified_kw:
cookies_specified_kw = {}
if cookiejar is None:
cookiejar = RequestsCookieJar()
if cookie_dict is not None:
names_from_jar = [cookie.name for cookie in cookiejar]
for name in cookie_dict:
if overwrite or (name not in names_from_jar):
# 添加参数 cookies_specified_kw
cookiejar.set_cookie(create_cookie(name, cookie_dict[name], **cookies_specified_kw))
return cookiejar
def get_cookie():
import robobrowser
browser = robobrowser.RoboBrowser(parser='html5lib')
browser.open('https://adventofcode.com/2021/auth/login')
browser.follow_link(browser.get_link('[GitHub]'))
form = browser.get_form()
form['login'].value = input('login?')
form['password'].value = input('password?')
browser.submit_form(form)
form2 = browser.get_form()
form2['otp'].value = input('otp?')
browser.submit_form(form2)
cookies = browser.session.cookies
#print(cookies)
for cookie in cookies:
#print(cookie, '\n')
cookiejar = RequestsCookieJar()
cookiejar.set_cookie(cookie)
if 'differ' not in requests.get(
'https://adventofcode.com/2020/day/10/input',
cookies=cookiejar).text:
found = cookie
print(found)
def get_spartan_token():
# Get the First Cookies
cookie_container = RequestsCookieJar()
first_response = requests.get(URL_TO_SCRAPE)
body = first_response.text.encode('utf-8', 'ignore')
for cookie in first_response.cookies: cookie_container.set_cookie(cookie)
# Get the PPFT
ppft_regex = re.compile("name=\"PPFT\".*?value=\"(.*?)\"")
ppft_match = re.findall(ppft_regex, body)
assert len(ppft_match) == 1
ppft = ppft_match[0]
# Prepare the login to Xbox
ppsx = "Pass"
query = "PPFT={ppft}&login={email}&passwd={password}&LoginOptions=3&NewUser=1&PPSX={ppsx}&type=11&i3={random}&m1=1680&m2=1050&m3=0&i12=1&i17=0&i18=__MobileLogin|1".format(
ppft = ppft, email = urllib.quote(EMAIL), password = PASSWORD, ppsx = ppsx, random = random.randint(15000, 50000))
headers = {"Content-Type": "application/x-www-form-urlencoded", "Host": "login.live.com", "Expect": "100-continue", "Connection": "Keep-Alive"}
# Stream the login to xbox
s = Session()
login_request = Request('POST', URL_TO_POST, headers = headers, data = query, cookies = cookie_container)
prepped = s.prepare_request(login_request)
login_response = s.send(prepped, stream = True, allow_redirects = False)
for cookie in login_response.cookies: cookie_container.set_cookie(cookie)
if "Location" not in login_response.headers: return None
next_location = login_response.headers['Location']
# Get Waypoint Cookies and Headers
waypoint_response = requests.get(next_location, allow_redirects = False)
if "WebAuth" not in waypoint_response.cookies: return None
for cookie in waypoint_response.cookies: cookie_container.set_cookie(cookie)
# Get the Spartan Token
headers = {"UserAgent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.52 Safari/537.17"}
token_response = requests.get(SPARTAN_TOKEN_GENERATOR, headers = headers, cookies = cookie_container)
spartan_token = token_response.text
spartan_token = json.loads(spartan_token)["SpartanToken"]
return spartan_token
class WxClient(object):
RUNTIME_KEYS = {
"device_id", "uuid", "sid", "uin", "skey", "pass_ticket", "syncKey",
"syncStr", "myUserName", "members", "groups", "_dn"
}
def __init__(self, client_name):
self.wx_host = 'wx.qq.com'
self.login_host = ''
self.file_host = ''
self.push_host = ''
self.conf_factory()
self.device_id = ('e%f' %
(random.random() * 1000000000000000)).split('.')[0]
self.client_name = client_name if client_name else self.device_id
self.qrcode_file = os.path.join(DATA_PATH, '%s.jpg' % self.client_name)
self.data_file = os.path.join(DATA_PATH, '%s.dat' % self.client_name)
self.online = STATUS_STOPPED
self._dn = int(time.time() * 1000) - 1
self.uuid = None
self.sid = None
self.uin = None
self.skey = None
self.pass_ticket = None
self.syncKey = None
self.syncStr = None
self.myUserName = None
self.members = {}
self.groups = {}
self.cookies = RequestsCookieJar()
self._request_d = {}
self._uptime = time.time()
self._recover()
def conf_factory(self):
e = self.wx_host
t, o, n = "login.weixin.qq.com", "file.wx.qq.com", "webpush.weixin.qq.com"
if e.find("wx2.qq.com") > -1:
t, o, n = "login.wx2.qq.com", "file.wx2.qq.com", "webpush.wx2.qq.com"
elif e.find("wx8.qq.com") > -1:
t, o, n = "login.wx8.qq.com", "file.wx8.qq.com", "webpush.wx8.qq.com"
elif e.find("qq.com") > -1:
t, o, n = "login.wx.qq.com", "file.wx.qq.com", "webpush.wx.qq.com"
elif e.find("web2.wechat.com") > -1:
t, o, n = "login.web2.wechat.com", "file.web2.wechat.com", "webpush.web2.wechat.com"
elif e.find("wechat.com") > -1:
t, o, n = "login.web.wechat.com", "file.web.wechat.com", "webpush.web.wechat.com"
self.login_host, self.file_host, self.push_host = t, o, n
def is_running(self):
return time.time() - self._uptime < 35.5
def is_online(self):
return self.is_running() and self.online == STATUS_ONLINE
def readable_status(self):
return ['STOPPED', 'WAITING', 'ONLINE'][self.online]
def cleanup(self):
dat = {}
for k in self.RUNTIME_KEYS:
dat[k] = getattr(self, k)
cookies = []
for cookie in self.cookies:
cookies.append(repr(cookie))
dat['cookies'] = cookies
with open(self.data_file, 'w') as f:
f.write(json.dumps(dat))
def stop(self):
for _d in self._request_d.values():
_d.cancel()
self.reset()
self.online = STATUS_STOPPED
def reset(self):
self.device_id = ('e%f' %
(random.random() * 1000000000000000)).split('.')[0]
self._dn = int(time.time() * 1000) - 1
self.uuid = None
self.sid = None
self.uin = None
self.skey = None
self.pass_ticket = None
self.syncKey = None
self.syncStr = None
self.myUserName = None
self.members = {}
self.groups = {}
self.cookies = RequestsCookieJar()
def _recover(self):
if not os.path.isfile(self.data_file):
return
with open(self.data_file) as f:
data = f.read()
data = json.loads(data or '{}')
for k, v in data.iteritems():
if k in self.RUNTIME_KEYS:
setattr(self, k, v)
for cookie_repr in data.get('cookies', []):
self.cookies.set_cookie(eval(cookie_repr, {'Cookie': Cookie}))
def _notice_log(self, msg):
logger.info('[%s] %s' % (self.client_name, msg))
def _warn_log(self, msg):
logger.warn('[%s] %s' % (self.client_name, msg))
def _error_log(self, msg):
logger.error('[%s] %s' % (self.client_name, msg))
@property
def dn(self):
self._dn += 1
return self._dn
@property
def _r(self):
return 1473173782527 - int(time.time() * 1000)
@property
def r(self):
return int(time.time() * 1000)
@defer.inlineCallbacks
def treq_request(self, url, data=None):
args = [url]
headers = {'Referer': 'https://wx.qq.com'}
method = 'get'
if data:
method = 'post'
data = json.dumps(data)
args.append(data)
headers['Content-Type'] = 'application/json; charset=UTF-8'
k = url.split('?')[0]
_d = self._request_d.get(k)
if isinstance(_d, defer.Deferred):
_d.cancel()
_d = self._request_d[k] = getattr(treq, method)(*args,
cookies=self.cookies,
headers=headers,
timeout=35)
try:
res = yield _d
except Exception as e:
if isinstance(e.message, list) and getattr(
e.message[0], 'type', None) is defer.CancelledError:
# print type(e.message[0]), e.message[0].type
self._warn_log('request %s cancelled' % k)
else:
import traceback
self._error_log(traceback.format_exc())
defer.returnValue('')
else:
self.cookies = res.cookies()
content = yield res.content()
defer.returnValue(content)
finally:
self._uptime = time.time()
self._request_d.pop(k, None)
@defer.inlineCallbacks
def run(self):
url = 'https://wx.qq.com'
try:
content = yield self.treq_request(url)
except Exception as e:
self._error_log('main page fail: %s' % e)
return
if content is None:
return
r = re.search(r'window\.MMCgi\s*=\s*\{\s*isLogin\s*:\s*(\S+)', content)
if r:
if r.group(1) == '!!"1"':
self.online = STATUS_ONLINE
self._notice_log(u"微信已登录")
d = self._sync_check()
else:
self.members = {}
self.groups = {}
d = self._get_uuid()
yield d
else:
self._error_log(u'主页返回格式有误')
@defer.inlineCallbacks
def _get_uuid(self):
self.online = STATUS_WAITING
url = 'https://' + self.login_host + '/jslogin?appid=wx782c26e4c19acffb&fun=new&lang=zh_CN&_=' + str(
self.dn)
try:
content = yield self.treq_request(url)
except:
self._error_log(u'获取uuid失败，准备重试...')
reactor.callLater(0.5, self.run)
return
if content is None:
return
r = re.match(
r'window\.QRLogin\.code = (\d+); window\.QRLogin\.uuid = "([^"]+)"',
content)
if r:
self.uuid = r.group(2)
self._get_qrcode()
yield self._login_check(1)
else:
self._error_log(u'uuid返回内容有误，%s' % content)
@defer.inlineCallbacks
def _get_qrcode(self):
url = 'https://' + self.login_host + '/qrcode/' + self.uuid
res = yield treq.get(url)
content = yield res.content()
with open(self.qrcode_file, 'wb') as f:
f.write(content)
self._notice_log(u'二维码准备就绪...')
@defer.inlineCallbacks
def _login_check(self, tip=0):
login_check_dict = {
'loginicon': 'true',
'uuid': self.uuid,
'tip': tip,
'_': self.dn,
'r': self._r
}
url = 'https://' + self.login_host + '/cgi-bin/mmwebwx-bin/login?%s' % urllib.urlencode(
login_check_dict)
content = yield self.treq_request(url)
if content is None:
return
r = re.search(r'window\.code=(\d+)', content)
if not r:
return
code = int(r.group(1))
if code == 200:
self._notice_log(u"正在登陆...")
r = re.search(r'window\.redirect_uri="([^"]+)"', content)
url = r.group(1) + '&fun=new&version=v2'
self.wx_host = url.split('://')[1].split('/')[0]
self.conf_factory()
content = yield self.treq_request(url)
dom = ElementTree.fromstring(content)
self.sid = dom.findtext('wxsid')
self.uin = dom.findtext('wxuin')
self.skey = dom.findtext('skey')
self.pass_ticket = dom.findtext('pass_ticket')
yield self._init()
elif code == 201:
self._notice_log(u"已扫码，请点击登录...")
yield self._login_check()
elif code == 408:
self._notice_log(u"等待手机扫描二维码...")
yield self._login_check()
elif code in {0, 400, 500}:
self._notice_log(u'等待超时，重新载入...')
yield self.run()
@defer.inlineCallbacks
def _init(self):
query_dict = {
'r': self._r,
'pass_ticket': self.pass_ticket,
'lang': 'zh_CN'
}
url = 'https://' + self.wx_host + '/cgi-bin/mmwebwx-bin/webwxinit?' + urllib.urlencode(
query_dict)
data = {
"BaseRequest": {
"Uin": self.uin,
"Sid": self.sid,
"Skey": self.skey,
"DeviceID": self.device_id
}
}
content = yield self.treq_request(url, data)
if content is None:
return
body_dic = json.loads(content)
if body_dic and body_dic['BaseResponse']['Ret'] == 0:
self.syncKey = body_dic['SyncKey']
self._form_sync_str()
self._parse_contact(body_dic['ContactList'])
self.myUserName = body_dic['User']['UserName']
self._notice_log(u"初始化成功，开始监听消息")
self.online = STATUS_ONLINE
self._status_notify()
self._get_contact()
yield self._sync_check()
else:
self._error_log(u'初始化失败%s' % unicode(body_dic))
@defer.inlineCallbacks
def _sync_check(self):
query_dict = {
'r': self.r,
'skey': self.skey,
'sid': self.sid,
'uin': self.uin,
'deviceid': self.device_id,
'synckey': self.syncStr,
'_': self.dn
}
url = 'https://' + self.push_host + '/cgi-bin/mmwebwx-bin/synccheck?' + urllib.urlencode(
query_dict)
try:
content = yield self.treq_request(url)
except Exception as e:
self._error_log(u'同步失败: ' + str(e))
yield self._sync_check()
return
if content is None:
return
r = re.match(r'window\.synccheck=\{retcode:"(\d+)",selector:"(\d+)"}',
content)
if not r:
self._error_log(u'同步失败: body格式有误，%s' % content)
yield self._sync_check()
return
retcode = int(r.group(1))
selector = int(r.group(2))
if retcode == 0:
if selector == 0:
self._notice_log(u'同步检查')
yield self._sync_check()
elif selector == 2:
self._notice_log(u'收到新消息')
elif selector == 4:
self._notice_log(u'朋友圈有新动态')
elif selector == 7:
self._notice_log(u'app操作消息')
else:
self._notice_log(u'未知消息')
if selector != 0:
yield self._sync()
elif retcode == 1100:
self._notice_log(u'你在手机上登出了微信，再见！')
elif retcode == 1101:
self._notice_log(u'你在其他地方登录了web微信，再见！')
elif retcode == 1102:
self._notice_log(u"未知登出，再见！")
else:
self._notice_log(u"未知retcode")
def _form_sync_str(self):
sync_str = ''
for i, sync_key in enumerate(self.syncKey['List']):
sync_str += '%s_%s' % (sync_key['Key'], sync_key['Val'])
if i != self.syncKey['Count'] - 1:
sync_str += '|'
self.syncStr = sync_str
@defer.inlineCallbacks
def _parse_contact(self, contact_list):
group_list = []
for contact in contact_list:
un = contact['UserName']
if un.find('@@') != -1:
self.groups[un] = contact
group_list.append(un)
else:
self.members[un] = contact
if group_list:
yield self._batch_get_contact(group_list)
@defer.inlineCallbacks
def _status_notify(self):
query_dict = {'pass_ticket': self.pass_ticket}
url = 'https://' + self.wx_host + '/cgi-bin/mmwebwx-bin/webwxstatusnotify?' + urllib.urlencode(
query_dict)
data = {
"BaseRequest": {
"Uin": self.uin,
"Sid": self.sid,
"Skey": self.skey,
"DeviceID": self.device_id
},
"Code": 3,
"FromUserName": self.myUserName,
"ToUserName": self.myUserName,
"ClientMsgId": int(time.time() * 1000)
}
content = yield self.treq_request(url, data)
if content:
body_dic = json.loads(content)
if body_dic['BaseResponse']['Ret'] == 0:
self._notice_log(u'状态同步成功')
else:
self._notice_log(u'状态同步失败: ' +
body_dic['BaseResponse']['ErrMsg'])
@defer.inlineCallbacks
def _get_contact(self):
query_dict = {
'pass_ticket': self.pass_ticket,
'skey': self.skey,
'r': self.r
}
url = 'https://' + self.wx_host + '/cgi-bin/mmwebwx-bin/webwxgetcontact?' + urllib.urlencode(
query_dict)
content = yield self.treq_request(url)
if content:
body_dic = json.loads(content)
yield self._parse_contact(body_dic['MemberList'])
@defer.inlineCallbacks
def _batch_get_contact(self, group_list):
query_dict = {
"type": "ex",
"pass_ticket": self.pass_ticket,
"r": self.r
}
url = 'https://' + self.wx_host + '/cgi-bin/mmwebwx-bin/webwxbatchgetcontact?' + urllib.urlencode(
query_dict)
_list = []
for un in group_list:
_list.append({'UserName': un, 'ChatRoomId': ''})
data = {
'BaseRequest': {
"DeviceID": self.device_id,
"Sid": self.sid,
"Skey": self.skey,
"Uin": self.uin,
},
'Count': len(_list),
'List': _list
}
content = yield self.treq_request(url, data)
if content is None:
return
body_dic = json.loads(content)
if not body_dic or body_dic['BaseResponse']['Ret'] != 0:
return
for contact in body_dic['ContactList']:
for member in contact['MemberList']:
self.members[member['UserName']] = member
@defer.inlineCallbacks
def _sync(self):
query_dict = {
'sid': self.sid,
'skey': self.skey,
'pass_ticket': self.pass_ticket,
'lang': 'zh_CN'
}
url = 'https://' + self.wx_host + '/cgi-bin/mmwebwx-bin/webwxsync?' + urllib.urlencode(
query_dict)
data = {
"BaseRequest": {
"Uin": self.uin,
"Sid": self.sid
},
"SyncKey": self.syncKey,
"rr": self._r
}
try:
content = yield self.treq_request(url, data)
except Exception as e:
self._warn_log(u'获取消息失败: %s' % e)
yield self._sync_check()
return
if content is None:
return
body_dic = json.loads(content)
if body_dic['BaseResponse']['Ret'] != 0:
self._warn_log(u'消息错误: %d' % body_dic['BaseResponse']['Ret'])
reactor.callLater(1, self._sync_check)
return
if body_dic['SyncKey'] and body_dic['SyncKey']['Count']:
self.syncKey = body_dic['SyncKey']
self._form_sync_str()
for contact in body_dic['DelContactList']:
self.members.pop(contact['UserName'], None)
self._parse_contact(body_dic['ModContactList'])
self._handle_msg(body_dic['AddMsgList'])
yield self._sync_check()
def _handle_msg(self, msgs):
for msg in msgs:
getattr(self, '_msg_%d' % msg['MsgType'], self._msg_default)(msg)
@defer.inlineCallbacks
def _get_user_remark_name(self, username):
remark_name = None
is_group = username.find('@@') == 0
if username in self.members:
remark_name = self.members[username].get('RemarkName')
remark_name = remark_name if remark_name else self.members[
username].get('NickName')
elif is_group:
if username in self.groups:
remark_name = self.groups[username].get('RemarkName')
remark_name = remark_name if remark_name else self.groups[
username].get('NickName')
else:
yield self._batch_get_contact([username])
if username in self.groups:
remark_name = self.groups[username].get('RemarkName')
remark_name = remark_name if remark_name else self.groups[
username].get('NickName')
defer.returnValue(remark_name if remark_name else u'未知')
@defer.inlineCallbacks
def _get_public_alias(self, username):
if username not in self.members:
yield self._batch_get_contact([username])
alias = None
if 'Alias' in self.members[username]:
alias = self.members[username]['Alias']
alias = alias if alias else (yield
self._get_user_remark_name(username))
defer.returnValue(alias if alias else u'未知')
@defer.inlineCallbacks
def _msg_1(self, msg):
src_name = yield self._get_user_remark_name(msg['FromUserName'])
dst_name = yield self._get_user_remark_name(msg['ToUserName'])
self._notice_log('%s -> %s: %s' % (src_name, dst_name, msg['Content']))
@defer.inlineCallbacks
def _msg_49(self, msg):
url = msg['Url'].replace('&', '&')
name = yield self._get_user_remark_name(msg['FromUserName'])
alias = yield self._get_public_alias(msg['FromUserName'])
title = msg['FileName']
self._notice_log(u'标题: %s' % title)
self._notice_log(u'链接: %s' % url)
self._notice_log(u'%s，分享了一个链接，请粘贴url到浏览器查看' % name)
Blog.parse_content(alias, msg['Content'])
# purge cache for wxdata
# noinspection PyBroadException
try:
res = yield treq.get(
'http://wxspider.pub.sina.com.cn:8086/_purge/v1/wxspider/blogs/%s'
% urllib.quote(alias.encode('utf-8')),
timeout=3)
content = yield res.content()
if content.find('Successful purge') > 0:
self._notice_log('purge cache ok')
elif content.find('Not Found') > 0:
self._notice_log('no cache there')
else:
self._warn_log('purge cache failed content: %s' % content)
except Exception as e:
self._warn_log('purge cache failed: %s' % str(e))
def _msg_default(self, msg):
self._notice_log(u'发现未定义的msgType: %d' % msg['MsgType'])
self._notice_log(msg['Content'])
port=None,
port_specified=False,
domain='.adventofcode.com',
domain_specified=True,
domain_initial_dot=True,
path='/',
path_specified=True,
secure=True,
expires=1953553997,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None},
rfc2109=False)
cookiejar = RequestsCookieJar()
cookiejar.set_cookie(cookie)
def download(year, day):
filename = f'aoc{year}_{day}input.txt'
if exists(filename):
return
text = requests.get(f'https://adventofcode.com/{year}/day/{day}/input',
cookies=cookiejar).text
with open(filename, 'w') as input_file:
input_file.write(text)
def get_cookie():
import robobrowser
browser = robobrowser.RoboBrowser(parser='html5lib')
def cookiejar_filter(cj, pattern):
filtered_cj = RequestsCookieJar()
for c in cj:
if re.match(pattern, c.domain):
filtered_cj.set_cookie(c)
return filtered_cj
def update(self, **kwargs):
login_page_url = "https://pass.neu.edu.cn/tpass/login?service=http%3A%2F%2F219.216.96.4%2Feams%2FhomeExt.action"
login_page_response = requests.get(login_page_url,
headers={'User-Agent': User_Agent})
if login_page_response.status_code != 200:
print("Get login page error")
login_page = BeautifulSoup(login_page_response.content.decode(),
features='html.parser')
page_cookies = login_page_response.cookies
# login_page = BeautifulSoup(open("test_html.txt", encoding='utf-8').read())
# rsa = login_page.find(id="rsa")['value']
# ul = login_page.find(id="ul")['value']
# pl = login_page.find(id="pl")['value']
login_form = login_page.find(id="loginForm")
login_url = login_form['action']
lt = login_page.find(id="lt")['value']
execution = login_form.find("input", attrs={'name':
'execution'})['value']
_eventId = login_form.find("input", attrs={'name':
'_eventId'})['value']
# rsa=20164617310410LT-3067-QjgEomOwuvnc0lIxzRWXafjreJ1i6u-tpass&
# ul=8&
# pl=6&
# lt=LT-3067-QjgEomOwuvnc0lIxzRWXafjreJ1i6u-tpass&
# execution=e2s1&
# _eventId=submit
# page_cookies['neu_cas_un'] = my_neu_id
page_cookies.set('neu_cas_un',
kwargs['uid'],
domain='pass.neu.edu.cn',
path='/')
login_response = requests.post(
"https://pass.neu.edu.cn/tpass/login?service=http%3A%2F%2F219.216.96.4%2Feams%2FhomeExt.action",
headers=dict(
headers, **{
'Referer':
'https://pass.neu.edu.cn/tpass/login?service=https%3A%2F%2Fportal.neu.edu.cn%2Ftp_up%2F',
'Origin': 'https://pass.neu.edu.cn',
'Host': 'pass.neu.edu.cn',
}),
cookies=page_cookies,
data={
'rsa': kwargs['uid'] + kwargs['password'] + lt,
'ul': len(kwargs['uid']),
'pl': len(kwargs['password']),
'lt': lt,
'execution': execution,
'_eventId': _eventId
})
login_response_page = BeautifulSoup(login_response.content.decode(),
features='html.parser')
stu_name = login_response_page.find('a', {'class': 'personal-name'})
if stu_name is None:
return False
stu_name = stu_name.text.strip()
self.name = stu_name.split('(')[0]
print(stu_name)
if login_response.content != b'':
with open('login_response.html', 'w', encoding='utf-8') as f:
f.write(login_response.content.decode('utf-8'))
if '教学周' in login_response.content.decode('utf-8'):
print("登录成功")
cookies_209 = RequestsCookieJar()
histories = [login_page_response
] + login_response.history + [login_response]
for h in histories:
for c in h.cookies:
c.domain = h.url.split('/')[2]
print(c)
cookies_209.set_cookie(c)
ids_url = 'http://219.216.96.4/eams/courseTableForStd.action?_=1557724872728'
ids_response = requests.get(ids_url, cookies=cookies_209)
ids = re.findall('bg\.form\.addInput\(form,"ids","(.+)"\);',
ids_response.content.decode())
table_url = 'http://219.216.96.4/eams/courseTableForStd!courseTable.action'
# semester.id=30: 2018-2019春季学期
table_response = requests.post(
table_url,
cookies=cookies_209,
headers=dict(
headers, **{
'Host': '219.216.96.4',
'Origin': 'http://219.216.96.4',
'Referer':
'http://219.216.96.4/eams/courseTableForStd.action'
}),
data={
'ignoreHead': '1',
'showPrintAndExport': '1',
'setting.kind': 'std',
'startWeek': '',
'semester.id': '30',
'ids': ids[0]
})
# if table_response.content != b'':
# with open('table.html', 'w', encoding='utf-8') as f:
# f.write(table_response.content.decode())
table_str = table_response.content.decode()
table_str = table_str.replace(' ', '')
table_str = table_str.replace('\t', '')
# table_str = table_str.replace('\r', '').replace('\n', '')
# with open('table2.html', 'w', encoding='utf-8') as f:
# f.write(table_str)
# with open('table2.html', encoding='utf-8') as f:
# stt = f.read()
# print(stt == table_str)
self.parse_table(table_str)
return True
