def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'loaded cookies file')
except Exception as msg:
ScholarUtils.log('warn',
'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def cmr_download(urls):
"""Download files from list of urls."""
if not urls:
return
url_count = len(urls)
print('Downloading {0} files...'.format(url_count))
credentials = None
for index, url in enumerate(urls, start=1):
print(url)
if not credentials and urlparse(url).scheme == 'https':
credentials = get_credentials(url)
filename = url.split('/')[-1]
print('{0}/{1}: {2}'.format(
str(index).zfill(len(str(url_count))), url_count, filename))
# -- dont overwrite --------------------------------------------------------------
if (os.path.exists(filename)):
print("already downloaded")
else:
# -- end of overwrite modif ------------------------------------------------------
try:
print("url:" + url)
# In Python 3 we could eliminate the opener and just do 2 lines:
# resp = requests.get(url, auth=(username, password))
# open(filename, 'wb').write(resp.content)
req = Request(url)
if credentials:
req.add_header('Authorization',
'Basic {0}'.format(credentials))
opener = build_opener(HTTPCookieProcessor())
data = opener.open(req).read()
open(filename, 'wb').write(data)
except HTTPError as e:
print('HTTP error {0}, {1}'.format(e.code, e.reason))
except URLError as e:
print('URL error: {0}'.format(e.reason))
except IOError:
raise
except KeyboardInterrupt:
quit()
def __init__(self, username, password):
self._cookie_jar = CookieJar()
self._opener = build_opener(HTTPCookieProcessor(self._cookie_jar))
self.username = username
self.session = self._login(username, password)
self.devices = self._devices()
self.master = None
for d in self.devices:
if d.master is not None:
self.master = d.master
break
if not self.master:
raise ValueError("couldn't find master unit")
self.ws = websocket.create_connection(self.API_URL_SOCKET)
self.ws.send(
json.dumps({
"jsonrpc": "2.0",
"id": 3,
"method": "srvWebSocketAuth",
"params": {
"varName": username,
"apiKey": self.session.api_key,
},
}))
try:
ws_auth = json.loads(self.ws.recv())
if not ws_auth:
raise ValueError("no socket auth returned")
params = ws_auth.get("params")
if not params:
raise ValueError("no socket auth params received")
authorized = params.get("authorized")
if not authorized:
raise ValueError("socket not authorized: {}".format(
pformat(ws_auth)))
except:
self.close()
raise
def sendSMSto(self, to_number, otp, ac):
url = 'http://site24.way2sms.com/Login1.action?'
your_number = '9999999999'
your_w2sms_pass = '******'
data = bytes(
f'username={your_number}&password={your_w2sms_pass}&Submit=Sign+in',
'utf-8')
# 7988367320
cj = CookieJar()
opener = build_opener(HTTPCookieProcessor(cj))
opener.addheaders = [(
'User-Agent',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120'
)]
try:
opener.open(url, data)
except IOError:
return "ERR: SENDMSG"
jession_id = str(cj).split('~')[1].split(' ')[0]
send_sms_url = 'http://site24.way2sms.com/smstoss.action?'
send_sms_data = bytes(
'ssaction=ss&Token=' + jession_id + '&mobile=' + to_number +
'&message=' + 'Your One Time Pass is ' + otp + '&msgLen=136',
'utf-8')
print('SENT OTP IS', otp)
opener.addheaders = [
('Referer',
'http://site25.way2sms.com/sendSMS?Token=' + jession_id)
]
try:
sms_sent_page = opener.open(send_sms_url, send_sms_data)
soup = BS(sms_sent_page.read())
errNode = soup.find('span', {'class': 'err'})
if errNode:
print(errNode)
return "ERR: SENDMSG"
else:
return "SUCCESS: SENDMSG"
except IOError:
return "ERR: SENDMSG"
return "SUCCESS: SENDMSG"
def __init__(self):
# establish connection
#
# make cookie
cj = CookieJar()
# if we wanna use https we mast add ssl=enable_ssl to cookie
c = Cookie(0, 'ssl', "enable_ssl", None, False, '.nnm-club.me',
True, False, '/', True, False, None, 'ParserCookie', None, None, None)
cj.set_cookie(c)
self.session = build_opener(HTTPCookieProcessor(cj))
# avoid endless waiting
self.blocked = False
# add proxy handler if needed
if self.config['proxy'] and any(self.config['proxies'].keys()):
self.session.add_handler(ProxyHandler(self.config['proxies']))
# change user-agent
self.session.addheaders.pop()
self.session.addheaders.append(('User-Agent', self.config['ua']))
response = self._catch_error_request(self.url + 'login.php')
if not self.blocked:
parser = self.WorstParser(self.url, True)
parser.feed(response.read().decode('cp1251'))
parser.close()
form_data = {"username": self.config['username'],
"password": self.config['password'],
"autologin": "******",
"code": parser.login_code,
"login": "******"}
# so we first encode keys to cp1251 then do default decode whole string
data_encoded = urlencode({k: v.encode('cp1251') for k, v in form_data.items()}).encode()
self._catch_error_request(self.url + 'login.php', data_encoded)
if 'phpbb2mysql_4_sid' not in [cookie.name for cookie in cj]:
logging.warning("we not authorized, please check your credentials")
else:
logging.info('We successfully authorized')
def get_cookies():
print("Fetching cookies")
endpoint = config["endpoint"]
username = config["username"]
password = config["password"]
stsurl = discover_sts(username)
try:
if stsurl is None:
authToken = getSecurityTokenManaged(username, password, endpoint)
else:
authToken = getSecurityTokenFederated(username, password, endpoint,
stsurl)
except RuntimeError as x:
print(x, sys.stderr)
endpointUrl = urlparse(endpoint)
if endpointUrl.scheme not in ["http", "https"] or not endpointUrl.netloc:
print("Invalid endpoint URL: {}".format(endpoint))
cookiejar = CookieJar()
opener = build_opener(HTTPCookieProcessor(cookiejar))
try:
parsed_url = "{0}://{1}/_forms/default.aspx?wa=wsignin1.0".format(
endpointUrl.scheme, endpointUrl.netloc)
request = Request(parsed_url)
response = opener.open(request, data=authToken.encode('utf-8'))
cookieStr = ""
cookiesFound = []
for cookie in cookiejar:
if cookie.name in ("FedAuth", "rtFa"):
cookieStr += cookie.name + "=" + cookie.value + "; "
cookiesFound.append(cookie.name)
if "FedAuth" not in cookiesFound or "rtFa" not in cookiesFound:
print("Incomplete cookies retrieved.", sys.stderr)
except URLError as x:
print("Failed to login to SharePoint site: {}".format(x.reason))
return cookieStr
def __init__(self, args):
self.modulus = None
self.exponent = None
self.args = args
self.jar = j = LWPCookieJar()
self.has_cookies = False
context = ssl.create_default_context()
if self.args.cookiefile:
self.has_cookies = True
try:
j.load(self.args.cookiefile, ignore_discard=True)
except IOError:
self.has_cookies = False
handlers = [HTTPCookieProcessor(j)]
if self.args.ssl_noverify:
handlers.append(HTTPSHandler(context=context))
self.opener = build_opener(*handlers)
self.nextfile = args.file
def grab(url):
cookie = MozillaCookieJar()
cookie.load('cookies.txt', ignore_discard=True, ignore_expires=True)
req = Request(url, headers=DEFAULT_HEADERS)
opener = build_opener(HTTPCookieProcessor(cookie))
response = opener.open(req, timeout=DEFAULT_TIMEOUT)
print(response.read().decode('utf8'))
result = opener.open('http://oa.epoint.com.cn')
html = result.read()
html=html.decode('utf-8')
print(html)
result = opener.open('http://oa.epoint.com.cn/netoffice8/ZReport/Pages/Problem/Problem_Add.aspx')
html = result.read()
html=html.decode('utf-8')
resu=html
#print(resu)
result = opener.open('http://oa2.epoint.com.cn/EpointCommunity/EpointCommunity/Home/Home.aspx')
html = result.read()
html=html.decode('utf-8')
resu=html
def _call_details(self,
page=1,
page_size=20,
begin_date=None,
end_date=None):
response = self.check_login()
if response['isLogin'] is not True:
raise PermissionError("you have not logged in.")
url = 'http://iservice.10010.com/e3/static/query/callDetail?menuid=000100030001'
request_data = {'pageNo': page, 'pageSize': page_size}
request_data['beginDate'] = begin_date or datetime.now()[:9] + '01'
request_data['endDate'] = end_date or datetime.now()[:11]
data = urlencode(request_data, encoding='utf-8').encode('utf-8')
request = Request(url, data=data, headers=self.headers, method='POST')
sleep(2)
opener = build_opener(HTTPCookieProcessor(self.cookie))
response = opener.open(request).read().decode('utf-8')
return json.loads(response)
def non_selenium_get(command_ip, flag_id, flag, vuln):
username, password, post_id = flag_id.split(":")
cookies = LWPCookieJar()
browser = build_opener(HTTPCookieProcessor(cookies), HTTPRedirectHandler)
try:
data = {"username": username, "password": password}
request = prepare_post_request("{}/login".format(command_ip), data)
request.add_header('User-Agent', get_useragent())
response = browser.open(request).read().decode()
session = {}
for cookie in cookies:
session[cookie.name] = cookie.value
return session
except MumbleException as e:
return {"code": MUMBLE, "public": str(e)}
except DownException as e:
return {"code": DOWN, "public": str(e)}
def get_flights(self, best_position=True):
self.data = {}
for i in range(0,10):
while True:
try:
cj = CookieJar()
opener = build_opener(HTTPCookieProcessor(cj))
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
response = opener.open(self.url)
str_response = response.read().decode('utf-8')
self.data = json.loads(str_response)
if best_position:
Notifier.best_position(self)
except:
time.sleep(30)
continue
break
return self.data
def get(url, params: dict = None, headers: dict = None) -> HTTPResponse:
"""
:param url:
:param params:
:param headers:
:return:
"""
if params:
parar_str = urlencode(params)
subfix = '?' if url.find('?') == -1 else '' if url.find(
'=') == -1 else '&'
url += subfix + parar_str
print(url)
if headers:
req = Request(url, headers=headers)
else:
req = Request(url)
opener = build_opener(HTTPHandler(), HTTPCookieProcessor(CookieJar()),
ProxyHandler(proxies={'http': '39.137.69.7:8080'}))
return opener.open(req)
def lstASXforms(xdirectory):
opener = build_opener(HTTPCookieProcessor())
url = "https://www.asxonline.com/companies/html/ASICForms.html"
html = opener.open(url)
soup = BeautifulSoup(html, 'html.parser')
tags = soup.select("td")
lstTitle = []
lstHref = []
for tag in tags:
xtitle = tag.get_text()
if not (tag.find('a')):
lstTitle.append(xtitle)
for c in tag.findAll('a'):
if (c.get('href', '')).startswith('/'):
xhref = 'https://www.asxonline.com' + c.get("href")
lstHref.append(xhref)
dfASXforms = pd.DataFrame({'Title': lstTitle, 'Link': lstHref})
xfilename = xdirectory + "/ASXForms.csv"
dfASXforms.to_csv(xfilename, encoding='utf-8', index=False)
return (dfASXforms)
def _make_request(self, url, data=None, headers={}):
if not data:
data = None
else:
data = urlencode(data)
data = data.encode('utf8')
req = Request(url, data, headers)
err = None
handler = build_opener(HTTPCookieProcessor(self.cookiejar))
try:
resp_obj = handler.open(req)
except HTTPError as e:
err = e.code
return err, e.read()
resp = resp_obj.read()
resp_obj.close()
return None, unistr(resp, encoding='utf8')
def grab(cookie):
engine = create_engine(constant.connectMysqlUrl)
Session = sessionmaker(bind=engine)
session = Session()
spiderSql = 'select spiderurl,fornumname from Spider_Record where isenable="1"'
logintokenSql = 'select * from login_token where isenable=true order by id desc'
spiderlist = session.execute(spiderSql)
logintoken = session.execute(logintokenSql).first()
for j in spiderlist:
requestUrl = j.spiderurl.replace(
j.spiderurl[j.spiderurl.index("token="):j.spiderurl.
index("&lang=zh_CN")], "token=" + logintoken.token)
#print(requestUrl)
req = Request(requestUrl, headers=DEFAULT_HEADERS)
opener = build_opener(HTTPCookieProcessor(cookie))
response = opener.open(req, timeout=DEFAULT_TIMEOUT)
encode_json = response.read().decode("utf8")
decode_json = jsonString2Dict(encode_json)
print(decode_json)
try:
print(decode_json['app_msg_cnt'])
except:
loginSaveCookie.updateLoginCookie()
endIndex = int(decode_json['app_msg_cnt'])
time.sleep(5)
while endIndex >= 0:
spiderUrl = requestUrl.replace('begin=0', 'begin=' + str(endIndex))
decode_json = grabContent(cookie, spiderUrl)
#print(decode_json)
for i in decode_json['app_msg_list']:
cnt = session.query(func.count(wechatspider_log.aid)).filter(
wechatspider_log.aid == i["aid"]).scalar()
if cnt == 0:
#publishWebsite(str(i['aid']),str(i['digest']),str(i['link']),str(j[1]))
print("文章抓取入库---" + j.fornumname + "------" +
str(i['digest']))
endIndex -= 6
time.sleep(100)
response.close()
session.close()
def GET_SELF_ASSERTED(params, data):
csrf = None
for c in params[0]:
if c.name == "x-ms-cpim-csrf":
csrf = c.value
url = "https://rcmnb2cprod.b2clogin.com/rcmnb2cprod.onmicrosoft.com/B2C_1A_ExternalClient_FrontEnd_Login/SelfAsserted?tx=StateProperties=" + params[
1].decode('utf-8') + "&p=B2C_1A_ExternalClient_FrontEnd_Login"
cookie_handler = HTTPCookieProcessor(params[0])
opener = build_opener(cookie_handler)
opener.addheaders = [
('X-CSRF-TOKEN', csrf),
('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8'),
('Accept', 'application/json, text/javascript, */*; q=0.01'),
('X-Requested-With', 'XMLHttpRequest'),
('Origin', 'https://rcmnb2cprod.b2clogin.com'),
('Referer',
'https://rcmnb2cprod.b2clogin.com/rcmnb2cprod.onmicrosoft.com/B2C_1A_ExternalClient_FrontEnd_Login/oauth2/v2.0/authorize?client_id=ebe6e7b0-3cc3-463d-9389-083c7b24399c&nonce=85e7800e-15ba-4153-ac2f-3e1491918111&redirect_uri=https%3A%2F%2Fici.tou.tv%2Fauth-changed&scope=openid%20offline_access%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Foidc4ropc%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fprofile%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Femail%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fid.write%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fmedia-validation.read%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fmedia-validation%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fmedia-meta%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fmedia-drmt%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Ftoutv-presentation%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Ftoutv-profiling%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fmetrik%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fsubscriptions.write%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fid.account.info%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fid.account.create%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fid.account.modify%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fid.account.reset-password%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fid.account.send-confirmation-email%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fid.account.delete%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Fsubscriptions.validate%20https%3A%2F%2Frcmnb2cprod.onmicrosoft.com%2F84593b65-0ef6-4a72-891c-d351ddd50aab%2Ftoutv&response_type=id_token%20token&response_mode=fragment&prompt=login&state=ZDY1Nzg2YzctZDQ4YS00YWRjLWEwNDMtMGI2MWIyM2UyZjUxfHsiYWN0aW9uIjoibG9naW4iLCJyZXR1cm5VcmwiOiIvIiwiZnJvbVN1YnNjcmlwdGlvbiI6ZmFsc2V9&state_value=ZDY1Nzg2YzctZDQ4YS00YWRjLWEwNDMtMGI2MWIyM2UyZjUxfHsiYWN0aW9uIjoibG9naW4iLCJyZXR1cm5VcmwiOiIvIiwiZnJvbVN1YnNjcmlwdGlvbiI6ZmFsc2V9&ui_locales=fr'
),
('Accept-Language', 'fr-CA,fr;q=0.9,en-CA;q=0.8,en-US;q=0.7,en;q=0.6'),
('Accept-Encoding', 'gzip, deflate, br'),
('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'
)
]
post_data = urlencode(data)
request = Request(url, data=BYTES_PY2(post_data))
request.get_method = lambda: "POST"
response = opener.open(request)
rawresp = handleHttpResponse(response)
print(rawresp)
return params[0], params[1]
def retrieve_by_letter(exchange, letter):
site = 'http://eoddata.com/stocklist/{0}/{1}.htm'.format(exchange, letter)
regex = '/stockquote/{}/(.*).htm'.format(exchange)
s = set()
res = []
opener = build_opener(HTTPCookieProcessor())
response = opener.open(site)
html_page = response.read()
soup = BeautifulSoup(html_page, "html.parser")
# Search for every symbol found on site
for link in soup.findAll('a'):
href = link.get('href')
search = re.search(regex, href)
if search:
symbol = search.group(1)
if symbol not in s:
s.add(symbol)
res.append(symbol)
return res
def __init__(self, args):
""" Start up... """
self.args = args
self.cj = http.cookiejar.MozillaCookieJar(COOKIES_FILENAME)
if os.access(COOKIES_FILENAME, os.F_OK):
self.cj.load(os.getcwd() + "/" + COOKIES_FILENAME)
self.opener = build_opener(HTTPRedirectHandler(),
HTTPHandler(debuglevel=0),
HTTPSHandler(debuglevel=0),
HTTPCookieProcessor(self.cj))
self.opener.addheaders = [
('User-Agent',
('Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36'
)),
('Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
)
]
if not os.path.exists(TMP_DIR):
os.makedirs(TMP_DIR)
def _init_urllib(self):
# Initialize a SSL context for all HTTPS calls
if self.verify_tls:
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1)
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = True
context.load_default_certs()
else:
context = ssl.create_default_context(
) # Should we enforce TLS 1.1 here?
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
# Cookie Jar
self.cj = http.cookiejar.CookieJar()
opener = build_opener(HTTPSHandler(context=context), \
HTTPHandler(), \
HTTPCookieProcessor(self.cj))
install_opener(opener)
def __request__(self):
Cookiejar = CookieJar()
opener = build_opener(HTTPCookieProcessor(Cookiejar))
_header = dict(self.headers.items())
if self.cookie:
_header.update({'Cookie': self.cookie})
req = Request(self.url, headers=_header, origin_req_host=self.host)
error_counter = 0
while error_counter < 3:
try:
res = opener.open(req)
break
except Exception as e:
# traceback.print_exc()
error_counter += 1
time.sleep(0.5)
else:
raise Exception('UrlNotRespond')
return res, Cookiejar._cookies
def check_market_order_sell_trade(self):
while not self.successed_market_order_sell_trade and datetime.datetime.now(
).hour < 6:
opener = build_opener(HTTPCookieProcessor(CookieJar()))
urlencoded_login_data = urlencode(
self.login_post_data).encode('utf-8')
try:
res = opener.open(self.login_url, urlencoded_login_data)
except:
logging.warning("failed to login")
exit("Failed to login.")
urlencoded_get_stock_holdings_data = urlencode(
self.get_stock_holdings_post_data).encode('utf-8')
try:
res = opener.open(self.get_stock_holdings_url,
urlencoded_get_stock_holdings_data)
html = res.read()
shift = html.decode('Shift_JIS')
stock_holdings_num = re.findall(
"(?<=<br>){}(?= )".format(self.stock_id), shift)
if not len(stock_holdings_num):
self.successed_market_order_sell_trade = True
logging.info(
"Successed market order sell trade of stock {}".format(
self.stock_id))
break
else:
self.successed_market_order_sell_trade = False
logging.info(
"have not yet done market order sell trade of stock {}"
.format(self.stock_id))
except:
logging.warning(
"failed to get stock holdings for check market order sell trade"
)
self.successed_market_order_sell_trade = False
return
def fzyz_exam(username, passwd, para_academic_Year, para_KEY):
# 获取登陆Cookies
# 登录时需要POST的数据
data = {
'staffCode': username,
'password': passwd,
'loginRole': '2',
}
# 登录时表单提交到的地址（用开发者工具可以看到）
login_url = 'http://fzyz.net/sys/login.shtml'
# 将数据转化为bytes格式
post_data = urlencode(data).encode('utf-8')
# 构造登录请求
req = Request(login_url, headers={}, data=post_data)
# 构造cookie
cookie = CookieJar()
# 由cookie构造opener
opener = build_opener(HTTPCookieProcessor(cookie))
# 发送登录请求，此后这个opener就携带了cookie，以证明自己登录过
opener.open(req)
# 获取考试详情json
# 考试详情查询界面
exam_url = (f'http://fzyz.net/education/score/score/'
f'getstuExamsByAcademicYearTermYears.shtml?'
f'para.academic_Year={para_academic_Year}&'
f'para.KEY={para_KEY}')
# 获取json
req = Request(exam_url, headers={})
resp = opener.open(req)
exam_js = resp.read().decode('gbk')
# 删除冗余信息，是小屏设备能正常显示。
exam_js = re.sub(r'(?<=高中部)[0-9\-]+', '', exam_js)
exam_js = re.sub(r'高中部', "", exam_js)
exam_js = re.sub(r'第一学期', "", exam_js)
exam_js = re.sub(r'第二学期', "", exam_js)
return exam_js
def cmr_download(urls):
"""Download files from list of urls."""
URS_URL = 'https://urs.earthdata.nasa.gov'
if not urls:
return
url_count = len(urls)
print('Downloading {0} files...'.format(url_count))
credentials = None
for index, url in enumerate(urls, start=1):
if not credentials and urlparse(url).scheme == 'https':
credentials = get_credentials(url)
filename = url.split('/')[-1]
filename = 'nsidc_api_output.zip' if filename.startswith(
'request') else filename
print('{0}/{1}: {2}'.format(
str(index).zfill(len(str(url_count))), url_count, filename))
try:
# In Python 3 we could eliminate the opener and just do 2 lines:
# resp = requests.get(url, auth=(username, password))
# open(filename, 'wb').write(resp.content)
req = Request(url)
if credentials:
req.add_header('Authorization',
'Basic {0}'.format(credentials))
opener = build_opener(HTTPCookieProcessor())
data = opener.open(req).read()
open(filename, 'wb').write(data)
except HTTPError as e:
print('HTTP error {0}, {1}'.format(e.code, e.reason))
except URLError as e:
print('URL error: {0}'.format(e.reason))
except IOError:
raise
except KeyboardInterrupt:
quit()
def parse_nyphil(To):
opener = build_opener(HTTPCookieProcessor())
response = opener.open('https://nyphil.org/rush')
html = response.read()
soup = bs(html, "html.parser")
divs = soup.findAll("div", {"id": "main"})
shows = divs[0].findAll("div",
{"id": "content"})[0].contents[7].contents[8:-6]
results = []
for i in range(len(shows)):
try:
if len(shows[i].contents) > 0:
show = shows[i].contents[0]
i += 1
date = shows[i].split('—')[1][1:]
results.append([date.replace(',', ''), show.replace(',', '')])
except:
pass
check_new_and_notify(results, 'NyPhil.csv', To)
def unrestrict(parameters):
cj_rd = cookielib.CookieJar()
opener_rd = build_opener(HTTPCookieProcessor(cj_rd))
opener_rd.addheaders = [
("Authorization", "Bearer " +
str(xbmcaddon.Addon('script.realdebrid').getSetting('rd_access')))
]
if 'url' in parameters:
link = parameters['url']
else:
link = util.searchDialog("Enter link to unrestrict")
if link:
data_rd = urlencode({'link': link}).encode("utf-8")
error = True
attempts = 0
while error:
try:
resp = opener_rd.open(
'https://api.real-debrid.com/rest/1.0/unrestrict/link',
data_rd)
content = resp.read()
credJSON = json.loads(content)
error = True
return credJSON
except Exception as e:
util.logError("realdebrid error: " + str(e))
attempts = attempts + 1
if attempts > 3:
error = True
util.notify("Unable to unrestrict link")
break
elif "Unauthorized" in e:
refreshToken()
return False
def __init__(self, cachedir = '/tmp', api_host_options = {}, urllist = [], http_debug = False,
cookiejar = None, offline = False, enable_cpio = True):
# set up progress bar callback
if sys.stdout.isatty() and TextMeter:
self.progress_obj = TextMeter(fo=sys.stdout)
else:
self.progress_obj = None
self.cachedir = cachedir
self.urllist = urllist
self.http_debug = http_debug
self.offline = offline
self.cpio = {}
self.enable_cpio = enable_cpio
passmgr = HTTPPasswordMgrWithDefaultRealm()
for host in api_host_options.keys():
passmgr.add_password(None, host, api_host_options[host]['user'], api_host_options[host]['pass'])
openers = (HTTPBasicAuthHandler(passmgr), )
if cookiejar:
openers += (HTTPCookieProcessor(cookiejar), )
self.gr = OscFileGrabber(progress_obj=self.progress_obj)
def create_user(username, password):
try:
print("Trying to create a test user ", username)
cj = CookieJar()
url = "http://{0}/signup".format(webhost)
data = urlencode([("email", ""), ("username", username),
("password", password),
("verify", password)]).encode('utf-8')
request = Request(url=url, data=data)
opener = build_opener(HTTPCookieProcessor(cj))
f = opener.open(request)
users = db.users
user = users.find_one({'_id': username})
if (user == None):
print("Could not find the test user ", username,
"in the users collection.")
return False
print("Found the test user ", username, " in the users collection")
# check that the user has been built
result = f.read()
expr = re.compile("Welcome[\s]+{}".format(username))
if expr.search(result.decode('utf-8')):
return True
print("When we tried to create a user, here is the output we got\n")
print(result)
return False
except:
print("the request to ", url,
" failed, so your blog may not be running.")
return False
def login(user_id, password):
url = 'https://beta.atcoder.jp/login'
cj = cookiejar.LWPCookieJar()
csrf_token = _get_csrf_token(url, cj)
opener = build_opener(HTTPCookieProcessor(cj))
install_opener(opener)
param = {
"username": user_id,
"password": password,
"csrf_token": csrf_token,
"User-Agent": settings.user_agent,
}
res = False
with opener.open(url, data=urlencode(param).encode('ascii')) as fs:
cj.save(settings.path_to_cookie)
html_str = re.compile(r'([\r\n\t]| )').sub('',
fs.read().decode('utf-8'))
if html_str.find('/login') >= 0:
root = PyQuery(html_str)
message = root.find('.alert').eq(0).text().replace('× ', '')
raise Exception(message)
return res
def __init__(self):
self.user = bugtracker_user
self.password = bugtracker_pass
self.login_page = 'https://bugs.archlinux.org/index.php?do=authenticate'
#self.target_page = 'http://bugs.archlinux.org/index.php?events%5B%5D=1&events%5B%5D=13&events%5B%5D=2&events%5B%5D=4&event_number=50&do=reports'
#self.target_page = 'http://bugs.archlinux.org/index.php?events[]=1&events[]=13&events[]=2&events[]=4&fromdate=&todate=&event_number=50&project=0&do=reports&submit='
self.target_page = 'https://bugs.archlinux.org/index.php?events%5B%5D=1&events%5B%5D=13&events%5B%5D=2&events%5B%5D=4&event_number=50&do=reports&project=0'
self.cj = CookieJar()
self.opener = build_opener(HTTPRedirectHandler(),
HTTPHandler(debuglevel=0),
HTTPSHandler(debuglevel=0),
HTTPCookieProcessor(self.cj))
# self.opener.addheaders = [
# ('User-agent', ('Mozilla/4.0 (compatible; MSIE 6.0; '
# 'Windows NT 5.2; .NET CLR 1.1.4322)'))
# ]
self.opener.addheaders = [('User-agent', 'Mozilla/5.0')]
# need this twice - once to set cookies, once to log in...
self.login()
#self.login()
self.old_events = set([])
def get_credentials(url):
URS_URL = 'https://urs.earthdata.nasa.gov'
"""Get user credentials from .netrc or prompt for input."""
credentials = None
errprefix = ''
try:
info = netrc.netrc()
username, account, password = info.authenticators(
urlparse(URS_URL).hostname)
errprefix = 'netrc error: '
except Exception as e:
if (not ('No such file' in str(e))):
print('netrc error: {0}'.format(str(e)))
username = None
password = None
while not credentials:
if not username:
username = get_username()
password = get_password()
credentials = '{0}:{1}'.format(username, password)
credentials = base64.b64encode(
credentials.encode('ascii')).decode('ascii')
if url:
try:
req = Request(url)
req.add_header('Authorization',
'Basic {0}'.format(credentials))
opener = build_opener(HTTPCookieProcessor())
opener.open(req)
except HTTPError:
print(errprefix + 'Incorrect username or password')
errprefix = ''
credentials = None
username = None
password = None
return credentials
