def get_cookie():
# 创建url
login_url = "*****"
# 构建伪装头
headers = {
"User-Agent": UserAgent().chrome
}
# 构建伪装用户登陆
form_data = {
"user": "******"
"pasword":"*****"
}
f_data = urlencode(form_data).encode()
re = Request(login_url, headers=headers, data=f_data) # 构建请求头
cookie_jar = MozillaCookieJar() # 保存cookie文件
handler = HTTPCookieProcessor(cookie_jar)
opener = build_opener(handler) # 构建opener对象
response = opener.open(re) # 准备返回值
# 保存cookie值到文件
cookie_jar.save("cookie.txt", ignore_expires = True, ignore_discard = True)
def url_request(self, action, url, **extra_data):
#get or post
abs_url = "http://%s:%s/%s" % (settings.configs['Server'],
settings.configs['ServerPort'], url)
if action in ('get', 'GET'):
print(abs_url, extra_data)
try:
req = request.Request(abs_url)
req_data = request.urlopen(
req, timeout=settings.configs['RequestTimeout'])
callback = req_data.read() # 获取回来的数据
#print(callback)
return callback
except error.URLError as e:
exit("\033[31;1m%s\033(0m" % e)
elif action in ('post', 'POST'):
# print(abs_url,extra_data['params'])
try:
data_encode = request.urlencode(extra_data['params'])
req = request.Request(url=abs_url, data=data_encode)
res_data = request.urlopen(
req, timeout=settings.configs['RequestTimeout'])
callback = res_data.read()
callback = json.loads(callback.decode("UTF-8"))
print
"\033[31;1m[%s]:[%s]\033[0m response:\n%s" % (action, abs_url,
callback)
return callback
except Exception as e:
print('---exec', e)
exit("\033[31;1m%s\033[0m" % e)
def make_signature(**params):
sign_data = [(k, v) for k, v in params.items() if k != 'signature' or k.startswith("__")]
sorted_parameters = sorted(sign_data, key=lambda kv: kv[0])
sorted_query_string = _standard_url(urlencode(sorted_parameters))
string_to_sign = _standard_url(pathname2url(sorted_query_string))
h = hmac.new(settings.YKT_ACCESS_KEY, string_to_sign, hashlib.sha1)
signature = encodestring(h.digest()).strip()
return signature
def give_access(doc, opener):
parser = FormParser()
parser.feed(doc)
parser.close()
if not parser.form_parsed or parser.url is None:
raise RuntimeError("Something wrong")
if parser.method == "POST":
response = opener.open(parser.url, urllib.urlencode(parser.params))
else:
raise NotImplementedError("Method '{}'".format(parser.method))
return response.geturl()
def give_access(doc, opener):
parser = FormParser()
parser.feed(doc)
parser.close()
if not parser.form_parsed or parser.url is None:
raise RuntimeError("Something wrong")
if parser.method == "POST":
response = opener.open(parser.url, urllib.urlencode(parser.params))
else:
raise NotImplementedError("Method '{}'".format(parser.method))
return response.geturl()
def check_profanity(qu):
print(getproxies())
connection = urlopen(r"http://www.wdyl.com/profanity?q="+ urlencode(str(qu)))
output = connection.read()
if "true" in output:
print("Profanity Alert!!!")
elif "false" in output:
print("This document has no curse words!");
else:
print("Please mannualy check for Profanity");
connection.close()
def check_profanity(qu):
print(getproxies())
connection = urlopen(r"http://www.wdyl.com/profanity?q=" +
urlencode(str(qu)))
output = connection.read()
if "true" in output:
print("Profanity Alert!!!")
elif "false" in output:
print("This document has no curse words!")
else:
print("Please mannualy check for Profanity")
connection.close()
def search(kwd=''):
search_url = "https://www.bing.com/search?q=" + urlencode(kwd)
if kwd == '':
kwd = choice(iJustWantUtoStudy)
wk_page = wikipedia(kwd)
if (wk_page):
return [
'在 Wikipedia 上找到如下信息：\n' + wk_page.url + '\n\n' + wk_page.summary,
wk_page.images[0]
]
else:
return search_url
def wiki_search(term, url=None):
term = urlencode(term.replace(" ", "_"))
url = (url or "https://en.wikipedia.org/w/api.php%s") % ("?action=query&prop=pageprops&format=json&generator=search&cllimit=max&gsrsearch=")
url += term
data = requests.get(url, headers=headers).json()
pages = sorted([v for k,v in data['query']['pages'].items()], key=lambda x: x['index'])
pages = [page for page in pages if not 'disambiguation' in page['pageprops']]
pages = [{key: t[key] for key in t if key not in ["ns", "index"]} for t in pages]
return [p['title'] for p in pages]
def pronounce(word, type=0):
'''
Pronounce English words
Args:
type: American pronunciation(0) or British pronunciation(1)
word: word to pronounce
Returns:
String, link
'''
info('Geting pronunciation:' + word)
api_url = 'http://dict.youdao.com/dictvoice?type=' + str(type) + '&audio='
return api_url + urlencode(word)
def wikipedia_get(bot, nick, chan, arg, root=None):
""" wiki <page> -> Get the first two sentences in a wikipedia article. """
if not arg:
return bot.msg(chan, get_doc())
name = arg
if ' (' in arg:
name = arg.split()[0]
term = arg.replace(" ", "_")
term = urlencode(term)
url = root or "http://en.wikipedia.org"
url += ("/w/api.php?action=parse&format=json&prop=text&page=%s&redirects=" % (term))
response = requests.get(url, headers=headers)
res = response.json()
if res.get("error", None):
return bot.msg(chan, "%s: Error: %s" % (nick, res['error']['info']))
soup = soupify(res['parse']['text']['*'], "lxml")
paragraph = soup.find('p')
url = "http://en.wikipedia.org/wiki/%s"
htmlurl = url % term
for i in soup.find_all('b'):
i.string = "%s" % (bot.hicolor(i.text))
if soup.find("table", id="disambigbox") is not None:
bot.msg(chan, "%s (%s) points to a disambiguation page." % (arg, shorten(htmlurl)))
return
if res['parse'].get('redirects', None):
if res['parse']['redirects'][0].get("tofragment", None):
anchor = res['parse']['redirects'][0]['tofragment']
paragraph = soup.find("span", {"id": anchor}).findNext("p")
htmlurl = url % (res['parse']['redirects'][0]['to']) + "#%s" % (anchor)
elif res['parse']['redirects'][0].get("to", None):
htmlurl = url % (res['parse']['redirects'][0]['to'].replace(" ", "_"))
sentences = bot.state.data["sentence_re"].findall(paragraph.text)[:2]
readmore = bot.style.color("\u2192 %s\x0f" % (bot.state.data['shortener'](bot, htmlurl)), color="blue")
text = ''.join(sentences)
if re.search("\[\d+\]", text):
text = re.sub("\[\d+\]", "", text)
output = "%s %s" % (text, readmore)
bot.msg(chan, '\n'.join(lines(output)))
time.time()
def getweather(location):
try:
url = "http://woeid.rosselliot.co.nz/lookup/"
loc = location
qry = urlencode(loc.lower().replace(",", " "))
req = url+qry
rsp = requests.get(req)
bsp = soupify(rsp.text, "lxml")
tbl = bsp.findAll("table")[0]
ids = [x.attrs for x in list(tbl)[1:]]
for x in ids[0:1]:
w = Weather(x)
if 'condition' in w.__dict__.keys():
yield w.form()
else:
print(w.__dict__)
except Exception as e:
yield "NoSuchPlaceError: '%s' not found (on earth)" %(loc)
def post(url, headers={}, data={}, json={}, as_json=False):
"""
Make a HTTP post request and return response
`Required`
:param str url: URL of target web page
`Optional`
:param dict headers: HTTP request headers
:param dict data: HTTP request POST data
:param dict json: POST data in JSON format
:param bool as_json: return JSON formatted output
"""
try:
import requests
req = requests.post(url, headers=headers, data=data, json=json)
output = req.content
if as_json:
try:
output = req.json()
except:
pass
return output
except ImportError:
import sys
if sys.version_info[0] > 2:
from urllib.request import urlopen, urlencode, Request
else:
from urllib import urlencode
from urllib2 import urlopen, Request
data = urlencode(data)
req = Request(str(url), data=data)
for key, value in headers.items():
req.headers[key] = value
output = urlopen(req).read()
if as_json:
import json
try:
output = json.loads(output)
except:
pass
return output
def post(url, headers={}, data={}, json={}, as_json=False):
"""
Make a HTTP post request and return response
`Required`
:param str url: URL of target web page
`Optional`
:param dict headers: HTTP request headers
:param dict data: HTTP request POST data
:param dict json: POST data in JSON format
:param bool as_json: return JSON formatted output
"""
try:
import requests
req = requests.post(url, headers=headers, data=data, json=json)
output = req.content
if as_json:
try:
output = req.json()
except: pass
return output
except ImportError:
import sys
if sys.version_info[0] > 2:
from urllib.request import urlopen,urlencode,Request
else:
from urllib import urlencode
from urllib2 import urlopen,Request
data = urlencode(data)
req = Request(str(url), data=data)
for key, value in headers.items():
req.headers[key] = value
output = urlopen(req).read()
if as_json:
import json
try:
output = json.loads(output)
except: pass
return output
def _open(cgi, params=None, get=1):
"""Open a handle to SCOP and return it (PRIVATE).
Open a handle to SCOP. cgi is the URL for the cgi script to access.
params is a dictionary with the options to pass to it. get is a boolean
that describes whether a GET should be used.
"""
from urllib.request import urlopen, urlencode
# Open a handle to SCOP.
if params is None:
params = {}
options = urlencode(params)
if get: # do a GET
if options:
cgi += "?" + options
handle = urlopen(cgi)
else: # do a POST
handle = urlopen(cgi, data=options)
return handle
# 3.需要登录的情况
# 3.1 cookie的处理
cookie_support = request.HTTPCookieProcessor(http.cookiejar.CookieJar)
opener = request.build_opener(cookie_support, request.HTTPHandler)
request.install_opener(opener)
# content = request.urlopen('https://www.baidu.com/').read()
# 3.2 表单的处理
postdata = request.urlencode({
'username': '******',
'password': '******',
'continueURI': 'http://www.verycd.com/',
'fk': 'fk',
'login_submit': '登录'
})
# 然后生成http请求，再发送请求：
req = request.Request(
url='http://secure.verycd.com/signin/*/http://www.verycd.com/',
data=postdata
)
result = request.urlopen(req).read()
# 3.3 伪装成浏览器访问
'''某些网站反感爬虫的到访，于是对爬虫一律拒绝请求。这时候我们需要伪装成浏览器，这可以通过修改http包中的header来实现'''
headers = {
if len(sys.argv) > 3:
endpoint = "http://" + sys.argv[3] + "/grafter-ws/formTransform"
print( endpoint )
headers = {}
headers["Content-type"] = "application/x-www-form-urlencoded"
csvString = ""
with open(input_csv, "r") as csv:
csvString = csv.read()
with open(output_rdf, "wb") as rdf:
request = Request(url = endpoint, data=urlencode({'csv': csvString, 'output': output_rdf}), headers=headers)
try:
request.get_method = lambda: "POST";
result = urlopen(request)
print ("HTML response: " , str(result.getcode()))
while True:
buffer = result.read()
if not buffer:
break
rdf.write(buffer)
except HTTPError, error:
print error
print (error.read())
def get_dce_daily(date=None):
"""
获取大连商品交易所日交易数据
Parameters
------
date: 日期 format：YYYY-MM-DD 或 YYYYMMDD 或 datetime.date对象 为空时为当天
Return
-------
DataFrame
大商所日交易数据(DataFrame):
symbol 合约代码
date 日期
open 开盘价
high 最高价
low 最低价
close 收盘价
pre_close 前收盘价
volume 成交量
open_interest 持仓量
turnover 成交额
settle 结算价
pre_settle 前结算价
variety 合约类别
或 None(给定日期没有交易数据)
"""
day = ct.convert_date(date) if date is not None else date.today()
url = ct.DCE_DAILY_URL + '?' + urlencode(
{
"currDate": day.strftime('%Y%m%d'),
"year": day.strftime('%Y'),
"month": str(int(day.strftime('%m')) - 1),
"day": day.strftime('%d')
})
response = urlopen(Request(url, method='POST',
headers=ct.DCE_HEADERS)).read().decode('utf8')
if u'错误：您所请求的网址（URL）无法获取' in response:
return
elif u'暂无数据' in response:
return
data = BeautifulSoup(response, 'html.parser').find_all('tr')
if len(data) == 0:
return
dict_data = list()
for idata in data[1:]:
if u'小计' in idata.text or u'总计' in idata.text:
continue
x = idata.find_all('td')
row_dict = dict()
for i, field in enumerate(ct.DCE_COLUMNS):
if i == 0:
row_dict[field] = ct.DCE_MAP[x[i].text.strip()]
elif i >= 2:
if '-' in x[i].text.strip():
row_dict[field] = 0.0
else:
row_dict[field] = float(x[i].text.strip().replace(',', ''))
else:
row_dict[field] = x[i].text.strip()
dict_data.append(row_dict)
df = pd.DataFrame(dict_data)
df['date'] = day.strftime('%Y%m%d')
df['pre_close'] = df.close - df.change1
df['symbol'] = df.variety + df.month
return df[ct.OUTPUT_COLUMNS]
#endpoint = "http://192.168.11.43:8080/jarfter/webresources/jarCreatorStandAlone"
endpoint = "http://localhost:8088/jarfter/webresources/jarCreatorStandAlone"
headers = {}
#headers["Content-Length"]= "%d" % os.stat(input_clj).st_size
#headers["Content-type"] = "txt/text; charset=utf-8"
headers["Content-type"] = "application/x-www-form-urlencoded"
clojureSource = ""
with open(input_clj, "r") as clj:
clojureSource = clj.read()
#with open(input_clj, "r") as clj:
with open(output_jar, "wb") as jar:
#params = ('clojure': clj)
request = Request(url = endpoint, data=urlencode({'clojure': clojureSource}), headers=headers)
try:
request.get_method = lambda: "POST";
result = urlopen(request)
print ("HTML response: " , str(result.getcode()))
while True:
buffer = result.read()
if not buffer:
break
jar.write(buffer)
except HTTPError, error:
print error
print (error.read())
#headers["Content-Length"]= "%d" % os.stat(input_clj).st_size
#headers["Content-type"] = "txt/text; charset=utf-8"
headers["Content-type"] = "application/x-www-form-urlencoded"
#headers["transformed_filename"] = output_rdf
clojureString = ""
with open(input_clj, "r") as clj:
clojureString = clj.read()
csvString = ""
with open(input_csv, "r") as csv:
csvString = csv.read()
with open(output_rdf, "wb") as rdf:
request = Request(url = endpoint, data=urlencode({'clojure': clojureString, 'csv': csvString, 'transformed_filename': output_rdf}), headers=headers)
try:
request.get_method = lambda: "POST";
result = urlopen(request)
print ("HTML response: " , str(result.getcode()))
while True:
buffer = result.read()
if not buffer:
break
rdf.write(buffer)
except HTTPError, error:
print error
print (error.read())
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Date : 2017-09-23 17:10:37 # @Author : Your Name ([email protected]) # @Link : http://example.org # @Version : $Id$ from urllib.request import urlopen, urlencode from urllib.error import URLError, HTTPError from bs4 import BeautifulSoup # POST values = {'username': "******", 'password': '******'} data = urlencode(values) # values = {} # values['username'] = "******" # values['password'] = "******" # data = urllib.urlencode(values) # print data url = "http://passport.csdn.net/account/login?from=http://my.csdn.net/my/mycsdn" html = urlopen(url, data) html = BeautifulSoup(html.read(), 'lxml') print(html)
def weblogo(self, fname, format="PNG", version="2.8.2", **kwds):
"""Download and save a weblogo using the Berkeley weblogo service.
Requires an internet connection.
The parameters from ``**kwds`` are passed directly to the weblogo server.
Currently, this method uses WebLogo version 3.3.
These are the arguments and their default values passed to
WebLogo 3.3; see their website at http://weblogo.threeplusone.com
for more information::
'stack_width' : 'medium',
'stacks_per_line' : '40',
'alphabet' : 'alphabet_dna',
'ignore_lower_case' : True,
'unit_name' : "bits",
'first_index' : '1',
'logo_start' : '1',
'logo_end': str(self.length),
'composition' : "comp_auto",
'percentCG' : '',
'scale_width' : True,
'show_errorbars' : True,
'logo_title' : '',
'logo_label' : '',
'show_xaxis': True,
'xaxis_label': '',
'show_yaxis': True,
'yaxis_label': '',
'yaxis_scale': 'auto',
'yaxis_tic_interval' : '1.0',
'show_ends' : True,
'show_fineprint' : True,
'color_scheme': 'color_auto',
'symbols0': '',
'symbols1': '',
'symbols2': '',
'symbols3': '',
'symbols4': '',
'color0': '',
'color1': '',
'color2': '',
'color3': '',
'color4': '',
"""
from urllib.request import urlopen, urlencode, Request
if set(self.alphabet) == set("ACDEFGHIKLMNPQRSTVWY"):
alpha = "alphabet_protein"
elif set(self.alphabet) == set("ACGU"):
alpha = "alphabet_rna"
elif set(self.alphabet) == set("ACGT"):
alpha = "alphabet_dna"
else:
alpha = "auto"
frequencies = self.format("transfac")
url = "http://weblogo.threeplusone.com/create.cgi"
values = {
"sequences": frequencies,
"format": format.lower(),
"stack_width": "medium",
"stacks_per_line": "40",
"alphabet": alpha,
"ignore_lower_case": True,
"unit_name": "bits",
"first_index": "1",
"logo_start": "1",
"logo_end": str(self.length),
"composition": "comp_auto",
"percentCG": "",
"scale_width": True,
"show_errorbars": True,
"logo_title": "",
"logo_label": "",
"show_xaxis": True,
"xaxis_label": "",
"show_yaxis": True,
"yaxis_label": "",
"yaxis_scale": "auto",
"yaxis_tic_interval": "1.0",
"show_ends": True,
"show_fineprint": True,
"color_scheme": "color_auto",
"symbols0": "",
"symbols1": "",
"symbols2": "",
"symbols3": "",
"symbols4": "",
"color0": "",
"color1": "",
"color2": "",
"color3": "",
"color4": "",
}
values.update({k: "" if v is False else str(v) for k, v in kwds.items()})
data = urlencode(values).encode("utf-8")
req = Request(url, data)
response = urlopen(req)
with open(fname, "wb") as f:
im = response.read()
f.write(im)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 3 19:19:56 2018
@author: wesleyz
"""
import json
try:
import urllib.request as urllib2
except ImportError:
import urllib2
baseurl = "https://query.yahooapis.com/v1/public/yql?"
yql_query = "select wind from weather.forecast where woeid=2460286"
yql_url = baseurl + urllib2.urlencode({'q': yql_query}) + "&format=json"
result = urllib2.urlopen(yql_url).read()
data = json.loads(result)
print(data['query']['results'])
p = optparse.OptionParser('%prog [(filename|url) [encoding]]',
version='%prog ' + __version__)
args = p.parse_args()[1]
if len(args) > 0:
file_ = args[0]
encoding = None
if len(args) == 2:
encoding = args[1]
if len(args) > 2:
p.error('Too many arguments')
if file_.startswith('http://') or file_.startswith('https://'):
baseurl = file_
req_headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13','Referer': 'http://python.org'}
j = urllib.urlopen(baseurl,urllib.urlencode(req_headers))
text = j.read()
if encoding is None:
try:
from feedparser import _getCharacterEncoding as enc
except ImportError:
enc = lambda x, y: ('utf-8', 1)
encoding = enc(j.headers, text)[0]
if encoding == 'us-ascii':
encoding = 'utf-8'
data = text.decode(encoding, 'ignore')
else:
data = open(file_, 'rb').read()
if encoding is None:
try:
'used': userData['used'],
'isTrial': int(userData['isTrial']),
'days': userData['days'],
'registerDate': userData['registerDate'],
'lastUsedDate': userData['lastUpdate'],
'referenceCount': len(referenceList),
'nameSpaceList': ','.join(nameSpaceList),
'os': str(cmds.about(operatingSystem=True))
}
url = 'https://hook.integromat.com/gnjcww5lcvgjhn9lpke8v255q6seov35'
if sys.version[0] == '3': #python 3
import urllib.parse
params = urllib.parse.urlencode(data)
else: #python 2
params = uLib.urlencode(data)
conn = uLib.urlopen('{}?{}'.format(url, params))
print(conn.read())
#print(conn.info())
# Supporter Coding
# Force Update for 1 month since 1 oct 2020
try:
updateSource = 'source "' + projectDir.replace(
'\\', '/') + '/BRS_DragNDrop_Update.mel' + '";'
mel.eval(updateSource)
except:
pass
# Fix Distance Slider
cmds.floatSlider(distanceS, e=True, minValue=0.01, maxValue=500, value=2)
def wiki_get(bot, nick, chan, arg, searchkey=None, api_url=None):
""" wiki <page> -> Get a summary of a wikipedia article. """
if not arg:
return bot.msg(chan, get_doc())
searchkey = searchkey or arg
# Get the article name, so we can highlight it later in the function
article_name = arg
if (' (' in arg):
article_name = arg.split()[0]
term = urlencode(arg.replace(" ", "_"))
url = (api_url or "https://en.wikipedia.org/w/api.php%s") % ("?action=parse&format=json&prop=text&redirects=&page=")
url += term
response = requests.get(url, headers=headers).json()
if response.get("error", None):
if response['error']['code'] == "missingtitle":
# the page we specified does not exist. We'll guess on the basis of search, then.
return wiki_get(bot, nick, chan, wiki_search(arg, url=api_url)[0], searchkey=arg, api_url=api_url)
else:
return bot.msg(chan, "Error: %s" % (response['error']['info']))
soup = soupify(response['parse']['text']['*'], "lxml")
# Is the page a disambiguation page? If so, search for the page title and get other pages with similar names
if soup.find("table", id="disambigbox") is not None:
return wiki_get(bot, nick, chan, wiki_search(arg, url=api_url)[0], searchkey=arg, api_url=api_url)
# Get the paragraph and the html url from the soup
paragraph = get_paragraph(response, soup)
html_url = get_html_url(response, soup, term)
print(html_url)
# First, highlight all of the bold terms in the page
for i in soup.find_all("b"):
i.string = "%s" % (bot.hicolor(i.text))
# Get the sentences
sentences = ''.join(bot.state.data["sentence_re"].findall(paragraph.text)[:2])
# Replace footnotes in the sentences
sentences = re.sub("\[\d+\]", "", sentences)
# Get other search results
results = wiki_search(searchkey)[1:4]
# Get a shortlink
short_link = bot.state.data["shortener"](bot, html_url)
output = "%s %s" % (sentences, bot.style.color("\u2192%s" % short_link, "blue"))
bot.msg(chan, output)
# other possible pages
bot.msg(chan, bot.style.color("Did you mean: %s" % (' - '.join(results)), "gray"))
server.sendmail('*****@*****.**', '*****@*****.**',
..."""to: [email protected]
...from: [email protected]
...
...baresasdada
...""")
server.quit()
# 处理get请求，不传data，则为get请求
import urllib
from urllib.request import urlopen
from urllib.request import urlencode
url='http://www.xxx.com/login'
data={"username":"******","password":123456}
req_data=urlencode(data) #将字典类型的请求数据转变为url编码
res=urlopen(url+'?'+req_data)#通过urlopen方法访问拼接好的url
#read()方法是读取返回数据内容，decode是转换返回数据的bytes格式为str
res=res.read().decode()
#处理post请求,如果传了data，则为post请求
url='http://www.xxx.com/login'
data={"username":"******","password":123456}
data = urlencode(data)
data=data.encode('ascii')#将url编码类型的请求数据转变为bytes类型
req_data=Request(url,data)#将url和请求数据处理为一个Request对象，供urlopen调用
#read()方法是读取返回数据内容，decode是转换返回数据的bytes格式为str
with urlopen(req_data) as res:
res=res.read().decode()
