def loadHtmlProxy(url,
method="GET",
headers=None,
contenttype='application/x-www-form-urlencoded',
cookie=''):
print "loadHtmlProxy"
#从redisdb获取免费代理
proxys = redisdb.hkeys("proxys_free")
if len(proxys) == 0:
print "proxys is none"
return
proxy = proxys[random.randint(0, len(proxys) - 1)]
ip = str(proxy).split(':')[0]
port = str(proxy).split(':')[1]
try:
h = httplib2.HTTPConnectionWithTimeout(ip, port, timeout=6)
h.request(url, method="GET", headers=headers)
response = h.getresponse()
print str(response.status)
content = response.read()
except Exception, e:
print repr(e)
def try_connect(self) -> None:
"""
Tries to connect to url. If have connection, checks the redirect. If redirect to 'https' protocol and
host is the same, reset scheme in 'href' attribute.
"""
try:
response = request.urlopen(self._options['href'])
# Check if we can connect to host.
if response.getcode() not in range(200, 400):
self.io.warning("Cannot connect to: '{0!s}'".format(
self._options['href']))
else:
# If we connected, check the redirect.
url = self._options['href'].lstrip('(http://)|(https://)')
split_url = url.split('/')
host = split_url[0]
address = '/'.join(split_url[1:])
connection = httplib2.HTTPConnectionWithTimeout(host)
connection.request('HEAD', address)
response = connection.getresponse()
if response.status in range(301, 304):
# If host of redirected is the same, reset 'href' option
if response.getheader('Location').startswith('https://' +
url):
self._options['href'].replace('http://', 'https://')
except error.URLError:
self.io.warning("Invalid url address: '{0!s}'".format(
self._options['href']))
def try_connect(self):
"""
Tries to connect to url. If have connection, checks the redirect. If redirect to 'https' protocol and
host is the same, reset scheme in 'href' attribute.
"""
try:
response = request.urlopen(self._options['href'])
# Check if we can connect to host.
if response.getcode() not in range(200, 400):
print("Warning! - Cannot connect to: '%s'" % self._options['href'])
else:
# If we connected, check the redirect.
url = self._options['href'].lstrip('(http://)|(https://)')
connection = httplib2.HTTPConnectionWithTimeout(url)
connection.request('HEAD', '/')
response = connection.getresponse()
if response.status in range(301, 304):
# If host of redirected is the same, reset 'href' option
if response.getheader('Location').startswith('https://' + url):
self._options['href'].replace('http://', 'https://')
except error.URLError:
print("Warning! - Invalid url address: '%s'" % self._options['href'])
def build(host, **kwargs):
proxy_info = http_proxy.GetHttpProxyInfo()
if callable(proxy_info):
proxy_info = proxy_info('http')
return httplib2.HTTPConnectionWithTimeout(host,
proxy_info=proxy_info,
**kwargs)
def leerPagina(url):
import httplib2
from urllib.parse import urlparse
import os, sys
parse = urlparse(url)
if parse.scheme == "http":
conn = httplib2.HTTPConnectionWithTimeout(parse.netloc, timeout=60)
else:
conn = httplib2.HTTPSConnectionWithTimeout(parse.netloc, timeout=60)
if parse.path == "":
# Si no disponemos de path le ponemos la barra
path = "/"
elif parse.query:
# Si disponemos de path y query, realizamos el montaje
path = "%s?%s" % (parse.path, parse.query)
else:
# Si solo disponemos de path
path = parse.path
# self.conn.putheader("User-agent", 'pywc')
try:
conn.request("GET", path)
response = conn.getresponse()
print("status: %s" % response.status)
print("------------------------------------------")
print("reason: %s" % response.reason)
print("------------------------------------------")
print("headers: %s" % response.getheaders())
print("------------------------------------------")
print("html: %s" % response.read())
except:
print(sys.exc_info()[1])
def html_connect(self, url):
socket.setdefaulttimeout(20)
try:
parse = urlparse(url)
if parse.scheme == "http":
#self.conn=httplib.HTTPConnection(parse.netloc,timeout=60)
self.conn = httplib2.HTTPConnectionWithTimeout(parse.netloc)
else:
#self.conn=httplib.HTTPSConnection(parse.netloc,timeout=60)
self.conn = httplib2.HTTPSConnectionWithTimeout(parse.netloc)
if parse.path == "":
# Si no disponemos de path le ponemos la barra
path = "/"
elif parse.query:
# Si disponemos de path y query, realizamos el montaje
path = "%s?%s" % (parse.path, parse.query)
else:
# Si solo disponemos de path
path = parse.path
self.conn.request("GET", path)
self.response1 = self.conn.getresponse()
self.status = self.response1.status
self.reason = self.response1.reason
self.headers = self.response1.getheaders()
except socket.error:
#errno, errstr = sys.exc_info()[:2]
#if errno == socket.timeout:
#print "There was a timeout"
#else:
#print "There was some other socket error"
self.status = 408
except:
self.status = 404
def get_local_location(refresh=False):
global CURRENT_CITY
if CURRENT_CITY:
return CURRENT_CITY
path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
LOC_CACHE_FILE)
if os.path.exists(path) and not refresh:
with open(path, "r") as cf:
data = cf.read()
data = json.loads(data)
CURRENT_CITY = data["data"]["city"][:-1]
else:
ip = get_local_ip()
if ip:
conn = httplib2.HTTPConnectionWithTimeout("ip.taobao.com",
timeout=20)
url = "/service/getIpInfo.php?ip=" + ip
conn.request("GET", url)
response = conn.getresponse()
if response.code == 200:
with open(path, "w") as cf:
data = json.loads(response.read())
json.dump(data, cf)
# print(data, file=cf)
CURRENT_CITY = data["data"]["city"][:-1]
return CURRENT_CITY
def transition(self, your_words):
appid = '20190817000327288' #你的appid
secretKey = 'QwHBwxGEZc2rsgYDZ3T0' #你的密钥
httpClient = None
myurl = '/api/trans/vip/translate'
q = your_words
fromLang = 'zh'
toLang = 'en'
salt = random.randint(32768, 65536)
sign = appid + q + str(salt) + secretKey
m1 = hashlib.md5()
m1.update(sign.encode('utf8'))
sign = m1.hexdigest()
myurl = myurl + '?appid=' + appid + '&q=' + urllib.parse.quote(
q) + '&from=' + fromLang + '&to=' + toLang + '&salt=' + str(
salt) + '&sign=' + sign
try:
httpClient = httplib2.HTTPConnectionWithTimeout(
'api.fanyi.baidu.com')
httpClient.request('GET', myurl)
# response是HTTPResponse对象
response = httpClient.getresponse()
b = response.read().decode('utf-8')
return (eval(b)['trans_result'][0]['dst'])
# print(eval(b)['trans_result'][0]['dst'])
except Exception as e:
print(e)
finally:
if httpClient:
httpClient.close()
def exec_sparksql(sql):
conn = None
try:
sql_str = sql
logging.info('SQL:\n[%s]\nJSON_SQL:\n[%s]', sql_str,
json.dumps({"sql": sql_str}))
conn = httplib2.HTTPConnectionWithTimeout(
options.spark_server_host,
port=options.spark_server_port,
timeout=60 * 5)
conn.request(method='POST',
url='/sql',
body=json.dumps({
"sql":
sql_str,
'job_id':
datetime.datetime.now().timestamp() * 1000000
}))
resp = conn.getresponse()
result = resp.read().decode(encoding='UTF-8', errors='strict')
if resp.code == 500:
return json.dumps({"error": result})
else:
return '{"data": %s}' % result
except:
logging.exception("SparkSQL job execution error")
return json.dumps({'error': str(sys.exc_info()[1])})
finally:
conn.close()
def _test():
connection = httplib2.HTTPConnectionWithTimeout(
'localhost', server.port)
connection.request('GET', '/')
response = connection.getresponse()
response.read()
connection.close()
def __init__(self, url, timeout):
r = urlparse(url)
self.conn = httplib2.HTTPConnectionWithTimeout(host=r.hostname,
port=r.port,
timeout=timeout)
self.url = url
self.path = r.path
def getjobinfo(servername, joburl):
conn = httplib2.HTTPConnectionWithTimeout(servername)
conn.request("GET", joburl + "api/json")
r1 = conn.getresponse()
data1 = r1.read()
json_decoder = JSONDecoder()
conn.close()
return json_decoder.decode(data1.decode())
def getjenkinsjobs(jenkinsserver):
conn = httplib2.HTTPConnectionWithTimeout(jenkinsserver)
conn.request("GET", "/jenkins/api/json")
r1 = conn.getresponse()
data1 = r1.read()
json_decoder = JSONDecoder()
conn.close()
return json_decoder.decode(data1.decode())
def run(self):
sleep(ENV.TIMEOUT) # sec
for url in self.urls:
connect = httplib2.HTTPConnectionWithTimeout(url)
connect.request("GET", "/")
c = connect.getresponse()
print(url, c.status, c.reason)
print("Call & Wake up url")
def checkURLexists(form, field):
try:
p = urlparse.urlparse(field.data)
conn = httplib2.HTTPConnectionWithTimeout(p.netloc)
conn.request('HEAD', p.path)
resp = conn.getresponse()
except:
raise ValidationError(
"URL doesn't resolve. Are you sure it's correct?")
def record_to_text(data,
request_id=uuid.uuid4().hex,
topic='notes',
lang='ru-RU',
key=KEY):
log("Считывание блока байтов")
chunks = read_chunks(CHUNK_SIZE, data)
log("Установление соединения")
connection = httplib2.HTTPConnectionWithTimeout(YANDEX_ASR_HOST)
url = YANDEX_ASR_PATH + '?uuid=%s&key=%s&topic=%s&lang=%s' % (
request_id, key, topic, lang)
log("Запрос к Yandex API")
connection.connect()
connection.putrequest('POST', url)
connection.putheader('Transfer-Encoding', 'chunked')
connection.putheader('Content-Type', 'audio/x-pcm;bit=16;rate=16000')
connection.endheaders()
log("Отправка записи")
for chunk in chunks:
connection.send(('%s\r\n' % hex(len(chunk))[2:]).encode())
connection.send(chunk)
connection.send('\r\n'.encode())
connection.send('0\r\n\r\n'.encode())
response = connection.getresponse()
log("Обработка ответа сервера")
if response.code == 200:
response_text = response.read()
xml = XmlElementTree.fromstring(response_text)
if int(xml.attrib['success']) == 1:
max_confidence = -float("inf")
text = ''
for child in xml:
if float(child.attrib['confidence']) > max_confidence:
text = child.text
max_confidence = float(child.attrib['confidence'])
if max_confidence != -float("inf"):
return text
else:
raise SpeechException('No text found.\n\nResponse:\n%s' %
(response_text))
else:
raise SpeechException('No text found.\n\nResponse:\n%s' %
(response_text))
else:
raise SpeechException('Unknown error.\nCode: %s\n\n%s' %
(response.code, response.read()))
def _make_request(self, method, bucket='', key='', query_args={}, headers={}, data='', metadata={}):
key = key.replace('\\', '')
server = ''
if bucket == '':
server = self.server
elif self.calling_format == CallingFormat.SUBDOMAIN:
server = "%s.%s" % (bucket, self.server)
elif self.calling_format == CallingFormat.VANITY:
server = bucket
else:
server = self.server
path = ''
if (bucket != '') and (self.calling_format == CallingFormat.PATH):
path += "/%s" % bucket
# add the slash after the bucket regardless
# the key will be appended if it is non-empty
path += "/%s" % urllib.quote_plus(key, '/')
# build the path_argument string
# add the ? in all cases since
# signature and credentials follow path args
if len(query_args):
path += "?" + query_args_hash_to_string(query_args)
is_secure = self.is_secure
host = "%s:%d" % (server, self.port)
while True:
if (is_secure):
connection = httplib.HTTPSConnectionWithTimeout(host)
else:
connection = httplib.HTTPConnectionWithTimeout(host)
final_headers = merge_meta(headers, metadata);
# add auth header
self._add_aws_auth_header(final_headers, method, bucket, key, query_args)
connection.request(method, path, data, final_headers)
resp = connection.getresponse()
if resp.status < 300 or resp.status >= 400:
return resp
# handle redirect
location = resp.getheader('location')
if not location:
return resp
# (close connection)
resp.read()
scheme, host, path, params, query, fragment \
= urlparse.urlparse(location)
if scheme == "http": is_secure = True
elif scheme == "https": is_secure = False
else: raise invalidURL("Not http/https: " + location)
if query: path += "?" + query
def check_internet_connection():
connection = httplib2.HTTPConnectionWithTimeout("www.google.com",
timeout=5)
try:
connection.request("HEAD", "/")
connection.close()
return True
except httplib2.HttpLib2Error:
connection.close()
return False
def _check_active(self):
"""Check a wsgi service is active by making a GET request."""
port = int(os.read(self.pipein, 5))
conn = httplib2.HTTPConnectionWithTimeout("localhost", port)
try:
conn.request("GET", "/")
resp = conn.getresponse()
return resp.status == 200
except socket.error:
return False
def check_for_https(url, port):
print(url)
print(port)
conn = httplib2.HTTPConnectionWithTimeout(url,
int(port),
timeout=CONNECTION_TIMEOUT)
conn.request("HEAD", "/")
if conn.getresponse():
return False
else:
return True
def speech_to_text(filename=None, bytes=None, request_id=uuid.uuid4().hex, topic='notes', lang='ru-RU',
key=YANDEX_API_KEY):
# Формирование тела запроса к Yandex API
url = YANDEX_ASR_PATH + '?uuid=%s&key=%s&topic=%s&lang=%s' % (
request_id,
key,
topic,
lang
)
# Считывание блока байтов
chunks = read_chunks(CHUNK_SIZE, bytes)
# Установление соединения и формирование запроса
connection = httplib2.HTTPConnectionWithTimeout(YANDEX_ASR_HOST)
connection.connect()
connection.putrequest('POST', url)
connection.putheader('Transfer-Encoding', 'chunked')
connection.putheader('Content-Type', 'audio/x-wav')
connection.endheaders()
# Отправка байтов блоками
for chunk in chunks:
connection.send(('%s\r\n' % hex(len(chunk))[2:]).encode())
connection.send(chunk)
connection.send('\r\n'.encode())
connection.send('0\r\n\r\n'.encode())
response = connection.getresponse()
# Обработка ответа сервера
if response.code == 200:
response_text = response.read()
xml = XmlElementTree.fromstring(response_text)
if int(xml.attrib['success']) == 1:
max_confidence = - float("inf")
text = ''
for child in xml:
if float(child.attrib['confidence']) > max_confidence:
text = child.text
max_confidence = float(child.attrib['confidence'])
if max_confidence != - float("inf"):
return text
else:
raise Exception('No text found.\n\nResponse:\n%s' % (response_text))
else:
raise Exception('No text found.\n\nResponse:\n%s' % (response_text))
else:
raise Exception('Unknown error.\nCode: %s\n\n%s' % (response.code, response.read()))
def voiceToText(voiceBytes):
""" Функция преобразования голосового сообщения в текст
:param voiceBytes: голосовое сообщение в виде набора байтов
:return: текст сообщения
:rtype: string
:raise: SpeechException
"""
url = YANDEX_ASR_PATH + '?uuid={0}&key={1}&topic={2}&lang={3}'.format(
uuid.uuid4().hex,
YANDEX_SPEECH_TOKEN,
'notes',
'ru-RU'
)
connection = httplib2.HTTPConnectionWithTimeout(YANDEX_ASR_HOST)
connection.connect()
connection.putrequest('POST', url)
connection.putheader('Content-Length', str(len(voiceBytes)))
connection.putheader('Content-Type', 'audio/ogg;codecs=opus')
connection.endheaders()
connection.send(voiceBytes)
response = connection.getresponse()
if response.code == 200:
responseText = response.read()
xml = XmlElementTree.fromstring(responseText)
if int(xml.attrib['success']) == 1:
maxConf = - float("inf")
text = ""
for child in xml:
if float(child.attrib['confidence']) > maxConf:
text = child.text
maxConf = float(child.attrib['confidence'])
if maxConf != - float("inf"):
return text
else:
raise SpeechException("No text found.\nResponse: {0}".format(responseText))
else:
raise SpeechException("No text found.\nResponse: {0}".format(responseText))
else:
raise SpeechException("Unknown error.\nCode: {0}\n{1}".format(
response.code, response.read()
))
def validate_url(url):
import httplib2
import urllib.parse
try:
p = urllib.parse.urlparse(url)
conn = httplib2.HTTPConnectionWithTimeout(p.netloc)
conn.request('HEAD', p.path)
resp = conn.getresponse()
return resp.status >= 400
except Exception as e:
print("\nException validating URL: %s (%s)" % (url, e))
return 1
def testInternet():
'''
Check Internet Connection
'''
# attempt a connection to google and report success or not
conn = httplib2.HTTPConnectionWithTimeout("www.google.com", timeout=None)
try:
conn.request("HEAD", "/")
return True
except socket.gaierror:
return False
def checkip(ip):
checkurl = str(ip) + ':80'
getcontent = ""
httplib2.socket.setdefaulttimeout(5)
conn = httplib2.HTTPConnectionWithTimeout(checkurl)
try:
conn.request('GET', '/', headers={'Host': appdomain})
r = conn.getresponse()
getcontent = r.read(15)
finally:
if getcontent == b'<!DOCTYPE html>':
print(str(ip) + '[OK]')
else:
print(str(ip) + '[Error]')
def check_ip(ip):
check_url = ip + ":80"
getcontent = ""
httplib2.socket.setdefaulttimeout(5) # 定义http连接超时时间为5s
conn = httplib2.HTTPConnectionWithTimeout(check_url) # 创建http连接对象
try:
conn.request("GET", "/", headers={"Host":
appdomain}) # 发起URL请求，添加host主机头
r = conn.getresponse()
getcontent = r.read(15) # 获取URL页面前15个字符作为可用性校验
finally:
if getcontent == "<!doctype html>": # 监控URL页内容一般为事先定义好的，例如”HTTP200“等
print(ip, " [OK]")
else:
print(ip, " [Error]") # 可以在这里增加报警程序，比如发送邮件
def checkip(ip: object) -> object:
checkurl = ip + ":80"
getcontent = ""
httplib2.socket.setdefaulttimeout(5) #超时
conn = httplib2.HTTPConnectionWithTimeout(checkurl)
try:
conn.request("GET", "/", headers={"host": appdomain})
result = conn.getresponse()
getcontent = result.read()
finally:
# if getcontent.find("<!doctype html")!=-1:
if getcontent.find("<!doctype html") == 0:
print("ipOK", checkurl)
else:
print("ipNO", checkurl)
def send_sms(self, mobile, sms_info):
"""发送手机通知短信，用的是-互亿无线-的测试短信"""
host = "106.ihuyi.com"
sms_send_uri = "/webservice/sms.php?method=Submit"
account = "C59782899"
pass_word = "19d4d9c0796532c7328e8b82e2812655"
params = parse.urlencode(
{'account': account, 'password': pass_word, 'content': sms_info, 'mobile': mobile, 'format': 'json'}
)
headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
conn = httplib2.HTTPConnectionWithTimeout(host, port=80, timeout=30)
conn.request("POST", sms_send_uri, params, headers)
response = conn.getresponse()
response_str = response.read()
conn.close()
return response_str
def GetCnblogsServerTime():
"""获取cnblogs服务器时间
GetCnblogsServerTime() -> list
NOTE: 原理是通过服务器头文件响应获取服务器时间
"""
conn = httplib2.HTTPConnectionWithTimeout('http://st.zzint.com/login.jsp')
conn.request('GET', '/')
response = conn.getresponse()
ts = response.getheader('date')
ltime = time.strptime(ts[5:25], '%d %b %Y %H:%M:%S') #按照特定时间格式将字符串转换为时间类型
serverTime = time.strftime(
'%H:%M:%S', time.localtime(time.mktime(ltime) + 8 * 3600)).split(
':') #将GMT时间转换为北京时间并以列表形式返回, -> [ hour, minute, second ]
print(serverTime)
return serverTime
def tampilan():
if request.method == "GET":
return render_template('Home.html')
else:
token = "Bearer " + request.form['Token']
tipedata = request.form['Tipedata']
conn = httplib2.HTTPConnectionWithTimeout("localhost:5001")
headers = {
"Authorization": token
}
conn.putrequest("GET", "/api/getdata" + tipedata)
conn.putheader("Authorization", token)
conn.endheaders()
response = conn.getresponse()
Data = response.read()
print (Data)
return render_template('Home.html', Data=Data)
