def try_different_adaptors():
"""
This method didn't return any positive results but it was work a try. We were
looking to see if using different SSLAdapters with the session object made a
difference. It certainly has different behaviour on different platforms but did
not fix the issue.
"""
from requests_toolbelt import SSLAdapter
import ssl
ssl3 = SSLAdapter(ssl.PROTOCOL_SSLv23)
tls1 = SSLAdapter(ssl.PROTOCOL_TLSv1)
tls11 = SSLAdapter(ssl.PROTOCOL_TLSv1_1)
print 'TLSv1'
session = session_post(adaptor=tls1)
session_post(session=session)
print 'TLSv1'
session = session_post(adaptor=tls11)
session_post(session=session)
print 'SSLv3'
session = session_post(adaptor=ssl3)
session_post(session=session)
def get_html_withheader(url, header):
try:
requests.adapters.DEFAULT_RETRIES = 5
s = requests.session()
s.keep_alive = False
try:
requests.packages.urllib3.disable_warnings()
html = s.get(url, timeout=30)
html.encoding = html.apparent_encoding
html.raise_for_status()
except:
adapter = SSLAdapter('TLSv1')
s.mount('https://', adapter)
html = s.get(url, verify=False, timeout=30, headers=header)
html.raise_for_status()
html.raise_for_status()
content = html.text
return content
except Exception as e:
My_logger.my_logger.error("访问网址{}出错header是{}，请检查后再试!".format(
url, header))
My_logger.my_logger.error(e)
time.sleep(3)
sys.exit(0)
return None
def __init__(self):
""" Create a UnifiClient object. """
self.version = settings.UNIFI_VERSION
self.site_id = settings.UNIFI_SITE_ID
self._unifiUser = settings.UNIFI_USER
self._unifiPass = settings.UNIFI_PASSWORD
self._unifiServer = settings.UNIFI_SERVER
self._unifiPort = settings.UNIFI_PORT
self._cookie_file = "/tmp/unifi_cookie"
# Use a Session object to handle cookies.
self._session = requests.Session()
cj = cookielib.LWPCookieJar(self._cookie_file)
# Load existing cookies (file might not yet exist)
try:
cj.load()
except:
pass
self._session.cookies = cj
# Use an SSLAdapter to work around SSL handshake issues.
self._session.mount(self._get_resource_url(), SSLAdapter(ssl.PROTOCOL_SSLv23))
pass
def text_to_speech(self):
print("はいってるぞよ！！！") # デバッグコメント
url = 'https://api.voicetext.jp/v1/tts' # apiのurl
API_KEY = '6otvdxnu4z5liju0' # API_KEY
self.decide_emotion()
self.decide_emotion_level()
parameters = { # パラメータ
'text': self.text,
'speaker': self.speaker,
"emotion": self.emotion,
"emotion_level": self.emotion_level,
"pitch": self.pitch,
"speed": self.speed,
"volume": self.volume
}
s = requests.Session()
s.mount(url, SSLAdapter(ssl.PROTOCOL_TLSv1))
r = s.post(url, params=parameters, auth=(API_KEY, ''))
print("status code:", r.status_code)
if r.status_code != 200: # エラーが起きた時のprintデバッグ
print("error:", r.json()['error']['message'])
sys.exit()
n = 1
while os.path.isfile("./voice/test" + str(n) + ".wav"):
n += 1
with open("./voice/test" + str(n) + ".wav", 'wb') as f: # wavファイルの生成
f.write(r.content)
return "./voice/test" + str(n) + ".wav"
def handle(self):
# print("1")
# print("from conn:", self.request)
conn = self.request # 每个客户端的连接
# print(self.client_address)
# conn.sendall(b"conn success!") # 回复客户端连接已建立
while True:
recv = conn.recv(1024)
if len(recv) <= 0:
break
json_data = str(recv, encoding='utf-8')
data = json.loads(json_data)
# print(type(data), data)
# 设置HTTPS
adapter = SSLAdapter('TLSv1.2') # 设置证书验证方式为TLSv1.2
r = requests.Session()
r.mount('https://', adapter) # 设置HTTPS的SSL适配器
ca_file = '../certs/chain-ca.pem' # 设置根证书
# 终端入网验证
dev_verify_api = 'https://127.0.0.1:5000/api/dev/dev_verify'
r = requests.post(dev_verify_api, json=data,
verify=ca_file) # 指定根证书
# print(r.json())
json_str = json.dumps(r.json())
conn.sendall(bytes(json_str, encoding='utf-8'))
def test_proxies(self, ProxyManager):
a = SSLAdapter('SSLv3')
a.proxy_manager_for('http://127.0.0.1:8888')
assert ProxyManager.call_count == 1
kwargs = ProxyManager.call_args_list[0][1]
assert kwargs['ssl_version'] == 'SSLv3'
def req_post(url, param='', type='json', token=True):
if token:
if param:
param.update({'token': leanote_token})
else:
param = {'token': leanote_token}
s = requests.Session()
if leanote_host.startswith('https'):
s.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
r = s.post(leanote_host + '/api/' + url, data=param)
if r.status_code == requests.codes.ok:
if type == 'json':
if is_ok(r.text):
rj = json.loads(r.text)
# if 'Msg' in rj:
# rj=rj['Msg']
return rj
else:
print '[Err] requests to url %s fail' % (r.url)
return None
else:
print '[Err] connect to url %s fail, error code %d ' % (r.url,
r.status_cde)
return None
def __init__(self):
self.os_name = platform.system() # 获取系统平台名，Linux、Windows
self.ca_file = "../certs/chain-ca.pem" # 设置根证书
adapter = SSLAdapter('TLSv1.2') # 设置证书验证方式为TLSv1.2
r = requests.Session()
r.mount('https://', adapter) # 设置HTTPS的SSL适配器
self.soft_list = list()
self.user_list = list()
def __init__(
self,
site_url, # type: str
auth=None, # type: Optional[Any]
authcookie=None, # type: Optional[requests.cookies.RequestsCookieJar]
verify_ssl=True, # type: bool
ssl_version=None, # type: Optional[float]
huge_tree=False, # type: bool
timeout=None, # type: Optional[int]
):
self.site_url = site_url
self._verify_ssl = verify_ssl
self._session = requests.Session()
if ssl_version is not None:
self._session.mount("https://", SSLAdapter(ssl_version))
self._session.headers.update(
{"user-agent": "shareplum/%s" % __version__})
if authcookie is not None:
self._session.cookies = authcookie
else:
self._session.auth = auth
self.huge_tree = huge_tree
self.timeout = timeout
self.last_request = None # type: Optional[str]
self._services_url = {
"Alerts": "/_vti_bin/Alerts.asmx",
"Authentication": "/_vti_bin/Authentication.asmx",
"Copy": "/_vti_bin/Copy.asmx",
"Dws": "/_vti_bin/Dws.asmx",
"Forms": "/_vti_bin/Forms.asmx",
"Imaging": "/_vti_bin/Imaging.asmx",
"DspSts": "/_vti_bin/DspSts.asmx",
"Lists": "/_vti_bin/lists.asmx",
"Meetings": "/_vti_bin/Meetings.asmx",
"People": "/_vti_bin/People.asmx",
"Permissions": "/_vti_bin/Permissions.asmx",
"SiteData": "/_vti_bin/SiteData.asmx",
"Sites": "/_vti_bin/Sites.asmx",
"Search": "/_vti_bin/Search.asmx",
"UserGroup": "/_vti_bin/usergroup.asmx",
"Versions": "/_vti_bin/Versions.asmx",
"Views": "/_vti_bin/Views.asmx",
"WebPartPages": "/_vti_bin/WebPartPages.asmx",
"Webs": "/_vti_bin/Webs.asmx",
} # type: Dict[str, str]
self.site_info = self.get_site()
self.users = self.get_users()
self.version = "2007" # For Debugging
def __init__(self,
site_url,
auth=None,
authcookie=None,
verify_ssl=True,
ssl_version=None,
huge_tree=False,
timeout=None):
self.site_url = site_url
self._verify_ssl = verify_ssl
self._session = requests.Session()
if ssl_version is not None:
self._session.mount('https://', SSLAdapter(ssl_version))
self._session.headers.update(
{'user-agent': 'shareplum/%s' % __version__})
if authcookie is not None:
self._session.cookies = authcookie
else:
self._session.auth = auth
self.huge_tree = huge_tree
self.timeout = timeout
self.last_request = None
self.xml_headers = {'accept': 'application/atom+xml'}
self._services_url = {
'Alerts': '/_vti_bin/Alerts.asmx',
'Authentication': '/_vti_bin/Authentication.asmx',
'Copy': '/_vti_bin/Copy.asmx',
'Dws': '/_vti_bin/Dws.asmx',
'Forms': '/_vti_bin/Forms.asmx',
'Imaging': '/_vti_bin/Imaging.asmx',
'DspSts': '/_vti_bin/DspSts.asmx',
'Lists': '/_vti_bin/lists.asmx',
'Meetings': '/_vti_bin/Meetings.asmx',
'People': '/_vti_bin/People.asmx',
'Permissions': '/_vti_bin/Permissions.asmx',
'SiteData': '/_vti_bin/SiteData.asmx',
'Sites': '/_vti_bin/Sites.asmx',
'Search': '/_vti_bin/Search.asmx',
'UserGroup': '/_vti_bin/usergroup.asmx',
'Versions': '/_vti_bin/Versions.asmx',
'Views': '/_vti_bin/Views.asmx',
'WebPartPages': '/_vti_bin/WebPartPages.asmx',
'Webs': '/_vti_bin/Webs.asmx',
'RequestDigest': '/_api/contextinfo',
'RestWeb': '/_api/web/'
}
self.users = self.GetUsers()
def get_method(context, url, user_role, check_response_code = True):
erase_last_response(context)
get_bearer_token(context,user_role)
#update_token(context, user_role)
s = requests.Session()
s.headers = context.config.userdata.get("bake_api_token")
s.mount(url, SSLAdapter(ssl.PROTOCOL_TLSv1))
response = s.get(url, verify=VERIFY_SSL_CERT)
set_api_response(context, response)
if check_response_code:
assert response.status_code == 200, "Error, status code is {code}, expected 200. The error is {error}".format(code=response.status_code, error=response.text)
print('Response code: '+str(response.status_code))
def __init__(self, api_key=None):
if api_key is None:
raise ApiKeyMissing
self._api_key = api_key
self._baseurl = 'https://api.meraki.com/api/v0'
self._headers = {
'X-Cisco-Meraki-API-Key': str(self._api_key),
'Content-type': 'application/json'
}
self._session = Session()
self._session.mount(self._baseurl, SSLAdapter(ssl.PROTOCOL_SSLv23))
def __init__(self, host='127.0.0.1', port=8443, version='v5'):
self.host = host
self.port = port
self.version = version
self.logged_in = False
self._username = None
self._password = None
self._site = None
self._baseurl = 'https://{}:{}'.format(self.host, self.port)
self._session = Session()
self._session.mount(self._baseurl, SSLAdapter(ssl.PROTOCOL_SSLv23))
def patch_method(context, url, payload, check_response_code=True, user_role = 'blippar_admin'):
erase_last_response(context)
get_bearer_token(context,user_role)
#update_token(context, user_role)
s = requests.Session()
s.headers = context.config.userdata.get("bake_api_token")
s.mount(url, SSLAdapter(ssl.PROTOCOL_TLSv1))
print('Running PATCH: '+url, "By role:", user_role)
print('Using payload: '+str(payload))
response = s.patch(url, data=payload, verify=VERIFY_SSL_CERT)
set_api_response(context, response)
if check_response_code:
assert response.status_code == 200 or response.status_code == 201, "Error, status code is {code}, expected 200 or 201. The error is {error}".format(code=response.status_code, error=response.text)
print('Response code: '+str(response.status_code))
def connect_server():
# 设置HTTPS
adapter = SSLAdapter('TLSv1.2') # 设置证书验证方式为TLSv1.2
r = requests.Session()
r.mount('https://', adapter) # 设置HTTPS的SSL适配器
ca_file = '../certs/chain-ca.pem' # 设置根证书
connect_server_api = "https://127.0.0.1:5000/"
while True:
try:
req = requests.get(connect_server_api, verify=ca_file)
if req.status_code == 200:
break
except Exception as e:
print(e)
time.sleep(10)
print("server connected!")
def __init__(self, host='127.0.0.1', port=8443, version='v5'):
""" host: (str) ipaddress of hosts, port: (int) tcp port, version: (str) default: 'v5' """
self.host = host
self.port = port
self.version = version
self.logged_in = False
self._username = None
self._password = None
self._site = None
self._baseurl = 'https://{}:{}'.format(self.host, self.port)
logger.debug(f'open & mount session to {self._baseurl}')
self._session = Session()
logger.trace(f'mount session begins')
self._session.mount(self._baseurl, SSLAdapter(ssl.PROTOCOL_SSLv23))
logger.trace(f'mount session complete')
def delete_method(context, url, user_role, check_response_code = True ):
erase_last_response(context)
get_bearer_token(context,user_role)
#update_token(context, user_role)
s = requests.Session()
s.headers = context.config.userdata.get("bake_api_token")
s.mount(url, SSLAdapter(ssl.PROTOCOL_TLSv1))
response = s.delete(url, verify=VERIFY_SSL_CERT)
set_api_response(context, response)
print('Running DELETE: '+url, "By role:", user_role)
if check_response_code:
assert response.status_code == 204 or response.status_code == 200, "Error, status code is {code}, expected 200 or 204. The error is {error}".format(code=response.status_code, error=response.text)
else:
#case of error 403
if user_role in USERS.keys() and user_role != 'blippar_admin' and response.status_code == 403:
print("Skip resp. check on if user is not admin and code is 403")
return response.status_code
print('Response code from delete: '+str(response.status_code))
def downLoadCsv(sourceFilePath: str):
proxy = {
# "https": "124.93.201.59:59618"
"https": getProxyURL()
}
s = requests.Session()
s.mount('https://', SSLAdapter('TLSv1'))
s.proxies = proxy
s.keep_alive = False
s.headers = {"User-Agent": USER_AGENT}
r = s.get(DOWNLOAD_URL, timeout=500)
sourceFile = f"{sourceFilePath}/raw-dimensions.csv"
with open(sourceFile, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return sourceFile
def _make_call_to_kobra(payload):
user = str(os.environ.get('KOBRA_USER'))
password = str(os.environ.get('KOBRA_PASSWORD'))
adapter = SSLAdapter('SSLv23')
s = requests.Session()
s.mount('https://', adapter)
r = s.post("https://kobra.ks.liu.se/students/api",
auth=(user, password),
data=payload)
if not r.status_code == requests.codes.ok:
if r.status_code == 404:
raise LiuNotFoundError
else:
raise LiuGetterError
r.encoding = "iso-8859-1"
result_dict = json.loads(r.text, encoding="iso-8859-1")
return result_dict
def getVideos(cls, channel, last_updated, proxy=''):
'''
通过 RSS 获取 YouTube 频道最近的视频列表
Args:
channel: 频道 ID
last_updated: 时间戳，只爬取该时间以后视频
proxy(optional): http/https 代理地址，e.g. '127.0.0.1:1080'
Returns:
包含视频 ID 的 list
'''
logging.debug('正在爬取 channel {} 视频 ...'.format(channel))
url = 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel
s = requests.Session()
s.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
requests.packages.urllib3.disable_warnings()
ssl._create_default_https_context = ssl._create_unverified_context
if proxy != '':
proxies = {'http': 'http://' + proxy, 'https': 'https://' + proxy}
response = s.get(url, verify=False, proxies=proxies)
else:
response = s.get(url, verify=False)
if response.status_code != 200:
raise 'requests 错误 {}'.format(response.status_code)
resp = response.text
r1 = r'<yt:videoId>(.*)</yt:videoId>'
r2 = r'<published>(.*)</published>\n\s*<updated>'
allVids = re.findall(r1, resp)
pubTimes = re.findall(r2, resp)
if len(allVids) != len(pubTimes):
raise 'allVids 与 pubTimes 长度不匹配'
res = []
for i in range(len(allVids) - 1, -1, -1): # 从旧到新
if pubTimes[i] > last_updated:
res.append(allVids[i])
logging.debug('共进队 {} 条视频：'.format(len(res)))
return res
def post_method(context, url, user_role, payload, check_response_code = True, files = None, custom_headers = None):
erase_last_response(context)
get_bearer_token(context,user_role)
#update_token(context, user_role)
s = requests.Session()
s.headers = context.config.userdata.get("bake_api_token")
if custom_headers != None:
custom_headers = ast.literal_eval(custom_headers)
for key, value in dict(custom_headers).iteritems():
s.headers[key] = value
print(s.headers)
s.mount(url, SSLAdapter(ssl.PROTOCOL_TLSv1))
print('Running POST: '+url, "By role:", user_role)
if files:
response = s.post(url, data=payload, files=files, verify=VERIFY_SSL_CERT)
else:
response = s.post(url, data=payload, verify=VERIFY_SSL_CERT)
set_api_response(context, response)
if check_response_code:
assert response.status_code == 200 or response.status_code == 201, "Error, status code is {code}, expected 200 or 201. The error is {error}".format(code=response.status_code, error=response.text)
print('Response code: '+str(response.status_code))
def login(url, username, password, domain="/", verifysslcert=False):
"""
Login and obtain a session to be used for subsequent API calls
Wrong username/password leads to HTTP error code 531
"""
args = {}
args["command"] = 'login'
args["username"] = username
args["password"] = password
args["domain"] = domain
args["response"] = "json"
sessionkey = ''
session = requests.Session()
session.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
try:
resp = session.post(url, params=args, verify=verifysslcert)
except requests.exceptions.ConnectionError as e:
writeError("Connection refused by server: %s" % e)
return None, None
if resp.status_code == 200:
sessionkey = resp.json()['loginresponse']['sessionkey']
elif resp.status_code == 405:
writeError("Method not allowed, unauthorized access on URL: %s" % url)
session = None
sessionkey = None
elif resp.status_code == 531:
writeError("Error authenticating at %s using username: %s"
", password: %s, domain: %s" %
(url, username, password, domain))
session = None
sessionkey = None
else:
resp.raise_for_status()
return session, sessionkey
def login(url, username, password, domain="/", verifysslcert=False):
"""
Login and obtain a session to be used for subsequent API calls
Wrong username/password leads to HTTP error code 531
"""
args = {}
args["command"] = 'login'
args["username"] = username
args["password"] = password
args["domain"] = domain
args["response"] = "json"
sessionkey = ''
session = requests.Session()
session.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
try:
resp = session.post(url, params=args, verify=verifysslcert)
except requests.exceptions.ConnectionError, e:
writeError("Connection refused by server: %s" % e)
return None, None
def __init__(self, site_url, auth=None, verify_ssl=True, ssl_version=None):
self.site_url = site_url
self._verify_ssl = verify_ssl
self._session = requests.Session()
if ssl_version is not None:
self._session.mount('https://', SSLAdapter(ssl_version))
self._session.headers.update({'user-agent':
'shareplum/%s' % __version__ })
if auth:
self._session.auth = auth
self.last_request = None
self._services_url = {'Alerts' : '/_vti_bin/Alerts.asmx',
'Authentication' : '/_vti_bin/Authentication.asmx',
'Copy' : '/_vti_bin/Copy.asmx',
'Dws' : '/_vti_bin/Dws.asmx',
'Forms' : '/_vti_bin/Forms.asmx',
'Imaging' : '/_vti_bin/Imaging.asmx',
'DspSts' : '/_vti_bin/DspSts.asmx',
'Lists' : '/_vti_bin/lists.asmx',
'Meetings' : '/_vti_bin/Meetings.asmx',
'People' : '/_vti_bin/People.asmx',
'Permissions' : '/_vti_bin/Permissions.asmx',
'SiteData' : '/_vti_bin/SiteData.asmx',
'Sites' : '/_vti_bin/Sites.asmx',
'Search' : '/_vti_bin/Search.asmx',
'UserGroup' : '/_vti_bin/usergroup.asmx',
'Versions' : '/_vti_bin/Versions.asmx',
'Views' : '/_vti_bin/Views.asmx',
'WebPartPages' : '/_vti_bin/WebPartPages.asmx',
'Webs' : '/_vti_bin/Webs.asmx'
}
self.users = self.GetUsers()
def __init__(self,
client_id,
client_secret,
access_token=None,
refresh_token=None):
self.client_id = client_id
self.client_secret = client_secret
self.access_token = access_token
self.refresh_token = refresh_token
self.expires_in = None
self._requests = requests.Session()
# Try to force requests to use the TLSv1
try:
self._requests.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
except:
self._requests = requests
parser = SafeConfigParser()
parser.read(os.path.dirname(os.path.abspath(__file__)) + '/config.ini')
self.API_URL = parser.get('config', 'api_url')
self.SDK_VERSION = parser.get('config', 'sdk_version')
self.AUTH_URL = parser.get('config', 'auth_url')
self.OAUTH_PATH = parser.get('config', 'oauth_path')
import urllib2
from requests_toolbelt import SSLAdapter
from bs4 import BeautifulSoup
import re
import time
#let's start the script. for function to select link id with ssl, copy it to Output.csv and print it
y = ("ni1", "ni2", "ni3")
for x in y: #range of ids
url = "http://www.ALGO.es/Portal/Consultas/Folletos/FolletosNARANJASOPV.aspx?nif=" + str(
x)
s = requests.Session() #ssl session
s.mount(url, SSLAdapter(ssl.PROTOCOL_TLSv1)) # ojo al protocolo ssl
r = requests.get(url, verify=False) #verificar aunque el cert no es valido
f = open(str(x + ".csv"), "a") #append to csv
f.write('\n' + url + ",") #new line url comma
f = open(str(x + ".sh"), "a") #append to csv
f.write('#!/bin/bash' + '\n') #start sh
f.close()
print url #print soup.prettify() #see the html
#script to scrap the soup, filter it, copy it in Output.csv and print
soup = BeautifulSoup(r.content)
for titulo in soup.find_all("span",
{'id': 'ctl00_ContentPrincipal_lblSubtitulo'}):
strtitulo = str(titulo)
tit = strtitulo[47:]
tit2 = re.sub("</span>", "", tit)
strtit = str(tit2)
# HOYA
payload = {
'text': txt,
'speaker': 'hikari', # free! haruka,hikari,show,takeru,santa,bear
'emotion': 'happiness', # happiness,anger,sadness
'emotion_level': 1, # 1,2,3,4
'pitch': 100, # 50-200
'speed': 120, # 50-400
'volume': 100, # 50-200
'format': 'wav' # wav,ogg,aac
}
s = requests.Session()
s.mount('https://api.voicetext.jp/v1/tts',
SSLAdapter(ssl.PROTOCOL_TLSv1))
r = s.post('https://api.voicetext.jp/v1/tts',
params=payload,
auth=('xx:', ''))
#print(' Status Code : ', r.status_code)
if r.status_code != 200:
print(' Error!', r.json()['error']['message'])
sys.exit()
wb = open(tmpFile, 'wb')
wb.write(r.content)
wb.close()
wb = None
except:
print(' Error!', sys.exc_info()[0])
hash_str = "&".join([
"=".join([
r[0].lower(),
urllib.quote_plus(str(r[1])).lower().replace("+",
"%20").replace(
"%3A", ":")
]) for r in request
])
return base64.encodestring(
hmac.new(secret_key, hash_str, hashlib.sha1).digest()).strip()
args['apiKey'] = credentials['apikey']
args["signature"] = sign_request(args, credentials['secretkey'])
session = requests.Session()
session.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
try:
response = session.get(url, params=args, verify=verifysslcert)
logger_debug(logger, "Request sent: %s" % response.url)
result = response.text
if response.status_code == 200: # success
error = None
elif response.status_code == 401: # auth issue
error = "401 Authentication error"
elif response.status_code != 200 and response.status_code != 401:
error = "{0}: {1}".format(response.status_code,
response.headers.get('X-Description'))
except requests.exceptions.ConnectionError, e:
return None, "Connection refused by server: %s" % e
import requests
from requests_toolbelt import SSLAdapter
from requests.packages.urllib3.exceptions import (InsecureRequestWarning,
InsecurePlatformWarning)
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
warnings.filterwarnings('ignore', category=InsecureRequestWarning)
warnings.filterwarnings('ignore', category=InsecurePlatformWarning)
from newslynx.core import settings
from newslynx.lib.serialize import json_to_obj
FAIL_ENCODING = 'ISO-8859-1'
ssl_adapter = SSLAdapter('SSLv3')
def retry(*dargs, **dkwargs):
"""A decorator for performing http requests and catching all concievable errors.
Useful for including in scrapers for unreliable webservers.
@retry(attempts=3)
def buggy_request():
return requests.get('http://www.gooooooooooooogle.com')
buggy_request()
>>> None
"""
# set defaults
attempts = dkwargs.get('attempts', settings.NETWORK_MAX_RETRIES)
wait = dkwargs.get('wait', settings.NETWORK_WAIT)
backoff = dkwargs.get('backoff', settings.NETWORK_BACKOFF)
def request(self,
cmd,
args,
exit_on_bad_request=False,
truncate_error=True):
if not self.session:
self.session = requests.Session()
self.session.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
http_cmd = 'POST' if cmd == 'login' else 'GET'
args = self.prep_args(cmd, args)
resp = None
def sign_request(params, secret_key):
self.trace('Signing request')
request = zip(params.keys(), params.values())
request.sort(key=lambda x: x[0].lower())
hash_str = '&'.join([
'='.join([
r[0].lower(),
urllib.quote_plus(str(r[1]), safe='*').lower().replace(
'+', '%20').replace('%3A', ':')
]) for r in request
])
return base64.encodestring(
hmac.new(secret_key, hash_str, hashlib.sha1).digest()).strip()
if self.api_key:
assert self.secret_key
args['apikey'] = self.api_key
args['signature'] = sign_request(args, self.secret_key)
elif cmd != 'login':
self.password_login()
args['sessionkey'] = self.session_key
try:
resp = self.http_request(http_cmd, args)
result = resp.text
if resp.status_code == 200: # success
error = None
elif resp.status_code == 401: # auth issue
error = '401 Authentication error'
elif resp.status_code == 405:
error = ('Method not allowed, unauthorized access on URL: %s' %
self.url)
elif resp.status_code == 531:
error = ('Error authenticating at %s using username: %s,'
'password: %s, domain: %s' %
(self.url, self.username, self.password, self.domain))
else:
error_msg = resp.headers.get('X-Description')
if truncate_error:
error_msg = error_msg.split(': {')[0]
error = '{0}: {1}'.format(resp.status_code, error_msg)
except requests.exceptions.ConnectionError as e:
raise BadRequest('Connection refused by server: %s' % e)
except Exception as pokemon:
result = None
error = pokemon.message
if error is not None:
self.debug('Error: {}'.format(error))
if result:
try:
response = json.loads(result, "utf-8")
self.trace('RESPONSE: %s: %s' % (resp.status_code, response))
except ValueError as e:
self.error('Received: {}'.format(result))
response = None
error = e
else:
response = None
if response and isinstance(response, dict):
m = list(v for v in response.keys() if 'response' in v.lower())
if not m:
error = 'Invalid response received: %s' % response
else:
response = response[filter(lambda x: 'response' in x,
response.keys())[0]]
if error:
if exit_on_bad_request:
self.error(error)
else:
raise BadRequest(error)
else:
return response