def request(url,
close=True,
redirect=True,
followredirect=False,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
limit=None,
referer=None,
cookie=None,
output='',
timeout='30',
httpsskip=False,
use_web_proxy=False,
XHR=False,
IPv4=False):
# output extended = 4, response = 2, responsecodeext = 2
try:
handlers = []
redirectURL = url
if IPv4 == True:
setIP4()
if error == False and not proxy == None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if error == False and output == 'cookie2' or output == 'cookie' or output == 'extended' or not close == True:
cookies = cookielib.LWPCookieJar()
if httpsskip or use_web_proxy:
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
else:
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
if error == False:
if sys.version_info < (2, 7, 9): raise Exception()
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
#headers['User-Agent'] = Constants.USER_AGENT
headers['User-Agent'] = randomagent()
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers:
pass
elif referer == None:
try:
headers['Referer'] = '%s://%s/' % (urlparse.urlparse(
url).scheme, urlparse.urlparse(url).netloc)
except:
try:
headers['Referer'] = url
except:
pass
else:
headers['Referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in headers:
pass
elif XHR == True:
headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in headers:
pass
elif not cookie == None:
headers['Cookie'] = cookie
if error == False and redirect == False:
class NoRedirection(urllib2.HTTPErrorProcessor):
def http_response(self, request, response):
if IPv4 == True:
setIP6()
return response
opener = urllib2.build_opener(NoRedirection)
opener = urllib2.install_opener(opener)
try:
del headers['Referer']
except:
pass
redirectHandler = None
urlList = []
if error == False and followredirect:
class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers,
newurl):
newreq = urllib2.HTTPRedirectHandler.redirect_request(
self, req, fp, code, msg, headers, newurl)
if newreq is not None:
self.redirections.append(newreq.get_full_url())
if IPv4 == True:
setIP6()
return newreq
redirectHandler = HTTPRedirectHandler()
redirectHandler.max_redirections = 10
redirectHandler.redirections = [url]
opener = urllib2.build_opener(redirectHandler)
opener = urllib2.install_opener(opener)
request = urllib2.Request(url, data=post, headers=headers)
#print request
try:
response = urllib2.urlopen(request, timeout=int(timeout))
if followredirect:
for redURL in redirectHandler.redirections:
urlList.append(redURL) # make a list, might be useful
redirectURL = redURL
except urllib2.HTTPError as response:
try:
resp_code = response.code
except:
resp_code = None
try:
content = response.read()
except:
content = ''
if response.code == 503:
#Log("AAAA- CODE %s|%s " % (url, response.code))
if 'cf-browser-verification' in content:
control.log('cf-browser-verification: CF-OK')
netloc = '%s://%s' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
#cf = cache.get(cfcookie, 168, netloc, headers['User-Agent'], timeout)
cfc = cfcookie()
cf = cfc.get(netloc, headers['User-Agent'], timeout)
headers['Cookie'] = cf
request = urllib2.Request(url, data=post, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
elif error == False:
if IPv4 == True:
setIP6()
return
elif error == True:
return '%s: %s' % (response.code, response.reason), content
elif response.code == 403:
if output == 'response':
return response.code, content
else:
return
elif response.code == 307:
#Log("AAAA- Response read: %s" % response.read(5242880))
#Log("AAAA- Location: %s" % (response.headers['Location'].rstrip()))
cookie = ''
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
headers['Cookie'] = cookie
request = urllib2.Request(response.headers['Location'],
data=post,
headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
#Log("AAAA- BBBBBBB %s" % response.code)
elif resp_code != None:
if IPv4 == True:
setIP6()
if output == 'response':
return (resp_code, None)
return resp_code
elif error == False:
#print ("Response code",response.code, response.msg,url)
if IPv4 == True:
setIP6()
return
else:
if IPv4 == True:
setIP6()
return
except Exception as e:
control.log('ERROR client.py>request : %s' % url)
control.log('ERROR client.py>request : %s' % e.args)
if IPv4 == True:
setIP6()
if output == 'response':
return (None, None)
return None
if output == 'cookie':
try:
result = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
result = cf
except:
pass
elif output == 'response':
if limit == '0':
result = (str(response.code), response.read(224 * 1024))
elif not limit == None:
result = (str(response.code), response.read(int(limit) * 1024))
else:
result = (str(response.code), response.read(5242880))
elif output == 'responsecodeext':
result = (str(response.code), redirectURL)
elif output == 'responsecode':
result = str(response.code)
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
#Log('CHUNK %s|%s' % (url,content))
if content < (2048 * 1024):
if IPv4 == True:
setIP6()
return
result = response.read(16 * 1024)
if close == True: response.close()
if IPv4 == True:
setIP6()
return result
elif output == 'extended':
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
cookie = cf
except:
pass
content = response.headers
result = response.read(5242880)
if IPv4 == True:
setIP6()
return (result, headers, content, cookie)
elif output == 'geturl':
result = response.geturl()
elif output == 'headers':
content = response.headers
if IPv4 == True:
setIP6()
return content
else:
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
if close == True:
response.close()
if IPv4 == True:
setIP6()
return result
except Exception as e:
control.log('ERROR client.py>request %s, %s' % (e.args, url))
traceback.print_exc()
if IPv4 == True:
setIP6()
return
if not os.path.exists(profileDir):
os.makedirs(profileDir)
urlopen = urllib2.urlopen
cj = cookielib.LWPCookieJar(xbmc.translatePath(cookiePath))
Request = urllib2.Request
handlers = [urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler()]
if (2, 7, 8) < sys.version_info < (2, 7, 12):
try:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
except:
pass
if cj != None:
if os.path.isfile(xbmc.translatePath(cookiePath)):
try:
cj.load()
except:
try:
os.remove(xbmc.translatePath(cookiePath))
pass
except:
dialog.ok('Oh oh',
'The Cookie file is locked, please restart Kodi')
pass
api_key = "JiSi7VtYVPLLWxQbrvnTJJOOq"
api_secret = "x7irzMC0FOMgLQn1LqrhnuQrrmVNa0snIGVv0GHebgcKxZ72B9"
access_token_key = "710935210775302145-eY3DWTKNGR0PQHchK62QbgSTIVBMBRI"
access_token_secret = "A2zMfbLpkEGnzltqzdjhFZsoHevegspGaBc7sTIixk78l"
_debug = 0
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
http_method = "GET"
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
'''
Construct, sign, and open a twitter request
using the hard-coded credentials above.
'''
def twitterreq(url, method, parameters):
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
def request(url, close=True, redirect=True, error=False, proxy=None, post=None, headers=None, mobile=False, XHR=False,
limit=None, referer=None, cookie=None, compression=True, output='', timeout='30'):
try:
if not url:
return
handlers = []
if not proxy == None:
handlers += [urllib2.ProxyHandler({'http': '%s' % (proxy)}), urllib2.HTTPHandler]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close == True:
cookies = cookielib.LWPCookieJar()
handlers += [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if (2, 7, 8) < sys.version_info < (2, 7, 12):
try:
import ssl;
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
if url.startswith('//'): url = 'http:' + url
_headers = {}
if headers:
try:
_headers.update(headers)
except:
log_exception()
if 'User-Agent' in _headers:
pass
elif not mobile == True:
# headers['User-Agent'] = agent()
_headers[
'User-Agent'] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.75 Safari/537.36"
else:
_headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in _headers:
pass
elif referer is not None:
_headers['Referer'] = referer
if not 'Accept-Language' in _headers:
_headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in _headers:
pass
elif XHR == True:
_headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in _headers:
pass
elif not cookie == None:
_headers['Cookie'] = cookie
if 'Accept-Encoding' in _headers:
pass
elif compression and limit is None:
_headers['Accept-Encoding'] = 'gzip'
if redirect == False:
# old implementation
# class NoRedirection(urllib2.HTTPErrorProcessor):
# def http_response(self, request, response): return response
# opener = urllib2.build_opener(NoRedirection)
# opener = urllib2.install_opener(opener)
class NoRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_302(self, req, fp, code, msg, headers):
infourl = urllib.addinfourl(fp, headers, req.get_full_url())
infourl.status = code
infourl.code = code
return infourl
http_error_300 = http_error_302
http_error_301 = http_error_302
http_error_303 = http_error_302
http_error_307 = http_error_302
opener = urllib2.build_opener(NoRedirectHandler())
urllib2.install_opener(opener)
try:
del _headers['Referer']
except:
pass
if isinstance(post, dict):
post = utils.byteify(post)
post = urllib.urlencode(post)
url = utils.byteify(url)
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
try:
import ssl
response = urllib2.urlopen(request, timeout=int(timeout), context=ssl._create_unverified_context())
except urllib2.HTTPError as response:
if response.code == 503 or ('shinden' in url and response.code == 403):
s = requests.Session()
headersok = {'User-Agent': _headers['User-Agent']}
s.headers = headersok
scraper = cloudflare6.create_scraper(s)
if output == 'session':
return s
if output == 'cookie':
return scraper.get_cookie_string(url, user_agent=_headers['User-Agent'])[0]
else:
return scraper.get(url).content
else:
log_utils.log('Request-Error (%s): %s' % (str(response.code), url), log_utils.LOGDEBUG)
if error == False: return
if output == 'cookie':
try:
result = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
if close == True: response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close == True: response.close()
return result
elif output == 'headers':
result = response.headers
if close == True: response.close()
return result
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
if close == True: response.close()
return result
elif output == 'file_size':
try:
content = int(response.headers['Content-Length'])
except:
content = '0'
response.close()
return content
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'sucuri_cloudproxy_js' in result:
su = sucuri().get(result)
_headers['Cookie'] = su
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
response = urllib2.urlopen(request, timeout=int(timeout))
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'Blazingfast.io' in result and 'xhr.open' in result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
ua = _headers['User-Agent']
_headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua, timeout)
result = _basic_request(url, headers=_headers, post=post, timeout=timeout, limit=limit)
if output == 'extended':
try:
response_headers = dict([(item[0].title(), item[1]) for item in response.info().items()])
except:
response_headers = response.headers
response_code = str(response.code)
try:
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
if close == True: response.close()
return (result, response_code, response_headers, _headers, cookie)
else:
if close == True: response.close()
return result
except Exception as e:
log_utils.log('Request-Error: (%s) => %s' % (str(e), url), log_utils.LOGDEBUG)
access_token = creds['access_token']
url = "https://api.fitbit.com/1/user/-/profile.json"
headers = {
"Authorization": "Bearer " + access_token,
"Accept-Language": "en_US"
}
#print 'Requesting: ' + url
req = urllib2.Request(url, headers=headers)
response = urllib2.urlopen(req)
the_page = response.read()
body = json.loads(the_page)
weight = body['user']['weight'] / 2.20462
print weight
http_logger = urllib2.HTTPSHandler() #debuglevel = 1)
#http_logger.set_http_debuglevel(1)
opener = urllib2.build_opener(http_logger) # put your other handlers here too!
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib2.install_opener(opener)
def auth():
global creds
# curl -X POST https://www.strava.com/api/v3/oauth/token -d client_id=20051 -d client_secret=[client_secret] -d code=[code] -d grant_type=authorization_code
client_id = creds['strava_client_id']
client_secret = creds['strava_client_secret']
refresh_token = creds['strava_refresh_token']
req = urllib2.Request(
'https://www.strava.com/api/v3/oauth/token',
urllib.urlencode({
def request(url,
close=True,
redirect=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
XHR=False,
limit=None,
referer=None,
cookie=None,
compression=True,
output='',
timeout='30',
ignoreSsl=False,
flare=True,
ignoreErrors=None):
try:
if not url: return None
handlers = []
if proxy is not None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close is True:
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if ignoreSsl or ((2, 7, 8) < sys.version_info < (2, 7, 12)):
try:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
if url.startswith('//'):
url = 'http:' + url
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers: pass
elif mobile is not True:
headers['User-Agent'] = cache.get(randomagent, 12)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers: pass
elif referer: headers['Referer'] = referer
if 'Accept-Language' not in headers:
headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in headers: pass
elif XHR: headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in headers: pass
elif cookie: headers['Cookie'] = cookie
if 'Accept-Encoding' in headers: pass
elif compression and limit is None: headers['Accept-Encoding'] = 'gzip'
if redirect is False:
class NoRedirection(urllib2.HTTPErrorProcessor):
def http_response(self, request, response):
return response
opener = urllib2.build_opener(NoRedirection)
opener = urllib2.install_opener(opener)
try:
del headers['Referer']
except:
pass
if isinstance(post, dict):
# Gets rid of the error: 'ascii' codec can't decode byte 0xd0 in position 0: ordinal not in range(128)
try:
iter_items = post.iteritems()
except:
iter_items = post.items()
for key, value in iter_items:
try:
post[key] = value.encode('utf-8')
except:
pass
post = urlencode(post)
request = urllib2.Request(url, data=post)
_add_request_header(request, headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except HTTPError as response:
try:
ignore = ignoreErrors and (int(response.code) == ignoreErrors
or int(
response.code) in ignoreErrors)
except:
ignore = False
if not ignore:
if response.code in [301, 307, 308, 503]:
cf_result = response.read(5242880)
try:
encoding = response.info().getheader(
'Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
cf_result = gzip.GzipFile(
fileobj=StringIO(cf_result)).read()
if flare and 'cloudflare' in str(response.info()).lower():
log_utils.log(
'client module calling cfscrape: url=%s' % url,
level=log_utils.LOGNOTICE)
try:
from fenomscrapers.modules import cfscrape
if isinstance(post, dict): data = post
else:
try:
data = parse_qs(post)
except:
data = None
scraper = cfscrape.CloudScraper()
response = scraper.request(
method='GET' if post is None else 'POST',
url=url,
headers=headers,
data=data,
timeout=int(timeout))
result = response.content
flare = 'cloudflare' # Used below
try:
cookies = response.request._cookies
except:
log_utils.error()
except:
log_utils.error()
elif 'cf-browser-verification' in cf_result:
netloc = '%s://%s' % (urlparse(url).scheme,
urlparse(url).netloc)
ua = headers['User-Agent']
cf = cache.get(cfcookie().get, 168, netloc, ua,
timeout)
headers['Cookie'] = cf
request = urllib2.Request(url, data=post)
_add_request_header(request, headers)
response = urllib2.urlopen(request,
timeout=int(timeout))
else:
log_utils.log('Request-Error (%s): %s' %
(str(response.code), url),
level=log_utils.LOGERROR)
if error is False: return None
else:
log_utils.log('Request-Error (%s): %s' %
(str(response.code), url),
level=log_utils.LOGERROR)
if error is False: return None
if output == 'cookie':
try:
result = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
result = cf
except:
pass
if close is True: response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close is True: response.close()
return result
elif output == 'headers':
result = response.headers
if close is True: response.close()
return result
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
if close is True: response.close()
return result
if flare != 'cloudflare':
if limit == '0': result = response.read(224 * 1024)
elif limit is not None: result = response.read(int(limit) * 1024)
else: result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO(result)).read()
if 'sucuri_cloudproxy_js' in result:
su = sucuri().get(result)
headers['Cookie'] = su
request = urllib2.Request(url, data=post)
_add_request_header(request, headers)
response = urllib2.urlopen(request, timeout=int(timeout))
if limit == '0': result = response.read(224 * 1024)
elif limit is not None: result = response.read(int(limit) * 1024)
else: result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO(result)).read()
if 'Blazingfast.io' in result and 'xhr.open' in result:
netloc = '%s://%s' % (urlparse(url).scheme, urlparse(url).netloc)
ua = headers['User-Agent']
headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua,
timeout)
result = _basic_request(url,
headers=headers,
post=post,
timeout=timeout,
limit=limit)
if output == 'extended':
try:
response_headers = dict([(item[0].title(), item[1])
for item in response.info().items()])
except:
response_headers = response.headers
try:
response_code = str(response.code)
except:
response_code = str(response.status_code
) # object from CFScrape Requests object.
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
cookie = cf
except:
pass
if close is True: response.close()
return (result, response_code, response_headers, headers, cookie)
else:
if close is True: response.close()
return result
except Exception as e:
log_utils.log('Request-Error: (%s) => %s' % (str(e), url),
level=log_utils.LOGERROR)
return None
def request(url, close=True, redirect=True, error=False, proxy=None, post=None, headers=None, mobile=False, XHR=False, limit=None, referer=None, cookie=None, compression=True, output='', timeout='30'):
try:
handlers = []
if not proxy == None:
handlers += [urllib2.ProxyHandler({'http':'%s' % (proxy)}), urllib2.HTTPHandler]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close == True:
cookies = cookielib.LWPCookieJar()
handlers += [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if (2, 7, 9) <= sys.version_info < (2, 7, 11):
try:
import ssl; ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
if url.startswith('//'): url = 'http:' + url
try: headers.update(headers)
except: headers = {}
if 'User-Agent' in headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers:
pass
elif referer == None:
headers['Referer'] = '%s://%s/' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
else:
headers['Referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in headers:
pass
elif XHR == True:
headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in headers:
pass
elif not cookie == None:
headers['Cookie'] = cookie
if 'Accept-Encoding' in headers:
pass
elif compression and limit is None:
headers['Accept-Encoding'] = 'gzip'
if redirect == False:
class NoRedirection(urllib2.HTTPErrorProcessor):
def http_response(self, request, response): return response
opener = urllib2.build_opener(NoRedirection)
opener = urllib2.install_opener(opener)
try: del headers['Referer']
except: pass
request = urllib2.Request(url, data=post, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
if response.code == 503:
cf_result = response.read(5242880)
try: encoding = response.info().getheader('Content-Encoding')
except: encoding = None
if encoding == 'gzip':
cf_result = gzip.GzipFile(fileobj=StringIO.StringIO(cf_result)).read()
if 'cf-browser-verification' in cf_result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
ua = headers['User-Agent']
cf = cache.get(cfcookie().get, 168, netloc, ua, timeout)
headers['Cookie'] = cf
request = urllib2.Request(url, data=post, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
elif error == False:
return
elif error == False:
return
if output == 'cookie':
try: result = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except: pass
try: result = cf
except: pass
if close == True: response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close == True: response.close()
return result
elif output == 'headers':
result = response.headers
if close == True: response.close()
return result
elif output == 'chunk':
try: content = int(response.headers['Content-Length'])
except: content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
if close == True: response.close()
return result
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try: encoding = response.info().getheader('Content-Encoding')
except: encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'sucuri_cloudproxy_js' in result:
su = sucuri().get(result)
headers['Cookie'] = su
request = urllib2.Request(url, data=post, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try: encoding = response.info().getheader('Content-Encoding')
except: encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if output == 'extended':
response_headers = response.headers
response_code = str(response.code)
try: cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except: pass
try: cookie = cf
except: pass
if close == True: response.close()
return (result, response_code, response_headers, headers, cookie)
else:
if close == True: response.close()
return result
except:
return
def cfcookie(netloc, ua, timeout):
try:
headers = {'User-Agent': ua}
request = urllib2.Request(netloc, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
result = response.read(5242880)
jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0]
init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};',
result)[-1]
builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0]
decryptVal = parseJSString(init)
lines = builder.split(';')
for line in lines:
if len(line) > 0 and '=' in line:
sections = line.split('=')
line_val = parseJSString(sections[1])
decryptVal = int(
eval(str(decryptVal) + sections[0][-1] + str(line_val)))
answer = decryptVal + len(urlparse.urlparse(netloc).netloc)
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (
netloc, jschl, answer)
if 'type="hidden" name="pass"' in result:
passval = re.findall('name="pass" value="(.*?)"', result)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % (
netloc, urllib.quote_plus(passval), jschl, answer)
time.sleep(5)
cookies = cookielib.LWPCookieJar()
handlers = [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
request = urllib2.Request(query, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
except:
pass
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
return cookie
except:
pass
def fireFuzz(type, fuzz, url, params, header, delay, outputFile, proxy, prefix, postfix):
"""
:Description: This function iterates through a list of fuzzing strings retrieved from the database, sends them to the target site and displays a progress bar of this process.
:param type: Type of the fuzzing strings to send [sql | xss].
:type type: String
:param fuzz: Fuzzing strings
:type fuzz: List
:param url: Target URL
:type url: String
:param params: POST Parameter
:type params: String
:param header: Cookie header
:type header: String
:param delay: Delay between requests
:type delay: Float
:param outputFile: Name of Output file
:type outputFile: String
:note: This function calls the showOutput() file with the saved outputs as argument.
:todo: Add threads in order to send requests simultaneously.
"""
print '''
___ ______________________ ______ ________
__ | / /__ |__ ____/__ | / /__(_)____________(_)_____ _
__ | /| / /__ /| |_ /_ __ |/ /__ /__ __ \____ /_ __ `/
__ |/ |/ / _ ___ | __/ _ /| / _ / _ / / /___ / / /_/ /
____/|__/ /_/ |_/_/ /_/ |_/ /_/ /_/ /_/___ / \__,_/
/___/
WAFNinja - Penetration testers favorite for WAF Bypassing
'''
pbar = ProgressBar(widgets=[SimpleProgress(), ' Fuzz sent! ', Percentage(), Bar()])
if proxy is not '':
httpProxy = urllib2.ProxyHandler({'http': proxy, 'https': proxy})
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
opener = urllib2.build_opener(urllib2.HTTPSHandler(context=ctx),httpProxy)
urllib2.install_opener(opener)
else:
opener = urllib2.build_opener()
opener.addheaders = [('User-Agent', 'Mozilla/5.0')]
for h in header:
opener.addheaders.append(h)
result = []
for fuzz in pbar(fuzz):
expected = fuzz[1]
fuzz = prefix + fuzz[0] + postfix
fuzz_enc = fuzz.encode('utf-8')
try:
sleep(float(delay))
if params is None: # GET parameter
randomString, url_with_fuzz = insertFuzz(url, fuzz_enc)
response = opener.open(url_with_fuzz)
else: # POST parameter
randomString, params_with_fuzz = setParams(params, fuzz_enc)
response = opener.open(url, urllib.urlencode(params_with_fuzz))
content = response.read()
occurence = content.find(randomString)+len(randomString) # get position of the randomString + length(randomString) to get to the fuzz
result.append({
'fuzz' : fuzz_enc,
'expected' : expected,
'httpCode' : response.getcode(),
'contentLength': len(content),
'output' : content[occurence:occurence+len(expected)]}) # take string from occurence to occurence+len(expected)
except urllib2.HTTPError, error: # HTTP Status != 200
if error.code == 404:
print 'ERROR: Target URL not reachable!'
sys.exit()
else: # HTTP Status != 404
result.append({
'fuzz' : fuzz_enc,
'expected' : expected,
'httpCode' : error.code,
'contentLength': '-',
'output' : '-'})
def request(url,
close=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
safe=False,
referer=None,
cookie=None,
output='',
timeout='30'):
try:
handlers = []
if not proxy == None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or not close == True:
import cookielib
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
if sys.version_info < (2, 7, 9): raise Exception()
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'referer' in headers:
pass
elif referer == None:
headers['referer'] = '%s://%s/' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
else:
headers['referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'cookie' in headers:
pass
elif not cookie == None:
headers['cookie'] = cookie
request = urllib2.Request(url, data=post, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
if error == False: return
if output == 'cookie':
result = []
for c in cookies:
result.append('%s=%s' % (c.name, c.value))
result = "; ".join(result)
elif output == 'response':
if safe == True:
result = (str(response.code), response.read(224 * 1024))
else:
result = (str(response.code), response.read())
elif output == 'chunk':
content = int(response.headers['Content-Length'])
if content < (2048 * 1024): return
result = response.read(16 * 1024)
elif output == 'geturl':
result = response.geturl()
else:
if safe == True:
result = response.read(224 * 1024)
else:
result = response.read()
if close == True:
response.close()
return result
except:
return
def request(url,
close=True,
redirect=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
limit=None,
referer=None,
cookie=None,
output='',
timeout='30'):
try:
handlers = []
if not proxy == None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close == True:
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
if sys.version_info < (2, 7, 9):
raise Exception()
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers:
pass
elif referer == None:
headers['Referer'] = '%s://%s/' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
else:
headers['Referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'Cookie' in headers:
pass
elif not cookie == None:
headers['Cookie'] = cookie
if redirect == False:
class NoRedirection(urllib2.HTTPErrorProcessor):
def http_response(self, request, response):
return response
opener = urllib2.build_opener(NoRedirection)
opener = urllib2.install_opener(opener)
try:
del headers['Referer']
except:
pass
request = urllib2.Request(url, data=post, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
if response.code == 503:
if 'cf-browser-verification' in response.read(5242880):
netloc = '%s://%s' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
cf = cache.get(cfcookie, 168, netloc,
headers['User-Agent'], timeout)
headers['Cookie'] = cf
request = urllib2.Request(url, data=post, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
elif error == False:
return
elif error == False:
return
if output == 'cookie':
try:
result = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
result = cf
except:
pass
elif output == 'response':
if limit == '0':
result = (str(response.code), response.read(224 * 1024))
elif not limit == None:
result = (str(response.code), response.read(int(limit) * 1024))
else:
result = (str(response.code), response.read(5242880))
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
elif output == 'extended':
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
cookie = cf
except:
pass
content = response.headers
result = response.read(5242880)
return (result, headers, content, cookie)
elif output == 'geturl':
result = response.geturl()
elif output == 'headers':
content = response.headers
return content
else:
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
if close == True:
response.close()
return result
except:
return
def request(url,
close=True,
redirect=True,
error=False,
verify=True,
proxy=None,
post=None,
headers=None,
mobile=False,
XHR=False,
limit=None,
referer=None,
cookie=None,
compression=True,
output='',
timeout='30'):
try:
if not url:
return
handlers = []
if proxy is not None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close is True:
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
try:
import platform
node = platform.node().lower()
is_XBOX = platform.uname()[1] == 'XboxOne'
except Exception:
node = ''
is_XBOX = False
if verify is False and sys.version_info >= (2, 7, 12):
try:
import ssl
ssl_context = ssl._create_unverified_context()
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
except:
pass
if verify is True and ((2, 7, 8) < sys.version_info <
(2, 7, 12) or is_XBOX):
try:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
except:
pass
if url.startswith('//'): url = 'http:' + url
_headers = {}
try:
_headers.update(headers)
except:
pass
if 'User-Agent' in _headers:
pass
elif mobile is True:
_headers['User-Agent'] = cache.get(randommobileagent, 1)
else:
_headers['User-Agent'] = cache.get(randomagent, 1)
if 'Referer' in _headers:
pass
elif referer is not None:
_headers['Referer'] = referer
if not 'Accept-Language' in _headers:
_headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in _headers:
pass
elif XHR is True:
_headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in _headers:
pass
elif cookie is not None:
_headers['Cookie'] = cookie
if 'Accept-Encoding' in _headers:
pass
elif compression and limit is None:
_headers['Accept-Encoding'] = 'gzip'
if redirect is False:
class NoRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_302(self, req, fp, code, msg, headers):
infourl = urllib.addinfourl(fp, headers,
req.get_full_url())
infourl.status = code
infourl.code = code
return infourl
http_error_300 = http_error_302
http_error_301 = http_error_302
http_error_303 = http_error_302
http_error_307 = http_error_302
opener = urllib2.build_opener(NoRedirectHandler())
urllib2.install_opener(opener)
try:
del _headers['Referer']
except:
pass
if isinstance(post, dict):
post = utils.byteify(post)
post = urllib.urlencode(post)
url = utils.byteify(url)
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
if response.code == 503:
cf_result = response.read()
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
cf_result = gzip.GzipFile(
fileobj=StringIO.StringIO(cf_result)).read()
if 'cf-browser-verification' in cf_result:
while 'cf-browser-verification' in cf_result:
netloc = '%s://%s/' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
ua = _headers['User-Agent']
cf = cache.get(cfcookie().get, 1, netloc, ua, timeout)
_headers['Cookie'] = cf
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
try:
response = urllib2.urlopen(request,
timeout=int(timeout))
cf_result = 'Success'
except urllib2.HTTPError as response:
cache.remove(cfcookie().get, netloc, ua, timeout)
cf_result = response.read()
else:
log_utils.log(
'Request-Error (%s): %s' % (str(response.code), url),
log_utils.LOGDEBUG)
if error is False:
return
else:
log_utils.log(
'Request-Error (%s): %s' % (str(response.code), url),
log_utils.LOGDEBUG)
if error is False:
return
if output == 'cookie':
try:
result = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
result = cf
except:
pass
if close is True:
response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close is True: response.close()
return result
elif output == 'headers':
result = response.headers
if close is True: response.close()
return result
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
if close is True: response.close()
return result
elif output == 'file_size':
try:
content = int(response.headers['Content-Length'])
except:
content = '0'
response.close()
return content
if limit == '0':
result = response.read(224 * 1024)
elif limit is not None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'sucuri_cloudproxy_js' in result:
su = sucuri().get(result)
_headers['Cookie'] = su
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
response = urllib2.urlopen(request, timeout=int(timeout))
if limit == '0':
result = response.read(224 * 1024)
elif limit is not None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(
fileobj=StringIO.StringIO(result)).read()
if 'Blazingfast.io' in result and 'xhr.open' in result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
ua = _headers['User-Agent']
_headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua,
timeout)
result = _basic_request(url,
headers=_headers,
post=post,
timeout=timeout,
limit=limit)
if output == 'extended':
try:
response_headers = dict([(item[0].title(), item[1])
for item in response.info().items()])
except:
response_headers = response.headers
response_code = str(response.code)
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
cookie = cf
except:
pass
if close is True: response.close()
return (result, response_code, response_headers, _headers, cookie)
else:
if close is True: response.close()
return result
except Exception as e:
log_utils.log('Request-Error: (%s) => %s' % (str(e), url),
log_utils.LOGDEBUG)
return
def do_request(self, json_obj):
headers = {
'Content-Type': 'application/json-rpc',
'User-Agent': 'python/zabbix_api'
}
if self.httpuser:
self.debug(logging.INFO, "HTTP Auth enabled")
auth = 'Basic ' + string.strip(
base64.encodestring(self.httpuser + ':' + self.httppasswd))
headers['Authorization'] = auth
self.r_query.append(str(json_obj))
self.debug(logging.INFO, "Sending: " + str(json_obj))
self.debug(logging.DEBUG, "Sending headers: " + str(headers))
request = urllib2.Request(url=self.url,
data=json_obj.encode('utf-8'),
headers=headers)
if self.proto == "https":
https_handler = urllib2.HTTPSHandler(debuglevel=0)
opener = urllib2.build_opener(https_handler)
elif self.proto == "http":
http_handler = urllib2.HTTPHandler(debuglevel=0)
opener = urllib2.build_opener(http_handler)
else:
raise ZabbixAPIException("Unknow protocol %s" % self.proto)
urllib2.install_opener(opener)
try:
response = opener.open(request, timeout=self.timeout)
except Exception as e:
raise ZabbixAPIException(
"Site needs HTTP authentication. Error: " + str(e))
self.debug(logging.INFO, "Response Code: " + str(response.code))
# NOTE: Getting a 412 response code means the headers are not in the
# list of allowed headers.
if response.code != 200:
raise ZabbixAPIException("HTTP ERROR %s: %s" %
(response.status, response.reason))
reads = response.read()
if len(reads) == 0:
raise ZabbixAPIException("Received zero answer")
try:
jobj = json.loads(reads.decode('utf-8'))
except ValueError as msg:
print("unable to decode. returned string: %s" % reads)
sys.exit(-1)
self.debug(logging.DEBUG, "Response Body: " + str(jobj))
self.id += 1
if 'error' in jobj: # some exception
msg = "Error %s: %s, %s while sending %s" % (
jobj['error']['code'], jobj['error']['message'],
jobj['error']['data'], str(json_obj))
if re.search(".*already\sexists.*", jobj["error"]["data"],
re.I): # already exists
raise Already_Exists(msg, jobj['error']['code'])
else:
raise ZabbixAPIException(msg, jobj['error']['code'])
return jobj
def getURLRequestData(self, params = {}, post_data = None):
def urlOpen(req, customOpeners, timeout):
if len(customOpeners) > 0:
opener = urllib2.build_opener( *customOpeners )
if timeout != None:
response = opener.open(req, timeout=timeout)
else:
response = opener.open(req)
else:
if timeout != None:
response = urllib2.urlopen(req, timeout=timeout)
else:
response = urllib2.urlopen(req)
return response
if IsMainThread():
msg1 = _('It is not allowed to call getURLRequestData from main thread.')
msg2 = _('You should never perform block I/O operations in the __init__.')
msg3 = _('In next release exception will be thrown instead of this message!')
GetIPTVNotify().push('%s\n\n%s\n\n%s' % (msg1, msg2, msg3), 'error', 40)
if not self.useMozillaCookieJar:
cj = cookielib.LWPCookieJar()
else:
cj = cookielib.MozillaCookieJar()
response = None
req = None
out_data = None
opener = None
metadata = None
timeout = params.get('timeout', None)
if 'host' in params:
host = params['host']
else:
host = self.HOST
if 'header' in params:
headers = params['header']
elif None != self.HEADER:
headers = self.HEADER
else:
headers = { 'User-Agent' : host }
if 'User-Agent' not in headers:
headers['User-Agent'] = host
metadata = {}
printDBG('pCommon - getURLRequestData() -> params: ' + str(params))
printDBG('pCommon - getURLRequestData() -> headers: ' + str(headers))
customOpeners = []
#cookie support
if 'use_cookie' not in params and 'cookiefile' in params and ('load_cookie' in params or 'save_cookie' in params):
params['use_cookie'] = True
if params.get('use_cookie', False):
if params.get('load_cookie', False):
try:
cj.load(params['cookiefile'], ignore_discard = True)
except IOError:
printDBG('Cookie file [%s] not exists' % params['cookiefile'])
except Exception:
printExc()
try:
for cookieKey in params.get('cookie_items', {}).keys():
printDBG("cookie_item[%s=%s]" % (cookieKey, params['cookie_items'][cookieKey]))
cookieItem = cookielib.Cookie(version=0, name=cookieKey, value=params['cookie_items'][cookieKey], port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
cj.set_cookie(cookieItem)
except Exception:
printExc()
customOpeners.append( urllib2.HTTPCookieProcessor(cj) )
if params.get('no_redirection', False):
customOpeners.append( NoRedirection() )
# debug
#customOpeners.append(urllib2.HTTPSHandler(debuglevel=1))
#customOpeners.append(urllib2.HTTPHandler(debuglevel=1))
if not IsHttpsCertValidationEnabled():
try:
if params.get('ssl_protocol', None) != None:
ctx = ssl._create_unverified_context(params['ssl_protocol'])
else:
ctx = ssl._create_unverified_context()
customOpeners.append(urllib2.HTTPSHandler(context=ctx))
except Exception: pass
elif params.get('ssl_protocol', None) != None:
ctx = ssl.SSLContext(params['ssl_protocol'])
customOpeners.append(urllib2.HTTPSHandler(context=ctx))
#proxy support
if self.useProxy:
http_proxy = self.proxyURL
else:
http_proxy = ''
#proxy from parameters (if available) overwrite default one
if 'http_proxy' in params:
http_proxy = params['http_proxy']
if '' != http_proxy:
printDBG('getURLRequestData USE PROXY')
customOpeners.append( urllib2.ProxyHandler({"http":http_proxy}) )
customOpeners.append( urllib2.ProxyHandler({"https":http_proxy}) )
pageUrl = params['url']
proxy_gateway = params.get('proxy_gateway', '')
if proxy_gateway != '':
pageUrl = proxy_gateway.format(urllib.quote_plus(pageUrl, ''))
printDBG("pageUrl: [%s]" % pageUrl)
if None != post_data:
printDBG('pCommon - getURLRequestData() -> post data: ' + str(post_data))
if params.get('raw_post_data', False):
dataPost = post_data
elif params.get('multipart_post_data', False):
customOpeners.append( MultipartPostHandler() )
dataPost = post_data
else:
dataPost = urllib.urlencode(post_data)
req = urllib2.Request(pageUrl, dataPost, headers)
else:
req = urllib2.Request(pageUrl, None, headers)
if not params.get('return_data', False):
out_data = urlOpen(req, customOpeners, timeout)
else:
gzip_encoding = False
try:
response = urlOpen(req, customOpeners, timeout)
if response.info().get('Content-Encoding') == 'gzip':
gzip_encoding = True
try:
metadata['url'] = response.geturl()
metadata['status_code'] = response.getcode()
if 'Content-Type' in response.info(): metadata['content-type'] = response.info()['Content-Type']
except Exception: pass
data = response.read()
response.close()
except urllib2.HTTPError, e:
ignoreCodeRanges = params.get('ignore_http_code_ranges', [(404, 404), (500, 500)])
ignoreCode = False
metadata['status_code'] = e.code
for ignoreCodeRange in ignoreCodeRanges:
if e.code >= ignoreCodeRange[0] and e.code <= ignoreCodeRange[1]:
ignoreCode = True
break
if ignoreCode:
printDBG('!!!!!!!! %s: getURLRequestData - handled' % e.code)
if e.fp.info().get('Content-Encoding', '') == 'gzip':
gzip_encoding = True
try:
metadata['url'] = e.fp.geturl()
if 'Content-Type' in e.fp.info(): metadata['content-type'] = e.fp.info()['Content-Type']
except Exception: pass
data = e.fp.read()
#e.msg
#e.headers
elif e.code == 503:
if params.get('use_cookie', False):
new_cookie = e.fp.info().get('Set-Cookie', '')
printDBG("> new_cookie[%s]" % new_cookie)
cj.save(params['cookiefile'], ignore_discard = True)
raise e
else:
if e.code in [300, 302, 303, 307] and params.get('use_cookie', False) and params.get('save_cookie', False):
new_cookie = e.fp.info().get('Set-Cookie', '')
printDBG("> new_cookie[%s]" % new_cookie)
#for cookieKey in params.get('cookie_items', {}).keys():
# cj.clear('', '/', cookieKey)
cj.save(params['cookiefile'], ignore_discard = True)
raise e
try:
if gzip_encoding:
printDBG('Content-Encoding == gzip')
buf = StringIO(data)
f = gzip.GzipFile(fileobj=buf)
out_data = f.read()
else:
out_data = data
except Exception as e:
printExc()
msg1 = _("Critical Error – Content-Encoding gzip cannot be handled!")
msg2 = _("Last error:\n%s" % str(e))
GetIPTVNotify().push('%s\n\n%s' % (msg1, msg2), 'error', 20)
out_data = data
def build_config(self):
"""Build the config as a Plugins object and return.
"""
config = monasca_setup.agent_config.Plugins()
# First watch the process
config.merge(monasca_setup.detection.watch_process(['apache2']))
log.info("\tWatching the apache webserver process.")
error_msg = '\n\t*** The Apache plugin is not configured ***\n\tPlease correct and re-run monasca-setup.'
# Attempt login, requires either an empty root password from localhost
# or relying on a configured /root/.apache.cnf
if self.dependencies_installed():
log.info("\tAttempting to use client credentials from {:s}".format(
apache_conf))
# Read the apache config file to extract the needed variables.
client_section = False
apache_url = None
apache_user = None
apache_pass = None
try:
with open(apache_conf, "r") as confFile:
for row in confFile:
if "[client]" in row:
client_section = True
pass
if client_section:
if "url=" in row:
apache_url = row.split("=")[1].strip()
if "user="******"=")[1].strip()
if "password="******"=")[1].strip()
except IOError:
log.info("\tUnable to read {:s}".format(apache_conf))
log.info("\tWill try to setup Apache plugin using defaults.")
if not apache_url:
apache_url = DEFAULT_APACHE_URL
opener = None
if apache_user and apache_pass:
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, apache_url, apache_user,
apache_pass)
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
else:
if 'https' in apache_url:
handler = urllib2.HTTPSHandler()
else:
handler = urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
response = None
try:
request = opener.open(apache_url)
response = request.read()
request.close()
if 'Total Accesses:' in response:
instance_vars = {'apache_status_url': apache_url}
if apache_user and apache_pass:
instance_vars.update({
'apache_user': apache_user,
'apache_password': apache_pass
})
config['apache'] = {
'init_config': None,
'instances': [instance_vars]
}
log.info("\tSuccessfully setup Apache plugin.")
else:
log.warn('Unable to access the Apache server-status URL;' +
error_msg)
except urllib2.URLError, e:
log.error(
'\tError {0} received when accessing url {1}.'.format(
e.reason, apache_url) +
'\n\tPlease ensure the Apache web server is running and your configuration '
+ 'information in /root/.apache.cnf is correct.' +
error_msg)
except urllib2.HTTPError, e:
log.error('\tError code {0} received when accessing {1}'.
format(e.code, apache_url) + error_msg)
def _request(method, url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
json=None):
if not headers:
headers = {}
pass
url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
handlers = []
import sys
# starting with python 2.7.9 urllib verifies every https request
if False == verify and sys.version_info >= (2, 7, 9):
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers.append(urllib2.HTTPSHandler(context=ssl_context))
pass
# handlers.append(urllib2.HTTPCookieProcessor())
# handlers.append(ErrorHandler)
if not allow_redirects:
handlers.append(NoRedirectHandler)
pass
opener = urllib2.build_opener(*handlers)
query = ''
if params:
for key in params:
value = params[key]
if isinstance(value, str):
value = value.decode('utf-8')
pass
params[key] = value.encode('utf-8')
pass
query = urllib.urlencode(params)
pass
if query:
url += '?%s' % query
pass
request = urllib2.Request(url)
if headers:
for key in headers:
request.add_header(key, str(unicode(headers[key]).encode('utf-8')))
pass
pass
if data or json:
if headers.get('Content-Type', '').startswith('application/x-www-form-urlencoded') or data:
# transform a string into a map of values
if isinstance(data, basestring):
_data = data.split('&')
data = {}
for item in _data:
name, value = item.split('=')
data[name] = value
pass
pass
# encode each value
for key in data:
data[key] = to_utf8(data[key])
pass
# urlencode
request.data = urllib.urlencode(data)
pass
elif headers.get('Content-Type', '').startswith('application/json') and data:
request.data = real_json.dumps(data).encode('utf-8')
pass
elif json:
request.data = real_json.dumps(json).encode('utf-8')
pass
else:
if not isinstance(data, basestring):
data = str(data)
pass
if isinstance(data, str):
data = data.encode('utf-8')
pass
request.data = data
pass
pass
elif method.upper() in ['POST', 'PUT']:
request.data = "null"
pass
request.get_method = lambda: method
result = Response()
response = None
try:
response = opener.open(request)
except urllib2.HTTPError, e:
# HTTPError implements addinfourl, so we can use the exception to construct a response
if isinstance(e, urllib2.addinfourl):
response = e
pass
pass
def send(self,
url,
method='GET',
payload=None,
headers=None,
cookiejar=None,
auth=None,
content=''):
'''Makes a web request and returns a response object.'''
if method.upper() != 'POST' and content:
raise RequestException(
'Invalid content type for the %s method: %s' %
(method, content))
# prime local mutable variables to prevent persistence
if payload is None: payload = {}
if headers is None: headers = {}
if auth is None: auth = ()
# set request arguments
# process user-agent header
headers['User-Agent'] = self.user_agent
# process payload
if content.upper() == 'JSON':
headers['Content-Type'] = 'application/json'
payload = json.dumps(payload)
else:
payload = urllib.urlencode(payload)
# process basic authentication
if len(auth) == 2:
authorization = ('%s:%s' %
(auth[0], auth[1])).encode('base64').replace(
'\n', '')
headers['Authorization'] = 'Basic %s' % (authorization)
# process socket timeout
if self.timeout:
socket.setdefaulttimeout(self.timeout)
# set handlers
# declare handlers list according to debug setting
handlers = [
urllib2.HTTPHandler(debuglevel=1),
urllib2.HTTPSHandler(debuglevel=1)
] if self.debug else []
# process cookiejar handler
if cookiejar != None:
handlers.append(urllib2.HTTPCookieProcessor(cookiejar))
# process redirect and add handler
if self.redirect == False:
handlers.append(NoRedirectHandler)
# process proxies and add handler
if self.proxy:
proxies = {'http': self.proxy, 'https': self.proxy}
handlers.append(urllib2.ProxyHandler(proxies))
# install opener
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
# process method and make request
if method == 'GET':
if payload: url = '%s?%s' % (url, payload)
req = urllib2.Request(url, headers=headers)
elif method == 'POST':
req = urllib2.Request(url, data=payload, headers=headers)
elif method == 'HEAD':
if payload: url = '%s?%s' % (url, payload)
req = urllib2.Request(url, headers=headers)
req.get_method = lambda: 'HEAD'
else:
raise RequestException(
'Request method \'%s\' is not a supported method.' % (method))
try:
resp = urllib2.urlopen(req)
except urllib2.HTTPError as e:
resp = e
# build and return response object
return ResponseObject(resp, cookiejar)
def __init__(self):
# Intialize urllib for camera verification
handler = urllib2.HTTPSHandler(debuglevel=1)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
def get_cookie(self, netloc, ua, timeout):
try:
headers = {'User-Agent': ua}
request = urllib2.Request(netloc)
_add_request_header(request, headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except HTTPError as response:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO(result)).read()
jschl = re.findall(
r'name\s*=\s*["\']jschl_vc["\']\s*value\s*=\s*["\'](.+?)["\']/>',
result, re.I)[0]
init = re.findall(r'setTimeout\(function\(\){\s*.*?.*:(.*?)};',
result, re.I)[-1]
builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result,
re.I)[0]
decryptVal = self.parseJSString(init)
lines = builder.split(';')
for line in lines:
if len(line) > 0 and '=' in line:
sections = line.split('=')
line_val = self.parseJSString(sections[1])
decryptVal = int(
eval(
str(decryptVal) + sections[0][-1] + str(line_val)))
answer = decryptVal + len(urlparse(netloc).netloc)
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (
netloc, jschl, answer)
if 'type="hidden" name="pass"' in result:
passval = re.findall(
r'name\s*=\s*["\']pass["\']\s*value\s*=\s*["\'](.*?)["\']',
result, re.I)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % (
netloc, quote_plus(passval), jschl, answer)
time.sleep(6)
cookies = cookielib.LWPCookieJar()
handlers = [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
request = urllib2.Request(query)
_add_request_header(request, headers)
response = urllib2.urlopen(request, timeout=int(timeout))
except:
pass
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
if 'cf_clearance' in cookie: self.cookie = cookie
except:
log_utils.error()
def request(url,
close=True,
redirect=True,
error=False,
verify=True,
proxy=None,
post=None,
headers=None,
mobile=False,
XHR=False,
limit=None,
referer=None,
cookie=None,
compression=False,
output='',
timeout='30',
username=None,
password=None,
as_bytes=False):
"""
Re-adapted from Twilight0's tulip module => https://github.com/Twilight0/script.module.tulip
post needs fixing
"""
try:
url = six.ensure_text(url, errors='ignore')
except Exception:
pass
if isinstance(post, dict):
post = bytes(urlencode(post), encoding='utf-8')
elif isinstance(post, str):
post = bytes(post, encoding='utf-8')
try:
handlers = []
if username is not None and password is not None and not proxy:
passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
passmgr.add_password(None, uri=url, user=username, passwd=password)
handlers += [urllib2.HTTPBasicAuthHandler(passmgr)]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
if proxy is not None:
if username is not None and password is not None:
if six.PY2:
passmgr = urllib2.ProxyBasicAuthHandler()
else:
passmgr = urllib2.HTTPPasswordMgr()
passmgr.add_password(None,
uri=url,
user=username,
passwd=password)
handlers += [
urllib2.ProxyHandler({'http': '{0}'.format(proxy)}),
urllib2.HTTPHandler,
urllib2.ProxyBasicAuthHandler(passmgr)
]
else:
handlers += [
urllib2.ProxyHandler({'http': '{0}'.format(proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or close is not True:
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
try:
import platform
is_XBOX = platform.uname()[1] == 'XboxOne'
except Exception:
is_XBOX = False
if not verify and sys.version_info >= (2, 7, 12):
try:
import ssl
ssl_context = ssl._create_unverified_context()
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
except Exception:
pass
elif verify and ((2, 7, 8) < sys.version_info < (2, 7, 12) or is_XBOX):
try:
import ssl
try:
import _ssl
CERT_NONE = _ssl.CERT_NONE
except Exception:
CERT_NONE = ssl.CERT_NONE
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
except Exception:
pass
try:
headers.update(headers)
except Exception:
headers = {}
if 'User-Agent' in headers:
pass
elif mobile is not True:
#headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 12)
else:
headers['User-Agent'] = cache.get(randommobileagent, 12)
if 'Referer' in headers:
pass
elif referer is None:
headers['Referer'] = '%s://%s/' % (urlparse(url).scheme,
urlparse(url).netloc)
else:
headers['Referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in headers:
pass
elif XHR is True:
headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in headers:
pass
elif cookie is not None:
headers['Cookie'] = cookie
if 'Accept-Encoding' in headers:
pass
elif compression and limit is None:
headers['Accept-Encoding'] = 'gzip'
if redirect is False:
class NoRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_302(self, reqst, fp, code, msg, head):
infourl = addinfourl(fp, head, reqst.get_full_url())
infourl.status = code
infourl.code = code
return infourl
http_error_300 = http_error_302
http_error_301 = http_error_302
http_error_303 = http_error_302
http_error_307 = http_error_302
opener = urllib2.build_opener(NoRedirectHandler())
urllib2.install_opener(opener)
try:
del headers['Referer']
except Exception:
pass
req = urllib2.Request(url, data=post, headers=headers)
try:
response = urllib2.urlopen(req, timeout=int(timeout))
except HTTPError as resp:
response = resp
if response.code == 503:
if 'cf-browser-verification' in response.read(5242880):
from resources.lib.modules import cfscrape
netloc = '{0}://{1}'.format(
urlparse(url).scheme,
urlparse(url).netloc)
ua = headers['User-Agent']
#cf = cache.get(Cfcookie.get, 168, netloc, ua, timeout)
try:
cf = cache.get(cfscrape.get_cookie_string, 1, netloc,
ua)[0]
except BaseException:
try:
cf = cfscrape.get_cookie_string(url, ua)[0]
except BaseException:
cf = None
finally:
headers['Cookie'] = cf
req = urllib2.Request(url, data=post, headers=headers)
response = urllib2.urlopen(req, timeout=int(timeout))
elif error is False:
return
elif error is False:
return
if output == 'cookie':
try:
result = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except Exception:
pass
try:
result = cf
except Exception:
pass
elif output == 'response':
if limit == '0':
result = (str(response.code), response.read(224 * 1024))
elif limit is not None:
result = (str(response.code), response.read(int(limit) * 1024))
else:
result = (str(response.code), response.read(5242880))
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except Exception:
content = (2049 * 1024)
if content < (2048 * 1024):
return
result = response.read(16 * 1024)
elif output == 'extended':
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except Exception:
pass
try:
cookie = cf
except Exception:
pass
content = response.headers
response_code = str(response.code)
result = response.read(5242880)
if not as_bytes:
result = six.ensure_text(result, errors='ignore')
return result, response_code, headers, content, cookie
elif output == 'geturl':
result = response.geturl()
elif output == 'headers':
content = response.headers
if close:
response.close()
return content
elif output == 'file_size':
try:
content = int(response.headers['Content-Length'])
except Exception:
content = '0'
response.close()
return content
elif output == 'json':
content = json.loads(response.read(5242880))
response.close()
return content
else:
if limit == '0':
result = response.read(224 * 1024)
elif limit is not None:
if isinstance(limit, int):
result = response.read(limit * 1024)
else:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
if close is True:
response.close()
if not as_bytes:
result = six.ensure_text(result, errors='ignore')
return result
except:
log_utils.log('Client request failed on url: ' + url + ' | Reason', 1)
return
def redirect_request(self, req, fp, code, msg, headers, newurl):
newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
newreq.get_method = lambda: req.get_method()
return newreq
exported_proxies = export_proxies(d)
handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
if export_proxies:
handlers.append(urllib2.ProxyHandler())
handlers.append(CacheHTTPHandler())
# XXX: Since Python 2.7.9 ssl cert validation is enabled by default
# see PEP-0476, this causes verification errors on some https servers
# so disable by default.
import ssl
if hasattr(ssl, '_create_unverified_context'):
handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context()))
opener = urllib2.build_opener(*handlers)
try:
uri = ud.url.split(";")[0]
r = urllib2.Request(uri)
r.get_method = lambda: "HEAD"
opener.open(r)
except urllib2.URLError as e:
# debug for now to avoid spamming the logs in e.g. remote sstate searches
logger.debug(2, "checkstatus() urlopen failed: %s" % e)
return False
return True
def _parse_path(self, regex, s):
"""
'keep-alive']'''
HeaderList = ["Server", "Via", "X-Powered-By"]
Pattern = re.compile('apache|JSP Engine|jetty|php|ssl', re.IGNORECASE)
Methods = ['post', 'get', 'put', 'trace']
Headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36'
}
############# URLLIB2 #############################
# proxy = urllib2.ProxyHandler({'https': '127.0.0.1:8080'})
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
# opener = urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
opener = urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
opener.addheaders = [(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36'
), ('Accept-Encoding', 'gzip, deflat')]
httplib.HTTPSConnection._http_vsn = 10
############# URLLIB2 #############################
def ChangeMethod(Link):
Status = StatusCode = ""
try:
if "http" not in Link:
Link = "https://" + Link + "/"
for Method in Methods:
Request = getattr(requests, Method)
import urllib
import json
import cookielib
import urllib2
from copy import deepcopy
from time import time
cookies = cookielib.LWPCookieJar()
handlers = [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
def fetch(uri):
req = urllib2.Request(uri)
return opener.open(req)
def dump():
for cookie in cookies:
print cookie.name, cookie.value
isMax = {
"aros": False,
"beach": False,
"downtown": False,
"price": False,
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=JARFILE,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=SESSIONJARFILE,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
requests_lib=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
handle=False,
agent=USERAGENT,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(syspaths.CONFIG_DIR, 'master'))
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(syspaths.CONFIG_DIR, 'minion'))
else:
opts = {}
if requests_lib is None:
requests_lib = opts.get('requests_lib', False)
if requests_lib is True:
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
log.debug('Using {0} Method'.format(method))
if method == 'POST':
log.trace('POST Data: {0}'.format(pprint.pformat(data)))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.fopen(session_cookie_jar, 'r') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if username and password:
auth = (username, password)
else:
auth = None
if requests_lib is True:
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: {0}'.format(sess.headers))
sess_cookies = sess.cookies
sess.verify = verify_ssl
else:
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(
cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(
cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
else:
sess_cookies.load()
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if requests_lib is True:
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error(
'The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.status_code
result_headers = result.headers
result_text = result.text
result_cookies = result.cookies
else:
request = urllib2.Request(url, data)
handlers = [
urllib2.HTTPHandler,
urllib2.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https') or port == 443:
if not HAS_MATCHHOSTNAME:
log.warn(
('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!'))
elif verify_ssl is False:
log.warn(('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!'))
else:
hostname = request.get_host()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, 443))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = ('The certificate was invalid. '
'Error returned was: {0}'.format(
pprint.pformat(exc)))
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: {0}'.format(cert))
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(urllib2.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[
0] = salt.ext.six.moves.http_client.HTTPSConnection(
**cert_kwargs)
opener = urllib2.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
result = opener.open(request)
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.code
result_headers = result.headers.headers
result_text = result.read()
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: {0}'.format(result_status_code))
log.trace('Response Headers: {0}'.format(result_headers))
log.trace('Response Cookies: {0}'.format(sess_cookies))
try:
log.trace('Response Text: {0}'.format(result_text))
except UnicodeEncodeError as exc:
log.trace(
('Cannot Trace Log Response Text: {0}. This may be due to '
'incompatibilities between requests and logging.').format(exc))
if text_out is not None and os.path.exists(text_out):
with salt.utils.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get('content-type',
'application/json')
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = ('Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)))
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
else:
text = True
if decode_out and os.path.exists(decode_out):
with salt.utils.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
urllib2.install_opener(opener)
# 4.使用HTTP的PUT和DELETE方法
import urllib2
uri = ''
request = urllib2.Request(uri, data=data)
request.get_method = lambda: 'PUT' # or 'DELETE'
response = urllib2.urlopen(request)
# 5.使用DebugLog
import urllib2
httpHandler = urllib2.HTTPHandler(debuglevel=1)
httpsHandler = urllib2.HTTPSHandler(debuglevel=1)
opener = urllib2.build_opener(httpHandler, httpsHandler)
urllib2.install_opener(opener)
response = urllib2.urlopen('http://www.baidu.com')
# 1.URLError
import urllib2
requset = urllib2.Request('http://www.xxxxx.com')
try:
urllib2.urlopen(request)
except urllib2.URLError, e:
print e.reason
# coding:utf-8
import urllib2
debug_mode = config.get('debug', '') == 'true' or sys.argv[-1] == 'debug'
START_AUTH_TOKEN = 'authenticityToken" value="'
END_AUTH_TOKEN = '"'
RESTAURANT_ID_FOOTER_KEY = 'Restaurant Id'
SET_LEAF_ID_FOOTER_KEY = 'Set Leaf Id'
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
cookieJar = cookielib.CookieJar()
handlers = [
urllib2.HTTPSHandler(context=context),
urllib2.HTTPCookieProcessor(cookieJar)
]
opener = urllib2.build_opener(*handlers)
def login_and_get_auth_token():
auth_token = login_page()
auth(auth_token)
return auth_token
def login_page():
try:
body = http('/login')
except:
import re
from bs4 import BeautifulSoup
import cookielib
import urllib
import urllib2
from captcha4 import captcha_to_string
import pickle
import pdb
# init
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(),
urllib2.HTTPHandler(debuglevel=0),
urllib2.HTTPSHandler(debuglevel=0),
urllib2.HTTPCookieProcessor(cookies))
# reslove captcha
res_dict = pickle.load(open('res_dict.p', 'rb'))
key = 'some string not from captcha'
print 'resolving captcha'
while key not in res_dict:
login_url = 'http://54.69.105.130/36e294b1d0dbe06a0cf626221ed54fc0/login.php'
response = opener.open(login_url)
captcha_url = "http://54.69.105.130/simple-php-captcha.php?_CAPTCHA&t=0.76120300+1419259319"
f = opener.open(captcha_url)
img = f.read()
open('out.png', 'wb').write(img)
key = captcha_to_string('out.png')
captcha_str = res_dict[key]
values = {
def install_debug_handler():
opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1),
urllib2.HTTPSHandler(debuglevel=1))
urllib2.install_opener(opener)
def __test_stream(self, stream_url):
'''
Returns True if the stream_url gets a non-failure http status (i.e. <400) back from the server
otherwise return False
Intended to catch stream urls returned by resolvers that would fail to playback
'''
# parse_qsl doesn't work because it splits elements by ';' which can be in a non-quoted UA
try:
headers = dict([
item.split('=')
for item in (stream_url.split('|')[1]).split('&')
])
except:
headers = {}
for header in headers:
headers[header] = urllib.unquote_plus(headers[header])
common.logger.log_debug('Setting Headers on UrlOpen: %s' % (headers))
try:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
opener = urllib2.build_opener(
urllib2.HTTPSHandler(context=ssl_context))
urllib2.install_opener(opener)
except:
pass
try:
msg = ''
request = urllib2.Request(stream_url.split('|')[0],
headers=headers)
# set urlopen timeout to 15 seconds
http_code = urllib2.urlopen(request, timeout=15).getcode()
except urllib2.URLError as e:
if hasattr(e, 'reason'):
# treat an unhandled url type as success
if 'unknown url type' in str(e.reason).lower():
return True
else:
msg = e.reason
if isinstance(e, urllib2.HTTPError):
http_code = e.code
else:
http_code = 600
if not msg: msg = str(e)
except Exception as e:
http_code = 601
msg = str(e)
# added this log line for now so that we can catch any logs on streams that are rejected due to test_stream failures
# we can remove it once we are sure this works reliably
if int(http_code) >= 400:
common.logger.log_warning(
'Stream UrlOpen Failed: Url: %s HTTP Code: %s Msg: %s' %
(stream_url, http_code, msg))
return int(http_code) < 400
def nsidc_icesat2_zarr(ddir,
PRODUCTS,
RELEASE,
VERSIONS,
GRANULES,
TRACKS,
USER='',
PASSWORD='',
YEARS=None,
SUBDIRECTORY=None,
AUXILIARY=False,
FLATTEN=False,
LOG=False,
LIST=False,
PROCESSES=0,
MODE=None,
CLOBBER=False):
#-- check if directory exists and recursively create if not
os.makedirs(ddir, MODE) if not os.path.exists(ddir) else None
#-- output of synchronized files
if LOG:
#-- format: NSIDC_IceBridge_sync_2002-04-01.log
today = time.strftime('%Y-%m-%d', time.localtime())
LOGFILE = 'NSIDC_IceSat-2_sync_{0}.log'.format(today)
fid = open(os.path.join(ddir, LOGFILE), 'w')
print('ICESat-2 Data Sync Log ({0})'.format(today), file=fid)
else:
#-- standard output (terminal output)
fid = sys.stdout
#-- https://docs.python.org/3/howto/urllib2.html#id5
#-- create a password manager
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
#-- Add the username and password for NASA Earthdata Login system
password_mgr.add_password(None, 'https://urs.earthdata.nasa.gov', USER,
PASSWORD)
#-- Encode username/password for request authorization headers
base64_string = base64.b64encode('{0}:{1}'.format(USER, PASSWORD).encode())
#-- compile HTML parser for lxml
parser = lxml.etree.HTMLParser()
#-- Create cookie jar for storing cookies. This is used to store and return
#-- the session cookie given to use by the data server (otherwise will just
#-- keep sending us back to Earthdata Login to authenticate).
cookie_jar = CookieJar()
#-- create "opener" (OpenerDirector instance)
opener = urllib2.build_opener(
urllib2.HTTPBasicAuthHandler(password_mgr),
urllib2.HTTPSHandler(context=ssl.SSLContext()),
urllib2.HTTPCookieProcessor(cookie_jar))
#-- add Authorization header to opener
authorization_header = "Basic {0}".format(base64_string.decode())
opener.addheaders = [("Authorization", authorization_header)]
#-- Now all calls to urllib2.urlopen use our opener.
urllib2.install_opener(opener)
#-- All calls to urllib2.urlopen will now use handler
#-- Make sure not to include the protocol in with the URL, or
#-- HTTPPasswordMgrWithDefaultRealm will be confused.
#-- remote https server for ICESat-2 Data
HOST = 'https://n5eil01u.ecs.nsidc.org'
#-- regular expression operator for finding files of a particular granule
#-- find ICESat-2 HDF5 files in the subdirectory for product and release
regex_track = '|'.join(['{0:04d}'.format(T) for T in TRACKS])
regex_granule = '|'.join(['{0:02d}'.format(G) for G in GRANULES])
regex_version = '|'.join(['{0:02d}'.format(V) for V in VERSIONS])
regex_suffix = '(.*?)' if AUXILIARY else '(h5)'
remote_regex_pattern = (
'{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})(\d{{2}})'
'(\d{{2}})(\d{{2}})_({1})(\d{{2}})({2})_({3})_({4})(.*?).{5}$')
#-- regular expression operator for finding subdirectories
if SUBDIRECTORY:
#-- Sync particular subdirectories for product
R2 = re.compile('(' + '|'.join(SUBDIRECTORY) + ')', re.VERBOSE)
elif YEARS:
#-- Sync particular years for product
regex_pattern = '|'.join('{0:d}'.format(y) for y in YEARS)
R2 = re.compile('({0}).(\d+).(\d+)'.format(regex_pattern), re.VERBOSE)
else:
#-- Sync all available subdirectories for product
R2 = re.compile('(\d+).(\d+).(\d+)', re.VERBOSE)
#-- build list of remote files, remote modification times and local files
remote_files = []
remote_mtimes = []
local_files = []
#-- for each ICESat-2 product listed
for p in PRODUCTS:
print('PRODUCT={0}'.format(p), file=fid)
#-- get directories from remote directory (* splat operator)
remote_directories = ['ATLAS', '{0}.{1}'.format(p, RELEASE)]
d = posixpath.join(HOST, *remote_directories)
req = urllib2.Request(url=d)
#-- compile regular expression operator for product, release and version
args = (p, regex_track, regex_granule, RELEASE, regex_version,
regex_suffix)
R1 = re.compile(remote_regex_pattern.format(*args), re.VERBOSE)
#-- read and parse request for subdirectories (find column names)
tree = lxml.etree.parse(urllib2.urlopen(req), parser)
colnames = tree.xpath('//td[@class="indexcolname"]//a/@href')
remote_sub = [sd for sd in colnames if R2.match(sd)]
#-- for each remote subdirectory
for sd in remote_sub:
#-- local directory for product and subdirectory
if FLATTEN:
local_dir = os.path.expanduser(ddir)
else:
local_dir = os.path.join(ddir, '{0}.{1}'.format(p, RELEASE),
sd)
#-- check if data directory exists and recursively create if not
os.makedirs(local_dir,
MODE) if not os.path.exists(local_dir) else None
#-- find ICESat-2 data files
req = urllib2.Request(url=posixpath.join(d, sd))
#-- read and parse request for remote files (columns and dates)
tree = lxml.etree.parse(urllib2.urlopen(req), parser)
colnames = tree.xpath('//td[@class="indexcolname"]//a/@href')
collastmod = tree.xpath('//td[@class="indexcollastmod"]/text()')
#-- find matching files (for granule, release, version, track)
remote_file_lines = [
i for i, f in enumerate(colnames) if R1.match(f)
]
#-- build lists of each ICESat-2 data file
for i in remote_file_lines:
#-- remote and local versions of the file
remote_files.append(posixpath.join(d, sd, colnames[i]))
local_files.append(os.path.join(local_dir, colnames[i]))
#-- get last modified date and convert into unix time
LMD = time.strptime(collastmod[i].rstrip(), '%Y-%m-%d %H:%M')
remote_mtimes.append(calendar.timegm(LMD))
#-- close request
req = None
#-- sync in series if PROCESSES = 0
if (PROCESSES == 0):
#-- sync each ICESat-2 data file
for i, remote_file in enumerate(remote_files):
#-- sync ICESat-2 files with NSIDC server
output = http_pull_file(remote_file, remote_mtimes[i],
local_files[i], LIST, CLOBBER, MODE)
#-- print the output string
print(output, file=fid)
else:
#-- sync in parallel with multiprocessing Pool
pool = mp.Pool(processes=PROCESSES)
#-- sync each ICESat-2 data file
output = []
for i, remote_file in enumerate(remote_files):
#-- sync ICESat-2 files with NSIDC server
args = (remote_file, remote_mtimes[i], local_files[i])
kwds = dict(LIST=LIST, CLOBBER=CLOBBER, MODE=MODE)
output.append(
pool.apply_async(multiprocess_sync, args=args, kwds=kwds))
#-- start multiprocessing jobs
#-- close the pool
#-- prevents more tasks from being submitted to the pool
pool.close()
#-- exit the completed processes
pool.join()
#-- print the output string
for out in output:
print(out.get(), file=fid)
#-- close log file and set permissions level to MODE
if LOG:
fid.close()
os.chmod(os.path.join(ddir, LOGFILE), MODE)
