def http_request(url, post=None, headers=None):
"""Perform an HTTP request."""
if headers is None:
headers = {}
request = Request(url)
request.add_header(
'User-Agent',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/1337 Safari/537.36'
)
for key in headers:
request.add_header(key, headers[key])
if post:
post = urlencode(post).encode("utf-8")
opener = build_opener(HTTPCookieProcessor(cookie_jar))
response = opener.open(request, data=post)
response_code = response.getcode()
if response_code not in [200, 204]:
raise Exception('Bad return code (' + str(response_code) + ') for: ' +
url)
return response.read(), response_code
def cookie_login(loginurl, values):
""" Open a session
login_url : the login url
values : dictionary containing login form field
"""
global urlOpener
# Enable cookie support for urllib2
mcookiejar = cookiejar.CookieJar()
urlOpener = build_opener(HTTPCookieProcessor(mcookiejar),
MultipartPostHandler)
data = urlencode(values)
request = Request(loginurl, data)
url = urlOpener.open(
request) # Our mcookiejar automatically receives the cookies
urllib.request.install_opener(urlOpener)
# Make sure we are logged in by checking the presence of the cookie "session_ser".
# (which is the cookie containing the session identifier.)
if not 'session_ser' in [cookie.name for cookie in mcookiejar]:
print("Login failed !")
return False
else:
print("We are logged in !")
return True
def get_cookies(proxy, url, useragent, timeout=.500):
proxy_support = ProxyHandler({'https': proxy})
cj = CookieJar()
opener = build_opener(proxy_support, HTTPCookieProcessor(cj))
install_opener(opener)
headers = {
'accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en-US,en;q=0.9,fr-FR;q=0.8,fr;q=0.7',
'cache-control': 'no-cache',
'dnt': '1',
'pragma': 'no-cache',
'referer': 'https://www.google.com',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'same-origin',
'sec-fetch-user': '******',
'upgrade-insecure-requests': '1',
'user-agent': useragent,
}
req = Request(url, data=None, headers=headers, method='GET')
try:
urlopen(req, timeout=timeout)
return dict_from_cookiejar(cj)
except URLError as e:
print(bcolors.FAIL + str(e) + bcolors.ENDC)
except HTTPError as e:
print(bcolors.FAIL + str(e) + bcolors.ENDC)
except Exception as e:
print(bcolors.FAIL + str(e) + bcolors.ENDC)
def _get_new_user_xvrttoken(self):
"""Get new 'user' X-VRT-Token from VRT NU website"""
# Get login json
login_json = self._get_login_json()
if login_json.get('errorCode') != 0:
return None
payload = dict(
UID=login_json.get('UID'),
UIDSignature=login_json.get('UIDSignature'),
signatureTimestamp=login_json.get('signatureTimestamp'),
client_id='vrtnu-site',
submit='submit',
)
data = urlencode(payload).encode()
cookiejar = cookielib.CookieJar()
opener = build_opener(HTTPCookieProcessor(cookiejar), ProxyHandler(self._proxies))
log(2, 'URL get: {url}', url=unquote(self._USER_TOKEN_GATEWAY_URL))
opener.open(self._USER_TOKEN_GATEWAY_URL)
log(2, 'URL post: {url}', url=unquote(self._VRT_LOGIN_URL))
opener.open(self._VRT_LOGIN_URL, data=data)
xvrttoken = TokenResolver._create_token_dictionary(cookiejar)
refreshtoken = TokenResolver._create_token_dictionary(cookiejar, cookie_name='vrtlogin-rt')
if xvrttoken is None:
return None
self._set_cached_token(xvrttoken, token_variant='user')
if refreshtoken is not None:
self._set_cached_token(refreshtoken)
return xvrttoken.get('X-VRT-Token')
def login(self, loc=None):
if loc is None:
loc = self.get_setting('BASE_LOCATION')
login_url = self.get_setting('BASE_URL') + \
self.get_setting('LOGIN_LOCATION')
cookies = HTTPCookieProcessor()
opener = build_opener(cookies)
opener.open(login_url)
try:
token = [
x.value for x in cookies.cookiejar if x.name == 'csrftoken'
][0]
except IndexError:
raise IOError("No csrf cookie found")
params = dict(username=self.get_setting('LOGIN_USERNAME'),
password=self.get_setting('LOGIN_PASSWORD'),
next=loc,
csrfmiddlewaretoken=token)
encoded_params = urlparse.urlencode(params).encode('utf-8')
req = Request(login_url, encoded_params)
req.add_header('Referer', login_url)
response = opener.open(req)
if response.geturl() == login_url:
raise IOError("Authentication refused")
return opener, response
def connect(self):
"""
authを行う
:return:
"""
# cookie付きのオープンハンドラ
self.opener = build_opener(HTTPCookieProcessor(CookieJar()))
o = {}
o["username"] = self.eve_user
o["password"] = self.eve_pass
o["html5"] = "-1"
stJson = json.dumps(o).encode("utf-8")
headers = {}
headers["Content-Type"] = "application/json"
# ログインの試行
u = "http://{0}/api/auth/login".format((self.host))
res = self.opener.open(u, stJson)
# 現在開いているlabの取得
u = "http://{0}/api/auth".format((self.host))
res = self.opener.open(u)
response_body = res.read().decode("utf-8")
d = json.loads(response_body)
data = d.get("data", {})
self.lab = data.get("lab", "")
def __init__(self):
# error message
self.error = None
# establish connection
self.session = build_opener()
# add proxy handler if needed
if config['proxy']:
if any(config['proxies'].values()):
self.session.add_handler(ProxyHandler(config['proxies']))
logger.debug("Proxy is set!")
else:
self.error = "Proxy enabled, but not set!"
# change user-agent
self.session.addheaders = [('User-Agent', config['ua'])]
# load local cookies
mcj = MozillaCookieJar()
try:
mcj.load(FILE_C, ignore_discard=True)
if 'uid' in [cookie.name for cookie in mcj]:
# if cookie.expires < int(time.time())
logger.info("Local cookies is loaded")
self.session.add_handler(HTTPCookieProcessor(mcj))
else:
logger.info("Local cookies expired or bad")
logger.debug(f"That we have: {[cookie for cookie in mcj]}")
mcj.clear()
self.login(mcj)
except FileNotFoundError:
self.login(mcj)
def start(args):
"""Login and session handler
"""
# create cookiejar
args._cj = LWPCookieJar()
# lets urllib handle cookies
opener = build_opener(HTTPCookieProcessor(args._cj))
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36"
), ("Accept-Encoding", "identity"), ("Accept-Charset", "utf-8"),
("DNT", "1")]
install_opener(opener)
# load cookies
try:
args._cj.load(getCookiePath(args), ignore_discard=True)
except IOError:
# cookie file does not exist
pass
args._cj.set_cookie(
Cookie(0, "timezoneoffset", str(timezone // 60), None, False,
"www.wakanim.tv", False, False, "/", True, False, None, False,
None, None, {"HttpOnly": None}, False))
def check_initial_sell_trade(self):
opener = build_opener(HTTPCookieProcessor(CookieJar()))
urlencoded_login_data = urlencode(self.login_post_data).encode('utf-8')
try:
res = opener.open(self.login_url, urlencoded_login_data)
except:
logging.warning("failed to login")
exit("Failed to login.")
urlencoded_get_stock_holdings_data = urlencode(
self.get_stock_holdings_post_data).encode('utf-8')
try:
res = opener.open(self.get_stock_holdings_url,
urlencoded_get_stock_holdings_data)
html = res.read()
shift = html.decode('Shift_JIS')
stock_holdings_num = re.findall(
"(?<=<br>){}(?= )".format(self.stock_id), shift)
if not len(stock_holdings_num):
self.successed_initial_sell_trade = True
logging.info("Successed initial sell trade of stock {}".format(
self.stock_id))
else:
self.successed_initial_sell_trade = False
logging.info(
"have not yet done initial sell trade of stock {}".format(
self.stock_id))
except:
logging.warning(
"failed to get stock holdings for check initital sell trade")
self.successed_initail_sell_trade = False
return
def look_up_ele():
BASE_URL = 'http://m.myhome.tsinghua.edu.cn/'
LOGIN_URL = BASE_URL + 'Loginwindow.aspx'
LOOKUP_URL = BASE_URL + 'weixin/weixin_student_electricity_search.aspx'
data = {
'__VIEWSTATE':
'/wEPDwUJMzUxODM3OTcwZGSzjX6gFtY6tdAxR1DJThQLsW0mqCd9bss+APDYXmKA8g==',
'__VIEWSTATEGENERATOR': '4B45C1AF',
'LoginCtrl1$txtUserName': settings.username,
'LoginCtrl1$txtPassword': settings.password,
'LoginCtrl1$btnLogin': '******'
}
cookie = CookieJar()
handler = HTTPCookieProcessor(cookie)
opener = build_opener(handler)
data = urlencode(data).encode('GBK')
res = opener.open(Request(LOGIN_URL, data)).read().decode('GBK')
res = opener.open(Request(LOOKUP_URL, data)).read().decode('GBK')
# with open('res.html', 'w') as f:
# f.write(res)
ele = -1
for line in res.split('\n'):
if 'lblele' in line:
line = line.strip()
get_ele = re.compile(r'>(\d+)')
ele = int(get_ele.findall(line)[0])
return ele
def initialReq():
cookiejar = CookieJar()
if ('ssl_validation'):
opener = build_opener(HTTPCookieProcessor(cookiejar), HTTPHandler())
else:
opener = build_opener(HTTPCookieProcessor(cookiejar), HTTPHandler(),
HTTPSHandler(context=sslvalidation()))
headers = {
"Host": "www.linkedin.com",
"Agent": user_agent,
}
req = Request("https://www.linkedin.com")
f = opener.open(req, timeout=timeout)
return cookiejar
def __init__(self,
mobile,
password=None,
status='0',
cachefile='Fetion.cache',
cookiesfile=''):
'''登录状态：
在线：400 隐身：0 忙碌：600 离开：100
'''
if cachefile:
self.cache = Cache(cachefile)
if not cookiesfile:
cookiesfile = '%s.cookies' % mobile
cookiejar = MozillaCookieJar(filename=cookiesfile)
if not os.path.isfile(cookiesfile):
open(cookiesfile, 'w').write(MozillaCookieJar.header)
cookiejar.load(filename=cookiesfile)
cookie_processor = HTTPCookieProcessor(cookiejar)
self.opener = build_opener(cookie_processor, HTTPHandler)
self.mobile, self.password = mobile, password
if not self.alive():
self._login()
cookiejar.save()
self.changestatus(status)
def login(self, mcj):
if self.error:
return
self.session.add_handler(HTTPCookieProcessor(mcj))
form_data = {
"username": config['username'],
"password": config['password']
}
logger.debug(f"Login. Data before: {form_data}")
# so we first encode vals to cp1251 then do default decode whole string
data_encoded = urlencode(
{k: v.encode('cp1251')
for k, v in form_data.items()}).encode()
logger.debug(f"Login. Data after: {data_encoded}")
self._catch_error_request(self.url + 'takelogin.php', data_encoded)
if self.error:
return
logger.debug(f"That we have: {[cookie for cookie in mcj]}")
if 'uid' in [cookie.name for cookie in mcj]:
mcj.save(FILE_C, ignore_discard=True, ignore_expires=True)
logger.info('We successfully authorized')
else:
self.error = "We not authorized, please check your credentials!"
logger.warning(self.error)
def __init__(self):
#build the opener
cookie_jar = CookieJar()
self._url_opener = build_opener(HTTPCookieProcessor(cookie_jar))
self._url_opener.addheaders =[
('Host', 'finance.yahoo.com'),
('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'),
('Accept', '*/*'),
('Accept-Language', 'en-US,en;q=0.5'),
('Accept-Encoding', 'gzip, deflate, br'),
('Referer', 'https://finance.yahoo.com/'),
('X-Requested-With', 'XMLHttpRequest'),
('Connection', 'keep-alive'),
('TE', 'Trailers'),
('Pragma', 'no-cache'),
('Cache-Control', 'no-cache')
]
#First Request to create cookies and access the CrumbStore
uri='https://finance.yahoo.com/quote/ALUA.BA/history?p=ALUA.BA'
with self._url_opener.open(Request(uri)) as response:
with gzip.GzipFile(fileobj=response) as deflated:
html=deflated.read().decode('utf-8')
self._crumb = json.loads(re.search('"CrumbStore":({"[^}]*})',html).group(1))["crumb"]
def get_credentials(url):
"""Get user credentials from .netrc or prompt for input."""
credentials = None
errprefix = ''
try:
info = netrc.netrc()
username, account, password = info.authenticators(urlparse(URS_URL).hostname)
errprefix = 'netrc error: '
except Exception as e:
if (not ('No such file' in str(e))):
print('netrc error: {0}'.format(str(e)))
username = None
password = None
while not credentials:
if not username:
username = get_username()
password = get_password()
credentials = '{0}:{1}'.format(username, password)
credentials = base64.b64encode(credentials.encode('ascii')).decode('ascii')
if url:
try:
req = Request(url)
req.add_header('Authorization', 'Basic {0}'.format(credentials))
opener = build_opener(HTTPCookieProcessor())
opener.open(req)
except HTTPError:
print(errprefix + 'Incorrect username or password')
errprefix = ''
credentials = None
username = None
password = None
return credentials
def unravel_message(url, cache):
"""Follows the riddle until the end — as in mission 04 — but this time
looking at the first cookie. Then returns the message decompressed."""
curr = 12345
cookie_jar = CookieJar()
url_opener = build_opener(HTTPCookieProcessor(cookie_jar))
msg = cache.get("msg", "")
if not msg:
while True:
try:
resp = url_opener.open(url.format(curr), timeout=3)
riddle = resp.read().decode()
cookie = list(cookie_jar)[0]
msg += cookie.value
except KeyboardInterrupt:
sys.exit(0)
except Exception as e:
print(f"\nBang! {e} ({curr})")
sys.exit(1)
try:
next_ = int(riddle.split(" ")[-1])
except ValueError:
break
curr = next_
cache["msg"] = msg
return bz2.decompress(unquote_to_bytes(msg.replace("+", " "))).decode()
def login(self, mcj):
if self.error:
return
# if we wanna use https we mast add ssl=enable_ssl to cookie
mcj.set_cookie(Cookie(0, 'ssl', "enable_ssl", None, False,
'.rutracker.org', True, False, '/', True,
False, None, 'ParserCookie', None, None, None))
self.session.add_handler(HTTPCookieProcessor(mcj))
form_data = {"login_username": config['username'],
"login_password": config['password'],
"login": "******"}
logger.debug(f"Login. Data before: {form_data}")
# so we first encode vals to cp1251 then do default decode whole string
data_encoded = urlencode(
{k: v.encode('cp1251') for k, v in form_data.items()}).encode()
logger.debug(f"Login. Data after: {data_encoded}")
self._catch_error_request(self.url + 'login.php', data_encoded)
if self.error:
return
logger.debug(f"That we have: {[cookie for cookie in mcj]}")
if 'bb_session' in [cookie.name for cookie in mcj]:
mcj.save(FILE_C, ignore_discard=True, ignore_expires=True)
logger.info("We successfully authorized")
else:
self.error = "We not authorized, please check your credentials!"
logger.warning(self.error)
def __request__(self):
Cookiejar = CookieJar()
opener = build_opener(HTTPCookieProcessor(Cookiejar))
_header = dict(self.headers.items())
if self.cookie:
_header.update({'Cookie': self.cookie})
req = Request(self.url, headers=_header, origin_req_host=self.host)
res = None
try:
res = opener.open(req)
# break
except HTTPError as e:
# if e.code >= 400 and e.code < 500:
return e, None
except (socket.timeout, URLError) as e:
return e, None
except Exception as e:
traceback.print_exc()
return e, None
return res, Cookiejar._cookies
def web_bruter(self):
while not self.password_q.empty() and not self.found:
brute = self.password_q.get().rstrip()
jar = cookielib.FileCookieJar("cookies")
opener = request.build_opener(HTTPCookieProcessor(jar))
response = opener.open(target_url)
page = response.read()
print("Trying: %s : %s (%d left)" % (self.username, brute, self.password_q.qsize()))
# parse out the hidden fields
parser = BruteParser()
parser.feed(page)
post_tags = parser.tar_results
# add our username and password fields
post_tags[username_field] = self.username
post_tags[password_field] = brute
login_data = urlencode(post_tags)
login_response = opener.open(target_post, login_data)
login_result = login_response.read()
if success_check in login_result:
self.found = True
print("[*] Bruteforce successful.")
print("[*] Username: %s" % username)
print("[*] Password: %s" % brute)
print("[*] Waiting for other threads to exit...")
def login(self, mcj):
if self.error:
return None
# if we wanna use https we mast add ssl=enable_ssl to cookie
mcj.set_cookie(Cookie(0, "ssl", "enable_ssl", None, False,
".nnmclub.to", True, False, "/", True,
False, None, False, None, None, {}))
self.session.add_handler(HTTPCookieProcessor(mcj))
response = self._catch_error_request(self.url_login)
if not response:
return None
code = RE_CODE.search(response.decode('cp1251'))[1]
form_data = {"username": config['username'],
"password": config['password'],
"autologin": "******",
"code": code,
"login": "******"}
# so we first encode vals to cp1251 then do default decode whole string
data_encoded = urlencode(
{k: v.encode('cp1251') for k, v in form_data.items()}
).encode()
self._catch_error_request(self.url_login, data_encoded)
if self.error:
return None
logger.debug(f"That we have: {[cookie for cookie in mcj]}")
if 'phpbb2mysql_4_sid' in [cookie.name for cookie in mcj]:
mcj.save(FILE_C, ignore_discard=True, ignore_expires=True)
logger.info('We successfully authorized')
else:
self.error = "We not authorized, please check your credentials!"
logger.warning(self.error)
def register(first_name, last_name, email, password):
# get register page
url = 'http://example.webscraping.com/places/default/user/register'
cj = CookieJar()
opener = build_opener(HTTPCookieProcessor(cj))
html = opener.open(url).read()
# try recognize captcha
img = get_captcha(html)
gray = img.convert('L')
bw = gray.point(lambda x: x if x < 1 else 255, '1')
captcha = pytesseract.image_to_string(bw)
# parse form and fill
form = parse_form(html)
form['first_name'] = first_name
form['last_name'] = last_name
form['email'] = email
form['password'] = password
form['password_two'] = password
form['recaptcha_response_field'] = captcha
data = urlencode(form).encode('utf-8')
r = Request(url=url, data=data)
register_page = opener.open(r)
return '/user/register' not in register_page.geturl()
def auth(*, email, password, client_id, scope):
def split_key_value(kv_pair):
kv = kv_pair.split("=")
return kv[0], kv[1]
# Authorization form
def auth_user(email, password, client_id, scope, opener):
response = opener.open(
"http://oauth.vk.com/oauth/authorize?" + \
"redirect_uri=http://oauth.vk.com/blank.html&response_type=token&" + \
"client_id=%s&scope=%s" % (client_id, ",".join(scope))
)
doc = response.read().decode('utf-8')
parser = AuthFormParser()
parser.feed(doc)
parser.close()
if not parser.form_parsed or parser.url is None or "pass" not in parser.params or \
"email" not in parser.params:
raise RuntimeError("Something wrong")
parser.params["email"] = email
parser.params["pass"] = password
if parser.method == "POST":
response = opener.open(parser.url,
urlencode(parser.params).encode('ascii'))
else:
raise NotImplementedError("Method '%s'" % parser.method)
return response.read(), response.geturl()
# Permission request form
def give_access(doc, opener):
parser = AuthFormParser()
parser.feed(doc)
parser.close()
if not parser.form_parsed or parser.url is None:
raise RuntimeError("Something wrong")
if parser.method == "POST":
response = opener.open(parser.url,
urlencode(parser.params).encode('ascii'))
else:
raise NotImplementedError("Method '%s'" % parser.method)
return response.geturl()
if not isinstance(scope, list):
scope = [scope]
opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()),
HTTPRedirectHandler())
doc, url = auth_user(email, password, client_id, scope, opener)
if urlparse(url).path != "/blank.html":
# Need to give access to requested scope
url = give_access(doc.decode('utf-8'), opener)
if urlparse(url).path != "/blank.html":
raise RuntimeError("Expected success here")
answer = dict(
split_key_value(kv_pair)
for kv_pair in urlparse(url).fragment.split("&"))
if "access_token" not in answer or "user_id" not in answer:
raise RuntimeError("Missing some values in answer")
return answer["access_token"], answer["user_id"]
def _setup_cookies(self):
# all requests should use the same cookie jar
self.cookiejar = LWPCookieJar(self.cookiefile)
if os.path.isfile(self.cookiefile):
self.cookiejar.load(ignore_discard=True)
self.cookieprocessor = HTTPCookieProcessor(self.cookiejar)
self.opener = build_opener(self.cookieprocessor)
install_opener(self.opener)
def __init__(self):
socket.setdefaulttimeout(20)
cookie = HTTPCookieProcessor(CookieJar())
opener = build_opener(cookie, HTTPHandler)
install_opener(opener)
agent = random.choice(AGENT)
opener.addheaders = [("User-agent", agent), ("Accept", "*/*")]
self.opener = opener
def __init__(self, auth_token, base_url):
self._token = auth_token
self._burl = base_url
# These object attributes are used to allow authentifying only once for
# the whole lifetime of the object.
self._cookie_processor = HTTPCookieProcessor(CookieJar())
self._authentified = False
def __init__(self, proxy=""):
# 生成cookie池
self.cookie = http.cookiejar.CookieJar()
self.opener = urllib.request.build_opener(
HTTPCookieProcessor(self.cookie))
if proxy:
# proxy format '127.0.0.1:8087'
# self.p = urllib2.ProxyHandler({'http': proxy})
# self.proxy_opener = urllib2.build_opener(self.p)
self.opener = urllib.request.build_opener(
HTTPCookieProcessor(self.cookie), ProxyHandler({'http':
proxy}))
urllib.request.install_opener(self.opener)
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
}
def nse_opener(self):
"""
builds opener for urllib2
:return: opener object
"""
cj = CookieJar()
bo = build_opener(HTTPCookieProcessor(cj))
return bo
def setup_method(self, method):
self.cookies = CookieJar()
self.opener = build_opener(HTTPRedirectHandler(),
HTTPHandler(debuglevel=0),
HTTPSHandler(debuglevel=0),
HTTPCookieProcessor(self.cookies))
self.application_process = Process(target=main)
self.application_process.start()
def __init__(self):
self.loginUrl = '****************'
self.cookies = http.cookiejar.CookieJar()
self.postdata = urlencode({
"txtId":"*********",
"txtMM":"*&******"
}).encode(encoding='gbk')
self.opener = build_opener(HTTPCookieProcessor(self.cookies))
def __opener__():
"""
Builds the opener for the url
:returns: opener object
"""
cookie_jar = CookieJar()
return build_opener(HTTPCookieProcessor(cookie_jar))
