async def verified_user_id(message, request):
print("VUI")
try:
action = json.loads(message)
# Get token, return if not specified
token = action.get("token")
headers = {k.decode(): v.decode() for k, v in request.headers}
print("H", headers)
cookies = SimpleCookie(headers.get('cookie'))
print("C", cookies)
for key, morsel in cookies.items():
if key == 'token':
token = morsel.value
if not token: return
# DEBUG
if token == 'machine':
debug_handle()
return 1
# decode the JWT token, get user_id from it
try:
identity = jwt.decode(token, 'secret', alorithms=['HS256'])
return identity['user_id']
except jwt.DecodeError as e:
return
except json.JSONDecodeError as e:
log.error("JSON decode error %s" % e)
return
def handle_route(self, route, params, cookies, set_cookie):
"""Common code for GET and POST commands to handle route and send json result."""
if route.____useCookies__:
t, cookie, result = route(cookies = cookies, **params)
else:
t, cookie, result = route(**params)
cookie is not None and cookies.load(cookie)
if set_cookie:
if cookie is None:
cookie = SimpleCookie(set_cookie)
else:
cookie.load(set_cookie)
if t == ROUTE_REDIRECT:
return self.redirect(result, cookie)
elif t == ROUTE_FORWARD:
route = self.dispatch(result[0])
return callable(route) and route or self.handle_route(route, result[1], cookies, cookie)
if route.____template__ is None:
content_type = 'text/json'
result = json.dumps(result)
else:
content_type = 'text/html'
result = self.template(route.____template__, result)
result = encodeUTF8(result)
headers = [('Content-type', content_type)]
if cookie is not None:
for key, value in sorted(cookie.items()):
headers.append(('Set-Cookie', value.OutputString()))
self.start_response('200 OK', headers)
return iter([result])
def __call__(self, env, start_response): # WSGI应用
method = env['REQUEST_METHOD'] # 请求方法
path = env['PATH_INFO'] # 请求路径
cookie_str = env['HTTP_COOKIE'] # Cookies
query = env['QUERY_STRING']
req_obj = None
if method == 'POST':
size = int(env.get('CONTENT_LENGTH', 0))
query = env['wsgi.input'].read(size).decode('utf-8')
req_obj = parse_qs(query) # 处理请求参数
sc = SimpleCookie()
sc.load(cookie_str) # 处理Cookie
req_obj['cookies'] = {k:m.value for k, m in sc.items()}
handler = self.handlers[method].get(path, False)# 获取handler
ck_dict = None
if not handler:
status = '404 Not Found'
content = page404.encode('utf-8')
else:
status = '200 OK'
content = handler(req_obj) # 执行handler
if isinstance(content, tuple):
content, ck_dict = content
content = content.encode('utf-8')
headers = [('Content-Type', 'text/html'),
('Content-Length', str(len(content)))]
if ck_dict:
set_ck = ';'.join([f'{k}={v}' for k, v in ck_dict.items()])
headers.append(('Set-Cookie', set_ck))
start_response(status, headers)
return [content]
def get_hedy_cookie(self, cookie_string):
cookie = SimpleCookie()
cookie.load(cookie_string)
for key, cookie in cookie.items():
if key == CONFIG['session']['cookie_name']:
return cookie
def __init__(self, username, cookies=""):
self.username = username
try:
cookie = SimpleCookie()
cookie.load(cookies)
self.cookies = {}
for key, morsel in cookie.items():
self.cookies[key] = morsel.value
except:
pass
self.similar = {}
self.info = {
"similar": {},
'bio': "",
"bio_url": "",
'followers': "",
"follow": "",
'name': "",
'profile_pic': "",
"descriptions": [],
"captions": [],
'locations': {},
'timestamps': {}
}
self.headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
}
def __init__(self, base_url, cookies=None):
self.base_url = base_url
if cookies is not None:
jar = SimpleCookie()
jar.load(cookies)
self.cookies = {key: morsel.value for key, morsel in jar.items()}
def cookie_dict(self, cookiestr):
cookie = SimpleCookie()
cookie.load(cookiestr)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def view(request, path):
morepath_request = MorepathRequest(request.environ, app)
# consume everything already seen by django
for i in range(resolved_segment_count(request.path, path)):
morepath_request.unconsumed.pop()
morepath_request.make_body_seekable()
response = app.publish(morepath_request)
cookies = SimpleCookie()
django_response = HttpResponse(
response.body, status=response.status_code
)
for (header, value) in response.headerlist:
if header.upper() == "SET-COOKIE":
cookies.load(value)
else:
django_response[header] = value
for cookie_name, cookie in cookies.items():
cookie_attributes = {
"key": cookie_name,
"value": cookie.value,
"expires": cookie["expires"],
"path": cookie["path"],
"domain": cookie["domain"],
}
if cookie["max-age"]:
# Starting in Django 1.3 it performs arithmetic operations
# with 'Max-Age'
cookie_attributes["max_age"] = int(cookie["max-age"])
django_response.set_cookie(**cookie_attributes)
return django_response
def parse_cookie(rawdata):
if rawdata == None:
return rawdata
cookie = SimpleCookie()
cookie.load(rawdata)
cookies = {}
for key, data in cookie.items():
cookies[key] = data.value
return cookies
# def parse_url(url):
# try:
# host = urllib3.util.url.parse_url(url).host
# except Exception as e:
# print("Invalid domain, try again..")
# sys.exit(1)
# return host
# def parse_wordlist(wordlist):
# try:
# wordlists = open(wordlist).read().splitlines()
# except Exception as e:
# print(e)
# sys.exit(1)
# return wordlists
def store_cookies(self, response):
if not "Set-Cookie" in response.headers:
return
cookies = SimpleCookie(response.headers["Set-Cookie"])
for key, item in cookies.items():
self.client.cookies.set(item.key, item.value)
def convertcookie(data):
cookie = SimpleCookie()
cookie.load(data)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def get_cookie_dict(cookie_str: str) -> dict:
cookie = SimpleCookie()
cookie.load(cookie_str)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def loginGET(environ, start_response):
unfromated_cookie = environ.get("HTTP_COOKIE", "")
cookie = SimpleCookie()
cookie.load(unfromated_cookie)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
passw = cookies['passw']
uname = cookies['uname']
passw = decript(passw)
login_status = login(uname, passw)
if login_status == 1:
start_response('200 OK', [('Content-text', 'text/plain')])
encoded_jwt = jwt.encode({
'uname': uname,
'iat': datetime.utcnow()
},
'secret',
algorithm='HS256')
yield encoded_jwt
elif login_status == 2:
start_response('401 Unauthorized', [('Content-text', 'text/plain')])
message = {"message": "Bad username or password"}
yield json.dumps(message).encode('utf-8')
elif login_status == 3:
start_response("500 Internal Server Error",
[('Content-text', 'text/plain')])
message = {"message": "Databease problem"}
yield json.dumps(message).encode('utf-8')
def start(self):
cookie = SimpleCookie()
request_cookies = {}
if self.cookies is not None:
cookie.load(self.cookies)
for key, morsel in cookie.items():
request_cookies[key] = morsel.value
else:
print("[*] No cookie set, continuing...")
count_matched = 0
for payload in self.payloads:
for url in self.urls:
urlTarget = url + payload
r = requests.get(urlTarget, cookies=request_cookies)
if self.verbose:
print('GET [{0}] {1}'.format(r.status_code, urlTarget))
for match in self.matches[payload]:
if match in r.text:
print("Interesting: " + url + payload)
count_matched = count_matched + 1
if "syntax error" in r.text:
print("PHP error: " + url + payload)
if count_matched == 0:
print("[-] Nothing found")
print("[*] Scan completed")
def loadCookie(rawdata):
cookie = SimpleCookie()
cookie.load(rawdata)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def on_open(self):
"""On Open
Will be called when a new client connects to the server
"""
if _verbose: print('on_open called')
# If there is no cookie
if 'HTTP_COOKIE' not in self.ws.environ:
if _verbose: print('HTTP_COOKIE missing')
return self._fail(11, 'HTTP_COOKIE missing')
# If the cookie lacks the PHPSESSID value
oCookies = SimpleCookie()
oCookies.load(self.ws.environ['HTTP_COOKIE'])
dCookies = {k: v.value for k, v in oCookies.items()}
if '_session' not in dCookies:
if _verbose: print('_session not in cookies')
return self._fail(12, '_session not in cookies')
# Store the session ID
self.token = dCookies['_session']
self.authorized = False
self.tracking = []
def updateheader(self, url, item):
headers = json.loads(item.get('item_headers'))
# if ':' in name]
if 'Host' in headers:
self.set_header('Host', headers.get('Host'))
if 'Referer' in headers:
self.set_header('Referer', headers.get('Referer'))
if 'Set-Cookie' in headers:
for cookiestr in commonutils.covertSetCookie2List(
headers.get('Set-Cookie')):
s = SimpleCookie()
s.load(cookiestr)
for name, cookie in s.items():
self.set_cookie(
name,
cookie.value,
domain=cookie.get('domain'),
# expires= commonutils.covertExpires2timestampe(cookie.get('expires')),
path=cookie.get('path'),
httponly=cookie.get('httponly'),
max_age=cookie.get('max-age'))
# try:
# self.set_cookie(name=name, value=value, **kw)
# except Exception as e:
# traceback.print_exc()
# 修正content-type
res = commonutils.Covert.covertStr2Bytes(item.get('item_res'))
if url.lower()[-4:] in self.imgtypes:
if url.lower()[-4:] == '.jpg':
self.set_header('Content-Type', 'image/jpeg')
else:
self.set_header('Content-Type',
'image/{imgtype}'.format(imgtype=url[-3:]))
self.write(res)
return
if len(re.findall(r'.*css.*', url.lower())) > 0:
self.set_header('Content-Type', 'text/css')
elif len(re.findall(r'.*js.*', url.lower())) > 0:
self.set_header('Content-Type', 'application/javascript')
# res = str(res, encoding='utf-8', errors='ignore')
encoding = requests.utils.get_encodings_from_content(str(res))
if len(encoding) > 0:
encoding = encoding[0]
else:
encoding = 'utf-8'
# print('encoding: ', encoding)
try:
res = str(res, encoding=encoding, errors='ignore')
except Exception as e:
print(e, res)
res = self.covertHTML2SafeHTML(res)
try:
res = json.loads(res)
except Exception as e:
print(e, )
self.write(res)
def get_cookie():
cookie = SimpleCookie()
cookie.load(COOKIE_STR)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def cookie_string_to_mapping(text):
cookie = SimpleCookie()
cookie.load(text)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def build_response(vcr_request, vcr_response, history):
request_info = RequestInfo(
url=URL(vcr_request.url),
method=vcr_request.method,
headers=_deserialize_headers(vcr_request.headers),
real_url=URL(vcr_request.url),
)
response = MockClientResponse(vcr_request.method, URL(vcr_response.get("url")), request_info=request_info)
response.status = vcr_response["status"]["code"]
response._body = vcr_response["body"].get("string", b"")
response.reason = vcr_response["status"]["message"]
response._headers = _deserialize_headers(vcr_response["headers"])
response._history = tuple(history)
# cookies
for hdr in response.headers.getall(hdrs.SET_COOKIE, ()):
try:
cookies = SimpleCookie(hdr)
for cookie_name, cookie in cookies.items():
expires = cookie.get("expires", "").strip()
if expires:
log.debug('Ignoring expiration date: %s="%s"', cookie_name, expires)
cookie["expires"] = ""
response.cookies.load(cookie.output(header="").strip())
except CookieError as exc:
log.warning("Can not load response cookies: %s", exc)
response.close()
return response
class Session(SyncCallMixin):
def __init__(self, cookies=None):
self.cookie = SimpleCookie(cookies)
def _get_cookie(self, response):
for c in response.headers.get_list("Set-Cookie"):
self.cookie.load(c)
def cookie_output(self, keys=None):
if keys:
_ck = SimpleCookie({
k: v
for k, v in self.cookie.items()
if k in keys
})
return _ck.output(attrs=[], header="", sep=";")
else:
return self.cookie.output(attrs=[], header="", sep=";")
async def call(self, client, *args, **kwargs):
headers = kwargs.get('headers', {})
if self.cookie:
headers['Cookie'] = self.cookie_output()
kwargs['headers'] = headers
cli = client()
data = await cli.call(*args, **kwargs)
self._get_cookie(cli.response)
return data
def cookies_from_scope(scope):
cookie = dict(scope.get("headers") or {}).get(b"cookie")
if not cookie:
return {}
simple_cookie = SimpleCookie()
simple_cookie.load(cookie.decode("utf8"))
return {key: morsel.value for key, morsel in simple_cookie.items()}
def parse_cookies(self):
cookies = {}
if self.headers.get('cookie'):
cookie_string = self.headers.get('cookie')[0]
s = SimpleCookie(cookie_string)
cookies = {v.key: v.value for k, v in s.items()}
self.cookies = cookies
def parse_cookies(query: str) -> dict:
cookie = SimpleCookie()
cookie.load(query)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def login():
if request.method == 'POST':
# session['username'] = request.form['username']
# session['password'] = request.form['password']
# session['validateCode'] = request.form['validateCode']
cookie_string = session['cookie']
cookie = SimpleCookie()
cookie.load(cookie_string)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
username = request.form['username']
password = request.form['password']
validcode = request.form['validateCode']
r = requests.post('http://elite.nju.edu.cn/jiaowu/login.do',
data={
'userName': username,
'password': password,
'ValidateCode': validcode
},
cookies=cookies)
# print(r.content.decode('utf-8'))
tmp = r.content.decode('utf-8')
if tmp.find("验证码错误！") != -1:
return jsonify({"status": "validate error"})
elif tmp.find("用户名错误！") != -1:
return jsonify({"status": "username error"})
elif tmp.find("密码错误！") != -1:
return jsonify({"status": "password error"})
else:
return jsonify({"status": "success"})
return redirect(url_for('index'))
def cookie_to_dict(c):
if c is None:
return c
cookie = SimpleCookie()
cookie.load(c)
return {k: v.value for (k, v) in cookie.items()}
def cookie_dict_from_cookie_str(cookie_str):
"""cookie_dict_from_str Cookie字符串返回成dict
:param cookie_str: cookies string
"""
cookie = SimpleCookie()
cookie.load(cookie_str)
return {key: morsel.value for key, morsel in cookie.items()}
def set_cookies(self, cookies):
"""
Sets new cookies from a string
"""
c = SimpleCookie()
c.load(cookies)
for key, m in c.items():
self.s.cookies.set(key, m.value)
def cookies(self) -> Mapping[str, str]:
"""Return request cookies.
A read-only dictionary-like object.
"""
raw = self.headers.get(hdrs.COOKIE, "")
parsed = SimpleCookie(raw) # type: SimpleCookie[str]
return MappingProxyType({key: val.value for key, val in parsed.items()})
def cookie_dict_from_cookie_str(cookie_str):
"""cookie_dict_from_str Cookie字符串返回成dict
:param cookie_str: cookies string
"""
cookie = SimpleCookie()
cookie.load(cookie_str)
return {key: morsel.value for key, morsel in cookie.items()}
def pull_cookies(environ):
"""
Pulls and formats cookies stored by user for domain.
<http://pwp.stevecassidy.net/wsgi/cookies.html>
"""
cookie = SimpleCookie(environ.get('HTTP_COOKIE', ''))
return {key: morsel.value for key, morsel in cookie.items()}
def cookies(self):
"""Return request cookies.
A read-only dictionary-like object.
"""
raw = self.headers.get(hdrs.COOKIE, '')
parsed = SimpleCookie(raw)
return MappingProxyType(
{key: val.value for key, val in parsed.items()})
def cookies(self):
if self._cookies is None:
cookie = self.headers.get('Cookie') or self.headers.get('cookie')
if cookie is not None:
cookies = SimpleCookie()
cookies.load(cookie)
self._cookies = {name: cookie.value
for name, cookie in cookies.items()}
else:
self._cookies = {}
return self._cookies
def COOKIES(self):
if self._cookies is None:
try:
c = SimpleCookie(self.ENV.get('HTTP_COOKIE', ''))
except CookieError: # pragma: no cover
self._cookies = {}
else:
res = {}
for name, value in c.items():
res[name] = value.value
self._cookies = res
return self._cookies
def accept_cookies_for_request(self, cookies: HTTPCookies):
"""
Insert all the cookies as a request cookie header.
"""
cookie_string = ""
if "cookie" in self.keys():
cookie_string += self["cookie"]
for cookie_name, cookie_morsel in cookies.items():
cookie_string += "%(cookie_name)s=%(cookie_value)s; " % {
"cookie_name": cookie_name,
"cookie_value": cookie_morsel.value
}
if cookie_string:
self["cookie"] = cookie_string
def addLinkSpider(add_link_dictionary):
# get user's download information from add_link_dictionary
for i in ['link', 'header', 'out', 'user-agent', 'load-cookies', 'referer']:
if not (i in add_link_dictionary):
add_link_dictionary[i] = None
link = add_link_dictionary['link']
header = add_link_dictionary['header']
user_agent = add_link_dictionary['user-agent']
raw_cookies = add_link_dictionary['load-cookies']
referer = add_link_dictionary['referer']
requests_session = requests.Session() # defining a requests Session
if raw_cookies: # set cookies
cookie = SimpleCookie()
cookie.load(raw_cookies)
cookies = {key: morsel.value for key, morsel in cookie.items()}
requests_session.cookies = cookiejar_from_dict(cookies)
if referer:
# set referer to the session
requests_session.headers.update({'referer': referer})
if user_agent:
# set user_agent to the session
requests_session.headers.update({'user-agent': user_agent})
# find headers
response = requests_session.head(link)
header = response.headers
file_size = None
if 'Content-Length' in header.keys(): # checking if file_size is available
file_size = int(header['Content-Length'])
if int(file_size/1073741824) != 0: # converting file_size to KB or MB or GB
file_size = file_size/1073741824
size_str = str(round(file_size, 2)) + " GB"
elif int(file_size/1048576) != 0:
size_str = str(int(file_size/1048576)) + " MB"
elif int(file_size/1024) != 0:
size_str = str(int(file_size/1024)) + " KB"
else:
size_str = str(file_size)
filesize = size_str
return filesize
def queueSpider(add_link_dictionary):
# get download information from add_link_dictionary
for i in ['link', 'header', 'out', 'user_agent', 'load_cookies', 'referer']:
if not (i in add_link_dictionary):
add_link_dictionary[i] = None
link = add_link_dictionary['link']
header = add_link_dictionary['header']
user_agent = add_link_dictionary['user_agent']
raw_cookies = add_link_dictionary['load_cookies']
referer = add_link_dictionary['referer']
requests_session = requests.Session() # defining a requests Session
if raw_cookies: # set cookies
cookie = SimpleCookie()
cookie.load(raw_cookies)
cookies = {key: morsel.value for key, morsel in cookie.items()}
requests_session.cookies = cookiejar_from_dict(cookies)
if referer:
# set referer to the session
requests_session.headers.update({'referer': referer})
if user_agent:
# set user_agent to the session
requests_session.headers.update({'user-agent': user_agent})
# find headers
try:
response = requests_session.head(link)
header = response.headers
except:
header = {}
filename = None
if 'Content-Disposition' in header.keys(): # checking if filename is available
content_disposition = header['Content-Disposition']
if content_disposition.find('filename') != -1:
filename_splited = content_disposition.split('filename=')
filename_splited = filename_splited[-1]
# getting file name in desired format
filename = filename_splited[1:-1]
if not(filename):
filename = link.split('/')[-1]
return filename
def addLinkSpider(add_link_dictionary):
# get user's download information from add_link_dictionary
for i in ['link', 'header', 'out', 'user_agent', 'load_cookies', 'referer']:
if not (i in add_link_dictionary):
add_link_dictionary[i] = None
link = add_link_dictionary['link']
header = add_link_dictionary['header']
user_agent = add_link_dictionary['user_agent']
raw_cookies = add_link_dictionary['load_cookies']
referer = add_link_dictionary['referer']
requests_session = requests.Session() # defining a requests Session
if raw_cookies: # set cookies
cookie = SimpleCookie()
cookie.load(raw_cookies)
cookies = {key: morsel.value for key, morsel in cookie.items()}
requests_session.cookies = cookiejar_from_dict(cookies)
if referer:
# set referer to the session
requests_session.headers.update({'referer': referer})
if user_agent:
# set user_agent to the session
requests_session.headers.update({'user-agent': user_agent})
# find headers
try:
response = requests_session.head(link)
header = response.headers
except:
header = {}
file_size = None
if 'Content-Length' in header.keys(): # checking if file_size is available
file_size = int(header['Content-Length'])
# converting file_size to KiB or MiB or GiB
file_size = humanReadbleSize(file_size)
return file_size # If no Content-Length ? fixed it.
def decompose_incoming_envelope(self, ctx, message):
assert message == SimpleDictDocument.REQUEST
ctx.transport.itself.decompose_incoming_envelope(self, ctx, message)
if self.parse_cookie:
cookies = ctx.in_header_doc.get('cookie', None)
if cookies is None:
cookies = ctx.in_header_doc.get('Cookie', None)
if cookies is not None:
for cookie_string in cookies:
cookie = SimpleCookie()
cookie.load(cookie_string)
for k,v in cookie.items():
l = ctx.in_header_doc.get(k, [])
l.append(v.coded_value)
ctx.in_header_doc[k] = l
logger.debug('\theader : %r' % (ctx.in_header_doc))
logger.debug('\tbody : %r' % (ctx.in_body_doc))
def build(resp):
response = Response()
# Pass settings over.
response.config = self.config
if resp:
# Fallback to None if there's no staus_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))
# Start off with our local cookies.
cookies = self.cookies or dict()
# Add new cookies from the server.
if 'set-cookie' in response.headers:
cookie_header = response.headers['set-cookie']
c = SimpleCookie()
c.load(cookie_header)
for k,v in list(c.items()):
cookies.update({k: v.value})
# Save cookies in Response.
response.cookies = cookies
# Save original resopnse for later.
response.raw = resp
if is_error:
response.error = resp
response.url = self.full_url
return response
def _get_cookies():
cookie = SimpleCookie()
cookie.load(config.get('cookie'))
return {key: morsel.value for key, morsel in cookie.items()}
def get_cookie_header(cookies: SimpleCookie) -> str:
return ';'.join(
["{}={}".format(name, value.value) for name, value in cookies.items()])
class CookieJar:
"""Implements cookie storage adhering to RFC 6265."""
DATE_TOKENS_RE = re.compile(
"[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E]*"
"(?P<token>[\x00-\x08\x0A-\x1F\d:a-zA-Z\x7F-\xFF]+)")
DATE_HMS_TIME_RE = re.compile("(\d{1,2}):(\d{1,2}):(\d{1,2})")
DATE_DAY_OF_MONTH_RE = re.compile("(\d{1,2})")
DATE_MONTH_RE = re.compile(
"(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)", re.I)
DATE_YEAR_RE = re.compile("(\d{2,4})")
def __init__(self, cookies=None, loop=None):
self._cookies = SimpleCookie()
self._loop = loop or asyncio.get_event_loop()
self._host_only_cookies = set()
if cookies is not None:
self.update_cookies(cookies)
@property
def cookies(self):
"""The session cookies."""
return self._cookies
def _expire_cookie(self, name):
if name in self._cookies:
del self._cookies[name]
def update_cookies(self, cookies, response_url=None):
"""Update cookies."""
url_parsed = urlsplit(response_url or "")
hostname = url_parsed.hostname
if is_ip_address(hostname):
# Don't accept cookies from IPs
return
if isinstance(cookies, dict):
cookies = cookies.items()
for name, value in cookies:
if isinstance(value, Morsel):
if not self._add_morsel(name, value, hostname):
continue
else:
self._cookies[name] = value
cookie = self._cookies[name]
if not cookie["domain"] and hostname is not None:
# Set the cookie's domain to the response hostname
# and set its host-only-flag
self._host_only_cookies.add(name)
cookie["domain"] = hostname
if not cookie["path"] or not cookie["path"].startswith("/"):
# Set the cookie's path to the response path
path = url_parsed.path
if not path.startswith("/"):
path = "/"
else:
# Cut everything from the last slash to the end
path = "/" + path[1:path.rfind("/")]
cookie["path"] = path
max_age = cookie["max-age"]
if max_age:
try:
delta_seconds = int(max_age)
self._loop.call_later(
delta_seconds, self._expire_cookie, name)
except ValueError:
cookie["max-age"] = ""
expires = cookie["expires"]
if not cookie["max-age"] and expires:
expire_time = self._parse_date(expires)
if expire_time:
self._loop.call_at(
expire_time.timestamp(),
self._expire_cookie, name)
else:
cookie["expires"] = ""
# Remove the host-only flags of nonexistent cookies
self._host_only_cookies -= (
self._host_only_cookies.difference(self._cookies.keys()))
def _add_morsel(self, name, value, hostname):
"""Add a Morsel to the cookie jar."""
cookie_domain = value["domain"]
if cookie_domain.startswith("."):
# Remove leading dot
cookie_domain = cookie_domain[1:]
value["domain"] = cookie_domain
if not cookie_domain or not hostname:
dict.__setitem__(self._cookies, name, value)
return True
if not self._is_domain_match(cookie_domain, hostname):
# Setting cookies for different domains is not allowed
return False
# use dict method because SimpleCookie class modifies value
# before Python 3.4
dict.__setitem__(self._cookies, name, value)
return True
def filter_cookies(self, request_url):
"""Returns this jar's cookies filtered by their attributes."""
url_parsed = urlsplit(request_url)
filtered = SimpleCookie()
for name, cookie in self._cookies.items():
cookie_domain = cookie["domain"]
# Send shared cookies
if not cookie_domain:
dict.__setitem__(filtered, name, cookie)
continue
hostname = url_parsed.hostname or ""
if is_ip_address(hostname):
continue
if name in self._host_only_cookies:
if cookie_domain != hostname:
continue
elif not self._is_domain_match(cookie_domain, hostname):
continue
if not self._is_path_match(url_parsed.path, cookie["path"]):
continue
is_secure = url_parsed.scheme in ("https", "wss")
if cookie["secure"] and not is_secure:
continue
dict.__setitem__(filtered, name, cookie)
return filtered
@staticmethod
def _is_domain_match(domain, hostname):
"""Implements domain matching adhering to RFC 6265."""
if hostname == domain:
return True
if not hostname.endswith(domain):
return False
non_matching = hostname[:-len(domain)]
if not non_matching.endswith("."):
return False
return not is_ip_address(hostname)
@staticmethod
def _is_path_match(req_path, cookie_path):
"""Implements path matching adhering to RFC 6265."""
if req_path == cookie_path:
return True
if not req_path.startswith(cookie_path):
return False
if cookie_path.endswith("/"):
return True
non_matching = req_path[len(cookie_path):]
return non_matching.startswith("/")
@classmethod
def _parse_date(cls, date_str):
"""Implements date string parsing adhering to RFC 6265."""
if not date_str:
return
found_time = False
found_day_of_month = False
found_month = False
found_year = False
hour = minute = second = 0
day_of_month = 0
month = ""
year = 0
for token_match in cls.DATE_TOKENS_RE.finditer(date_str):
token = token_match.group("token")
if not found_time:
time_match = cls.DATE_HMS_TIME_RE.match(token)
if time_match:
found_time = True
hour, minute, second = [
int(s) for s in time_match.groups()]
continue
if not found_day_of_month:
day_of_month_match = cls.DATE_DAY_OF_MONTH_RE.match(token)
if day_of_month_match:
found_day_of_month = True
day_of_month = int(day_of_month_match.group())
continue
if not found_month:
month_match = cls.DATE_MONTH_RE.match(token)
if month_match:
found_month = True
month = month_match.group()
continue
if not found_year:
year_match = cls.DATE_YEAR_RE.match(token)
if year_match:
found_year = True
year = int(year_match.group())
if 70 <= year <= 99:
year += 1900
elif 0 <= year <= 69:
year += 2000
if False in (found_day_of_month, found_month, found_year, found_time):
return
if not 1 <= day_of_month <= 31:
return
if year < 1601 or hour > 23 or minute > 59 or second > 59:
return
dt = datetime.datetime.strptime(
"%s %d %d:%d:%d %d" % (
month, day_of_month, hour, minute, second, year
), "%b %d %H:%M:%S %Y")
return dt.replace(tzinfo=datetime.timezone.utc)
def spider(add_link_dictionary):
# get user's download request from add_link_dictionary
link = add_link_dictionary['link']
ip = add_link_dictionary['ip']
port = add_link_dictionary['port']
proxy_user = add_link_dictionary['proxy_user']
proxy_passwd = add_link_dictionary['proxy_passwd']
download_user = add_link_dictionary['download_user']
download_passwd = add_link_dictionary['download_passwd']
header = add_link_dictionary['header']
out = add_link_dictionary['out']
user_agent = add_link_dictionary['user_agent']
raw_cookies = add_link_dictionary['load_cookies']
referer = add_link_dictionary['referer']
# defin a requests session
requests_session = requests.Session()
if ip:
ip_port = 'http://' + str(ip) + ":" + str(port)
if proxy_user:
ip_port = 'http://' + proxy_user + ':' + proxy_passwd + '@' + ip_port
# set proxy to the session
requests_session.proxies = {'http': ip_port}
if download_user:
# set download user pass to the session
requests_session.auth(download_user, download_passwd)
# set cookies
if raw_cookies:
cookie = SimpleCookie()
cookie.load(raw_cookies)
cookies = {key: morsel.value for key, morsel in cookie.items()}
requests_session.cookies = cookiejar_from_dict(cookies)
# set referer
if referer:
requests_session.headers.update({'referer': referer }) #setting referer to the session
# set user_agent
if user_agent:
requests_session.headers.update({'user-agent':user_agent }) #setting user_agent to the session
#find headers
try:
response = requests_session.head(link)
header = response.headers
except:
header = {}
filename = None
filesize = None
if 'Content-Disposition' in header.keys(): # checking if filename is available
content_disposition = header['Content-Disposition']
if content_disposition.find('filename') != -1:
filename_splited = content_disposition.split('filename=')
filename_splited = filename_splited[-1]
# getting file name in desired format
filename = filename_splited[1:-1]
if not(filename):
filename = link.split('/')[-1]
# if user set file name before in add_link_dictionary['out'],
# then set "out" for filename
if out:
filename = out
# check if file_size is available
if 'Content-Length' in header.keys():
file_size = int(header['Content-Length'])
# converting file_size to KiB or MiB or GiB
file_size = humanReadbleSize(file_size)
# return results
return filename, filesize
