async def verified_user_id(message, request): print("VUI") try: action = json.loads(message) # Get token, return if not specified token = action.get("token") headers = {k.decode(): v.decode() for k, v in request.headers} print("H", headers) cookies = SimpleCookie(headers.get('cookie')) print("C", cookies) for key, morsel in cookies.items(): if key == 'token': token = morsel.value if not token: return # DEBUG if token == 'machine': debug_handle() return 1 # decode the JWT token, get user_id from it try: identity = jwt.decode(token, 'secret', alorithms=['HS256']) return identity['user_id'] except jwt.DecodeError as e: return except json.JSONDecodeError as e: log.error("JSON decode error %s" % e) return
def handle_route(self, route, params, cookies, set_cookie): """Common code for GET and POST commands to handle route and send json result.""" if route.____useCookies__: t, cookie, result = route(cookies = cookies, **params) else: t, cookie, result = route(**params) cookie is not None and cookies.load(cookie) if set_cookie: if cookie is None: cookie = SimpleCookie(set_cookie) else: cookie.load(set_cookie) if t == ROUTE_REDIRECT: return self.redirect(result, cookie) elif t == ROUTE_FORWARD: route = self.dispatch(result[0]) return callable(route) and route or self.handle_route(route, result[1], cookies, cookie) if route.____template__ is None: content_type = 'text/json' result = json.dumps(result) else: content_type = 'text/html' result = self.template(route.____template__, result) result = encodeUTF8(result) headers = [('Content-type', content_type)] if cookie is not None: for key, value in sorted(cookie.items()): headers.append(('Set-Cookie', value.OutputString())) self.start_response('200 OK', headers) return iter([result])
def __call__(self, env, start_response): # WSGI应用 method = env['REQUEST_METHOD'] # 请求方法 path = env['PATH_INFO'] # 请求路径 cookie_str = env['HTTP_COOKIE'] # Cookies query = env['QUERY_STRING'] req_obj = None if method == 'POST': size = int(env.get('CONTENT_LENGTH', 0)) query = env['wsgi.input'].read(size).decode('utf-8') req_obj = parse_qs(query) # 处理请求参数 sc = SimpleCookie() sc.load(cookie_str) # 处理Cookie req_obj['cookies'] = {k:m.value for k, m in sc.items()} handler = self.handlers[method].get(path, False)# 获取handler ck_dict = None if not handler: status = '404 Not Found' content = page404.encode('utf-8') else: status = '200 OK' content = handler(req_obj) # 执行handler if isinstance(content, tuple): content, ck_dict = content content = content.encode('utf-8') headers = [('Content-Type', 'text/html'), ('Content-Length', str(len(content)))] if ck_dict: set_ck = ';'.join([f'{k}={v}' for k, v in ck_dict.items()]) headers.append(('Set-Cookie', set_ck)) start_response(status, headers) return [content]
def get_hedy_cookie(self, cookie_string): cookie = SimpleCookie() cookie.load(cookie_string) for key, cookie in cookie.items(): if key == CONFIG['session']['cookie_name']: return cookie
def __init__(self, username, cookies=""): self.username = username try: cookie = SimpleCookie() cookie.load(cookies) self.cookies = {} for key, morsel in cookie.items(): self.cookies[key] = morsel.value except: pass self.similar = {} self.info = { "similar": {}, 'bio': "", "bio_url": "", 'followers': "", "follow": "", 'name': "", 'profile_pic': "", "descriptions": [], "captions": [], 'locations': {}, 'timestamps': {} } self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" }
def __init__(self, base_url, cookies=None): self.base_url = base_url if cookies is not None: jar = SimpleCookie() jar.load(cookies) self.cookies = {key: morsel.value for key, morsel in jar.items()}
def cookie_dict(self, cookiestr): cookie = SimpleCookie() cookie.load(cookiestr) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value return cookies
def view(request, path): morepath_request = MorepathRequest(request.environ, app) # consume everything already seen by django for i in range(resolved_segment_count(request.path, path)): morepath_request.unconsumed.pop() morepath_request.make_body_seekable() response = app.publish(morepath_request) cookies = SimpleCookie() django_response = HttpResponse( response.body, status=response.status_code ) for (header, value) in response.headerlist: if header.upper() == "SET-COOKIE": cookies.load(value) else: django_response[header] = value for cookie_name, cookie in cookies.items(): cookie_attributes = { "key": cookie_name, "value": cookie.value, "expires": cookie["expires"], "path": cookie["path"], "domain": cookie["domain"], } if cookie["max-age"]: # Starting in Django 1.3 it performs arithmetic operations # with 'Max-Age' cookie_attributes["max_age"] = int(cookie["max-age"]) django_response.set_cookie(**cookie_attributes) return django_response
def parse_cookie(rawdata): if rawdata == None: return rawdata cookie = SimpleCookie() cookie.load(rawdata) cookies = {} for key, data in cookie.items(): cookies[key] = data.value return cookies # def parse_url(url): # try: # host = urllib3.util.url.parse_url(url).host # except Exception as e: # print("Invalid domain, try again..") # sys.exit(1) # return host # def parse_wordlist(wordlist): # try: # wordlists = open(wordlist).read().splitlines() # except Exception as e: # print(e) # sys.exit(1) # return wordlists
def store_cookies(self, response): if not "Set-Cookie" in response.headers: return cookies = SimpleCookie(response.headers["Set-Cookie"]) for key, item in cookies.items(): self.client.cookies.set(item.key, item.value)
def convertcookie(data): cookie = SimpleCookie() cookie.load(data) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value return cookies
def get_cookie_dict(cookie_str: str) -> dict: cookie = SimpleCookie() cookie.load(cookie_str) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value return cookies
def loginGET(environ, start_response): unfromated_cookie = environ.get("HTTP_COOKIE", "") cookie = SimpleCookie() cookie.load(unfromated_cookie) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value passw = cookies['passw'] uname = cookies['uname'] passw = decript(passw) login_status = login(uname, passw) if login_status == 1: start_response('200 OK', [('Content-text', 'text/plain')]) encoded_jwt = jwt.encode({ 'uname': uname, 'iat': datetime.utcnow() }, 'secret', algorithm='HS256') yield encoded_jwt elif login_status == 2: start_response('401 Unauthorized', [('Content-text', 'text/plain')]) message = {"message": "Bad username or password"} yield json.dumps(message).encode('utf-8') elif login_status == 3: start_response("500 Internal Server Error", [('Content-text', 'text/plain')]) message = {"message": "Databease problem"} yield json.dumps(message).encode('utf-8')
def start(self): cookie = SimpleCookie() request_cookies = {} if self.cookies is not None: cookie.load(self.cookies) for key, morsel in cookie.items(): request_cookies[key] = morsel.value else: print("[*] No cookie set, continuing...") count_matched = 0 for payload in self.payloads: for url in self.urls: urlTarget = url + payload r = requests.get(urlTarget, cookies=request_cookies) if self.verbose: print('GET [{0}] {1}'.format(r.status_code, urlTarget)) for match in self.matches[payload]: if match in r.text: print("Interesting: " + url + payload) count_matched = count_matched + 1 if "syntax error" in r.text: print("PHP error: " + url + payload) if count_matched == 0: print("[-] Nothing found") print("[*] Scan completed")
def loadCookie(rawdata): cookie = SimpleCookie() cookie.load(rawdata) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value return cookies
def on_open(self): """On Open Will be called when a new client connects to the server """ if _verbose: print('on_open called') # If there is no cookie if 'HTTP_COOKIE' not in self.ws.environ: if _verbose: print('HTTP_COOKIE missing') return self._fail(11, 'HTTP_COOKIE missing') # If the cookie lacks the PHPSESSID value oCookies = SimpleCookie() oCookies.load(self.ws.environ['HTTP_COOKIE']) dCookies = {k: v.value for k, v in oCookies.items()} if '_session' not in dCookies: if _verbose: print('_session not in cookies') return self._fail(12, '_session not in cookies') # Store the session ID self.token = dCookies['_session'] self.authorized = False self.tracking = []
def updateheader(self, url, item): headers = json.loads(item.get('item_headers')) # if ':' in name] if 'Host' in headers: self.set_header('Host', headers.get('Host')) if 'Referer' in headers: self.set_header('Referer', headers.get('Referer')) if 'Set-Cookie' in headers: for cookiestr in commonutils.covertSetCookie2List( headers.get('Set-Cookie')): s = SimpleCookie() s.load(cookiestr) for name, cookie in s.items(): self.set_cookie( name, cookie.value, domain=cookie.get('domain'), # expires= commonutils.covertExpires2timestampe(cookie.get('expires')), path=cookie.get('path'), httponly=cookie.get('httponly'), max_age=cookie.get('max-age')) # try: # self.set_cookie(name=name, value=value, **kw) # except Exception as e: # traceback.print_exc() # 修正content-type res = commonutils.Covert.covertStr2Bytes(item.get('item_res')) if url.lower()[-4:] in self.imgtypes: if url.lower()[-4:] == '.jpg': self.set_header('Content-Type', 'image/jpeg') else: self.set_header('Content-Type', 'image/{imgtype}'.format(imgtype=url[-3:])) self.write(res) return if len(re.findall(r'.*css.*', url.lower())) > 0: self.set_header('Content-Type', 'text/css') elif len(re.findall(r'.*js.*', url.lower())) > 0: self.set_header('Content-Type', 'application/javascript') # res = str(res, encoding='utf-8', errors='ignore') encoding = requests.utils.get_encodings_from_content(str(res)) if len(encoding) > 0: encoding = encoding[0] else: encoding = 'utf-8' # print('encoding: ', encoding) try: res = str(res, encoding=encoding, errors='ignore') except Exception as e: print(e, res) res = self.covertHTML2SafeHTML(res) try: res = json.loads(res) except Exception as e: print(e, ) self.write(res)
def get_cookie(): cookie = SimpleCookie() cookie.load(COOKIE_STR) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value return cookies
def cookie_string_to_mapping(text): cookie = SimpleCookie() cookie.load(text) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value return cookies
def build_response(vcr_request, vcr_response, history): request_info = RequestInfo( url=URL(vcr_request.url), method=vcr_request.method, headers=_deserialize_headers(vcr_request.headers), real_url=URL(vcr_request.url), ) response = MockClientResponse(vcr_request.method, URL(vcr_response.get("url")), request_info=request_info) response.status = vcr_response["status"]["code"] response._body = vcr_response["body"].get("string", b"") response.reason = vcr_response["status"]["message"] response._headers = _deserialize_headers(vcr_response["headers"]) response._history = tuple(history) # cookies for hdr in response.headers.getall(hdrs.SET_COOKIE, ()): try: cookies = SimpleCookie(hdr) for cookie_name, cookie in cookies.items(): expires = cookie.get("expires", "").strip() if expires: log.debug('Ignoring expiration date: %s="%s"', cookie_name, expires) cookie["expires"] = "" response.cookies.load(cookie.output(header="").strip()) except CookieError as exc: log.warning("Can not load response cookies: %s", exc) response.close() return response
class Session(SyncCallMixin): def __init__(self, cookies=None): self.cookie = SimpleCookie(cookies) def _get_cookie(self, response): for c in response.headers.get_list("Set-Cookie"): self.cookie.load(c) def cookie_output(self, keys=None): if keys: _ck = SimpleCookie({ k: v for k, v in self.cookie.items() if k in keys }) return _ck.output(attrs=[], header="", sep=";") else: return self.cookie.output(attrs=[], header="", sep=";") async def call(self, client, *args, **kwargs): headers = kwargs.get('headers', {}) if self.cookie: headers['Cookie'] = self.cookie_output() kwargs['headers'] = headers cli = client() data = await cli.call(*args, **kwargs) self._get_cookie(cli.response) return data
def cookies_from_scope(scope): cookie = dict(scope.get("headers") or {}).get(b"cookie") if not cookie: return {} simple_cookie = SimpleCookie() simple_cookie.load(cookie.decode("utf8")) return {key: morsel.value for key, morsel in simple_cookie.items()}
def parse_cookies(self): cookies = {} if self.headers.get('cookie'): cookie_string = self.headers.get('cookie')[0] s = SimpleCookie(cookie_string) cookies = {v.key: v.value for k, v in s.items()} self.cookies = cookies
def parse_cookies(query: str) -> dict: cookie = SimpleCookie() cookie.load(query) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value return cookies
def login(): if request.method == 'POST': # session['username'] = request.form['username'] # session['password'] = request.form['password'] # session['validateCode'] = request.form['validateCode'] cookie_string = session['cookie'] cookie = SimpleCookie() cookie.load(cookie_string) cookies = {} for key, morsel in cookie.items(): cookies[key] = morsel.value username = request.form['username'] password = request.form['password'] validcode = request.form['validateCode'] r = requests.post('http://elite.nju.edu.cn/jiaowu/login.do', data={ 'userName': username, 'password': password, 'ValidateCode': validcode }, cookies=cookies) # print(r.content.decode('utf-8')) tmp = r.content.decode('utf-8') if tmp.find("验证码错误!") != -1: return jsonify({"status": "validate error"}) elif tmp.find("用户名错误!") != -1: return jsonify({"status": "username error"}) elif tmp.find("密码错误!") != -1: return jsonify({"status": "password error"}) else: return jsonify({"status": "success"}) return redirect(url_for('index'))
def cookie_to_dict(c): if c is None: return c cookie = SimpleCookie() cookie.load(c) return {k: v.value for (k, v) in cookie.items()}
def cookie_dict_from_cookie_str(cookie_str): """cookie_dict_from_str Cookie字符串返回成dict :param cookie_str: cookies string """ cookie = SimpleCookie() cookie.load(cookie_str) return {key: morsel.value for key, morsel in cookie.items()}
def set_cookies(self, cookies): """ Sets new cookies from a string """ c = SimpleCookie() c.load(cookies) for key, m in c.items(): self.s.cookies.set(key, m.value)
def cookies(self) -> Mapping[str, str]: """Return request cookies. A read-only dictionary-like object. """ raw = self.headers.get(hdrs.COOKIE, "") parsed = SimpleCookie(raw) # type: SimpleCookie[str] return MappingProxyType({key: val.value for key, val in parsed.items()})
def pull_cookies(environ): """ Pulls and formats cookies stored by user for domain. <http://pwp.stevecassidy.net/wsgi/cookies.html> """ cookie = SimpleCookie(environ.get('HTTP_COOKIE', '')) return {key: morsel.value for key, morsel in cookie.items()}
def cookies(self): """Return request cookies. A read-only dictionary-like object. """ raw = self.headers.get(hdrs.COOKIE, '') parsed = SimpleCookie(raw) return MappingProxyType( {key: val.value for key, val in parsed.items()})
def cookies(self): if self._cookies is None: cookie = self.headers.get('Cookie') or self.headers.get('cookie') if cookie is not None: cookies = SimpleCookie() cookies.load(cookie) self._cookies = {name: cookie.value for name, cookie in cookies.items()} else: self._cookies = {} return self._cookies
def COOKIES(self): if self._cookies is None: try: c = SimpleCookie(self.ENV.get('HTTP_COOKIE', '')) except CookieError: # pragma: no cover self._cookies = {} else: res = {} for name, value in c.items(): res[name] = value.value self._cookies = res return self._cookies
def accept_cookies_for_request(self, cookies: HTTPCookies): """ Insert all the cookies as a request cookie header. """ cookie_string = "" if "cookie" in self.keys(): cookie_string += self["cookie"] for cookie_name, cookie_morsel in cookies.items(): cookie_string += "%(cookie_name)s=%(cookie_value)s; " % { "cookie_name": cookie_name, "cookie_value": cookie_morsel.value } if cookie_string: self["cookie"] = cookie_string
def addLinkSpider(add_link_dictionary): # get user's download information from add_link_dictionary for i in ['link', 'header', 'out', 'user-agent', 'load-cookies', 'referer']: if not (i in add_link_dictionary): add_link_dictionary[i] = None link = add_link_dictionary['link'] header = add_link_dictionary['header'] user_agent = add_link_dictionary['user-agent'] raw_cookies = add_link_dictionary['load-cookies'] referer = add_link_dictionary['referer'] requests_session = requests.Session() # defining a requests Session if raw_cookies: # set cookies cookie = SimpleCookie() cookie.load(raw_cookies) cookies = {key: morsel.value for key, morsel in cookie.items()} requests_session.cookies = cookiejar_from_dict(cookies) if referer: # set referer to the session requests_session.headers.update({'referer': referer}) if user_agent: # set user_agent to the session requests_session.headers.update({'user-agent': user_agent}) # find headers response = requests_session.head(link) header = response.headers file_size = None if 'Content-Length' in header.keys(): # checking if file_size is available file_size = int(header['Content-Length']) if int(file_size/1073741824) != 0: # converting file_size to KB or MB or GB file_size = file_size/1073741824 size_str = str(round(file_size, 2)) + " GB" elif int(file_size/1048576) != 0: size_str = str(int(file_size/1048576)) + " MB" elif int(file_size/1024) != 0: size_str = str(int(file_size/1024)) + " KB" else: size_str = str(file_size) filesize = size_str return filesize
def queueSpider(add_link_dictionary): # get download information from add_link_dictionary for i in ['link', 'header', 'out', 'user_agent', 'load_cookies', 'referer']: if not (i in add_link_dictionary): add_link_dictionary[i] = None link = add_link_dictionary['link'] header = add_link_dictionary['header'] user_agent = add_link_dictionary['user_agent'] raw_cookies = add_link_dictionary['load_cookies'] referer = add_link_dictionary['referer'] requests_session = requests.Session() # defining a requests Session if raw_cookies: # set cookies cookie = SimpleCookie() cookie.load(raw_cookies) cookies = {key: morsel.value for key, morsel in cookie.items()} requests_session.cookies = cookiejar_from_dict(cookies) if referer: # set referer to the session requests_session.headers.update({'referer': referer}) if user_agent: # set user_agent to the session requests_session.headers.update({'user-agent': user_agent}) # find headers try: response = requests_session.head(link) header = response.headers except: header = {} filename = None if 'Content-Disposition' in header.keys(): # checking if filename is available content_disposition = header['Content-Disposition'] if content_disposition.find('filename') != -1: filename_splited = content_disposition.split('filename=') filename_splited = filename_splited[-1] # getting file name in desired format filename = filename_splited[1:-1] if not(filename): filename = link.split('/')[-1] return filename
def addLinkSpider(add_link_dictionary): # get user's download information from add_link_dictionary for i in ['link', 'header', 'out', 'user_agent', 'load_cookies', 'referer']: if not (i in add_link_dictionary): add_link_dictionary[i] = None link = add_link_dictionary['link'] header = add_link_dictionary['header'] user_agent = add_link_dictionary['user_agent'] raw_cookies = add_link_dictionary['load_cookies'] referer = add_link_dictionary['referer'] requests_session = requests.Session() # defining a requests Session if raw_cookies: # set cookies cookie = SimpleCookie() cookie.load(raw_cookies) cookies = {key: morsel.value for key, morsel in cookie.items()} requests_session.cookies = cookiejar_from_dict(cookies) if referer: # set referer to the session requests_session.headers.update({'referer': referer}) if user_agent: # set user_agent to the session requests_session.headers.update({'user-agent': user_agent}) # find headers try: response = requests_session.head(link) header = response.headers except: header = {} file_size = None if 'Content-Length' in header.keys(): # checking if file_size is available file_size = int(header['Content-Length']) # converting file_size to KiB or MiB or GiB file_size = humanReadbleSize(file_size) return file_size # If no Content-Length ? fixed it.
def decompose_incoming_envelope(self, ctx, message): assert message == SimpleDictDocument.REQUEST ctx.transport.itself.decompose_incoming_envelope(self, ctx, message) if self.parse_cookie: cookies = ctx.in_header_doc.get('cookie', None) if cookies is None: cookies = ctx.in_header_doc.get('Cookie', None) if cookies is not None: for cookie_string in cookies: cookie = SimpleCookie() cookie.load(cookie_string) for k,v in cookie.items(): l = ctx.in_header_doc.get(k, []) l.append(v.coded_value) ctx.in_header_doc[k] = l logger.debug('\theader : %r' % (ctx.in_header_doc)) logger.debug('\tbody : %r' % (ctx.in_body_doc))
def build(resp): response = Response() # Pass settings over. response.config = self.config if resp: # Fallback to None if there's no staus_code, for whatever reason. response.status_code = getattr(resp, 'status', None) # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) # Start off with our local cookies. cookies = self.cookies or dict() # Add new cookies from the server. if 'set-cookie' in response.headers: cookie_header = response.headers['set-cookie'] c = SimpleCookie() c.load(cookie_header) for k,v in list(c.items()): cookies.update({k: v.value}) # Save cookies in Response. response.cookies = cookies # Save original resopnse for later. response.raw = resp if is_error: response.error = resp response.url = self.full_url return response
def _get_cookies(): cookie = SimpleCookie() cookie.load(config.get('cookie')) return {key: morsel.value for key, morsel in cookie.items()}
def get_cookie_header(cookies: SimpleCookie) -> str: return ';'.join( ["{}={}".format(name, value.value) for name, value in cookies.items()])
class CookieJar: """Implements cookie storage adhering to RFC 6265.""" DATE_TOKENS_RE = re.compile( "[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E]*" "(?P<token>[\x00-\x08\x0A-\x1F\d:a-zA-Z\x7F-\xFF]+)") DATE_HMS_TIME_RE = re.compile("(\d{1,2}):(\d{1,2}):(\d{1,2})") DATE_DAY_OF_MONTH_RE = re.compile("(\d{1,2})") DATE_MONTH_RE = re.compile( "(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)", re.I) DATE_YEAR_RE = re.compile("(\d{2,4})") def __init__(self, cookies=None, loop=None): self._cookies = SimpleCookie() self._loop = loop or asyncio.get_event_loop() self._host_only_cookies = set() if cookies is not None: self.update_cookies(cookies) @property def cookies(self): """The session cookies.""" return self._cookies def _expire_cookie(self, name): if name in self._cookies: del self._cookies[name] def update_cookies(self, cookies, response_url=None): """Update cookies.""" url_parsed = urlsplit(response_url or "") hostname = url_parsed.hostname if is_ip_address(hostname): # Don't accept cookies from IPs return if isinstance(cookies, dict): cookies = cookies.items() for name, value in cookies: if isinstance(value, Morsel): if not self._add_morsel(name, value, hostname): continue else: self._cookies[name] = value cookie = self._cookies[name] if not cookie["domain"] and hostname is not None: # Set the cookie's domain to the response hostname # and set its host-only-flag self._host_only_cookies.add(name) cookie["domain"] = hostname if not cookie["path"] or not cookie["path"].startswith("/"): # Set the cookie's path to the response path path = url_parsed.path if not path.startswith("/"): path = "/" else: # Cut everything from the last slash to the end path = "/" + path[1:path.rfind("/")] cookie["path"] = path max_age = cookie["max-age"] if max_age: try: delta_seconds = int(max_age) self._loop.call_later( delta_seconds, self._expire_cookie, name) except ValueError: cookie["max-age"] = "" expires = cookie["expires"] if not cookie["max-age"] and expires: expire_time = self._parse_date(expires) if expire_time: self._loop.call_at( expire_time.timestamp(), self._expire_cookie, name) else: cookie["expires"] = "" # Remove the host-only flags of nonexistent cookies self._host_only_cookies -= ( self._host_only_cookies.difference(self._cookies.keys())) def _add_morsel(self, name, value, hostname): """Add a Morsel to the cookie jar.""" cookie_domain = value["domain"] if cookie_domain.startswith("."): # Remove leading dot cookie_domain = cookie_domain[1:] value["domain"] = cookie_domain if not cookie_domain or not hostname: dict.__setitem__(self._cookies, name, value) return True if not self._is_domain_match(cookie_domain, hostname): # Setting cookies for different domains is not allowed return False # use dict method because SimpleCookie class modifies value # before Python 3.4 dict.__setitem__(self._cookies, name, value) return True def filter_cookies(self, request_url): """Returns this jar's cookies filtered by their attributes.""" url_parsed = urlsplit(request_url) filtered = SimpleCookie() for name, cookie in self._cookies.items(): cookie_domain = cookie["domain"] # Send shared cookies if not cookie_domain: dict.__setitem__(filtered, name, cookie) continue hostname = url_parsed.hostname or "" if is_ip_address(hostname): continue if name in self._host_only_cookies: if cookie_domain != hostname: continue elif not self._is_domain_match(cookie_domain, hostname): continue if not self._is_path_match(url_parsed.path, cookie["path"]): continue is_secure = url_parsed.scheme in ("https", "wss") if cookie["secure"] and not is_secure: continue dict.__setitem__(filtered, name, cookie) return filtered @staticmethod def _is_domain_match(domain, hostname): """Implements domain matching adhering to RFC 6265.""" if hostname == domain: return True if not hostname.endswith(domain): return False non_matching = hostname[:-len(domain)] if not non_matching.endswith("."): return False return not is_ip_address(hostname) @staticmethod def _is_path_match(req_path, cookie_path): """Implements path matching adhering to RFC 6265.""" if req_path == cookie_path: return True if not req_path.startswith(cookie_path): return False if cookie_path.endswith("/"): return True non_matching = req_path[len(cookie_path):] return non_matching.startswith("/") @classmethod def _parse_date(cls, date_str): """Implements date string parsing adhering to RFC 6265.""" if not date_str: return found_time = False found_day_of_month = False found_month = False found_year = False hour = minute = second = 0 day_of_month = 0 month = "" year = 0 for token_match in cls.DATE_TOKENS_RE.finditer(date_str): token = token_match.group("token") if not found_time: time_match = cls.DATE_HMS_TIME_RE.match(token) if time_match: found_time = True hour, minute, second = [ int(s) for s in time_match.groups()] continue if not found_day_of_month: day_of_month_match = cls.DATE_DAY_OF_MONTH_RE.match(token) if day_of_month_match: found_day_of_month = True day_of_month = int(day_of_month_match.group()) continue if not found_month: month_match = cls.DATE_MONTH_RE.match(token) if month_match: found_month = True month = month_match.group() continue if not found_year: year_match = cls.DATE_YEAR_RE.match(token) if year_match: found_year = True year = int(year_match.group()) if 70 <= year <= 99: year += 1900 elif 0 <= year <= 69: year += 2000 if False in (found_day_of_month, found_month, found_year, found_time): return if not 1 <= day_of_month <= 31: return if year < 1601 or hour > 23 or minute > 59 or second > 59: return dt = datetime.datetime.strptime( "%s %d %d:%d:%d %d" % ( month, day_of_month, hour, minute, second, year ), "%b %d %H:%M:%S %Y") return dt.replace(tzinfo=datetime.timezone.utc)
def spider(add_link_dictionary): # get user's download request from add_link_dictionary link = add_link_dictionary['link'] ip = add_link_dictionary['ip'] port = add_link_dictionary['port'] proxy_user = add_link_dictionary['proxy_user'] proxy_passwd = add_link_dictionary['proxy_passwd'] download_user = add_link_dictionary['download_user'] download_passwd = add_link_dictionary['download_passwd'] header = add_link_dictionary['header'] out = add_link_dictionary['out'] user_agent = add_link_dictionary['user_agent'] raw_cookies = add_link_dictionary['load_cookies'] referer = add_link_dictionary['referer'] # defin a requests session requests_session = requests.Session() if ip: ip_port = 'http://' + str(ip) + ":" + str(port) if proxy_user: ip_port = 'http://' + proxy_user + ':' + proxy_passwd + '@' + ip_port # set proxy to the session requests_session.proxies = {'http': ip_port} if download_user: # set download user pass to the session requests_session.auth(download_user, download_passwd) # set cookies if raw_cookies: cookie = SimpleCookie() cookie.load(raw_cookies) cookies = {key: morsel.value for key, morsel in cookie.items()} requests_session.cookies = cookiejar_from_dict(cookies) # set referer if referer: requests_session.headers.update({'referer': referer }) #setting referer to the session # set user_agent if user_agent: requests_session.headers.update({'user-agent':user_agent }) #setting user_agent to the session #find headers try: response = requests_session.head(link) header = response.headers except: header = {} filename = None filesize = None if 'Content-Disposition' in header.keys(): # checking if filename is available content_disposition = header['Content-Disposition'] if content_disposition.find('filename') != -1: filename_splited = content_disposition.split('filename=') filename_splited = filename_splited[-1] # getting file name in desired format filename = filename_splited[1:-1] if not(filename): filename = link.split('/')[-1] # if user set file name before in add_link_dictionary['out'], # then set "out" for filename if out: filename = out # check if file_size is available if 'Content-Length' in header.keys(): file_size = int(header['Content-Length']) # converting file_size to KiB or MiB or GiB file_size = humanReadbleSize(file_size) # return results return filename, filesize