def authenticate(url: str, cookies: str, username: str, password: str, headers: Headers) -> Tuple[str, List[Role]]: """ Authenitcate with user credentials to IdP. Args: url: ECP endpoint URL for the IdP. cookies: A path to a cookie jar. username: Username to provide to the IdP. password: Password to provide to the IdP. headers: Optional headers to provide to the IdP. Returns: A base 64 encoded SAML assertion string, and a list of tuples containing a SAML provider ARN and a Role ARN. """ jar = LWPCookieJar(cookies) soap = saml_login(url, jar, username, password, headers) mesg = "Successfully authenticated with username/password" logger.info(mesg + " to endpoint: " + url) secure_touch(cookies) jar.save(ignore_discard=True) logger.info("Saved cookies to jar: " + jar.filename) return parse_soap_response(soap)
def init_basicauth(config, config_mtime): """initialize urllib2 with the credentials for Basic Authentication""" def filterhdrs(meth, ishdr, *hdrs): # this is so ugly but httplib doesn't use # a logger object or such def new_method(self, *args, **kwargs): # check if this is a recursive call (note: we do not # have to care about thread safety) is_rec_call = getattr(self, '_orig_stdout', None) is not None try: if not is_rec_call: self._orig_stdout = sys.stdout sys.stdout = StringIO() meth(self, *args, **kwargs) hdr = sys.stdout.getvalue() finally: # restore original stdout if not is_rec_call: sys.stdout = self._orig_stdout del self._orig_stdout for i in hdrs: if ishdr: hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr) else: hdr = re.sub(i, '', hdr) sys.stdout.write(hdr) new_method.__name__ = meth.__name__ return new_method if config['http_debug'] and not config['http_full_debug']: HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie', 'Authorization') HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False, 'header: Set-Cookie.*\n') if config['http_debug']: # brute force def urllib2_debug_init(self, debuglevel=0): self._debuglevel = 1 AbstractHTTPHandler.__init__ = urllib2_debug_init cookie_file = os.path.expanduser(config['cookiejar']) global cookiejar cookiejar = LWPCookieJar(cookie_file) try: cookiejar.load(ignore_discard=True) if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime): cookiejar.clear() cookiejar.save() except IOError: try: fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600) os.close(fd) except IOError: # hmm is any good reason why we should catch the IOError? #print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file cookiejar = CookieJar()
def get_cookie(self): """ 1. dev has option to not to save cookies 2. it will come in handy for most scenarios, so until user explicitly says no, we will send :return: """ if self.args.no_cookie: cookie = None request_logger.debug(f'cookies set to `{self.args.no_cookie}`') else: cookie = LWPCookieJar(DOTHTTP_COOKIEJAR) request_logger.debug( f'cookie {cookie} loaded from {DOTHTTP_COOKIEJAR}') try: if not os.path.exists(DOTHTTP_COOKIEJAR): cookie.save() else: cookie.load() except Exception as e: # mostly permission exception # traceback.print_exc() eprint("cookie save action failed") base_logger.debug("error while saving cookies", exc_info=True) cookie = None # FUTURE # instead of saving (short curiting here, could lead to bad logic error) self.args.no_cookie = True self._cookie = cookie return self._cookie
def cookiejar_create(self, cookiejar_file, session): cookie_jar = LWPCookieJar(cookiejar_file.name) cookie_jar.set_cookie( Cookie(0, self.COOKIE_NAME, session, None, False, '', False, True, '/', True, True, None, None, None, None, {})) cookie_jar.save() cookiejar_file.flush()
def save_cookies(self, cookie_storage): """Save to cookielib's CookieJar or Set-Cookie3 format text file. :param cookie_storage: file location string or CookieJar instance. """ def toPyCookieJar(QtCookieJar, PyCookieJar): for c in QtCookieJar.allCookies(): PyCookieJar.set_cookie(toPyCookie(c)) def toPyCookie(QtCookie): port = None port_specified = False secure = QtCookie.isSecure() name = str(QtCookie.name()) value = str(QtCookie.value()) v = str(QtCookie.path()) path_specified = bool(v != "") path = v if path_specified else None v = str(QtCookie.domain()) domain_specified = bool(v != "") domain = v if domain_specified: domain_initial_dot = v.startswith('.') else: domain_initial_dot = None v = int(QtCookie.expirationDate().toTime_t()) # Long type boundary on 32bit platfroms; avoid ValueError expires = 2147483647 if v > 2147483647 else v rest = {} discard = False return Cookie( 0, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, None, None, rest, ) if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) toPyCookieJar(self.cookie_jar, cj) cj.save() elif cookie_storage.__class__.__name__.endswith('CookieJar'): toPyCookieJar(self.cookie_jar, cookie_storage) else: raise ValueError('unsupported cookie_storage type.')
def save_cookie(self, file_name=None, **kwargs): file_name = file_name or self._cookie_file lwp_jar = LWPCookieJar() for item in self._cookie_jar: args = dict(vars(item).items()) args['rest'] = args['_rest'] del (args['_rest']) cookie = Cookie(**args) lwp_jar.set_cookie(cookie) lwp_jar.save(file_name, **kwargs)
def cookiejar_create(self, cookiejar_file, session): cookie_jar = LWPCookieJar(cookiejar_file.name) cookie_jar.set_cookie(Cookie(0, self.COOKIE_NAME, session, None, False, '', False, True, '/', True, True, None, None, None, None, {})) cookie_jar.save() cookiejar_file.flush()
def save_cookie_on_file(path): # 第一种保存cookie到文件的方式 #cookie = MozillaCookieJar(path) # 第二种保存cookie到文件的方式 cookie = LWPCookieJar(path) cookie_manage = urllib.request.HTTPCookieProcessor(cookie) opener = urllib.request.build_opener(cookie_manage) response = opener.open('https://www.baidu.com') cookie.save(ignore_discard=True, ignore_expires=True)
class CookieJar: """A cookie jar that can be persisted.""" def __init__(self, cookie_file: Optional[Path] = None) -> None: """Create a new cookie jar at the given path. If the path is None, the cookies will not be persisted. """ self._cookies: LWPCookieJar if cookie_file is None: self._cookies = LWPCookieJar() else: self._cookies = LWPCookieJar(str(cookie_file.resolve())) @property def cookies(self) -> LWPCookieJar: """Return the requests cookie jar.""" return self._cookies def load_cookies(self) -> None: """Load all cookies from the file given in the constructor.""" if self._cookies.filename is None: return try: LOGGER.info("Loading old cookies from %s", self._cookies.filename) self._cookies.load(ignore_discard=True) except (FileNotFoundError, LoadError): LOGGER.warning( "No valid cookie file found at %s, continuing with no cookies", self._cookies.filename) def save_cookies(self, reason: Optional[str] = None) -> None: """Save the cookies in the file given in the constructor.""" if self._cookies.filename is None: return if reason is None: LOGGER.info("Saving cookies") else: LOGGER.info("Saving cookies (%s)", reason) # TODO figure out why ignore_discard is set # TODO possibly catch a few more exceptions self._cookies.save(ignore_discard=True) def create_session(self) -> requests.Session: """Create a new session using the cookie jar.""" sess = requests.Session() # From the request docs: "All requests code should work out of the box # with externally provided instances of CookieJar, e.g. LWPCookieJar # and FileCookieJar." sess.cookies = self.cookies # type: ignore return sess
class SimpleCookieConnector(ConnectorMixin, BaseConnector): def __init__(self, cookie_file, delay=3, extra_headers=None): """ Keywords -------- - cookie_file """ super(SimpleCookieConnector, self).__init__(delay, extra_headers) self._cookie_file = cookie_file self._cookie_jar = LWPCookieJar(filename=self._cookie_file) self._cookie_processor = HTTPCookieProcessor(self._cookie_jar) self._opener = build_opener(self._cookie_processor) self._last_request = None self._last_response = None self._last_content = '' def request(self, url, method='GET', params=None, data=None, headers=None): if not url: return '' params = params or {} data = data or {} headers = (headers or {}).update(self._extra_headers) if method.upper() == 'GET': _url = self.create_get_url(url, params) return self.resolve(Request(_url, data=None, headers=headers)) elif method.upper() == 'POST': return self.resolve(Request(url, data=bytes(urlencode(data), 'utf-8'), headers=headers)) def resolve(self, request): self._last_request = request self._last_response = self._opener.open(self._last_request) content = self._last_response.read() self._last_response.close() sleep(self._delay) charset = self.detect_charset(headers=self._last_response.headers, content=content) self._last_content = content.decode(charset) return self._last_content def save_cookie(self): self._cookie_jar.save(self._cookie_file) def get_cookie(self, name): for cookie in self._cookie_jar: if cookie.name == name: return cookie return None
def get_opener(): jar = LWPCookieJar(COOKIES_FILENAME) if os.path.exists(COOKIES_FILENAME): try: jar.load(ignore_discard=True) except LoadError: log.error('Cookie file corrupt, skipping') opener = build_opener(HTTPCookieProcessor(jar)) yield opener jar.save(ignore_discard=True)
def get_cookie_jar(self): """Returns our cookie jar.""" cookie_file = self._get_cookie_file() cookie_jar = LWPCookieJar(cookie_file) if os.path.exists(cookie_file): cookie_jar.load() else: safe_mkdir_for(cookie_file) # Save an empty cookie jar so we can change the file perms on it before writing data to it. with self._lock: cookie_jar.save() os.chmod(cookie_file, 0o600) return cookie_jar
def requests_session_with_selenium_cookies(driver, config): """ Return a Requests library session object initialized with the cookies from Selenium. We have already logged into Volunteer Connection using selenium; use those cookies to initialize a Requests session that we will use to download files (Selenium has trouble intercepting file downloads) """ cookies = LWPCookieJar(config.COOKIE_FILE) selenium_cookies = driver.get_cookies() for c in selenium_cookies: log.debug(f"selenium cookie: { c }") path = c['path'] path_specified = path != None domain = c['domain'] domain_specified = domain != None domain_initial_dot = domain_specified and domain[0] == '.' if 'expiry' in c: expires = c['expiry'] + 86400 * 365 * 10 # add 10 years to expiry else: expires = None cookie = Cookie(version=0, name=c['name'], value=c['value'], port=None, port_specified=False, discard=False, comment=None, comment_url=None, domain=c['domain'], domain_specified=domain_specified, domain_initial_dot=domain_initial_dot, expires=expires, path=path, path_specified=path_specified, rest={'HttpOnly': c['httpOnly']}, secure=c['secure']) log.debug(f"cookejar cookie: { cookie }\n") cookies.set_cookie(cookie) cookies.save(ignore_discard=True, ignore_expires=True) return cookies
def init_cookie_jar(cookie_file=None, cookie_t=None, cookie_y=None, cookie_euconsent=None): cookie_jar = LWPCookieJar( cookie_file) if cookie_file else RequestsCookieJar() if cookie_file and os.path.exists(cookie_file): cookie_jar.load(ignore_discard=True) if cookie_t: cookie_jar.set_cookie(create_cookie('T', cookie_t)) if cookie_y: cookie_jar.set_cookie(create_cookie('Y', cookie_y)) if cookie_euconsent: cookie_jar.set_cookie(create_cookie('EuConsent', cookie_euconsent)) if cookie_file: cookie_jar.save(ignore_discard=True) return cookie_jar
def _initialize_cookies(cookies_path=None) -> LWPCookieJar: """ 初始化 Cookies """ if cookies_path: jar = LWPCookieJar(cookies_path) if os.path.isfile(cookies_path): jar.load() else: jar = LWPCookieJar() for cookie in jar: if cookie.is_expired(): jar.clear() break for k, v in c.BASE_COOKIES.items(): jar.set_cookie(make_cookie(k, v)) if cookies_path: jar.save() return jar
def login(self, data, url=None): if re.match("^1{0-9}{10}$", data["account"]): account_type = "phone_num" url_login = url or "https://www.zhihu.com/login/phone_num" elif re.match(".+@.+\.com", data["account"]): account_type = "email" url_login = url or "https://www.zhihu.com/login/email" else: print("账号类型错误") self._data = { "_xsrf": self._session.cookies.get("_xsrf", ""), "password": data.get("password", ""), "captcha": self._captcha, "remember_me": "true", account_type: data.get("account", "") } self._headers["X-Xsrftoken"] = self._session.cookies.get("_xsrf", "") self._r = self._session.post(url_login, data=self._data, headers=self._headers) if self._r.status_code != 200: print("提交数据失败") else: self._response_json = json.loads(self._r.content.decode("utf-8")) if self._response_json["r"] == 0: print(self._response_json["msg"]) # save cookies lwpcookie = LWPCookieJar('cookie.txt') cookiejar_from_dict({ c.name: c.value for c in self._session.cookies}, lwpcookie) lwpcookie.save(ignore_discard=True) else: if self._response_json["errcode"] in [1991829, 100005]: print(self._response_json["msg"]) self.get_captcha() self.login() else: print("未知的错误")
def get_persistent_login(username, password): global session session = requests.Session() session.headers.update( { 'DNT': '1', 'Upgrade-Insecure-Requests': '1', 'User-Agent' : BROWSER } ) cj = LWPCookieJar('cookies.txt') if os.path.isfile('cookies.txt'): cj.load() session.cookies = cj logged_in = check_auth_status() if not logged_in: login(username, password) if not check_auth_status(): print("Login failed") return False have_csrf = cookie_in('csrf', session.cookies) if not have_csrf: r = session.get('https://alexa.amazon.co.uk/api/language') have_csrf = cookie_in('csrf', session.cookies) if not have_csrf: r = session.get('https://alexa.amazon.co.uk/templates/oobe/d-device-pick.handlebars') have_csrf = cookie_in('csrf', session.cookies) if not have_csrf: print('No CSRF cookie, cannot continue') return False session.headers.update({'csrf' : get_cookie_value('csrf', session.cookies)}) cj.save() return True
class Ggrks: USER_AGENT = 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0)' def __init__(self): self.home_folder = os.getenv('HOME') if not self.home_folder: self.home_folder = os.getenv('USERHOME') if not home_folder: self.home_folder = '.' # Use the current folder on error. self.cookie_jar = LWPCookieJar(os.path.join(self.home_folder, '.google-cookie')) try: self.cookie_jar.load() except Exception: pass def search(self, url:str) -> str: ''' ググったHTMLを返す ''' request = Request(url) request.add_header('User-Agent', Ggrks.USER_AGENT) # cookie_jar.add_cookie_header(request) # GET with urlopen(request) as response: # cookieの保管 self.cookie_jar.extract_cookies(response, request) html = response.read() # cookieをファイル保管 self.cookie_jar.save() return html return '' def next(): return ''
class Opener(DefaultOpener): """ 高级的Opener对象. """ def __init__(self, cookie_path=None): self.opener = build_opener() self.cookiejar = LWPCookieJar(cookie_path) cookie_path and os.path.exists(cookie_path) and self.cookiejar.load() self.opener.add_handler(HTTPCookieProcessor(self.cookiejar)) self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')] def set_headers(self, arg={}, **kwargs): """ 设置headers. @arg(pairs/dict) @kwargs(mixed) """ if not arg and not kwargs: return headers = dict(self.opener.addheaders) headers.update(arg) headers.update(kwargs) self.opener.addheaders = list(headers.items()) def set_cookies(self, *args, **kwargs): """ 设置cookies. @args(pairs/dict) @kwargs(mixed) """ for arg in args: cookie = DefaultCookiePolicy(**dict(arg)) self.cookiejar.set_cookie(cookie) kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs)) def save_cookies(self, cookie_path=None): if cookie_path or self.cookiejar.filename: self.cookiejar.save(cookie_path)
class PLIdeabank(object): def __init__(self, credstore=None, **kwargs): self.sso_url_base = 'https://sso.cloud.ideabank.pl/' self.cloud_url_base = 'https://cloud.ideabank.pl/' self.log = structlog.get_logger() self.log = self.log.bind(cls=__class__.__name__, sso_url_base=self.sso_url_base, cloud_url_base=self.cloud_url_base) self.credstore = credstore if self.credstore: self.log = self.log.bind(credstore=credstore.name) self.log.msg('bound to credentials store') self.browser = mechanicalsoup.StatefulBrowser() def _get_auth_token(self): # Check for valid authentication token in the cookie store return [c.value for c in self.cookiejar if c.name == 'Authentication-Token' and not c.is_expired()][0] def _we_have_cookie(self, n): return len([c.value for c in self.cookiejar if c.name == n and not c.is_expired()]) > 0 def _sso_authenticate(self): self.log.msg('sso auth') # sso_login_info = self.browser.get(urljoin(self.sso_url_base, '/authenticate/login'), params={ 'login': login }) self.browser.open(self.sso_url_base) sso_auth_params=dict(login=self.creds['identity'], password=self.creds['password']) sso_auth_resp = self.browser.post(urljoin(self.sso_url_base, '/authenticate/login'), json=sso_auth_params) self.cookiejar.save() return sso_auth_resp.json()['token'] def _perform_2fa(self, sso_token): # The twofactor info is not known in the beginning twofactor = None while True: headers = {'Exchange-Token': sso_token} if twofactor is not None: headers.update(twofactor) # Claim device is trusted trusted_dev_claim_res = self.browser.post(urljoin(self.cloud_url_base, '/api/strong-authentication'), json={'trustedDevice': True}, headers=headers) self.cookiejar.save() # Check if claim was successful if self._we_have_cookie('ib_trusted_device'): break else: # Check for what to do now claim_resp = trusted_dev_claim_res.json() if claim_resp['code'] == 'MISSING_VERIFICATION_CODE': code_number = claim_resp['content']['codeNumber'] code_date = claim_resp['content']['generatedDate'] twofactor = {'Verification-Code-Number': str(code_number)} # The user should have received an SMS with the 2FA code at this point # Request it from the user twofactor_code = getpass.getpass( 'Enter 2FA code number {} generated on {}: '.format(code_number, code_date)) twofactor['Verification-Code'] = twofactor_code else: self.log.msg('unrecognized trusted device claim response code', code=claim_resp['code']) raise RuntimeError() def usable_identities(self): return list(filter(lambda i: i is not None, [ self.credstore.get_identity(self.sso_url_base) ])) def authenticate(self, identity=None): self.log.msg('authentication attempt') self.identity = identity or self.credstore.get_identity(self.sso_url_base) self.log = self.log.bind(identity=self.identity) self.creds = self.credstore.get_credentials(self.sso_url_base, self.identity) self.log.msg('got creds', password='******'.format(self.creds['password'][0:2], self.creds['password'][-1])) # Load cookiejar cache_dir = Path(XDG_CACHE_HOME, 'ideabank-fetch-balances', self.creds['identity']) cache_dir.mkdir(parents=True, exist_ok=True) cookiejar_filename = Path(cache_dir, 'cookies.txt') self.cookiejar = LWPCookieJar(str(cookiejar_filename)) if cookiejar_filename.exists(): self.cookiejar.load() # Use the cookiejar we just loaded self.browser.set_cookiejar(self.cookiejar) # Check if the authentication token we have is valid if self._we_have_cookie('Authentication-Token'): self.session_headers = {'Authentication-Token': self._get_auth_token()} user_info_resp = self.browser.get( urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=self.session_headers) if not user_info_resp.ok: self.log.msg('session token not valid, authenticating') # The cached token is not valid, remove it from the cookie store self.cookiejar.clear('.cloud.ideabank.pl', '/', 'Authentication-Token') # Check if we have an authentication token if not self._we_have_cookie('Authentication-Token'): # Authenticate, we do not have a valid token # First do SSO sso_token = self._sso_authenticate() # Now attempt to authenticate to main application (Cloud) cloud_auth_resp = self.browser.post( urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token}) self.cookiejar.save() # We need to perform 2FA if no token at this point if not self._we_have_cookie('Authentication-Token'): self.log.msg('no auth token') final_uri = cloud_auth_resp.url self.log.msg('redirected', redirect_uri=final_uri) # We have no token, check if we were redirected to the 2FA page if final_uri != urljoin(self.cloud_url_base, '/strong-authentication'): # The target URI is not what we expected self.log.msg('unexpected 2fa URI') return False self._perform_2fa(sso_token) # Try to get the cloud token again cloud_auth_resp = self.browser.post( urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token}) self.cookiejar.save() if not self._we_have_cookie('Authentication-Token'): self.log.msg("cannot authenticate after 2FA") return False self.session_headers = {'Authentication-Token': self._get_auth_token()} return True return True def get_balances(self): self.log.msg('fetching balances') # user_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=session_headers) # print(json.dumps(user_info_resp.json(), indent=2)) accounts = [] accounts_info_resp = self.browser.get( urljoin(self.cloud_url_base, '/api/accounts'), headers=self.session_headers) # print(json.dumps(accounts_info_resp.json(), indent=2)) for acct_group in accounts_info_resp.json()['personAccountGroups']: for account in acct_group['accounts']: name = account['accountName'] balance = account['activeBalance'] accounts.append({ 'description': name, 'value': 'PLN {}'.format(balance) }) deposits = [] deposits_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/deposits'), headers=self.session_headers, params={ 'status': 'ACTIVE' }) # print(json.dumps(deposits_info_resp.json(), indent=2)) for deposit in deposits_info_resp.json()['deposits']: name = deposit['name'] amount = deposit['amount'] deposits.append({ 'description': name, 'value': 'PLN {}'.format(amount) }) return dict(name='Ideabank', bags=accounts + deposits) def logout(self): self.log.msg('logout') self.browser.get(urljoin(self.cloud_url_base, '/api/logout'), headers=self.session_headers) self.cookiejar.save() def close(self): self.log.msg('closing') self.browser.close()
class Client: """ Web client class with automatic charset detection and decoding """ def __init__(self, info=None, request_charset='utf-8', response_charset=None): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.url = None self.user_agent = USER_AGENT self.content = None self.status = None self.username = None self.token = None self.passkey = None self.info = info self.proxy_url = None self.request_charset = request_charset self.response_charset = response_charset self.needs_proxylock = False self.headers = dict() self.request_headers = None self.session = requests.session() self.session.verify = False # Enabling retrying on failed requests retries = Retry(total=3, read=2, connect=2, redirect=3, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]) self.session.mount('http://', HTTPAdapter(max_retries=retries)) self.session.mount('https://', HTTPAdapter(max_retries=retries)) # self.session = cfscrape.create_scraper() # self.scraper = cfscrape.create_scraper() # self.session = self.scraper.session() global dns_public_list global dns_opennic_list dns_public_list = public_dns_list.replace(" ", "").split(",") dns_opennic_list = opennic_dns_list.replace(" ", "").split(",") # socket.setdefaulttimeout(60) # Parsing proxy information proxy = { 'enabled': proxy_enabled, 'use_type': proxy_use_type, 'type': proxy_types[0], 'host': proxy_host, 'port': proxy_port, 'login': proxy_login, 'password': proxy_password, } try: proxy['type'] = proxy_types[proxy_type] except: pass if use_public_dns: connection.create_connection = patched_create_connection if use_elementum_proxy: elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum') if elementum_addon and elementum_addon.getSetting( 'internal_proxy_enabled') == "true": self.proxy_url = "{0}://{1}:{2}".format( "http", "127.0.0.1", "65222") if info and "internal_proxy_url" in info: self.proxy_url = info["internal_proxy_url"] self.session.proxies = { 'http': self.proxy_url, 'https': self.proxy_url, } elif proxy['enabled']: if proxy['use_type'] == 0 and info and "proxy_url" in info: log.debug("Setting proxy from Elementum: %s" % (info["proxy_url"])) elif proxy['use_type'] == 1: log.debug("Setting proxy with custom settings: %s" % (repr(proxy))) if proxy['login'] or proxy['password']: self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format( proxy['type'], proxy['login'], proxy['password'], proxy['host'], proxy['port']) else: self.proxy_url = "{0}://{1}:{2}".format( proxy['type'], proxy['host'], proxy['port']) if self.proxy_url: self.session.proxies = { 'http': self.proxy_url, 'https': self.proxy_url, } def _create_cookies(self, payload): return urlencode(payload) def _locate_cookies(self, url=''): cookies_path = os.path.join(PATH_TEMP, 'burst') if not os.path.exists(cookies_path): try: os.makedirs(cookies_path) except Exception as e: log.debug("Error creating cookies directory: %s" % repr(e)) return os.path.join(cookies_path, 'common_cookies.jar') def _read_cookies(self, url=''): self._cookies_filename = self._locate_cookies(url) if os.path.exists(self._cookies_filename): try: self._cookies.load(self._cookies_filename) except Exception as e: log.debug("Reading cookies error: %s" % repr(e)) def save_cookies(self): self._cookies_filename = self._locate_cookies(self.url) try: self._cookies.save(self._cookies_filename) except Exception as e: log.debug("Saving cookies error: %s" % repr(e)) def _good_spider(self): self._counter += 1 if self._counter > 1: sleep(0.25) def cookies(self): """ Saved client cookies Returns: list: A list of saved Cookie objects """ return self._cookies def open(self, url, language='en', post_data=None, get_data=None, headers=None): """ Opens a connection to a webpage and saves its HTML content in ``self.content`` Args: url (str): The URL to open language (str): The language code for the ``Content-Language`` header post_data (dict): POST data for the request get_data (dict): GET data for the request """ if get_data: url += '?' + urlencode(get_data) log.debug("Opening URL: %s" % repr(url)) if self.session.proxies: log.debug("Proxies: %s" % (repr(self.session.proxies))) self._read_cookies(url) self.session.cookies = self._cookies # log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies))) # Default headers for any request. Pretend like we are the usual browser. req_headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7,uk;q=0.6,pl;q=0.5', 'Cache-Control': 'max-age=0', 'Content-Language': language, 'Origin': url, 'Referer': url, 'User-Agent': self.user_agent } # If headers passed to open() call - we overwrite headers. if headers: for key, value in iteritems(headers): if key == ':path': u = urlparse(url) value = u.path if value: req_headers[key] = value elif key.capitalize() in req_headers: del req_headers[key.capitalize()] if self.token: req_headers["Authorization"] = self.token req = None if post_data: req = requests.Request('POST', url, data=post_data, headers=req_headers) else: req = requests.Request('GET', url, headers=req_headers) prepped = self.session.prepare_request(req) self.request_headers = prepped.headers try: self._good_spider() with self.session.send(prepped) as response: self.headers = response.headers self.status = response.status_code self.url = response.url if self.response_charset: self.content = response.content.decode( self.response_charset, 'ignore') else: self.content = response.text except requests.exceptions.InvalidSchema as e: # If link points to a magnet: then it can be used as a content matches = re.findall( 'No connection adapters were found for \'(.*?)\'', str(e)) if matches: self.content = matches[0] return True import traceback log.error("%s failed with %s:" % (repr(url), repr(e))) map(log.debug, traceback.format_exc().split("\n")) except Exception as e: import traceback log.error("%s failed with %s:" % (repr(url), repr(e))) map(log.debug, traceback.format_exc().split("\n")) log.debug("Status for %s : %s" % (repr(url), str(self.status))) return self.status == 200 def login(self, root_url, url, data, headers, fails_with): """ Login wrapper around ``open`` Args: url (str): The URL to open data (dict): POST login data fails_with (str): String that must **not** be included in the response's content Returns: bool: Whether or not login was successful """ if not url.startswith('http'): url = root_url + url if self.open(url.encode('utf-8'), post_data=encode_dict(data, self.request_charset), headers=headers): try: if fails_with in self.content: self.status = 'Wrong username or password' return False except Exception as e: log.debug("Login failed with: %s" % e) try: if fails_with in self.content.decode('utf-8'): self.status = 'Wrong username or password' return False except: return False return True return False
class DimClient(object): def __init__(self, server_url, cookie_file=None, cookie_umask=None): self.server_url = server_url self.cookie_jar = LWPCookieJar() self.session = build_opener(HTTPCookieProcessor(self.cookie_jar)) if cookie_file: self._use_cookie_file(cookie_file, cookie_umask) def login(self, username, password, permanent_session=False): try: self.session.open( self.server_url + '/login', urlencode( dict(username=username, password=password, permanent_session=permanent_session)).encode('utf8')) self.check_protocol_version() if self.cookie_jar.filename and self.save_cookie: # Use umask when saving cookie if self.cookie_umask is not None: old_mask = os.umask(self.cookie_umask) self.cookie_jar.save() if self.cookie_umask is not None: os.umask(old_mask) return True except HTTPError as e: logger.error("Login failed: " + str(e)) return False @property def logged_in(self): try: self.session.open(self.server_url + '/index.html') return True except HTTPError as e: if e.code == 403: return False else: raise def _use_cookie_file(self, cookie_file, cookie_umask, save_cookie=True): self.cookie_jar.filename = cookie_file self.save_cookie = save_cookie self.cookie_umask = cookie_umask try: self.cookie_jar.load() except: pass def login_prompt(self, username=None, password=None, permanent_session=False, ignore_cookie=False): if not ignore_cookie and self.logged_in: return True else: if username is None: print("Username:"******"The server does not have the JSONRPC interface enabled (%s)" % e) if server_protocol != PROTOCOL_VERSION: raise ProtocolError( "Server protocol version (%s) does not match client protocol version (%s)" % (server_protocol, PROTOCOL_VERSION)) def raw_call(self, function, *args): url = self.server_url + "/jsonrpc" json_call = json.dumps( dict(jsonrpc='2.0', method=function, params=args, id=None)) logger.debug('dim call: %s(%s)' % (function, ', '.join([repr(x) for x in args]))) start = time.time() request = Request(url, data=json_call.encode('utf8'), headers={'Content-Type': 'application/json'}) response = self.session.open(request).read() rpc_response = json.loads(response.decode('utf8')) logger.debug('time taken: %.3f' % (time.time() - start)) if 'error' in rpc_response: logger.debug('dim error: ' + str(rpc_response['error'])) raise DimError(message=rpc_response['error']['message'], code=rpc_response['error']['code']) else: if logger.isEnabledFor(logging.DEBUG): logger.debug('dim result: ' + pformat(rpc_response['result'])) return rpc_response['result'] def call(self, function, *args, **kwargs): ''' Instead of passing the last argument as a dictionary (usually called "options"), you can use keyword arguments. .. note:: Keyword arguments cannot be used for positional jsonrpc arguments. ''' passed_args = args if kwargs: passed_args += (kwargs, ) return self.raw_call(function, *passed_args) def __getattr__(self, name): return lambda *args, **kwargs: self.call(name, *args, **kwargs) def ip_list_all(self, **options): total = options.get('limit', 10) result = [] while len(result) < total: batch = self.ip_list(**options) if len(batch) == 0: break result.extend(batch) options['limit'] -= len(batch) options['after'] = batch[-1]['ip'] return result
class CCPConnection(object): """ Netcup CCP API """ def __init__(self, cachepath=None): """ Creates CCP connection """ self.__cache = False self.__sessionhash = None self.__nocsrftoken = None # creates urllib with custom headers and cookie management self.__jar = LWPCookieJar() self.__network = build_opener(HTTPCookieProcessor(self.__jar)) self.__network.addheaders = [ ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299" ), ("Accept-Encoding", "gzip, deflate, br"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" ) ] # load session cache from disk if cachepath: # check if path is writeable path = os.path.dirname(cachepath) if not path: path = "." if not os.access(path, os.W_OK): raise IOError("cachepath is not writeable") self.__cache = True self.__cachepath = cachepath # load cookies try: self.__jar.load(cachepath, ignore_discard=True) except IOError: # cookie file does not exist pass def start(self, username, password, token_2FA=None): """ Performs login if session is invalid """ # check if arguments are strings if not isinstance(username, str): username = str(username) if not isinstance(password, str): password = str(password) if token_2FA and not isinstance(token_2FA, str): token_2FA = str(token_2FA) # validate cached session if self.__cache: resource = self.__network.open( "https://ccp.netcup.net/run/domains.php") content = decompress(resource.read()).decode( resource.headers.get_content_charset()) if username in content: self.__getTokens(content) return True # create login post payload = { "action": "login", "nocsrftoken": "", "ccp_user": username, "ccp_password": password, "language": "DE", "login": "******" } # 2FA Auth if token_2FA: payload.pop("ccp_password", False) payload["pwdb64"] = b64encode(password.encode("ascii")) payload["tan"] = token_2FA # send login payload = urlencode(payload) resource = self.__network.open("https://ccp.netcup.net/run/start.php", payload.encode("utf-8")) content = decompress(resource.read()).decode( resource.headers.get_content_charset()) # check if login successful if username in content: resource = self.__network.open( "https://ccp.netcup.net/run/domains.php") content = decompress(resource.read()).decode( resource.headers.get_content_charset()) self.__getTokens(content) # check tokens if not self.__sessionhash or not self.__nocsrftoken: if token_2FA: raise CCPLoginError("2FA token invalid") else: raise CCPLoginError("Your account has 2FA enabled") return True else: raise CCPLoginError("Login failed check your credentials") def close(self): """ Save session or perform logout """ # check if caching is enabled if self.__cache: # save session self.__jar.save(self.__cachepath, ignore_discard=True) else: # logout resource = self.__network.open( "https://ccp.netcup.net/run/logout.php") content = decompress(resource.read()).decode( resource.headers.get_content_charset()) def getDomainList(self, search="", page=1): """ Returns dict containing domain id and name """ # get domain list resource = self.__network.open( "https://ccp.netcup.net/run/domains_ajax.php?suchstrg=" + quote_plus(search) + "&action=listdomains&seite=" + str(page) + "&sessionhash=" + self.__sessionhash + "&nocsrftoken=" + self.__nocsrftoken) content = decompress(resource.read()).decode( resource.headers.get_content_charset()) self.__getTokens(content) # check if domains found if "Es wurden keine Domains zu ihrer Suche nach" in content or "Sie haben keine Domains gebucht" in content: return {} # parse list domain_id = findall(r"showDomainsDetails\((.*?)\);", content) domain_name = findall(r"<td>([a-zA-Z0-9-_\.äöüß]+?)\s", content) # check result if not domain_id or not domain_name: raise CCPWebsiteChanges("Could not find domains and keys") # check if len equals if not len(domain_id) == len(domain_name): raise CCPWebsiteChanges( "Number of domains does not match number of keys") # return dict return dict(zip(domain_id, domain_name)) def getDomain(self, domain_id): """ Return Domain object """ # get domain info resource = self.__network.open( "https://ccp.netcup.net/run/domains_ajax.php?domain_id=" + str(domain_id) + "&action=showdomainsdetails&sessionhash=" + self.__sessionhash + "&nocsrftoken=" + self.__nocsrftoken) content = decompress(resource.read()).decode( resource.headers.get_content_charset()) self.__getTokens(content) # parse html soup = BeautifulSoup(content, "html.parser") div = soup.find("div", {"id": "domainsdetail_detail_dns_" + str(domain_id)}) if not div: raise CCPWebsiteChanges("Could not get DNS tab") table = div.find_all("table") if not table: raise CCPWebsiteChanges("Could not get RR table") # create CCPDomain object try: webhosting = True if "restoredefaultslabel_" + str( domain_id) in str(div) else False dnssec = True if "checked" in str( div.find("input", {"id": "dnssecenabled_" + str(domain_id)})) else None domain_obj = CCPDomain( domain_id=domain_id, domain_name=div.find("input", { "name": "zone" }).get("value"), domain_zone=div.find("input", { "name": "zoneid" }).get("value"), domain_serial=div.find("input", { "name": "serial" }).get("value"), domain_dnssec=dnssec, domain_webhosting=webhosting, domain_ttl=div.find( "input", { "name": "zone_settings_ttl_" + str(domain_id) }).get("value"), domain_retry=div.find( "input", { "name": "zone_settings_retry_" + str(domain_id) }).get("value"), domain_expire=div.find( "input", { "name": "zone_settings_expire_" + str(domain_id) }).get("value"), domain_refresh=div.find( "input", { "name": "zone_settings_refresh_" + str(domain_id) }).get("value")) except (AttributeError, TypeError) as e: raise CCPWebsiteChanges("Could not get domain infos") # for every dns entry del_lines = -2 if not webhosting else -4 for row in table[-2].find_all("tr")[1:del_lines]: column = row.find_all("td") # get values try: rr_host = column[0].input.get("value") rr_pri = column[2].input.get("value") rr_destination = column[3].input.get("value") rr_type = "" for option in column[1].find_all("option"): if option.get("selected"): rr_type = option.get("value") break except (AttributeError, TypeError, KeyError) as e: raise CCPWebsiteChanges("Could not get RR row") # if record contain values if rr_host: domain_obj.addRecord(rr_host, rr_type, rr_destination, rr_pri, column[0].input.get("name")[:-6]) return domain_obj def saveDomain(self, domain_obj): """ Saves domain object on netcup """ # check if domain_obj is CCPDomain if not isinstance(domain_obj, CCPDomain): raise TypeError("Object of type CCPDomain expected") # check if object changed if not domain_obj.hasChanged(): return True # create post payload payload = { "zone": domain_obj.getDomainName(), "zoneid": domain_obj.getDomainZone(), "serial": domain_obj.getDomainSerial(), "order": "", "formchanged": "", "zone_settings_ttl_" + domain_obj.getDomainID(): domain_obj.getTTL(), "zone_settings_expire_" + domain_obj.getDomainID(): domain_obj.getExpire(), "zone_settings_retry_" + domain_obj.getDomainID(): domain_obj.getRetry(), "zone_settings_refresh_" + domain_obj.getDomainID(): domain_obj.getRefresh(), "restoredefaults_" + domain_obj.getDomainID(): "false", "submit": "DNS Records speichern" } # set dnssec state if isinstance(domain_obj.getDNSSEC(), bool): payload["dnssecenabled"] = str(not domain_obj.getDNSSEC()).lower() # add dns records to payload for key, value in domain_obj.getAllRecords().items(): try: payload[key + "[host]"] = value["host"] payload[key + "[type]"] = value["type"] payload[key + "[pri]"] = value["pri"] payload[key + "[destination]"] = value["destination"] if "delete" in value: payload[key + "[delete]"] = value["delete"] except KeyError: raise ValueError("Invalid CCPDomain object") # send update payload = urlencode(payload) resource = self.__network.open( "https://ccp.netcup.net/run/domains_ajax.php?action=editzone&domain_id=" + domain_obj.getDomainID() + "&sessionhash=" + self.__sessionhash + "&nocsrftoken=" + self.__nocsrftoken, payload.encode("utf-8")) content = decompress(resource.read()).decode( resource.headers.get_content_charset()) self.__getTokens(content) # check if update was successful if not "Eintrag erfolgreich!" in content: raise CCPSaveDomainError("Could not save domain") # parse html soup = BeautifulSoup(content, "html.parser") try: # update domain serial domain_obj.setDomainSerial( soup.find("input", { "name": "serial" }).get("value")) except (AttributeError, TypeError) as e: raise CCPWebsiteChanges("Could not get domain serial") return True def isRecordLive(self, domain_id): """ Checks if domain dns records are live """ # get domain info resource = self.__network.open( "https://ccp.netcup.net/run/domains_ajax.php?domain_id=" + str(domain_id) + "&action=showdomainsdetails&sessionhash=" + self.__sessionhash + "&nocsrftoken=" + self.__nocsrftoken) content = decompress(resource.read()).decode( resource.headers.get_content_charset()) self.__getTokens(content) if "<td>yes</td>" in content: return True else: return False def __getTokens(self, html): """ Retrieves session and csrf token """ # check session if "Your session has expired" in html: raise CCPSessionExpired("CCP session expired") try: self.__sessionhash = search(r"sessionhash = \"(.*?)\";", html).group(1) except AttributeError: pass try: self.__nocsrftoken = search(r"nocsrftoken = \"(.*?)\";", html).group(1) except AttributeError: try: self.__nocsrftoken = search(r"nocsrftoken = '(.*?)';", html).group(1) except AttributeError: self.__getNewCSRF() def __getNewCSRF(self): """ Gets new csrf token from api """ # request token resource = self.__network.open( "https://ccp.netcup.net/run/nocrfs_ajax.php?&action=getnocsrftoken&sessionhash=" + self.__sessionhash) self.__nocsrftoken = decompress(resource.read()).decode( resource.headers.get_content_charset()) if "Your session has expired" in self.__nocsrftoken: raise CCPSessionExpired("CCP session expired")
opener = urllib.request.build_opener(cookie_handle) # 指定url url = "https://www.baidu.com/" # 构造请求头部 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', 'Host': 'baidu.com' } # 构造post表单数据 forms = {'word': 'hello'} postdata = urllib.parse.urlencode(forms).encode() # 构造请求实例 req = urllib.request.Request(url, postdata, headers) # 发送请求 try: response = opener.open(req, timeout=5) cookie.save(ignore_discard=True, ignore_expires=True) if response.code == 200: print("headers:", response.headers) print("url:", response.url) print("msg:", response.msg) print("Receive data: %d bytes" % len(response.read().decode('utf-8'))) except urllib.error.URLError as e: print("URLError reason:", e.reason) except urllib.error.HTTPError as e: print("HTTPError reason:", e.reason) print("HTTPError code:", e.code) print("HTTPError headers:", e.headers)
def save_cookies(self, cookies): _cookies = LWPCookieJar(self.filename) for cookie in cookies: _cookies.set_cookie(copy.copy(cookie)) _cookies.save()
class Usr_login: def __init__(self, user="", password="", cookieDir='.', proxy_IP=''): http_header = [ ('User-Agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36'), ('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), ('Accept-Charset','utf-8,ISO-8859-1;q=0.7,*;q=0.3'), ('Accept-Encoding','none'), ('Accept-Language','en-US,en;q=0.8'), ('Connection','keep-alive'), ('Host','s.weibo.com'), ] self.cookieFile = cookieDir+'/'+user ouf = open(self.cookieFile, 'w') ouf.write('#LWP-Cookies-2.0') ouf.close() self.cj = LWPCookieJar(self.cookieFile) self.cj.load(ignore_discard=True, ignore_expires=True) self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cj)) self.opener.add_handler(urllib.request.ProxyHandler(proxies={"http":proxy_IP})) self.opener.addheaders = http_header #get user information from params self.user,self.password = user,password #encode userid userid = bytes(urllib.parse.quote(self.user),'utf-8') self.encode_userid = base64.encodestring(userid)[:-1] #https to login #rsa2 def ssologin(self): #get prelogin #fetch pubkey rsakv servertime nonce url = "https://login.sina.com.cn/sso/prelogin.php?entry=sso&callback=sinaSSOController.preloginCallBack&su=&rsakt=mod&client=ssologin.js(v1.4.11)" time.sleep(0.5) print('Pubkey, rsakv, servertime and nonce obtained') res = self.opener.open(url).read().decode('utf-8') res = res[res.find("(")+1:-1] data = json.loads(res) nonce = data["nonce"] pubkey = data["pubkey"] rsakv = data["rsakv"] servertime = data["servertime"] #init rsa object rsaPublickey = int(pubkey, 16) key = rsa.PublicKey(rsaPublickey,65537) message = str(servertime) + '\t' + str(nonce) + '\n' + str(self.password) message = bytes(message,"utf-8") sp = rsa.encrypt(message,key) sp = binascii.b2a_hex(sp) #to login baseurl = "https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)" params = {"entry":"sinaoauth","gateway":1,"from":"","savestate":0,\ "useticket":1,"vsnf":0,"s":1,"su":self.encode_userid,\ "service":"sinaoauth","servertime":servertime,"nonce":nonce,\ "pwencode":"rsa2","rsakv":rsakv,"sp":sp,"encoding":"UTF-8",\ "callback":"sinaSSOController.loginCallBack","cdult":2,"prelt":83,\ "returntype":"TEXT"} rurl = baseurl + "?" + urllib.parse.urlencode(params) time.sleep(0.5) res = self.opener.open(rurl).read() res = res.decode('utf-8') print('Login responsed obtained') self.cj.save(ignore_discard=True, ignore_expires=True) pos1 = res.find('(') pos2 = res.find(')') js = json.loads(res[pos1+1:pos2]) retcode = js["retcode"] print(js) if retcode == '0': url = js["crossDomainUrlList"][0] time.sleep(0.5) try: self.opener.open(url, timeout=60) except: return False, 'Connection Timeout' self.cj.save(ignore_discard=True, ignore_expires=True) self.opener.close() return True, self.cookieFile else: reason = js["reason"] self.opener.close() return False, str(js)
def login(): global username, my_password totals = { # '*****@*****.**': 'z3vgwakvy', # '*****@*****.**': 'pvyqhkzvg', # '*****@*****.**': 'b3vzc7cq', # '*****@*****.**': 'vba9tq37', # '*****@*****.**': 'vzd2thpmwx5', # '*****@*****.**': 'zuaavgy4qg9' #'18902304731': 'gwk2014081029' '15602200534': 'guoweikuang2016' } # username = '******' # my_password = '******' # totals = sorted(totals.items(), key=lambda d:d[1], reverse=True) print(totals) for (user, passwd) in totals.items(): # username = user # my_password = passwd filenames = os.listdir( '/home/guoweikuang/weibo_showing/app/python_analyse/weibo_crawl01/cookies' ) name = user[:4] + '.txt' if name in filenames: continue else: username = user my_password = passwd # username = input('请输入你要模拟登录的微博账号名:') # my_password = input('请输入你的密码:') # username = '******' # my_password = '******' message = before_login(url_login) post_url, password, vk, cap_id = message[0] captcha = get_cha(cap_id) login_url = 'https://login.weibo.cn/login/' postdata = { "mobile": username, "code": captcha, 'remember': "on", "backURL": "http%3A%2F%2Fweibo.cn", "backTitle": "手机新浪网", "tryCount": "", "vk": vk, "capId": cap_id, "submit": "登录", password: my_password } post_url = login_url + post_url html = session.post(post_url, data=postdata, headers=headers) print(html.content.decode('utf-8')) print(html.cookies.get_dict()) url = 'http://weibo.cn/gzyhl?page=1' html1 = session.get(url) print(html1.status_code) print(html1.cookies) print(session.cookies.get_dict()) file = 'home/guoweikuang/weibo_showing/app/python_analyse/weibo_crawl01/cookies/' file1 = '{}.txt'.format(username[:5]) file += file1 file = 'cookie.txt' if not os.path.exists(file): os.mkdir(file) file_cookiejar = LWPCookieJar(file) requests.utils.cookiejar_from_dict( {c.name: c.value for c in session.cookies}, file_cookiejar) file_cookiejar.save(file, ignore_discard=True, ignore_expires=True) print(html1.content.decode('utf-8'))
class PrettyBugz: def __init__(self, args): self.columns = args.columns or terminal_width() self.user = args.user self.password = args.password self.passwordcmd = args.passwordcmd self.skip_auth = args.skip_auth cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) try: self.cookiejar.load() except IOError: pass self.token_file = os.path.join(os.environ['HOME'], DEFAULT_TOKEN_FILE) try: self.token = open(self.token_file).read().strip() except IOError: self.token = None if getattr(args, 'encoding'): log_info("The --encoding option is deprecated."); log_info("Using %s " % args.base) self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar) def get_input(self, prompt): return input(prompt) def set_token(self, *args): if args and self.token: args[0]['Bugzilla_token'] = self.token return args def call_bz(self, method, *args): """Attempt to call method with args. Log in if authentication is required. """ try: return method(*self.set_token(*args)) except xmlrpc.client.Fault as fault: # Fault code 410 means login required if fault.faultCode == 410 and not self.skip_auth: self.login() return method(*self.set_token(*args)) raise def login(self, args=None): """Authenticate a session. """ # prompt for username if we were not supplied with it if not self.user: log_info('No username given.') self.user = self.get_input('Username: '******'No password given.') self.password = getpass.getpass() else: process = subprocess.Popen(self.passwordcmd, shell=True, stdout=subprocess.PIPE) self.password, _ = process.communicate() self.password = self.password.split('\n')[0] # perform login params = {} params['login'] = self.user params['password'] = self.password if args is not None: params['remember'] = True log_info('Logging in') try: self.bz.User.login(params) except xmlrpc.client.Fault as fault: raise BugzError("Can't login: "******"Failed to logout: " + fault.faultString) def search(self, args): """Performs a search on the bugzilla database with the keywords given on the title (or the body if specified). """ valid_keys = ['alias', 'assigned_to', 'component', 'creator', 'limit', 'offset', 'op_sys', 'platform', 'priority', 'product', 'resolution', 'severity', 'status', 'version', 'whiteboard'] search_opts = sorted([(opt, val) for opt, val in list(args.__dict__.items()) if val is not None and opt in valid_keys]) params = {} for key in args.__dict__: if key in valid_keys and getattr(args, key) is not None: params[key] = getattr(args, key) if getattr(args, 'terms'): params['summary'] = args.terms search_term = ' '.join(args.terms).strip() if not (params or search_term): raise BugzError('Please give search terms or options.') if search_term: log_msg = 'Searching for \'%s\' ' % search_term else: log_msg = 'Searching for bugs ' if search_opts: log_info(log_msg + 'with the following options:') for opt, val in search_opts: log_info(' %-20s = %s' % (opt, val)) else: log_info(log_msg) if 'status' not in params: params['status'] = ['CONFIRMED', 'IN_PROGRESS', 'UNCONFIRMED'] else: for x in params['status'][:]: if x in ['all', 'ALL']: del params['status'] result = self.call_bz(self.bz.Bug.search, params)['bugs'] if not len(result): log_info('No bugs found.') else: self.list_bugs(result, args) def get(self, args): """ Fetch bug details given the bug id """ log_info('Getting bug %s ..' % args.bugid) try: result = self.call_bz(self.bz.Bug.get, {'ids':[args.bugid]}) except xmlrpc.client.Fault as fault: raise BugzError("Can't get bug #" + str(args.bugid) + ": " \ + fault.faultString) for bug in result['bugs']: self.show_bug_info(bug, args.attachments, args.comments) def post(self, args): """Post a new bug""" # load description from file if possible if args.description_from is not None: try: if args.description_from == '-': args.description = sys.stdin.read() else: args.description = open( args.description_from, 'r').read() except IOError as e: raise BugzError('Unable to read from file: %s: %s' % (args.description_from, e)) if not args.batch: log_info('Press Ctrl+C at any time to abort.') # # Check all bug fields. # XXX: We use "if not <field>" for mandatory fields # and "if <field> is None" for optional ones. # # check for product if not args.product: while not args.product or len(args.product) < 1: args.product = self.get_input('Enter product: ') else: log_info('Enter product: %s' % args.product) # check for component if not args.component: while not args.component or len(args.component) < 1: args.component = self.get_input('Enter component: ') else: log_info('Enter component: %s' % args.component) # check for version # FIXME: This default behaviour is not too nice. if not args.version: line = self.get_input('Enter version (default: unspecified): ') if len(line): args.version = line else: args.version = 'unspecified' else: log_info('Enter version: %s' % args.version) # check for title if not args.summary: while not args.summary or len(args.summary) < 1: args.summary = self.get_input('Enter title: ') else: log_info('Enter title: %s' % args.summary) # check for description if not args.description: line = block_edit('Enter bug description: ') if len(line): args.description = line else: log_info('Enter bug description: %s' % args.description) # check for operating system if not args.op_sys: op_sys_msg = 'Enter operating system where this bug occurs: ' line = self.get_input(op_sys_msg) if len(line): args.op_sys = line else: log_info('Enter operating system: %s' % args.op_sys) # check for platform if not args.platform: platform_msg = 'Enter hardware platform where this bug occurs: ' line = self.get_input(platform_msg) if len(line): args.platform = line else: log_info('Enter hardware platform: %s' % args.platform) # check for default priority if args.priority is None: priority_msg ='Enter priority (eg. Normal) (optional): ' line = self.get_input(priority_msg) if len(line): args.priority = line else: log_info('Enter priority (optional): %s' % args.priority) # check for default severity if args.severity is None: severity_msg ='Enter severity (eg. normal) (optional): ' line = self.get_input(severity_msg) if len(line): args.severity = line else: log_info('Enter severity (optional): %s' % args.severity) # check for default alias if args.alias is None: alias_msg ='Enter an alias for this bug (optional): ' line = self.get_input(alias_msg) if len(line): args.alias = line else: log_info('Enter alias (optional): %s' % args.alias) # check for default assignee if args.assigned_to is None: assign_msg ='Enter assignee (eg. [email protected]) (optional): ' line = self.get_input(assign_msg) if len(line): args.assigned_to = line else: log_info('Enter assignee (optional): %s' % args.assigned_to) # check for CC list if args.cc is None: cc_msg = 'Enter a CC list (comma separated) (optional): ' line = self.get_input(cc_msg) if len(line): args.cc = line.split(', ') else: log_info('Enter a CC list (optional): %s' % args.cc) # check for URL if args.url is None: url_msg = 'Enter a URL (optional): ' line = self.get_input(url_msg) if len(line): args.url = line else: log_info('Enter a URL (optional): %s' % args.url) # fixme: groups # fixme: status # fixme: milestone if args.append_command is None: args.append_command = self.get_input('Append the output of the following command (leave blank for none): ') else: log_info('Append command (optional): %s' % args.append_command) # raise an exception if mandatory fields are not specified. if args.product is None: raise RuntimeError('Product not specified') if args.component is None: raise RuntimeError('Component not specified') if args.summary is None: raise RuntimeError('Title not specified') if args.description is None: raise RuntimeError('Description not specified') if not args.version: args.version = 'unspecified' # append the output from append_command to the description if args.append_command is not None and args.append_command != '': append_command_output = subprocess.getoutput(args.append_command) args.description = args.description + '\n\n' + '$ ' + args.append_command + '\n' + append_command_output # print submission confirmation print('-' * (self.columns - 1)) print('%-12s: %s' % ('Product', args.product)) print('%-12s: %s' %('Component', args.component)) print('%-12s: %s' % ('Title', args.summary)) print('%-12s: %s' % ('Version', args.version)) print('%-12s: %s' % ('Description', args.description)) print('%-12s: %s' % ('Operating System', args.op_sys)) print('%-12s: %s' % ('Platform', args.platform)) print('%-12s: %s' % ('Priority', args.priority)) print('%-12s: %s' % ('Severity', args.severity)) print('%-12s: %s' % ('Alias', args.alias)) print('%-12s: %s' % ('Assigned to', args.assigned_to)) print('%-12s: %s' % ('CC', args.cc)) print('%-12s: %s' % ('URL', args.url)) # fixme: groups # fixme: status # fixme: Milestone print('-' * (self.columns - 1)) if not args.batch: if args.default_confirm in ['Y','y']: confirm = input('Confirm bug submission (Y/n)? ') else: confirm = input('Confirm bug submission (y/N)? ') if len(confirm) < 1: confirm = args.default_confirm if confirm[0] not in ('y', 'Y'): log_info('Submission aborted') return params={} params['product'] = args.product params['component'] = args.component params['version'] = args.version params['summary'] = args.summary if args.description is not None: params['description'] = args.description if args.op_sys is not None: params['op_sys'] = args.op_sys if args.platform is not None: params['platform'] = args.platform if args.priority is not None: params['priority'] = args.priority if args.severity is not None: params['severity'] = args.severity if args.alias is not None: params['alias'] = args.alias if args.assigned_to is not None: params['assigned_to'] = args.assigned_to if args.cc is not None: params['cc'] = args.cc if args.url is not None: params['url'] = args.url result = self.call_bz(self.bz.Bug.create, params) log_info('Bug %d submitted' % result['id']) def modify(self, args): """Modify an existing bug (eg. adding a comment or changing resolution.)""" if args.comment_from: try: if args.comment_from == '-': args.comment = sys.stdin.read() else: args.comment = open(args.comment_from, 'r').read() except IOError as e: raise BugzError('unable to read file: %s: %s' % \ (args.comment_from, e)) if args.comment_editor: args.comment = block_edit('Enter comment:') params = {} if args.blocks_add is not None or args.blocks_remove is not None: params['blocks'] = {} if args.depends_on_add is not None \ or args.depends_on_remove is not None: params['depends_on'] = {} if args.cc_add is not None or args.cc_remove is not None: params['cc'] = {} if args.comment is not None: params['comment'] = {} if args.groups_add is not None or args.groups_remove is not None: params['groups'] = {} if args.keywords_set is not None: params['keywords'] = {} if args.see_also_add is not None or args.see_also_remove is not None: params['see_also'] = {} params['ids'] = [args.bugid] if args.alias is not None: params['alias'] = args.alias if args.assigned_to is not None: params['assigned_to'] = args.assigned_to if args.blocks_add is not None: params['blocks']['add'] = args.blocks_add if args.blocks_remove is not None: params['blocks']['remove'] = args.blocks_remove if args.depends_on_add is not None: params['depends_on']['add'] = args.depends_on_add if args.depends_on_remove is not None: params['depends_on']['remove'] = args.depends_on_remove if args.cc_add is not None: params['cc']['add'] = args.cc_add if args.cc_remove is not None: params['cc']['remove'] = args.cc_remove if args.comment is not None: params['comment']['body'] = args.comment if args.component is not None: params['component'] = args.component if args.dupe_of: params['dupe_of'] = args.dupe_of args.status = None args.resolution = None if args.groups_add is not None: params['groups']['add'] = args.groups_add if args.groups_remove is not None: params['groups']['remove'] = args.groups_remove if args.keywords_set is not None: params['keywords']['set'] = args.keywords_set if args.op_sys is not None: params['op_sys'] = args.op_sys if args.platform is not None: params['platform'] = args.platform if args.priority is not None: params['priority'] = args.priority if args.product is not None: params['product'] = args.product if args.resolution is not None: params['resolution'] = args.resolution if args.see_also_add is not None: params['see_also']['add'] = args.see_also_add if args.see_also_remove is not None: params['see_also']['remove'] = args.see_also_remove if args.severity is not None: params['severity'] = args.severity if args.status is not None: params['status'] = args.status if args.summary is not None: params['summary'] = args.summary if args.url is not None: params['url'] = args.url if args.version is not None: params['version'] = args.version if args.whiteboard is not None: params['whiteboard'] = args.whiteboard if args.fixed: params['status'] = 'RESOLVED' params['resolution'] = 'FIXED' if args.invalid: params['status'] = 'RESOLVED' params['resolution'] = 'INVALID' if len(params) < 2: raise BugzError('No changes were specified') result = self.call_bz(self.bz.Bug.update, params) for bug in result['bugs']: changes = bug['changes'] if not len(changes): log_info('Added comment to bug %s' % bug['id']) else: log_info('Modified the following fields in bug %s' % bug['id']) for key in changes: log_info('%-12s: removed %s' %(key, changes[key]['removed'])) log_info('%-12s: added %s' %(key, changes[key]['added'])) def attachment(self, args): """ Download or view an attachment given the id.""" log_info('Getting attachment %s' % args.attachid) params = {} params['attachment_ids'] = [args.attachid] result = self.call_bz(self.bz.Bug.attachments, params) result = result['attachments'][args.attachid] action = {True:'Viewing', False:'Saving'} log_info('%s attachment: "%s"' % (action[args.view], result['file_name'])) safe_filename = os.path.basename(re.sub(r'\.\.', '', result['file_name'])) if args.view: print(result['data'].data) else: if os.path.exists(result['file_name']): raise RuntimeError('Filename already exists') fd = open(safe_filename, 'wb') fd.write(result['data'].data) fd.close() def attach(self, args): """ Attach a file to a bug given a filename. """ filename = args.filename content_type = args.content_type bugid = args.bugid summary = args.summary is_patch = args.is_patch comment = args.comment if not os.path.exists(filename): raise BugzError('File not found: %s' % filename) if content_type is None: content_type = get_content_type(filename) if comment is None: comment = block_edit('Enter optional long description of attachment') if summary is None: summary = os.path.basename(filename) params = {} params['ids'] = [bugid] fd = open(filename, 'rb') params['data'] = xmlrpc.client.Binary(fd.read()) fd.close() params['file_name'] = os.path.basename(filename) params['summary'] = summary if not is_patch: params['content_type'] = content_type; params['comment'] = comment params['is_patch'] = is_patch result = self.call_bz(self.bz.Bug.add_attachment, params) log_info("'%s' has been attached to bug %s" % (filename, bugid)) def list_bugs(self, buglist, args): for bug in buglist: bugid = bug['id'] status = bug['status'] priority = bug['priority'] severity = bug['severity'] assignee = bug['assigned_to'].split('@')[0] desc = bug['summary'] line = '%s' % (bugid) if args.show_status: line = '%s %-12s' % (line, status) if args.show_priority: line = '%s %-12s' % (line, priority) if args.show_severity: line = '%s %-12s' % (line, severity) line = '%s %-20s' % (line, assignee) line = '%s %s' % (line, desc) print(line[:self.columns]) log_info("%i bug(s) found." % len(buglist)) def show_bug_info(self, bug, show_attachments, show_comments): FieldMap = { 'alias': 'Alias', 'summary': 'Title', 'status': 'Status', 'resolution': 'Resolution', 'product': 'Product', 'component': 'Component', 'version': 'Version', 'platform': 'Hardware', 'op_sys': 'OpSystem', 'priority': 'Priority', 'severity': 'Severity', 'target_milestone': 'TargetMilestone', 'assigned_to': 'AssignedTo', 'url': 'URL', 'whiteboard': 'Whiteboard', 'keywords': 'Keywords', 'depends_on': 'dependsOn', 'blocks': 'Blocks', 'creation_time': 'Reported', 'creator': 'Reporter', 'last_change_time': 'Updated', 'cc': 'CC', 'see_also': 'See Also', } SkipFields = ['is_open', 'id', 'is_confirmed', 'is_creator_accessible', 'is_cc_accessible', 'update_token'] for field in bug: if field in SkipFields: continue if field in FieldMap: desc = FieldMap[field] else: desc = field value = bug[field] if field in ['cc', 'see_also']: for x in value: print('%-12s: %s' % (desc, x)) elif isinstance(value, list): s = ', '.join(["%s" % x for x in value]) if s: print('%-12s: %s' % (desc, s)) elif value is not None and value != '': print('%-12s: %s' % (desc, value)) if show_attachments: bug_attachments = self.call_bz(self.bz.Bug.attachments, {'ids':[bug['id']]}) bug_attachments = bug_attachments['bugs']['%s' % bug['id']] print('%-12s: %d' % ('Attachments', len(bug_attachments))) print() for attachment in bug_attachments: aid = attachment['id'] desc = attachment['summary'] when = attachment['creation_time'] print('[Attachment] [%s] [%s]' % (aid, desc)) if show_comments: bug_comments = self.call_bz(self.bz.Bug.comments, {'ids':[bug['id']]}) bug_comments = bug_comments['bugs']['%s' % bug['id']]['comments'] print() print('%-12s: %d' % ('Comments', len(bug_comments))) print() i = 0 wrapper = textwrap.TextWrapper(width = self.columns, break_long_words = False, break_on_hyphens = False) for comment in bug_comments: who = comment['creator'] when = comment['time'] what = comment['text'] print('[Comment #%d] %s : %s' % (i, who, when)) print('-' * (self.columns - 1)) if what is None: what = '' # print wrapped version for line in what.split('\n'): if len(line) < self.columns: print(line) else: for shortline in wrapper.wrap(line): print(shortline) print() i += 1
class URLHandler: def __init__(self): self.cookie_filename = os.path.join(__profile__, "cookiejar.txt") self.cookie_jar = LWPCookieJar(self.cookie_filename) if os.access(self.cookie_filename, os.F_OK): self.cookie_jar.load() self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar)) self.opener.addheaders = [ ('Accept-Encoding', 'gzip'), ('Accept-Language', 'en-us,en;q=0.5'), ('Pragma', 'no-cache'), ('Cache-Control', 'no-cache'), ('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36' ) ] def request(self, url, query_string=None, data=None, ajax=False, referer=None, cookie=None, decode_zlib=True): if data is not None: self.opener.addheaders += [ ('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8') ] data = urlencode(data).encode("utf-8") if query_string is not None: query_string = urlencode(query_string) url += "?" + query_string if ajax: self.opener.addheaders += [('X-Requested-With', 'XMLHttpRequest')] if referer is not None: self.opener.addheaders += [('Referer', referer)] if cookie is not None: self.opener.addheaders += [('Cookie', cookie)] content = None log("Getting url: %s\nData: %s" % (url, data)) try: response = self.opener.open(url, data) if response.code == 200: content = response.read() if decode_zlib and 'gzip' in response.headers.get( 'content-encoding', ''): try: content = zlib.decompress(content, 16 + zlib.MAX_WBITS) except zlib.error: pass if 'application/json' in response.headers.get( 'content-type', ''): content = json.loads(content, encoding="utf-8") elif 'text/html' in response.headers.get('content-type', ''): content = content.decode('utf-8') response.close() except Exception as e: log("Failed to get url: %s\n%s" % (url, e)) # Second parameter is the filename return content def save_cookie(self): self.cookie_jar.save()
class ClansSession(object): """ This object is created on each `clans` incantation as a storage place for configuration info, command-line arguments, and other stuff """ def __init__(self, profile_dir=None): # profile folder: either passed directly (for testing only), # set by CLANS_DIR environment variable, or the standard user # data directory for this OS self.profile_dir = profile_dir or ( os.environ.get('CLANS_DIR', '') or appdirs.user_data_dir(appname='clans', appauthor='baldwint')) # config file location: in data directory self.config_loc = os.path.join(self.profile_dir, 'clans.cfg') # load config, extensions, and define command line args self.config = self._load_config() self.extensions = self._load_extensions() self.formatters = self._load_formatters() self.commands = self._load_commands() # let extensions modify command list self.hook('post_load_commands') def run(self, argv=None): """ Clans main function, run with specified arguments """ # get command line arguments self.args = self.commands.main.parse_args(argv) self.args = vars(self.args) # let command line args override equivalent config file settings self.username = (self.args['username'] or self.config.get('login', 'username')) try: # pass execution to the subcommand func = self.args['func'] func(self) finally: # do this part always, even if subcommand fails self.finish() def _load_config(self): # set config file defaults config = ConfigParser() config.add_section('login') config.set('login', 'username', '') config.set('login', 'url', 'https://www.grinnellplans.com') config.add_section('clans') # text editor: either specified in config file, or $EDITOR, or pico config.set('clans', 'editor', os.environ.get('EDITOR', 'pico')) config.set('clans', 'format', 'raw') config.set('clans', 'timezone', '') config.set('clans', 'date_format', '') # create profile directory if it doesn't exist try: # 0700 for secure-ish cookie storage. os.makedirs(self.profile_dir, 0o700) except OSError: pass # already exists # read user's config file, if present config.read(self.config_loc) return config def _load_extensions(self): """ Load Clans extensions. reads the config file for extension information, and loads extensions as python modules into an ordered dictionary. """ extensions = OrderedDict() if self.config.has_section('extensions'): for name, path in self.config.items('extensions'): try: if not path: # if no value is specified, # assume it is for a built-in extension path = 'clans.ext.%s' % name mod = importlib.import_module(path) assert mod.__name__ == path except ImportError: print('Failed to load extension "%s".' % name, file=sys.stderr) else: extensions[name] = mod return extensions def _load_formatters(self): """ Load output formatters. """ formatters = { 'raw': clans.fmt.RawFormatter, 'json': clans.fmt.JSONFormatter, 'text': clans.fmt.TextFormatter, 'color': clans.fmt.ColorFormatter, } return formatters def hook(self, name, *args, **kwargs): """ Call the method named ``name`` in every loaded extension. Returns: a list of return values. """ def run_hook(ext, name): func = getattr(ext, name, None) if func is not None: return func(self, *args, **kwargs) else: return None results = [run_hook(ext, name) for ext in self.extensions.values()] return results def _load_commands(self): # define command line arguments # globals: options/arguments inherited by all parsers, including root global_parser = argparse.ArgumentParser(add_help=False) global_parser.add_argument( '-u', '--username', dest='username', default='', help='GrinnellPlans username, no brackets.') global_parser.add_argument( '-p', '--password', dest='password', default='', help='GrinnellPlans password. Omit for secure entry.') global_parser.add_argument( '--logout', dest='logout', action='store_true', default=False, help='Log out before quitting.') global_parser.add_argument( '--version', action='version', version='%(prog)s ' + clans.__version__, help='Show clans version number and exit.') # filters: options/arguments for those commands that format text filter_parser = argparse.ArgumentParser(add_help=False) filter_parser.add_argument( '--format', dest='fmt', default=self.config.get('clans', 'format'), choices=self.formatters, help="Display format to use") # main parser: has subcommands for everything commands = CommandSet( description=__doc__, parents=[global_parser], formatter_class=argparse.RawTextHelpFormatter) # edit parser: options/arguments for editing plans commands.add_command( 'edit', edit, parents=[global_parser], description='Opens your plan for editing in a text editor.', help='Edit your plan.') commands["edit"].add_argument( '--from-file', dest='source_file', default=False, metavar='FILE', help="Replace plan with the contents of FILE. " "Skips interactive editing." " Use with caution!") # read parser commands.add_command( 'read', read, parents=[global_parser, filter_parser], description="Read someone else's plan.", help="Read a plan.",) commands["read"].add_argument( 'plan', default=False, metavar='PLAN', help="Name of plan to be read.") # autoread list parser commands.add_command( 'list', autoread, parents=[global_parser, filter_parser], description="Display unread plans on your autoread list.", help="Display autoread list.",) # quicklove parser commands.add_command( 'love', love, parents=[global_parser, filter_parser], description="Search for other users giving you planlove.", help="Check quicklove.",) # search parser commands.add_command( 'search', search, parents=[global_parser, filter_parser], description="Search plans for any word or phrase.", help="Search plans for any word or phrase.",) commands["search"].add_argument( 'term', default=False, metavar='TERM', help="Term to search for.") commands["search"].add_argument( '-l', '--love', dest='love', action='store_true', default=False, help="Restrict search to planlove.") # watch parser commands.add_command( 'watch', watch, parents=[global_parser, filter_parser], description="See recently updated plans.", help="See recently updated plans.",) commands["watch"].add_argument( 'hours', type=int, nargs='?', default=12, metavar='HOURS', help="Specify how many hours' worth of plan updates to show.") # config parser commands.add_command( 'config', config, parents=[global_parser, filter_parser], description="The clans config file sets the default" " behavior of the client." " (Not to be confused with Plans preferences!)", help="Edit clans configuration file.") commands["config"].add_argument( '--dir', dest='profile_dir', action='store_true', default=False, help="Print the path to the clans profile directory.") return commands def make_plans_connection(self): """ Connects to plans, prompting for passwords if necessary """ # create a cookie self.cookie = LWPCookieJar( os.path.join(self.profile_dir, '%s.cookie' % self.username)) try: self.cookie.load() # fails with IOError if it does not exist except IOError: pass # no cookie saved for this user # create plans connection using cookie pc = PlansConnection(self.cookie, base_url=self.config.get('login', 'url')) if pc.plans_login(): pass # we're still logged in else: # we're not logged in, prompt for password if necessary password = (self.args['password'] or getpass("[%s]'s password: "******""" Initialize and return the appropriate output formatter. """ Fmt = self.formatters[self.args['fmt']] kwargs = {} kwargs['timezone'] = self.config.get('clans', 'timezone') or None kwargs['date_format'] = self.config.get('clans', 'date_format') or None # if a key is None, remove it and rely on the default kwargs = dict((k,v) for k,v in kwargs.items() if v is not None) fmt = Fmt(**kwargs) return fmt def finish(self): """ Cookie-related cleanup. Either save the updated cookie, or delete it to log out """ if not hasattr(self, 'cookie'): return # no plans connection was made elif self.args['logout']: os.unlink(self.cookie.filename) else: # save cookie self.cookie.save()
def getCookie(self): ''' save cookie to file and test cookie, if invaild, get new one and bind to self.headers ''' s = requests.Session() # get cookie file to test, if invaild, to step 1, else fit to self.header and return load_cookiejar = LWPCookieJar() if os.path.exists(self.cookie_filename): #从文件中加载cookies(LWP格式) load_cookiejar.load(self.cookie_filename, ignore_discard=True, ignore_expires=True) #工具方法转换成字典 load_cookies = requests.utils.dict_from_cookiejar(load_cookiejar) #工具方法将字典转换成RequestsCookieJar,赋值给session的cookies. s.cookies = requests.utils.cookiejar_from_dict(load_cookies) # s.cookies = MozillaCookieJar(filename=self.cookie_filename) # set max retry entries -> 5 s.mount('http://', HTTPAdapter(max_retries=Retry(total=5, method_whitelist=frozenset(['GET', 'POST'])))) try: test_resp = s.get(self.cookie_test_url) except requests.exceptions.RequestException as e: print('test cookie failed') return None # Test cookie vaild if not self.invaild_text in test_resp.text: print('Cookie vaild') self.session = s return s.cookies # Invaild Cookie, get new Cookie print('Store Cookie invaild, get new one...') try: # Get login info pre_login_resp = s.get(self.pre_login_url) if not pre_login_resp.status_code == 200: print('Get key or sessId failed') return None # find key and sessId key_match = re.search(r"\('input\[name=password\]'\)\.val\(\), '(.*?)'", pre_login_resp.text, re.S) if not key_match: print('get key failed') return None key = key_match.group(1) sessId_match = re.search('<input.*?name="sessId".*?value="(.*?)".*?>', pre_login_resp.text, re.S) if not sessId_match: print('get sessid failed') return None sessId = sessId_match.group(1) rsaPwd = '' with open(self.rsa_js_filename, 'r', encoding='utf-8') as f: des = execjs.compile(f.read()) rsaPwd = str(des.call('encMe', self.login_password, key)) if not rsaPwd: print('get rsaPwd failed') return None login_data = { 'redirectUrl': '', 'productCode': 'Portal', 'extParams': '', 'vu': '', 'sessId': sessId, 'username': self.login_username, 'rsaPassword': rsaPwd, 'password': self.login_password, 'savePassword': '******', 'autoLogin': '******' } # Login Action login_resp = s.post(self.login_url, data=login_data) if login_resp.status_code == 200: print('Get new cookie done ') #保存cookie到文件 #ignore_discard的意思是即使cookies将被丢弃也将它保存下来 #ignore_expires的意思是如果在该文件中cookies已经存在,则覆盖原文件写入 with open(self.cookie_filename, 'w') as f: f.seek(0) f.truncate() new_cookie_jar = LWPCookieJar(self.cookie_filename) requests.utils.cookiejar_from_dict({c.name: c.value for c in s.cookies}, new_cookie_jar) new_cookie_jar.save(ignore_discard=True,ignore_expires=True) self.session = s return s.cookies else: print('Get new Cookie failed') return None except requests.exceptions.RequestException as e: print('Get Pre login info failed') return None
class URLHandler: def __init__(self): cookie_filename = os.path.join(__profile__, "cookiejar.txt") self.cookie_jar = LWPCookieJar(cookie_filename) if os.access(cookie_filename, os.F_OK): self.cookie_jar.load() self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar)) self.opener.addheaders = [ ('Accept-Encoding', 'gzip'), ('Accept-Language', 'en-us,en;q=0.5'), ('Pragma', 'no-cache'), ('Cache-Control', 'no-cache'), ('Content-type', 'application/json'), ('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Kodi/%s Chrome/78.0.3904.97 Safari/537.36' % (__kodi_version__)) ] def request(self, url, data=None, query_string=None, referrer=None, cookie=None): if data is not None: data = json.dumps(data).encode('utf8') if query_string is not None: url += '?' + urlencode(query_string) if referrer is not None: self.opener.addheaders += [('Referrer', referrer)] if cookie is not None: self.opener.addheaders += [('Cookie', cookie)] content = None log("Getting url: %s" % (url)) if data is not None: log("Post Data: %s" % (data)) try: req = Request(url, data, headers={'Content-Type': 'application/json'}) response = self.opener.open(req) content = None if response.code != 200 else response.read() if response.headers.get('content-encoding', '') == 'gzip': try: content = zlib.decompress(content, 16 + zlib.MAX_WBITS) except zlib.error: pass if response.headers.get('content-type', '').startswith('application/json'): parsed_content = json.loads(content, encoding="utf-8") content = json.loads(parsed_content["d"], encoding="utf-8") response.close() except Exception as e: log("Failed to get url: %s\n%s" % (url, e)) # Second parameter is the filename return content def save_cookie(self): # extend cookie expiration for cookie in self.cookie_jar: if cookie.expires is not None: cookie.expires += 2 * 12 * 30 * 24 * 60 * 60 self.cookie_jar.save()
class PrettyBugz: def __init__(self, args): self.user = args['user'] self.password = args['password'] self.skip_auth = args['skip_auth'] cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) try: self.cookiejar.load() except IOError: pass self.token_file = os.path.join(os.environ['HOME'], DEFAULT_TOKEN_FILE) try: self.token = open(self.token_file).read().strip() except IOError: self.token = None log_info("Using %s " % args['url']) self.bz = BugzillaProxy(args['url'], cookiejar=self.cookiejar) def set_token(self, *args): if args and self.token: args[0]['Bugzilla_token'] = self.token return args def call_bz(self, method, *args): """Attempt to call method with args. Log in if authentication is required. """ try: return method(*self.set_token(*args)) except xmlrpc.client.Fault as fault: # Fault code 410 means login required if fault.faultCode == 410 and not self.skip_auth: self.login() return method(*self.set_token(*args)) raise def login(self, args=None): # Authenticate a session. # perform login params = {} params['login'] = self.user params['password'] = self.password if args is not None: params['remember'] = True log_info('Logging in') try: self.bz.User.login(params) except xmlrpc.client.Fault as fault: raise BugzError("Can't login: "******"Failed to logout: " + fault.faultString) def search(self, args): """Performs a search on the bugzilla database with the keywords given on the title (or the body if specified). """ valid_keys = ['alias', 'assigned_to', 'component', 'creator', 'limit', 'offset', 'op_sys', 'platform', 'priority', 'product', 'resolution', 'severity', 'status', 'version', 'whiteboard'] search_opts = sorted([(opt, val) for opt, val in list(args.__dict__.items()) if val is not None and opt in valid_keys]) params = {} for key in args.__dict__: if key in valid_keys and getattr(args, key) is not None: params[key] = getattr(args, key) if getattr(args, 'terms'): params['summary'] = args.terms search_term = ' '.join(args.terms).strip() if not (params or search_term): raise BugzError('Please give search terms or options.') if search_term: log_msg = 'Searching for \'%s\' ' % search_term else: log_msg = 'Searching for bugs ' if search_opts: log_info(log_msg + 'with the following options:') for opt, val in search_opts: log_info(' %-20s = %s' % (opt, val)) else: log_info(log_msg) if 'status' not in params: params['status'] = ['CONFIRMED', 'IN_PROGRESS', 'UNCONFIRMED'] else: for x in params['status'][:]: if x in ['all', 'ALL']: del params['status'] result = self.call_bz(self.bz.Bug.search, params)['bugs'] if not len(result): log_info('No bugs found.') else: self.list_bugs(result, args) def get(self, args): """ Fetch bug details given the bug id """ log_info('Getting bug %s ..' % args['bugid']) try: result = self.call_bz(self.bz.Bug.get, {'ids':[args['bugid']]}) except xmlrpc.client.Fault as fault: raise BugzError("Can't get bug #" + str(args['bugid']) + ": " \ + fault.faultString) for bug in result['bugs']: self.show_bug_info(bug, args.attachments, args.comments) def post(self, args): """Post a new bug""" params={} params['product'] = args['product'] params['component'] = args['component'] params['summary'] = args['summary'] params['description'] = args['description'] #params['assigned_to'] = args['assigned_to'] params['version'] = args['version'] result = self.call_bz(self.bz.Bug.create, params) log_info('Bug %d submitted' % result['id']) return result def modify(self, args): """Modify an existing bug (eg. adding a comment or changing resolution.)""" params = {} if args['cc_add'] is not None: params['cc'] = {} if args['comment'] is not None: params['comment'] = {} params['ids'] = args['bugid'] # if args['assigned_to'] is not None: # params['assigned_to'] = args['assigned_to'] if args['cc_add'] is not None: params['cc']['add'] = args['cc_add'] if args['comment'] is not None: params['comment']['body'] = args['comment'] if len(params) < 2: raise BugzError('No changes were specified') result = self.call_bz(self.bz.Bug.update, params) for bug in result['bugs']: changes = bug['changes'] if not len(changes): log_info('Added comment to bug %s' % bug['id']) else: log_info('Modified the following fields in bug %s' % bug['id']) for key in changes: log_info('%-12s: removed %s' %(key, changes[key]['removed'])) log_info('%-12s: added %s' %(key, changes[key]['added'])) def attachment(self, args): """ Download or view an attachment given the id.""" log_info('Getting attachment %s' % args.attachid) params = {} params['attachment_ids'] = [args.attachid] result = self.call_bz(self.bz.Bug.attachments, params) result = result['attachments'][args.attachid] action = {True:'Viewing', False:'Saving'} log_info('%s attachment: "%s"' % (action[args.view], result['file_name'])) safe_filename = os.path.basename(re.sub(r'\.\.', '', result['file_name'])) if args.view: print(result['data'].data) else: if os.path.exists(result['file_name']): raise RuntimeError('Filename already exists') fd = open(safe_filename, 'wb') fd.write(result['data'].data) fd.close() def attach(self, args): """ Attach a file to a bug given a filename. """ filename = args['filename'] summary = os.path.basename(filename) comment = args['comment'] if not os.path.exists(filename): raise BugzError('File not found: %s' % filename) params = {} params['ids'] = args['bugid'] fd = open(filename, 'rb') params['data'] = xmlrpc.client.Binary(fd.read()) fd.close() params['file_name'] = os.path.basename(filename) params['summary'] = summary params['content_type'] = args['content_type'] params['comment'] = comment #params['is_patch'] = is_patch result = self.call_bz(self.bz.Bug.add_attachment, params) log_info("'%s' has been attached to bug %s" % (filename, args['bugid'])) def list_bugs(self, buglist, args): for bug in buglist: bugid = bug['id'] status = bug['status'] priority = bug['priority'] severity = bug['severity'] assignee = bug['assigned_to'].split('@')[0] desc = bug['summary'] line = '%s' % (bugid) if args.show_status: line = '%s %-12s' % (line, status) if args.show_priority: line = '%s %-12s' % (line, priority) if args.show_severity: line = '%s %-12s' % (line, severity) line = '%s %-20s' % (line, assignee) line = '%s %s' % (line, desc) print(line[:self.columns]) log_info("%i bug(s) found." % len(buglist)) def show_bug_info(self, bug, show_attachments, show_comments): FieldMap = { 'alias': 'Alias', 'summary': 'Title', 'status': 'Status', 'resolution': 'Resolution', 'product': 'Product', 'component': 'Component', 'version': 'Version', 'platform': 'Hardware', 'op_sys': 'OpSystem', 'priority': 'Priority', 'severity': 'Severity', 'target_milestone': 'TargetMilestone', 'assigned_to': 'AssignedTo', 'url': 'URL', 'whiteboard': 'Whiteboard', 'keywords': 'Keywords', 'depends_on': 'dependsOn', 'blocks': 'Blocks', 'creation_time': 'Reported', 'creator': 'Reporter', 'last_change_time': 'Updated', 'cc': 'CC', 'see_also': 'See Also', } SkipFields = ['is_open', 'id', 'is_confirmed', 'is_creator_accessible', 'is_cc_accessible', 'update_token'] for field in bug: if field in SkipFields: continue if field in FieldMap: desc = FieldMap[field] else: desc = field value = bug[field] if field in ['cc', 'see_also']: for x in value: print('%-12s: %s' % (desc, x)) elif isinstance(value, list): s = ', '.join(["%s" % x for x in value]) if s: print('%-12s: %s' % (desc, s)) elif value is not None and value != '': print('%-12s: %s' % (desc, value)) if show_attachments: bug_attachments = self.call_bz(self.bz.Bug.attachments, {'ids':[bug['id']]}) bug_attachments = bug_attachments['bugs']['%s' % bug['id']] print('%-12s: %d' % ('Attachments', len(bug_attachments))) print() for attachment in bug_attachments: aid = attachment['id'] desc = attachment['summary'] when = attachment['creation_time'] print('[Attachment] [%s] [%s]' % (aid, desc)) if show_comments: bug_comments = self.call_bz(self.bz.Bug.comments, {'ids':[bug['id']]}) bug_comments = bug_comments['bugs']['%s' % bug['id']]['comments'] print() print('%-12s: %d' % ('Comments', len(bug_comments))) print() i = 0 wrapper = textwrap.TextWrapper(width = self.columns, break_long_words = False, break_on_hyphens = False) for comment in bug_comments: who = comment['creator'] when = comment['time'] what = comment['text'] print('[Comment #%d] %s : %s' % (i, who, when)) print('-' * (self.columns - 1)) if what is None: what = '' # print wrapped version for line in what.split('\n'): if len(line) < self.columns: print(line) else: for shortline in wrapper.wrap(line): print(shortline) print() i += 1
class CookiePageGetter(object): def __init__(self, username=None, password=None, cookiefile=None, logger=None): self.bandwidth = 0 self.logger = logger or SilentLogger() self.cookiefile = cookiefile or COOKIEFILE self.cj = LWPCookieJar() if os.path.isfile(self.cookiefile): self.cj.load(self.cookiefile) self.opener = build_opener(HTTPCookieProcessor(self.cj)) self.opener.addheaders = [("User-Agent", USER_AGENT)] self.opener.addheaders.append(("Accept-Encoding", "gzip")) if not os.path.isfile(self.cookiefile) and username is not None and password is not None: self.logger.log("Logging in") self.log_in(username, password) self.html_encoding = "utf-8" def log_in(self): pass def _read_in_chunks(self, response): data = bytes() while True: small_data = response.read(1024) self.bandwidth += len(small_data) if len(small_data) == 0: break data += small_data return data def _open_with_retry(self, request): # sometimes there are timeouts and such - try to open the page 10 times until giving up for i in range(10): try: response = self.opener.open(request) if i != 0: self.logger.log("* Recovered from open error after %d tries" % i) return response except: pass return None def _prepare_request(self, request, additional_headers, post_data): if additional_headers is not None: for k, v in additional_headers.items(): request.add_header(k, v) if post_data is not None: request.add_data(urlencode(post_data).encode("ascii")) def _request(self, url, additional_headers=None, post_data=None): request = Request(url) self._prepare_request(request, additional_headers, post_data) response = self._open_with_retry(request) if response is None: self.logger.log("* ERROR Could not open <%s>" % url) return bytes() data = self._read_in_chunks(response) self.cj.save(self.cookiefile) # handle gzip'd data if response.info().get("Content-Encoding") == "gzip": gzip_stream = BytesIO(data) gzip_file = gzip.GzipFile(fileobj=gzip_stream) data = gzip_file.read() self._update_encoding(response) return data def get_page_html(self, url, additional_headers=None, post_data=None): data = self._request(url, additional_headers, post_data) data = data.decode(self.html_encoding) return data def download_binary(self, url, fname): data = self._request(url) open(fname, "wb").write(data) def _update_encoding(self, response): content_type = response.info().get('content-type') if content_type is None: return matchobj = re.search("charset=([^;]+)", content_type) if matchobj is None: return self.html_encoding = matchobj.group(1) def _format_size(self, bytes): suffixes = ['T', 'G', 'M', 'K', ''] bytes = [bytes] for i in range(len(suffixes)-1): bytes = list(divmod(bytes[0], 1024)) + bytes[1:] return ', '.join(["%d %sB" % (val, suf) for val, suf in zip(bytes, suffixes) if val != 0]) def log_bandwidth(self): self.logger.log("%s transferred" % (self._format_size(self.bandwidth)))
class GoogleSearch(RequestHandler): """ 偷懒耍滑之 google search 主要还是用来减少访问目标站点的次数,应该是有更好的方法的 并且检查google search得到的网页是否正确 """ def __init__(self, cfg): super().__init__(cfg) cookie = Path(__file__).parent.parent.joinpath(".google-cookie") if cookie.exists(): self.cookie_jar = LWPCookieJar(cookie) # noinspection PyBroadException try: self.cookie_jar.load(ignore_discard=True, ignore_expires=True) except Exception: pass else: self.cookie_jar = None self.get_page("https://www.google.com/") def get_page(self, url): """ 加载 cookie, 获取网页,之后保存 Args: url: 搜索链接 Returns: """ response = self.session.get( url, timeout=self.timeout, proxies=self.proxy_strategy, cookies=self.cookie_jar, ) response.encoding = "utf-8" html = response.text # noinspection PyBroadException try: self.cookie_jar.save(ignore_discard=True, ignore_expires=True) except Exception: pass return html @staticmethod def filter(link): """ 获得的原始 url """ # 参考自 https://github.com/MarioVilas/googlesearch # noinspection PyBroadException try: if link.startswith("/url?"): link = parse_qs(urlparse(link, "http").query)["q"][0] url = urlparse(link, "http") if url.netloc and "google" not in url.netloc: return link except Exception: ... @staticmethod def extract(html, number): """ 从搜索页面提取标题和 url, 标题中含有番号则返回 url 估计这个用xpath提取标题很容易失效 Args: html: number: Returns: """ html = HTML(html=html) link_content = html.xpath("//a") # 直接维护列表 title_xpath = ["//h3/div/text()", "//h3/span/text()"] for content in link_content: for xpath in title_xpath: title = content.xpath(xpath, first=True) if not title: continue if re.search( "".join(filter(str.isalnum, number)), "".join(filter(str.isalnum, title)), flags=re.I, ): link = content.xpath("//@href", first=True) if link: return link def google_search(self, number, site): """ 通过加 site 指定网站,语言使用 en,我试了一下,不然搜不出来 Args: number: site: Returns: """ query = quote_plus(number + "+site:" + site) html = self.get_page( f"https://google.com/search?hl=en&q={query}&safe=off") return self.filter(self.extract(html, number))
class BMClient(): def _read_rcfile(self, rcfile, site): config = configparser.ConfigParser() config.read(rcfile) self.url = config.get(site, "url") self.username = config.get(site, "username") self.password = config.get(site, "password") self.cookiefile = os.path.expanduser(config.get(site, "cookiefile")) try: self.cachedir = os.path.expanduser(config.get(site, "cachedir")) except (configparser.NoOptionError): pass def _setup_cookies(self): # all requests should use the same cookie jar self.cookiejar = LWPCookieJar(self.cookiefile) if os.path.isfile(self.cookiefile): self.cookiejar.load(ignore_discard=True) self.cookieprocessor = HTTPCookieProcessor(self.cookiejar) self.opener = build_opener(self.cookieprocessor) install_opener(self.opener) def __init__(self, rcfile, site): self.username = None self.password = None self.cookiefile = None self.cachedir = None self._read_rcfile(rcfile, site) self._setup_cookies() def _make_request(self, args): data = json.dumps(args) headers = { 'Content-Type': 'application/x-www-form-urlencoded', } req = Request(self.url, data, headers) response = urlopen(req) jsonval = response.read() try: retval = json.loads(jsonval.decode('ascii')) return BMAPIResponse(retval) except (Exception) as e: print("could not parse return: " + jsonval) return False def login(self): args = { 'type': 'login', 'username': self.username, 'password': self.password, } retval = self._make_request(args) if retval.status == 'ok': self.cookiejar.save(ignore_discard=True) return True return False def load_player_name(self): args = { 'type': 'loadPlayerName', } return self._make_request(args) def verify_login(self): retval = self.load_player_name() if retval.status == 'ok': return True return self.login() def load_button_names(self): args = { 'type': 'loadButtonData', } return self._make_request(args) def load_player_names(self): args = { 'type': 'loadPlayerNames', } return self._make_request(args) def load_active_games(self): args = { 'type': 'loadActiveGames', } return self._make_request(args) def load_new_games(self): args = { 'type': 'loadNewGames', } return self._make_request(args) def react_to_new_game(self, gameId, action): args = {'type': 'reactToNewGame', 'gameId': gameId, 'action': action} return self._make_request(args) def load_completed_games(self): args = { 'type': 'loadCompletedGames', } return self._make_request(args) def load_game_data(self, gameId, logEntryLimit=10): args = { 'type': 'loadGameData', 'game': gameId, 'logEntryLimit': logEntryLimit, } return self._make_request(args) def create_game(self, pbutton, obutton='', player='', opponent='', description='', max_wins=3, use_prev_game=False, custom_recipe_array=None): if player == None or player == '': player = self.username if not obutton: obutton = '' player_info_array = [ [ player, pbutton, ], [ opponent, obutton, ], ] args = { 'type': 'createGame', 'playerInfoArray': player_info_array, 'maxWins': max_wins, } if use_prev_game: args['previousGameId'] = use_prev_game if description: args['description'] = description if custom_recipe_array: args['customRecipeArray'] = custom_recipe_array return self._make_request(args) def submit_turn(self, gameId, attackerIdx, defenderIdx, dieSelectStatus, attackType, roundNumber, timestamp, turboVals, chat=''): args = { 'type': 'submitTurn', 'game': gameId, 'attackerIdx': attackerIdx, 'defenderIdx': defenderIdx, 'attackType': attackType, 'roundNumber': roundNumber, 'timestamp': timestamp, 'chat': chat, 'dieSelectStatus': dieSelectStatus, } if turboVals: args['turboVals'] = turboVals return self._make_request(args) def submit_die_values(self, gameId, swingArray, optionArray, roundNumber, timestamp): args = { 'type': 'submitDieValues', 'game': gameId, 'roundNumber': roundNumber, 'timestamp': timestamp, } if swingArray: for [key, value] in sorted(swingArray.items()): args['swingValueArray'] = swingArray if optionArray: for [key, value] in sorted(optionArray.items()): args['optionValueArray'] = optionArray return self._make_request(args) def react_to_new_game(self, gameId, action): args = { 'type': 'reactToNewGame', 'gameId': gameId, 'action': action, } return self._make_request(args) def react_to_initiative(self, gameId, action, idxArray, valueArray, roundNumber, timestamp): args = { 'type': 'reactToInitiative', 'game': gameId, 'roundNumber': roundNumber, 'timestamp': timestamp, 'action': action, 'dieIdxArray': idxArray, 'dieValueArray': valueArray, } return self._make_request(args) def adjust_fire_dice(self, gameId, action, idxArray, valueArray, roundNumber, timestamp): args = { 'type': 'adjustFire', 'game': gameId, 'roundNumber': roundNumber, 'timestamp': timestamp, 'action': action, 'dieIdxArray': idxArray, 'dieValueArray': valueArray, } return self._make_request(args) def choose_reserve_dice(self, gameId, action, dieIdx=None): args = { 'type': 'reactToReserve', 'game': gameId, 'action': action, } if dieIdx is not None: args['dieIdx'] = dieIdx return self._make_request(args) def choose_auxiliary_dice(self, gameId, action, dieIdx=None): args = { 'type': 'reactToAuxiliary', 'game': gameId, 'action': action, } if dieIdx is not None: args['dieIdx'] = dieIdx return self._make_request(args)
class StreamSession(object): """ Top-level stream session interface Individual stream providers can be implemented by inheriting from this class and implementing methods for login flow, getting streams, etc. """ HEADERS = {"User-agent": USER_AGENT} SESSION_CLASS = requests.Session def __init__(self, provider_id, proxies=None, *args, **kwargs): self.provider_id = provider_id self.session = self.SESSION_CLASS() self.cookies = LWPCookieJar() if not os.path.exists(self.COOKIES_FILE): self.cookies.save(self.COOKIES_FILE) self.cookies.load(self.COOKIES_FILE, ignore_discard=True) self.session.headers = self.HEADERS self._state = AttrDict([("proxies", proxies)]) if proxies: self.proxies = proxies self._cache_responses = False @property def provider(self): return providers.get(self.provider_id) def login(self): pass @classmethod def session_type(cls): return cls.__name__.replace("StreamSession", "").lower() @classmethod def _COOKIES_FILE(cls): return os.path.join(config.settings.CONFIG_DIR, f"{cls.session_type()}.cookies") @property def COOKIES_FILE(self): return self._COOKIES_FILE() @classmethod def _SESSION_FILE(cls): return os.path.join(config.settings.CONFIG_DIR, f"{cls.session_type()}.session") @property def SESSION_FILE(self): return self._SESSION_FILE() @classmethod def new(cls, provider_id, *args, **kwargs): try: return cls.load(provider_id, **kwargs) except (FileNotFoundError, TypeError): logger.debug(f"creating new session: {args}, {kwargs}") return cls(provider_id, **kwargs) @property def cookies(self): return self.session.cookies @cookies.setter def cookies(self, value): self.session.cookies = value @classmethod def destroy(cls): if os.path.exists(cls.COOKIES_FILE): os.remove(cls.COOKIES_FILE) if os.path.exists(cls.SESSION_FILE): os.remove(cls.SESSION_FILE) @classmethod def load(cls, provider_id, **kwargs): state = yaml.load(open(cls._SESSION_FILE()), Loader=AttrDictYAMLLoader) logger.trace(f"load: {cls.__name__}, {state}") return cls(provider_id, **dict(kwargs, **state)) def save(self): logger.trace(f"load: {self.__class__.__name__}, {self._state}") with open(self.SESSION_FILE, 'w') as outfile: yaml.dump(self._state, outfile, default_flow_style=False) self.cookies.save(self.COOKIES_FILE) def get_cookie(self, name): return requests.utils.dict_from_cookiejar(self.cookies).get(name) def __getattr__(self, attr): if attr in [ "delete", "get", "head", "options", "post", "put", "patch" ]: # return getattr(self.session, attr) session_method = getattr(self.session, attr) return functools.partial(self.request, session_method) # raise AttributeError(attr) @db_session def request(self, method, url, *args, **kwargs): response = None use_cache = not self.no_cache and self._cache_responses # print(self.proxies) if use_cache: logger.debug("getting cached response for %s" % (url)) e = model.CacheEntry.get(url=url) if e: # (pickled_response, last_seen) = self.cursor.fetchone() td = datetime.now() - e.last_seen if td.seconds >= self._cache_responses: logger.debug("cache expired for %s" % (url)) else: response = pickle.loads(e.response) logger.debug("using cached response for %s" % (url)) else: logger.debug("no cached response for %s" % (url)) if not response: response = method(url, *args, **kwargs) # logger.trace(dump.dump_all(response).encode("utf-8")) if use_cache and not e: pickled_response = pickle.dumps(response) e = model.CacheEntry(url=url, response=pickled_response, last_seen=datetime.now()) return response @property def headers(self): return [] @property def proxies(self): return self._state.proxies @proxies.setter def proxies(self, value): # Override proxy environment variables if proxies are defined on session if value is None: self.session.proxies = {} else: self.session.trust_env = (len(value) == 0) self._state.proxies = value self.session.proxies.update(value) @contextmanager def cache_responses(self, duration=model.CACHE_DURATION_DEFAULT): self._cache_responses = duration try: yield finally: self._cache_responses = False def cache_responses_short(self): return self.cache_responses(model.CACHE_DURATION_SHORT) def cache_responses_medium(self): return self.cache_responses(model.CACHE_DURATION_MEDIUM) def cache_responses_long(self): return self.cache_responses(model.CACHE_DURATION_LONG)
class Opener: def __init__(self, cookie_path=None): self.opener = build_opener() self.cookiejar = LWPCookieJar() self.set_cookie_path(cookie_path) self.opener.add_handler(HTTPCookieProcessor(self.cookiejar)) self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')] self.last_request = None self.last_response = None @property def request_info(self): pass @property def response_info(self): pass def set_cookie_path(self, path): self.cookiejar.filename = path try: os.path.exists(path) and self.cookiejar.load() except: pass def set_headers(self, arg={}, **kwargs): if not arg and not kwargs: return headers = dict(self.opener.addheaders) headers.update(arg) headers.update(kwargs) self.opener.addheaders = list(headers.items()) def set_cookies(self, *args, **kwargs): for arg in args: cookie = DefaultCookiePolicy(**arg) self.cookiejar.set_cookie(cookie) kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs)) def save_cookies(self, cookie_path=None): if cookie_path or self.cookiejar.filename: self.cookiejar.save(cookie_path) def urlopen(self, url, param=None, data=None, headers={}, proxies={}, timeout=None, encoding='utf8', errors='strict'): """ 打开目标链接, 返回一个 HttpResponse对象. @url(str/Request): 目标链接. @param(str/dict/pairs tuple): query string. @data(bytes/str/dict): post data. @headers(dict): http request headers. @proxies(dict): 代理, 如:{'http': 'xx.xx.xx.xx:3128', 'https': 'xxx.xxx.xxx.xxx:8080'}. @timeout(int): http request timeout. @encoding/errors(str): url编码. """ if param: full_url = isinstance(url, Request) and url.get_full_url() or url url_parse_dict = urlparse(full_url)._asdict() query_param = url_parse_dict.get('query') + (isinstance(param, str) and param or urlencode(param, encoding, errors)) url_parse_dict['query'] = query_param full_url = urlunparse(url_parse_dict.values()) request = Request(full_url) else: request = isinstance(url, Request) and url or Request(url) if data: if isinstance(data, bytes): request.data = data elif isinstance(data, str): request.data = data.encode(encoding, errors) else: request.data = urlencode(data).encode(encoding, errors) for key, value in headers.items(): request.add_header(key, value) for proxy_type, proxy_host in proxies.items(): request.set_proxy(proxy_host, proxy_type) self.last_request = request self.last_response = self.opener.open(request, timeout=timeout) return self.last_response def clear(self): self.last_request = None self.last_response = None
print(year_results) fp.write(year_results + '\n') time.sleep(0.8) fp.close() if __name__ == "__main__": if len(sys.argv) < 3: print("******") print("Academic word relevance") print("******") print("") print( "Usage: python extract_occurrences.py '<search term>' <start date> <end date> [<include_patents> [<include_citations>]]" ) else: try: search_term = sys.argv[1] start_date = int(sys.argv[2]) end_date = int(sys.argv[3]) include_patents = bool(sys.argv[4]) if len(sys.argv) > 4 else False include_citations = bool( sys.argv[4]) if len(sys.argv) > 5 else False html = get_range(search_term, start_date, end_date, include_patents, include_citations) finally: cookies.save()
def init_basicauth(config, config_mtime): """initialize urllib2 with the credentials for Basic Authentication""" def filterhdrs(meth, ishdr, *hdrs): # this is so ugly but httplib doesn't use # a logger object or such def new_method(self, *args, **kwargs): # check if this is a recursive call (note: we do not # have to care about thread safety) is_rec_call = getattr(self, '_orig_stdout', None) is not None try: if not is_rec_call: self._orig_stdout = sys.stdout sys.stdout = StringIO() meth(self, *args, **kwargs) hdr = sys.stdout.getvalue() finally: # restore original stdout if not is_rec_call: sys.stdout = self._orig_stdout del self._orig_stdout for i in hdrs: if ishdr: hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr) else: hdr = re.sub(i, '', hdr) sys.stdout.write(hdr) new_method.__name__ = meth.__name__ return new_method if config['http_debug'] and not config['http_full_debug']: HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie', 'Authorization') HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False, 'header: Set-Cookie.*\n') if sys.version_info < (2, 6): # HTTPS proxy is not supported in old urllib2. It only leads to an error # or, at best, a warning. if 'https_proxy' in os.environ: del os.environ['https_proxy'] if 'HTTPS_PROXY' in os.environ: del os.environ['HTTPS_PROXY'] if config['http_debug']: # brute force def urllib2_debug_init(self, debuglevel=0): self._debuglevel = 1 AbstractHTTPHandler.__init__ = urllib2_debug_init cookie_file = os.path.expanduser(config['cookiejar']) global cookiejar cookiejar = LWPCookieJar(cookie_file) try: cookiejar.load(ignore_discard=True) if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime): cookiejar.clear() cookiejar.save() except IOError: try: fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600) os.close(fd) except IOError: # hmm is any good reason why we should catch the IOError? #print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file cookiejar = CookieJar()