class FileDownloader(): USER_AGENT = 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/12.04 Chromium/18.0.1025.168 Chrome/18.0.1025.168 Safari/535.19' opener_installed = False def __init__(self, cookiefile, core = None): self.cookiefile = cookiefile self.logged_in = False from socket import setdefaulttimeout setdefaulttimeout(30) self.opener_installed = False # This controls the use of the cache-headers in requests to allow/deny minified answers # as provided by some mobile operators. self.allow_minified_answers = True self.cj = LWPCookieJar(self.cookiefile) if DEBUG_HTTP: opener = build_opener(HTTPHandler(debuglevel=1), HTTPCookieProcessor(self.cj)) else: opener = build_opener(HTTPCookieProcessor(self.cj)) install_opener(opener) try: self.cj.load() logger.info("Loaded cookie file") except IOError, e: logger.info("Couldn't load cookie file") if core != None: core.connect('save-settings', self._on_save_settings)
def performTVPLogin(): cookieJar = LWPCookieJar() try: url="https://tvplayer.com/account" username=ADDON.getSetting( "tvpusername" ) if username=="": return False,cookieJar pwd=ADDON.getSetting( "tvppwd" ) lasstusername=ADDON.getSetting( "lasttvpusername" ) lasstpwd=ADDON.getSetting( "lasttvppwd" ) cookieJar=getTVPCookieJar(lasstusername!=username or lasstpwd!= pwd) mainpage = open_url(url,cj=cookieJar) if 'Login to TVPlayer' in mainpage: token = urllib.unquote(re.findall('name="token" value="(.*?)"' ,mainpage)[0]) print 'LOGIN NOW' url="https://tvplayer.com/account/login" headers={'Referer':"https://tvplayer.com/account/login",'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36','Origin':'https://tvplayer.com'} post = {'email':username,'password':pwd,'token':token} post = urllib.urlencode(post) mainpage = open_url(url,cj=cookieJar,data=post,headers=headers ) cookieJar.save (TVPCOOKIEFILE,ignore_discard=True) ADDON.setSetting( id="lasttvpusername" ,value=username) ADDON.setSetting( id="lasttvppwd" ,value=pwd) return not '>Login</a>' in mainpage,cookieJar except: traceback.print_exc(file=sys.stdout) return False,cookieJar
def __init__(self, request, username,password,context=''): """ Al instanciar la classe, es loguejarà utilitzant el nom d'usuari i password proporcionats, Si browser_login conté algo són les cookies guardades de la última sessio loguejada que s'ha fet. Recarregarem les cookies en un Browser nou, i aixi estalviarà uns segons que consumiria del login. """ self.context=context self.request=request registry = self.request.registry self.epitool=registry.getUtility(IEPIUtility) self.username = username self.password = password self.browser_login, elk = self.epitool.recoverBrowserSession(self.request, self.username,'presencia') if self.browser_login: self.br=Browser() self.br.set_handle_robots(False) cj = LWPCookieJar() self.br.set_cookiejar(cj) for co in self.browser_login: ck = Cookie(version=co['version'], name=co['name'], value=co['value'], port=co['port'], port_specified=co['port_specified'], domain=co['domain'], domain_specified=co['domain_specified'], domain_initial_dot=co['domain_initial_dot'], path=co['path'], path_specified=co['path_specified'], secure=co['secure'], expires=co['expires'], discard=co['discard'], comment=co['comment'], comment_url=co['comment_url'], rest=co['rest']) cj.set_cookie(ck) print "Logging-in into presencia via browser" else: self.br = Browser() self.br.set_handle_equiv(False) self.login(message=u"Logging-in into presencia via regular login") return
def printLatest(self, channel, irc): user = self.getConfig("News", "username") passwd = self.getConfig("News", "password") cookie = "cookie.jar" # login and parse the latest post parser = PhpugphParser() cookieJar = LWPCookieJar() opener = build_opener(HTTPCookieProcessor(cookieJar)) install_opener(opener) data = urlencode({"user": user, "passwrd": passwd}) f = urlopen("http://www.phpugph.com/talk/index.php?action=login2", data) cookieJar.save(cookie) f.close() cookieJar.load(cookie) f = urlopen("http://www.phpugph.com/talk/SSI.php?ssi_function=recentTopics") parser.feed(f.read().decode("utf-8", "replace")) parser.close() f.close() lines = [] latest = parser.latest for l in latest: g = GoogleShortener() g.shorten_and_do(l["link"], self.pretty_print, l, channel, irc)
def __init__(self, request, username, password, eid='', tid=''): """ """ self.request = request registry = request.registry self.epitool=registry.getUtility(IEPIUtility) self.initialized = True self.username = username self.password = password self.equipID=eid self.tecnicID=tid self.browser_login,self.externalLoginKey=self.epitool.recoverBrowserSession(request, self.username,'operacions') if self.browser_login: #Si tenim cookies anteriors, creem un browser nou i li passem les cookies guardades self.br=Browser() self.br.set_handle_robots(False) cj = LWPCookieJar() self.br.set_cookiejar(cj) for co in self.browser_login: ck = Cookie(version=co['version'], name=co['name'], value=co['value'], port=co['port'], port_specified=co['port_specified'], domain=co['domain'], domain_specified=co['domain_specified'], domain_initial_dot=co['domain_initial_dot'], path=co['path'], path_specified=co['path_specified'], secure=co['secure'], expires=co['expires'], discard=co['discard'], comment=co['comment'], comment_url=co['comment_url'], rest=co['rest']) cj.set_cookie(ck) print "Logging-in into operacions via browser" else: #self.br = Browser() try: self.login() except: self.initialized=False return
def getres(url, cookie_in=None, cookie_out=None, post_params=None, require_ssl=False): """ HTTPリクエスト :url: リクエストを送信するURL :cookie_in: 読み込むクッキーファイル :cookie_out: 書き込むクッキーファイル :post_params: POSTする辞書リスト :require_ssl: 接続にSSL(v3)を使用するか """ if require_ssl: #handler = urllib2_ssl.HTTPSHandler(ca_certs=os.path.dirname(__file__)+'/cert/cacert.pem') handler = urllib2_ssl.TLS1Handler() else: handler = urllib2.HTTPHandler() if cookie_in != None or cookie_out != None: cookiejar = LWPCookieJar(cookie_in or cookie_out) if cookie_in != None: cookiejar.load() cProcessor = urllib2.HTTPCookieProcessor(cookiejar) opener = urllib2.build_opener(cProcessor, handler) else: opener = urllib2.build_opener(handler) if post_params != None: res = opener.open(url, urllib.urlencode(post_params)) else: res = opener.open(url) if cookie_out != None: cookiejar.save() return res.read()
def http_req(url, data=None, headers={}, cookie=None): if cookie: cj = LWPCookieJar() if os.path.isfile(cookie): cj.load(cookie, ignore_discard=True) opener = urllib2.build_opener(NoRedirection, urllib2.HTTPCookieProcessor(cj)) urllib2.install_opener(opener) if data is not None: data = urllib.urlencode(data) req = urllib2.Request(url, data) for k, v in common.HEADERS.items(): req.add_header(k, v) if headers: for k, v in headers.items(): req.add_header(k, v) conn = urllib2.urlopen(req) if cookie: cj.save(cookie, ignore_discard=True) return conn
def send_cookie_auth(self, connection): """ Include Cookie Authentication data in a header """ cj = LWPCookieJar() cj.load(COOKIES_FILE, ignore_discard=True, ignore_expires=True) for cookie in cj: connection.putheader('Cookie', '%s=%s' % (cookie.name, cookie.value))
def save_cookies(self, cookie_storage): """Save to cookielib's CookieJar or Set-Cookie3 format text file. :param cookie_storage: file location string or CookieJar instance. """ def toPyCookieJar(QtCookieJar, PyCookieJar): for c in QtCookieJar.allCookies(): PyCookieJar.set_cookie(toPyCookie(c)) def toPyCookie(QtCookie): port = None port_specified = False secure = QtCookie.isSecure() name = str(QtCookie.name()) value = str(QtCookie.value()) v = str(QtCookie.path()) path_specified = bool(v != "") path = v if path_specified else None v = str(QtCookie.domain()) domain_specified = bool(v != "") domain = v if domain_specified: domain_initial_dot = v.startswith(".") else: domain_initial_dot = None v = long(QtCookie.expirationDate().toTime_t()) # Long type boundary on 32bit platfroms; avoid ValueError expires = 2147483647 if v > 2147483647 else v rest = {} discard = False return Cookie( 0, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, None, None, rest, ) if cookie_storage.__class__.__name__ == "str": cj = LWPCookieJar(cookie_storage) toPyCookieJar(self.cookie_jar, cj) cj.save() elif cookie_storage.__class__.__name__.endswith("CookieJar"): toPyCookieJar(self.cookie_jar, cookie_storage) else: raise ValueError("unsupported cookie_storage type.")
def initCookies(self): """ Stud.IP requires cookies for session-handling. Cookies are stored in a file to limit login requests""" cookieFile = open(cookieFilename, "r+") cookies = LWPCookieJar(cookieFile.name) if len(cookieFile.read()) > 0: cookies.load(cookieFile.name, ignore_discard=True) return cookies
class PrettyBugz: def __init__(self, args): self.quiet = args.quiet self.columns = args.columns or terminal_width() self.user = args.user self.password = args.password self.passwordcmd = args.passwordcmd self.skip_auth = args.skip_auth cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) try: self.cookiejar.load() except IOError: pass if getattr(args, 'encoding'): self.enc = args.encoding else: try: self.enc = locale.getdefaultlocale()[1] except: self.enc = 'utf-8' if not self.enc: self.enc = 'utf-8' self.log("Using %s " % args.base) self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar) def log(self, status_msg, newline = True): if not self.quiet: if newline: print ' * %s' % status_msg else: print ' * %s' % status_msg, def warn(self, warn_msg): if not self.quiet: print ' ! Warning: %s' % warn_msg def get_input(self, prompt): return raw_input(prompt) def bzcall(self, method, *args): """Attempt to call method with args. Log in if authentication is required. """ try: return method(*args) except xmlrpclib.Fault, fault: # Fault code 410 means login required if fault.faultCode == 410 and not self.skip_auth: self.login() return method(*args) raise
def _get_cookie_jar(self): if not hasattr(self, '_cookie_jar'): cj = LWPCookieJar() if not isfile(self._cookie_file): # @todo:use notifications or login? raise IOError('Snap! Who stole the cookie in the cookie jar?') cj.load(self._cookie_file) self._cookie_jar = cj return self._cookie_jar
class PrettyBugz: def __init__(self, args): self.columns = args.columns or terminal_width() self.user = args.user self.password = args.password self.passwordcmd = args.passwordcmd self.skip_auth = args.skip_auth cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) try: self.cookiejar.load() except IOError: pass self.token_file = os.path.join(os.environ['HOME'], DEFAULT_TOKEN_FILE) try: self.token = open(self.token_file).read().strip() except IOError: self.token = None if getattr(args, 'encoding'): self.enc = args.encoding else: try: self.enc = locale.getdefaultlocale()[1] except: self.enc = 'utf-8' if not self.enc: self.enc = 'utf-8' log_info("Using %s " % args.base) self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar) def get_input(self, prompt): return raw_input(prompt) def set_token(self, *args): if args and self.token: args[0]['Bugzilla_token'] = self.token return args def bzcall(self, method, *args): """Attempt to call method with args. Log in if authentication is required. """ try: return method(*self.set_token(*args)) except xmlrpclib.Fault, fault: # Fault code 410 means login required if fault.faultCode == 410 and not self.skip_auth: self.login() return method(*self.set_token(*args)) raise
def send_host(self, connection, host): """ Override send_host to add cookie info to XML-RPC requests """ self.crequest = CookieRequest('http://'+host+'/') # Read the cookie file and add the Token header if os.path.exists(self.cookie_file): cj = LWPCookieJar() cj.load(self.cookie_file) for cookie in cj: if cookie.name == 'Token': connection.putheader(cookie.name, cookie.value) xmlrpclib.Transport.send_host(self, connection, host)
def configureCookieProcessor(cookiefile="/tmp/searchmyhash.cookie"): """Set a Cookie Handler to accept cookies from the different Web sites. @param cookiefile Path of the cookie store.""" cookieHandler = LWPCookieJar() if cookieHandler is not None: if path.isfile(cookiefile): cookieHandler.load(cookiefile) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieHandler)) urllib2.install_opener(opener)
def save_cookies_to_file2(url): cookiefile = 'cookies.log' lwpcookiejar = LWPCookieJar(filename=cookiefile) opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(lwpcookiejar)) opener.open(url) lwpcookiejar.save() return lwpcookiejar
class katsomo(): URL = { 'login' : 'http://m.katsomo.fi/katsomo/login', 'programdir' : 'http://m.katsomo.fi/katsomo/programs', 'programs' : 'http://m.katsomo.fi/katsomo/?treeId=', 'videolink' : 'http://m.katsomo.fi/?progId=' } clearCache=False def __init__(self, username="", password="", cookie_file=""): self.cj = LWPCookieJar(cookie_file) if username == "": self.noLogin = True else: self.noLogin = False self.username = username self.password = password try: self.cj.revert(ignore_discard = True) except IOError: pass self.cj.set_cookie(self.makeCookie('hq','1')) self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) self.user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3' def makeCookie(self, name, value): return Cookie( version=0, name=name, value=value, port=None, port_specified=False, domain="katsomo.fi", domain_specified=True, domain_initial_dot=False, path="/", path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None ) def getPage(self, url,postvars={}, header_data = {}): req = urllib2.Request(url, urllib.urlencode(postvars), header_data) req.add_header('user-agent', self.user_agent) try: resp = self.opener.open(req).read() except HTTPError, error: raise NetworkError('HTTPError: %s' % error) except URLError, error: raise NetworkError('URLError: %s' % error)
def getTVPCookieJar(updatedUName=False): cookieJar=None print 'updatedUName',updatedUName try: cookieJar = LWPCookieJar() if not updatedUName: cookieJar.load(TVPCOOKIEFILE,ignore_discard=True) except: cookieJar=None if not cookieJar: cookieJar = LWPCookieJar() return cookieJar
def getFirefoxCookies(domain = ''): """Returns Mozilla Firefox cookies for the specified domain.""" isWindows = system().lower().startswith('win') profilesPath = unicode(expanduser(FIREFOX_PROFILES_WINDOWS if isWindows else FIREFOX_PROFILES_LINUX)) cookieFiles = (join(profilesPath, f, FIREFOX_COOKIE_FILE) for f in listdir(profilesPath)) cookieDB = sorted((f for f in cookieFiles if isfile(f)), key = getmtime)[-1] cursor = connect(cookieDB).cursor() cursor.execute(FIREFOX_COOKIES_SQL_REQUEST % domain) cookieJar = LWPCookieJar() for (host, path, isSecure, expiry, name, value) in cursor.fetchall(): cookieJar.set_cookie(Cookie(0, name, value, None, False, host, host.startswith('.'), host.startswith('.'), path, False, isSecure, expiry, not expiry, None, None, {})) return cookieJar
def login(self): if self.username == '' or self.password == '': raise Exception("Please configure your username/password and restart the application") logger.info("Checking Login status") from cookielib import LWPCookieJar cj = LWPCookieJar(self.cookiefile) if not self.opener_installed: from urllib2 import build_opener, install_opener, HTTPCookieProcessor opener = build_opener(HTTPCookieProcessor(cj)) install_opener(opener) self.opener_installed = True try: cj.load() logger.info("Loaded cookie file") except: logger.info("Couldn't load cookie file") else: logger.info("Checking if still logged in...") url = 'http://www.geocaching.com/seek/nearest.aspx' page = self.get_reader(url, login = False) for line in page: if 'You are logged in as' in line: self.logged_in = True logger.info("Seems as we're still logged in") return elif 'You are not logged in.' in line: logger.info("Nope, not logged in anymore") break logger.info("Logging in") url, values = self.login_callback(self.username, self.password) page = self.get_reader(url, values, login = False) for line in page: if 'You are logged in as' in line: break elif 'You are not logged in.' in line or 'combination does not match' in line: raise Exception("Wrong password or username!") else: logger.info("Seems as if the language is set to something other than english") raise Exception("Please go to geocaching.com and set the website language to english!") logger.info("Great success.") self.logged_in = True try: cj.save() except Exception, e: logger.info("Could not save cookies: %s" % e)
def send_cookie_auth(self, connection): """ Include Cookie Authentication data in a header """ try: cj = LWPCookieJar() cj.load(COOKIES_FILE, ignore_discard=True, ignore_expires=True) for cookie in cj: connection.putheader('Cookie', '%s=%s' % (cookie.name, cookie.value)) return True except LoadError : # mmc-cookies file is sometimes on bad format # (probably caused by interrupted twisted sessions) log.warn("mmc-cookies: invalid LWP format file, resending the credentials") return False
def initialize_cookie_jar(self): """Load cookies from disk, creating a new file if needed.""" self.cookies = LWPCookieJar(self.cfg['cookie_jar_path']) if not os.path.exists('cookiejar'): # Create a new cookies file and set our Session's cookies self.cookies.save() self.cookies.load(ignore_discard=True)
def login(username, password, cookiefile='cookies'): cj = LWPCookieJar(cookiefile) try: cj.load() except IOError: pass opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) formdata = {'sUsername': username, 'sPassword': password} data_encoded = urllib.urlencode(formdata) response = opener.open('/'.join([cost_prefix, cost_login]), data_encoded) content = response.read() # print(content) # print() cj.save(ignore_discard=True) return (cj, opener)
def _get_cookiejar(self): """Return a LWPCookieJar object loaded from our .cookies file. The same .cookies file is returned every time, located in the project root, same directory as config.yml and bot.py. If it doesn't exist, we will create the file and set it to be readable and writeable only by us. If it exists but the information inside is bogus, we'll ignore it. This is normally called by _make_site_object() (in turn called by get_site()), and the cookiejar is passed to our Site's constructor, used when it makes API queries. This way, we can easily preserve cookies between sites (e.g., for CentralAuth), making logins easier. """ if self._cookiejar: return self._cookiejar self._cookiejar = LWPCookieJar(self._cookie_file) try: self._cookiejar.load() except LoadError: pass # File contains bad data, so ignore it completely except IOError as e: if e.errno == errno.ENOENT: # "No such file or directory" # Create the file and restrict reading/writing only to the # owner, so others can't peak at our cookies: open(self._cookie_file, "w").close() chmod(self._cookie_file, stat.S_IRUSR|stat.S_IWUSR) else: raise return self._cookiejar
def setUp(self): ''' ''' # {{{ self.size = 100 self.user_agent = 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)' #self.prefix_url = 'http://www.google.com/search?q=' self.server_ip = '192.168.1.100' self.server_port = '8011' self.server_path = 'hello?q=' self.prefix_url = 'http://%s:%s/%s' % (self.server_ip, self.server_port, self.server_path) self.proxy = 'http://*****:*****@31.220.11.117:1212' self.numbers = ['09%08d' % randint(10000001, 89000000) for i in xrange(self.size)] # Cookie jar # {{{ home_folder = os.getenv('HOME') if not home_folder: home_folder = os.getenv('USERHOME') if not home_folder: home_folder = '.' # Use the current folder on error. self.cookie_jar = LWPCookieJar(os.path.join(home_folder, '.google-cookie')) try: self.cookie_jar.load() except Exception: print '> cookie_jar failed' pass # }}} # make google to cache this query? # {{{ for n in self.numbers: c = requests.get(self.prefix_url+n).content # }}} print 'benchmarking...'
def __init__(self, hostname, username, password, fields=[], groups=[]): self.hostname = hostname # requests session self.session = requests.Session() # request headers token = base64.standard_b64encode("%s:%s" % (username, password)) self.headers = { "Accept": "application/json", "Content-Type": "application/json", "Authorization": "Basic %s" % token, } # request cookies self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session") try: self.cookies.load(ignore_discard=True) except IOError: pass self.session.cookies = self.cookies # extra fields (table columns) self.fields = fields # extra groups (table columns) self.groups = groups # initialize inventory self.inventory = {'_meta': {'hostvars': { }}} return
def __init__(self, cookiefile, core = None): self.cookiefile = cookiefile self.logged_in = False from socket import setdefaulttimeout setdefaulttimeout(30) self.opener_installed = False # This controls the use of the cache-headers in requests to allow/deny minified answers # as provided by some mobile operators. self.allow_minified_answers = True self.cj = LWPCookieJar(self.cookiefile) if DEBUG_HTTP: opener = build_opener(HTTPHandler(debuglevel=1), HTTPCookieProcessor(self.cj)) else: opener = build_opener(HTTPCookieProcessor(self.cj)) install_opener(opener) try: self.cj.load() logger.info("Loaded cookie file") except IOError, e: logger.info("Couldn't load cookie file")
def __init__(self, args): self.columns = args.columns or terminal_width() self.user = args.user self.password = args.password self.passwordcmd = args.passwordcmd self.skip_auth = args.skip_auth cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) try: self.cookiejar.load() except IOError: pass if getattr(args, 'encoding'): self.enc = args.encoding else: try: self.enc = locale.getdefaultlocale()[1] except: self.enc = 'utf-8' if not self.enc: self.enc = 'utf-8' log_info("Using %s " % args.base) self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)
def download(self,url=""): if url == "": url = self.url else: url = url try: with open('cookies.txt'): pass except (OSError, IOError): cookies = LWPCookieJar('cookies.txt') cookies.save() cookies = self.scraper cookies.cookies = LWPCookieJar('cookies.txt') cookies.cookies.load() html = self.scraper.get(url).content cookies.cookies.save() return html
class zhihuCrawler(object): def __init__(self, login_url, login_info, header): self.base_url = 'http://www.zhihu.com' self.login_url = login_url self.login_info = login_info self.header = header self.cookies = LWPCookieJar('cookies.jar') try: self.cookies.revert() # print open(cookies.filename).read() except Exception,e: print 'First time to login, setting up cookies...' opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookies)) urllib2.install_opener(opener)
class Bilibili(): def __init__(self, appkey=APPKEY, appsecret=APPSECRET): self.appkey = appkey self.appsecret = appsecret self.is_login = False cookie_path = os.path.dirname(os.path.abspath(__file__)) + '/.cookie' self.cj = LWPCookieJar(cookie_path) if os.path.isfile(cookie_path): self.cj.load() if requests.utils.dict_from_cookiejar( self.cj).has_key('DedeUserID'): self.is_login = True self.mid = str( requests.utils.dict_from_cookiejar(self.cj)['DedeUserID']) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) urllib2.install_opener(opener) def get_captcha(self, path=None): utils.get_page_content('https://passport.bilibili.com/login') result = utils.get_page_content( LOGIN_CAPTCHA_URL, headers={ 'Referer': 'https://passport.bilibili.com/ajax/miniLogin/minilogin' }) if path == None: path = tempfile.gettempdir() + '/captcha.jpg' with open(path, 'wb') as f: f.write(result) return path def get_encryped_pwd(self, pwd): import rsa result = json.loads( utils.get_page_content( LOGIN_HASH_URL, headers={ 'Referer': 'https://passport.bilibili.com/ajax/miniLogin/minilogin' })) pwd = result['hash'] + pwd key = result['key'] pub_key = rsa.PublicKey.load_pkcs1_openssl_pem(key) pwd = rsa.encrypt(pwd.encode('utf-8'), pub_key) pwd = base64.b64encode(pwd) pwd = urllib.quote(pwd) return pwd def api_sign(self, params): params['appkey'] = self.appkey data = "" keys = params.keys() keys.sort() for key in keys: if data != "": data += "&" value = params[key] if type(value) == int: value = str(value) data += key + "=" + str(urllib.quote(value)) if self.appsecret == None: return data m = hashlib.md5() m.update(data + self.appsecret) return data + '&sign=' + m.hexdigest() def get_category_from_web_page(self): category_dict = {'0': {'title': u'全部', 'url': HOME_URL, 'subs': []}} node = category_dict['0'] url = node['url'] result = BeautifulSoup(utils.get_page_content(url), "html.parser").findAll('li', {'class': 'm-i'}) for item in result: if len(item['class']) != 1: continue tid = item['data-tid'] title = item.em.contents[0] url = 'http:' + item.a['href'] category_dict[tid] = {'title': title, 'url': url, 'subs': []} node['subs'].append(tid) #Fix video and movie if '11' not in category_dict['0']['subs']: category_dict['0']['subs'].append('11') if '23' not in category_dict['0']['subs']: category_dict['0']['subs'].append('23') category_dict['11'] = { 'title': u'电视剧', 'url': 'http://bangumi.bilibili.com/tv/', 'subs': [] } category_dict['23'] = { 'title': u'电影', 'url': 'http://bangumi.bilibili.com/movie/', 'subs': [] } for sub in category_dict['0']['subs']: node = category_dict[sub] url = node['url'] result = BeautifulSoup(utils.get_page_content(url), "html.parser").select('ul.n_num li') for item in result[1:]: if not item.has_attr('tid'): continue if not hasattr(item, 'a'): continue if item.has_attr('class'): continue tid = item['tid'] title = item.a.contents[0] url = HOME_URL + item.a['href'] category_dict[tid] = {'title': title, 'url': url, 'subs': []} node['subs'].append(tid) return category_dict def get_category(self, tid='0'): items = [{ tid: { 'title': '全部', 'url': CATEGORY[tid]['url'], 'subs': [] } }] for sub in CATEGORY[tid]['subs']: items.append({sub: CATEGORY[sub]}) return items def get_category_name(self, tid): return CATEGORY[str(tid)]['title'] def get_order(self): return ORDER def get_category_list(self, tid=0, order='default', days=30, page=1, pagesize=10): params = { 'tid': tid, 'order': order, 'days': days, 'page': page, 'pagesize': pagesize } url = LIST_URL.format(self.api_sign(params)) result = json.loads(utils.get_page_content(url)) results = [] for i in range(pagesize): if result['list'].has_key(str(i)): results.append(result['list'][str(i)]) else: break return results, result['pages'] def get_my_info(self): if self.is_login == False: return [] result = json.loads(utils.get_page_content(MY_INFO_URL)) return result['data'] def get_bangumi_chase(self, page=1, pagesize=10): if self.is_login == False: return [] url = BANGUMI_CHASE_URL.format(self.mid, page, pagesize) result = json.loads(utils.get_page_content(url)) return result['data']['result'], result['data']['pages'] def get_bangumi_detail(self, season_id): url = BANGUMI_SEASON_URL.format(season_id) result = utils.get_page_content(url) if result[0] != '{': start = result.find('(') + 1 end = result.find(');') result = result[start:end] result = json.loads(result) return result['result'] def get_history(self, page=1, pagesize=10): if self.is_login == False: return [] url = HISTORY_URL.format(page, pagesize) result = json.loads(utils.get_page_content(url)) if len(result['data']) >= int(pagesize): total_page = int(page) + 1 else: total_page = int(page) return result['data'], total_page def get_dynamic(self, page=1, pagesize=10): if self.is_login == False: return [] url = DYNAMIC_URL.format(pagesize, page) result = json.loads(utils.get_page_content(url)) total_page = int( (result['data']['page']['count'] + pagesize - 1) / pagesize) return result['data']['feeds'], total_page def get_attention(self, page=1, pagesize=10): if self.is_login == False: return [] url = ATTENTION_URL.format(self.mid, page, pagesize) result = json.loads(utils.get_page_content(url)) return result['data']['list'], result['data']['pages'] def get_attention_video(self, mid, tid=0, page=1, pagesize=10): if self.is_login == False: return [] url = ATTENTION_VIDEO_URL.format(mid, page, pagesize, tid) result = json.loads(utils.get_page_content(url)) return result['data'], result['data']['pages'] def get_attention_channel(self, mid): if self.is_login == False: return [] url = ATTENTION_CHANNEL_URL.format(mid) result = json.loads(utils.get_page_content(url)) return result['data']['list'] def get_attention_channel_list(self, mid, cid, page=1, pagesize=10): if self.is_login == False: return [] url = ATTENTION_CHANNEL_LIST_URL.format(mid, cid, page, pagesize) result = json.loads(utils.get_page_content(url)) return result['data']['list'], result['data']['total'] def get_fav_box(self): if self.is_login == False: return [] url = FAV_BOX_URL.format(self.mid) result = json.loads(utils.get_page_content(url)) return result['data']['list'] def get_fav(self, fav_box, page=1, pagesize=10): if self.is_login == False: return [] url = FAV_URL.format(self.mid, page, pagesize, fav_box) result = json.loads(utils.get_page_content(url)) return result['data']['vlist'], result['data']['pages'] def login(self, userid, pwd, captcha): #utils.get_page_content('http://www.bilibili.com') if self.is_login == True: return True, '' pwd = self.get_encryped_pwd(pwd) data = 'userid={0}&pwd={1}&keep=1&captcha={2}'.format( userid, pwd, captcha) result = utils.get_page_content( LOGIN_URL, data, { 'Origin': 'https://passport.bilibili.com', 'Referer': 'https://passport.bilibili.com/ajax/miniLogin/minilogin' }) if not requests.utils.dict_from_cookiejar( self.cj).has_key('DedeUserID'): return False, LOGIN_ERROR_MAP[json.loads(result)['message'] ['code']] self.cj.save() self.is_login = True self.mid = str( requests.utils.dict_from_cookiejar(self.cj)['DedeUserID']) return True, '' def logout(self): self.cj.clear() self.cj.save() self.is_login = False def get_av_list_detail(self, aid, page=1, fav=0, pagesize=10): params = {'id': aid, 'page': page} if fav != 0: params['fav'] = fav url = VIEW_URL.format(self.api_sign(params)) result = json.loads(utils.get_page_content(url)) results = [result] if (int(page) < result['pages']) and (pagesize > 1): results += self.get_av_list_detail(aid, int(page) + 1, fav, pagesize=pagesize - 1)[0] return results, result['pages'] def get_av_list(self, aid): url = AV_URL.format(aid) result = json.loads(utils.get_page_content(url)) return result def get_video_urls(self, cid): m = hashlib.md5() m.update(INTERFACE_PARAMS.format(str(cid), SECRETKEY_MINILOADER)) url = INTERFACE_URL.format(str(cid), m.hexdigest()) doc = minidom.parseString(utils.get_page_content(url)) urls = [ durl.getElementsByTagName('url')[0].firstChild.nodeValue for durl in doc.getElementsByTagName('durl') ] urls = [ url if not re.match(r'.*\.qqvideo\.tc\.qq\.com', url) else re.sub( r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', url) for url in urls ] return urls def add_history(self, aid, cid): url = ADD_HISTORY_URL.format(str(cid), str(aid)) utils.get_page_content(url)
from urllib2 import build_opener, HTTPCookieProcessor, Request, HTTPHandler from urllib import urlencode, quote from cookielib import FileCookieJar, CookieJar, LWPCookieJar from tempfile import mkstemp from contextlib import closing from subprocess import Popen, PIPE from lxml import html COOKIES_FILE = '/usr/local/etc/bandcamp.cookies' URL = 'https://bandcamp.com' CDN_COVERS = 'https://f4.bcbits.com/img' cj = LWPCookieJar() if os.path.isfile(COOKIES_FILE): cj.load(COOKIES_FILE) handler = HTTPHandler(debuglevel=0) opener = build_opener(handler, HTTPCookieProcessor(cj)) opener.addheaders = [('User-agent', 'Enter you own user agent !'), ('Accept', '*/*'), ('Accept-Encoding', 'deflate')] TMP_PATH = '' TMP_FILE_PREFIX = 'tmpS_' queue = Queue() # Do we have to download then add cover to downloaded music file ? ADD_COVER = 1
class Client: """ Web client class with automatic charset detection and decoding """ def __init__(self, info=None): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.user_agent = USER_AGENT self.info = info self.proxy_type = None self.proxy_url = None self.content = None self.status = None self.headers = dict() if get_setting("use_elementum_proxy", bool): elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum') if elementum_addon and elementum_addon.getSetting('internal_proxy_enabled') == "true": self.proxy_url = "{0}://{1}:{2}".format("http", "127.0.0.1", "65222") if info and "internal_proxy_url" in info: log.debug("Use Internal Elementum Proxy") self.proxy_url = info["internal_proxy_url"] if elementum_addon.getSetting("proxy_enabled") == "true" and get_setting("use_proxy_setting", bool): self.proxy_type = int(elementum_addon.getSetting("proxy_type")) log.debug("Use users proxy from elementum settings: {0}".format(proxy_types[self.proxy_type])) prx_host = elementum_addon.getSetting("proxy_host") prx_port = elementum_addon.getSetting("proxy_port") self.proxy_url = "{0}://{1}:{2}".format(proxy_types[self.proxy_type], prx_host, prx_port) @classmethod def _create_cookies(self, payload): return urlencode(payload) def _read_cookies(self, url=''): cookies_path = os.path.join(PATH_TEMP, 'nova') if not os.path.exists(cookies_path): try: os.makedirs(cookies_path) except Exception as e: log.debug("Error creating cookies directory: %s" % repr(e)) self._cookies_filename = os.path.join(cookies_path, urlparse(url).netloc + '_cookies.jar') if os.path.exists(self._cookies_filename): try: self._cookies.load(self._cookies_filename) except Exception as e: log.debug("Reading cookies error: %s" % repr(e)) def _save_cookies(self): try: self._cookies.save(self._cookies_filename) except Exception as e: log.debug("Saving cookies error: %s" % repr(e)) def _good_spider(self): self._counter += 1 if self._counter > 1: sleep(0.25) def cookies(self): """ Saved client cookies Returns: list: A list of saved Cookie objects """ return self._cookies def open(self, url, language='en', post_data=None, get_data=None): """ Opens a connection to a webpage and saves its HTML content in ``self.content`` Args: url (str): The URL to open language (str): The language code for the ``Content-Language`` header post_data (dict): POST data for the request get_data (dict): GET data for the request """ if not post_data: post_data = {} if get_data: url += '?' + urlencode(get_data) log.debug("Opening URL: %s" % repr(url)) result = False data = urlencode(post_data) if len(post_data) > 0 else None req = urllib2.Request(url, data) self._read_cookies(url) log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies))) handlers = [] if get_setting("use_elementum_proxy", bool) and self.proxy_url: if self.proxy_type: if self.proxy_type == 2: proxyHandler = urllib2.ProxyHandler({ 'http': self.proxy_url, 'https': self.proxy_url, }) handlers.append(proxyHandler) elif self.proxy_type == 1: import proxy.socks as socks from proxy.sockshandler import SocksiPyHandler prx_info = self.proxy_url.split(':') handlers.append(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, prx_info[1].replace("//", ''), int(prx_info[2]))) else: proxyHandler = urllib2.ProxyHandler({ 'http': self.proxy_url, 'https': self.proxy_url, }) handlers.append(proxyHandler) cookieHandler = urllib2.HTTPCookieProcessor(self._cookies) handlers.append(cookieHandler) opener = urllib2.build_opener(*handlers) req.add_header('User-Agent', self.user_agent) req.add_header('Content-Language', language) req.add_header("Accept-Encoding", "gzip") req.add_header("Origin", url) req.add_header("Referer", url) try: self._good_spider() with closing(opener.open(req)) as response: self.headers = response.headers self._save_cookies() if response.headers.get("Content-Encoding", "") == "gzip": import zlib self.content = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(response.read()) else: self.content = response.read() charset = response.headers.getparam('charset') if not charset: match = re.search("""<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""", self.content) if match: charset = match.group(1) if charset and charset.lower() == 'utf-8': charset = 'utf-8-sig' # Changing to utf-8-sig to remove BOM if found on decode from utf-8 if charset: log.debug('Decoding charset from %s for %s' % (charset, repr(url))) self.content = self.content.decode(charset, 'replace') self.status = response.getcode() result = True except urllib2.HTTPError as e: self.status = e.code log.warning("Status for %s : %s" % (repr(url), str(self.status))) except urllib2.URLError as e: self.status = repr(e.reason) log.warning("Status for %s : %s" % (repr(url), self.status)) except Exception as e: import traceback log.error("%s failed with %s:" % (repr(url), repr(e))) map(log.debug, traceback.format_exc().split("\n")) log.debug("Status for %s : %s" % (repr(url), str(self.status))) return result def login(self, url, data, fails_with): """ Login wrapper around ``open`` Args: url (str): The URL to open data (dict): POST login data fails_with (str): String that must **not** be included in the response's content Returns: bool: Whether or not login was successful """ result = False if self.open(url.encode('utf-8'), post_data=encode_dict(data)): result = True if fails_with in self.content: self.status = 'Wrong username or password' result = False return result
#!/bin/env python # -*- coding=utf-8 -*- import ssl import sys import urllib2 import random import httplib import json from cookielib import LWPCookieJar import urllib import re import getpass reload(sys) sys.setdefaultencoding('UTF8') cookiejar = LWPCookieJar() cookiesuppor = urllib2.HTTPCookieProcessor(cookiejar) opener = urllib2.build_opener(cookiesuppor, urllib2.HTTPHandler) urllib2.install_opener(opener) ssl._create_default_https_context = ssl._create_unverified_context codeimg = 'https://kyfw.12306.cn/otn/passcodeNew/getPassCodeNew?module=login&rand=sjrand&%s' % random.random( ) baner = """ ################################## 12306登录脚本,作者Mr RJL python版本:2.7,适用于linux 验证码输入方式: 输入问题对应的图片序号,1-8; 多个以','分隔.如:1,2,3 ##################################
class AlarmDotComScraper: def __init__(self, user, password, event_log_path): self.user = user self.password = password self.event_log_path = event_log_path self.cj = LWPCookieJar(COOKIEFILE) def getCurAlarmState(self): events = self.getLatestEvents() state_found = 0 # assume we'll find no state change current_state = "UNKNOWN" for event in events: #print event state = self.__getStateFromString(event[1]) if state != "": #print "Found State:",state state_found = 1 current_state = state if not state_found: current_state = self.__getCurStateFromLog() self.addEventsToLog(events) return current_state def getLatestEvents(self): try: self.cj.load(None, True, True) except IOError: pass # Ignore the open error if the cookies file doesn't exist retry = False while True: opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) response = opener.open(alarmComBaseURL + alarmComStatus) #response = opener.open("file:/home/tnorris/Desktop/Home.html") soup = BeautifulSoup(response.read()) event_tab_heading = soup.find("th", text=re.compile(".*Last 5 Events.*")) # Assume if we can't find the event table on the 1st try we need to login # on the 2nd time (retry=True) assume there's an issue if (event_tab_heading is None): if retry: raise AlarmDotComScraperError( "Unable to locate Alarm Event Table Header") retry = True self.__login() else: break event_table = event_tab_heading.parent.parent if (event_table is None): raise AlarmDotComScraperError("Error locating Events in Table", soup) date_time_tags = event_table.find_all("div") event_tags = event_table.find_all("span") if len(date_time_tags) == 0: raise AlarmDotComScraperError("Unable to locate time tags") if len(event_tags) == 0: raise AlarmDotComScraperError("Unable to locate event tags") if len(event_tags) != len(date_time_tags): raise AlarmDotComScraperError( "Mismatched length of events and times", len(event_tags), len(date_time_tags)) date_times = [ date_time_tags[i].string for i in reversed(range(len(date_time_tags))) ] events = [ event_tags[i].string for i in reversed(range(len(event_tags))) ] return zip(date_times, events) def addEventsToLog(self, events): new_events = [] try: handle = open(self.event_log_path, "r") for i in range(len( events)): # assumes events are in oldest to newest order temp_line = self.__makeLine(events[i][0], events[i][1]) found = 0 handle.seek(0) for line in handle: line = line.strip() if line == temp_line: found = 1 break if not found: new_events.append(events[i]) handle.close() except IOError: new_events = events if len(new_events) > 0: handle = open(self.event_log_path, "a") for i in range(len(new_events)): out_line = self.__makeLine(new_events[i][0], new_events[i][1]) handle.write(out_line) handle.write("\n") handle.close() def __getCurStateFromLog(self): try: current_state = "UNKNOWN" handle = open(self.event_log_path, "r") for line in handle: state = self.__getStateFromString(line) if state != "": current_state = state handle.close() except IOError: # if no log exists, there's no state to find pass return current_state def __getStateFromString(self, string): state = "" # return no state is none found if re.search(r"\bARMED\b", string, flags=re.IGNORECASE) >= 0: # ARMED? if re.search(r"\bSTAY\b", string, flags=re.IGNORECASE) >= 0: # STAY? state = "STAY" elif re.search(r"\bAWAY\b", string, flags=re.IGNORECASE) >= 0: # AWAY? state = "AWAY" else: state = "ARMED" # ARMED, but unknown type elif re.search(r"\bDISARMED\b", string, flags=re.IGNORECASE) >= 0: # DISARMED? state = "DISARMED" #print state_found,state return state def __makeLine(self, date_time, event): first_space = date_time.find(" ") date = date_time[:first_space] time = date_time[first_space + 1:].swapcase() return "[{date} {time:>11}] {event}".format(date=date, time=time, event=event) def __login(self): formdata = { "ctl00$ContentPlaceHolder_InnerCommon$loginform$txtUserName": self.user, "ctl00$ContentPlaceHolder_InnerCommon$loginform$txtPassword": self.password, "IsFromNewSite": "1", "JavaScriptTest": "1" } data_encoded = urllib.urlencode(formdata) # Actually login, response should contain status page opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) opener.open(alarmComBaseURL + alarmComLanding, data_encoded) self.cj.save(None, True, True)
def authenticate(self): if self.test: return {'status': True, 'inkdrops': mylar.INKDROPS_32P} feedinfo = [] try: with cfscrape.create_scraper() as s: s.headers = self.headers cj = LWPCookieJar( os.path.join(mylar.CONFIG.CACHE_DIR, ".32p_cookies.dat")) cj.load() s.cookies = cj if mylar.CONFIG.VERIFY_32P == 1 or mylar.CONFIG.VERIFY_32P == True: verify = True else: verify = False logger.fdebug('[32P] Verify SSL set to : %s' % verify) if not verify: #32P throws back an insecure warning because it can't validate against the CA. The below suppresses the message just for 32P instead of being displa$ from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings( InsecureRequestWarning) # post to the login form r = s.post(self.url, verify=verify, allow_redirects=True) #logger.debug(self.module + " Content session reply" + r.text) #need a way to find response code (200=OK), but returns 200 for everything even failed signons (returns a blank page) #logger.info('[32P] response: ' + str(r.content)) soup = BeautifulSoup(r.content, "html.parser") soup.prettify() if self.searchterm: logger.info( '[32P] Successfully authenticated. Initiating search for : %s' % self.searchterm) return self.search32p(s) logger.info('[32P] Successfully authenticated.') all_script = soup.find_all("script", {"src": False}) all_script2 = soup.find_all("link", {"rel": "alternate"}) authfound = False logger.info( '%s Atttempting to integrate with all of your 32P Notification feeds.' % self.module) #get inkdrop count ... #user_info = soup.find_all(attrs={"class": "stat"}) #inkdrops = user_info[0]['title'] #logger.info('INKDROPS: ' + str(inkdrops)) for al in all_script2: alurl = al['href'] if 'auth=' in alurl and 'torrents_notify' in alurl and not authfound: f1 = alurl.find('auth=') f2 = alurl.find('&', f1 + 1) auth = alurl[f1 + 5:f2] authfound = True #logger.fdebug(self.module + ' Auth:' + str(auth)) #p1 = alurl.find('passkey=') #p2 = alurl.find('&', p1 + 1) #passkey = alurl[p1 +8:p2] #logger.fdebug(self.module + ' Passkey:' + str(passkey)) if self.reauthenticate: break if 'torrents_notify' in alurl and ( 'torrents_notify_' + str(self.passkey)) not in alurl: notifyname_st = alurl.find('name=') notifyname_en = alurl.find('&', notifyname_st + 1) if notifyname_en == -1: notifyname_en = len(alurl) notifyname = alurl[notifyname_st + 5:notifyname_en] notifynumber_st = alurl.find('torrents_notify_') notifynumber_en = alurl.find('_', notifynumber_st + 17) notifynumber = alurl[notifynumber_st:notifynumber_en] logger.fdebug( '%s [NOTIFICATION: %s] Notification ID: %s' % (self.module, notifyname, notifynumber)) #generate the rss-url here feedinfo.append({ 'feed': notifynumber + '_' + str(self.passkey), 'feedname': notifyname, 'user': str(self.uid), 'auth': auth, 'passkey': str(self.passkey), 'authkey': str(self.authkey) }) except (requests.exceptions.Timeout, EnvironmentError): logger.warn( 'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.' ) return #set the keys here that will be used to download. try: mylar.CONFIG.PASSKEY_32P = str(self.passkey) mylar.AUTHKEY_32P = str(self.authkey) # probably not needed here. mylar.KEYS_32P = {} mylar.KEYS_32P = { "user": str(self.uid), "auth": auth, "passkey": str(self.passkey), "authkey": str(self.authkey) } except NameError: logger.warn( 'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.' ) return if self.reauthenticate: return else: mylar.FEEDINFO_32P = feedinfo return feedinfo
def init_cookie_jar(cookie_file=None, cookie_t=None, cookie_y=None, cookie_euconsent=None): cookie_jar = LWPCookieJar( cookie_file) if cookie_file else RequestsCookieJar() if cookie_file and os.path.exists(cookie_file): cookie_jar.load(ignore_discard=True) if args.cookie_t: cookie_jar.set_cookie(create_cookie('T', cookie_t)) if cookie_y: cookie_jar.set_cookie(create_cookie('Y', cookie_y)) if cookie_euconsent: cookie_jar.set_cookie(create_cookie('EuConsent', cookie_euconsent)) if cookie_file: cookie_jar.save(ignore_discard=True) return cookie_jar
def main(): banner() print print(C+' Diperbarui Tanggal : '+W+'11 April 2020') print tiks(W+50*'=') tiks(C+'\tTOOLS\t\t LIMIT\t\tSTATUS') tiks(W+50*'=') print tiks(C+'['+W+'1'+C+']'+W+' SPAM CALL V1' +W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94') tiks(C+50*'=') tiks(C+'['+W+'2'+C+']'+W+' SPAM CALL V2'+W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94') tiks(C+50*'=') tiks(C+'['+W+'3'+C+']'+W+' SPAM CALL V3'+W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94') tiks(C+50*'=') tiks(C+'['+W+'4'+C+']'+W+' REPORT BUG'+K+'\t\t -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94') tiks(C+50*'=') tiks(C+'['+W+'5'+C+']'+W+' SUBSCRIBE YT'+K+'\t -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94') tiks(C+50*'=') tiks(C+'['+W+'X'+C+']'+W+' EXIT / KELUAR'+K+'\t -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94') tiks(C+50*'=') try: print pilih = raw_input(W+'Pilih Tools'+C+' : '+W+'') if pilih == '1': print tiks(W+'\t\tSPAM OTP V1') tiks(C+'\t\t'+12*'=') print phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 62812xxxx '+W+') : ') jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ') if jumlah > 3: print print(W+'Jumlah Melewati Limit'+C+' ^_^') sleep(3) main() elif len(phone) < 10: print print(M+'\tNomor Tidak Valid !') sys.exit() elif '62' not in phone[0:2]: print print(C+'\tPakai 62'+W+' !') sleep(2.5) main() else: pass print for _ in range(int(jumlah)): sleep(5) data = { 'phoneNumber' : phone, 'workFlow' : 'GLOBAL_SIGNUP_LOGIN', 'otpMethod' : 'CALL' } Cookies = Cookie('.cookieslog','w') Sess = requests.Session() Sess.cookies = Cookies Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'} send = Sess.post('https://www.airbnb.co.id/api/v2/phone_one_time_passwords?currency=USD&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&locale=id', json = data) if 'error' in send.text: print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96') else: print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94') Cookies.save() elif pilih == '2': print tiks(W+'\t\tSPAM OTP V2') tiks(C+'\t\t'+12*'=') print phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 812xxxx '+W+') : ') jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ') if jumlah > 3: print print(W+'Jumlah Melewati Limit'+C+' ^_^') sleep(3) main() elif len(phone) < 10: print print(M+'\tNomor Tidak Valid !') sys.exit() elif '62' in phone[0:2] or '+62' in phone[0:2] or '08' in phone[0:2]: print print(C+'\tTanpa'+W+' +62/62/08'+C+' Langsung'+W+' 812xxxx'+C+' !') sleep(4) main() else: pass print for _ in range(int(jumlah)): sleep(5) data = { 'contactNo' : phone, 'countryCode' : 'ID', 'requestMode' : '2', 'isSignup' : 'true' } Cookies = Cookie('.cookieslog','w') Sess = requests.Session() Sess.cookies = Cookies Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'} send = Sess.post('https://bitapples.com/api/v1/customer/auth/requestotpwithoutauth?t=1586576496&lang=en', json = data, headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}) if 'Otp sent, please check your mobile.' in send.text: print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94') else: print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96') Cookies.save() elif pilih == '3': print tiks(W+'\t\tSPAM OTP V3') tiks(C+'\t\t'+12*'=') print phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 812xxxx '+W+') : ') jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ') if jumlah > 3: print print(W+'Jumlah Melewati Limit'+C+' ^_^') sleep(3) main() elif len(phone) < 10: print print(M+'\tNomor Tidak Valid !') sys.exit() elif '62' in phone[0:2] or '+62' in phone[0:2] or '08' in phone[0:2]: print print(C+'\tTanpa'+W+' +62/62/08'+C+' Langsung'+W+' 812xxxx'+C+' !') sleep(4) main() else: pass print for _ in range(int(jumlah)): sleep(5) data = { 'contactNo' : phone, 'countryCode' : 'ID', 'requestMode' : '2', 'isSignup' : 'true' } Cookies = Cookie('.cookieslog','w') Sess = requests.Session() Sess.cookies = Cookies Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'} send = Sess.post('https://citihash.com/api/v1/auth/otpRequest?lang=en', json = data, headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}) if 'Otp sent, please check your mobile.' in send.text: print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94') else: print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96') import pdb;pdb.set_trace() Cookies.save() elif pilih == '4': print os.system('xdg-open http://wa.me/6285880818385?text=Hallo%20Min.%20Saya%20Nemu%20Kesalahan%20:%0A%0A') sleep(2) print(W+'Hayooo Abis Report Apa'+C+' ^_^') sys.exit() elif pilih == '5': print os.system('xdg-open https://www.youtube.com/channel/UCzsADl8XRJeZXJ6CKZLX5KQ') sleep(2) tanya = raw_input(W+'Udah Subscribe Belum ?'+C+' ^_^'+W+' ['+C+'Y'+W+'/'+C+'n'+W+']'+C+' : '+W+'') if tanya.upper() == 'Y': print print(W+'Terima Kasih'+C+' ^_^') sys.exit() else: print print(W+'Ok Gpp Kok'+C+' ^_^') sys.exit() elif pilih == 'X' or pilih == 'x': print print(W+'Thanks, Jangan Lupa Balik Lagi'+C+' ^_^') sys.exit() else: print print(W+'Tools Tidak Di Temukan '+C+'^_^') sys.exit() except ConnectionError: print print(M+' Jaringan Tidak Ada !') sys.exit() except ValueError: print print(M+' Yang Anda Masukkan Salah !') sleep(3) main() except NameError: print print(M+' Masukkan Angka !') sleep(3) main()
def __login__(self, email, passwd, secret_answer, platform='pc', emulate=None): """Just log in.""" # TODO: split into smaller methods secret_answer_hash = EAHashingAlgorithm().EAHash(secret_answer) self.credits = 0 # create session self.r = requests.Session() # init/reset requests session object # load saved cookies/session if self.cookies_file: self.r.cookies = LWPCookieJar(self.cookies_file) try: self.r.cookies.load( ignore_discard=True ) # is it good idea to load discarded cookies after long time? except IOError: pass #self.r.cookies.save(ignore_discard=True) # create empty file for cookies if emulate == 'and': self.r.headers = headers_and.copy() # i'm android now ;-) elif emulate == 'ios': self.r.headers = headers_ios.copy() # i'm ios phone now ;-) else: self.r.headers = headers.copy() # i'm chrome browser now ;-) self.urls = urls(platform) # TODO: urls won't be loaded if we drop here #if self.r.get(self.urls['main_site']+'/fifa/api/isUserLoggedIn').json()['isLoggedIn']: # return True # no need to log in again # emulate if emulate == 'ios': sku = 'FUT15IOS' clientVersion = 11 elif emulate == 'and': sku = 'FUT15AND' clientVersion = 11 # TODO: need more info about log in procedure in game # elif emulate == 'xbox': # sku = 'FFA15XBX' # FFA14CAP ? # clientVersion = 1 # elif emulate == 'ps3': # sku = 'FFA15PS3' # FFA14KTL ? # clientVersion = 1 # elif emulate == 'pc': # sku = '' # dunno # clientVersion = 1 elif not emulate: sku = 'FUT15WEB' clientVersion = 1 else: raise FutError( 'Invalid emulate parameter. (Valid ones are and/ios).' ) # pc/ps3/xbox/ # === login self.urls['login'] = self.r.get(self.urls['fut_home']).url self.r.headers['Referer'] = self.urls['login'] # prepare headers data = { 'email': email, 'password': passwd, '_rememberMe': 'on', 'rememberMe': 'on', '_eventId': 'submit', 'facebookAuth': '' } rc = self.r.post(self.urls['login'], data=data) self.logger.debug(rc.content) if self.r.get( self.urls['main_site'] + '/fifa/api/isUserLoggedIn').json()['isLoggedIn'] is not True: raise FutError( 'Error during login process (probably invalid email or password).' ) # TODO: catch invalid data exception #self.nucleus_id = re.search('userid : "([0-9]+)"', rc.text).group(1) # we'll get it later # === lanuch futweb self.r.headers['Referer'] = self.urls['fut_home'] # prepare headers rc = self.r.get(self.urls['futweb']) self.logger.debug(rc.content) rc = rc.text # if 'EASW_ID' not in rc: # raise FutError('Error during login process (probably invalid email or password).') self.nucleus_id = re.search("var EASW_ID = '([0-9]+)';", rc).group(1) self.build_cl = re.search("var BUILD_CL = '([0-9]+)';", rc).group(1) #self.urls['fut_base'] = re.search("var BASE_FUT_URL = '(https://.+?)';", rc).group(1) #self.urls['fut_home'] = re.search("var GUEST_APP_URI = '(http://.+?)';", rc).group(1) self.urls = urls(platform, self.build_cl) # acc info self.r.headers.update({ # prepare headers 'Content-Type': 'application/json', 'Accept': 'text/json', 'Easw-Session-Data-Nucleus-Id': self.nucleus_id, 'X-UT-Embed-Error': 'true', 'X-Requested-With': 'XMLHttpRequest', 'X-UT-Route': self.urls['fut_host'], 'Referer': self.urls['futweb'], }) rc = self.r.get(self.urls['acc_info']) self.logger.debug(rc.content) rc = rc.json()['userAccountInfo']['personas'][0] self.persona_id = rc['personaId'] self.persona_name = rc['personaName'] self.clubs = [i for i in rc['userClubList']] # sort clubs by lastAccessTime (latest firts) self.clubs.sort(key=lambda i: i['lastAccessTime'], reverse=True) # authorization self.r.headers.update({ # prepare headers 'Accept': 'application/json, text/javascript', 'Origin': 'http://www.easports.com', }) data = { 'isReadOnly': False, 'sku': sku, 'clientVersion': clientVersion, #'nuc': self.nucleus_id, 'nucleusPersonaId': self.persona_id, 'nucleusPersonaDisplayName': self.persona_name, 'nucleusPersonaPlatform': platform, 'locale': 'en-GB', 'method': 'authcode', 'priorityLevel': 4, 'identification': { 'AuthCode': '' } } rc = self.r.post(self.urls['fut']['authentication'], data=json.dumps(data)) self.logger.debug(rc.content) if rc.status_code == 500: raise InternalServerError('Servers are probably temporary down.') rc = rc.json() #self.urls['fut_host'] = '{0}://{1}'.format(rc['protocol']+rc['ipPort']) if rc.get('reason') == 'multiple session': raise MultipleSession elif rc.get('reason') == 'max sessions': raise MaxSessions elif rc.get('reason') == 'doLogin: doLogin failed': raise doLoginFail elif rc.get('reason'): raise UnknownError(rc.__str__()) self.r.headers['X-UT-SID'] = self.sid = rc['sid'] # validate (secret question) self.r.headers['Accept'] = 'text/json' # prepare headers del self.r.headers['Origin'] rc = self.r.get(self.urls['fut_question']) self.logger.debug(rc.content) rc = rc.json() if rc.get('string') != 'Already answered question.': # answer question data = {'answer': secret_answer_hash} self.r.headers[ 'Content-Type'] = 'application/x-www-form-urlencoded' # requests bug? rc = self.r.post(self.urls['fut_validate'], data=data) self.logger.debug(rc.content) rc = rc.json() if rc['string'] != 'OK': # we've got error if 'Answers do not match' in rc['reason']: raise FutError( 'Error during login process (invalid secret answer).') else: raise UnknownError self.r.headers['Content-Type'] = 'application/json' self.r.headers['X-UT-PHISHING-TOKEN'] = self.token = rc['token'] # prepare headers for ut operations del self.r.headers['Easw-Session-Data-Nucleus-Id'] del self.r.headers['X-Requested-With'] del self.r.headers['X-UT-Route'] self.r.headers.update({ #'X-HTTP-Method-Override': 'GET', # __request__ method manages this 'Referer': 'https://www.easports.com/iframe/fut15/bundles/futweb/web/flash/FifaUltimateTeam.swf', 'Origin': 'https://www.easports.com', #'Content-Type': 'application/json', # already set 'Accept': 'application/json', }) # get basic user info # TODO: parse response (https://gist.github.com/oczkers/526577572c097eb8172f) self.__get__(self.urls['fut']['user']) # size of piles piles = self.pileSize() self.tradepile_size = piles['tradepile'] self.watchlist_size = piles['watchlist'] self.saveSession()
class Client: """ Web client class with automatic charset detection and decoding """ def __init__(self, info=None, request_charset='utf-8', response_charset=None): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.url = None self.user_agent = USER_AGENT self.content = None self.status = None self.token = None self.passkey = None self.info = info self.proxy_url = None self.request_charset = request_charset self.response_charset = response_charset self.needs_proxylock = False self.headers = dict() self.request_headers = None self.session = requests.session() self.session.verify = False # Enabling retrying on failed requests retries = Retry(total=2, read=2, connect=2, redirect=3, backoff_factor=0.1 # status_forcelist=[ 500, 502, 503, 504 ]) ) self.session.mount('http://', HTTPAdapter(max_retries=retries)) self.session.mount('https://', HTTPAdapter(max_retries=retries)) # self.session = cfscrape.create_scraper() # self.scraper = cfscrape.create_scraper() # self.session = self.scraper.session() global dns_public_list global dns_opennic_list dns_public_list = get_setting("public_dns_list", unicode).replace(" ", "").split(",") dns_opennic_list = get_setting("opennic_dns_list", unicode).replace(" ", "").split(",") # socket.setdefaulttimeout(60) # Parsing proxy information proxy = { 'enabled': get_setting("proxy_enabled", bool), 'use_type': get_setting("proxy_use_type", int), 'type': proxy_types[0], 'host': get_setting("proxy_host", unicode), 'port': get_setting("proxy_port", int), 'login': get_setting("proxy_login", unicode), 'password': get_setting("proxy_password", unicode), } try: proxy['type'] = proxy_types[get_setting("proxy_type", int)] except: pass if get_setting("use_public_dns", bool): connection.create_connection = patched_create_connection if get_setting("use_elementum_proxy", bool): elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum') if elementum_addon and elementum_addon.getSetting( 'internal_proxy_enabled') == "true": self.proxy_url = "{0}://{1}:{2}".format( "http", "127.0.0.1", "65222") if info and "internal_proxy_url" in info: self.proxy_url = info["internal_proxy_url"] self.session.proxies = { 'http': self.proxy_url, 'https': self.proxy_url, } elif proxy['enabled']: if proxy['use_type'] == 0 and info and "proxy_url" in info: log.debug("Setting proxy from Elementum: %s" % (info["proxy_url"])) elif proxy['use_type'] == 1: log.debug("Setting proxy with custom settings: %s" % (repr(proxy))) if proxy['login'] or proxy['password']: self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format( proxy['type'], proxy['login'], proxy['password'], proxy['host'], proxy['port']) else: self.proxy_url = "{0}://{1}:{2}".format( proxy['type'], proxy['host'], proxy['port']) if self.proxy_url: self.session.proxies = { 'http': self.proxy_url, 'https': self.proxy_url, } def _create_cookies(self, payload): return urlencode(payload) def _locate_cookies(self, url=''): cookies_path = os.path.join(PATH_TEMP, 'burst') if not os.path.exists(cookies_path): try: os.makedirs(cookies_path) except Exception as e: log.debug("Error creating cookies directory: %s" % repr(e)) # return os.path.join(cookies_path, urlparse(url).netloc + '_cookies.jar') # Do we really need to split cookies for each domain? return os.path.join(cookies_path, 'common_cookies.jar') def _read_cookies(self, url=''): self._cookies_filename = self._locate_cookies(url) if os.path.exists(self._cookies_filename): try: self._cookies.load(self._cookies_filename) except Exception as e: log.debug("Reading cookies error: %s" % repr(e)) def _save_cookies(self): self._cookies_filename = self._locate_cookies(self.url) try: self._cookies.save(self._cookies_filename) except Exception as e: log.debug("Saving cookies error: %s" % repr(e)) def _good_spider(self): self._counter += 1 if self._counter > 1: sleep(0.25) def cookies(self): """ Saved client cookies Returns: list: A list of saved Cookie objects """ return self._cookies def open(self, url, language='en', post_data=None, get_data=None, headers=None): """ Opens a connection to a webpage and saves its HTML content in ``self.content`` Args: url (str): The URL to open language (str): The language code for the ``Content-Language`` header post_data (dict): POST data for the request get_data (dict): GET data for the request """ if get_data: url += '?' + urlencode(get_data) log.debug("Opening URL: %s" % repr(url)) if self.session.proxies: log.debug("Proxies: %s" % (repr(self.session.proxies))) self._read_cookies(url) self.session.cookies = self._cookies log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies))) # Default headers for any request. Pretend like we are the usual browser. req_headers = { 'User-Agent': self.user_agent, 'Content-Language': language, 'Cache-Control': 'no-cache', 'Accept-Encoding': 'deflate, compress, gzip', 'Origin': url, 'Referer': url } # If headers passed to open() call - we overwrite headers. if headers: for key, value in headers.iteritems(): if value: req_headers[key] = value elif key.capitalize() in req_headers: del req_headers[key.capitalize()] if self.token: req_headers["Authorization"] = self.token req = None if post_data: req = requests.Request('POST', url, data=post_data, headers=req_headers) else: req = requests.Request('GET', url, headers=req_headers) prepped = self.session.prepare_request(req) self.request_headers = prepped.headers try: self._good_spider() with self.session.send(prepped) as response: self.headers = response.headers self.status = response.status_code self.url = response.url self._save_cookies() if self.response_charset: self.content = response.content.decode( self.response_charset, 'ignore') else: self.content = response.text except Exception as e: import traceback log.error("%s failed with %s:" % (repr(url), repr(e))) map(log.debug, traceback.format_exc().split("\n")) log.debug("Status for %s : %s" % (repr(url), str(self.status))) return self.status == 200 def login(self, root_url, url, data, headers, fails_with): """ Login wrapper around ``open`` Args: url (str): The URL to open data (dict): POST login data fails_with (str): String that must **not** be included in the response's content Returns: bool: Whether or not login was successful """ if not url.startswith('http'): url = root_url + url if self.open(url.encode('utf-8'), post_data=encode_dict(data, self.request_charset), headers=headers): try: if fails_with in self.content: self.status = 'Wrong username or password' return False except Exception as e: log.debug("Login failed with: %s" % e) try: if fails_with in self.content.decode('utf-8'): self.status = 'Wrong username or password' return False except: return False return True return False
REFER_URL = URL URL_AFTER_LOGIN = '******' USERNAME = '******' PASSWORD = '******' # 这个完全可以不用bs库,直接从网页源代码提取 def get_lt(str_): f = re.compile(r"name=\"lt\"\svalue=\"(.*)\"\s/>", flags=0) return f.findall(str_)[0] # cookie setting s = requests.Session() s.cookies = LWPCookieJar('cookiejar') header = { 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0', 'Referer': REFER_URL } payload = { 'username': USERNAME, 'password': PASSWORD, 'execution': 'e1s1', 'lt': None, '_eventId': 'submit' } # if cookies existed, or not expried
def __init__(self, info=None, request_charset='utf-8', response_charset=None): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.url = None self.user_agent = USER_AGENT self.content = None self.status = None self.token = None self.passkey = None self.info = info self.proxy_url = None self.request_charset = request_charset self.response_charset = response_charset self.needs_proxylock = False self.headers = dict() self.request_headers = None self.session = requests.session() self.session.verify = False # Enabling retrying on failed requests retries = Retry(total=2, read=2, connect=2, redirect=3, backoff_factor=0.1 # status_forcelist=[ 500, 502, 503, 504 ]) ) self.session.mount('http://', HTTPAdapter(max_retries=retries)) self.session.mount('https://', HTTPAdapter(max_retries=retries)) # self.session = cfscrape.create_scraper() # self.scraper = cfscrape.create_scraper() # self.session = self.scraper.session() global dns_public_list global dns_opennic_list dns_public_list = get_setting("public_dns_list", unicode).replace(" ", "").split(",") dns_opennic_list = get_setting("opennic_dns_list", unicode).replace(" ", "").split(",") # socket.setdefaulttimeout(60) # Parsing proxy information proxy = { 'enabled': get_setting("proxy_enabled", bool), 'use_type': get_setting("proxy_use_type", int), 'type': proxy_types[0], 'host': get_setting("proxy_host", unicode), 'port': get_setting("proxy_port", int), 'login': get_setting("proxy_login", unicode), 'password': get_setting("proxy_password", unicode), } try: proxy['type'] = proxy_types[get_setting("proxy_type", int)] except: pass if get_setting("use_public_dns", bool): connection.create_connection = patched_create_connection if get_setting("use_elementum_proxy", bool): elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum') if elementum_addon and elementum_addon.getSetting( 'internal_proxy_enabled') == "true": self.proxy_url = "{0}://{1}:{2}".format( "http", "127.0.0.1", "65222") if info and "internal_proxy_url" in info: self.proxy_url = info["internal_proxy_url"] self.session.proxies = { 'http': self.proxy_url, 'https': self.proxy_url, } elif proxy['enabled']: if proxy['use_type'] == 0 and info and "proxy_url" in info: log.debug("Setting proxy from Elementum: %s" % (info["proxy_url"])) elif proxy['use_type'] == 1: log.debug("Setting proxy with custom settings: %s" % (repr(proxy))) if proxy['login'] or proxy['password']: self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format( proxy['type'], proxy['login'], proxy['password'], proxy['host'], proxy['port']) else: self.proxy_url = "{0}://{1}:{2}".format( proxy['type'], proxy['host'], proxy['port']) if self.proxy_url: self.session.proxies = { 'http': self.proxy_url, 'https': self.proxy_url, }
ITUNESCONNECT_MAIN_PAGE_URL = '/WebObjects/iTunesConnect.woa' KEYRING_SERVICE_NAME = 'itc.cli' class DEVICE_TYPE: iPad = 0 iPhone = 1 iPhone5 = 2 deviceStrings = ['iPad', 'iPhone', 'iPhone 5'] temp_dir = gettempdir() default_file_format = 'images/{language}/{device_type} {index}.png' cookie_file_name = '.itc-cli-cookies.txt' cookie_file = os.path.join(temp_dir, cookie_file_name) cookie_jar = LWPCookieJar(cookie_file) class ALIASES: language_aliases = {} device_type_aliases = {} class config: options = {} def __initCookies(): if cookie_file: try: cookie_jar.load(cookie_file, ignore_discard=True)
class SitesDB(object): """ **EarwigBot: Wiki Toolset: Sites Database Manager** This class controls the :file:`sites.db` file, which stores information about all wiki sites known to the bot. Three public methods act as bridges between the bot's config files and :py:class:`~earwigbot.wiki.site.Site` objects: - :py:meth:`get_site`: returns a Site object corresponding to a site - :py:meth:`add_site`: stores a site in the database - :py:meth:`remove_site`: removes a site from the database There's usually no need to use this class directly. All public methods here are available as :py:meth:`bot.wiki.get_site`, :py:meth:`bot.wiki.add_site`, and :py:meth:`bot.wiki.remove_site`, which use a :file:`sites.db` file located in the same directory as our :file:`config.yml` file. Lower-level access can be achieved by importing the manager class (``from earwigbot.wiki import SitesDB``). """ def __init__(self, bot): """Set up the manager with an attribute for the base Bot object.""" self.config = bot.config self._logger = bot.logger.getChild("wiki") self._sites = {} # Internal site cache self._sitesdb = path.join(bot.config.root_dir, "sites.db") self._cookie_file = path.join(bot.config.root_dir, ".cookies") self._cookiejar = None excl_db = path.join(bot.config.root_dir, "exclusions.db") excl_logger = self._logger.getChild("exclusionsdb") self._exclusions_db = ExclusionsDB(self, excl_db, excl_logger) def __repr__(self): """Return the canonical string representation of the SitesDB.""" res = "SitesDB(config={0!r}, sitesdb={1!r}, cookie_file={2!r})" return res.format(self.config, self._sitesdb, self._cookie_file) def __str__(self): """Return a nice string representation of the SitesDB.""" return "<SitesDB at {0}>".format(self._sitesdb) def _get_cookiejar(self): """Return a LWPCookieJar object loaded from our .cookies file. The same .cookies file is returned every time, located in the project root, same directory as config.yml and bot.py. If it doesn't exist, we will create the file and set it to be readable and writeable only by us. If it exists but the information inside is bogus, we'll ignore it. This is normally called by _make_site_object() (in turn called by get_site()), and the cookiejar is passed to our Site's constructor, used when it makes API queries. This way, we can easily preserve cookies between sites (e.g., for CentralAuth), making logins easier. """ if self._cookiejar: return self._cookiejar self._cookiejar = LWPCookieJar(self._cookie_file) try: self._cookiejar.load() except LoadError: pass # File contains bad data, so ignore it completely except IOError as e: if e.errno == errno.ENOENT: # "No such file or directory" # Create the file and restrict reading/writing only to the # owner, so others can't peak at our cookies: open(self._cookie_file, "w").close() chmod(self._cookie_file, stat.S_IRUSR|stat.S_IWUSR) else: raise return self._cookiejar def _create_sitesdb(self): """Initialize the sitesdb file with its three necessary tables.""" script = """ CREATE TABLE sites (site_name, site_project, site_lang, site_base_url, site_article_path, site_script_path); CREATE TABLE sql_data (sql_site, sql_data_key, sql_data_value); CREATE TABLE namespaces (ns_site, ns_id, ns_name, ns_is_primary_name); """ with sqlite.connect(self._sitesdb) as conn: conn.executescript(script) def _get_site_object(self, name): """Return the site from our cache, or create it if it doesn't exist. This is essentially just a wrapper around _make_site_object that returns the same object each time a specific site is asked for. """ try: return self._sites[name] except KeyError: site = self._make_site_object(name) self._sites[name] = site return site def _load_site_from_sitesdb(self, name): """Return all information stored in the sitesdb relating to given site. The information will be returned as a tuple, containing the site's name, project, language, base URL, article path, script path, SQL connection data, and namespaces, in that order. If the site is not found in the database, SiteNotFoundError will be raised. An empty database will be created before the exception is raised if none exists. """ query1 = "SELECT * FROM sites WHERE site_name = ?" query2 = "SELECT sql_data_key, sql_data_value FROM sql_data WHERE sql_site = ?" query3 = "SELECT ns_id, ns_name, ns_is_primary_name FROM namespaces WHERE ns_site = ?" error = "Site '{0}' not found in the sitesdb.".format(name) with sqlite.connect(self._sitesdb) as conn: try: site_data = conn.execute(query1, (name,)).fetchone() except sqlite.OperationalError: self._create_sitesdb() raise SiteNotFoundError(error) if not site_data: raise SiteNotFoundError(error) sql_data = conn.execute(query2, (name,)).fetchall() ns_data = conn.execute(query3, (name,)).fetchall() name, project, lang, base_url, article_path, script_path = site_data sql = dict(sql_data) namespaces = {} for ns_id, ns_name, ns_is_primary_name in ns_data: try: if ns_is_primary_name: # "Primary" name goes first in list namespaces[ns_id].insert(0, ns_name) else: # Ordering of the aliases doesn't matter namespaces[ns_id].append(ns_name) except KeyError: namespaces[ns_id] = [ns_name] return (name, project, lang, base_url, article_path, script_path, sql, namespaces) def _make_site_object(self, name): """Return a Site object associated with the site *name* in our sitesdb. This calls _load_site_from_sitesdb(), so SiteNotFoundError will be raised if the site is not in our sitesdb. """ cookiejar = self._get_cookiejar() (name, project, lang, base_url, article_path, script_path, sql, namespaces) = self._load_site_from_sitesdb(name) config = self.config login = (config.wiki.get("username"), config.wiki.get("password")) user_agent = config.wiki.get("userAgent") use_https = config.wiki.get("useHTTPS", False) assert_edit = config.wiki.get("assert") maxlag = config.wiki.get("maxlag") wait_between_queries = config.wiki.get("waitTime", 2) logger = self._logger.getChild(name) search_config = config.wiki.get("search", OrderedDict()).copy() if user_agent: user_agent = user_agent.replace("$1", __version__) user_agent = user_agent.replace("$2", python_version()) if search_config: nltk_dir = path.join(self.config.root_dir, ".nltk") search_config["nltk_dir"] = nltk_dir search_config["exclusions_db"] = self._exclusions_db if not sql: sql = config.wiki.get("sql", OrderedDict()).copy() for key, value in sql.iteritems(): if isinstance(value, basestring) and "$1" in value: sql[key] = value.replace("$1", name) return Site(name=name, project=project, lang=lang, base_url=base_url, article_path=article_path, script_path=script_path, sql=sql, namespaces=namespaces, login=login, cookiejar=cookiejar, user_agent=user_agent, use_https=use_https, assert_edit=assert_edit, maxlag=maxlag, wait_between_queries=wait_between_queries, logger=logger, search_config=search_config) def _get_site_name_from_sitesdb(self, project, lang): """Return the name of the first site with the given project and lang. If we can't find the site with the given information, we'll also try searching for a site whose base_url contains "{lang}.{project}". There are a few sites, like the French Wikipedia, that set their project to something other than the expected "wikipedia" ("wikipédia" in this case), but we should correctly find them when doing get_site(lang="fr", project="wikipedia"). If the site is not found, return None. An empty sitesdb will be created if none exists. """ query1 = "SELECT site_name FROM sites WHERE site_project = ? and site_lang = ?" query2 = "SELECT site_name FROM sites WHERE site_base_url LIKE ?" with sqlite.connect(self._sitesdb) as conn: try: site = conn.execute(query1, (project, lang)).fetchone() if site: return site[0] else: url = "%{0}.{1}%".format(lang, project) site = conn.execute(query2, (url,)).fetchone() return site[0] if site else None except sqlite.OperationalError: self._create_sitesdb() def _add_site_to_sitesdb(self, site): """Extract relevant info from a Site object and add it to the sitesdb. Works like a reverse _load_site_from_sitesdb(); the site's project, language, base URL, article path, script path, SQL connection data, and namespaces are extracted from the site and inserted into the sites database. If the sitesdb doesn't exist, we'll create it first. """ name = site.name sites_data = (name, site.project, site.lang, site._base_url, site._article_path, site._script_path) sql_data = [(name, key, val) for key, val in site._sql_data.iteritems()] ns_data = [] for ns_id, ns_names in site._namespaces.iteritems(): ns_data.append((name, ns_id, ns_names.pop(0), True)) for ns_name in ns_names: ns_data.append((name, ns_id, ns_name, False)) with sqlite.connect(self._sitesdb) as conn: check_exists = "SELECT 1 FROM sites WHERE site_name = ?" try: exists = conn.execute(check_exists, (name,)).fetchone() except sqlite.OperationalError: self._create_sitesdb() else: if exists: conn.execute("DELETE FROM sites WHERE site_name = ?", (name,)) conn.execute("DELETE FROM sql_data WHERE sql_site = ?", (name,)) conn.execute("DELETE FROM namespaces WHERE ns_site = ?", (name,)) conn.execute("INSERT INTO sites VALUES (?, ?, ?, ?, ?, ?)", sites_data) conn.executemany("INSERT INTO sql_data VALUES (?, ?, ?)", sql_data) conn.executemany("INSERT INTO namespaces VALUES (?, ?, ?, ?)", ns_data) def _remove_site_from_sitesdb(self, name): """Remove a site by name from the sitesdb and the internal cache.""" try: del self._sites[name] except KeyError: pass with sqlite.connect(self._sitesdb) as conn: cursor = conn.execute("DELETE FROM sites WHERE site_name = ?", (name,)) if cursor.rowcount == 0: return False else: conn.execute("DELETE FROM sql_data WHERE sql_site = ?", (name,)) conn.execute("DELETE FROM namespaces WHERE ns_site = ?", (name,)) self._logger.info("Removed site '{0}'".format(name)) return True def get_site(self, name=None, project=None, lang=None): """Return a Site instance based on information from the sitesdb. With no arguments, return the default site as specified by our config file. This is ``config.wiki["defaultSite"]``. With *name* specified, return the site with that name. This is equivalent to the site's ``wikiid`` in the API, like *enwiki*. With *project* and *lang* specified, return the site whose project and language match these values. If there are multiple sites with the same values (unlikely), this is not a reliable way of loading a site. Call the function with an explicit *name* in that case. We will attempt to login to the site automatically using ``config.wiki["username"]`` and ``config.wiki["password"]`` if both are defined. Specifying a project without a lang or a lang without a project will raise :py:exc:`TypeError`. If all three args are specified, *name* will be first tried, then *project* and *lang* if *name* doesn't work. If a site cannot be found in the sitesdb, :py:exc:`~earwigbot.exceptions.SiteNotFoundError` will be raised. An empty sitesdb will be created if none is found. """ # Someone specified a project without a lang, or vice versa: if (project and not lang) or (not project and lang): e = "Keyword arguments 'lang' and 'project' must be specified together." raise TypeError(e) # No args given, so return our default site: if not name and not project and not lang: try: default = self.config.wiki["defaultSite"] except KeyError: e = "Default site is not specified in config." raise SiteNotFoundError(e) return self._get_site_object(default) # Name arg given, but don't look at others unless `name` isn't found: if name: try: return self._get_site_object(name) except SiteNotFoundError: if project and lang: name = self._get_site_name_from_sitesdb(project, lang) if name: return self._get_site_object(name) raise # If we end up here, then project and lang are the only args given: name = self._get_site_name_from_sitesdb(project, lang) if name: return self._get_site_object(name) e = "Site '{0}:{1}' not found in the sitesdb.".format(project, lang) raise SiteNotFoundError(e) def add_site(self, project=None, lang=None, base_url=None, script_path="/w", sql=None): """Add a site to the sitesdb so it can be retrieved with get_site(). If only a project and a lang are given, we'll guess the *base_url* as ``"//{lang}.{project}.org"`` (which is protocol-relative, becoming ``"https"`` if *useHTTPS* is ``True`` in config otherwise ``"http"``). If this is wrong, provide the correct *base_url* as an argument (in which case project and lang are ignored). Most wikis use ``"/w"`` as the script path (meaning the API is located at ``"{base_url}{script_path}/api.php"`` -> ``"//{lang}.{project}.org/w/api.php"``), so this is the default. If your wiki is different, provide the script_path as an argument. SQL connection settings are guessed automatically using config's template value. If this is wrong or not specified, provide a dict of kwargs as *sql* and Site will pass it to :py:func:`oursql.connect(**sql) <oursql.connect>`, allowing you to make queries with :py:meth:`site.sql_query <earwigbot.wiki.site.Site.sql_query>`. Returns ``True`` if the site was added successfully or ``False`` if the site is already in our sitesdb (this can be done purposefully to update old site info). Raises :py:exc:`~earwigbot.exception.SiteNotFoundError` if not enough information has been provided to identify the site (e.g. a *project* but not a *lang*). """ if not base_url: if not project or not lang: e = "Without a base_url, both a project and a lang must be given." raise SiteNotFoundError(e) base_url = "//{0}.{1}.org".format(lang, project) cookiejar = self._get_cookiejar() config = self.config login = (config.wiki.get("username"), config.wiki.get("password")) user_agent = config.wiki.get("userAgent") use_https = config.wiki.get("useHTTPS", True) assert_edit = config.wiki.get("assert") maxlag = config.wiki.get("maxlag") wait_between_queries = config.wiki.get("waitTime", 2) if user_agent: user_agent = user_agent.replace("$1", __version__) user_agent = user_agent.replace("$2", python_version()) # Create a Site object to log in and load the other attributes: site = Site(base_url=base_url, script_path=script_path, sql=sql, login=login, cookiejar=cookiejar, user_agent=user_agent, use_https=use_https, assert_edit=assert_edit, maxlag=maxlag, wait_between_queries=wait_between_queries) self._logger.info("Added site '{0}'".format(site.name)) self._add_site_to_sitesdb(site) return self._get_site_object(site.name) def remove_site(self, name=None, project=None, lang=None): """Remove a site from the sitesdb. Returns ``True`` if the site was removed successfully or ``False`` if the site was not in our sitesdb originally. If all three args (*name*, *project*, and *lang*) are given, we'll first try *name* and then try the latter two if *name* wasn't found in the database. Raises :py:exc:`TypeError` if a project was given but not a language, or vice versa. Will create an empty sitesdb if none was found. """ # Someone specified a project without a lang, or vice versa: if (project and not lang) or (not project and lang): e = "Keyword arguments 'lang' and 'project' must be specified together." raise TypeError(e) if name: was_removed = self._remove_site_from_sitesdb(name) if not was_removed: if project and lang: name = self._get_site_name_from_sitesdb(project, lang) if name: return self._remove_site_from_sitesdb(name) return was_removed if project and lang: name = self._get_site_name_from_sitesdb(project, lang) if name: return self._remove_site_from_sitesdb(name) return False
# -*- coding: utf-8 -*- import urllib2 import parse_captcha import urllib from urllib2 import Request, urlopen from cookielib import LWPCookieJar import os posturl = "http://ecard.sjtu.edu.cn/loginstudent.action" picurl = "http://ecard.sjtu.edu.cn/getCheckpic.action" homeurl = "http://ecard.sjtu.edu.cn/homeLogin.action" userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36" home_folder = os.getenv('HOME') cookie_jar = LWPCookieJar(os.path.join(home_folder, '.ecard-cookie')) try: cookie_jar.load() except Exception: pass def get_page(url, data=""): request = Request(url) request.add_header('User-Agent', userAgent) if data: request.add_data(data) cookie_jar.add_cookie_header(request) response = urlopen(request) cookie_jar.extract_cookies(response, request) html = response.read() response.close()
def __login__(self, email, passwd, secret_answer, platform='pc', code=None, emulate=None): """Log in. :params email: Email. :params passwd: Password. :params secret_answer: Answer for secret question. :params platform: (optional) [pc/xbox/xbox360/ps3/ps4] Platform. :params code: (optional) Security code generated in origin or send via mail/sms. :params emulate: (optional) [and/ios] Emulate mobile device. """ # TODO: split into smaller methods # TODO: check first if login is needed (https://www.easports.com/fifa/api/isUserLoggedIn) # TODO: get gamesku, url from shards !! self.emulate = emulate secret_answer_hash = EAHashingAlgorithm().EAHash(secret_answer) # create session self.r = requests.Session() # init/reset requests session object # load saved cookies/session if self.cookies_file: self.r.cookies = LWPCookieJar(self.cookies_file) try: self.r.cookies.load(ignore_discard=True) # is it good idea to load discarded cookies after long time? except IOError: pass # self.r.cookies.save(ignore_discard=True) # create empty file for cookies if emulate == 'and': self.r.headers = headers_and.copy() # i'm android now ;-) elif emulate == 'ios': self.r.headers = headers_ios.copy() # i'm ios phone now ;-) else: self.r.headers = headers.copy() # i'm chrome browser now ;-) self.urls = urls(platform) # TODO: urls won't be loaded if we drop here if platform == 'pc': game_sku = 'FFA17PCC' elif platform == 'xbox': game_sku = 'FFA17XBO' elif platform == 'xbox360': game_sku = 'FFA17XBX' elif platform == 'ps3': game_sku = 'FFA17PS3' # not tested elif platform == 'ps4': game_sku = 'FFA17PS4' platform = 'ps3' # ps4 not available? else: raise FutError(reason='Wrong platform. (Valid ones are pc/xbox/xbox360/ps3/ps4)') # if self.r.get(self.urls['main_site']+'/fifa/api/isUserLoggedIn', timeout=self.timeout).json()['isLoggedIn']: # return True # no need to log in again # emulate if emulate == 'ios': sku = 'FUT17IOS' clientVersion = 21 elif emulate == 'and': sku = 'FUT17AND' clientVersion = 21 # TODO: need more info about log in procedure in game # elif emulate == 'xbox': # sku = 'FFA16XBX' # FFA14CAP ? # clientVersion = 1 # elif emulate == 'ps3': # sku = 'FFA16PS3' # FFA14KTL ? # clientVersion = 1 # elif emulate == 'pc': # sku = '' # dunno # clientVersion = 1 elif not emulate: sku = 'FUT17WEB' clientVersion = 1 else: raise FutError(reason='Invalid emulate parameter. (Valid ones are and/ios).') # pc/ps3/xbox/ # === login self.urls['login'] = self.r.get(self.urls['fut_home'], timeout=self.timeout).url self.r.headers['Referer'] = self.urls['login'] # prepare headers data = {'email': email, 'password': passwd, '_rememberMe': 'on', 'rememberMe': 'on', '_eventId': 'submit'} rc = self.r.post(self.urls['login'], data=data, timeout=self.timeout) self.logger.debug(rc.content) ''' # pops out only on first launch if 'FIFA Ultimate Team</strong> needs to update your Account to help protect your gameplay experience.' in rc.text: # request email/sms code self.r.headers['Referer'] = rc.url # s2 rc = self.r.post(rc.url.replace('s2', 's3'), {'_eventId': 'submit'}, timeout=self.timeout).content self.r.headers['Referer'] = rc.url # s3 rc = self.r.post(rc.url, {'twofactorType': 'EMAIL', 'country': 0, 'phoneNumber': '', '_eventId': 'submit'}, timeout=self.timeout) ''' if 'We sent a security code to your' in rc.text or 'Your security code was sent to' in rc.text or 'Enter the 6-digit verification code' in rc.text: # post code # TODO: 'We sent a security code to your email' / 'We sent a security code to your ?' # TODO: pick code from codes.txt? if not code: self.saveSession() raise FutError(reason='Error during login process - code is required.') self.r.headers['Referer'] = url = rc.url # self.r.headers['Upgrade-Insecure-Requests'] = 1 # ? # self.r.headers['Origin'] = 'https://signin.ea.com' rc = self.r.post(url, {'twofactorCode': code, '_trustThisDevice': 'on', 'trustThisDevice': 'on', '_eventId': 'submit'}, timeout=self.timeout).text if 'Incorrect code entered' in rc or 'Please enter a valid security code' in rc: raise FutError(reason='Error during login process - provided code is incorrect.') self.logger.debug(rc) if 'Set Up an App Authenticator' in rc: rc = self.r.post(url.replace('s2', 's3'), {'_eventId': 'cancel', 'appDevice': 'IPHONE'}, timeout=self.timeout).text self.logger.debug(rc) self.r.headers['Referer'] = self.urls['login'] if self.r.get(self.urls['main_site'] + '/fifa/api/isUserLoggedIn', timeout=self.timeout).json()['isLoggedIn'] is not True: # TODO: parse error? raise FutError(reason='Error during login process (probably invalid email, password or code).') # TODO: catch invalid data exception # self.nucleus_id = re.search('userid : "([0-9]+)"', rc.text).group(1) # we'll get it later # === lanuch futweb self.r.headers['Referer'] = self.urls['fut_home'] # prepare headers rc = self.r.get(self.urls['futweb'], timeout=self.timeout) self.logger.debug(rc.content) rc = rc.text # if 'EASW_ID' not in rc: # raise FutError(reason='Error during login process (probably invalid email or password).') self.nucleus_id = re.search("var EASW_ID = '([0-9]+)';", rc).group(1) self.build_cl = re.search("var BUILD_CL = '([0-9]+)';", rc).group(1) # self.urls['fut_base'] = re.search("var BASE_FUT_URL = '(https://.+?)';", rc).group(1) # self.urls['fut_home'] = re.search("var GUEST_APP_URI = '(http://.+?)';", rc).group(1) self.urls = urls(platform, self.build_cl) # acc info self.r.headers.update({ # prepare headers 'Content-Type': 'application/json', 'Accept': 'text/json', 'Easw-Session-Data-Nucleus-Id': self.nucleus_id, 'X-UT-Embed-Error': 'true', 'X-Requested-With': 'XMLHttpRequest', 'X-UT-Route': self.urls['fut_host'], 'Referer': self.urls['futweb'], }) rc = self.r.get(self.urls['acc_info'], params={'_': int(time.time() * 1000)}, timeout=self.timeout) self.logger.debug(rc.content) # pick persona (first valid for given game_sku) personas = rc.json()['userAccountInfo']['personas'] for p in personas: # self.clubs = [i for i in p['userClubList']] # sort clubs by lastAccessTime (latest first but looks like ea is doing this for us(?)) # self.clubs.sort(key=lambda i: i['lastAccessTime'], reverse=True) for c in p['userClubList']: if c['skuAccessList'] and game_sku in c['skuAccessList']: self.persona_id = p['personaId'] self.persona_name = p['personaName'] break if not hasattr(self, 'persona_id') or not hasattr(self, 'persona_name'): raise FutError(reason='Error during login process (no persona found).') # authorization self.r.headers.update({ # prepare headers 'Accept': 'application/json, text/javascript', 'Origin': 'http://www.easports.com', }) data = {'isReadOnly': False, 'sku': sku, 'clientVersion': clientVersion, # 'nuc': self.nucleus_id, 'nucleusPersonaId': self.persona_id, 'nucleusPersonaDisplayName': self.persona_name, 'gameSku': game_sku, 'nucleusPersonaPlatform': platform, 'locale': 'en-GB', 'method': 'authcode', 'priorityLevel': 4, 'identification': {'AuthCode': ''}} rc = self.r.post(self.urls['fut']['authentication'], data=json.dumps(data), timeout=self.timeout) self.logger.debug(rc.content) if rc.status_code == 500: raise InternalServerError('Servers are probably temporary down.') rc = rc.json() # self.urls['fut_host'] = '{0}://{1}'.format(rc['protocol']+rc['ipPort']) if rc.get('reason') == 'multiple session': raise MultipleSession elif rc.get('reason') == 'max sessions': raise MaxSessions elif rc.get('reason') == 'doLogin: doLogin failed': raise doLoginFail elif rc.get('reason'): raise UnknownError(rc.__str__()) self.r.headers['X-UT-SID'] = self.sid = rc['sid'] # validate (secret question) self.r.headers['Accept'] = 'text/json' # prepare headers del self.r.headers['Origin'] rc = self.r.get(self.urls['fut_question'], params={'_': int(time.time() * 1000)}, timeout=self.timeout) self.logger.debug(rc.content) rc = rc.json() if rc.get('string') != 'Already answered question.': # answer question data = {'answer': secret_answer_hash} self.r.headers['Content-Type'] = 'application/x-www-form-urlencoded' # requests bug? rc = self.r.post(self.urls['fut_validate'], data=data, timeout=self.timeout) self.logger.debug(rc.content) rc = rc.json() if rc['string'] != 'OK': # we've got error if 'Answers do not match' in rc['reason']: raise FutError(reason='Error during login process (invalid secret answer).') else: raise UnknownError self.r.headers['Content-Type'] = 'application/json' self.r.headers['X-UT-PHISHING-TOKEN'] = self.token = rc['token'] # prepare headers for ut operations del self.r.headers['Easw-Session-Data-Nucleus-Id'] del self.r.headers['X-Requested-With'] del self.r.headers['X-UT-Route'] self.r.headers.update({ # 'X-HTTP-Method-Override': 'GET', # __request__ method manages this 'X-Requested-With': flash_agent, 'Referer': 'https://www.easports.com/iframe/fut16/bundles/futweb/web/flash/FifaUltimateTeam.swf', 'Origin': 'https://www.easports.com', # 'Content-Type': 'application/json', # already set 'Accept': 'application/json', }) # get basic user info # TODO: parse response (https://gist.github.com/oczkers/526577572c097eb8172f) self.__get__(self.urls['fut']['user']) # size of piles piles = self.pileSize() self.tradepile_size = piles['tradepile'] self.watchlist_size = piles['watchlist'] self.saveSession()
def searchit(self): chk_id = None #logger.info('searchterm: %s' % self.searchterm) series_search = self.searchterm['series'] #self.searchterm is a tuple containing series name, issue number, volume and publisher. if series_search.startswith('0-Day Comics Pack'): torrentid = 22247 #2018 issue_search = self.searchterm['issue'] #'21' #Wed volume_search = self.searchterm['volume'] #'2' #2nd month publisher_search = None #'2' #2nd month comic_id = None else: comic_id = self.searchterm['id'] annualize = False if 'annual' in series_search.lower(): series_search = re.sub(' annual', '', series_search.lower()).strip() annualize = True issue_search = self.searchterm['issue'] volume_search = self.searchterm['volume'] publisher_search = self.searchterm['publisher'] spl = [x for x in self.publisher_list if x in publisher_search] for x in spl: publisher_search = re.sub(x, '', publisher_search).strip() #logger.info('publisher search set to : %s' % publisher_search) # lookup the ComicID in the 32p sqlite3 table to pull the series_id to use. if comic_id: chk_id = helpers.checkthe_id(comic_id) if any([chk_id is None, mylar.CONFIG.DEEP_SEARCH_32P is True]): #generate the dynamic name of the series here so we can match it up as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(series_search) mod_series = re.sub('\|', '', as_dinfo['mod_seriesname']).strip() as_puinfo = as_d.dynamic_replace(publisher_search) pub_series = as_puinfo['mod_seriesname'] logger.fdebug('series_search: %s' % series_search) if '/' in series_search: series_search = series_search[:series_search.find('/')] if ':' in series_search: series_search = series_search[:series_search.find(':')] if ',' in series_search: series_search = series_search[:series_search.find(',')] logger.fdebug('config.search_32p: %s' % mylar.CONFIG.SEARCH_32P) if mylar.CONFIG.SEARCH_32P is False: url = 'https://walksoftly.itsaninja.party/serieslist.php' params = { 'series': re.sub('\|', '', mod_series.lower()).strip() } #series_search} logger.fdebug('search query: %s' % re.sub('\|', '', mod_series.lower()).strip()) try: t = requests.get( url, params=params, verify=True, headers={ 'USER-AGENT': mylar.USER_AGENT[:mylar.USER_AGENT.find('/') + 7] + mylar.USER_AGENT[mylar.USER_AGENT.find('(') + 1] }) except requests.exceptions.RequestException as e: logger.warn(e) return "no results" if t.status_code == '619': logger.warn('[%s] Unable to retrieve data from site.' % t.status_code) return "no results" elif t.status_code == '999': logger.warn( '[%s] No series title was provided to the search query.' % t.status_code) return "no results" try: results = t.json() except: results = t.text if len(results) == 0: logger.warn('No results found for search on 32P.') return "no results" with cfscrape.create_scraper() as s: s.headers = self.headers cj = LWPCookieJar( os.path.join(mylar.CONFIG.CACHE_DIR, ".32p_cookies.dat")) cj.load() s.cookies = cj data = [] pdata = [] pubmatch = False if series_search.startswith('0-Day Comics Pack'): data.append({"id": torrentid, "series": series_search}) else: if any([not chk_id, mylar.CONFIG.DEEP_SEARCH_32P is True]): if mylar.CONFIG.SEARCH_32P is True: url = 'https://32pag.es/torrents.php' #?action=serieslist&filter=' + series_search #&filter=F params = { 'action': 'serieslist', 'filter': series_search } time.sleep( 1) #just to make sure we don't hammer, 1s pause. t = s.get(url, params=params, verify=True, allow_redirects=True) soup = BeautifulSoup(t.content, "html.parser") results = soup.find_all("a", {"class": "object-qtip"}, {"data-type": "torrentgroup"}) for r in results: if mylar.CONFIG.SEARCH_32P is True: torrentid = r['data-id'] torrentname = r.findNext(text=True) torrentname = torrentname.strip() else: torrentid = r['id'] torrentname = r['series'] as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(torrentname) seriesresult = re.sub( '\|', '', as_dinfo['mod_seriesname']).strip() logger.fdebug( 'searchresult: %s --- %s [%s]' % (seriesresult, mod_series, publisher_search)) if seriesresult.lower() == mod_series.lower(): logger.fdebug('[MATCH] %s [%s]' % (torrentname, torrentid)) data.append({ "id": torrentid, "series": torrentname }) elif publisher_search.lower() in seriesresult.lower(): logger.fdebug('[MATCH] Publisher match.') tmp_torrentname = re.sub( publisher_search.lower(), '', seriesresult.lower()).strip() as_t = filechecker.FileChecker() as_tinfo = as_t.dynamic_replace(tmp_torrentname) if re.sub('\|', '', as_tinfo['mod_seriesname'] ).strip() == mod_series.lower(): logger.fdebug('[MATCH] %s [%s]' % (torrentname, torrentid)) pdata.append({ "id": torrentid, "series": torrentname }) pubmatch = True logger.fdebug( '%s series listed for searching that match.' % len(data)) else: logger.fdebug( 'Exact series ID already discovered previously. Setting to : %s [%s]' % (chk_id['series'], chk_id['id'])) pdata.append({ "id": chk_id['id'], "series": chk_id['series'] }) pubmatch = True if all([len(data) == 0, len(pdata) == 0]): return "no results" else: dataset = [] if len(data) > 0: dataset += data if len(pdata) > 0: dataset += pdata logger.fdebug( str(len(dataset)) + ' series match the tile being searched for on 32P...') if all([ chk_id is None, not series_search.startswith('0-Day Comics Pack') ]) and any([len(data) == 1, len(pdata) == 1]): #update the 32p_reference so we avoid doing a url lookup next time helpers.checkthe_id(comic_id, dataset) else: logger.debug( 'Unable to properly verify reference on 32P - will update the 32P reference point once the issue has been successfully matched against.' ) results32p = [] resultlist = {} for x in dataset: #for 0-day packs, issue=week#, volume=month, id=0-day year pack (ie.issue=21&volume=2 for feb.21st) payload = { "action": "groupsearch", "id": x['id'], #searchid, "issue": issue_search } #in order to match up against 0-day stuff, volume has to be none at this point #when doing other searches tho, this should be allowed to go through #if all([volume_search != 'None', volume_search is not None]): # payload.update({'volume': re.sub('v', '', volume_search).strip()}) if series_search.startswith('0-Day Comics Pack'): payload.update({"volume": volume_search}) payload = json.dumps(payload) payload = json.loads(payload) logger.fdebug('payload: %s' % payload) url = 'https://32pag.es/ajax.php' time.sleep(1) #just to make sure we don't hammer, 1s pause. try: d = s.get(url, params=payload, verify=True, allow_redirects=True) except Exception as e: logger.error('%s [%s] Could not POST URL %s' % (self.module, e, url)) try: searchResults = d.json() except: searchResults = d.text logger.debug( '%s Search Result did not return valid JSON, falling back on text: %s' % (self.module, searchResults.text)) return False if searchResults[ 'status'] == 'success' and searchResults['count'] > 0: logger.fdebug('successfully retrieved %s search results' % searchResults['count']) for a in searchResults['details']: if series_search.startswith('0-Day Comics Pack'): title = series_search else: title = self.searchterm['series'] + ' v' + a[ 'volume'] + ' #' + a['issues'] results32p.append({ 'link': a['id'], 'title': title, 'filesize': a['size'], 'issues': a['issues'], 'pack': a['pack'], 'format': a['format'], 'language': a['language'], 'seeders': a['seeders'], 'leechers': a['leechers'], 'scanner': a['scanner'], 'chkit': { 'id': x['id'], 'series': x['series'] }, 'pubdate': datetime.datetime.fromtimestamp( float(a['upload_time'])).strftime( '%a, %d %b %Y %H:%M:%S'), 'int_pubdate': float(a['upload_time']) }) else: logger.fdebug( '32P did not return any valid search results.') if len(results32p) > 0: resultlist['entries'] = sorted(results32p, key=itemgetter('pack', 'title'), reverse=False) logger.debug('%s Resultslist: %s' % (self.module, resultlist)) else: resultlist = 'no results' return resultlist
class Client: """ Web client class with automatic charset detection and decoding """ def __init__(self): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.user_agent = USER_AGENT self.clearance = None self.content = None self.status = None self.token = None self.passkey = None self.headers = dict() global dns_public_list global dns_opennic_list dns_public_list = get_setting("public_dns_list", unicode).replace(" ", "").split(",") dns_opennic_list = get_setting("opennic_dns_list", unicode).replace(" ", "").split(",") socket.setdefaulttimeout(60) def _create_cookies(self, payload): return urlencode(payload) def _read_cookies(self, url=''): cookies_path = os.path.join(PATH_TEMP, 'burst') if not os.path.exists(cookies_path): try: os.makedirs(cookies_path) except Exception as e: log.debug("Error creating cookies directory: %s" % repr(e)) self._cookies_filename = os.path.join( cookies_path, urlparse(url).netloc + '_cookies.jar') if os.path.exists(self._cookies_filename): try: self._cookies.load(self._cookies_filename) except Exception as e: log.debug("Reading cookies error: %s" % repr(e)) # Check for cf_clearance cookie # https://github.com/scakemyer/cloudhole-api if self.clearance and not any(cookie.name == 'cf_clearance' for cookie in self._cookies): c = Cookie(version=None, name='cf_clearance', value=self.clearance[13:], port=None, port_specified=False, domain='.{uri.netloc}'.format(uri=urlparse(url)), domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None, rfc2109=False) self._cookies.set_cookie(c) def _save_cookies(self): try: self._cookies.save(self._cookies_filename) except Exception as e: log.debug("Saving cookies error: %s" % repr(e)) def _good_spider(self): self._counter += 1 if self._counter > 1: sleep(0.25) def cookies(self): """ Saved client cookies Returns: list: A list of saved Cookie objects """ return self._cookies def open(self, url, language='en', post_data=None, get_data=None, headers=None, proxy_url=None, charset='utf8'): """ Opens a connection to a webpage and saves its HTML content in ``self.content`` Args: url (str): The URL to open language (str): The language code for the ``Content-Language`` header post_data (dict): POST data for the request get_data (dict): GET data for the request """ if not post_data: post_data = {} if get_data: url += '?' + urlencode(get_data) log.debug("Opening URL: %s" % repr(url)) result = False data = urlencode(post_data) if len(post_data) > 0 else None req = urllib2.Request(url, data) self._read_cookies(url) log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies))) # Parsing proxy information proxy = { 'enabled': get_setting("proxy_enabled", bool), 'use_type': get_setting("proxy_use_type", int), 'type': proxy_types[0], 'host': get_setting("proxy_host", unicode), 'port': get_setting("proxy_port", int), 'login': get_setting("proxy_login", unicode), 'password': get_setting("proxy_password", unicode), } try: proxy['type'] = proxy_types[get_setting("proxy_type", int)] except: pass handlers = [urllib2.HTTPCookieProcessor(self._cookies)] if get_setting("use_public_dns", bool): handlers.append(MyHTTPHandler) if proxy['enabled']: if proxy['use_type'] == 0 and proxy_url: log.debug("Setting proxy from Elementum: %s" % (proxy_url)) handlers.append(parse_proxy_url(proxy_url)) elif proxy['use_type'] == 1: log.debug("Setting proxy with custom settings: %s" % (repr(proxy))) handlers.append( SocksiPyHandler(proxytype=proxy['type'], proxyaddr=proxy['host'], proxyport=int(proxy['port']), username=proxy['login'], password=proxy['password'], rdns=True)) elif proxy['use_type'] == 2: try: handlers.append(antizapret.AntizapretProxyHandler()) except Exception as e: log.info("Could not create antizapret configuration: %s" % (e)) opener = urllib2.build_opener(*handlers) req.add_header('User-Agent', self.user_agent) req.add_header('Content-Language', language) req.add_header("Accept-Encoding", "gzip") req.add_header("Origin", url) req.add_header("Referer", url) if headers: for key, value in headers.iteritems(): if value: req.add_header(key, value) else: del req.headers[key.capitalize()] if self.token: req.add_header("Authorization", self.token) try: self._good_spider() with closing(opener.open(req)) as response: self.headers = response.headers self._save_cookies() if response.headers.get("Content-Encoding", "") == "gzip": import zlib self.content = zlib.decompressobj( 16 + zlib.MAX_WBITS).decompress(response.read()) else: self.content = response.read() charset = response.headers.getparam('charset') if not charset: match = re.search( """<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""", self.content) if match: charset = match.group(1) # We try to remove non-utf chars. Should we? if (charset and charset.lower() == 'utf-8') or charset is None: charset = 'utf-8-sig' # Changing to utf-8-sig to remove BOM if found on decode from utf-8 if charset: log.debug('Decoding charset from %s for %s' % (charset, repr(url))) self.content = self.content.decode(charset, 'replace') self.status = response.getcode() result = True except urllib2.HTTPError as e: self.status = e.code log.warning("Status for %s : %s" % (repr(url), str(self.status))) if e.code == 403 or e.code == 503: log.warning("CloudFlared at %s, try enabling CloudHole" % url) except urllib2.URLError as e: self.status = repr(e.reason) log.warning("Status for %s : %s" % (repr(url), self.status)) except Exception as e: import traceback log.error("%s failed with %s:" % (repr(url), repr(e))) map(log.debug, traceback.format_exc().split("\n")) log.debug("Status for %s : %s" % (repr(url), str(self.status))) return result def login(self, url, data, fails_with, charset='utf8'): """ Login wrapper around ``open`` Args: url (str): The URL to open data (dict): POST login data fails_with (str): String that must **not** be included in the response's content Returns: bool: Whether or not login was successful """ result = False if self.open(url.encode('utf-8'), post_data=encode_dict(data, charset)): result = True try: if fails_with in self.content: self.status = 'Wrong username or password' result = False except Exception as e: log.debug("Login failed with: %s" % e) try: if fails_with in self.content.decode('utf-8'): self.status = 'Wrong username or password' result = False except: result = False return result
class NowInventory(object): def __init__(self, hostname, username, password, fields=None, groups=None, selection=None, proxy=None): self.hostname = hostname # requests session self.session = requests.Session() self.auth = requests.auth.HTTPBasicAuth(username, password) # request headers self.headers = { "Accept": "application/json", "Content-Type": "application/json", } # request cookies self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session") try: self.cookies.load(ignore_discard=True) except IOError: pass self.session.cookies = self.cookies if fields is None: fields = [] if groups is None: groups = [] if selection is None: selection = [] if proxy is None: proxy = [] # extra fields (table columns) self.fields = fields # extra groups (table columns) self.groups = groups # selection order self.selection = selection # proxy settings self.proxy = proxy # initialize inventory self.inventory = {'_meta': {'hostvars': {}}} return def _put_cache(self, name, value): cache_dir = os.environ.get('SN_CACHE_DIR') if not cache_dir and config.has_option('defaults', 'cache_dir'): cache_dir = os.path.expanduser(config.get('defaults', 'cache_dir')) if cache_dir: if not os.path.exists(cache_dir): os.makedirs(cache_dir) cache_file = os.path.join(cache_dir, name) with open(cache_file, 'w') as cache: json.dump(value, cache) def _get_cache(self, name, default=None): cache_dir = os.environ.get('SN_CACHE_DIR') if not cache_dir and config.has_option('defaults', 'cache_dir'): cache_dir = config.get('defaults', 'cache_dir') if cache_dir: cache_file = os.path.join(cache_dir, name) if os.path.exists(cache_file): cache_max_age = os.environ.get('SN_CACHE_MAX_AGE') if not cache_max_age: if config.has_option('defaults', 'cache_max_age'): cache_max_age = config.getint('defaults', 'cache_max_age') else: cache_max_age = 0 cache_stat = os.stat(cache_file) if (cache_stat.st_mtime + int(cache_max_age)) >= time.time(): with open(cache_file) as cache: return json.load(cache) return default def __del__(self): self.cookies.save(ignore_discard=True) def _invoke(self, verb, path, data): cache_name = '__snow_inventory__' if not SN_TOWER: inventory = self._get_cache(cache_name, None) if inventory is not None: return inventory # build url url = "https://%s/%s" % (self.hostname, path) # perform REST operation response = self.session.get(url, auth=self.auth, headers=self.headers, proxies={ 'http': self.proxy, 'https': self.proxy }) if response.status_code != 200: print >> sys.stderr, "http error (%s): %s" % (response.status_code, response.text) if not SN_TOWER: self._put_cache(cache_name, response.json()) return response.json() def add_group(self, target, group): ''' Transform group names: 1. lower() 2. non-alphanumerical characters to '_' ''' # Ignore empty group names if group == '' or group is None: return group = group.lower() group = re.sub(r' ', '_', group) self.inventory.setdefault(group, {'hosts': []}) self.inventory[group]['hosts'].append(target) return def add_var(self, target, key, val): if target not in self.inventory['_meta']['hostvars']: self.inventory['_meta']['hostvars'][target] = {} self.inventory['_meta']['hostvars'][target]["sn_" + key] = val return def generate(self): table = 'cmdb_ci_server' # table = 'cmdb_ci_linux_server' base_fields = [ u'name', u'host_name', u'fqdn', u'ip_address', u'sys_class_name' ] base_groups = [u'sys_class_name'] options = "?sysparm_exclude_reference_link=true&sysparm_display_value=true" columns = list( set(base_fields + base_groups + self.fields + self.groups)) path = '/api/now/table/' + table + options + \ "&sysparm_fields=" + ','.join(columns) # Default, mandatory group 'sys_class_name' groups = list(set(base_groups + self.groups)) content = self._invoke('GET', path, None) for record in content['result']: ''' Ansible host target selection order: 1. ip_address 2. fqdn 3. host_name ''' target = None selection = self.selection if not selection: selection = ['host_name', 'fqdn', 'ip_address'] for k in selection: if record[k] != '': target = record[k] # Skip if no target available if target is None: continue # hostvars for k in record.keys(): self.add_var(target, k, record[k]) # groups for k in groups: self.add_group(target, record[k]) return def json(self): return json.dumps(self.inventory)
class Client: """ Web client class with automatic charset detection and decoding """ def __init__(self): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.user_agent = USER_AGENT self.clearance = None self.content = None self.status = None self.token = None self.passkey = None self.headers = dict() def _create_cookies(self, payload): return urlencode(payload) def _read_cookies(self, url=''): cookies_path = os.path.join(PATH_TEMP, 'burst') if not os.path.exists(cookies_path): try: os.makedirs(cookies_path) except Exception as e: log.debug("Error creating cookies directory: %s" % repr(e)) self._cookies_filename = os.path.join( cookies_path, urlparse(url).netloc + '_cookies.jar') if os.path.exists(self._cookies_filename): try: self._cookies.load(self._cookies_filename) except Exception as e: log.debug("Reading cookies error: %s" % repr(e)) # Check for cf_clearance cookie # https://github.com/scakemyer/cloudhole-api if self.clearance and not any(cookie.name == 'cf_clearance' for cookie in self._cookies): c = Cookie(version=None, name='cf_clearance', value=self.clearance[13:], port=None, port_specified=False, domain='.{uri.netloc}'.format(uri=urlparse(url)), domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None, rfc2109=False) self._cookies.set_cookie(c) def _save_cookies(self): try: self._cookies.save(self._cookies_filename) except Exception as e: log.debug("Saving cookies error: %s" % repr(e)) def _good_spider(self): self._counter += 1 if self._counter > 1: sleep(0.25) def cookies(self): """ Saved client cookies Returns: list: A list of saved Cookie objects """ return self._cookies def open(self, url, language='en', post_data=None, get_data=None): """ Opens a connection to a webpage and saves its HTML content in ``self.content`` Args: url (str): The URL to open language (str): The language code for the ``Content-Language`` header post_data (dict): POST data for the request get_data (dict): GET data for the request """ if not post_data: post_data = {} if get_data: url += '?' + urlencode(get_data) log.debug("Opening URL: %s" % repr(url)) result = False data = urlencode(post_data) if len(post_data) > 0 else None req = urllib2.Request(url, data) self._read_cookies(url) log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies))) opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self._cookies)) if get_setting("use_public_dns", bool): opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self._cookies), MyHTTPHandler) urllib2.install_opener(opener) req.add_header('User-Agent', self.user_agent) req.add_header('Content-Language', language) req.add_header("Accept-Encoding", "gzip") req.add_header("Origin", url) req.add_header("Referer", url) if self.token: req.add_header("Authorization", self.token) try: self._good_spider() with closing(opener.open(req)) as response: self.headers = response.headers self._save_cookies() if response.headers.get("Content-Encoding", "") == "gzip": import zlib self.content = zlib.decompressobj( 16 + zlib.MAX_WBITS).decompress(response.read()) else: self.content = response.read() charset = response.headers.getparam('charset') if not charset: match = re.search( """<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""", self.content) if match: charset = match.group(1) if charset and charset.lower() == 'utf-8': charset = 'utf-8-sig' # Changing to utf-8-sig to remove BOM if found on decode from utf-8 if charset: log.debug('Decoding charset from %s for %s' % (charset, repr(url))) self.content = self.content.decode(charset, 'replace') self.status = response.getcode() result = True except urllib2.HTTPError as e: self.status = e.code log.warning("Status for %s : %s" % (repr(url), str(self.status))) if e.code == 403 or e.code == 503: log.warning("CloudFlared at %s, try enabling CloudHole" % url) except urllib2.URLError as e: self.status = repr(e.reason) log.warning("Status for %s : %s" % (repr(url), self.status)) except Exception as e: import traceback log.error("%s failed with %s:" % (repr(url), repr(e))) map(log.debug, traceback.format_exc().split("\n")) log.debug("Status for %s : %s" % (repr(url), str(self.status))) return result def login(self, url, data, fails_with): """ Login wrapper around ``open`` Args: url (str): The URL to open data (dict): POST login data fails_with (str): String that must **not** be included in the response's content Returns: bool: Whether or not login was successful """ result = False if self.open(url.encode('utf-8'), post_data=encode_dict(data)): result = True if fails_with in self.content: self.status = 'Wrong username or password' result = False return result
class Browser: _cookies = None cookies = LWPCookieJar() content = None status = None headers = "" def __init__(self): pass @classmethod def create_cookies(cls, payload): cls._cookies = urlencode(payload) # to open any web page @classmethod def open(cls, url='', language='en', post_data=None, get_data=None): if post_data is None: post_data = {} if get_data is not None: url += '?' + urlencode(get_data) print(url) result = True if len(post_data) > 0: cls.create_cookies(post_data) if cls._cookies is not None: req = urllib2.Request(url, cls._cookies) cls._cookies = None else: req = urllib2.Request(url) req.add_header('User-Agent', USER_AGENT) req.add_header('Content-Language', language) req.add_header("Accept-Encoding", "gzip") opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cls.cookies)) # open cookie jar try: response = opener.open(req) # send cookies and open url cls.headers = response.headers # borrow from provider.py Steeve if response.headers.get("Content-Encoding", "") == "gzip": import zlib cls.content = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(response.read()) else: cls.content = response.read() response.close() cls.status = 200 except urllib2.HTTPError as e: cls.status = e.code result = False # if e.code == 503: # # trying to open with antibots tool # import cfscrape # scraper = cfscrape.create_scraper() # returns a CloudflareScraper instance # cls.content = scraper.get(url).content # cls.status = 200 # result = True except urllib2.URLError as e: cls.status = e.reason result = False log.debug("Status: " + str(cls.status)) log.debug(cls.content) return result # alternative when it is problem with https @classmethod def open2(cls, url=''): import httplib word = url.split("://") pos = word[1].find("/") conn = httplib.HTTPConnection(re.search[:pos]) conn.request("GET", re.search[pos:]) r1 = conn.getresponse() cls.status = str(r1.status) + " " + r1.reason cls.content = r1.read() if r1.status == 200: return True else: return False # used for sites with login @classmethod def login(cls, url, payload, word): result = False cls.create_cookies(payload) if cls.open(url): result = True data = cls.content if word in data: cls.status = 'Wrong Username or Password' result = False return result def json(self): return loads(self.content)
class Browser: """ Mini Web Browser with cookies handle """ _counter = 0 _cookies_filename = '' _cookies = LWPCookieJar() user_agent = USER_AGENT clearance = None content = None raw_content = None status = None headers = dict() def __init__(self): pass @classmethod def _create_cookies(cls, payload): return urlencode(payload) @classmethod def _read_cookies(cls, url=''): cls._cookies_filename = path.join( PATH_TEMP, urlparse(url).netloc + '_cookies.jar') if path.exists(cls._cookies_filename): try: cls._cookies.load(cls._cookies_filename) except Exception as e: logger.debug("Reading cookies error: %s" % repr(e)) # Check for cf_clearance cookie provided by scakemyer # https://github.com/scakemyer/cloudhole-api if cls.clearance and not any(cookie.name == 'cf_clearance' for cookie in cls._cookies): t = str(int(time()) + 604800) c = Cookie(None, 'cf_clearance', cls.clearance[13:], None, False, '.{uri.netloc}'.format(uri=urlparse(url)), True, True, '/', True, False, t, False, None, None, None, False) cls._cookies.set_cookie(c) @classmethod def _save_cookies(cls): try: cls._cookies.save(cls._cookies_filename) except Exception as e: logger.debug("Saving cookies error: %s" % repr(e)) @classmethod def _good_spider(cls): """ Delay of 0.5 seconds to to call too many requests per second. Some pages start to block """ cls._counter += 1 if cls._counter > 1: sleep(0.5) # good spider @classmethod def cookies(cls): """ Cookies :return: LWPCookieJar format. """ return cls._cookies @classmethod def open(cls, url='', language='en', post_data=None, get_data=None, use_cache=True, charset='utf-8'): """ Open a web page and returns its contents :param charset: :param use_cache: if it uses the information stored in the cache :type use_cache: bool :param url: url address web page :type url: str or unicode :param language: language encoding web page :type language: str :param post_data: parameters for POST request :type post_data: dict :param get_data: parameters for GET request :type get_data: dict :return: True if the web page was opened successfully. False, otherwise. """ if len(url) == 0: cls.status = 404 cls.raw_content = '' logger.debug('Empty url') return False # Check the cache cache_file = quote_plus(repr(url)) + '.cache' if use_cache: cache = Storage.open(cache_file, ttl=15) if 'content' in cache: cls.status = 200 cls.content = cache['content'] cls.raw_content = cache['raw_content'] cls.headers = cache['headers'] logger.debug('Using cache for %s' % url) cache.close() logger.debug("Status: " + str(cls.status)) logger.debug(repr(cls.content)) return True # Creating request if post_data is None: post_data = {} if get_data is not None: url += '?' + urlencode(get_data) logger.debug(url) result = True cls.status = 200 data = urlencode(post_data) if len(post_data) > 0 else None req = urllib2.Request(url, data) # Cookies and cloudhole info cls._read_cookies(url) logger.debug("Cookies: %s" % repr(cls._cookies)) # open cookie jar opener = urllib2.build_opener(urllib2.HTTPCookieProcessor( cls._cookies)) # Headers req.add_header('User-Agent', cls.user_agent) req.add_header('Content-Language', language) req.add_header("Accept-Encoding", "gzip") try: cls._good_spider() # send cookies and open url with closing(opener.open(req)) as response: cls.headers = response.headers cls._save_cookies() # borrow from provider.py Steeve if response.headers.get('Content-Encoding', '') == 'gzip': import zlib cls.raw_content = zlib.decompressobj( 16 + zlib.MAX_WBITS).decompress(response.read()) else: cls.raw_content = response.read() logger.debug("Raw Content:") logger.debug(repr(cls.raw_content)) if 'charset' in cls.raw_content: match = re.search('charset=(\S+)"', cls.raw_content, re.IGNORECASE) if match: charset = match.group(1) logger.debug('charset=' + charset) except urllib2.HTTPError as e: result = False cls.status = e.code logger.warning("Status: " + str(cls.status)) if e.code == 403: logger.warning("CloudFlared at %s" % url) except urllib2.URLError as e: result = False cls.status = e.reason logger.warning("Status: " + str(cls.status)) except Exception as e: result = False logger.error("Error in the browser: %s" % repr(e)) if result: # Normalization try: cls.content = cls.raw_content.decode(charset) except UnicodeDecodeError: cls.content = normalize_string(cls.raw_content, charset=charset) except LookupError: cls.content = normalize_string(cls.raw_content) cls.content = cls.content.replace('<![CDATA[', '').replace(']]', '') cls.content = HTMLParser().unescape(cls.content) # Save in the cache if use_cache: cache = Storage.open(cache_file, ttl=15) cache['content'] = cls.content cache['raw_content'] = cls.raw_content cache['headers'] = cls.headers cache.close() # status logger.debug("Status: " + str(cls.status)) logger.debug(repr(cls.content)) return result @classmethod def login(cls, url='', payload=None, word=''): """ Login to web site :param url: url address from web site :type url: str :param payload: parameters for the login request :type payload: dict :param word: message from the web site when the login fails :type word: str :return: True if the login was successful. False, otherwise. """ result = False if cls.open(url, post_data=payload): result = True data = cls.raw_content if word in data: cls.status = 'Wrong Username or Password' result = False return result
favdatei = os.path.join(seventv_favorites, masterNEW) baseURL = "https://www.7tv.de" xbmcplugin.setContent(pluginhandle, 'tvshows') if not xbmcvfs.exists(seventv_favorites): xbmcvfs.mkdirs(seventv_favorites) if xbmcvfs.exists(source): xbmcvfs.copy(source, favdatei) if xbmcvfs.exists(temp) and os.path.isdir(temp): shutil.rmtree(temp, ignore_errors=True) xbmcvfs.mkdirs(temp) cookie = os.path.join(temp, 'cookie.lwp') cj = LWPCookieJar() if xbmcvfs.exists(cookie): cj.load(cookie, ignore_discard=True, ignore_expires=True) def py2_enc(s, encoding='utf-8'): if PY2 and isinstance(s, unicode): s = s.encode(encoding) return s def py2_uni(s, encoding='utf-8'): if PY2 and isinstance(s, str): s = unicode(s, encoding) return s def py3_dec(d, encoding='utf-8'):
class MsgAutoSender(object): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', "Accept-Language": "en-US,en;q=0.5", #"Cookie": "; ".join(map(lambda x: "=".join(x), data.items())) } messages = ["你好,你的眼睛很好看哦[调皮],可以交个朋友吗[微笑]?", "你的眼睛很漂亮哦,可以和你交个朋友吗[微笑]?", "很喜欢你的眼睛,可以和你交个朋友吗[微笑]?", "你的眼睛很好看嘛,可以认识一下交个朋友吗[微笑]?"] data = { 'txtLoginEMail': "", 'txtLoginPwd': "", 'chkRememberMe': "", 'codeId': "", 'codeValue': '', 'event':'3', 'spmp':'4.20.53.225.685', '_': "%d"%(time.time()*1000) } # 预登陆url url1 = 'http://my.baihe.com/Getinterlogin/gotoLogin?jsonCallBack=jQuery18308807729283968166_%d&'%(time.time()*1000) # 登陆成功后跳转到主页 url2 = "http://u.baihe.com/?userid=&time=%d"%(time.time()*1000) # 用来获取一些默认的搜索条件(百合会根据你个人信息筛选出一些基本符合你要求的人群) url3 = "http://search.baihe.com/mystruts/nextSolrSearch.action?jsoncallback=jQuery183042376943520885857_1479472584212&callType=next&pageId=%%s&ord=1&_=%d"%(time.time()*1000) # 用来搜索默认条件下的妹纸 url4 = "http://search.baihe.com/solrAdvanceSearch.action" # 向妹纸发送消息 url5 = "http://msg.baihe.com/owner/api/sendMessage?jsonCallBack=jQuery18304671662130587029_1479300393335&toUserID=%%s&content=%%s&type=1&pathID=01.00.10402&_=%d&"%(time.time()*1000) # 登陆过频繁的话,会要求输验证,此url用来获取验证码图片 url6 = "http://my.baihe.com/Getinterlogin/getVerifyPic?jsonCallBack=?&tmpId=%s" # 用来检查登陆次数,来判定是否需要输验证码了 url7 = "http://my.baihe.com/Getinterlogin/getAccountTimes?jsonCallBack=jQuery183013238800936369732_1479556200186&userAccount=18353130797&_=1479556585384" # 用来验证验证码是否正确 url8 = "http://my.baihe.com/Getinterlogin/checkVerifyPic?jsonCallBack=jQuery183010981413646438898_1480223919788&tmpId=%%s&checkcode=%%s&_=%d"%(time.time()*1000) # access_token生成 acc_token = Cookie(version=0, name="accessToken", value='BH%d%d'%(time.time()*1000,math.floor(random.random()*1000000)), domain=".baihe.com", path="/", port=None, port_specified=False, domain_specified=True, domain_initial_dot=False, path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={}, rfc2109=False) def __init__(self): self.page = 1 self.order = 1 self.product_ids = set() self.error_count = 0 self.lock = RLock() self.alive = True self.cookie = LWPCookieJar() try: self.have_send_list = set(open("have_send_list.txt").read().strip(",").split(",")) except IOError: self.have_send_list = set() self.logger = logging.getLogger("send_msg") self.logger.setLevel(logging.DEBUG) self.logger.addHandler(logging.StreamHandler(sys.stdout)) def get_account_times(self, opener): resp = opener.open(self.url7) buf = resp.read() data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk") self.logger.debug("Check whether need input captcha or not. ") if data["data"]["showCode"]: return self.get_captcha(opener) else: return "", "" def get_captcha(self, opener): tmpId = "%d.%s"%(time.time()*1000, str(round(random.random(), 4))[2:]) resp = opener.open(self.url6%tmpId) img = Image.open( StringIO(resp.read())) img.show() return raw_input("Please input captcha recognization: "), tmpId def send_captcha(self, opener, captcha, tmpId): self.logger.debug("Send captcha. ") url = self.url8 % (tmpId, captcha) req = Request(url=url, headers=self.headers) resp = opener.open(req) data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk") if data["data"] == 1: return tmpId, captcha def login(self, opener): url = self.url1 + urlencode(self.data) req = Request(url=url, headers=self.headers) resp = opener.open(req) data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk") self.logger.debug("Login jsonp response state:%s"%data["state"]) if data["state"] == 0: return "Wrong account or password. " req = Request(url=self.url2, headers=self.headers) resp = opener.open(req) self.logger.debug("Login redirect response code:%s"%resp.code) def get_auth_cookies(self, opener): while True: self.enter_password() captcha, tmpId = self.get_account_times(opener) if tmpId: while not self.send_captcha(opener, captcha, tmpId): captcha, tmpId = self.get_account_times(opener) self.data["codeValue"] = captcha self.data["codeId"] = tmpId result = self.login(opener) if result: self.logger.info(result) else: break def get_search_cookies(self, opener): req = Request(url=self.url4, headers=self.headers) resp = opener.open(req) self.logger.debug("Finish get default search cookies, response code:%s" % resp.code) def search(self, opener): conti = True while conti: while True: try: id = self.product_ids.pop() except KeyError: break self.send_msg(opener, id) self.order += 1 req = Request(url=self.url3 % self.page, headers=self.headers) self.logger.debug("Start to find girls in page NO.%s. "%self.page) resp = opener.open(req) self.logger.debug("Search response code:%s" % resp.code) buf = resp.read() data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk") if data["result"]: self.product_ids = set([i.split(":")[0] for i in data["result"].split(",")]) self.page += 1 elif self.page > 100: return "finished" else: raise SendMessageError("You need relogin. ") def send_msg(self, opener, id): if id not in self.have_send_list: msg = random.choice(self.messages) d = quote(msg) url = self.url5 % (id, d) req = Request(url=url, headers=self.headers) resp = opener.open(req) buf = resp.read() recv = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk") code = recv["code"] self.logger.info("Send %s to No.%s girl whose id is %s, status code is %s" % (msg.decode("utf-8"), self.order, id, code)) if code in [200, u"-603", u"-804", u"-602", u"-611"]: self.logger.info("Normal message is %s" % (recv.get("msg") or u"empty").encode("gbk")) if self.error_count > 0: self.error_count -= 1 self.have_send_list.add(id) else: self.logger.info("Error message is %s"%(recv.get("msg") or u"empty").encode("gbk")) self.error_count += 1 if code in [u"-701", u"-803"]: self.alive = False self.logger.error(u"坑爹的百合每天每个账号只允许给100个人发消息。。") sys.exit(0) if self.error_count > 3: raise SendMessageError("code: %s error: %s" % (code.encode("gbk"), (recv.get("msg") or u"empty").encode("gbk"))) time.sleep(1) else: self.logger.info("The No.%s girl whose id is %s has been sent, don't molesting her any more. "%(self.order, id)) def pwd_input(self, msg=''): if msg != '': sys.stdout.write(msg) chars = [] while True: newChar = getch() if newChar in '\3\r\n': # 如果是换行,Ctrl+C,则输入结束 print '' if newChar in '\3': # 如果是Ctrl+C,则将输入清空,返回空字符串 chars = [] break elif newChar == '\b' or ord(newChar) == 127: # 如果是退格,则删除末尾一位 if chars: del chars[-1] sys.stdout.write('\b \b') # 左移一位,用空格抹掉星号,再退格 else: chars.append(newChar) sys.stdout.write('*') # 显示为星号 return ''.join(chars) def enter_password(self): account = raw_input("Please input your baihe account number: ") self.data["txtLoginEMail"] = account self.data["txtLoginPwd"] = self.pwd_input("Please input your baihe account password: "******"Please input what you want to send, input empty to break. ") if not msg: break else: try: msg = msg.decode("gbk").encode("utf-8") except UnicodeDecodeError: pass self.messages.append(msg) def start(self): self.enter_msg() self.cookie.set_cookie(self.acc_token) have_load = False try: if os.path.exists(("baihe.cookie")): self.cookie.load("baihe.cookie", True, True) have_load = True opener = build_opener(HTTPCookieProcessor(self.cookie)) if not have_load: self.get_auth_cookies(opener) self.get_search_cookies(opener) # 有时意外不正常关闭cookie和send_list无法保存,所以启动一个进程来做这件事。 Thread(target=self.saveing).start() while True: try: if self.search(opener) == "finished": self.logger.info("No more girls to send. ") break except Exception, e: time.sleep(1) self.logger.error(e) self.get_auth_cookies(opener) self.get_search_cookies(opener) except KeyboardInterrupt: self.logger.info("Closing...") self.alive = False finally: self.save() self.alive = False def saveing(self): while self.alive: self.save() time.sleep(2) def save(self): self.lock.acquire() open("have_send_list.txt", "w").write(",".join(self.have_send_list)) self.cookie.save("baihe.cookie", True, True) self.lock.release()
def __init__(self, user, password, event_log_path): self.user = user self.password = password self.event_log_path = event_log_path self.cj = LWPCookieJar(COOKIEFILE)
'Accept-Encoding': 'gzip,deflate,sdch', 'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4', 'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded', 'Host': 'music.163.com', 'Referer': 'http://music.163.com/search/', "User-Agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" "(KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36") } cookies = {'appver': '1.5.2'} session = requests.Session() session.cookies = LWPCookieJar('cookies') session.cookies.load() session.cookies.save() def phone_login(username, password): action = 'https://music.163.com/weapi/login/cellphone' text = {'phone': username, 'password': password, 'rememberLogin': '******'} data = encrypted_request(text) connect = session.post(action, data=data, headers=header, timeout=10) connect.encoding = 'UTF-8' return connect.content
#!/usr/bin/env python # By: Volker Strobel from bs4 import BeautifulSoup import urllib from urllib2 import Request, build_opener, HTTPCookieProcessor from cookielib import LWPCookieJar import re import time import sys cookies = LWPCookieJar('./cookies') try: cookies.load() except IOError: pass def get_num_results(search_term, start_date, end_date): """ Helper method, sends HTTP request and returns response payload. """ # Open website and read html user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36' query_params = { 'q' : search_term, 'as_ylo' : start_date, 'as_yhi' : end_date} url = "https://scholar.google.com/scholar?as_vis=1&hl=en&as_sdt=1,5&" + urllib.urlencode(query_params) opener = build_opener(HTTPCookieProcessor(cookies)) request = Request(url=url, headers={'User-Agent': user_agent}) handler = opener.open(request) html = handler.read() # Create soup for parsing HTML and extracting the relevant information
def login(self, username=None, password=None, is_beta=False): if (not username or not password): if not self.credentials: raise CredentialsError() else: credentials = self.credentials else: credentials = { 'email': username, 'password': password, 'beta': is_beta } if self.credentials == credentials and self.state == Session.STATE_OK: return # already logged in if not self.credentials or self.credentials['email'] != credentials[ 'email'] or self.credentials['beta'] != credentials['beta']: # changed account self.close() self.session = requests.Session() self.session.headers[ 'User-Agent'] = 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Mobile/11D257' cookiefile = join( config.app_dir, 'cookies-%s.txt' % hashlib.md5(credentials['email']).hexdigest()) if not isfile(cookiefile) and isfile( join(config.app_dir, 'cookies.txt')): os.rename(join(config.app_dir, 'cookies.txt'), cookiefile) # migration from <= 2.25 self.session.cookies = LWPCookieJar(cookiefile) try: self.session.cookies.load() except IOError: pass self.server = credentials['beta'] and SERVER_BETA or SERVER_LIVE self.credentials = credentials self.state = Session.STATE_INIT try: r = self.session.post(self.server + URL_LOGIN, data=self.credentials, timeout=timeout) except: if __debug__: print_exc() raise ServerError() if r.status_code != requests.codes.ok or 'server error' in r.text: self.dump(r) raise ServerError() elif r.url == self.server + URL_LOGIN: # would have redirected away if success self.dump(r) if 'purchase' in r.text.lower(): raise SKUError() else: raise CredentialsError() elif r.url == self.server + URL_CONFIRM: # redirected to verification page self.state = Session.STATE_AUTH raise VerificationRequired() else: self.state = Session.STATE_OK return r.status_code
def __init__(self, hostname, username, password, table=None, fields=None, groups=None, selection=None, filter_results=None, proxy=None): self.hostname = hostname # requests session self.session = requests.Session() self.auth = requests.auth.HTTPBasicAuth(username, password) # request headers self.headers = { "Accept": "application/json", "Content-Type": "application/json", } # request cookies self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session") try: self.cookies.load(ignore_discard=True) except IOError: pass self.session.cookies = self.cookies if table is None: table = 'cmdb_ci_server' if fields is None: fields = [] if groups is None: groups = [] if selection is None: selection = [] if filter_results is None: filter_results = '' if proxy is None: proxy = [] # table self.table = table # extra fields (table columns) self.fields = fields # extra groups (table columns) self.groups = groups # selection order self.selection = selection # filter results (sysparm_query encoded query string) self.filter_results = filter_results # proxy settings self.proxy = proxy # initialize inventory self.inventory = {'_meta': {'hostvars': {}}} return
import requests import string import sys import zlib from requests.packages import chardet from StringIO import StringIO import gzip from cookielib import LWPCookieJar import os s = requests.Session() s.cookies = LWPCookieJar("cookiejar") def login(): url = 'http://jiaowu.swjtu.edu.cn/servlet/GetRandomNumberToJPEG' hraders = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36', 'Accept-Language': 'zh-CN,zh;q=0.8', 'Accept': 'image/webp,image/*,*/*;q=0.8' } response = s.get(url, headers=hraders, stream=True) with open('demo.jpg', 'wb') as fd: for chunk in response.iter_content(128): fd.write(chunk) url = 'http://jiaowu.swjtu.edu.cn/servlet/UserLoginSQLAction' hraders = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded',