class FileDownloader():
    USER_AGENT = 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/12.04 Chromium/18.0.1025.168 Chrome/18.0.1025.168 Safari/535.19'
    opener_installed = False

    def __init__(self, cookiefile, core = None):
        self.cookiefile = cookiefile
        self.logged_in = False
        from socket import setdefaulttimeout
        setdefaulttimeout(30)
        self.opener_installed = False
        
        # This controls the use of the cache-headers in requests to allow/deny minified answers
        # as provided by some mobile operators.
        self.allow_minified_answers = True
        
        
        self.cj = LWPCookieJar(self.cookiefile)

        
        if DEBUG_HTTP:
            opener = build_opener(HTTPHandler(debuglevel=1), HTTPCookieProcessor(self.cj))
        else:
            opener = build_opener(HTTPCookieProcessor(self.cj))
        install_opener(opener)

        try:
            self.cj.load()
            logger.info("Loaded cookie file")
        except IOError, e:
            logger.info("Couldn't load cookie file")
            
        if core != None:
            core.connect('save-settings', self._on_save_settings)
def performTVPLogin():
    cookieJar = LWPCookieJar()
    try:
        
        url="https://tvplayer.com/account"
        username=ADDON.getSetting( "tvpusername" ) 
        if username=="": return False,cookieJar
        pwd=ADDON.getSetting( "tvppwd" ) 
        lasstusername=ADDON.getSetting( "lasttvpusername" )
        lasstpwd=ADDON.getSetting( "lasttvppwd" )         
        cookieJar=getTVPCookieJar(lasstusername!=username or lasstpwd!= pwd)
        mainpage = open_url(url,cj=cookieJar)
        

        if 'Login to TVPlayer' in mainpage:
            token   = urllib.unquote(re.findall('name="token" value="(.*?)"' ,mainpage)[0])
            print 'LOGIN NOW'
            url="https://tvplayer.com/account/login"
            headers={'Referer':"https://tvplayer.com/account/login",'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36','Origin':'https://tvplayer.com'}            
            post = {'email':username,'password':pwd,'token':token}
            post = urllib.urlencode(post)
            mainpage = open_url(url,cj=cookieJar,data=post,headers=headers )
            cookieJar.save (TVPCOOKIEFILE,ignore_discard=True)
            ADDON.setSetting( id="lasttvpusername" ,value=username)
            ADDON.setSetting( id="lasttvppwd" ,value=pwd)
        
        return not '>Login</a>' in mainpage,cookieJar
    except: 
            traceback.print_exc(file=sys.stdout)
    return False,cookieJar
Beispiel #3
0
    def __init__(self, request, username,password,context=''):
        """
        Al instanciar la classe, es loguejarà utilitzant el nom d'usuari i password proporcionats,
        Si browser_login conté algo són les cookies guardades de la última sessio loguejada que s'ha fet.
        Recarregarem les cookies en un Browser nou, i aixi estalviarà uns segons que consumiria del login.
        """
        self.context=context
        self.request=request
        registry = self.request.registry
        self.epitool=registry.getUtility(IEPIUtility)

        self.username = username
        self.password = password
        self.browser_login, elk = self.epitool.recoverBrowserSession(self.request, self.username,'presencia')
        if self.browser_login:
          self.br=Browser()
          self.br.set_handle_robots(False)
          cj = LWPCookieJar()
          self.br.set_cookiejar(cj)
          for co in self.browser_login:
              ck = Cookie(version=co['version'], name=co['name'], value=co['value'], port=co['port'], port_specified=co['port_specified'], domain=co['domain'], domain_specified=co['domain_specified'], domain_initial_dot=co['domain_initial_dot'], path=co['path'], path_specified=co['path_specified'], secure=co['secure'], expires=co['expires'], discard=co['discard'], comment=co['comment'], comment_url=co['comment_url'], rest=co['rest'])
              cj.set_cookie(ck)
          print "Logging-in into presencia via browser"
        else:
          self.br = Browser()
          self.br.set_handle_equiv(False)
          self.login(message=u"Logging-in into presencia via regular login")
          return
Beispiel #4
0
    def printLatest(self, channel, irc):
        user = self.getConfig("News", "username")
        passwd = self.getConfig("News", "password")
        cookie = "cookie.jar"

        # login and parse the latest post
        parser = PhpugphParser()

        cookieJar = LWPCookieJar()
        opener = build_opener(HTTPCookieProcessor(cookieJar))
        install_opener(opener)
        data = urlencode({"user": user, "passwrd": passwd})

        f = urlopen("http://www.phpugph.com/talk/index.php?action=login2", data)
        cookieJar.save(cookie)
        f.close()

        cookieJar.load(cookie)
        f = urlopen("http://www.phpugph.com/talk/SSI.php?ssi_function=recentTopics")
        parser.feed(f.read().decode("utf-8", "replace"))
        parser.close()
        f.close()

        lines = []
        latest = parser.latest
        for l in latest:
            g = GoogleShortener()
            g.shorten_and_do(l["link"], self.pretty_print, l, channel, irc)
Beispiel #5
0
    def __init__(self, request, username, password, eid='', tid=''):
        """
        """
        self.request = request
        registry = request.registry
        self.epitool=registry.getUtility(IEPIUtility)

        self.initialized = True
        self.username = username
        self.password = password
        self.equipID=eid
        self.tecnicID=tid
        self.browser_login,self.externalLoginKey=self.epitool.recoverBrowserSession(request, self.username,'operacions')
        if self.browser_login:
          #Si tenim cookies anteriors, creem un browser nou i li passem les cookies guardades
          self.br=Browser()
          self.br.set_handle_robots(False)
          cj = LWPCookieJar()
          self.br.set_cookiejar(cj)
          for co in self.browser_login:
              ck = Cookie(version=co['version'], name=co['name'], value=co['value'], port=co['port'], port_specified=co['port_specified'], domain=co['domain'], domain_specified=co['domain_specified'], domain_initial_dot=co['domain_initial_dot'], path=co['path'], path_specified=co['path_specified'], secure=co['secure'], expires=co['expires'], discard=co['discard'], comment=co['comment'], comment_url=co['comment_url'], rest=co['rest'])
              cj.set_cookie(ck)

          print "Logging-in into operacions via browser"
        else:
          #self.br = Browser()
          try:
              self.login()
          except:
              self.initialized=False
        return
Beispiel #6
0
def getres(url, cookie_in=None, cookie_out=None, post_params=None, require_ssl=False):
    """
    HTTPリクエスト
    :url: リクエストを送信するURL
    :cookie_in: 読み込むクッキーファイル
    :cookie_out: 書き込むクッキーファイル
    :post_params: POSTする辞書リスト
    :require_ssl: 接続にSSL(v3)を使用するか
    """

    if require_ssl:
        #handler = urllib2_ssl.HTTPSHandler(ca_certs=os.path.dirname(__file__)+'/cert/cacert.pem')
        handler = urllib2_ssl.TLS1Handler()
    else:
        handler = urllib2.HTTPHandler()

    if cookie_in != None or cookie_out != None:
        cookiejar = LWPCookieJar(cookie_in or cookie_out)
        if cookie_in != None:
            cookiejar.load()
        cProcessor = urllib2.HTTPCookieProcessor(cookiejar)
        opener = urllib2.build_opener(cProcessor, handler)
    else:
        opener = urllib2.build_opener(handler)

    if post_params != None:
        res = opener.open(url, urllib.urlencode(post_params))
    else:
        res = opener.open(url)

    if cookie_out != None:
        cookiejar.save()

    return res.read()
Beispiel #7
0
def http_req(url, data=None, headers={}, cookie=None):
    if cookie:
        cj = LWPCookieJar()
        
        if os.path.isfile(cookie):
            cj.load(cookie, ignore_discard=True)
        
        opener = urllib2.build_opener(NoRedirection, urllib2.HTTPCookieProcessor(cj))
        urllib2.install_opener(opener)
    
    if data is not None:
        data = urllib.urlencode(data)
    
    req = urllib2.Request(url, data)
    
    for k, v in common.HEADERS.items():
        req.add_header(k, v)
    
    if headers:
        for k, v in headers.items():
            req.add_header(k, v)
    
    conn = urllib2.urlopen(req)
    
    if cookie:
        cj.save(cookie, ignore_discard=True)
    
    return conn
Beispiel #8
0
    def send_cookie_auth(self, connection):
        """ Include Cookie Authentication data in a header
        """
        cj = LWPCookieJar()
        cj.load(COOKIES_FILE, ignore_discard=True, ignore_expires=True)

        for cookie in cj:
            connection.putheader('Cookie', '%s=%s' % (cookie.name, cookie.value))
Beispiel #9
0
    def save_cookies(self, cookie_storage):
        """Save to cookielib's CookieJar or Set-Cookie3 format text file.

        :param cookie_storage: file location string or CookieJar instance.
        """

        def toPyCookieJar(QtCookieJar, PyCookieJar):
            for c in QtCookieJar.allCookies():
                PyCookieJar.set_cookie(toPyCookie(c))

        def toPyCookie(QtCookie):
            port = None
            port_specified = False
            secure = QtCookie.isSecure()
            name = str(QtCookie.name())
            value = str(QtCookie.value())
            v = str(QtCookie.path())
            path_specified = bool(v != "")
            path = v if path_specified else None
            v = str(QtCookie.domain())
            domain_specified = bool(v != "")
            domain = v
            if domain_specified:
                domain_initial_dot = v.startswith(".")
            else:
                domain_initial_dot = None
            v = long(QtCookie.expirationDate().toTime_t())
            # Long type boundary on 32bit platfroms; avoid ValueError
            expires = 2147483647 if v > 2147483647 else v
            rest = {}
            discard = False
            return Cookie(
                0,
                name,
                value,
                port,
                port_specified,
                domain,
                domain_specified,
                domain_initial_dot,
                path,
                path_specified,
                secure,
                expires,
                discard,
                None,
                None,
                rest,
            )

        if cookie_storage.__class__.__name__ == "str":
            cj = LWPCookieJar(cookie_storage)
            toPyCookieJar(self.cookie_jar, cj)
            cj.save()
        elif cookie_storage.__class__.__name__.endswith("CookieJar"):
            toPyCookieJar(self.cookie_jar, cookie_storage)
        else:
            raise ValueError("unsupported cookie_storage type.")
Beispiel #10
0
  def initCookies(self):
    """ Stud.IP requires cookies for session-handling.
      Cookies are stored in a file to limit login requests"""
    cookieFile = open(cookieFilename, "r+")
    cookies = LWPCookieJar(cookieFile.name)

    if len(cookieFile.read()) > 0:
      cookies.load(cookieFile.name, ignore_discard=True)
    return cookies
Beispiel #11
0
class PrettyBugz:
	def __init__(self, args):
		self.quiet = args.quiet
		self.columns = args.columns or terminal_width()
		self.user = args.user
		self.password = args.password
		self.passwordcmd = args.passwordcmd
		self.skip_auth = args.skip_auth

		cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
		self.cookiejar = LWPCookieJar(cookie_file)

		try:
			self.cookiejar.load()
		except IOError:
			pass

		if getattr(args, 'encoding'):
			self.enc = args.encoding
		else:
			try:
				self.enc = locale.getdefaultlocale()[1]
			except:
				self.enc = 'utf-8'
			if not self.enc:
				self.enc = 'utf-8'

		self.log("Using %s " % args.base)
		self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)

	def log(self, status_msg, newline = True):
		if not self.quiet:
			if newline:
				print ' * %s' % status_msg
			else:
				print ' * %s' % status_msg,

	def warn(self, warn_msg):
		if not self.quiet:
			print ' ! Warning: %s' % warn_msg

	def get_input(self, prompt):
		return raw_input(prompt)

	def bzcall(self, method, *args):
		"""Attempt to call method with args. Log in if authentication is required.
		"""
		try:
			return method(*args)
		except xmlrpclib.Fault, fault:
			# Fault code 410 means login required
			if fault.faultCode == 410 and not self.skip_auth:
				self.login()
				return method(*args)
			raise
Beispiel #12
0
	def _get_cookie_jar(self):
		if not hasattr(self, '_cookie_jar'):
			cj = LWPCookieJar()
			if not isfile(self._cookie_file):
				# @todo:use notifications or login?
				raise IOError('Snap! Who stole the cookie in the cookie jar?')

			cj.load(self._cookie_file)
			self._cookie_jar = cj

		return self._cookie_jar
Beispiel #13
0
class PrettyBugz:
	def __init__(self, args):
		self.columns = args.columns or terminal_width()
		self.user = args.user
		self.password = args.password
		self.passwordcmd = args.passwordcmd
		self.skip_auth = args.skip_auth

		cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
		self.cookiejar = LWPCookieJar(cookie_file)

		try:
			self.cookiejar.load()
		except IOError:
			pass

		self.token_file = os.path.join(os.environ['HOME'], DEFAULT_TOKEN_FILE)
		try:
			self.token = open(self.token_file).read().strip()
		except IOError:
			self.token = None

		if getattr(args, 'encoding'):
			self.enc = args.encoding
		else:
			try:
				self.enc = locale.getdefaultlocale()[1]
			except:
				self.enc = 'utf-8'
			if not self.enc:
				self.enc = 'utf-8'

		log_info("Using %s " % args.base)
		self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)

	def get_input(self, prompt):
		return raw_input(prompt)

	def set_token(self, *args):
		if args and self.token:
			args[0]['Bugzilla_token'] = self.token
		return args

	def bzcall(self, method, *args):
		"""Attempt to call method with args. Log in if authentication is required.
		"""
		try:
			return method(*self.set_token(*args))
		except xmlrpclib.Fault, fault:
			# Fault code 410 means login required
			if fault.faultCode == 410 and not self.skip_auth:
				self.login()
				return method(*self.set_token(*args))
			raise
Beispiel #14
0
 def send_host(self, connection, host):
     """ Override send_host to add cookie info
     to XML-RPC requests """
     self.crequest = CookieRequest('http://'+host+'/')
     # Read the cookie file and add the Token header
     if os.path.exists(self.cookie_file):
         cj = LWPCookieJar()
         cj.load(self.cookie_file)
         for cookie in cj:
             if cookie.name == 'Token':
                 connection.putheader(cookie.name, cookie.value)
     xmlrpclib.Transport.send_host(self, connection, host)
Beispiel #15
0
def configureCookieProcessor(cookiefile="/tmp/searchmyhash.cookie"):
    """Set a Cookie Handler to accept cookies from the different Web sites.
	
	@param cookiefile Path of the cookie store."""

    cookieHandler = LWPCookieJar()
    if cookieHandler is not None:
        if path.isfile(cookiefile):
            cookieHandler.load(cookiefile)

        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieHandler))
        urllib2.install_opener(opener)
def save_cookies_to_file2(url):
    cookiefile = 'cookies.log'

    lwpcookiejar = LWPCookieJar(filename=cookiefile)

    opener = urllib2.build_opener(
        urllib2.HTTPCookieProcessor(lwpcookiejar))
    opener.open(url)

    lwpcookiejar.save()

    return lwpcookiejar
Beispiel #17
0
class katsomo():
	URL = { 
		'login' : 'http://m.katsomo.fi/katsomo/login',
		'programdir' : 'http://m.katsomo.fi/katsomo/programs',
		'programs' : 'http://m.katsomo.fi/katsomo/?treeId=',
		'videolink' : 'http://m.katsomo.fi/?progId='
	}
	clearCache=False
	def __init__(self, username="", password="", cookie_file=""):
		self.cj = LWPCookieJar(cookie_file)
		if username == "":
			self.noLogin = True
		else:
			self.noLogin = False
			self.username = username
			self.password = password
			try:
				self.cj.revert(ignore_discard = True)
			except IOError:
				pass
		self.cj.set_cookie(self.makeCookie('hq','1'))
		self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
		self.user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3'
	
	def makeCookie(self, name, value):
		return Cookie(
		    version=0, 
		    name=name, 
		    value=value,
		    port=None, 
		    port_specified=False,
		    domain="katsomo.fi", 
		    domain_specified=True, 
		    domain_initial_dot=False,
		    path="/", 
		    path_specified=True,
		    secure=False,
		    expires=None,
		    discard=False,
		    comment=None,
		    comment_url=None,
		    rest=None
		)
		
	def getPage(self, url,postvars={}, header_data = {}):
		req = urllib2.Request(url, urllib.urlencode(postvars), header_data)			
		req.add_header('user-agent', self.user_agent)
		try:
			resp = self.opener.open(req).read()
		except HTTPError, error:
			raise NetworkError('HTTPError: %s' % error)
		except URLError, error:
			raise NetworkError('URLError: %s' % error)
def getTVPCookieJar(updatedUName=False):
    cookieJar=None
    print 'updatedUName',updatedUName
    try:
        cookieJar = LWPCookieJar()
        if not updatedUName:
            cookieJar.load(TVPCOOKIEFILE,ignore_discard=True)
    except: 
        cookieJar=None

    if not cookieJar:
        cookieJar = LWPCookieJar()
    return cookieJar
Beispiel #19
0
def getFirefoxCookies(domain = ''):
    """Returns Mozilla Firefox cookies for the specified domain."""
    isWindows = system().lower().startswith('win')
    profilesPath = unicode(expanduser(FIREFOX_PROFILES_WINDOWS if isWindows else FIREFOX_PROFILES_LINUX))
    cookieFiles = (join(profilesPath, f, FIREFOX_COOKIE_FILE) for f in listdir(profilesPath))
    cookieDB = sorted((f for f in cookieFiles if isfile(f)), key = getmtime)[-1]
    cursor = connect(cookieDB).cursor()
    cursor.execute(FIREFOX_COOKIES_SQL_REQUEST % domain)
    cookieJar = LWPCookieJar()
    for (host, path, isSecure, expiry, name, value) in cursor.fetchall():
        cookieJar.set_cookie(Cookie(0, name, value, None, False,
            host, host.startswith('.'), host.startswith('.'),
            path, False, isSecure, expiry, not expiry, None, None, {}))
    return cookieJar
Beispiel #20
0
    def login(self):
        if self.username == '' or self.password == '':
            raise Exception("Please configure your username/password and restart the application")
        logger.info("Checking Login status")
        from cookielib import LWPCookieJar
        cj = LWPCookieJar(self.cookiefile)
        if not self.opener_installed:
            from urllib2 import build_opener, install_opener, HTTPCookieProcessor
            opener = build_opener(HTTPCookieProcessor(cj))
            install_opener(opener)
            self.opener_installed = True

        try:
            cj.load()
            logger.info("Loaded cookie file")
        except:
            logger.info("Couldn't load cookie file")
        else:
            logger.info("Checking if still logged in...")
            url = 'http://www.geocaching.com/seek/nearest.aspx'
            page = self.get_reader(url, login = False)
            for line in page:
                if 'You are logged in as' in line:
                    self.logged_in = True
                    logger.info("Seems as we're still logged in")
                    return
                elif 'You are not logged in.' in line:
                    logger.info("Nope, not logged in anymore")
                    break
        
        logger.info("Logging in")
        url, values = self.login_callback(self.username, self.password)

        page = self.get_reader(url, values, login = False)

        for line in page:
            if 'You are logged in as' in line:
                break
            elif 'You are not logged in.' in line or 'combination does not match' in line:
                raise Exception("Wrong password or username!")
        else:
            logger.info("Seems as if the language is set to something other than english")
            raise Exception("Please go to geocaching.com and set the website language to english!")

        logger.info("Great success.")
        self.logged_in = True
        try:
            cj.save()
        except Exception, e:
            logger.info("Could not save cookies: %s" % e)
Beispiel #21
0
    def send_cookie_auth(self, connection):
        """ Include Cookie Authentication data in a header
        """
        try:
            cj = LWPCookieJar()
            cj.load(COOKIES_FILE, ignore_discard=True, ignore_expires=True)

            for cookie in cj:
                connection.putheader('Cookie', '%s=%s' % (cookie.name, cookie.value))
            return True
        except LoadError :
            # mmc-cookies file is sometimes on bad format 
            # (probably caused by interrupted twisted sessions)
            log.warn("mmc-cookies: invalid LWP format file, resending the credentials")
            return False
Beispiel #22
0
 def initialize_cookie_jar(self):
     """Load cookies from disk, creating a new file if needed."""
     self.cookies = LWPCookieJar(self.cfg['cookie_jar_path'])
     if not os.path.exists('cookiejar'):
         # Create a new cookies file and set our Session's cookies
         self.cookies.save()
     self.cookies.load(ignore_discard=True)
def login(username, password, cookiefile='cookies'):
    cj = LWPCookieJar(cookiefile)
    try:
        cj.load()
    except IOError:
        pass

    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    formdata = {'sUsername': username, 'sPassword': password}
    data_encoded = urllib.urlencode(formdata)
    response = opener.open('/'.join([cost_prefix, cost_login]), data_encoded)
    content = response.read()
    # print(content)
    # print()
    cj.save(ignore_discard=True)
    return (cj, opener)
Beispiel #24
0
    def _get_cookiejar(self):
        """Return a LWPCookieJar object loaded from our .cookies file.

        The same .cookies file is returned every time, located in the project
        root, same directory as config.yml and bot.py. If it doesn't exist, we
        will create the file and set it to be readable and writeable only by
        us. If it exists but the information inside is bogus, we'll ignore it.

        This is normally called by _make_site_object() (in turn called by
        get_site()), and the cookiejar is passed to our Site's constructor,
        used when it makes API queries. This way, we can easily preserve
        cookies between sites (e.g., for CentralAuth), making logins easier.
        """
        if self._cookiejar:
            return self._cookiejar

        self._cookiejar = LWPCookieJar(self._cookie_file)

        try:
            self._cookiejar.load()
        except LoadError:
            pass  # File contains bad data, so ignore it completely
        except IOError as e:
            if e.errno == errno.ENOENT:  # "No such file or directory"
                # Create the file and restrict reading/writing only to the
                # owner, so others can't peak at our cookies:
                open(self._cookie_file, "w").close()
                chmod(self._cookie_file, stat.S_IRUSR|stat.S_IWUSR)
            else:
                raise

        return self._cookiejar
Beispiel #25
0
    def setUp(self):
        ''' ''' # {{{
        self.size = 100
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)'
        #self.prefix_url = 'http://www.google.com/search?q='
        self.server_ip = '192.168.1.100'
        self.server_port = '8011'
        self.server_path = 'hello?q='
        self.prefix_url = 'http://%s:%s/%s' % (self.server_ip, self.server_port, self.server_path)
        self.proxy = 'http://*****:*****@31.220.11.117:1212'

        self.numbers = ['09%08d' % randint(10000001, 89000000) for i in xrange(self.size)]

        # Cookie jar
        # {{{
        home_folder = os.getenv('HOME')
        if not home_folder:
            home_folder = os.getenv('USERHOME')
            if not home_folder:
                home_folder = '.'   # Use the current folder on error.
        self.cookie_jar = LWPCookieJar(os.path.join(home_folder, '.google-cookie'))
        try:
            self.cookie_jar.load()
        except Exception:
            print '> cookie_jar failed'
            pass
        # }}}

        # make google to cache this query?
        # {{{
        for n in self.numbers:
            c = requests.get(self.prefix_url+n).content
        # }}}

        print 'benchmarking...'
Beispiel #26
0
	def __init__(self, hostname, username, password, fields=[], groups=[]):
		self.hostname = hostname

		# requests session
		self.session = requests.Session()
		
		# request headers
		token = base64.standard_b64encode("%s:%s" % (username, password))
		self.headers = {
			"Accept": "application/json", 
			"Content-Type": "application/json", 
			"Authorization": "Basic %s" % token,
		}

		# request cookies
		self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session") 
		try:
			self.cookies.load(ignore_discard=True)
		except IOError:
			pass
		self.session.cookies = self.cookies

		# extra fields (table columns)
		self.fields = fields

		# extra groups (table columns)
		self.groups = groups

		# initialize inventory
		self.inventory = {'_meta': {'hostvars': { }}}

		return
    def __init__(self, cookiefile, core = None):
        self.cookiefile = cookiefile
        self.logged_in = False
        from socket import setdefaulttimeout
        setdefaulttimeout(30)
        self.opener_installed = False
        
        # This controls the use of the cache-headers in requests to allow/deny minified answers
        # as provided by some mobile operators.
        self.allow_minified_answers = True
        
        
        self.cj = LWPCookieJar(self.cookiefile)

        
        if DEBUG_HTTP:
            opener = build_opener(HTTPHandler(debuglevel=1), HTTPCookieProcessor(self.cj))
        else:
            opener = build_opener(HTTPCookieProcessor(self.cj))
        install_opener(opener)

        try:
            self.cj.load()
            logger.info("Loaded cookie file")
        except IOError, e:
            logger.info("Couldn't load cookie file")
Beispiel #28
0
	def __init__(self, args):
		self.columns = args.columns or terminal_width()
		self.user = args.user
		self.password = args.password
		self.passwordcmd = args.passwordcmd
		self.skip_auth = args.skip_auth

		cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
		self.cookiejar = LWPCookieJar(cookie_file)

		try:
			self.cookiejar.load()
		except IOError:
			pass

		if getattr(args, 'encoding'):
			self.enc = args.encoding
		else:
			try:
				self.enc = locale.getdefaultlocale()[1]
			except:
				self.enc = 'utf-8'
			if not self.enc:
				self.enc = 'utf-8'

		log_info("Using %s " % args.base)
		self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)
Beispiel #29
0
 def download(self,url=""):
     if url == "":
         url = self.url
     else:
         url = url
     try:
             with open('cookies.txt'): pass
     except (OSError, IOError):
             cookies = LWPCookieJar('cookies.txt')
             cookies.save()      
     cookies = self.scraper
     cookies.cookies = LWPCookieJar('cookies.txt')
     cookies.cookies.load()
     html = self.scraper.get(url).content
     cookies.cookies.save()
     return html
Beispiel #30
0
class zhihuCrawler(object):
	
	def __init__(self, login_url, login_info, header):
		self.base_url = 'http://www.zhihu.com'
		self.login_url = login_url
		self.login_info = login_info
		self.header = header
		self.cookies = LWPCookieJar('cookies.jar')

		try:
			self.cookies.revert()
		#	print open(cookies.filename).read()
		except Exception,e:
			print 'First time to login, setting up cookies...'

		opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookies))
		urllib2.install_opener(opener)
Beispiel #31
0
class Bilibili():
    def __init__(self, appkey=APPKEY, appsecret=APPSECRET):
        self.appkey = appkey
        self.appsecret = appsecret
        self.is_login = False
        cookie_path = os.path.dirname(os.path.abspath(__file__)) + '/.cookie'
        self.cj = LWPCookieJar(cookie_path)
        if os.path.isfile(cookie_path):
            self.cj.load()
            if requests.utils.dict_from_cookiejar(
                    self.cj).has_key('DedeUserID'):
                self.is_login = True
                self.mid = str(
                    requests.utils.dict_from_cookiejar(self.cj)['DedeUserID'])
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
        urllib2.install_opener(opener)

    def get_captcha(self, path=None):
        utils.get_page_content('https://passport.bilibili.com/login')
        result = utils.get_page_content(
            LOGIN_CAPTCHA_URL,
            headers={
                'Referer':
                'https://passport.bilibili.com/ajax/miniLogin/minilogin'
            })
        if path == None:
            path = tempfile.gettempdir() + '/captcha.jpg'
        with open(path, 'wb') as f:
            f.write(result)
        return path

    def get_encryped_pwd(self, pwd):
        import rsa
        result = json.loads(
            utils.get_page_content(
                LOGIN_HASH_URL,
                headers={
                    'Referer':
                    'https://passport.bilibili.com/ajax/miniLogin/minilogin'
                }))
        pwd = result['hash'] + pwd
        key = result['key']
        pub_key = rsa.PublicKey.load_pkcs1_openssl_pem(key)
        pwd = rsa.encrypt(pwd.encode('utf-8'), pub_key)
        pwd = base64.b64encode(pwd)
        pwd = urllib.quote(pwd)
        return pwd

    def api_sign(self, params):
        params['appkey'] = self.appkey
        data = ""
        keys = params.keys()
        keys.sort()
        for key in keys:
            if data != "":
                data += "&"
            value = params[key]
            if type(value) == int:
                value = str(value)
            data += key + "=" + str(urllib.quote(value))
        if self.appsecret == None:
            return data
        m = hashlib.md5()
        m.update(data + self.appsecret)
        return data + '&sign=' + m.hexdigest()

    def get_category_from_web_page(self):
        category_dict = {'0': {'title': u'全部', 'url': HOME_URL, 'subs': []}}
        node = category_dict['0']
        url = node['url']
        result = BeautifulSoup(utils.get_page_content(url),
                               "html.parser").findAll('li', {'class': 'm-i'})
        for item in result:
            if len(item['class']) != 1:
                continue
            tid = item['data-tid']
            title = item.em.contents[0]
            url = 'http:' + item.a['href']
            category_dict[tid] = {'title': title, 'url': url, 'subs': []}
            node['subs'].append(tid)

        #Fix video and movie
        if '11' not in category_dict['0']['subs']:
            category_dict['0']['subs'].append('11')
        if '23' not in category_dict['0']['subs']:
            category_dict['0']['subs'].append('23')
        category_dict['11'] = {
            'title': u'电视剧',
            'url': 'http://bangumi.bilibili.com/tv/',
            'subs': []
        }
        category_dict['23'] = {
            'title': u'电影',
            'url': 'http://bangumi.bilibili.com/movie/',
            'subs': []
        }

        for sub in category_dict['0']['subs']:
            node = category_dict[sub]
            url = node['url']
            result = BeautifulSoup(utils.get_page_content(url),
                                   "html.parser").select('ul.n_num li')
            for item in result[1:]:
                if not item.has_attr('tid'):
                    continue
                if not hasattr(item, 'a'):
                    continue
                if item.has_attr('class'):
                    continue
                tid = item['tid']
                title = item.a.contents[0]
                url = HOME_URL + item.a['href']
                category_dict[tid] = {'title': title, 'url': url, 'subs': []}
                node['subs'].append(tid)
        return category_dict

    def get_category(self, tid='0'):
        items = [{
            tid: {
                'title': '全部',
                'url': CATEGORY[tid]['url'],
                'subs': []
            }
        }]
        for sub in CATEGORY[tid]['subs']:
            items.append({sub: CATEGORY[sub]})
        return items

    def get_category_name(self, tid):
        return CATEGORY[str(tid)]['title']

    def get_order(self):
        return ORDER

    def get_category_list(self,
                          tid=0,
                          order='default',
                          days=30,
                          page=1,
                          pagesize=10):
        params = {
            'tid': tid,
            'order': order,
            'days': days,
            'page': page,
            'pagesize': pagesize
        }
        url = LIST_URL.format(self.api_sign(params))
        result = json.loads(utils.get_page_content(url))
        results = []
        for i in range(pagesize):
            if result['list'].has_key(str(i)):
                results.append(result['list'][str(i)])
            else:
                break
        return results, result['pages']

    def get_my_info(self):
        if self.is_login == False:
            return []
        result = json.loads(utils.get_page_content(MY_INFO_URL))
        return result['data']

    def get_bangumi_chase(self, page=1, pagesize=10):
        if self.is_login == False:
            return []
        url = BANGUMI_CHASE_URL.format(self.mid, page, pagesize)
        result = json.loads(utils.get_page_content(url))
        return result['data']['result'], result['data']['pages']

    def get_bangumi_detail(self, season_id):
        url = BANGUMI_SEASON_URL.format(season_id)
        result = utils.get_page_content(url)
        if result[0] != '{':
            start = result.find('(') + 1
            end = result.find(');')
            result = result[start:end]
        result = json.loads(result)
        return result['result']

    def get_history(self, page=1, pagesize=10):
        if self.is_login == False:
            return []
        url = HISTORY_URL.format(page, pagesize)
        result = json.loads(utils.get_page_content(url))
        if len(result['data']) >= int(pagesize):
            total_page = int(page) + 1
        else:
            total_page = int(page)
        return result['data'], total_page

    def get_dynamic(self, page=1, pagesize=10):
        if self.is_login == False:
            return []
        url = DYNAMIC_URL.format(pagesize, page)
        result = json.loads(utils.get_page_content(url))
        total_page = int(
            (result['data']['page']['count'] + pagesize - 1) / pagesize)
        return result['data']['feeds'], total_page

    def get_attention(self, page=1, pagesize=10):
        if self.is_login == False:
            return []
        url = ATTENTION_URL.format(self.mid, page, pagesize)
        result = json.loads(utils.get_page_content(url))
        return result['data']['list'], result['data']['pages']

    def get_attention_video(self, mid, tid=0, page=1, pagesize=10):
        if self.is_login == False:
            return []
        url = ATTENTION_VIDEO_URL.format(mid, page, pagesize, tid)
        result = json.loads(utils.get_page_content(url))
        return result['data'], result['data']['pages']

    def get_attention_channel(self, mid):
        if self.is_login == False:
            return []
        url = ATTENTION_CHANNEL_URL.format(mid)
        result = json.loads(utils.get_page_content(url))
        return result['data']['list']

    def get_attention_channel_list(self, mid, cid, page=1, pagesize=10):
        if self.is_login == False:
            return []
        url = ATTENTION_CHANNEL_LIST_URL.format(mid, cid, page, pagesize)
        result = json.loads(utils.get_page_content(url))
        return result['data']['list'], result['data']['total']

    def get_fav_box(self):
        if self.is_login == False:
            return []
        url = FAV_BOX_URL.format(self.mid)
        result = json.loads(utils.get_page_content(url))
        return result['data']['list']

    def get_fav(self, fav_box, page=1, pagesize=10):
        if self.is_login == False:
            return []
        url = FAV_URL.format(self.mid, page, pagesize, fav_box)
        result = json.loads(utils.get_page_content(url))
        return result['data']['vlist'], result['data']['pages']

    def login(self, userid, pwd, captcha):
        #utils.get_page_content('http://www.bilibili.com')
        if self.is_login == True:
            return True, ''
        pwd = self.get_encryped_pwd(pwd)
        data = 'userid={0}&pwd={1}&keep=1&captcha={2}'.format(
            userid, pwd, captcha)
        result = utils.get_page_content(
            LOGIN_URL, data, {
                'Origin': 'https://passport.bilibili.com',
                'Referer':
                'https://passport.bilibili.com/ajax/miniLogin/minilogin'
            })
        if not requests.utils.dict_from_cookiejar(
                self.cj).has_key('DedeUserID'):
            return False, LOGIN_ERROR_MAP[json.loads(result)['message']
                                          ['code']]
        self.cj.save()
        self.is_login = True
        self.mid = str(
            requests.utils.dict_from_cookiejar(self.cj)['DedeUserID'])
        return True, ''

    def logout(self):
        self.cj.clear()
        self.cj.save()
        self.is_login = False

    def get_av_list_detail(self, aid, page=1, fav=0, pagesize=10):
        params = {'id': aid, 'page': page}
        if fav != 0:
            params['fav'] = fav
        url = VIEW_URL.format(self.api_sign(params))
        result = json.loads(utils.get_page_content(url))
        results = [result]
        if (int(page) < result['pages']) and (pagesize > 1):
            results += self.get_av_list_detail(aid,
                                               int(page) + 1,
                                               fav,
                                               pagesize=pagesize - 1)[0]
        return results, result['pages']

    def get_av_list(self, aid):
        url = AV_URL.format(aid)
        result = json.loads(utils.get_page_content(url))
        return result

    def get_video_urls(self, cid):
        m = hashlib.md5()
        m.update(INTERFACE_PARAMS.format(str(cid), SECRETKEY_MINILOADER))
        url = INTERFACE_URL.format(str(cid), m.hexdigest())
        doc = minidom.parseString(utils.get_page_content(url))
        urls = [
            durl.getElementsByTagName('url')[0].firstChild.nodeValue
            for durl in doc.getElementsByTagName('durl')
        ]
        urls = [
            url if not re.match(r'.*\.qqvideo\.tc\.qq\.com', url) else re.sub(
                r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', url)
            for url in urls
        ]
        return urls

    def add_history(self, aid, cid):
        url = ADD_HISTORY_URL.format(str(cid), str(aid))
        utils.get_page_content(url)
Beispiel #32
0
from urllib2 import build_opener, HTTPCookieProcessor, Request, HTTPHandler
from urllib import urlencode, quote
from cookielib import FileCookieJar, CookieJar, LWPCookieJar
from tempfile import mkstemp
from contextlib import closing
from subprocess import Popen, PIPE

from lxml import html

COOKIES_FILE = '/usr/local/etc/bandcamp.cookies'

URL = 'https://bandcamp.com'
CDN_COVERS = 'https://f4.bcbits.com/img'

cj = LWPCookieJar()

if os.path.isfile(COOKIES_FILE):
    cj.load(COOKIES_FILE)

handler = HTTPHandler(debuglevel=0)
opener = build_opener(handler, HTTPCookieProcessor(cj))
opener.addheaders = [('User-agent', 'Enter you own user agent !'),
                     ('Accept', '*/*'), ('Accept-Encoding', 'deflate')]

TMP_PATH = ''
TMP_FILE_PREFIX = 'tmpS_'
queue = Queue()

# Do we have to download then add cover to downloaded music file ?
ADD_COVER = 1
Beispiel #33
0
class Client:
    """
    Web client class with automatic charset detection and decoding
    """
    def __init__(self, info=None):
        self._counter = 0
        self._cookies_filename = ''
        self._cookies = LWPCookieJar()
        self.user_agent = USER_AGENT
        self.info = info
        self.proxy_type = None
        self.proxy_url = None
        self.content = None
        self.status = None
        self.headers = dict()

        if get_setting("use_elementum_proxy", bool):
            elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
            if elementum_addon and elementum_addon.getSetting('internal_proxy_enabled') == "true":
                self.proxy_url = "{0}://{1}:{2}".format("http", "127.0.0.1", "65222")
                if info and "internal_proxy_url" in info:
                    log.debug("Use Internal Elementum Proxy")
                    self.proxy_url = info["internal_proxy_url"]
            if elementum_addon.getSetting("proxy_enabled") == "true" and get_setting("use_proxy_setting", bool):
                self.proxy_type = int(elementum_addon.getSetting("proxy_type"))
                log.debug("Use users proxy from elementum settings: {0}".format(proxy_types[self.proxy_type]))
                prx_host = elementum_addon.getSetting("proxy_host")
                prx_port = elementum_addon.getSetting("proxy_port")
                self.proxy_url = "{0}://{1}:{2}".format(proxy_types[self.proxy_type], prx_host, prx_port)

    @classmethod
    def _create_cookies(self, payload):
        return urlencode(payload)

    def _read_cookies(self, url=''):
        cookies_path = os.path.join(PATH_TEMP, 'nova')
        if not os.path.exists(cookies_path):
            try:
                os.makedirs(cookies_path)
            except Exception as e:
                log.debug("Error creating cookies directory: %s" % repr(e))
        self._cookies_filename = os.path.join(cookies_path, urlparse(url).netloc + '_cookies.jar')
        if os.path.exists(self._cookies_filename):
            try:
                self._cookies.load(self._cookies_filename)
            except Exception as e:
                log.debug("Reading cookies error: %s" % repr(e))

    def _save_cookies(self):
        try:
            self._cookies.save(self._cookies_filename)
        except Exception as e:
            log.debug("Saving cookies error: %s" % repr(e))

    def _good_spider(self):
        self._counter += 1
        if self._counter > 1:
            sleep(0.25)

    def cookies(self):
        """ Saved client cookies

        Returns:
            list: A list of saved Cookie objects
        """
        return self._cookies

    def open(self, url, language='en', post_data=None, get_data=None):
        """ Opens a connection to a webpage and saves its HTML content in ``self.content``

        Args:
            url        (str): The URL to open
            language   (str): The language code for the ``Content-Language`` header
            post_data (dict): POST data for the request
            get_data  (dict): GET data for the request
        """
        if not post_data:
            post_data = {}
        if get_data:
            url += '?' + urlencode(get_data)

        log.debug("Opening URL: %s" % repr(url))
        result = False

        data = urlencode(post_data) if len(post_data) > 0 else None
        req = urllib2.Request(url, data)

        self._read_cookies(url)
        log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))

        handlers = []

        if get_setting("use_elementum_proxy", bool) and self.proxy_url:
            if self.proxy_type:
                if self.proxy_type == 2:
                    proxyHandler = urllib2.ProxyHandler({
                        'http': self.proxy_url,
                        'https': self.proxy_url,
                    })
                    handlers.append(proxyHandler)
                elif self.proxy_type == 1:
                    import proxy.socks as socks
                    from proxy.sockshandler import SocksiPyHandler
                    prx_info = self.proxy_url.split(':')
                    handlers.append(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, prx_info[1].replace("//", ''), int(prx_info[2])))
            else:
                proxyHandler = urllib2.ProxyHandler({
                    'http': self.proxy_url,
                    'https': self.proxy_url,
                })
                handlers.append(proxyHandler)

        cookieHandler = urllib2.HTTPCookieProcessor(self._cookies)
        handlers.append(cookieHandler)

        opener = urllib2.build_opener(*handlers)
        req.add_header('User-Agent', self.user_agent)
        req.add_header('Content-Language', language)
        req.add_header("Accept-Encoding", "gzip")
        req.add_header("Origin", url)
        req.add_header("Referer", url)

        try:
            self._good_spider()
            with closing(opener.open(req)) as response:
                self.headers = response.headers
                self._save_cookies()
                if response.headers.get("Content-Encoding", "") == "gzip":
                    import zlib
                    self.content = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(response.read())
                else:
                    self.content = response.read()

                charset = response.headers.getparam('charset')

                if not charset:
                    match = re.search("""<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""", self.content)
                    if match:
                        charset = match.group(1)

                if charset and charset.lower() == 'utf-8':
                    charset = 'utf-8-sig'  # Changing to utf-8-sig to remove BOM if found on decode from utf-8

                if charset:
                    log.debug('Decoding charset from %s for %s' % (charset, repr(url)))
                    self.content = self.content.decode(charset, 'replace')

                self.status = response.getcode()
            result = True

        except urllib2.HTTPError as e:
            self.status = e.code
            log.warning("Status for %s : %s" % (repr(url), str(self.status)))

        except urllib2.URLError as e:
            self.status = repr(e.reason)
            log.warning("Status for %s : %s" % (repr(url), self.status))

        except Exception as e:
            import traceback
            log.error("%s failed with %s:" % (repr(url), repr(e)))
            map(log.debug, traceback.format_exc().split("\n"))

        log.debug("Status for %s : %s" % (repr(url), str(self.status)))

        return result

    def login(self, url, data, fails_with):
        """ Login wrapper around ``open``

        Args:
            url        (str): The URL to open
            data      (dict): POST login data
            fails_with (str): String that must **not** be included in the response's content

        Returns:
            bool: Whether or not login was successful
        """
        result = False
        if self.open(url.encode('utf-8'), post_data=encode_dict(data)):
            result = True
            if fails_with in self.content:
                self.status = 'Wrong username or password'
                result = False
        return result
Beispiel #34
0
#!/bin/env python
# -*- coding=utf-8 -*-
import ssl
import sys
import urllib2
import random
import httplib
import json
from cookielib import LWPCookieJar
import urllib
import re
import getpass

reload(sys)
sys.setdefaultencoding('UTF8')
cookiejar = LWPCookieJar()
cookiesuppor = urllib2.HTTPCookieProcessor(cookiejar)
opener = urllib2.build_opener(cookiesuppor, urllib2.HTTPHandler)
urllib2.install_opener(opener)
ssl._create_default_https_context = ssl._create_unverified_context
codeimg = 'https://kyfw.12306.cn/otn/passcodeNew/getPassCodeNew?module=login&rand=sjrand&%s' % random.random(
)

baner = """
##################################
    12306登录脚本,作者Mr RJL
    python版本:2.7,适用于linux
    验证码输入方式:
    输入问题对应的图片序号,1-8;
    多个以','分隔.如:1,2,3
##################################
class AlarmDotComScraper:
    def __init__(self, user, password, event_log_path):
        self.user = user
        self.password = password
        self.event_log_path = event_log_path
        self.cj = LWPCookieJar(COOKIEFILE)

    def getCurAlarmState(self):
        events = self.getLatestEvents()

        state_found = 0  # assume we'll find no state change
        current_state = "UNKNOWN"
        for event in events:
            #print event
            state = self.__getStateFromString(event[1])
            if state != "":
                #print "Found State:",state
                state_found = 1
                current_state = state

        if not state_found:
            current_state = self.__getCurStateFromLog()

        self.addEventsToLog(events)
        return current_state

    def getLatestEvents(self):
        try:
            self.cj.load(None, True, True)
        except IOError:
            pass  # Ignore the open error if the cookies file doesn't exist

        retry = False
        while True:
            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
            response = opener.open(alarmComBaseURL + alarmComStatus)
            #response = opener.open("file:/home/tnorris/Desktop/Home.html")

            soup = BeautifulSoup(response.read())

            event_tab_heading = soup.find("th",
                                          text=re.compile(".*Last 5 Events.*"))

            # Assume if we can't find the event table on the 1st try we need to login
            # on the 2nd time (retry=True) assume there's an issue
            if (event_tab_heading is None):
                if retry:
                    raise AlarmDotComScraperError(
                        "Unable to locate Alarm Event Table Header")
                retry = True
                self.__login()
            else:
                break

        event_table = event_tab_heading.parent.parent

        if (event_table is None):
            raise AlarmDotComScraperError("Error locating Events in Table",
                                          soup)

        date_time_tags = event_table.find_all("div")
        event_tags = event_table.find_all("span")
        if len(date_time_tags) == 0:
            raise AlarmDotComScraperError("Unable to locate time tags")

        if len(event_tags) == 0:
            raise AlarmDotComScraperError("Unable to locate event tags")

        if len(event_tags) != len(date_time_tags):
            raise AlarmDotComScraperError(
                "Mismatched length of events and times", len(event_tags),
                len(date_time_tags))

        date_times = [
            date_time_tags[i].string
            for i in reversed(range(len(date_time_tags)))
        ]
        events = [
            event_tags[i].string for i in reversed(range(len(event_tags)))
        ]
        return zip(date_times, events)

    def addEventsToLog(self, events):
        new_events = []
        try:
            handle = open(self.event_log_path, "r")
            for i in range(len(
                    events)):  # assumes events are in oldest to newest order
                temp_line = self.__makeLine(events[i][0], events[i][1])
                found = 0
                handle.seek(0)
                for line in handle:
                    line = line.strip()
                    if line == temp_line:
                        found = 1
                        break

                if not found:
                    new_events.append(events[i])

            handle.close()
        except IOError:
            new_events = events

        if len(new_events) > 0:
            handle = open(self.event_log_path, "a")
            for i in range(len(new_events)):
                out_line = self.__makeLine(new_events[i][0], new_events[i][1])
                handle.write(out_line)
                handle.write("\n")
            handle.close()

    def __getCurStateFromLog(self):
        try:
            current_state = "UNKNOWN"
            handle = open(self.event_log_path, "r")
            for line in handle:
                state = self.__getStateFromString(line)
                if state != "":
                    current_state = state
            handle.close()
        except IOError:  # if no log exists, there's no state to find
            pass

        return current_state

    def __getStateFromString(self, string):
        state = ""  # return no state is none found
        if re.search(r"\bARMED\b", string, flags=re.IGNORECASE) >= 0:  # ARMED?
            if re.search(r"\bSTAY\b", string,
                         flags=re.IGNORECASE) >= 0:  # STAY?
                state = "STAY"
            elif re.search(r"\bAWAY\b", string,
                           flags=re.IGNORECASE) >= 0:  # AWAY?
                state = "AWAY"
            else:
                state = "ARMED"  # ARMED, but unknown type
        elif re.search(r"\bDISARMED\b", string,
                       flags=re.IGNORECASE) >= 0:  # DISARMED?
            state = "DISARMED"

        #print state_found,state
        return state

    def __makeLine(self, date_time, event):
        first_space = date_time.find(" ")
        date = date_time[:first_space]
        time = date_time[first_space + 1:].swapcase()
        return "[{date} {time:>11}] {event}".format(date=date,
                                                    time=time,
                                                    event=event)

    def __login(self):
        formdata = {
            "ctl00$ContentPlaceHolder_InnerCommon$loginform$txtUserName":
            self.user,
            "ctl00$ContentPlaceHolder_InnerCommon$loginform$txtPassword":
            self.password,
            "IsFromNewSite": "1",
            "JavaScriptTest": "1"
        }
        data_encoded = urllib.urlencode(formdata)

        # Actually login, response should contain status page
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
        opener.open(alarmComBaseURL + alarmComLanding, data_encoded)
        self.cj.save(None, True, True)
Beispiel #36
0
    def authenticate(self):

        if self.test:
            return {'status': True, 'inkdrops': mylar.INKDROPS_32P}

        feedinfo = []

        try:
            with cfscrape.create_scraper() as s:
                s.headers = self.headers
                cj = LWPCookieJar(
                    os.path.join(mylar.CONFIG.CACHE_DIR, ".32p_cookies.dat"))
                cj.load()
                s.cookies = cj

                if mylar.CONFIG.VERIFY_32P == 1 or mylar.CONFIG.VERIFY_32P == True:
                    verify = True
                else:
                    verify = False

                logger.fdebug('[32P] Verify SSL set to : %s' % verify)

                if not verify:
                    #32P throws back an insecure warning because it can't validate against the CA. The below suppresses the message just for 32P instead of being displa$
                    from requests.packages.urllib3.exceptions import InsecureRequestWarning
                    requests.packages.urllib3.disable_warnings(
                        InsecureRequestWarning)

                # post to the login form
                r = s.post(self.url, verify=verify, allow_redirects=True)

                #logger.debug(self.module + " Content session reply" + r.text)

                #need a way to find response code (200=OK), but returns 200 for everything even failed signons (returns a blank page)
                #logger.info('[32P] response: ' + str(r.content))
                soup = BeautifulSoup(r.content, "html.parser")
                soup.prettify()

                if self.searchterm:
                    logger.info(
                        '[32P] Successfully authenticated. Initiating search for : %s'
                        % self.searchterm)
                    return self.search32p(s)

                logger.info('[32P] Successfully authenticated.')
                all_script = soup.find_all("script", {"src": False})
                all_script2 = soup.find_all("link", {"rel": "alternate"})

                authfound = False
                logger.info(
                    '%s Atttempting to integrate with all of your 32P Notification feeds.'
                    % self.module)

                #get inkdrop count ...
                #user_info = soup.find_all(attrs={"class": "stat"})
                #inkdrops = user_info[0]['title']
                #logger.info('INKDROPS: ' + str(inkdrops))

                for al in all_script2:
                    alurl = al['href']
                    if 'auth=' in alurl and 'torrents_notify' in alurl and not authfound:
                        f1 = alurl.find('auth=')
                        f2 = alurl.find('&', f1 + 1)
                        auth = alurl[f1 + 5:f2]
                        authfound = True
                        #logger.fdebug(self.module + ' Auth:' + str(auth))
                        #p1 = alurl.find('passkey=')
                        #p2 = alurl.find('&', p1 + 1)
                        #passkey = alurl[p1 +8:p2]
                        #logger.fdebug(self.module + ' Passkey:' + str(passkey))
                        if self.reauthenticate: break

                    if 'torrents_notify' in alurl and (
                            'torrents_notify_' +
                            str(self.passkey)) not in alurl:
                        notifyname_st = alurl.find('name=')
                        notifyname_en = alurl.find('&', notifyname_st + 1)
                        if notifyname_en == -1: notifyname_en = len(alurl)
                        notifyname = alurl[notifyname_st + 5:notifyname_en]
                        notifynumber_st = alurl.find('torrents_notify_')
                        notifynumber_en = alurl.find('_', notifynumber_st + 17)
                        notifynumber = alurl[notifynumber_st:notifynumber_en]
                        logger.fdebug(
                            '%s [NOTIFICATION: %s] Notification ID: %s' %
                            (self.module, notifyname, notifynumber))

                        #generate the rss-url here
                        feedinfo.append({
                            'feed':
                            notifynumber + '_' + str(self.passkey),
                            'feedname':
                            notifyname,
                            'user':
                            str(self.uid),
                            'auth':
                            auth,
                            'passkey':
                            str(self.passkey),
                            'authkey':
                            str(self.authkey)
                        })

        except (requests.exceptions.Timeout, EnvironmentError):
            logger.warn(
                'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
            )
            return

        #set the keys here that will be used to download.
        try:
            mylar.CONFIG.PASSKEY_32P = str(self.passkey)
            mylar.AUTHKEY_32P = str(self.authkey)  # probably not needed here.
            mylar.KEYS_32P = {}
            mylar.KEYS_32P = {
                "user": str(self.uid),
                "auth": auth,
                "passkey": str(self.passkey),
                "authkey": str(self.authkey)
            }

        except NameError:
            logger.warn(
                'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
            )
            return

        if self.reauthenticate:
            return
        else:
            mylar.FEEDINFO_32P = feedinfo
            return feedinfo
Beispiel #37
0
def init_cookie_jar(cookie_file=None,
                    cookie_t=None,
                    cookie_y=None,
                    cookie_euconsent=None):
    cookie_jar = LWPCookieJar(
        cookie_file) if cookie_file else RequestsCookieJar()

    if cookie_file and os.path.exists(cookie_file):
        cookie_jar.load(ignore_discard=True)

    if args.cookie_t:
        cookie_jar.set_cookie(create_cookie('T', cookie_t))
    if cookie_y:
        cookie_jar.set_cookie(create_cookie('Y', cookie_y))
    if cookie_euconsent:
        cookie_jar.set_cookie(create_cookie('EuConsent', cookie_euconsent))

    if cookie_file:
        cookie_jar.save(ignore_discard=True)

    return cookie_jar
Beispiel #38
0
def main():
	banner()
	print
	print(C+' Diperbarui Tanggal : '+W+'11 April 2020')
	print
	tiks(W+50*'=')
	tiks(C+'\tTOOLS\t\t LIMIT\t\tSTATUS')
	tiks(W+50*'=')
	print
	tiks(C+'['+W+'1'+C+']'+W+' SPAM CALL V1' +W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94')
	tiks(C+50*'=')
	tiks(C+'['+W+'2'+C+']'+W+' SPAM CALL V2'+W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94')
	tiks(C+50*'=')
	tiks(C+'['+W+'3'+C+']'+W+' SPAM CALL V3'+W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94')
	tiks(C+50*'=')
	tiks(C+'['+W+'4'+C+']'+W+' REPORT BUG'+K+'\t\t    -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94')
	tiks(C+50*'=')
	tiks(C+'['+W+'5'+C+']'+W+' SUBSCRIBE YT'+K+'\t    -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94')
	tiks(C+50*'=')
	tiks(C+'['+W+'X'+C+']'+W+' EXIT / KELUAR'+K+'\t    -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94')
	tiks(C+50*'=')
	
	try:
		print
		pilih = raw_input(W+'Pilih Tools'+C+' : '+W+'')
		
		if pilih == '1':
			print
			tiks(W+'\t\tSPAM OTP V1')
			tiks(C+'\t\t'+12*'=')
			print
			
			phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 62812xxxx '+W+') : ')
			jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ')
			
			if jumlah > 3:
				print
				print(W+'Jumlah Melewati Limit'+C+' ^_^')
				sleep(3)
				main()
				
			elif len(phone) < 10:
				print
				print(M+'\tNomor Tidak Valid !')
				sys.exit()
			
			elif '62' not in phone[0:2]:
				print
				print(C+'\tPakai 62'+W+' !')
				sleep(2.5)
				main()
					
			else:
				pass
			
			print
			for _ in range(int(jumlah)):
				
				sleep(5)
				
				data = {
				'phoneNumber' : phone,
				'workFlow' : 'GLOBAL_SIGNUP_LOGIN',
				'otpMethod' : 'CALL'
				}
				
				Cookies = Cookie('.cookieslog','w')
				Sess = requests.Session()
				Sess.cookies = Cookies
				Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}
				
				send = Sess.post('https://www.airbnb.co.id/api/v2/phone_one_time_passwords?currency=USD&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&locale=id', json = data)
			
				if 'error' in send.text:
					print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96')
				
				else:
					print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94')
					Cookies.save()
					
		elif pilih == '2':
			print
			tiks(W+'\t\tSPAM OTP V2')
			tiks(C+'\t\t'+12*'=')
			print
			phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 812xxxx '+W+') : ')
			jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ')
			
			if jumlah > 3:
				print
				print(W+'Jumlah Melewati Limit'+C+' ^_^')
				sleep(3)
				main()
				
			elif len(phone) < 10:
				print
				print(M+'\tNomor Tidak Valid !')
				sys.exit()
			
			elif '62' in phone[0:2] or '+62' in phone[0:2] or '08' in phone[0:2]:
				print
				print(C+'\tTanpa'+W+' +62/62/08'+C+' Langsung'+W+' 812xxxx'+C+' !')
				sleep(4)
				main()
					
			else:
				pass
			
			print
			for _ in range(int(jumlah)):
				
				sleep(5)
				data = {
				'contactNo' : phone,
				'countryCode' : 'ID',
				'requestMode' : '2',
				'isSignup' : 'true'
				}

				Cookies = Cookie('.cookieslog','w')
				Sess = requests.Session()
				Sess.cookies = Cookies
				Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}
				
				send = Sess.post('https://bitapples.com/api/v1/customer/auth/requestotpwithoutauth?t=1586576496&lang=en', json = data, headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'})
			
				if 'Otp sent, please check your mobile.' in send.text:
					print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94')
				
				else:
					print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96')
					Cookies.save()
					
		elif pilih == '3':
			print
			tiks(W+'\t\tSPAM OTP V3')
			tiks(C+'\t\t'+12*'=')
			print
			phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 812xxxx '+W+') : ')
			jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ')
			
			if jumlah > 3:
				print
				print(W+'Jumlah Melewati Limit'+C+' ^_^')
				sleep(3)
				main()
				
			elif len(phone) < 10:
				print
				print(M+'\tNomor Tidak Valid !')
				sys.exit()
			
			elif '62' in phone[0:2] or '+62' in phone[0:2] or '08' in phone[0:2]:
				print
				print(C+'\tTanpa'+W+' +62/62/08'+C+' Langsung'+W+' 812xxxx'+C+' !')
				sleep(4)
				main()
					
			else:
				pass
			
			print
			for _ in range(int(jumlah)):
				
				sleep(5)
				
				data = {
				'contactNo' : phone,
				'countryCode' : 'ID',
				'requestMode' : '2',
				'isSignup' : 'true'
				}
				
				Cookies = Cookie('.cookieslog','w')
				Sess = requests.Session()
				Sess.cookies = Cookies
				Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}
				
				send = Sess.post('https://citihash.com/api/v1/auth/otpRequest?lang=en', json = data, headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'})
			
				if 'Otp sent, please check your mobile.' in send.text:
					print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94')
				
				else:
					print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96')
					import pdb;pdb.set_trace()
					Cookies.save()
					
		elif pilih == '4':
			print
			os.system('xdg-open http://wa.me/6285880818385?text=Hallo%20Min.%20Saya%20Nemu%20Kesalahan%20:%0A%0A')
			sleep(2)
			print(W+'Hayooo Abis Report Apa'+C+' ^_^')
			sys.exit()
			
		elif pilih == '5':
			print
			os.system('xdg-open https://www.youtube.com/channel/UCzsADl8XRJeZXJ6CKZLX5KQ')
			sleep(2)
			tanya = raw_input(W+'Udah Subscribe Belum ?'+C+' ^_^'+W+' ['+C+'Y'+W+'/'+C+'n'+W+']'+C+' : '+W+'')
			if tanya.upper() == 'Y':
				print
				print(W+'Terima Kasih'+C+' ^_^')
				sys.exit() 
				
			else:
				print
				print(W+'Ok Gpp Kok'+C+' ^_^')
				sys.exit() 
			
		elif pilih == 'X' or pilih == 'x':
			print
			print(W+'Thanks, Jangan Lupa Balik Lagi'+C+' ^_^')
			sys.exit()
			
		else:
			print
			print(W+'Tools Tidak Di Temukan '+C+'^_^')
			sys.exit()
			
	except ConnectionError:
		print
		print(M+' Jaringan Tidak Ada  !')
		sys.exit()
	
	except ValueError:
		print
		print(M+' Yang Anda Masukkan Salah  !')
		sleep(3)
		main()
		
	except NameError:
		print
		print(M+' Masukkan Angka !')
		sleep(3)
		main()
Beispiel #39
0
    def __login__(self,
                  email,
                  passwd,
                  secret_answer,
                  platform='pc',
                  emulate=None):
        """Just log in."""
        # TODO: split into smaller methods
        secret_answer_hash = EAHashingAlgorithm().EAHash(secret_answer)
        self.credits = 0
        # create session
        self.r = requests.Session()  # init/reset requests session object
        # load saved cookies/session
        if self.cookies_file:
            self.r.cookies = LWPCookieJar(self.cookies_file)
            try:
                self.r.cookies.load(
                    ignore_discard=True
                )  # is it good idea to load discarded cookies after long time?
            except IOError:
                pass
                #self.r.cookies.save(ignore_discard=True)  # create empty file for cookies
        if emulate == 'and':
            self.r.headers = headers_and.copy()  # i'm android now ;-)
        elif emulate == 'ios':
            self.r.headers = headers_ios.copy()  # i'm ios phone now ;-)
        else:
            self.r.headers = headers.copy()  # i'm chrome browser now ;-)
        self.urls = urls(platform)
        # TODO: urls won't be loaded if we drop here
        #if self.r.get(self.urls['main_site']+'/fifa/api/isUserLoggedIn').json()['isLoggedIn']:
        #    return True  # no need to log in again
        # emulate
        if emulate == 'ios':
            sku = 'FUT15IOS'
            clientVersion = 11
        elif emulate == 'and':
            sku = 'FUT15AND'
            clientVersion = 11
#        TODO: need more info about log in procedure in game
#        elif emulate == 'xbox':
#            sku = 'FFA15XBX'  # FFA14CAP ?
#            clientVersion = 1
#        elif emulate == 'ps3':
#            sku = 'FFA15PS3'  # FFA14KTL ?
#            clientVersion = 1
#        elif emulate == 'pc':
#            sku = ''  # dunno
#            clientVersion = 1
        elif not emulate:
            sku = 'FUT15WEB'
            clientVersion = 1
        else:
            raise FutError(
                'Invalid emulate parameter. (Valid ones are and/ios).'
            )  # pc/ps3/xbox/
        # === login
        self.urls['login'] = self.r.get(self.urls['fut_home']).url
        self.r.headers['Referer'] = self.urls['login']  # prepare headers
        data = {
            'email': email,
            'password': passwd,
            '_rememberMe': 'on',
            'rememberMe': 'on',
            '_eventId': 'submit',
            'facebookAuth': ''
        }
        rc = self.r.post(self.urls['login'], data=data)
        self.logger.debug(rc.content)
        if self.r.get(
                self.urls['main_site'] +
                '/fifa/api/isUserLoggedIn').json()['isLoggedIn'] is not True:
            raise FutError(
                'Error during login process (probably invalid email or password).'
            )
        # TODO: catch invalid data exception
        #self.nucleus_id = re.search('userid : "([0-9]+)"', rc.text).group(1)  # we'll get it later

        # === lanuch futweb
        self.r.headers['Referer'] = self.urls['fut_home']  # prepare headers
        rc = self.r.get(self.urls['futweb'])
        self.logger.debug(rc.content)
        rc = rc.text
        #        if 'EASW_ID' not in rc:
        #            raise FutError('Error during login process (probably invalid email or password).')
        self.nucleus_id = re.search("var EASW_ID = '([0-9]+)';", rc).group(1)
        self.build_cl = re.search("var BUILD_CL = '([0-9]+)';", rc).group(1)
        #self.urls['fut_base'] = re.search("var BASE_FUT_URL = '(https://.+?)';", rc).group(1)
        #self.urls['fut_home'] = re.search("var GUEST_APP_URI = '(http://.+?)';", rc).group(1)

        self.urls = urls(platform, self.build_cl)

        # acc info
        self.r.headers.update({  # prepare headers
            'Content-Type': 'application/json',
            'Accept': 'text/json',
            'Easw-Session-Data-Nucleus-Id': self.nucleus_id,
            'X-UT-Embed-Error': 'true',
            'X-Requested-With': 'XMLHttpRequest',
            'X-UT-Route': self.urls['fut_host'],
            'Referer': self.urls['futweb'],
        })
        rc = self.r.get(self.urls['acc_info'])
        self.logger.debug(rc.content)
        rc = rc.json()['userAccountInfo']['personas'][0]
        self.persona_id = rc['personaId']
        self.persona_name = rc['personaName']
        self.clubs = [i for i in rc['userClubList']]
        # sort clubs by lastAccessTime (latest firts)
        self.clubs.sort(key=lambda i: i['lastAccessTime'], reverse=True)

        # authorization
        self.r.headers.update({  # prepare headers
            'Accept': 'application/json, text/javascript',
            'Origin': 'http://www.easports.com',
        })
        data = {
            'isReadOnly': False,
            'sku': sku,
            'clientVersion': clientVersion,
            #'nuc': self.nucleus_id,
            'nucleusPersonaId': self.persona_id,
            'nucleusPersonaDisplayName': self.persona_name,
            'nucleusPersonaPlatform': platform,
            'locale': 'en-GB',
            'method': 'authcode',
            'priorityLevel': 4,
            'identification': {
                'AuthCode': ''
            }
        }
        rc = self.r.post(self.urls['fut']['authentication'],
                         data=json.dumps(data))
        self.logger.debug(rc.content)
        if rc.status_code == 500:
            raise InternalServerError('Servers are probably temporary down.')
        rc = rc.json()
        #self.urls['fut_host'] = '{0}://{1}'.format(rc['protocol']+rc['ipPort'])
        if rc.get('reason') == 'multiple session':
            raise MultipleSession
        elif rc.get('reason') == 'max sessions':
            raise MaxSessions
        elif rc.get('reason') == 'doLogin: doLogin failed':
            raise doLoginFail
        elif rc.get('reason'):
            raise UnknownError(rc.__str__())
        self.r.headers['X-UT-SID'] = self.sid = rc['sid']

        # validate (secret question)
        self.r.headers['Accept'] = 'text/json'  # prepare headers
        del self.r.headers['Origin']
        rc = self.r.get(self.urls['fut_question'])
        self.logger.debug(rc.content)
        rc = rc.json()
        if rc.get('string') != 'Already answered question.':
            # answer question
            data = {'answer': secret_answer_hash}
            self.r.headers[
                'Content-Type'] = 'application/x-www-form-urlencoded'  # requests bug?
            rc = self.r.post(self.urls['fut_validate'], data=data)
            self.logger.debug(rc.content)
            rc = rc.json()
            if rc['string'] != 'OK':  # we've got error
                if 'Answers do not match' in rc['reason']:
                    raise FutError(
                        'Error during login process (invalid secret answer).')
                else:
                    raise UnknownError
            self.r.headers['Content-Type'] = 'application/json'
        self.r.headers['X-UT-PHISHING-TOKEN'] = self.token = rc['token']

        # prepare headers for ut operations
        del self.r.headers['Easw-Session-Data-Nucleus-Id']
        del self.r.headers['X-Requested-With']
        del self.r.headers['X-UT-Route']
        self.r.headers.update({
            #'X-HTTP-Method-Override': 'GET',  # __request__ method manages this
            'Referer':
            'https://www.easports.com/iframe/fut15/bundles/futweb/web/flash/FifaUltimateTeam.swf',
            'Origin': 'https://www.easports.com',
            #'Content-Type': 'application/json',  # already set
            'Accept': 'application/json',
        })

        # get basic user info
        # TODO: parse response (https://gist.github.com/oczkers/526577572c097eb8172f)
        self.__get__(self.urls['fut']['user'])
        # size of piles
        piles = self.pileSize()
        self.tradepile_size = piles['tradepile']
        self.watchlist_size = piles['watchlist']

        self.saveSession()
class Client:
    """
    Web client class with automatic charset detection and decoding
    """
    def __init__(self,
                 info=None,
                 request_charset='utf-8',
                 response_charset=None):
        self._counter = 0
        self._cookies_filename = ''
        self._cookies = LWPCookieJar()
        self.url = None
        self.user_agent = USER_AGENT
        self.content = None
        self.status = None
        self.token = None
        self.passkey = None
        self.info = info
        self.proxy_url = None
        self.request_charset = request_charset
        self.response_charset = response_charset

        self.needs_proxylock = False

        self.headers = dict()
        self.request_headers = None

        self.session = requests.session()
        self.session.verify = False

        # Enabling retrying on failed requests
        retries = Retry(total=2,
                        read=2,
                        connect=2,
                        redirect=3,
                        backoff_factor=0.1
                        # status_forcelist=[ 500, 502, 503, 504 ])
                        )

        self.session.mount('http://', HTTPAdapter(max_retries=retries))
        self.session.mount('https://', HTTPAdapter(max_retries=retries))
        # self.session = cfscrape.create_scraper()
        # self.scraper = cfscrape.create_scraper()
        # self.session = self.scraper.session()

        global dns_public_list
        global dns_opennic_list
        dns_public_list = get_setting("public_dns_list",
                                      unicode).replace(" ", "").split(",")
        dns_opennic_list = get_setting("opennic_dns_list",
                                       unicode).replace(" ", "").split(",")
        # socket.setdefaulttimeout(60)

        # Parsing proxy information
        proxy = {
            'enabled': get_setting("proxy_enabled", bool),
            'use_type': get_setting("proxy_use_type", int),
            'type': proxy_types[0],
            'host': get_setting("proxy_host", unicode),
            'port': get_setting("proxy_port", int),
            'login': get_setting("proxy_login", unicode),
            'password': get_setting("proxy_password", unicode),
        }

        try:
            proxy['type'] = proxy_types[get_setting("proxy_type", int)]
        except:
            pass

        if get_setting("use_public_dns", bool):
            connection.create_connection = patched_create_connection

        if get_setting("use_elementum_proxy", bool):
            elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
            if elementum_addon and elementum_addon.getSetting(
                    'internal_proxy_enabled') == "true":
                self.proxy_url = "{0}://{1}:{2}".format(
                    "http", "127.0.0.1", "65222")
                if info and "internal_proxy_url" in info:
                    self.proxy_url = info["internal_proxy_url"]

                self.session.proxies = {
                    'http': self.proxy_url,
                    'https': self.proxy_url,
                }
        elif proxy['enabled']:
            if proxy['use_type'] == 0 and info and "proxy_url" in info:
                log.debug("Setting proxy from Elementum: %s" %
                          (info["proxy_url"]))
            elif proxy['use_type'] == 1:
                log.debug("Setting proxy with custom settings: %s" %
                          (repr(proxy)))

                if proxy['login'] or proxy['password']:
                    self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format(
                        proxy['type'], proxy['login'], proxy['password'],
                        proxy['host'], proxy['port'])
                else:
                    self.proxy_url = "{0}://{1}:{2}".format(
                        proxy['type'], proxy['host'], proxy['port'])

            if self.proxy_url:
                self.session.proxies = {
                    'http': self.proxy_url,
                    'https': self.proxy_url,
                }

    def _create_cookies(self, payload):
        return urlencode(payload)

    def _locate_cookies(self, url=''):
        cookies_path = os.path.join(PATH_TEMP, 'burst')
        if not os.path.exists(cookies_path):
            try:
                os.makedirs(cookies_path)
            except Exception as e:
                log.debug("Error creating cookies directory: %s" % repr(e))

        # return os.path.join(cookies_path, urlparse(url).netloc + '_cookies.jar')
        # Do we really need to split cookies for each domain?
        return os.path.join(cookies_path, 'common_cookies.jar')

    def _read_cookies(self, url=''):
        self._cookies_filename = self._locate_cookies(url)
        if os.path.exists(self._cookies_filename):
            try:
                self._cookies.load(self._cookies_filename)
            except Exception as e:
                log.debug("Reading cookies error: %s" % repr(e))

    def _save_cookies(self):
        self._cookies_filename = self._locate_cookies(self.url)

        try:
            self._cookies.save(self._cookies_filename)
        except Exception as e:
            log.debug("Saving cookies error: %s" % repr(e))

    def _good_spider(self):
        self._counter += 1
        if self._counter > 1:
            sleep(0.25)

    def cookies(self):
        """ Saved client cookies

        Returns:
            list: A list of saved Cookie objects
        """
        return self._cookies

    def open(self,
             url,
             language='en',
             post_data=None,
             get_data=None,
             headers=None):
        """ Opens a connection to a webpage and saves its HTML content in ``self.content``

        Args:
            url        (str): The URL to open
            language   (str): The language code for the ``Content-Language`` header
            post_data (dict): POST data for the request
            get_data  (dict): GET data for the request
        """

        if get_data:
            url += '?' + urlencode(get_data)

        log.debug("Opening URL: %s" % repr(url))
        if self.session.proxies:
            log.debug("Proxies: %s" % (repr(self.session.proxies)))

        self._read_cookies(url)
        self.session.cookies = self._cookies

        log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))

        # Default headers for any request. Pretend like we are the usual browser.
        req_headers = {
            'User-Agent': self.user_agent,
            'Content-Language': language,
            'Cache-Control': 'no-cache',
            'Accept-Encoding': 'deflate, compress, gzip',
            'Origin': url,
            'Referer': url
        }

        # If headers passed to open() call - we overwrite headers.
        if headers:
            for key, value in headers.iteritems():
                if value:
                    req_headers[key] = value
                elif key.capitalize() in req_headers:
                    del req_headers[key.capitalize()]

        if self.token:
            req_headers["Authorization"] = self.token

        req = None
        if post_data:
            req = requests.Request('POST',
                                   url,
                                   data=post_data,
                                   headers=req_headers)
        else:
            req = requests.Request('GET', url, headers=req_headers)

        prepped = self.session.prepare_request(req)
        self.request_headers = prepped.headers

        try:
            self._good_spider()
            with self.session.send(prepped) as response:
                self.headers = response.headers
                self.status = response.status_code
                self.url = response.url

                self._save_cookies()

                if self.response_charset:
                    self.content = response.content.decode(
                        self.response_charset, 'ignore')
                else:
                    self.content = response.text

        except Exception as e:
            import traceback
            log.error("%s failed with %s:" % (repr(url), repr(e)))
            map(log.debug, traceback.format_exc().split("\n"))

        log.debug("Status for %s : %s" % (repr(url), str(self.status)))

        return self.status == 200

    def login(self, root_url, url, data, headers, fails_with):
        """ Login wrapper around ``open``

        Args:
            url        (str): The URL to open
            data      (dict): POST login data
            fails_with (str): String that must **not** be included in the response's content

        Returns:
            bool: Whether or not login was successful
        """
        if not url.startswith('http'):
            url = root_url + url

        if self.open(url.encode('utf-8'),
                     post_data=encode_dict(data, self.request_charset),
                     headers=headers):
            try:
                if fails_with in self.content:
                    self.status = 'Wrong username or password'
                    return False
            except Exception as e:
                log.debug("Login failed with: %s" % e)
                try:
                    if fails_with in self.content.decode('utf-8'):
                        self.status = 'Wrong username or password'
                        return False
                except:
                    return False

            return True

        return False
Beispiel #41
0
REFER_URL = URL
URL_AFTER_LOGIN = '******'

USERNAME = '******'
PASSWORD = '******'


# 这个完全可以不用bs库,直接从网页源代码提取
def get_lt(str_):
    f = re.compile(r"name=\"lt\"\svalue=\"(.*)\"\s/>", flags=0)
    return f.findall(str_)[0]


# cookie setting
s = requests.Session()
s.cookies = LWPCookieJar('cookiejar')
header = {
    'User-Agent':
    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0',
    'Referer': REFER_URL
}

payload = {
    'username': USERNAME,
    'password': PASSWORD,
    'execution': 'e1s1',
    'lt': None,
    '_eventId': 'submit'
}

# if cookies existed, or not expried
    def __init__(self,
                 info=None,
                 request_charset='utf-8',
                 response_charset=None):
        self._counter = 0
        self._cookies_filename = ''
        self._cookies = LWPCookieJar()
        self.url = None
        self.user_agent = USER_AGENT
        self.content = None
        self.status = None
        self.token = None
        self.passkey = None
        self.info = info
        self.proxy_url = None
        self.request_charset = request_charset
        self.response_charset = response_charset

        self.needs_proxylock = False

        self.headers = dict()
        self.request_headers = None

        self.session = requests.session()
        self.session.verify = False

        # Enabling retrying on failed requests
        retries = Retry(total=2,
                        read=2,
                        connect=2,
                        redirect=3,
                        backoff_factor=0.1
                        # status_forcelist=[ 500, 502, 503, 504 ])
                        )

        self.session.mount('http://', HTTPAdapter(max_retries=retries))
        self.session.mount('https://', HTTPAdapter(max_retries=retries))
        # self.session = cfscrape.create_scraper()
        # self.scraper = cfscrape.create_scraper()
        # self.session = self.scraper.session()

        global dns_public_list
        global dns_opennic_list
        dns_public_list = get_setting("public_dns_list",
                                      unicode).replace(" ", "").split(",")
        dns_opennic_list = get_setting("opennic_dns_list",
                                       unicode).replace(" ", "").split(",")
        # socket.setdefaulttimeout(60)

        # Parsing proxy information
        proxy = {
            'enabled': get_setting("proxy_enabled", bool),
            'use_type': get_setting("proxy_use_type", int),
            'type': proxy_types[0],
            'host': get_setting("proxy_host", unicode),
            'port': get_setting("proxy_port", int),
            'login': get_setting("proxy_login", unicode),
            'password': get_setting("proxy_password", unicode),
        }

        try:
            proxy['type'] = proxy_types[get_setting("proxy_type", int)]
        except:
            pass

        if get_setting("use_public_dns", bool):
            connection.create_connection = patched_create_connection

        if get_setting("use_elementum_proxy", bool):
            elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
            if elementum_addon and elementum_addon.getSetting(
                    'internal_proxy_enabled') == "true":
                self.proxy_url = "{0}://{1}:{2}".format(
                    "http", "127.0.0.1", "65222")
                if info and "internal_proxy_url" in info:
                    self.proxy_url = info["internal_proxy_url"]

                self.session.proxies = {
                    'http': self.proxy_url,
                    'https': self.proxy_url,
                }
        elif proxy['enabled']:
            if proxy['use_type'] == 0 and info and "proxy_url" in info:
                log.debug("Setting proxy from Elementum: %s" %
                          (info["proxy_url"]))
            elif proxy['use_type'] == 1:
                log.debug("Setting proxy with custom settings: %s" %
                          (repr(proxy)))

                if proxy['login'] or proxy['password']:
                    self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format(
                        proxy['type'], proxy['login'], proxy['password'],
                        proxy['host'], proxy['port'])
                else:
                    self.proxy_url = "{0}://{1}:{2}".format(
                        proxy['type'], proxy['host'], proxy['port'])

            if self.proxy_url:
                self.session.proxies = {
                    'http': self.proxy_url,
                    'https': self.proxy_url,
                }
Beispiel #43
0
ITUNESCONNECT_MAIN_PAGE_URL = '/WebObjects/iTunesConnect.woa'
KEYRING_SERVICE_NAME = 'itc.cli'


class DEVICE_TYPE:
    iPad = 0
    iPhone = 1
    iPhone5 = 2
    deviceStrings = ['iPad', 'iPhone', 'iPhone 5']


temp_dir = gettempdir()
default_file_format = 'images/{language}/{device_type} {index}.png'
cookie_file_name = '.itc-cli-cookies.txt'
cookie_file = os.path.join(temp_dir, cookie_file_name)
cookie_jar = LWPCookieJar(cookie_file)


class ALIASES:
    language_aliases = {}
    device_type_aliases = {}


class config:
    options = {}


def __initCookies():
    if cookie_file:
        try:
            cookie_jar.load(cookie_file, ignore_discard=True)
Beispiel #44
0
class SitesDB(object):
    """
    **EarwigBot: Wiki Toolset: Sites Database Manager**

    This class controls the :file:`sites.db` file, which stores information
    about all wiki sites known to the bot. Three public methods act as bridges
    between the bot's config files and :py:class:`~earwigbot.wiki.site.Site`
    objects:

    - :py:meth:`get_site`:    returns a Site object corresponding to a site
    - :py:meth:`add_site`:    stores a site in the database
    - :py:meth:`remove_site`: removes a site from the database

    There's usually no need to use this class directly. All public methods
    here are available as :py:meth:`bot.wiki.get_site`,
    :py:meth:`bot.wiki.add_site`, and :py:meth:`bot.wiki.remove_site`, which
    use a :file:`sites.db` file located in the same directory as our
    :file:`config.yml` file. Lower-level access can be achieved by importing
    the manager class (``from earwigbot.wiki import SitesDB``).
    """

    def __init__(self, bot):
        """Set up the manager with an attribute for the base Bot object."""
        self.config = bot.config
        self._logger = bot.logger.getChild("wiki")

        self._sites = {}  # Internal site cache
        self._sitesdb = path.join(bot.config.root_dir, "sites.db")
        self._cookie_file = path.join(bot.config.root_dir, ".cookies")
        self._cookiejar = None

        excl_db = path.join(bot.config.root_dir, "exclusions.db")
        excl_logger = self._logger.getChild("exclusionsdb")
        self._exclusions_db = ExclusionsDB(self, excl_db, excl_logger)

    def __repr__(self):
        """Return the canonical string representation of the SitesDB."""
        res = "SitesDB(config={0!r}, sitesdb={1!r}, cookie_file={2!r})"
        return res.format(self.config, self._sitesdb, self._cookie_file)

    def __str__(self):
        """Return a nice string representation of the SitesDB."""
        return "<SitesDB at {0}>".format(self._sitesdb)

    def _get_cookiejar(self):
        """Return a LWPCookieJar object loaded from our .cookies file.

        The same .cookies file is returned every time, located in the project
        root, same directory as config.yml and bot.py. If it doesn't exist, we
        will create the file and set it to be readable and writeable only by
        us. If it exists but the information inside is bogus, we'll ignore it.

        This is normally called by _make_site_object() (in turn called by
        get_site()), and the cookiejar is passed to our Site's constructor,
        used when it makes API queries. This way, we can easily preserve
        cookies between sites (e.g., for CentralAuth), making logins easier.
        """
        if self._cookiejar:
            return self._cookiejar

        self._cookiejar = LWPCookieJar(self._cookie_file)

        try:
            self._cookiejar.load()
        except LoadError:
            pass  # File contains bad data, so ignore it completely
        except IOError as e:
            if e.errno == errno.ENOENT:  # "No such file or directory"
                # Create the file and restrict reading/writing only to the
                # owner, so others can't peak at our cookies:
                open(self._cookie_file, "w").close()
                chmod(self._cookie_file, stat.S_IRUSR|stat.S_IWUSR)
            else:
                raise

        return self._cookiejar

    def _create_sitesdb(self):
        """Initialize the sitesdb file with its three necessary tables."""
        script = """
        CREATE TABLE sites (site_name, site_project, site_lang, site_base_url,
                            site_article_path, site_script_path);
        CREATE TABLE sql_data (sql_site, sql_data_key, sql_data_value);
        CREATE TABLE namespaces (ns_site, ns_id, ns_name, ns_is_primary_name);
        """
        with sqlite.connect(self._sitesdb) as conn:
            conn.executescript(script)

    def _get_site_object(self, name):
        """Return the site from our cache, or create it if it doesn't exist.

        This is essentially just a wrapper around _make_site_object that
        returns the same object each time a specific site is asked for.
        """
        try:
            return self._sites[name]
        except KeyError:
            site = self._make_site_object(name)
            self._sites[name] = site
            return site

    def _load_site_from_sitesdb(self, name):
        """Return all information stored in the sitesdb relating to given site.

        The information will be returned as a tuple, containing the site's
        name, project, language, base URL, article path, script path, SQL
        connection data, and namespaces, in that order. If the site is not
        found in the database, SiteNotFoundError will be raised. An empty
        database will be created before the exception is raised if none exists.
        """
        query1 = "SELECT * FROM sites WHERE site_name = ?"
        query2 = "SELECT sql_data_key, sql_data_value FROM sql_data WHERE sql_site = ?"
        query3 = "SELECT ns_id, ns_name, ns_is_primary_name FROM namespaces WHERE ns_site = ?"
        error = "Site '{0}' not found in the sitesdb.".format(name)
        with sqlite.connect(self._sitesdb) as conn:
            try:
                site_data = conn.execute(query1, (name,)).fetchone()
            except sqlite.OperationalError:
                self._create_sitesdb()
                raise SiteNotFoundError(error)
            if not site_data:
                raise SiteNotFoundError(error)
            sql_data = conn.execute(query2, (name,)).fetchall()
            ns_data = conn.execute(query3, (name,)).fetchall()

        name, project, lang, base_url, article_path, script_path = site_data
        sql = dict(sql_data)
        namespaces = {}
        for ns_id, ns_name, ns_is_primary_name in ns_data:
            try:
                if ns_is_primary_name:  # "Primary" name goes first in list
                    namespaces[ns_id].insert(0, ns_name)
                else:  # Ordering of the aliases doesn't matter
                    namespaces[ns_id].append(ns_name)
            except KeyError:
                namespaces[ns_id] = [ns_name]

        return (name, project, lang, base_url, article_path, script_path, sql,
                namespaces)

    def _make_site_object(self, name):
        """Return a Site object associated with the site *name* in our sitesdb.

        This calls _load_site_from_sitesdb(), so SiteNotFoundError will be
        raised if the site is not in our sitesdb.
        """
        cookiejar = self._get_cookiejar()
        (name, project, lang, base_url, article_path, script_path, sql,
         namespaces) = self._load_site_from_sitesdb(name)

        config = self.config
        login = (config.wiki.get("username"), config.wiki.get("password"))
        user_agent = config.wiki.get("userAgent")
        use_https = config.wiki.get("useHTTPS", False)
        assert_edit = config.wiki.get("assert")
        maxlag = config.wiki.get("maxlag")
        wait_between_queries = config.wiki.get("waitTime", 2)
        logger = self._logger.getChild(name)
        search_config = config.wiki.get("search", OrderedDict()).copy()

        if user_agent:
            user_agent = user_agent.replace("$1", __version__)
            user_agent = user_agent.replace("$2", python_version())

        if search_config:
            nltk_dir = path.join(self.config.root_dir, ".nltk")
            search_config["nltk_dir"] = nltk_dir
            search_config["exclusions_db"] = self._exclusions_db

        if not sql:
            sql = config.wiki.get("sql", OrderedDict()).copy()
            for key, value in sql.iteritems():
                if isinstance(value, basestring) and "$1" in value:
                    sql[key] = value.replace("$1", name)

        return Site(name=name, project=project, lang=lang, base_url=base_url,
                    article_path=article_path, script_path=script_path,
                    sql=sql, namespaces=namespaces, login=login,
                    cookiejar=cookiejar, user_agent=user_agent,
                    use_https=use_https, assert_edit=assert_edit,
                    maxlag=maxlag, wait_between_queries=wait_between_queries,
                    logger=logger, search_config=search_config)

    def _get_site_name_from_sitesdb(self, project, lang):
        """Return the name of the first site with the given project and lang.

        If we can't find the site with the given information, we'll also try
        searching for a site whose base_url contains "{lang}.{project}". There
        are a few sites, like the French Wikipedia, that set their project to
        something other than the expected "wikipedia" ("wikipédia" in this
        case), but we should correctly find them when doing get_site(lang="fr",
        project="wikipedia").

        If the site is not found, return None. An empty sitesdb will be created
        if none exists.
        """
        query1 = "SELECT site_name FROM sites WHERE site_project = ? and site_lang = ?"
        query2 = "SELECT site_name FROM sites WHERE site_base_url LIKE ?"
        with sqlite.connect(self._sitesdb) as conn:
            try:
                site = conn.execute(query1, (project, lang)).fetchone()
                if site:
                    return site[0]
                else:
                    url = "%{0}.{1}%".format(lang, project)
                    site = conn.execute(query2, (url,)).fetchone()
                    return site[0] if site else None
            except sqlite.OperationalError:
                self._create_sitesdb()

    def _add_site_to_sitesdb(self, site):
        """Extract relevant info from a Site object and add it to the sitesdb.

        Works like a reverse _load_site_from_sitesdb(); the site's project,
        language, base URL, article path, script path, SQL connection data, and
        namespaces are extracted from the site and inserted into the sites
        database. If the sitesdb doesn't exist, we'll create it first.
        """
        name = site.name
        sites_data = (name, site.project, site.lang, site._base_url,
                      site._article_path, site._script_path)
        sql_data = [(name, key, val) for key, val in site._sql_data.iteritems()]
        ns_data = []
        for ns_id, ns_names in site._namespaces.iteritems():
            ns_data.append((name, ns_id, ns_names.pop(0), True))
            for ns_name in ns_names:
                ns_data.append((name, ns_id, ns_name, False))

        with sqlite.connect(self._sitesdb) as conn:
            check_exists = "SELECT 1 FROM sites WHERE site_name = ?"
            try:
                exists = conn.execute(check_exists, (name,)).fetchone()
            except sqlite.OperationalError:
                self._create_sitesdb()
            else:
                if exists:
                    conn.execute("DELETE FROM sites WHERE site_name = ?", (name,))
                    conn.execute("DELETE FROM sql_data WHERE sql_site = ?", (name,))
                    conn.execute("DELETE FROM namespaces WHERE ns_site = ?", (name,))
            conn.execute("INSERT INTO sites VALUES (?, ?, ?, ?, ?, ?)", sites_data)
            conn.executemany("INSERT INTO sql_data VALUES (?, ?, ?)", sql_data)
            conn.executemany("INSERT INTO namespaces VALUES (?, ?, ?, ?)", ns_data)

    def _remove_site_from_sitesdb(self, name):
        """Remove a site by name from the sitesdb and the internal cache."""
        try:
            del self._sites[name]
        except KeyError:
            pass

        with sqlite.connect(self._sitesdb) as conn:
            cursor = conn.execute("DELETE FROM sites WHERE site_name = ?", (name,))
            if cursor.rowcount == 0:
                return False
            else:
                conn.execute("DELETE FROM sql_data WHERE sql_site = ?", (name,))
                conn.execute("DELETE FROM namespaces WHERE ns_site = ?", (name,))
                self._logger.info("Removed site '{0}'".format(name))
                return True

    def get_site(self, name=None, project=None, lang=None):
        """Return a Site instance based on information from the sitesdb.

        With no arguments, return the default site as specified by our config
        file. This is ``config.wiki["defaultSite"]``.

        With *name* specified, return the site with that name. This is
        equivalent to the site's ``wikiid`` in the API, like *enwiki*.

        With *project* and *lang* specified, return the site whose project and
        language match these values. If there are multiple sites with the same
        values (unlikely), this is not a reliable way of loading a site. Call
        the function with an explicit *name* in that case.

        We will attempt to login to the site automatically using
        ``config.wiki["username"]`` and ``config.wiki["password"]`` if both are
        defined.

        Specifying a project without a lang or a lang without a project will
        raise :py:exc:`TypeError`. If all three args are specified, *name* will
        be first tried, then *project* and *lang* if *name* doesn't work. If a
        site cannot be found in the sitesdb,
        :py:exc:`~earwigbot.exceptions.SiteNotFoundError` will be raised. An
        empty sitesdb will be created if none is found.
        """
        # Someone specified a project without a lang, or vice versa:
        if (project and not lang) or (not project and lang):
            e = "Keyword arguments 'lang' and 'project' must be specified together."
            raise TypeError(e)

        # No args given, so return our default site:
        if not name and not project and not lang:
            try:
                default = self.config.wiki["defaultSite"]
            except KeyError:
                e = "Default site is not specified in config."
                raise SiteNotFoundError(e)
            return self._get_site_object(default)

        # Name arg given, but don't look at others unless `name` isn't found:
        if name:
            try:
                return self._get_site_object(name)
            except SiteNotFoundError:
                if project and lang:
                    name = self._get_site_name_from_sitesdb(project, lang)
                    if name:
                        return self._get_site_object(name)
                raise

        # If we end up here, then project and lang are the only args given:
        name = self._get_site_name_from_sitesdb(project, lang)
        if name:
            return self._get_site_object(name)
        e = "Site '{0}:{1}' not found in the sitesdb.".format(project, lang)
        raise SiteNotFoundError(e)

    def add_site(self, project=None, lang=None, base_url=None,
                 script_path="/w", sql=None):
        """Add a site to the sitesdb so it can be retrieved with get_site().

        If only a project and a lang are given, we'll guess the *base_url* as
        ``"//{lang}.{project}.org"`` (which is protocol-relative, becoming
        ``"https"`` if *useHTTPS* is ``True`` in config otherwise ``"http"``).
        If this is wrong, provide the correct *base_url* as an argument (in
        which case project and lang are ignored). Most wikis use ``"/w"`` as
        the script path (meaning the API is located at
        ``"{base_url}{script_path}/api.php"`` ->
        ``"//{lang}.{project}.org/w/api.php"``), so this is the default. If
        your wiki is different, provide the script_path as an argument. SQL
        connection settings are guessed automatically using config's template
        value. If this is wrong or not specified, provide a dict of kwargs as
        *sql* and Site will pass it to :py:func:`oursql.connect(**sql)
        <oursql.connect>`, allowing you to make queries with
        :py:meth:`site.sql_query <earwigbot.wiki.site.Site.sql_query>`.

        Returns ``True`` if the site was added successfully or ``False`` if the
        site is already in our sitesdb (this can be done purposefully to update
        old site info). Raises :py:exc:`~earwigbot.exception.SiteNotFoundError`
        if not enough information has been provided to identify the site (e.g.
        a *project* but not a *lang*).
        """
        if not base_url:
            if not project or not lang:
                e = "Without a base_url, both a project and a lang must be given."
                raise SiteNotFoundError(e)
            base_url = "//{0}.{1}.org".format(lang, project)
        cookiejar = self._get_cookiejar()

        config = self.config
        login = (config.wiki.get("username"), config.wiki.get("password"))
        user_agent = config.wiki.get("userAgent")
        use_https = config.wiki.get("useHTTPS", True)
        assert_edit = config.wiki.get("assert")
        maxlag = config.wiki.get("maxlag")
        wait_between_queries = config.wiki.get("waitTime", 2)

        if user_agent:
            user_agent = user_agent.replace("$1", __version__)
            user_agent = user_agent.replace("$2", python_version())

        # Create a Site object to log in and load the other attributes:
        site = Site(base_url=base_url, script_path=script_path, sql=sql,
                    login=login, cookiejar=cookiejar, user_agent=user_agent,
                    use_https=use_https, assert_edit=assert_edit,
                    maxlag=maxlag, wait_between_queries=wait_between_queries)

        self._logger.info("Added site '{0}'".format(site.name))
        self._add_site_to_sitesdb(site)
        return self._get_site_object(site.name)

    def remove_site(self, name=None, project=None, lang=None):
        """Remove a site from the sitesdb.

        Returns ``True`` if the site was removed successfully or ``False`` if
        the site was not in our sitesdb originally. If all three args (*name*,
        *project*, and *lang*) are given, we'll first try *name* and then try
        the latter two if *name* wasn't found in the database. Raises
        :py:exc:`TypeError` if a project was given but not a language, or vice
        versa. Will create an empty sitesdb if none was found.
        """
        # Someone specified a project without a lang, or vice versa:
        if (project and not lang) or (not project and lang):
            e = "Keyword arguments 'lang' and 'project' must be specified together."
            raise TypeError(e)

        if name:
            was_removed = self._remove_site_from_sitesdb(name)
            if not was_removed:
                if project and lang:
                    name = self._get_site_name_from_sitesdb(project, lang)
                    if name:
                        return self._remove_site_from_sitesdb(name)
            return was_removed

        if project and lang:
            name = self._get_site_name_from_sitesdb(project, lang)
            if name:
                return self._remove_site_from_sitesdb(name)

        return False
Beispiel #45
0
# -*- coding: utf-8 -*-
import urllib2
import parse_captcha
import urllib
from urllib2 import Request, urlopen
from cookielib import LWPCookieJar
import os

posturl = "http://ecard.sjtu.edu.cn/loginstudent.action"
picurl = "http://ecard.sjtu.edu.cn/getCheckpic.action"
homeurl = "http://ecard.sjtu.edu.cn/homeLogin.action"
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36"
home_folder = os.getenv('HOME')

cookie_jar = LWPCookieJar(os.path.join(home_folder, '.ecard-cookie'))
try:
    cookie_jar.load()
except Exception:
    pass


def get_page(url, data=""):
    request = Request(url)
    request.add_header('User-Agent', userAgent)
    if data:
        request.add_data(data)
    cookie_jar.add_cookie_header(request)
    response = urlopen(request)
    cookie_jar.extract_cookies(response, request)
    html = response.read()
    response.close()
Beispiel #46
0
    def __login__(self, email, passwd, secret_answer, platform='pc', code=None, emulate=None):
        """Log in.

        :params email: Email.
        :params passwd: Password.
        :params secret_answer: Answer for secret question.
        :params platform: (optional) [pc/xbox/xbox360/ps3/ps4] Platform.
        :params code: (optional) Security code generated in origin or send via mail/sms.
        :params emulate: (optional) [and/ios] Emulate mobile device.
        """
        # TODO: split into smaller methods
        # TODO: check first if login is needed (https://www.easports.com/fifa/api/isUserLoggedIn)
        # TODO: get gamesku, url from shards !!
        self.emulate = emulate
        secret_answer_hash = EAHashingAlgorithm().EAHash(secret_answer)
        # create session
        self.r = requests.Session()  # init/reset requests session object
        # load saved cookies/session
        if self.cookies_file:
            self.r.cookies = LWPCookieJar(self.cookies_file)
            try:
                self.r.cookies.load(ignore_discard=True)  # is it good idea to load discarded cookies after long time?
            except IOError:
                pass
                # self.r.cookies.save(ignore_discard=True)  # create empty file for cookies
        if emulate == 'and':
            self.r.headers = headers_and.copy()  # i'm android now ;-)
        elif emulate == 'ios':
            self.r.headers = headers_ios.copy()  # i'm ios phone now ;-)
        else:
            self.r.headers = headers.copy()  # i'm chrome browser now ;-)
        self.urls = urls(platform)
        # TODO: urls won't be loaded if we drop here
        if platform == 'pc':
            game_sku = 'FFA17PCC'
        elif platform == 'xbox':
            game_sku = 'FFA17XBO'
        elif platform == 'xbox360':
            game_sku = 'FFA17XBX'
        elif platform == 'ps3':
            game_sku = 'FFA17PS3'  # not tested
        elif platform == 'ps4':
            game_sku = 'FFA17PS4'
            platform = 'ps3'  # ps4 not available?
        else:
            raise FutError(reason='Wrong platform. (Valid ones are pc/xbox/xbox360/ps3/ps4)')
        # if self.r.get(self.urls['main_site']+'/fifa/api/isUserLoggedIn', timeout=self.timeout).json()['isLoggedIn']:
        #    return True  # no need to log in again
        # emulate
        if emulate == 'ios':
            sku = 'FUT17IOS'
            clientVersion = 21
        elif emulate == 'and':
            sku = 'FUT17AND'
            clientVersion = 21
#        TODO: need more info about log in procedure in game
#        elif emulate == 'xbox':
#            sku = 'FFA16XBX'  # FFA14CAP ?
#            clientVersion = 1
#        elif emulate == 'ps3':
#            sku = 'FFA16PS3'  # FFA14KTL ?
#            clientVersion = 1
#        elif emulate == 'pc':
#            sku = ''  # dunno
#            clientVersion = 1
        elif not emulate:
            sku = 'FUT17WEB'
            clientVersion = 1
        else:
            raise FutError(reason='Invalid emulate parameter. (Valid ones are and/ios).')  # pc/ps3/xbox/
        # === login
        self.urls['login'] = self.r.get(self.urls['fut_home'], timeout=self.timeout).url
        self.r.headers['Referer'] = self.urls['login']  # prepare headers
        data = {'email': email,
                'password': passwd,
                '_rememberMe': 'on',
                'rememberMe': 'on',
                '_eventId': 'submit'}
        rc = self.r.post(self.urls['login'], data=data, timeout=self.timeout)
        self.logger.debug(rc.content)

        '''  # pops out only on first launch
        if 'FIFA Ultimate Team</strong> needs to update your Account to help protect your gameplay experience.' in rc.text:  # request email/sms code
            self.r.headers['Referer'] = rc.url  # s2
            rc = self.r.post(rc.url.replace('s2', 's3'), {'_eventId': 'submit'}, timeout=self.timeout).content
            self.r.headers['Referer'] = rc.url  # s3
            rc = self.r.post(rc.url, {'twofactorType': 'EMAIL', 'country': 0, 'phoneNumber': '', '_eventId': 'submit'}, timeout=self.timeout)
        '''
        if 'We sent a security code to your' in rc.text or 'Your security code was sent to' in rc.text or 'Enter the 6-digit verification code' in rc.text:  # post code
            # TODO: 'We sent a security code to your email' / 'We sent a security code to your ?'
            # TODO: pick code from codes.txt?
            if not code:
                self.saveSession()
                raise FutError(reason='Error during login process - code is required.')
            self.r.headers['Referer'] = url = rc.url
            # self.r.headers['Upgrade-Insecure-Requests'] = 1  # ?
            # self.r.headers['Origin'] = 'https://signin.ea.com'
            rc = self.r.post(url, {'twofactorCode': code, '_trustThisDevice': 'on', 'trustThisDevice': 'on', '_eventId': 'submit'}, timeout=self.timeout).text
            if 'Incorrect code entered' in rc or 'Please enter a valid security code' in rc:
                raise FutError(reason='Error during login process - provided code is incorrect.')
            self.logger.debug(rc)
            if 'Set Up an App Authenticator' in rc:
                rc = self.r.post(url.replace('s2', 's3'), {'_eventId': 'cancel', 'appDevice': 'IPHONE'}, timeout=self.timeout).text
            self.logger.debug(rc)

        self.r.headers['Referer'] = self.urls['login']
        if self.r.get(self.urls['main_site'] + '/fifa/api/isUserLoggedIn', timeout=self.timeout).json()['isLoggedIn'] is not True:  # TODO: parse error?
            raise FutError(reason='Error during login process (probably invalid email, password or code).')
        # TODO: catch invalid data exception
        # self.nucleus_id = re.search('userid : "([0-9]+)"', rc.text).group(1)  # we'll get it later

        # === lanuch futweb
        self.r.headers['Referer'] = self.urls['fut_home']  # prepare headers
        rc = self.r.get(self.urls['futweb'], timeout=self.timeout)
        self.logger.debug(rc.content)
        rc = rc.text
#        if 'EASW_ID' not in rc:
#            raise FutError(reason='Error during login process (probably invalid email or password).')
        self.nucleus_id = re.search("var EASW_ID = '([0-9]+)';", rc).group(1)
        self.build_cl = re.search("var BUILD_CL = '([0-9]+)';", rc).group(1)
        # self.urls['fut_base'] = re.search("var BASE_FUT_URL = '(https://.+?)';", rc).group(1)
        # self.urls['fut_home'] = re.search("var GUEST_APP_URI = '(http://.+?)';", rc).group(1)

        self.urls = urls(platform, self.build_cl)

        # acc info
        self.r.headers.update({  # prepare headers
            'Content-Type': 'application/json',
            'Accept': 'text/json',
            'Easw-Session-Data-Nucleus-Id': self.nucleus_id,
            'X-UT-Embed-Error': 'true',
            'X-Requested-With': 'XMLHttpRequest',
            'X-UT-Route': self.urls['fut_host'],
            'Referer': self.urls['futweb'],
        })
        rc = self.r.get(self.urls['acc_info'], params={'_': int(time.time() * 1000)}, timeout=self.timeout)
        self.logger.debug(rc.content)
        # pick persona (first valid for given game_sku)
        personas = rc.json()['userAccountInfo']['personas']
        for p in personas:
            # self.clubs = [i for i in p['userClubList']]
            # sort clubs by lastAccessTime (latest first but looks like ea is doing this for us(?))
            # self.clubs.sort(key=lambda i: i['lastAccessTime'], reverse=True)
            for c in p['userClubList']:
                if c['skuAccessList'] and game_sku in c['skuAccessList']:
                    self.persona_id = p['personaId']
                    self.persona_name = p['personaName']
                    break
        if not hasattr(self, 'persona_id') or not hasattr(self, 'persona_name'):
            raise FutError(reason='Error during login process (no persona found).')

        # authorization
        self.r.headers.update({  # prepare headers
            'Accept': 'application/json, text/javascript',
            'Origin': 'http://www.easports.com',
        })
        data = {'isReadOnly': False,
                'sku': sku,
                'clientVersion': clientVersion,
                # 'nuc': self.nucleus_id,
                'nucleusPersonaId': self.persona_id,
                'nucleusPersonaDisplayName': self.persona_name,
                'gameSku': game_sku,
                'nucleusPersonaPlatform': platform,
                'locale': 'en-GB',
                'method': 'authcode',
                'priorityLevel': 4,
                'identification': {'AuthCode': ''}}
        rc = self.r.post(self.urls['fut']['authentication'], data=json.dumps(data), timeout=self.timeout)
        self.logger.debug(rc.content)
        if rc.status_code == 500:
            raise InternalServerError('Servers are probably temporary down.')
        rc = rc.json()
        # self.urls['fut_host'] = '{0}://{1}'.format(rc['protocol']+rc['ipPort'])
        if rc.get('reason') == 'multiple session':
            raise MultipleSession
        elif rc.get('reason') == 'max sessions':
            raise MaxSessions
        elif rc.get('reason') == 'doLogin: doLogin failed':
            raise doLoginFail
        elif rc.get('reason'):
            raise UnknownError(rc.__str__())
        self.r.headers['X-UT-SID'] = self.sid = rc['sid']

        # validate (secret question)
        self.r.headers['Accept'] = 'text/json'  # prepare headers
        del self.r.headers['Origin']
        rc = self.r.get(self.urls['fut_question'], params={'_': int(time.time() * 1000)}, timeout=self.timeout)
        self.logger.debug(rc.content)
        rc = rc.json()
        if rc.get('string') != 'Already answered question.':
            # answer question
            data = {'answer': secret_answer_hash}
            self.r.headers['Content-Type'] = 'application/x-www-form-urlencoded'  # requests bug?
            rc = self.r.post(self.urls['fut_validate'], data=data, timeout=self.timeout)
            self.logger.debug(rc.content)
            rc = rc.json()
            if rc['string'] != 'OK':  # we've got error
                if 'Answers do not match' in rc['reason']:
                    raise FutError(reason='Error during login process (invalid secret answer).')
                else:
                    raise UnknownError
            self.r.headers['Content-Type'] = 'application/json'
        self.r.headers['X-UT-PHISHING-TOKEN'] = self.token = rc['token']

        # prepare headers for ut operations
        del self.r.headers['Easw-Session-Data-Nucleus-Id']
        del self.r.headers['X-Requested-With']
        del self.r.headers['X-UT-Route']
        self.r.headers.update({
            # 'X-HTTP-Method-Override': 'GET',  # __request__ method manages this
            'X-Requested-With': flash_agent,
            'Referer': 'https://www.easports.com/iframe/fut16/bundles/futweb/web/flash/FifaUltimateTeam.swf',
            'Origin': 'https://www.easports.com',
            # 'Content-Type': 'application/json',  # already set
            'Accept': 'application/json',
        })

        # get basic user info
        # TODO: parse response (https://gist.github.com/oczkers/526577572c097eb8172f)
        self.__get__(self.urls['fut']['user'])
        # size of piles
        piles = self.pileSize()
        self.tradepile_size = piles['tradepile']
        self.watchlist_size = piles['watchlist']

        self.saveSession()
Beispiel #47
0
    def searchit(self):
        chk_id = None
        #logger.info('searchterm: %s' % self.searchterm)
        series_search = self.searchterm['series']
        #self.searchterm is a tuple containing series name, issue number, volume and publisher.
        if series_search.startswith('0-Day Comics Pack'):
            torrentid = 22247  #2018
            issue_search = self.searchterm['issue']  #'21'  #Wed
            volume_search = self.searchterm['volume']  #'2'  #2nd month
            publisher_search = None  #'2'  #2nd month
            comic_id = None
        else:
            comic_id = self.searchterm['id']

            annualize = False
            if 'annual' in series_search.lower():
                series_search = re.sub(' annual', '',
                                       series_search.lower()).strip()
                annualize = True
            issue_search = self.searchterm['issue']
            volume_search = self.searchterm['volume']
            publisher_search = self.searchterm['publisher']
            spl = [x for x in self.publisher_list if x in publisher_search]
            for x in spl:
                publisher_search = re.sub(x, '', publisher_search).strip()
            #logger.info('publisher search set to : %s' % publisher_search)

            # lookup the ComicID in the 32p sqlite3 table to pull the series_id to use.
            if comic_id:
                chk_id = helpers.checkthe_id(comic_id)

            if any([chk_id is None, mylar.CONFIG.DEEP_SEARCH_32P is True]):
                #generate the dynamic name of the series here so we can match it up
                as_d = filechecker.FileChecker()
                as_dinfo = as_d.dynamic_replace(series_search)
                mod_series = re.sub('\|', '',
                                    as_dinfo['mod_seriesname']).strip()
                as_puinfo = as_d.dynamic_replace(publisher_search)
                pub_series = as_puinfo['mod_seriesname']

                logger.fdebug('series_search: %s' % series_search)

                if '/' in series_search:
                    series_search = series_search[:series_search.find('/')]
                if ':' in series_search:
                    series_search = series_search[:series_search.find(':')]
                if ',' in series_search:
                    series_search = series_search[:series_search.find(',')]

                logger.fdebug('config.search_32p: %s' %
                              mylar.CONFIG.SEARCH_32P)
                if mylar.CONFIG.SEARCH_32P is False:
                    url = 'https://walksoftly.itsaninja.party/serieslist.php'
                    params = {
                        'series': re.sub('\|', '', mod_series.lower()).strip()
                    }  #series_search}
                    logger.fdebug('search query: %s' %
                                  re.sub('\|', '', mod_series.lower()).strip())
                    try:
                        t = requests.get(
                            url,
                            params=params,
                            verify=True,
                            headers={
                                'USER-AGENT':
                                mylar.USER_AGENT[:mylar.USER_AGENT.find('/') +
                                                 7] +
                                mylar.USER_AGENT[mylar.USER_AGENT.find('(') +
                                                 1]
                            })
                    except requests.exceptions.RequestException as e:
                        logger.warn(e)
                        return "no results"

                    if t.status_code == '619':
                        logger.warn('[%s] Unable to retrieve data from site.' %
                                    t.status_code)
                        return "no results"
                    elif t.status_code == '999':
                        logger.warn(
                            '[%s] No series title was provided to the search query.'
                            % t.status_code)
                        return "no results"

                    try:
                        results = t.json()
                    except:
                        results = t.text

                    if len(results) == 0:
                        logger.warn('No results found for search on 32P.')
                        return "no results"

        with cfscrape.create_scraper() as s:
            s.headers = self.headers
            cj = LWPCookieJar(
                os.path.join(mylar.CONFIG.CACHE_DIR, ".32p_cookies.dat"))
            cj.load()
            s.cookies = cj
            data = []
            pdata = []
            pubmatch = False

            if series_search.startswith('0-Day Comics Pack'):
                data.append({"id": torrentid, "series": series_search})
            else:
                if any([not chk_id, mylar.CONFIG.DEEP_SEARCH_32P is True]):
                    if mylar.CONFIG.SEARCH_32P is True:
                        url = 'https://32pag.es/torrents.php'  #?action=serieslist&filter=' + series_search #&filter=F
                        params = {
                            'action': 'serieslist',
                            'filter': series_search
                        }
                        time.sleep(
                            1)  #just to make sure we don't hammer, 1s pause.
                        t = s.get(url,
                                  params=params,
                                  verify=True,
                                  allow_redirects=True)
                        soup = BeautifulSoup(t.content, "html.parser")
                        results = soup.find_all("a", {"class": "object-qtip"},
                                                {"data-type": "torrentgroup"})

                    for r in results:
                        if mylar.CONFIG.SEARCH_32P is True:
                            torrentid = r['data-id']
                            torrentname = r.findNext(text=True)
                            torrentname = torrentname.strip()
                        else:
                            torrentid = r['id']
                            torrentname = r['series']

                        as_d = filechecker.FileChecker()
                        as_dinfo = as_d.dynamic_replace(torrentname)
                        seriesresult = re.sub(
                            '\|', '', as_dinfo['mod_seriesname']).strip()
                        logger.fdebug(
                            'searchresult: %s --- %s [%s]' %
                            (seriesresult, mod_series, publisher_search))
                        if seriesresult.lower() == mod_series.lower():
                            logger.fdebug('[MATCH] %s [%s]' %
                                          (torrentname, torrentid))
                            data.append({
                                "id": torrentid,
                                "series": torrentname
                            })
                        elif publisher_search.lower() in seriesresult.lower():
                            logger.fdebug('[MATCH] Publisher match.')
                            tmp_torrentname = re.sub(
                                publisher_search.lower(), '',
                                seriesresult.lower()).strip()
                            as_t = filechecker.FileChecker()
                            as_tinfo = as_t.dynamic_replace(tmp_torrentname)
                            if re.sub('\|', '', as_tinfo['mod_seriesname']
                                      ).strip() == mod_series.lower():
                                logger.fdebug('[MATCH] %s [%s]' %
                                              (torrentname, torrentid))
                                pdata.append({
                                    "id": torrentid,
                                    "series": torrentname
                                })
                                pubmatch = True

                    logger.fdebug(
                        '%s series listed for searching that match.' %
                        len(data))
                else:
                    logger.fdebug(
                        'Exact series ID already discovered previously. Setting to : %s [%s]'
                        % (chk_id['series'], chk_id['id']))
                    pdata.append({
                        "id": chk_id['id'],
                        "series": chk_id['series']
                    })
                    pubmatch = True

            if all([len(data) == 0, len(pdata) == 0]):
                return "no results"
            else:
                dataset = []
                if len(data) > 0:
                    dataset += data
                if len(pdata) > 0:
                    dataset += pdata
                logger.fdebug(
                    str(len(dataset)) +
                    ' series match the tile being searched for on 32P...')

            if all([
                    chk_id is None,
                    not series_search.startswith('0-Day Comics Pack')
            ]) and any([len(data) == 1, len(pdata) == 1]):
                #update the 32p_reference so we avoid doing a url lookup next time
                helpers.checkthe_id(comic_id, dataset)
            else:
                logger.debug(
                    'Unable to properly verify reference on 32P - will update the 32P reference point once the issue has been successfully matched against.'
                )

            results32p = []
            resultlist = {}

            for x in dataset:
                #for 0-day packs, issue=week#, volume=month, id=0-day year pack (ie.issue=21&volume=2 for feb.21st)
                payload = {
                    "action": "groupsearch",
                    "id": x['id'],  #searchid,
                    "issue": issue_search
                }
                #in order to match up against 0-day stuff, volume has to be none at this point
                #when doing other searches tho, this should be allowed to go through
                #if all([volume_search != 'None', volume_search is not None]):
                #    payload.update({'volume': re.sub('v', '', volume_search).strip()})
                if series_search.startswith('0-Day Comics Pack'):
                    payload.update({"volume": volume_search})

                payload = json.dumps(payload)
                payload = json.loads(payload)

                logger.fdebug('payload: %s' % payload)
                url = 'https://32pag.es/ajax.php'
                time.sleep(1)  #just to make sure we don't hammer, 1s pause.
                try:
                    d = s.get(url,
                              params=payload,
                              verify=True,
                              allow_redirects=True)
                except Exception as e:
                    logger.error('%s [%s] Could not POST URL %s' %
                                 (self.module, e, url))

                try:
                    searchResults = d.json()
                except:
                    searchResults = d.text
                    logger.debug(
                        '%s Search Result did not return valid JSON, falling back on text: %s'
                        % (self.module, searchResults.text))
                    return False

                if searchResults[
                        'status'] == 'success' and searchResults['count'] > 0:
                    logger.fdebug('successfully retrieved %s search results' %
                                  searchResults['count'])
                    for a in searchResults['details']:
                        if series_search.startswith('0-Day Comics Pack'):
                            title = series_search
                        else:
                            title = self.searchterm['series'] + ' v' + a[
                                'volume'] + ' #' + a['issues']
                        results32p.append({
                            'link':
                            a['id'],
                            'title':
                            title,
                            'filesize':
                            a['size'],
                            'issues':
                            a['issues'],
                            'pack':
                            a['pack'],
                            'format':
                            a['format'],
                            'language':
                            a['language'],
                            'seeders':
                            a['seeders'],
                            'leechers':
                            a['leechers'],
                            'scanner':
                            a['scanner'],
                            'chkit': {
                                'id': x['id'],
                                'series': x['series']
                            },
                            'pubdate':
                            datetime.datetime.fromtimestamp(
                                float(a['upload_time'])).strftime(
                                    '%a, %d %b %Y %H:%M:%S'),
                            'int_pubdate':
                            float(a['upload_time'])
                        })

                else:
                    logger.fdebug(
                        '32P did not return any valid search results.')

            if len(results32p) > 0:
                resultlist['entries'] = sorted(results32p,
                                               key=itemgetter('pack', 'title'),
                                               reverse=False)
                logger.debug('%s Resultslist: %s' % (self.module, resultlist))
            else:
                resultlist = 'no results'

        return resultlist
Beispiel #48
0
class Client:
    """
    Web client class with automatic charset detection and decoding
    """
    def __init__(self):
        self._counter = 0
        self._cookies_filename = ''
        self._cookies = LWPCookieJar()
        self.user_agent = USER_AGENT
        self.clearance = None
        self.content = None
        self.status = None
        self.token = None
        self.passkey = None
        self.headers = dict()
        global dns_public_list
        global dns_opennic_list
        dns_public_list = get_setting("public_dns_list",
                                      unicode).replace(" ", "").split(",")
        dns_opennic_list = get_setting("opennic_dns_list",
                                       unicode).replace(" ", "").split(",")
        socket.setdefaulttimeout(60)

    def _create_cookies(self, payload):
        return urlencode(payload)

    def _read_cookies(self, url=''):
        cookies_path = os.path.join(PATH_TEMP, 'burst')
        if not os.path.exists(cookies_path):
            try:
                os.makedirs(cookies_path)
            except Exception as e:
                log.debug("Error creating cookies directory: %s" % repr(e))
        self._cookies_filename = os.path.join(
            cookies_path,
            urlparse(url).netloc + '_cookies.jar')
        if os.path.exists(self._cookies_filename):
            try:
                self._cookies.load(self._cookies_filename)
            except Exception as e:
                log.debug("Reading cookies error: %s" % repr(e))

        # Check for cf_clearance cookie
        # https://github.com/scakemyer/cloudhole-api
        if self.clearance and not any(cookie.name == 'cf_clearance'
                                      for cookie in self._cookies):
            c = Cookie(version=None,
                       name='cf_clearance',
                       value=self.clearance[13:],
                       port=None,
                       port_specified=False,
                       domain='.{uri.netloc}'.format(uri=urlparse(url)),
                       domain_specified=True,
                       domain_initial_dot=True,
                       path='/',
                       path_specified=True,
                       secure=False,
                       expires=None,
                       discard=False,
                       comment=None,
                       comment_url=None,
                       rest=None,
                       rfc2109=False)
            self._cookies.set_cookie(c)

    def _save_cookies(self):
        try:
            self._cookies.save(self._cookies_filename)
        except Exception as e:
            log.debug("Saving cookies error: %s" % repr(e))

    def _good_spider(self):
        self._counter += 1
        if self._counter > 1:
            sleep(0.25)

    def cookies(self):
        """ Saved client cookies

        Returns:
            list: A list of saved Cookie objects
        """
        return self._cookies

    def open(self,
             url,
             language='en',
             post_data=None,
             get_data=None,
             headers=None,
             proxy_url=None,
             charset='utf8'):
        """ Opens a connection to a webpage and saves its HTML content in ``self.content``

        Args:
            url        (str): The URL to open
            language   (str): The language code for the ``Content-Language`` header
            post_data (dict): POST data for the request
            get_data  (dict): GET data for the request
        """
        if not post_data:
            post_data = {}
        if get_data:
            url += '?' + urlencode(get_data)

        log.debug("Opening URL: %s" % repr(url))
        result = False

        data = urlencode(post_data) if len(post_data) > 0 else None
        req = urllib2.Request(url, data)

        self._read_cookies(url)
        log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))

        # Parsing proxy information
        proxy = {
            'enabled': get_setting("proxy_enabled", bool),
            'use_type': get_setting("proxy_use_type", int),
            'type': proxy_types[0],
            'host': get_setting("proxy_host", unicode),
            'port': get_setting("proxy_port", int),
            'login': get_setting("proxy_login", unicode),
            'password': get_setting("proxy_password", unicode),
        }

        try:
            proxy['type'] = proxy_types[get_setting("proxy_type", int)]
        except:
            pass

        handlers = [urllib2.HTTPCookieProcessor(self._cookies)]

        if get_setting("use_public_dns", bool):
            handlers.append(MyHTTPHandler)

        if proxy['enabled']:
            if proxy['use_type'] == 0 and proxy_url:
                log.debug("Setting proxy from Elementum: %s" % (proxy_url))
                handlers.append(parse_proxy_url(proxy_url))
            elif proxy['use_type'] == 1:
                log.debug("Setting proxy with custom settings: %s" %
                          (repr(proxy)))
                handlers.append(
                    SocksiPyHandler(proxytype=proxy['type'],
                                    proxyaddr=proxy['host'],
                                    proxyport=int(proxy['port']),
                                    username=proxy['login'],
                                    password=proxy['password'],
                                    rdns=True))
            elif proxy['use_type'] == 2:
                try:
                    handlers.append(antizapret.AntizapretProxyHandler())
                except Exception as e:
                    log.info("Could not create antizapret configuration: %s" %
                             (e))

        opener = urllib2.build_opener(*handlers)

        req.add_header('User-Agent', self.user_agent)
        req.add_header('Content-Language', language)
        req.add_header("Accept-Encoding", "gzip")
        req.add_header("Origin", url)
        req.add_header("Referer", url)

        if headers:
            for key, value in headers.iteritems():
                if value:
                    req.add_header(key, value)
                else:
                    del req.headers[key.capitalize()]

        if self.token:
            req.add_header("Authorization", self.token)

        try:
            self._good_spider()
            with closing(opener.open(req)) as response:
                self.headers = response.headers
                self._save_cookies()
                if response.headers.get("Content-Encoding", "") == "gzip":
                    import zlib
                    self.content = zlib.decompressobj(
                        16 + zlib.MAX_WBITS).decompress(response.read())
                else:
                    self.content = response.read()

                charset = response.headers.getparam('charset')

                if not charset:
                    match = re.search(
                        """<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""",
                        self.content)
                    if match:
                        charset = match.group(1)

                # We try to remove non-utf chars. Should we?
                if (charset and charset.lower() == 'utf-8') or charset is None:
                    charset = 'utf-8-sig'  # Changing to utf-8-sig to remove BOM if found on decode from utf-8

                if charset:
                    log.debug('Decoding charset from %s for %s' %
                              (charset, repr(url)))
                    self.content = self.content.decode(charset, 'replace')

                self.status = response.getcode()
            result = True

        except urllib2.HTTPError as e:
            self.status = e.code
            log.warning("Status for %s : %s" % (repr(url), str(self.status)))
            if e.code == 403 or e.code == 503:
                log.warning("CloudFlared at %s, try enabling CloudHole" % url)

        except urllib2.URLError as e:
            self.status = repr(e.reason)
            log.warning("Status for %s : %s" % (repr(url), self.status))

        except Exception as e:
            import traceback
            log.error("%s failed with %s:" % (repr(url), repr(e)))
            map(log.debug, traceback.format_exc().split("\n"))

        log.debug("Status for %s : %s" % (repr(url), str(self.status)))

        return result

    def login(self, url, data, fails_with, charset='utf8'):
        """ Login wrapper around ``open``

        Args:
            url        (str): The URL to open
            data      (dict): POST login data
            fails_with (str): String that must **not** be included in the response's content

        Returns:
            bool: Whether or not login was successful
        """
        result = False
        if self.open(url.encode('utf-8'), post_data=encode_dict(data,
                                                                charset)):
            result = True
            try:
                if fails_with in self.content:
                    self.status = 'Wrong username or password'
                    result = False
            except Exception as e:
                log.debug("Login failed with: %s" % e)
                try:
                    if fails_with in self.content.decode('utf-8'):
                        self.status = 'Wrong username or password'
                        result = False
                except:
                    result = False

        return result
Beispiel #49
0
class NowInventory(object):
    def __init__(self,
                 hostname,
                 username,
                 password,
                 fields=None,
                 groups=None,
                 selection=None,
                 proxy=None):
        self.hostname = hostname

        # requests session
        self.session = requests.Session()

        self.auth = requests.auth.HTTPBasicAuth(username, password)
        # request headers
        self.headers = {
            "Accept": "application/json",
            "Content-Type": "application/json",
        }

        # request cookies
        self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session")
        try:
            self.cookies.load(ignore_discard=True)
        except IOError:
            pass
        self.session.cookies = self.cookies

        if fields is None:
            fields = []

        if groups is None:
            groups = []

        if selection is None:
            selection = []

        if proxy is None:
            proxy = []

        # extra fields (table columns)
        self.fields = fields

        # extra groups (table columns)
        self.groups = groups

        # selection order
        self.selection = selection

        # proxy settings
        self.proxy = proxy

        # initialize inventory
        self.inventory = {'_meta': {'hostvars': {}}}

        return

    def _put_cache(self, name, value):
        cache_dir = os.environ.get('SN_CACHE_DIR')
        if not cache_dir and config.has_option('defaults', 'cache_dir'):
            cache_dir = os.path.expanduser(config.get('defaults', 'cache_dir'))
        if cache_dir:
            if not os.path.exists(cache_dir):
                os.makedirs(cache_dir)
            cache_file = os.path.join(cache_dir, name)
            with open(cache_file, 'w') as cache:
                json.dump(value, cache)

    def _get_cache(self, name, default=None):
        cache_dir = os.environ.get('SN_CACHE_DIR')
        if not cache_dir and config.has_option('defaults', 'cache_dir'):
            cache_dir = config.get('defaults', 'cache_dir')
        if cache_dir:
            cache_file = os.path.join(cache_dir, name)
            if os.path.exists(cache_file):
                cache_max_age = os.environ.get('SN_CACHE_MAX_AGE')
                if not cache_max_age:
                    if config.has_option('defaults', 'cache_max_age'):
                        cache_max_age = config.getint('defaults',
                                                      'cache_max_age')
                    else:
                        cache_max_age = 0
                cache_stat = os.stat(cache_file)
                if (cache_stat.st_mtime + int(cache_max_age)) >= time.time():
                    with open(cache_file) as cache:
                        return json.load(cache)
        return default

    def __del__(self):
        self.cookies.save(ignore_discard=True)

    def _invoke(self, verb, path, data):

        cache_name = '__snow_inventory__'
        if not SN_TOWER:
            inventory = self._get_cache(cache_name, None)
            if inventory is not None:
                return inventory

        # build url
        url = "https://%s/%s" % (self.hostname, path)

        # perform REST operation
        response = self.session.get(url,
                                    auth=self.auth,
                                    headers=self.headers,
                                    proxies={
                                        'http': self.proxy,
                                        'https': self.proxy
                                    })
        if response.status_code != 200:
            print >> sys.stderr, "http error (%s): %s" % (response.status_code,
                                                          response.text)

        if not SN_TOWER:
            self._put_cache(cache_name, response.json())
        return response.json()

    def add_group(self, target, group):
        ''' Transform group names:
                        1. lower()
                        2. non-alphanumerical characters to '_'
        '''

        # Ignore empty group names
        if group == '' or group is None:
            return

        group = group.lower()
        group = re.sub(r' ', '_', group)

        self.inventory.setdefault(group, {'hosts': []})
        self.inventory[group]['hosts'].append(target)
        return

    def add_var(self, target, key, val):
        if target not in self.inventory['_meta']['hostvars']:
            self.inventory['_meta']['hostvars'][target] = {}

        self.inventory['_meta']['hostvars'][target]["sn_" + key] = val
        return

    def generate(self):

        table = 'cmdb_ci_server'
        # table = 'cmdb_ci_linux_server'
        base_fields = [
            u'name', u'host_name', u'fqdn', u'ip_address', u'sys_class_name'
        ]
        base_groups = [u'sys_class_name']
        options = "?sysparm_exclude_reference_link=true&sysparm_display_value=true"

        columns = list(
            set(base_fields + base_groups + self.fields + self.groups))
        path = '/api/now/table/' + table + options + \
            "&sysparm_fields=" + ','.join(columns)

        # Default, mandatory group 'sys_class_name'
        groups = list(set(base_groups + self.groups))

        content = self._invoke('GET', path, None)

        for record in content['result']:
            ''' Ansible host target selection order:
                        1. ip_address
                        2. fqdn
                        3. host_name

                        '''
            target = None

            selection = self.selection

            if not selection:
                selection = ['host_name', 'fqdn', 'ip_address']
            for k in selection:
                if record[k] != '':
                    target = record[k]

            # Skip if no target available
            if target is None:
                continue

            # hostvars
            for k in record.keys():
                self.add_var(target, k, record[k])

            # groups
            for k in groups:
                self.add_group(target, record[k])

        return

    def json(self):
        return json.dumps(self.inventory)
Beispiel #50
0
class Client:
    """
    Web client class with automatic charset detection and decoding
    """
    def __init__(self):
        self._counter = 0
        self._cookies_filename = ''
        self._cookies = LWPCookieJar()
        self.user_agent = USER_AGENT
        self.clearance = None
        self.content = None
        self.status = None
        self.token = None
        self.passkey = None
        self.headers = dict()

    def _create_cookies(self, payload):
        return urlencode(payload)

    def _read_cookies(self, url=''):
        cookies_path = os.path.join(PATH_TEMP, 'burst')
        if not os.path.exists(cookies_path):
            try:
                os.makedirs(cookies_path)
            except Exception as e:
                log.debug("Error creating cookies directory: %s" % repr(e))
        self._cookies_filename = os.path.join(
            cookies_path,
            urlparse(url).netloc + '_cookies.jar')
        if os.path.exists(self._cookies_filename):
            try:
                self._cookies.load(self._cookies_filename)
            except Exception as e:
                log.debug("Reading cookies error: %s" % repr(e))

        # Check for cf_clearance cookie
        # https://github.com/scakemyer/cloudhole-api
        if self.clearance and not any(cookie.name == 'cf_clearance'
                                      for cookie in self._cookies):
            c = Cookie(version=None,
                       name='cf_clearance',
                       value=self.clearance[13:],
                       port=None,
                       port_specified=False,
                       domain='.{uri.netloc}'.format(uri=urlparse(url)),
                       domain_specified=True,
                       domain_initial_dot=True,
                       path='/',
                       path_specified=True,
                       secure=False,
                       expires=None,
                       discard=False,
                       comment=None,
                       comment_url=None,
                       rest=None,
                       rfc2109=False)
            self._cookies.set_cookie(c)

    def _save_cookies(self):
        try:
            self._cookies.save(self._cookies_filename)
        except Exception as e:
            log.debug("Saving cookies error: %s" % repr(e))

    def _good_spider(self):
        self._counter += 1
        if self._counter > 1:
            sleep(0.25)

    def cookies(self):
        """ Saved client cookies

        Returns:
            list: A list of saved Cookie objects
        """
        return self._cookies

    def open(self, url, language='en', post_data=None, get_data=None):
        """ Opens a connection to a webpage and saves its HTML content in ``self.content``

        Args:
            url        (str): The URL to open
            language   (str): The language code for the ``Content-Language`` header
            post_data (dict): POST data for the request
            get_data  (dict): GET data for the request
        """
        if not post_data:
            post_data = {}
        if get_data:
            url += '?' + urlencode(get_data)

        log.debug("Opening URL: %s" % repr(url))
        result = False

        data = urlencode(post_data) if len(post_data) > 0 else None
        req = urllib2.Request(url, data)

        self._read_cookies(url)
        log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))

        opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self._cookies))
        if get_setting("use_public_dns", bool):
            opener = urllib2.build_opener(
                urllib2.HTTPCookieProcessor(self._cookies), MyHTTPHandler)
        urllib2.install_opener(opener)

        req.add_header('User-Agent', self.user_agent)
        req.add_header('Content-Language', language)
        req.add_header("Accept-Encoding", "gzip")
        req.add_header("Origin", url)
        req.add_header("Referer", url)

        if self.token:
            req.add_header("Authorization", self.token)

        try:
            self._good_spider()
            with closing(opener.open(req)) as response:
                self.headers = response.headers
                self._save_cookies()
                if response.headers.get("Content-Encoding", "") == "gzip":
                    import zlib
                    self.content = zlib.decompressobj(
                        16 + zlib.MAX_WBITS).decompress(response.read())
                else:
                    self.content = response.read()

                charset = response.headers.getparam('charset')

                if not charset:
                    match = re.search(
                        """<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""",
                        self.content)
                    if match:
                        charset = match.group(1)

                if charset and charset.lower() == 'utf-8':
                    charset = 'utf-8-sig'  # Changing to utf-8-sig to remove BOM if found on decode from utf-8

                if charset:
                    log.debug('Decoding charset from %s for %s' %
                              (charset, repr(url)))
                    self.content = self.content.decode(charset, 'replace')

                self.status = response.getcode()
            result = True

        except urllib2.HTTPError as e:
            self.status = e.code
            log.warning("Status for %s : %s" % (repr(url), str(self.status)))
            if e.code == 403 or e.code == 503:
                log.warning("CloudFlared at %s, try enabling CloudHole" % url)

        except urllib2.URLError as e:
            self.status = repr(e.reason)
            log.warning("Status for %s : %s" % (repr(url), self.status))

        except Exception as e:
            import traceback
            log.error("%s failed with %s:" % (repr(url), repr(e)))
            map(log.debug, traceback.format_exc().split("\n"))

        log.debug("Status for %s : %s" % (repr(url), str(self.status)))

        return result

    def login(self, url, data, fails_with):
        """ Login wrapper around ``open``

        Args:
            url        (str): The URL to open
            data      (dict): POST login data
            fails_with (str): String that must **not** be included in the response's content

        Returns:
            bool: Whether or not login was successful
        """
        result = False
        if self.open(url.encode('utf-8'), post_data=encode_dict(data)):
            result = True
            if fails_with in self.content:
                self.status = 'Wrong username or password'
                result = False
        return result
class Browser:
    _cookies = None
    cookies = LWPCookieJar()
    content = None
    status = None
    headers = ""

    def __init__(self):
        pass

    @classmethod
    def create_cookies(cls, payload):

        cls._cookies = urlencode(payload)

    # to open any web page
    @classmethod
    def open(cls, url='', language='en', post_data=None, get_data=None):
        if post_data is None:
            post_data = {}
        if get_data is not None:
            url += '?' + urlencode(get_data)
        print(url)
        result = True
        if len(post_data) > 0:
            cls.create_cookies(post_data)
        if cls._cookies is not None:
            req = urllib2.Request(url, cls._cookies)
            cls._cookies = None
        else:
            req = urllib2.Request(url)
        req.add_header('User-Agent', USER_AGENT)
        req.add_header('Content-Language', language)
        req.add_header("Accept-Encoding", "gzip")
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cls.cookies))  # open cookie jar
        try:
            response = opener.open(req)  # send cookies and open url
            cls.headers = response.headers
            # borrow from provider.py Steeve
            if response.headers.get("Content-Encoding", "") == "gzip":
                import zlib
                cls.content = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(response.read())
            else:
                cls.content = response.read()
            response.close()
            cls.status = 200
        except urllib2.HTTPError as e:
            cls.status = e.code
            result = False
            # if e.code == 503:
            #     # trying to open with antibots tool
            #     import cfscrape
            #     scraper = cfscrape.create_scraper()  # returns a CloudflareScraper instance
            #     cls.content = scraper.get(url).content
            #     cls.status = 200
            #     result = True
        except urllib2.URLError as e:
            cls.status = e.reason
            result = False
        log.debug("Status: " + str(cls.status))
        log.debug(cls.content)
        return result

    # alternative when it is problem with https
    @classmethod
    def open2(cls, url=''):
        import httplib

        word = url.split("://")
        pos = word[1].find("/")
        conn = httplib.HTTPConnection(re.search[:pos])
        conn.request("GET", re.search[pos:])
        r1 = conn.getresponse()
        cls.status = str(r1.status) + " " + r1.reason
        cls.content = r1.read()
        if r1.status == 200:
            return True
        else:
            return False

    # used for sites with login
    @classmethod
    def login(cls, url, payload, word):
        result = False
        cls.create_cookies(payload)
        if cls.open(url):
            result = True
            data = cls.content
            if word in data:
                cls.status = 'Wrong Username or Password'
                result = False
        return result

    def json(self):
        return loads(self.content)
Beispiel #52
0
class Browser:
    """
    Mini Web Browser with cookies handle
    """
    _counter = 0
    _cookies_filename = ''
    _cookies = LWPCookieJar()
    user_agent = USER_AGENT
    clearance = None
    content = None
    raw_content = None
    status = None
    headers = dict()

    def __init__(self):
        pass

    @classmethod
    def _create_cookies(cls, payload):
        return urlencode(payload)

    @classmethod
    def _read_cookies(cls, url=''):
        cls._cookies_filename = path.join(
            PATH_TEMP,
            urlparse(url).netloc + '_cookies.jar')
        if path.exists(cls._cookies_filename):
            try:
                cls._cookies.load(cls._cookies_filename)
            except Exception as e:
                logger.debug("Reading cookies error: %s" % repr(e))

        # Check for cf_clearance cookie provided by scakemyer
        # https://github.com/scakemyer/cloudhole-api
        if cls.clearance and not any(cookie.name == 'cf_clearance'
                                     for cookie in cls._cookies):
            t = str(int(time()) + 604800)
            c = Cookie(None, 'cf_clearance', cls.clearance[13:], None, False,
                       '.{uri.netloc}'.format(uri=urlparse(url)), True, True,
                       '/', True, False, t, False, None, None, None, False)
            cls._cookies.set_cookie(c)

    @classmethod
    def _save_cookies(cls):
        try:
            cls._cookies.save(cls._cookies_filename)
        except Exception as e:
            logger.debug("Saving cookies error: %s" % repr(e))

    @classmethod
    def _good_spider(cls):
        """
        Delay of 0.5 seconds to to call too many requests per second. Some pages start to block
        """
        cls._counter += 1
        if cls._counter > 1:
            sleep(0.5)  # good spider

    @classmethod
    def cookies(cls):
        """
        Cookies
        :return: LWPCookieJar format.
        """
        return cls._cookies

    @classmethod
    def open(cls,
             url='',
             language='en',
             post_data=None,
             get_data=None,
             use_cache=True,
             charset='utf-8'):
        """
        Open a web page and returns its contents
        :param charset:
        :param use_cache: if it uses the information stored in the cache
        :type use_cache: bool
        :param url: url address web page
        :type url: str or unicode
        :param language: language encoding web page
        :type language: str
        :param post_data: parameters for POST request
        :type post_data: dict
        :param get_data: parameters for GET request
        :type get_data: dict
        :return: True if the web page was opened successfully. False, otherwise.
        """
        if len(url) == 0:
            cls.status = 404
            cls.raw_content = ''
            logger.debug('Empty url')
            return False

        # Check the cache
        cache_file = quote_plus(repr(url)) + '.cache'
        if use_cache:
            cache = Storage.open(cache_file, ttl=15)
            if 'content' in cache:
                cls.status = 200
                cls.content = cache['content']
                cls.raw_content = cache['raw_content']
                cls.headers = cache['headers']
                logger.debug('Using cache for %s' % url)
                cache.close()
                logger.debug("Status: " + str(cls.status))
                logger.debug(repr(cls.content))
                return True

        # Creating request
        if post_data is None:
            post_data = {}
        if get_data is not None:
            url += '?' + urlencode(get_data)

        logger.debug(url)
        result = True
        cls.status = 200
        data = urlencode(post_data) if len(post_data) > 0 else None
        req = urllib2.Request(url, data)
        # Cookies and cloudhole info
        cls._read_cookies(url)
        logger.debug("Cookies: %s" % repr(cls._cookies))
        # open cookie jar
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(
            cls._cookies))
        # Headers
        req.add_header('User-Agent', cls.user_agent)
        req.add_header('Content-Language', language)
        req.add_header("Accept-Encoding", "gzip")
        try:
            cls._good_spider()
            # send cookies and open url
            with closing(opener.open(req)) as response:
                cls.headers = response.headers
                cls._save_cookies()
                # borrow from provider.py Steeve
                if response.headers.get('Content-Encoding', '') == 'gzip':
                    import zlib
                    cls.raw_content = zlib.decompressobj(
                        16 + zlib.MAX_WBITS).decompress(response.read())

                else:
                    cls.raw_content = response.read()

                logger.debug("Raw Content:")
                logger.debug(repr(cls.raw_content))
                if 'charset' in cls.raw_content:
                    match = re.search('charset=(\S+)"', cls.raw_content,
                                      re.IGNORECASE)
                    if match:
                        charset = match.group(1)

                logger.debug('charset=' + charset)

        except urllib2.HTTPError as e:
            result = False
            cls.status = e.code
            logger.warning("Status: " + str(cls.status))
            if e.code == 403:
                logger.warning("CloudFlared at %s" % url)

        except urllib2.URLError as e:
            result = False
            cls.status = e.reason
            logger.warning("Status: " + str(cls.status))

        except Exception as e:
            result = False
            logger.error("Error in the browser: %s" % repr(e))

        if result:
            # Normalization
            try:
                cls.content = cls.raw_content.decode(charset)

            except UnicodeDecodeError:
                cls.content = normalize_string(cls.raw_content,
                                               charset=charset)

            except LookupError:
                cls.content = normalize_string(cls.raw_content)

            cls.content = cls.content.replace('<![CDATA[',
                                              '').replace(']]', '')
            cls.content = HTMLParser().unescape(cls.content)
            # Save in the cache
            if use_cache:
                cache = Storage.open(cache_file, ttl=15)
                cache['content'] = cls.content
                cache['raw_content'] = cls.raw_content
                cache['headers'] = cls.headers
                cache.close()

            # status
            logger.debug("Status: " + str(cls.status))
            logger.debug(repr(cls.content))

        return result

    @classmethod
    def login(cls, url='', payload=None, word=''):
        """
        Login to web site
        :param url:  url address from web site
        :type url: str
        :param payload: parameters for the login request
        :type payload: dict
        :param word:  message from the web site when the login fails
        :type word: str
        :return: True if the login was successful. False, otherwise.
        """
        result = False
        if cls.open(url, post_data=payload):
            result = True
            data = cls.raw_content
            if word in data:
                cls.status = 'Wrong Username or Password'
                result = False

        return result
Beispiel #53
0
favdatei = os.path.join(seventv_favorites, masterNEW)
baseURL = "https://www.7tv.de"

xbmcplugin.setContent(pluginhandle, 'tvshows')

if not xbmcvfs.exists(seventv_favorites):
	xbmcvfs.mkdirs(seventv_favorites)

if xbmcvfs.exists(source):
	xbmcvfs.copy(source, favdatei)

if xbmcvfs.exists(temp) and os.path.isdir(temp):
	shutil.rmtree(temp, ignore_errors=True)
xbmcvfs.mkdirs(temp)
cookie = os.path.join(temp, 'cookie.lwp')
cj = LWPCookieJar()

if xbmcvfs.exists(cookie):
	cj.load(cookie, ignore_discard=True, ignore_expires=True)

def py2_enc(s, encoding='utf-8'):
	if PY2 and isinstance(s, unicode):
		s = s.encode(encoding)
	return s

def py2_uni(s, encoding='utf-8'):
	if PY2 and isinstance(s, str):
		s = unicode(s, encoding)
	return s

def py3_dec(d, encoding='utf-8'):
Beispiel #54
0
class MsgAutoSender(object):

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        "Accept-Language": "en-US,en;q=0.5",
        #"Cookie": "; ".join(map(lambda x: "=".join(x), data.items()))
    }

    messages = ["你好,你的眼睛很好看哦[调皮],可以交个朋友吗[微笑]?",
                "你的眼睛很漂亮哦,可以和你交个朋友吗[微笑]?",
                "很喜欢你的眼睛,可以和你交个朋友吗[微笑]?",
                "你的眼睛很好看嘛,可以认识一下交个朋友吗[微笑]?"]

    data = {
        'txtLoginEMail': "",
        'txtLoginPwd': "",
        'chkRememberMe': "",
        'codeId': "",
        'codeValue': '',
        'event':'3',
        'spmp':'4.20.53.225.685',
        '_': "%d"%(time.time()*1000)
    }

    # 预登陆url
    url1 = 'http://my.baihe.com/Getinterlogin/gotoLogin?jsonCallBack=jQuery18308807729283968166_%d&'%(time.time()*1000)
    # 登陆成功后跳转到主页
    url2 = "http://u.baihe.com/?userid=&time=%d"%(time.time()*1000)
    # 用来获取一些默认的搜索条件(百合会根据你个人信息筛选出一些基本符合你要求的人群)
    url3 = "http://search.baihe.com/mystruts/nextSolrSearch.action?jsoncallback=jQuery183042376943520885857_1479472584212&callType=next&pageId=%%s&ord=1&_=%d"%(time.time()*1000)
    # 用来搜索默认条件下的妹纸
    url4 = "http://search.baihe.com/solrAdvanceSearch.action"
    # 向妹纸发送消息
    url5 = "http://msg.baihe.com/owner/api/sendMessage?jsonCallBack=jQuery18304671662130587029_1479300393335&toUserID=%%s&content=%%s&type=1&pathID=01.00.10402&_=%d&"%(time.time()*1000)
    # 登陆过频繁的话,会要求输验证,此url用来获取验证码图片
    url6 = "http://my.baihe.com/Getinterlogin/getVerifyPic?jsonCallBack=?&tmpId=%s"
    # 用来检查登陆次数,来判定是否需要输验证码了
    url7 = "http://my.baihe.com/Getinterlogin/getAccountTimes?jsonCallBack=jQuery183013238800936369732_1479556200186&userAccount=18353130797&_=1479556585384"
    # 用来验证验证码是否正确
    url8 = "http://my.baihe.com/Getinterlogin/checkVerifyPic?jsonCallBack=jQuery183010981413646438898_1480223919788&tmpId=%%s&checkcode=%%s&_=%d"%(time.time()*1000)
    # access_token生成
    acc_token = Cookie(version=0,
                                 name="accessToken",
                                 value='BH%d%d'%(time.time()*1000,math.floor(random.random()*1000000)),
                                 domain=".baihe.com",
                                 path="/",
                                 port=None,
                                 port_specified=False,
                                 domain_specified=True,
                                 domain_initial_dot=False,
                                 path_specified=True,
                                 secure=False,
                                 expires=None,
                                 discard=False,
                                 comment=None,
                                 comment_url=None,
                                 rest={},
                                 rfc2109=False)

    def __init__(self):
        self.page = 1
        self.order = 1
        self.product_ids = set()
        self.error_count = 0
        self.lock = RLock()
        self.alive = True
        self.cookie = LWPCookieJar()
        try:
            self.have_send_list = set(open("have_send_list.txt").read().strip(",").split(","))
        except IOError:
            self.have_send_list = set()
        self.logger = logging.getLogger("send_msg")
        self.logger.setLevel(logging.DEBUG)
        self.logger.addHandler(logging.StreamHandler(sys.stdout))

    def get_account_times(self, opener):
        resp = opener.open(self.url7)
        buf = resp.read()
        data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
        self.logger.debug("Check whether need input captcha or not. ")
        if data["data"]["showCode"]:
            return self.get_captcha(opener)
        else:
            return "", ""

    def get_captcha(self, opener):
        tmpId =  "%d.%s"%(time.time()*1000, str(round(random.random(), 4))[2:])
        resp = opener.open(self.url6%tmpId)
        img = Image.open( StringIO(resp.read()))
        img.show()
        return raw_input("Please input captcha recognization: "), tmpId

    def send_captcha(self, opener, captcha, tmpId):
        self.logger.debug("Send captcha. ")
        url = self.url8 % (tmpId, captcha)
        req = Request(url=url, headers=self.headers)
        resp = opener.open(req)
        data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk")
        if data["data"] == 1:
            return tmpId, captcha

    def login(self, opener):
        url = self.url1 + urlencode(self.data)
        req = Request(url=url, headers=self.headers)
        resp = opener.open(req)
        data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk")
        self.logger.debug("Login jsonp response state:%s"%data["state"])

        if data["state"] == 0:
            return "Wrong account or password. "

        req = Request(url=self.url2, headers=self.headers)
        resp = opener.open(req)
        self.logger.debug("Login redirect response code:%s"%resp.code)

    def get_auth_cookies(self, opener):
        while True:
            self.enter_password()
            captcha, tmpId = self.get_account_times(opener)
            if tmpId:
                while not self.send_captcha(opener, captcha, tmpId):
                    captcha, tmpId = self.get_account_times(opener)
            self.data["codeValue"] = captcha
            self.data["codeId"] = tmpId
            result = self.login(opener)
            if result:
                self.logger.info(result)
            else:
                break

    def get_search_cookies(self, opener):
        req = Request(url=self.url4, headers=self.headers)
        resp = opener.open(req)
        self.logger.debug("Finish get default search cookies, response code:%s" % resp.code)

    def search(self, opener):
        conti = True
        while conti:

            while True:
                try:
                    id = self.product_ids.pop()
                except KeyError:
                    break
                self.send_msg(opener, id)
                self.order += 1

            req = Request(url=self.url3 % self.page, headers=self.headers)
            self.logger.debug("Start to find girls in page NO.%s. "%self.page)
            resp = opener.open(req)
            self.logger.debug("Search response code:%s" % resp.code)
            buf = resp.read()
            data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
            if data["result"]:
                self.product_ids = set([i.split(":")[0] for i in data["result"].split(",")])
                self.page += 1
            elif self.page > 100:
                return "finished"
            else:
                raise SendMessageError("You need relogin. ")

    def send_msg(self, opener, id):

        if id not in self.have_send_list:
            msg = random.choice(self.messages)
            d = quote(msg)
            url = self.url5 % (id, d)
            req = Request(url=url, headers=self.headers)
            resp = opener.open(req)
            buf = resp.read()
            recv = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
            code = recv["code"]
            self.logger.info("Send %s to No.%s girl whose id is %s, status code is %s" % (msg.decode("utf-8"), self.order, id, code))
            if code in [200, u"-603", u"-804", u"-602", u"-611"]:
                self.logger.info("Normal message is %s" % (recv.get("msg") or u"empty").encode("gbk"))
                if self.error_count > 0:
                    self.error_count -= 1
                self.have_send_list.add(id)
            else:
                self.logger.info("Error message is %s"%(recv.get("msg") or u"empty").encode("gbk"))
                self.error_count += 1
                if code in [u"-701", u"-803"]:
                    self.alive = False
                    self.logger.error(u"坑爹的百合每天每个账号只允许给100个人发消息。。")
                    sys.exit(0)
                if self.error_count > 3:
                    raise SendMessageError("code: %s error: %s" % (code.encode("gbk"), (recv.get("msg") or u"empty").encode("gbk")))
            time.sleep(1)
        else:
            self.logger.info("The No.%s girl whose id is %s has been sent, don't molesting her any more. "%(self.order, id))

    def pwd_input(self, msg=''):

        if msg != '':
            sys.stdout.write(msg)
        chars = []
        while True:
            newChar = getch()
            if newChar in '\3\r\n':  # 如果是换行,Ctrl+C,则输入结束
                print ''
                if newChar in '\3':  # 如果是Ctrl+C,则将输入清空,返回空字符串
                    chars = []
                break
            elif newChar == '\b' or ord(newChar) == 127:  # 如果是退格,则删除末尾一位
                if chars:
                    del chars[-1]
                    sys.stdout.write('\b \b')  # 左移一位,用空格抹掉星号,再退格
            else:
                chars.append(newChar)
                sys.stdout.write('*')  # 显示为星号
        return ''.join(chars)

    def enter_password(self):
        account = raw_input("Please input your baihe account number: ")
        self.data["txtLoginEMail"] = account
        self.data["txtLoginPwd"] = self.pwd_input("Please input your baihe account password: "******"Please input what you want to send, input empty to break. ")
            if not msg:
                break
            else:
                try:
                    msg = msg.decode("gbk").encode("utf-8")
                except UnicodeDecodeError:
                    pass
                self.messages.append(msg)

    def start(self):
        self.enter_msg()
        self.cookie.set_cookie(self.acc_token)
        have_load = False
        try:
            if os.path.exists(("baihe.cookie")):
                self.cookie.load("baihe.cookie", True, True)
                have_load = True
            opener = build_opener(HTTPCookieProcessor(self.cookie))
            if not have_load:
                self.get_auth_cookies(opener)
                self.get_search_cookies(opener)
            # 有时意外不正常关闭cookie和send_list无法保存,所以启动一个进程来做这件事。
            Thread(target=self.saveing).start()
            while True:
                try:
                    if self.search(opener) == "finished":
                        self.logger.info("No more girls to send. ")
                        break
                except Exception, e:
                    time.sleep(1)
                    self.logger.error(e)
                    self.get_auth_cookies(opener)
                    self.get_search_cookies(opener)

        except KeyboardInterrupt:
            self.logger.info("Closing...")
            self.alive = False
        finally:
            self.save()
            self.alive = False

    def saveing(self):
        while self.alive:
            self.save()
            time.sleep(2)

    def save(self):
        self.lock.acquire()
        open("have_send_list.txt", "w").write(",".join(self.have_send_list))
        self.cookie.save("baihe.cookie", True, True)
        self.lock.release()
 def __init__(self, user, password, event_log_path):
     self.user = user
     self.password = password
     self.event_log_path = event_log_path
     self.cj = LWPCookieJar(COOKIEFILE)
Beispiel #56
0
    'Accept-Encoding':
    'gzip,deflate,sdch',
    'Accept-Language':
    'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4',
    'Connection':
    'keep-alive',
    'Content-Type':
    'application/x-www-form-urlencoded',
    'Host':
    'music.163.com',
    'Referer':
    'http://music.163.com/search/',
    "User-Agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
                   "(KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36")
}

cookies = {'appver': '1.5.2'}
session = requests.Session()
session.cookies = LWPCookieJar('cookies')
session.cookies.load()
session.cookies.save()


def phone_login(username, password):
    action = 'https://music.163.com/weapi/login/cellphone'
    text = {'phone': username, 'password': password, 'rememberLogin': '******'}
    data = encrypted_request(text)
    connect = session.post(action, data=data, headers=header, timeout=10)
    connect.encoding = 'UTF-8'
    return connect.content
Beispiel #57
0
#!/usr/bin/env python
# By: Volker Strobel
from bs4 import BeautifulSoup
import urllib
from urllib2 import Request, build_opener, HTTPCookieProcessor
from cookielib import LWPCookieJar
import re
import time
import sys

cookies = LWPCookieJar('./cookies')
try:
    cookies.load()
except IOError:
    pass

def get_num_results(search_term, start_date, end_date):
    """
    Helper method, sends HTTP request and returns response payload.
    """

    # Open website and read html
    user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36'
    query_params = { 'q' : search_term, 'as_ylo' : start_date, 'as_yhi' : end_date}
    url = "https://scholar.google.com/scholar?as_vis=1&hl=en&as_sdt=1,5&" + urllib.urlencode(query_params)
    opener = build_opener(HTTPCookieProcessor(cookies))
    request = Request(url=url, headers={'User-Agent': user_agent})
    handler = opener.open(request)
    html = handler.read() 

    # Create soup for parsing HTML and extracting the relevant information
Beispiel #58
0
    def login(self, username=None, password=None, is_beta=False):
        if (not username or not password):
            if not self.credentials:
                raise CredentialsError()
            else:
                credentials = self.credentials
        else:
            credentials = {
                'email': username,
                'password': password,
                'beta': is_beta
            }

        if self.credentials == credentials and self.state == Session.STATE_OK:
            return  # already logged in

        if not self.credentials or self.credentials['email'] != credentials[
                'email'] or self.credentials['beta'] != credentials['beta']:
            # changed account
            self.close()
            self.session = requests.Session()
            self.session.headers[
                'User-Agent'] = 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Mobile/11D257'
            cookiefile = join(
                config.app_dir, 'cookies-%s.txt' %
                hashlib.md5(credentials['email']).hexdigest())
            if not isfile(cookiefile) and isfile(
                    join(config.app_dir, 'cookies.txt')):
                os.rename(join(config.app_dir, 'cookies.txt'),
                          cookiefile)  # migration from <= 2.25
            self.session.cookies = LWPCookieJar(cookiefile)
            try:
                self.session.cookies.load()
            except IOError:
                pass

        self.server = credentials['beta'] and SERVER_BETA or SERVER_LIVE
        self.credentials = credentials
        self.state = Session.STATE_INIT
        try:
            r = self.session.post(self.server + URL_LOGIN,
                                  data=self.credentials,
                                  timeout=timeout)
        except:
            if __debug__: print_exc()
            raise ServerError()

        if r.status_code != requests.codes.ok or 'server error' in r.text:
            self.dump(r)
            raise ServerError()
        elif r.url == self.server + URL_LOGIN:  # would have redirected away if success
            self.dump(r)
            if 'purchase' in r.text.lower():
                raise SKUError()
            else:
                raise CredentialsError()
        elif r.url == self.server + URL_CONFIRM:  # redirected to verification page
            self.state = Session.STATE_AUTH
            raise VerificationRequired()
        else:
            self.state = Session.STATE_OK
            return r.status_code
Beispiel #59
0
    def __init__(self,
                 hostname,
                 username,
                 password,
                 table=None,
                 fields=None,
                 groups=None,
                 selection=None,
                 filter_results=None,
                 proxy=None):
        self.hostname = hostname

        # requests session
        self.session = requests.Session()

        self.auth = requests.auth.HTTPBasicAuth(username, password)
        # request headers
        self.headers = {
            "Accept": "application/json",
            "Content-Type": "application/json",
        }

        # request cookies
        self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session")
        try:
            self.cookies.load(ignore_discard=True)
        except IOError:
            pass
        self.session.cookies = self.cookies

        if table is None:
            table = 'cmdb_ci_server'

        if fields is None:
            fields = []

        if groups is None:
            groups = []

        if selection is None:
            selection = []

        if filter_results is None:
            filter_results = ''

        if proxy is None:
            proxy = []

        # table
        self.table = table

        # extra fields (table columns)
        self.fields = fields

        # extra groups (table columns)
        self.groups = groups

        # selection order
        self.selection = selection

        # filter results (sysparm_query encoded query string)
        self.filter_results = filter_results

        # proxy settings
        self.proxy = proxy

        # initialize inventory
        self.inventory = {'_meta': {'hostvars': {}}}

        return
Beispiel #60
0
import requests
import string
import sys
import zlib
from requests.packages import chardet
from StringIO import StringIO
import gzip
from cookielib import LWPCookieJar
import os
s = requests.Session()
s.cookies = LWPCookieJar("cookiejar")


def login():
    url = 'http://jiaowu.swjtu.edu.cn/servlet/GetRandomNumberToJPEG'
    hraders = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Accept': 'image/webp,image/*,*/*;q=0.8'
    }
    response = s.get(url, headers=hraders, stream=True)
    with open('demo.jpg', 'wb') as fd:
        for chunk in response.iter_content(128):
            fd.write(chunk)

    url = 'http://jiaowu.swjtu.edu.cn/servlet/UserLoginSQLAction'
    hraders = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
        'Content-Type': 'application/x-www-form-urlencoded',