Exemplo n.º 1
0
 def init(self):
     """Initialize connection"""
     logger.debug(u"Initializing %s" % self.__class__.__name__) if sys.platform != "win32" else logger.debug(
         "Log line suppressed on windows"
     )
     self.session = requests.session()
     self.session.headers.update({"User-Agent": self.user_agent})
Exemplo n.º 2
0
 def __addImg(self,data):
     api = conf.read_config(conf.path, 'API', 'addImgApi')
     url = base + api
     s = requests.session()
     result = s.post(url,data)
     s.keep_alive = False       
     return result.text
Exemplo n.º 3
0
    def init(self):
       
        super(Itasa, self).init()
        login_pattern = '<input type="hidden" name="return" value="([^\n\r\t ]+?)" /><input type="hidden" name="([^\n\r\t ]+?)" value="([^\n\r\t ]+?)" />'

        response = requests.get(self.server_url + 'index.php')
        if response.status_code != 200:
            raise ServiceError('Initiate failed')
        
        match = re.search(login_pattern, response.content, re.IGNORECASE | re.DOTALL)
        if not match:
            raise ServiceError('Can not find unique id parameter on page')
        
        login_parameter = {'username': '******',
                           'passwd': 'subliminal',
                           'remember': 'yes',
                           'Submit': 'Login',
                           'remember': 'yes',
                           'option': 'com_user',
                           'task': 'login',
                           'silent': 'true',
                           'return': match.group(1), 
                            match.group(2): match.group(3)
                          }

        self.session = requests.session()
        r = self.session.post(self.server_url + 'index.php', data=login_parameter)
        if not re.search('logouticon.png', r.content, re.IGNORECASE | re.DOTALL):
            raise ServiceError('Itasa Login Failed')
Exemplo n.º 4
0
 def __getAssetByType(self,pid):
     api = conf.read_config(conf.path, 'API', 'getAssetByTypeApi')
     url = base + api + '?project_id=' + pid
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False 
     return result.json()
Exemplo n.º 5
0
 def __getLastVersion(self,entity_id,entity_type):
     api = conf.read_config(conf.path, 'API', 'getLastVersionApi')
     url = base + api + '?entity_id=' + entity_id + '&entity_type=' + entity_type
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False     
     return result.content
Exemplo n.º 6
0
    def __init__(self, name):
        # these need to be set in the subclass
        self.providerType = None
        self.name = name

        self.proxy = ProviderProxy()
        self.proxyGlypeProxySSLwarning = None
        self.urls = {}
        self.url = ''

        self.show = None

        self.supportsBacklog = False
        self.supportsAbsoluteNumbering = False
        self.anime_only = False

        self.search_mode = None
        self.search_fallback = False
        self.enable_daily = False
        self.enable_backlog = False

        self.cache = tvcache.TVCache(self)

        self.session = requests.session()

        self.headers = {'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT}
Exemplo n.º 7
0
 def __upLoadFile(self,filePath,directory):
     api = conf.read_config(conf.path, 'API', 'publishFileApi')
     url = base + api + '?f=' + directory
     files = {'file': open(filePath, 'rb')}
     s = requests.session()
     s.post(url, files=files)
     s.keep_alive = False   
Exemplo n.º 8
0
 def __getTaskStep(self,uid,pid,tableName):
     api = conf.read_config(conf.path, 'API', 'getTaskStepApi')
     url = base + api + '?user_id=' +  uid  + '&project_id='+ pid
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False   
     return result.json()[tableName]
Exemplo n.º 9
0
def authenticate_with_server():
    http_session = requests.session()
    http_session.get(LOGIN_PAGE, headers=HEADER)
    data = {"pseudo": USERNAME, "passe": PASSWORD, "souvenir": 1}
    http_session.post(POST_LOGIN_PAGE, data, headers=HEADER)
    logger.warning('LOGIN')
    save_session_in_cache(http_session)
    save_session_to_db()
    return http_session
Exemplo n.º 10
0
 def __getSingleAsset(self,pid,entityId,entityType):
     api = conf.read_config(conf.path, 'API', 'getSingleAssetApi')
     url = base + api +  '?project_id=' + pid + '&entity_id=' + entityId + '&entity_type=' + entityType 
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False   
     if result.text != u"null":
         return result.json()['SA']
     else:
         return ""
Exemplo n.º 11
0
 def __getReferenceVersion(self,version_id):
     api = conf.read_config(conf.path, 'API', 'getReferenceVersionApi')
     url = base + api + '?version_id=' + version_id 
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False    
     if result.text != u"null":
         return result.json()['ATTACHMENTID']
     else:
         return ""
Exemplo n.º 12
0
 def __getVersion(self,vname,pid):
     api = conf.read_config(conf.path, 'API', 'getVersionByNameApi')
     url = base + api + '?versionName=' + vname + '&project_id=' + pid
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False  
     if result.text != u"null":
         return result.json()['VERSION']
     else:
         return ""
Exemplo n.º 13
0
 def __getImg(self,imgId):
     api = conf.read_config(conf.path, 'API', 'getImgApi')
     url = base + api + '?image_id=' + imgId
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False
     if result.text != u"null":
         return result.json()['THUMBNAIL']
     else:
         return ""
Exemplo n.º 14
0
 def __getProject(self,uid):
     api = conf.read_config(conf.path, 'API', 'getProjectApi')
     url = base + api + '?user_id=' + uid
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False
     if result.text != u"null":
         return result.json()['PROJECT']
     else:
         return ""
Exemplo n.º 15
0
 def __getSingleTask(self,uid,taskId,stepId):
     api = conf.read_config(conf.path, 'API', 'getSingleTaskApi')
     url = (base + api + '?uid=' + uid + '&task_id=' + taskId + '&step_id=' + stepId)
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False   
     if result.text != u"null":
         return result.json()['TASK']
     else:
         return ""   
Exemplo n.º 16
0
 def __getAssetType(self):
     api = conf.read_config(conf.path, 'API', 'getAssetTypeAPi')
     url = base + api
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False  
     if result.text != u"null":
         return result.json()['ASSETTYPE']
     else:
         return ""
Exemplo n.º 17
0
 def __getAsset(self,uid,project_id,start,length):
     api = conf.read_config(conf.path, 'API', 'getAssetApi')
     url = base + api + '?uid=' + uid +  '&project_id=' + project_id + '&start=' + start + '&length=' + length
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False   
     if result.text != u"null":
         return result.json()['ASSET']
     else:
         return ""
Exemplo n.º 18
0
 def __getSequences(self,pid):
     api = conf.read_config(conf.path, 'API', 'sequencesApi')
     url = base + api + pid
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False   
     
     if result.text != u"null":
         return result.json()['SEQUENCENAME']
     else:
         return ""
Exemplo n.º 19
0
 def __lastVersion(self,vid):
     api = conf.read_config(conf.path, 'API', 'checkVersionApi2')
     url = base + api
     s = requests.session()
     data = {'id':vid}
     result = s.post(url,data = data)
     s.keep_alive = False 
     if result.text != u"null":
         return result.json()['LASTVERSION']
     else:
         return ""
Exemplo n.º 20
0
    def __init__(self, name, host=None, username=None, password=None):

        self.name = name
        self.username = sickbeard.TORRENT_USERNAME if username is None else username
        self.password = sickbeard.TORRENT_PASSWORD if password is None else password
        self.host = sickbeard.TORRENT_HOST if host is None else host
        
        self.url = None
        self.response = None
        self.auth = None
        self.last_time = time.time()
        self.session = requests.session(auth=(self.username, self.password),timeout=60)
Exemplo n.º 21
0
 def __getFile(self,entity_id,entity_type,project_id):
     api = conf.read_config(conf.path, 'API', 'workfileApi')
     url = (base + api + '?entity_id=' + entity_id + 
            '&entity_type=' + entity_type +
            '&project_id=' + project_id)
     s = requests.session()
     result = s.post(url)
     s.keep_alive = False   
     if result.text != u"null":
         return result.json()['VERSION']
     else:
         return ""
Exemplo n.º 22
0
 def __getUserInfo(self,userName,password):
     base = conf.read_config(conf.path, 'API', 'baseUrl')
     api = conf.read_config(conf.path, 'API', 'loginApi')
     url = base + api 
     data = {'name': userName,'password':password}
     s = requests.session()
     result = s.post(url,data = data)
     s.keep_alive = False  
     if  result.json() != "error":
         return result.json()['Table']
     else:
         return "error"
Exemplo n.º 23
0
 def __getCompetence(self,uid):
     base = conf.read_config(conf.path, 'API', 'baseUrl')
     api = conf.read_config(conf.path, 'API', 'getUserRule')
     url = base + api
     s = requests.session()
     idData = { 'ID': uid, 'VALUE': '5' };
     result = s.post(url,data = idData)
     s.keep_alive = False 
     if result.text != u"null":
         return result.json()['AUTH']
     else:
         return ""
Exemplo n.º 24
0
    def __init__(self, name, host=None, username=None, password=None):

        self.name = name
        self.username = sickbeard.TORRENT_USERNAME if username is None else username
        self.password = sickbeard.TORRENT_PASSWORD if password is None else password
        self.host = sickbeard.TORRENT_HOST if host is None else host

        self.url = None
        self.response = None
        self.auth = None
        self.last_time = time.time()
        self.session = requests.session(auth=(self.username, self.password),
                                        timeout=60)
Exemplo n.º 25
0
    def __init__(self, name, host=None, username=None, password=None):

        self.name = name
        self.username = sickbeard.TORRENT_USERNAME if username is None else username
        self.password = sickbeard.TORRENT_PASSWORD if password is None else password
        self.host = sickbeard.TORRENT_HOST if host is None else host.rstrip(
            '/') + '/'

        self.url = None
        self.auth = None
        self.last_time = time.time()
        self.session = requests.session()
        self.session.auth = (self.username, self.password)
        self.created_id = None
Exemplo n.º 26
0
    def __init__(self):
        generic.TorrentProvider.__init__(self, "Rarbg")

        self.enabled = False

        self.supportsBacklog = True

        self.ratio = None

        self.cache = RarbgCache(self)

        self.urls = {
            'url': 'https://rarbg.com',
            'base_url': 'https://rarbg.com/torrents.php',
            'search':
            'https://rarbg.com/torrents.php?search=%s&category=%s&page=%s',
            'download': 'https://rarbg.com/download.php?id=%s&f=%s',
        }

        self.url = self.urls['base_url']

        self.subcategories = [18, 41]
        self.pages = [1, 2, 3, 4, 5]

        self.cookie = {
            "version": 0,
            "name": '7fAY799j',
            "value": 'VtdTzG69',
            "port": None,
            # "port_specified": False,
            "domain": 'rarbg.com',
            # "domain_specified": False,
            # "domain_initial_dot": False,
            "path": '/',
            # "path_specified": True,
            "secure": False,
            "expires": None,
            "discard": True,
            "comment": None,
            "comment_url": None,
            "rest": {},
            "rfc2109": False
        }

        self.session = requests.session()
        self.session.headers.update({
            'User-Agent':
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36'
        })
        self.session.cookies.set(**self.cookie)
Exemplo n.º 27
0
    def __init__(self, name):

        # these need to be set in the subclass
        self.providerType = None
        self.name = name
        self.url = ''

        self.show = None
        self.supportsBacklog = False

        self.cache = tvcache.TVCache(self)

        self.session = requests.session()
        self.session.verify = False
        self.session.headers.update({'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'})
Exemplo n.º 28
0
    def _doLogin(self):

        login_params = {'username': self.username,
                        'password': self.password,
                        'ssl': 'yes'
        }

        if not self.session:
            self.session = requests.session()

        try:
            response = self.session.post(self.urls['login'], data=login_params, timeout=30, verify=False)
        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
            logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR)
            return False
    def __init__(self, name):

        # these need to be set in the subclass
        self.providerType = None
        self.name = name
        self.url = ''

        self.show = None
        self.supportsBacklog = False

        self.cache = tvcache.TVCache(self)

        self.session = requests.session()
        self.session.verify = False
        self.session.headers.update({
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'})
Exemplo n.º 30
0
    def __init__(self):
        generic.TorrentProvider.__init__(self, "Rarbg")

        self.enabled = False

        self.supportsBacklog = True
        
        self.supportsFrench = False

        self.ratio = None

        self.cache = RarbgCache(self)

        self.urls = {'url': 'https://rarbg.com',
                     'base_url': 'https://rarbg.com/torrents.php',
                     'search': 'https://rarbg.com/torrents.php?search=%s&category=%s&page=%s',
                     'download': 'https://rarbg.com/download.php?id=%s&f=%s',
                     }

        self.url = self.urls['base_url']

        self.subcategories = [18,41]
        self.pages = [1,2,3,4,5]

        self.cookie = {
            "version": 0,
            "name": '7fAY799j',
            "value": 'VtdTzG69',
            "port": None,
            # "port_specified": False,
            "domain": 'rarbg.com',
            # "domain_specified": False,
            # "domain_initial_dot": False,
            "path": '/',
            # "path_specified": True,
            "secure": False,
            "expires": None,
            "discard": True,
            "comment": None,
            "comment_url": None,
            "rest": {},
            "rfc2109": False
        }

        self.session = requests.session()
        self.session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36'})
        self.session.cookies.set(**self.cookie)
Exemplo n.º 31
0
 def __init__(self, username=None, password=None):
     self.session = requests.session()
     self.session.headers = self.default_headers
     self.username = username
     self.password = password
     self.authkey = None
     self.passkey = None
     self.userid = None
     self.logged_in_user = None
     self.cached_users = {}
     self.cached_artists = {}
     self.cached_tags = {}
     self.cached_torrent_groups = {}
     self.cached_torrents = {}
     self.cached_requests = {}
     self.cached_categories = {}
     self.site = "https://what.cd/"
     self.past_request_timestamps = []
Exemplo n.º 32
0
 def __init__(self, username=None, password=None):
     self.session = requests.session(headers=self.default_headers)
     self.username = username
     self.password = password
     self.authkey = None
     self.passkey = None
     self.userid = None
     self.logged_in_user = None
     self.cached_users = {}
     self.cached_artists = {}
     self.cached_tags = {}
     self.cached_torrent_groups = {}
     self.cached_torrents = {}
     self.cached_requests = {}
     self.cached_categories = {}
     self.site = "https://what.cd/"
     self.rate_limit = 2.0 # seconds between requests
     self._login()
Exemplo n.º 33
0
 def __init__(self, username=None, password=None):
     self.session = requests.session()
     self.session.headers = self.default_headers
     self.username = username
     self.password = password
     self.authkey = None
     self.passkey = None
     self.userid = None
     self.logged_in_user = None
     self.cached_users = {}
     self.cached_artists = {}
     self.cached_tags = {}
     self.cached_torrent_groups = {}
     self.cached_torrents = {}
     self.cached_requests = {}
     self.cached_categories = {}
     self.site = "https://what.cd/"
     self.past_request_timestamps = []
Exemplo n.º 34
0
 def __init__(self, username=None, password=None):
     self.session = requests.session(headers=self.default_headers)
     self.username = username
     self.password = password
     self.authkey = None
     self.passkey = None
     self.userid = None
     self.logged_in_user = None
     self.cached_users = {}
     self.cached_artists = {}
     self.cached_tags = {}
     self.cached_torrent_groups = {}
     self.cached_torrents = {}
     self.cached_requests = {}
     self.cached_categories = {}
     self.site = "https://what.cd/"
     self.rate_limit = 2.0  # seconds between requests
     self._login()
Exemplo n.º 35
0
    def _doLogin(self):

        login_params = {
            'username': self.username,
            'password': self.password,
            'ssl': 'yes'
        }

        if not self.session:
            self.session = requests.session()

        try:
            response = self.session.post(self.urls['login'],
                                         data=login_params,
                                         timeout=30,
                                         verify=False)
        except (requests.exceptions.ConnectionError,
                requests.exceptions.HTTPError), e:
            logger.log(
                u'Unable to connect to ' + self.name + ' provider: ' + ex(e),
                logger.ERROR)
            return False
Exemplo n.º 36
0
def downloadpage(url, **opt):
    # logger.info()
    """
       Open a url and return the data obtained

        @param url: url to open.
        @type url: str
        @param post: If it contains any value, it is sent by POST.
        @type post: str
        @param headers: Headers for the request, if it contains nothing the default headers will be used.
        @type headers: dict, list
        @param timeout: Timeout for the request.
        @type timeout: int
        @param follow_redirects: Indicates if redirects are to be followed.
        @type follow_redirects: bool
        @param cookies: Indicates whether cookies are to be used.
        @type cookies: bool
        @param replace_headers: If True, headers passed by the "headers" parameter will completely replace the default headers.
                                If False, the headers passed by the "headers" parameter will modify the headers by default.
        @type replace_headers: bool
        @param add_referer: Indicates whether to add the "Referer" header using the domain of the url as a value.
        @type add_referer: bool
        @param only_headers: If True, only headers will be downloaded, omitting the content of the url.
        @type only_headers: bool
        @param random_headers: If True, use the method of selecting random headers.
        @type random_headers: bool
        @param ignore_response_code: If True, ignore the method for WebErrorException for error like 404 error in veseriesonline, but it is a functional data
        @type ignore_response_code: bool
        @return: Result of the petition
        @rtype: HTTPResponse
        @param use_requests: Use requests.session()
        @type: bool

                Parameter Type Description
                -------------------------------------------------- -------------------------------------------------- ------------
                HTTPResponse.sucess: bool True: Request successful | False: Error when making the request
                HTTPResponse.code: int Server response code or error code if an error occurs
                HTTPResponse.error: str Description of the error in case of an error
                HTTPResponse.headers: dict Dictionary with server response headers
                HTTPResponse.data: str Response obtained from server
                HTTPResponse.json: dict Response obtained from the server in json format
                HTTPResponse.time: float Time taken to make the request

        """
    url = scrapertools.unescape(url)
    domain = urlparse.urlparse(url).netloc
    global CF_LIST
    CF = False

    if domain in FORCE_CLOUDSCRAPER_LIST:
        from lib import cloudscraper
        session = cloudscraper.create_scraper()
        CF = True
    else:
        from lib import requests
        session = requests.session()

        if domain in CF_LIST or opt.get('CF', False):
            url = 'https://web.archive.org/save/' + url
            CF = True

    if config.get_setting('resolver_dns') and not opt.get(
            'use_requests', False):
        from specials import resolverdns
        session.mount('https://', resolverdns.CipherSuiteAdapter(domain, CF))

    req_headers = default_headers.copy()

    # Headers passed as parameters
    if opt.get('headers', None) is not None:
        if not opt.get('replace_headers', False):
            req_headers.update(dict(opt['headers']))
        else:
            req_headers = dict(opt['headers'])

    if opt.get('random_headers', False) or HTTPTOOLS_DEFAULT_RANDOM_HEADERS:
        req_headers['User-Agent'] = random_useragent()
    url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")

    opt['url_save'] = url
    opt['post_save'] = opt.get('post', None)

    response = {}
    info_dict = []
    payload = dict()
    files = {}
    file_name = ''

    session.verify = opt.get('verify', True)

    if opt.get('cookies', True):
        session.cookies = cj
    session.headers.update(req_headers)

    proxy_data = {'dict': {}}

    inicio = time.time()

    if opt.get(
            'timeout',
            None) is None and HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT is not None:
        opt['timeout'] = HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT
    if opt['timeout'] == 0: opt['timeout'] = None

    if len(url) > 0:
        try:
            if opt.get('post', None) is not None or opt.get('file',
                                                            None) is not None:
                if opt.get('post', None) is not None:
                    # Convert string post in dict
                    try:
                        json.loads(opt['post'])
                        payload = opt['post']
                    except:
                        if not isinstance(opt['post'], dict):
                            post = urlparse.parse_qs(opt['post'],
                                                     keep_blank_values=1)
                            payload = dict()

                            for key, value in post.items():
                                try:
                                    payload[key] = value[0]
                                except:
                                    payload[key] = ''
                        else:
                            payload = opt['post']

                # Verify 'file' and 'file_name' options to upload a buffer or file
                if opt.get('file', None) is not None:
                    if os.path.isfile(opt['file']):
                        if opt.get('file_name', None) is None:
                            path_file, opt['file_name'] = os.path.split(
                                opt['file'])
                        files = {
                            'file': (opt['file_name'], open(opt['file'], 'rb'))
                        }
                        file_name = opt['file']
                    else:
                        files = {
                            'file': (opt.get('file_name',
                                             'Default'), opt['file'])
                        }
                        file_name = opt.get('file_name',
                                            'Default') + ', Buffer de memoria'

                info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
                if opt.get('only_headers', False):
                    # Makes the request with HEAD method
                    req = session.head(url,
                                       allow_redirects=opt.get(
                                           'follow_redirects', True),
                                       timeout=opt['timeout'])
                else:
                    # Makes the request with POST method
                    req = session.post(url,
                                       data=payload,
                                       allow_redirects=opt.get(
                                           'follow_redirects', True),
                                       files=files,
                                       timeout=opt['timeout'])

            elif opt.get('only_headers', False):
                info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
                # Makes the request with HEAD method
                req = session.head(url,
                                   allow_redirects=opt.get(
                                       'follow_redirects', True),
                                   timeout=opt['timeout'])
            else:
                info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
                # Makes the request with GET method
                req = session.get(url,
                                  allow_redirects=opt.get(
                                      'follow_redirects', True),
                                  timeout=opt['timeout'])
        except Exception as e:
            from lib import requests
            req = requests.Response()
            if not opt.get('ignore_response_code',
                           False) and not proxy_data.get('stat', ''):
                response['data'] = ''
                response['sucess'] = False
                info_dict.append(('Success', 'False'))
                response['code'] = str(e)
                info_dict.append(('Response code', str(e)))
                info_dict.append(('Finalizado en', time.time() - inicio))
                if not opt.get('alfa_s', False):
                    show_infobox(info_dict)
                return type('HTTPResponse', (), response)
            else:
                req.status_code = str(e)

    else:
        response['data'] = ''
        response['sucess'] = False
        response['code'] = ''
        return type('HTTPResponse', (), response)

    response_code = req.status_code

    if req.headers.get('Server',
                       '').startswith('cloudflare') and response_code in [
                           429, 503, 403
                       ] and not opt.get('CF', False):
        if domain not in CF_LIST:
            opt["CF"] = True
            with open(CF_LIST_PATH, "a") as CF_File:
                CF_File.write("%s\n" % domain)
            logger.debug("CF retry... for domain: %s" % domain)
            return downloadpage(url, **opt)

    response['data'] = req.content if req.content else ''
    if CF:
        import re
        response['data'] = re.sub('["|\']/save/[^"]*(https?://[^"]+)', '"\\1',
                                  response['data'])
    response['url'] = req.url

    if type(response['data']) != str:
        response['data'] = response['data'].decode('UTF-8')

    if not response['data']:
        response['data'] = ''
    try:
        response['json'] = to_utf8(req.json())
    except:
        response['json'] = dict()
    response['code'] = response_code
    response['headers'] = req.headers
    response['cookies'] = req.cookies

    info_dict, response = fill_fields_post(info_dict, req, response,
                                           req_headers, inicio)

    if opt.get('cookies', True):
        save_cookies(alfa_s=opt.get('alfa_s', False))

    # is_channel = inspect.getmodule(inspect.currentframe().f_back)
    # is_channel = scrapertools.find_single_match(str(is_channel), "<module '(channels).*?'")
    # if is_channel and isinstance(response_code, int):
    #     if not opt.get('ignore_response_code', False) and not proxy_data.get('stat', ''):
    #         if response_code > 399:
    #             show_infobox(info_dict)
    #             raise WebErrorException(urlparse.urlparse(url)[1])

    if not 'api.themoviedb' in url and not opt.get('alfa_s', False):
        show_infobox(info_dict)

    return type('HTTPResponse', (), response)
Exemplo n.º 37
0
    def authenticate(self):

        feedinfo = []

        try:
            with requests.session() as s:
                if mylar.VERIFY_32P == 1 or mylar.VERIFY_32P == True:
                    verify = True
                else:
                    verify = False

                logger.fdebug('[32P] Verify SSL set to : ' + str(verify))

                if not verify:
                #32P throws back an insecure warning because it can't validate against the CA. The below suppresses the message just for 32P instead of being displa$
                    from lib.requests.packages.urllib3.exceptions import InsecureRequestWarning
                    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


                # fetch the login page

                s.headers = self.headers
                try:
                    s.get(self.url, verify=verify, timeout=30)
                except (requests.exceptions.SSLError, requests.exceptions.Timeout) as e:
                    logger.error(self.module + ' Unable to establish connection to 32P: ' + str(e))
                    return
                    
                # post to the login form
                r = s.post(self.url, data=self.payload, verify=verify)

                #need a way to find response code (200=OK), but returns 200 for everything even failed signons (returns a blank page)
                #logger.info('[32P] response: ' + str(r.content))

                if self.searchterm:
                    logger.info('[32P] Successfully authenticated. Initiating search for : ' + self.searchterm)
                    return self.search32p(s)
                soup = BeautifulSoup(r.content)
                all_script = soup.find_all("script", {"src": False})
                all_script2 = soup.find_all("link", {"rel": "alternate"})

                for ind_s in all_script:
                    all_value = str(ind_s)
                    all_items = all_value.split()
                    auth_found = False
                    user_found = False
                    for al in all_items:
                        if al == 'authkey':
                            auth_found = True
                        elif auth_found == True and al != '=':
                            authkey = re.sub('["/;]', '', al).strip()
                            auth_found = False
                            logger.fdebug(self.module + ' Authkey found: ' + str(authkey))
                        if al == 'userid':
                            user_found = True
                        elif user_found == True and al != '=':
                            userid = re.sub('["/;]', '', al).strip()
                            user_found = False
                            logger.fdebug(self.module + ' Userid found: ' + str(userid))

                authfound = False
                logger.info(self.module + ' Atttempting to integrate with all of your 32P Notification feeds.')

                for al in all_script2:
                    alurl = al['href']
                    if 'auth=' in alurl and 'torrents_notify' in alurl and not authfound:
                        f1 = alurl.find('auth=')
                        f2 = alurl.find('&', f1 + 1)
                        auth = alurl[f1 +5:f2]
                        logger.fdebug(self.module + ' Auth:' + str(auth))
                        authfound = True
                        p1 = alurl.find('passkey=')
                        p2 = alurl.find('&', p1 + 1)
                        passkey = alurl[p1 +8:p2]
                        logger.fdebug(self.module + ' Passkey:' + str(passkey))
                        if self.reauthenticate: break

                    if 'torrents_notify' in alurl and ('torrents_notify_' + str(passkey)) not in alurl:
                        notifyname_st = alurl.find('name=')
                        notifyname_en = alurl.find('&', notifyname_st +1)
                        if notifyname_en == -1: notifyname_en = len(alurl)
                        notifyname = alurl[notifyname_st +5:notifyname_en]
                        notifynumber_st = alurl.find('torrents_notify_')
                        notifynumber_en = alurl.find('_', notifynumber_st +17)
                        notifynumber = alurl[notifynumber_st:notifynumber_en]
                        logger.fdebug(self.module + ' [NOTIFICATION: ' + str(notifyname) + '] Notification ID: ' + str(notifynumber))

                        #generate the rss-url here
                        feedinfo.append({'feed':     notifynumber + '_' + str(passkey),
                                         'feedname': notifyname,
                                         'user':     userid,
                                         'auth':     auth,
                                         'passkey':  passkey,
                                         'authkey':  authkey})
        except (requests.exceptions.Timeout, EnvironmentError):
            logger.warn('Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.')
            return

        #set the keys here that will be used to download.
        try:
            mylar.PASSKEY_32P = passkey
            mylar.AUTHKEY_32P = authkey  # probably not needed here.
            mylar.KEYS_32P = {}
            mylar.KEYS_32P = {"user": userid,
                              "auth": auth,
                              "passkey": passkey,
                              "authkey": authkey}
        except NameError:
            logger.warn('Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.')
            return
           
        if self.reauthenticate:
            return
        else:
            mylar.FEEDINFO_32P = feedinfo
            return feedinfo
Exemplo n.º 38
0
def execute(args):  
    pp = pprint.PrettyPrinter(indent=2)
    try:
    
        # verbose
        verbose = args['v']
        
        # veryverbose
        veryverbose = args['vv']
        if veryverbose:
            verbose = True
        
        # istSOS service
        service = args['s']
        
        # const constraint role
        role = "urn:ogc:def:classifiers:x-istsos:1.0:qualityIndex:check:reasonable"
        
        # filename
        csvfile = args['f']
        
        req = requests.session()
        
        # Open CSV file
        fo = open(csvfile, "rw+")
        
        #check file validity
        rlines = [ row.strip().split(",") for row in fo.readlines() if row.strip() is not ""]
        lines = []
        for line in rlines:
            lines.append([c.strip() for c in line ])
        # load sensor description
        res = req.get("%s/procedures/operations/getlist" % (service), 
                      prefetch=True, verify=False)
        
        jj = json.loads(res.content)
        if veryverbose:
            print "RETRIVING PRECEDURES..."
            pp.pprint(res.json)
            print "---------------------"
        elif verbose:
            if jj['success'] is False:
                pp.pprint(res.json)
                print "---------------------"
        
        procedures = dict( ( i["name"], [ j["name"] for j in i["observedproperties"] ] ) for i in jj["data"] )
        
        for nr,line in enumerate(lines):
            line = [ l.strip() for l in line ]
            if len(line)==4:
                if not line[0] in procedures.keys():
                    raise Exception("[line %s]: procedure '%s' not observed by the istsos service!" %(nr,line[0]) )
                if not "-".join(line[1].split(":")[-2:]) in procedures[line[0]]:
                    raise Exception("[line %s]: procedure '%s' does not observe property '%s'!" %(nr,line[0],line[1]) )
                if not (is_number(line[2]) or line[2] is ""):
                    raise Exception("[line %s]: value '%s' at column 3 should represent min values if present, check it is a number!" %(nr,line[2]) )
                if not (is_number(line[3]) or line[3] is ""):
                    raise Exception("[line %s]: value '%s' at column 4 should represent min values if present, check it is a number!" %(nr,line[3]) )
            else:
                raise Exception("[line %s]: %s input file must contain 4 row: station name, observed property URI, min, max" %(nr,line))
        
        for nr,line in enumerate(lines):
            if line:
                # load sensor description
                res = req.get("%s/procedures/%s" % (service,line[0]), 
                              prefetch=True, verify=False)
                
                ds = json.loads(res.content)
                if veryverbose:
                    print "RETRIVING PRECEDURES..."
                    pp.pprint(res.json)
                    print "---------------------"
                elif verbose:
                    if ds['success'] is False:
                        pp.pprint(res.json)
                        print "---------------------"
                                
                #update constraints in Json
                for opr in ds["data"]["outputs"]:
                    if opr["definition"] == line[1]:
                        opr["constraint"] = {}
                        opr["constraint"]["role"]=role
                        if line[2] and line[3]:
                            opr["constraint"]["interval"]=[float(line[2]),float(line[3])]
                        elif not line[2] and line[3]:
                            opr["constraint"]["max"]=float(line[3])
                        elif line[2] and not line[3]:
                            opr["constraint"]["min"]=float(line[2])
                
                # send Json request to update constrain on service
                res = req.put("%s/procedures/%s" % (service,line[0]),
                            prefetch=True,
                            verify=False,
                            data=json.dumps(ds["data"])
                        )
                # read response
                jj = json.loads(res.content)
                if veryverbose:
                    print "SAVING PRECEDURE %s..." % line[0]
                    pp.pprint(json.dumps(ds["data"]))
                    print "---------------------"
                
                print "---------------------"
                print " > Updated %s procedure success: %s" %(line[0],res.json['success'])

                if verbose:
                    if jj['success'] is False:
                        pp.pprint(res.json)
                
                print "---------------------"
                
                
    except Exception as e:
        print "ERROR: %s\n\n" % e
        traceback.print_exc()
Exemplo n.º 39
0
def downloadpage(url, **opt):
    # logger.info()
    """
       Open a url and return the data obtained

        @param url: url to open.
        @type url: str
        @param post: If it contains any value, it is sent by POST.
        @type post: str
        @param headers: Headers for the request, if it contains nothing the default headers will be used.
        @type headers: dict, list
        @param timeout: Timeout for the request.
        @type timeout: int
        @param follow_redirects: Indicates if redirects are to be followed.
        @type follow_redirects: bool
        @param cookies: Indicates whether cookies are to be used.
        @type cookies: bool
        @param replace_headers: If True, headers passed by the "headers" parameter will completely replace the default headers.
                                If False, the headers passed by the "headers" parameter will modify the headers by default.
        @type replace_headers: bool
        @param add_referer: Indicates whether to add the "Referer" header using the domain of the url as a value.
        @type add_referer: bool
        @param only_headers: If True, only headers will be downloaded, omitting the content of the url.
        @type only_headers: bool
        @param random_headers: If True, use the method of selecting random headers.
        @type random_headers: bool
        @param ignore_response_code: If True, ignore the method for WebErrorException for error like 404 error in veseriesonline, but it is a functional data
        @type ignore_response_code: bool
        @return: Result of the petition
        @rtype: HTTPResponse
        @param use_requests: Use requests.session()
        @type: bool

                Parameter Type Description
                -------------------------------------------------- -------------------------------------------------- ------------
                HTTPResponse.success: bool True: Request successful | False: Error when making the request
                HTTPResponse.code: int Server response code or error code if an error occurs
                HTTPResponse.error: str Description of the error in case of an error
                HTTPResponse.headers: dict Dictionary with server response headers
                HTTPResponse.data: str Response obtained from server
                HTTPResponse.json: dict Response obtained from the server in json format
                HTTPResponse.time: float Time taken to make the request

        """
    url = scrapertools.unescape(url)
    parse = urlparse.urlparse(url)
    domain = parse.netloc

    from lib import requests
    session = requests.session()

    if config.get_setting('resolver_dns') and not opt.get(
            'use_requests', False):
        from core import resolverdns
        session.mount('https://', resolverdns.CipherSuiteAdapter(domain))

    req_headers = default_headers.copy()

    # Headers passed as parameters
    if opt.get('headers', None) is not None:
        if not opt.get('replace_headers', False):
            req_headers.update(dict(opt['headers']))
        else:
            req_headers = dict(opt['headers'])

    if domain in directIP.keys() and not opt.get('disable_directIP', False):
        req_headers['Host'] = domain
        url = urlparse.urlunparse(parse._replace(netloc=directIP.get(domain)))

    if opt.get('random_headers', False) or HTTPTOOLS_DEFAULT_RANDOM_HEADERS:
        req_headers['User-Agent'] = random_useragent()
    url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")

    opt['url_save'] = url
    opt['post_save'] = opt.get('post', None)

    response = {}
    info_dict = []
    payload = dict()
    files = {}
    file_name = ''

    session.verify = opt.get('verify', True)

    if opt.get('cookies', True):
        session.cookies = cj
    session.headers.update(req_headers)

    proxy_data = {'dict': {}}

    inicio = time.time()

    if opt.get(
            'timeout',
            None) is None and HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT is not None:
        opt['timeout'] = HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT
    if opt['timeout'] == 0: opt['timeout'] = None

    if len(url) > 0:
        try:
            if opt.get('post', None) is not None or opt.get('file',
                                                            None) is not None:
                if opt.get('post', None) is not None:
                    # Convert string post in dict
                    try:
                        json.loads(opt['post'])
                        payload = opt['post']
                    except:
                        if not isinstance(opt['post'], dict):
                            post = urlparse.parse_qs(opt['post'],
                                                     keep_blank_values=1)
                            payload = dict()

                            for key, value in post.items():
                                try:
                                    payload[key] = value[0]
                                except:
                                    payload[key] = ''
                        else:
                            payload = opt['post']

                # Verify 'file' and 'file_name' options to upload a buffer or file
                if opt.get('file', None) is not None:
                    if os.path.isfile(opt['file']):
                        if opt.get('file_name', None) is None:
                            path_file, opt['file_name'] = os.path.split(
                                opt['file'])
                        files = {
                            'file': (opt['file_name'], open(opt['file'], 'rb'))
                        }
                        file_name = opt['file']
                    else:
                        files = {
                            'file': (opt.get('file_name',
                                             'Default'), opt['file'])
                        }
                        file_name = opt.get('file_name',
                                            'Default') + ', Buffer de memoria'

                info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
                if opt.get('only_headers', False):
                    # Makes the request with HEAD method
                    req = session.head(url,
                                       allow_redirects=opt.get(
                                           'follow_redirects', True),
                                       timeout=opt['timeout'])
                else:
                    # Makes the request with POST method
                    req = session.post(url,
                                       data=payload,
                                       allow_redirects=opt.get(
                                           'follow_redirects', True),
                                       files=files,
                                       timeout=opt['timeout'])

            elif opt.get('only_headers', False):
                info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
                # Makes the request with HEAD method
                req = session.head(url,
                                   allow_redirects=opt.get(
                                       'follow_redirects', True),
                                   timeout=opt['timeout'])
            else:
                info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
                # Makes the request with GET method
                req = session.get(url,
                                  allow_redirects=opt.get(
                                      'follow_redirects', True),
                                  timeout=opt['timeout'])
        except Exception as e:
            from lib import requests
            req = requests.Response()
            if not opt.get('ignore_response_code',
                           False) and not proxy_data.get('stat', ''):
                response['data'] = ''
                response['success'] = False
                info_dict.append(('Success', 'False'))
                import traceback
                response['code'] = traceback.format_exc()
                info_dict.append(('Response code', str(e)))
                info_dict.append(('Finished in', time.time() - inicio))
                if not opt.get('alfa_s', False):
                    show_infobox(info_dict)
                return type('HTTPResponse', (), response)
            else:
                req.status_code = str(e)

    else:
        response['data'] = ''
        response['success'] = False
        response['code'] = ''
        return type('HTTPResponse', (), response)

    response_code = req.status_code
    response['url'] = req.url

    response['data'] = req.content if req.content else ''

    if type(response['data']) != str:
        try:
            response['data'] = response['data'].decode('utf-8')
        except:
            response['data'] = response['data'].decode('ISO-8859-1')

    if req.headers.get('Server', '').startswith('cloudflare') and response_code in [429, 503, 403]\
            and not opt.get('CF', False) and 'Ray ID' in response['data'] and not opt.get('post', None):
        logger.debug("CF retry... for domain: %s" % domain)
        from lib import proxytranslate
        gResp = proxytranslate.process_request_proxy(url)
        if gResp:
            req = gResp['result']
            response_code = req.status_code
            response['url'] = gResp['url']
            response['data'] = gResp['data']

    if not response['data']:
        response['data'] = ''

    try:
        response['json'] = to_utf8(req.json())
    except:
        response['json'] = dict()

    response['code'] = response_code
    response['headers'] = req.headers
    response['cookies'] = req.cookies

    info_dict, response = fill_fields_post(info_dict, req, response,
                                           req_headers, inicio)

    if opt.get('cookies', True):
        save_cookies(alfa_s=opt.get('alfa_s', False))

    if not 'api.themoviedb' in url and not opt.get('alfa_s', False):
        show_infobox(info_dict)
    if not config.get_setting("debug"): logger.info('Page URL:', url)
    return type('HTTPResponse', (), response)
Exemplo n.º 40
0
    def __init__(self,
                 name,
                 url,
                 service,
                 folderIn,
                 pattern,
                 folderOut=None,
                 qualityIndex=False,
                 exceptionBehaviour={},
                 user=None,
                 password=None,
                 debug=False,
                 csvlength=5000,
                 filenamecheck=None,
                 archivefolder=None,
                 extra={}):
        """
        Info:

        name: procedure name
        url: url of the istSOS service
        service: service instance name
        folderIn: folder where raw data are stored (file:///home/pippo/rawdata)
        pattern: name of the raw data file (can contains wildcard *,
                 eg: "T_TRE_*.dat")
        folderOut: folder where the formatted istSOS type files are created
        qualityIndex: force a quality index value for all the observed
                      properties
        exceptionBehaviour: example {
            "RedundacyError": "overwrite",
            "StrictTimeSeriesError": "raise"
        }
        user and password: if user and password are required
        ...
        filenamecheck = {
            'dateformat': '12_%Y-%m-%d_%H%M%S.dat',
            'datetz': '+01:00',
            'replace': ['_P','_M'],
            'timedelta': timedelta(days=1)
        }
        """
        # Can be used to speedup directory reading doinng it only once
        #  > "folderIn" and "pattern" must be identical
        self.extra = extra

        self.fileArray = None
        self.name = name
        self.url = url
        self.service = service
        self.folderIn = folderIn
        self.pattern = pattern

        # Messages collected during processing
        self.messages = []
        self.warnings = []
        self.exceptions = []

        self.debugfile = False
        self.debugConverter = False
        if debug == 'file':
            self.debug = False
            try:
                self.debugfile = open(os.path.join(self.folderOut, "log.txt"),
                                      "w")
            except Exception as e:
                self.log(str(e))
                self.debug = True
                self.debugfile = False

        elif isinstance(debug, DebugConverter):
            self.debugConverter = debug
            self.debugConverter.addConverter(self)
        else:
            self.debug = debug

        self.addMessage("%s initialization" % name)

        self.req = requests.session()

        self.folderOut = folderOut if folderOut is not None else tempfile.mkdtemp(
        )

        self.qualityIndex = qualityIndex
        self.user = user
        self.password = password
        self.auth = (self.user,
                     self.password) if (self.user != None
                                        and self.password != None) else None

        self.archivefolder = archivefolder

        # Used inf the function "skipFile"
        self.fndtz = '+01:00'
        self.fntd = self.fnre = self.fndf = None
        if type(filenamecheck) == type({}):
            if 'dateformat' in filenamecheck:
                self.fndf = filenamecheck['dateformat']
            if 'datetz' in filenamecheck:
                self.fndtz = filenamecheck['datetz']
            if 'replace' in filenamecheck:
                self.fnre = filenamecheck['replace']
            if 'timedelta' in filenamecheck:
                if not isinstance(filenamecheck['timedelta'], timedelta):
                    raise InitializationError(
                        "filenamecheck configuration contains a timedelta attribute, it shall be and instance of datetime.timedelta."
                    )
                self.fntd = filenamecheck['timedelta']

        # >>> year = timedelta(days=365)

        # Array where Observation are stored during the parse operation
        self.observations = []
        self.observationsCheck = {}
        self.describe = None
        self.endPosition = None

        self.log("%s initialized." % self.name)

        # Single loop execution information
        self.executing = {'file': None}

        # Load describeSensor from istSOS WALib (http://localhost/istsos/wa/istsos/services/demo/procedures/T_LUGANO)
        self.loadSensorMetadata()
Exemplo n.º 41
0
    def authenticate(self):

        feedinfo = []

        try:
            with requests.session() as s:
                if mylar.VERIFY_32P == 1 or mylar.VERIFY_32P == True:
                    verify = True
                else:
                    verify = False

                logger.fdebug('[32P] Verify SSL set to : ' + str(verify))

                if not verify:
                    #32P throws back an insecure warning because it can't validate against the CA. The below suppresses the message just for 32P instead of being displa$
                    from lib.requests.packages.urllib3.exceptions import InsecureRequestWarning
                    requests.packages.urllib3.disable_warnings(
                        InsecureRequestWarning)

                # fetch the login page

                s.headers = self.headers
                try:
                    t = s.get(self.url, verify=verify, timeout=30)
                except (requests.exceptions.SSLError,
                        requests.exceptions.Timeout) as e:
                    logger.error(self.module +
                                 ' Unable to establish connection to 32P: ' +
                                 str(e))
                    return

                chksoup = BeautifulSoup(t.content)
                chksoup.prettify()
                chk_login = chksoup.find_all("form", {"id": "loginform"})
                if not chk_login:
                    logger.warn(
                        self.module +
                        ' Something is wrong - either 32p is offline, or your account has been temporarily banned (possibly).'
                    )
                    logger.warn(
                        self.module +
                        ' Disabling provider until this gets addressed by manual intervention.'
                    )
                    return "disable"

                for ck in chk_login:
                    #<div><div id='recaptchadiv'></div><input type='hidden' id='recaptchainp' value='' name='recaptchainp' /></div>
                    captcha = ck.find("div", {"id": "recaptchadiv"})
                    capt_error = ck.find("span", {
                        "class": "notice hidden",
                        "id": "formnotice"
                    })
                    error_msg = ck.find("span", {"id": "formerror"})
                    if error_msg:
                        loginerror = " ".join(list(error_msg.stripped_strings))
                        logger.warn(self.module + ' Warning: ' + loginerror)

                    if capt_error:
                        aleft = ck.find("span", {"class": "info"})
                        attemptsleft = " ".join(list(aleft.stripped_strings))
                        if int(attemptsleft) < 6:
                            logger.warn(self.module + ' ' + str(attemptsleft) +
                                        ' sign-on attempts left.')

                    if captcha:
                        logger.warn(
                            self.module +
                            ' Captcha detected. Temporariliy disabling 32p (to re-enable answer the captcha manually in a normal browswer or wait ~10 minutes...'
                        )
                        return "disable"
                    else:
                        logger.fdebug(
                            self.module +
                            ' Captcha currently not present - continuing to signon...'
                        )

                if self.test:
                    rtnmsg = ''
                    if (not capt_error
                            and not error_msg) or (capt_error
                                                   and int(attemptsleft) == 6):
                        rtnmsg += '[No Warnings/Errors]'
                    else:
                        if capt_error and int(attemptsleft) < 6:
                            rtnmsg = '[' + str(
                                attemptsleft) + ' sign-on attempts left]'
                        if error_msg:
                            rtnmsg += '[' + error_msg + ']'
                    if not captcha:
                        rtnmsg += '[No Captcha]'
                    else:
                        rtnmsg += '[Captcha Present!]'

                    return rtnmsg

                # post to the login form
                r = s.post(self.url, data=self.payload, verify=verify)

                #need a way to find response code (200=OK), but returns 200 for everything even failed signons (returns a blank page)
                #logger.info('[32P] response: ' + str(r.content))
                soup = BeautifulSoup(r.content)
                soup.prettify()
                #check for invalid username/password and if it's invalid - disable provider so we don't autoban (manual intervention is required after).
                chk_login = soup.find_all("form", {"id": "loginform"})

                for ck in chk_login:
                    captcha = ck.find("div", {"id": "recaptchadiv"})
                    errorlog = ck.find("span", {"id": "formerror"})
                    errornot = ck.find("span", {
                        "class": "notice hidden",
                        "id": "formnotice"
                    })
                    loginerror = " ".join(list(errorlog.stripped_strings)
                                          )  #login_error.findNext(text=True)
                    noticeerror = " ".join(list(errornot.stripped_strings)
                                           )  #notice_error.findNext(text=True)
                    if captcha:
                        logger.warn(
                            self.module +
                            ' Captcha detected. Temporariliy disabling 32p (to re-enable answer the captcha manually in a normal browswer or wait ~10 minutes'
                        )
                    if errorlog:
                        logger.error(self.module + ' Error: ' + loginerror)
                    if errornot:
                        aleft = ck.find("span", {"class": "info"})
                        attemptsleft = " ".join(list(aleft.stripped_strings))
                        if int(attemptsleft) < 6:
                            logger.warn(self.module + ' ' + str(attemptsleft) +
                                        ' sign-on attempts left.')
                    logger.error(
                        self.module +
                        ' Disabling 32P provider until errors can be fixed in order to avoid temporary bans.'
                    )
                    return "disable"

                if not self.searchterm:
                    logger.info(
                        '[32P] Successfully authenticated. Verifying authentication & passkeys for usage.'
                    )
                else:
                    logger.info(
                        '[32P] Successfully authenticated. Initiating search for : '
                        + self.searchterm)
                    return self.search32p(s)

                all_script = soup.find_all("script", {"src": False})
                all_script2 = soup.find_all("link", {"rel": "alternate"})

                for ind_s in all_script:
                    all_value = str(ind_s)
                    all_items = all_value.split()
                    auth_found = False
                    user_found = False
                    for al in all_items:
                        if al == 'authkey':
                            auth_found = True
                        elif auth_found == True and al != '=':
                            authkey = re.sub('["/;]', '', al).strip()
                            auth_found = False
                            logger.fdebug(self.module + ' Authkey found: ' +
                                          str(authkey))
                        if al == 'userid':
                            user_found = True
                        elif user_found == True and al != '=':
                            userid = re.sub('["/;]', '', al).strip()
                            user_found = False
                            logger.fdebug(self.module + ' Userid found: ' +
                                          str(userid))

                authfound = False
                logger.info(
                    self.module +
                    ' Atttempting to integrate with all of your 32P Notification feeds.'
                )

                for al in all_script2:
                    alurl = al['href']
                    if 'auth=' in alurl and 'torrents_notify' in alurl and not authfound:
                        f1 = alurl.find('auth=')
                        f2 = alurl.find('&', f1 + 1)
                        auth = alurl[f1 + 5:f2]
                        logger.fdebug(self.module + ' Auth:' + str(auth))
                        authfound = True
                        p1 = alurl.find('passkey=')
                        p2 = alurl.find('&', p1 + 1)
                        passkey = alurl[p1 + 8:p2]
                        logger.fdebug(self.module + ' Passkey:' + str(passkey))
                        if self.reauthenticate: break

                    if 'torrents_notify' in alurl and (
                            'torrents_notify_' + str(passkey)) not in alurl:
                        notifyname_st = alurl.find('name=')
                        notifyname_en = alurl.find('&', notifyname_st + 1)
                        if notifyname_en == -1: notifyname_en = len(alurl)
                        notifyname = alurl[notifyname_st + 5:notifyname_en]
                        notifynumber_st = alurl.find('torrents_notify_')
                        notifynumber_en = alurl.find('_', notifynumber_st + 17)
                        notifynumber = alurl[notifynumber_st:notifynumber_en]
                        logger.fdebug(self.module + ' [NOTIFICATION: ' +
                                      str(notifyname) + '] Notification ID: ' +
                                      str(notifynumber))

                        #generate the rss-url here
                        feedinfo.append({
                            'feed':
                            notifynumber + '_' + str(passkey),
                            'feedname':
                            notifyname,
                            'user':
                            userid,
                            'auth':
                            auth,
                            'passkey':
                            passkey,
                            'authkey':
                            authkey
                        })
        except (requests.exceptions.Timeout, EnvironmentError):
            logger.warn(
                'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
            )
            return

        #set the keys here that will be used to download.
        try:
            mylar.PASSKEY_32P = passkey
            mylar.AUTHKEY_32P = authkey  # probably not needed here.
            mylar.KEYS_32P = {}
            mylar.KEYS_32P = {
                "user": userid,
                "auth": auth,
                "passkey": passkey,
                "authkey": authkey
            }
        except NameError:
            logger.warn(
                'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
            )
            return

        if self.reauthenticate:
            return
        else:
            mylar.FEEDINFO_32P = feedinfo
            return feedinfo
Exemplo n.º 42
0
    def authenticate(self):

        feedinfo = []

        try:
            with requests.session() as s:
                if mylar.VERIFY_32P == 1 or mylar.VERIFY_32P == True:
                    verify = True
                else:
                    verify = False

                logger.fdebug('[32P] Verify SSL set to : ' + str(verify))

                if not verify:
                    #32P throws back an insecure warning because it can't validate against the CA. The below suppresses the message just for 32P instead of being displa$
                    from lib.requests.packages.urllib3.exceptions import InsecureRequestWarning
                    requests.packages.urllib3.disable_warnings(
                        InsecureRequestWarning)

                # fetch the login page

                s.headers = self.headers
                try:
                    s.get(self.url, verify=verify, timeout=30)
                except (requests.exceptions.SSLError,
                        requests.exceptions.Timeout) as e:
                    logger.error(self.module +
                                 ' Unable to establish connection to 32P: ' +
                                 str(e))
                    return

                # post to the login form
                r = s.post(self.url, data=self.payload, verify=verify)

                #need a way to find response code (200=OK), but returns 200 for everything even failed signons (returns a blank page)
                #logger.info('[32P] response: ' + str(r.content))

                if self.searchterm:
                    logger.info(
                        '[32P] Successfully authenticated. Initiating search for : '
                        + self.searchterm)
                    return self.search32p(s)
                soup = BeautifulSoup(r.content)
                all_script = soup.find_all("script", {"src": False})
                all_script2 = soup.find_all("link", {"rel": "alternate"})

                for ind_s in all_script:
                    all_value = str(ind_s)
                    all_items = all_value.split()
                    auth_found = False
                    user_found = False
                    for al in all_items:
                        if al == 'authkey':
                            auth_found = True
                        elif auth_found == True and al != '=':
                            authkey = re.sub('["/;]', '', al).strip()
                            auth_found = False
                            logger.fdebug(self.module + ' Authkey found: ' +
                                          str(authkey))
                        if al == 'userid':
                            user_found = True
                        elif user_found == True and al != '=':
                            userid = re.sub('["/;]', '', al).strip()
                            user_found = False
                            logger.fdebug(self.module + ' Userid found: ' +
                                          str(userid))

                authfound = False
                logger.info(
                    self.module +
                    ' Atttempting to integrate with all of your 32P Notification feeds.'
                )

                for al in all_script2:
                    alurl = al['href']
                    if 'auth=' in alurl and 'torrents_notify' in alurl and not authfound:
                        f1 = alurl.find('auth=')
                        f2 = alurl.find('&', f1 + 1)
                        auth = alurl[f1 + 5:f2]
                        logger.fdebug(self.module + ' Auth:' + str(auth))
                        authfound = True
                        p1 = alurl.find('passkey=')
                        p2 = alurl.find('&', p1 + 1)
                        passkey = alurl[p1 + 8:p2]
                        logger.fdebug(self.module + ' Passkey:' + str(passkey))
                        if self.reauthenticate: break

                    if 'torrents_notify' in alurl and (
                            'torrents_notify_' + str(passkey)) not in alurl:
                        notifyname_st = alurl.find('name=')
                        notifyname_en = alurl.find('&', notifyname_st + 1)
                        if notifyname_en == -1: notifyname_en = len(alurl)
                        notifyname = alurl[notifyname_st + 5:notifyname_en]
                        notifynumber_st = alurl.find('torrents_notify_')
                        notifynumber_en = alurl.find('_', notifynumber_st + 17)
                        notifynumber = alurl[notifynumber_st:notifynumber_en]
                        logger.fdebug(self.module + ' [NOTIFICATION: ' +
                                      str(notifyname) + '] Notification ID: ' +
                                      str(notifynumber))

                        #generate the rss-url here
                        feedinfo.append({
                            'feed':
                            notifynumber + '_' + str(passkey),
                            'feedname':
                            notifyname,
                            'user':
                            userid,
                            'auth':
                            auth,
                            'passkey':
                            passkey,
                            'authkey':
                            authkey
                        })
        except (requests.exceptions.Timeout, EnvironmentError):
            logger.warn(
                'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
            )
            return

        #set the keys here that will be used to download.
        try:
            mylar.PASSKEY_32P = passkey
            mylar.AUTHKEY_32P = authkey  # probably not needed here.
            mylar.KEYS_32P = {}
            mylar.KEYS_32P = {
                "user": userid,
                "auth": auth,
                "passkey": passkey,
                "authkey": authkey
            }
        except NameError:
            logger.warn(
                'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
            )
            return

        if self.reauthenticate:
            return
        else:
            mylar.FEEDINFO_32P = feedinfo
            return feedinfo