def normalize(title): try: if py_tools.isPY2: try: return py_tools.ensure_str( py_tools.ensure_text(title, encoding='ascii')) except: pass # return str(''.join(c for c in unicodedata.normalize('NFKD', unicode(title.decode('utf-8'))) if unicodedata.category(c) != 'Mn')) return ''.join(c for c in unicodedata.normalize( 'NFKD', py_tools.ensure_text(py_tools.ensure_str(title))) if unicodedata.category(c) != 'Mn') except: log_utils.error() return title
def normalize(title): try: return ''.join(c for c in unicodedata.normalize( 'NFKD', py_tools.ensure_text(py_tools.ensure_str(title))) if unicodedata.category(c) != 'Mn') except: error() return title
def playListItems(): rpc = '{"jsonrpc": "2.0", "method": "Playlist.GetItems", "params": {"playlistid" : %s}, "id": 1 }' % Id result = control.jsonrpc(rpc) limits =jsloads(result)['result']['limits'] total = limits['total'] if int(total) <= 0: return [] result = py_tools.ensure_text(result, errors='ignore') result = jsloads(result)['result']['items'] try: return [i['label'] for i in result] except: return []
def openMenuFile(self, menuFile): try: req = Request(menuFile) req.add_header('User-Agent', self.agent) response = urlopen(req) link = response.read() link = py_tools.ensure_text(link, errors='ignore') response.close() return link except: log_utils.error()
def log(msg, caller=None, level=LOGNOTICE): debug_enabled = control.setting('debug.enabled') == 'true' if not debug_enabled: return debug_level = control.setting('debug.level') if level == LOGDEBUG and debug_level != '1': return debug_location = control.setting('debug.location') if isinstance(msg, int): msg = control.lang(msg) # for strings.po translations try: if py_tools.isPY3: if not msg.isprintable( ): # ex. "\n" is not a printable character so returns False on those sort of cases msg = '%s (NORMALIZED by log_utils.log())' % normalize(msg) if isinstance(msg, py_tools.binary_type): msg = '%s (ENCODED by log_utils.log())' % (py_tools.ensure_str( msg, errors='replace')) else: if not is_printable( msg ): # if not all(c in printable for c in msg): # .isprintable() not available in py2 msg = normalize(msg) if isinstance(msg, py_tools.binary_type): msg = '%s (ENCODED by log_utils.log())' % ( py_tools.ensure_text(msg)) if caller is not None and level != LOGERROR: func = inspect.currentframe().f_back.f_code line_number = inspect.currentframe().f_back.f_lineno caller = "%s.%s()" % (caller, func.co_name) msg = 'From func name: %s Line # :%s\n msg : %s' % ( caller, line_number, msg) elif caller is not None and level == LOGERROR: msg = 'From func name: %s.%s() Line # :%s\n msg : %s' % ( caller[0], caller[1], caller[2], msg) if debug_location == '1': log_file = control.joinPath(LOGPATH, 'venom.log') if not control.existsPath(log_file): f = open(log_file, 'w') f.close() with open(log_file, 'a', encoding='utf-8') as f: line = '[%s %s] %s: %s' % ( datetime.now().date(), str(datetime.now().time())[:8], DEBUGPREFIX % debug_list[level], msg) f.write(line.rstrip('\r\n') + '\n') # f.writelines([line1, line2]) ## maybe an option for the 2 lines without using "\n" else: xbmc.log('%s: %s' % (DEBUGPREFIX % debug_list[level], msg, level)) except Exception as e: import traceback traceback.print_exc() xbmc.log( '[ plugin.video.venom ] log_utils.log() Logging Failure: %s' % (e), LOGERROR)
def ckKodiSources(self, paths=None): contains = False try: if paths is None: paths = [] movie_LibraryFolder = control.joinPath(control.transPath(control.setting('library.movie')), '') special_movie_LibraryFolder = control.joinPath(control.setting('library.movie'), '') paths.append(movie_LibraryFolder) paths.append(special_movie_LibraryFolder) tvShows_LibraryFolder = control.joinPath(control.transPath(control.setting('library.tv')),'') speical_tvShows_LibraryFolder = control.joinPath(control.setting('library.tv'),'') paths.append(tvShows_LibraryFolder) paths.append(speical_tvShows_LibraryFolder) paths = [i.rstrip('/').rstrip('\\') for i in paths] result = control.jsonrpc('{"jsonrpc": "2.0", "method": "Files.GetSources", "params": {"media" : "video"}, "id": 1}') result = py_tools.ensure_text(result, errors='ignore') result = jsloads(result)['result']['sources'] for i in result: if i['file'].rstrip('/').rstrip('\\') in paths: contains = True break except: log_utils.error() if not contains: try: global folder_setup global service_update if control.setting('library.service.update') == 'false' or service_update is False: return contains if folder_setup: contains = True return contains msg = 'Your Library Folders do not exist in Kodi Sources. Would you like to run full setup of Library Folders to Kodi Sources now?' if control.yesnoDialog(msg, '', ''): lib_tools.total_setup() folder_setup = True contains = True else: msg = 'Would you like to turn off Library Auto Update Service?' if control.yesnoDialog(msg, '', ''): service_update = False control.setSetting('library.service.update', 'false') contains = False control.notification(message=32112) # control.refresh() except: log_utils.error() return contains
def add(self, name, title, year, imdb, tmdb, range=False): try: contains = lib_tools().ckKodiSources() if general_notification: if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'): control.notification(title=name, message=32552) try: if not self.dupe_chk == 'true': raise Exception() id = [imdb, tmdb] if tmdb != '0' else [imdb] lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovies", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties" : ["imdbnumber", "title", "originaltitle", "year"]}, "id": 1}' % (year, str(int(year)+1), str(int(year)-1))) lib = py_tools.ensure_text(lib, errors='ignore') lib = jsloads(lib)['result']['movies'] # lib = [i for i in lib if str(i['imdbnumber']) in id or (cleantitle.get(title) in [cleantitle.get(i['title'].encode('utf-8')), cleantitle.get(i['originaltitle'].encode('utf-8'))] and str(i['year']) == year)] lib = [i for i in lib if str(i['imdbnumber']) in id or (cleantitle.get(title) in [cleantitle.get(i['title']), cleantitle.get(i['originaltitle'])] and str(i['year']) == year)] except: lib = [] files_added = 0 try: if lib != []: raise Exception() if self.check_setting == 'true': src = lib_tools.check_sources(title, year, imdb, None, None, None, None, None) if not src: raise Exception() self.strmFile({'name': name, 'title': title, 'year': year, 'imdb': imdb, 'tmdb': tmdb}) files_added += 1 except: pass if files_added == 0 and general_notification: control.notification(title=name, message=32652) if range: return files_added if self.library_update == 'true' and not control.condVisibility('Library.IsScanningVideo') and files_added > 0: if contains: if general_notification: control.notification(title=name, message=32554) control.sleep(10000) control.execute('UpdateLibrary(video)') elif general_notification: control.notification(title=name, message=32104) except: pass
def getMeta(self, meta): try: if not meta: raise Exception() poster = meta.get('poster3') or meta.get('poster2') or meta.get( 'poster') #poster2 and poster3 may not be passed anymore thumb = meta.get('thumb') thumb = thumb or poster or control.addonThumb() season_poster = meta.get('season_poster') or poster fanart = meta.get('fanart') banner = meta.get('banner') clearart = meta.get('clearart') clearlogo = meta.get('clearlogo') discart = meta.get('discart') if 'mediatype' not in meta: meta.update( {'mediatype': 'episode' if self.episode else 'movie'}) if self.episode: meta.update({'season': self.season}) meta.update({'episode': self.episode}) meta.update({'tvshowtitle': self.title}) return (poster, thumb, season_poster, fanart, banner, clearart, clearlogo, discart, meta) except: log_utils.error() try: if self.media_type != 'movie': raise Exception() meta = control.jsonrpc( '{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovies", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties" : ["title", "originaltitle", "year", "genre", "studio", "country", "runtime", "rating", "votes", "mpaa", "director", "writer", "plot", "plotoutline", "tagline", "thumbnail", "file"]}, "id": 1}' % (self.year, str(int(self.year) + 1), str(int(self.year) - 1))) meta = py_tools.ensure_text(meta, errors='ignore') meta = jsloads(meta)['result']['movies'] t = cleantitle.get(self.title) meta = [ i for i in meta if self.year == str(i['year']) and (t == cleantitle.get( i['title']) or t == cleantitle.get(i['originaltitle'])) ][0] if 'mediatype' not in meta: meta.update({'mediatype': 'movie'}) if 'duration' not in meta: meta.update({'duration': meta.get('runtime') / 60 }) # this doesn't make sense for k, v in control.iteritems(meta): if type(v) == list: try: meta[k] = str(' / '.join([i for i in v])) except: meta[k] = '' else: try: meta[k] = str(v) except: meta[k] = str(v) if 'plugin' not in control.infoLabel('Container.PluginName'): self.DBID = meta.get('movieid') poster = thumb = meta.get('thumbnail') return (poster, thumb, '', '', '', '', '', '', meta) except: log_utils.error() try: if self.media_type != 'episode': raise Exception() meta = control.jsonrpc( '{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties" : ["title", "year", "thumbnail", "file"]}, "id": 1}' % (self.year, str(int(self.year) + 1), str(int(self.year) - 1))) meta = py_tools.ensure_text(meta, errors='ignore') meta = jsloads(meta)['result']['tvshows'] t = cleantitle.get(self.title) meta = [ i for i in meta if self.year == str(i['year']) and t == cleantitle.get(i['title']) ][0] tvshowid = meta['tvshowid'] poster = meta['thumbnail'] meta = control.jsonrpc( '{"jsonrpc": "2.0", "method": "VideoLibrary.GetEpisodes", "params":{ "tvshowid": %d, "filter":{"and": [{"field": "season", "operator": "is", "value": "%s"}, {"field": "episode", "operator": "is", "value": "%s"}]}, "properties": ["title", "season", "episode", "showtitle", "firstaired", "runtime", "rating", "director", "writer", "plot", "thumbnail", "file"]}, "id": 1}' % (tvshowid, self.season, self.episode)) meta = py_tools.ensure_text(meta, errors='ignore') meta = jsloads(meta)['result']['episodes'][0] if 'mediatype' not in meta: meta.update({'mediatype': 'episode'}) if 'duration' not in meta: meta.update({'duration': meta.get('runtime') / 60 }) # this doesn't make sense for k, v in control.iteritems(meta): if type(v) == list: try: meta[k] = str(' / '.join([i for i in v])) except: meta[k] = '' else: try: meta[k] = str(v) except: meta[k] = str(v) if 'plugin' not in control.infoLabel('Container.PluginName'): self.DBID = meta.get('episodeid') thumb = meta['thumbnail'] return (poster, thumb, '', '', '', '', '', '', meta) except: log_utils.error() poster, thumb, season_poster, fanart, banner, clearart, clearlogo, discart, meta = '', '', '', '', '', '', '', '', { 'title': self.name } return (poster, thumb, season_poster, fanart, banner, clearart, clearlogo, discart, meta)
def update(self): if control.setting('library.service.update') == 'false': control.notification(message=32106) contains = lib_tools().ckKodiSources() if not contains: control.notification(message=32107) return try: items, season, episode = [], [], [] show = [control.joinPath(self.library_folder, i) for i in control.listDir(self.library_folder)[0]] if show == []: control.notification(message=32108) return for s in show: try: season += [control.joinPath(s, i) for i in control.listDir(s)[0]] except: pass for s in season: try: episode.append([control.joinPath(s, i) for i in control.listDir(s)[1] if i.endswith('.strm')][-1]) except: pass for file in episode: try: file = control.openFile(file) read = file.read() # read = read.encode('utf-8') file.close() if not read.startswith(sys.argv[0]): continue params = dict(parse_qsl(read.replace('?',''))) try: tvshowtitle = params['tvshowtitle'] except: tvshowtitle = None try: tvshowtitle = params['show'] except: pass if tvshowtitle is None or tvshowtitle == '': continue year, imdb, tvdb = params['year'], params['imdb'], params['tvdb'] imdb = 'tt' + re.sub(r'[^0-9]', '', str(imdb)) tmdb = params.get('tmdb', '0') items.append({'tvshowtitle': tvshowtitle, 'year': year, 'imdb': imdb, 'tmdb': tmdb, 'tvdb': tvdb}) except: pass items = [i for x, i in enumerate(items) if i not in items[x + 1:]] if len(items) == 0: return except: log_utils.error() return try: lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"properties": ["imdbnumber", "title", "year"]}, "id": 1 }') # lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties": ["imdbnumber", "title", "year"]}, "id": 1}' % (year, str(int(year)+1), str(int(year)-1))) # lib = unicode(lib, 'utf-8', errors='ignore') lib = py_tools.ensure_text(lib, errors='ignore') lib = jsloads(lib)['result']['tvshows'] except: log_utils.error() return if service_notification and not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'): control.notification(message=32553) try: control.makeFile(control.dataPath) dbcon = database.connect(control.libcacheFile) dbcur = dbcon.cursor() dbcur.execute('''CREATE TABLE IF NOT EXISTS tvshows (id TEXT, items TEXT, UNIQUE(id));''') dbcur.connection.commit() # from resources.lib.menus import episodes from resources.lib.menus import seasons except: log_utils.error() try: dbcur.close() ; dbcon.close() except: pass return files_added = 0 # __init__ doesn't get called from services so self.date never gets updated and new episodes are not added to the library # self.date_time = (datetime.utcnow() - timedelta(hours=5)) self.date_time = datetime.utcnow() if control.setting('library.importdelay') != 'true': self.date = self.date_time.strftime('%Y%m%d') else: self.date = (self.date_time - timedelta(hours=24)).strftime('%Y%m%d') for item in items: it = None if control.monitor.abortRequested(): try: dbcur.close() ; dbcon.close() except: pass return sys.exit() try: fetch = dbcur.execute("SELECT * FROM tvshows WHERE id = '%s'" % item['tvdb']).fetchone() # if fetch: it = eval(fetch[1].encode('utf-8')) if fetch: it = eval(fetch[1]) except: log_utils.error() try: if it: raise Exception() # it = episodes.Episodes().get(item['tvshowtitle'], item['year'], item['imdb'], item['tmdb'], item['tvdb'], idx=False) it = seasons.Seasons().tvdb_list(item['tvshowtitle'], item['year'], item['imdb'], item['tmdb'], item['tvdb'], control.apiLanguage()['tvdb'], '-1') # fetch fresh meta (uncached) if not it: continue status = it[0]['status'].lower() it = [{'title': i['title'], 'year': i['year'], 'imdb': i['imdb'], 'tmdb': i['tmdb'], 'tvdb': i['tvdb'], 'season': i['season'], 'episode': i['episode'], 'tvshowtitle': i['tvshowtitle'], 'premiered': i['premiered']} for i in it] if status == 'continuing': raise Exception() dbcur.execute("INSERT INTO tvshows Values (?, ?)", (item['tvdb'], repr(it))) dbcur.connection.commit() except: log_utils.error() try: id = [item['imdb'], item['tvdb']] if item['tmdb'] != '0': id += [item['tmdb']] # ep = [x['title'].encode('utf-8') for x in lib if str(x['imdbnumber']) in id or (x['title'].encode('utf-8') == item['tvshowtitle'] and str(x['year']) == item['year'])][0] ep = [x['title'] for x in lib if str(x['imdbnumber']) in id or (x['title'] == item['tvshowtitle'] and str(x['year']) == item['year'])][0] ep = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetEpisodes", "params": {"filter":{"and": [{"field": "tvshow", "operator": "is", "value": "%s"}]}, "properties": ["season", "episode"]}, "id": 1}' % ep) ep = py_tools.ensure_text(ep, errors='ignore') ep = jsloads(ep).get('result', {}).get('episodes', {}) ep = [{'season': int(i['season']), 'episode': int(i['episode'])} for i in ep] ep = sorted(ep, key = lambda x: (x['season'], x['episode']))[-1] num = [x for x,y in enumerate(it) if str(y['season']) == str(ep['season']) and str(y['episode']) == str(ep['episode'])][-1] it = [y for x,y in enumerate(it) if x > num] if len(it) == 0: continue except: # log_utils.error() continue for i in it: if control.monitor.abortRequested(): return sys.exit() try: # Show Season Special(Season0). if str(i.get('season')) == '0' and control.setting('tv.specials') == 'false': continue # Show Unaired items. premiered = i.get('premiered', '0') if premiered == '0' and self.include_unknown == 'false': continue elif int(re.sub(r'[^0-9]', '', str(premiered))) > int(re.sub(r'[^0-9]', '', str(self.date))): if self.showunaired != 'true': continue libtvshows().strmFile(i) files_added += 1 if service_notification : control.notification(title=item['tvshowtitle'], message=32678) except: log_utils.error() try: dbcur.close() ; dbcon.close() except: pass if files_added == 0 and service_notification : control.notification(message=32109) if self.library_update == 'true' and not control.condVisibility('Library.IsScanningVideo') and files_added > 0: if contains: if service_notification: control.notification(message=32554) control.sleep(10000) control.execute('UpdateLibrary(video)') elif service_notification: control.notification(message=32103)
def add(self, tvshowtitle, year, imdb, tmdb, tvdb, range=False): try: contains = lib_tools().ckKodiSources() if general_notification: if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'): control.notification(title=tvshowtitle, message=32552) try: # from resources.lib.menus import episodes # items = episodes.Episodes().get(tvshowtitle, year, imdb, tmdb, tvdb, idx=False) from resources.lib.menus import seasons items = seasons.Seasons().tvdb_list(tvshowtitle, year, imdb, tmdb, tvdb, control.apiLanguage()['tvdb'], '-1') # fetch fresh meta (uncached) except: log_utils.error() return status = items[0]['status'].lower() try: items = [{'title': i['title'], 'year': i['year'], 'imdb': i['imdb'], 'tmdb': i['tmdb'], 'tvdb': i['tvdb'], 'season': i['season'], 'episode': i['episode'], 'tvshowtitle': i['tvshowtitle'], 'premiered': i['premiered']} for i in items] except: items = [] if items == []: return try: if self.dupe_chk != 'true': raise Exception() id = [items[0]['imdb'], items[0]['tvdb']] # lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"properties" : ["imdbnumber", "title", "year"]}, "id": 1}') lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties": ["imdbnumber", "title", "year"]}, "id": 1}' % (year, str(int(year)+1), str(int(year)-1))) lib = py_tools.ensure_text(lib, errors='ignore') lib = jsloads(lib)['result']['tvshows'] # lib = [i['title'].encode('utf-8') for i in lib if str(i['imdbnumber']) in id or (i['title'].encode('utf-8') == items[0]['tvshowtitle'] and str(i['year']) == items[0]['year'])][0] lib = [i['title'] for i in lib if str(i['imdbnumber']) in id or (i['title'] == items[0]['tvshowtitle'] and str(i['year']) == items[0]['year'])][0] lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetEpisodes", "params": {"filter":{"and": [{"field": "tvshow", "operator": "is", "value": "%s"}]}, "properties": ["season", "episode"]}, "id": 1}' % lib) lib = py_tools.ensure_text(lib, errors='ignore') lib = jsloads(lib)['result']['episodes'] lib = ['S%02dE%02d' % (int(i['season']), int(i['episode'])) for i in lib] items = [i for i in items if not 'S%02dE%02d' % (int(i['season']), int(i['episode'])) in lib] except: lib = [] files_added = 0 for i in items: if lib != []: continue if control.monitor.abortRequested(): return sys.exit() try: if self.check_setting == 'true': if i['episode'] == '1': self.block = True src = lib_tools.check_sources(i['title'], i['year'], i['imdb'], i['tvdb'], i['season'], i['episode'], i['tvshowtitle'], i['premiered']) if src: self.block = False if self.block: continue # Show Season Special(Season0). if str(i.get('season')) == '0' and control.setting('tv.specials') == 'false': continue # Show Unaired or Unknown items. premiered = i.get('premiered', '0') if premiered == '0' and self.include_unknown == 'false': continue elif int(re.sub(r'[^0-9]', '', str(premiered))) > int(re.sub(r'[^0-9]', '', str(self.date))): if self.showunaired != 'true': continue self.strmFile(i) files_added += 1 except: log_utils.error() if files_added == 0 and general_notification: control.notification(title=tvshowtitle, message=32652) if range: return files_added if self.library_update == 'true' and not control.condVisibility('Library.IsScanningVideo') and files_added > 0: if contains: if general_notification: control.notification(title=tvshowtitle, message=32554) control.execute('UpdateLibrary(video)') elif general_notification: control.notification(title=tvshowtitle, message=32104) except: pass
def request(url, close=True, redirect=True, error=False, proxy=None, post=None, headers=None, mobile=False, XHR=False, limit=None, referer=None, cookie=None, compression=True, output='', timeout='30', verifySsl=True, flare=True, ignoreErrors=None, as_bytes=False): try: if not url: return None if url.startswith('//'): url = 'http:' + url try: url = py_tools.ensure_text(url, errors='ignore') except: pass if isinstance(post, dict): post = bytes(urlencode(post), encoding='utf-8') elif isinstance(post, str) and py_tools.isPY3: post = bytes(post, encoding='utf-8') handlers = [] if proxy is not None: handlers += [ urllib2.ProxyHandler({'http': '%s' % (proxy)}), urllib2.HTTPHandler ] opener = urllib2.build_opener(*handlers) urllib2.install_opener(opener) if output == 'cookie' or output == 'extended' or close is not True: cookies = cookielib.LWPCookieJar() handlers += [ urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies) ] opener = urllib2.build_opener(*handlers) urllib2.install_opener(opener) if not verifySsl and sys.version_info >= (2, 7, 12): try: import ssl ssl_context = ssl._create_unverified_context() handlers += [urllib2.HTTPSHandler(context=ssl_context)] opener = urllib2.build_opener(*handlers) urllib2.install_opener(opener) except: log_utils.error() if verifySsl and ((2, 7, 8) < sys.version_info < (2, 7, 12)): # try: # import ssl # ssl_context = ssl.create_default_context() # ssl_context.check_hostname = False # ssl_context.verify_mode = ssl.CERT_NONE # handlers += [urllib2.HTTPSHandler(context=ssl_context)] # opener = urllib2.build_opener(*handlers) # urllib2.install_opener(opener) # except: # log_utils.error() try: import ssl try: import _ssl CERT_NONE = _ssl.CERT_NONE except Exception: CERT_NONE = ssl.CERT_NONE ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = CERT_NONE handlers += [urllib2.HTTPSHandler(context=ssl_context)] opener = urllib2.build_opener(*handlers) urllib2.install_opener(opener) except: log_utils.error() try: headers.update(headers) except: headers = {} if 'User-Agent' in headers: pass elif mobile is not True: headers['User-Agent'] = cache.get(randomagent, 12) else: headers['User-Agent'] = 'Apple-iPhone/701.341' if 'Referer' in headers: pass elif referer is not None: headers['Referer'] = referer if 'Accept-Language' not in headers: headers['Accept-Language'] = 'en-US' if 'X-Requested-With' in headers: pass elif XHR: headers['X-Requested-With'] = 'XMLHttpRequest' if 'Cookie' in headers: pass elif cookie: headers['Cookie'] = cookie if 'Accept-Encoding' in headers: pass elif compression and limit is None: headers['Accept-Encoding'] = 'gzip' # if redirect is False: # class NoRedirection(urllib2.HTTPErrorProcessor): # def http_response(self, request, response): # return response # opener = urllib2.build_opener(NoRedirection) # urllib2.install_opener(opener) # try: del headers['Referer'] # except: pass if redirect is False: class NoRedirectHandler(urllib2.HTTPRedirectHandler): def http_error_302(self, reqst, fp, code, msg, head): infourl = addinfourl(fp, head, reqst.get_full_url()) infourl.status = code infourl.code = code return infourl http_error_300 = http_error_302 http_error_301 = http_error_302 http_error_303 = http_error_302 http_error_307 = http_error_302 opener = urllib2.build_opener(NoRedirectHandler()) urllib2.install_opener(opener) try: del headers['Referer'] except: pass req = urllib2.Request(url, data=post) _add_request_header(req, headers) try: response = urllib2.urlopen(req, timeout=int(timeout)) except HTTPError as error_response: # if HTTPError, using "as response" will be reset after entire Exception code runs and throws error around line 247 as "local variable 'response' referenced before assignment", re-assign it response = error_response try: ignore = ignoreErrors and (int(response.code) == ignoreErrors or int( response.code) in ignoreErrors) except: ignore = False if not ignore: if response.code in [ 301, 307, 308, 503, 403 ]: # 403:Forbidden added 3/3/21 for cloudflare, fails on bad User-Agent cf_result = response.read(5242880) log_utils.log('cf_result = %s' % str(cf_result), level=log_utils.LOGDEBUG) try: encoding = response.headers["Content-Encoding"] except: encoding = None if encoding == 'gzip': cf_result = gzip.GzipFile( fileobj=StringIO(cf_result)).read() if flare and 'cloudflare' in str(response.info()).lower(): log_utils.log( 'client module calling cfscrape: url=%s' % url, level=log_utils.LOGDEBUG) try: from fenomscrapers.modules import cfscrape if isinstance(post, dict): data = post else: try: data = parse_qs(post) except: data = None scraper = cfscrape.CloudScraper() if response.code == 403: # possible bad User-Agent in headers, let cfscrape assign response = scraper.request( method='GET' if post is None else 'POST', url=url, data=data, timeout=int(timeout)) else: response = scraper.request( method='GET' if post is None else 'POST', url=url, headers=headers, data=data, timeout=int(timeout)) result = response.content flare = 'cloudflare' # Used below try: cookies = response.request._cookies except: log_utils.error() if response.status_code == 403: # if cfscrape server still responds with 403 log_utils.log( 'cfscrape-Error url=(%s): %s' % (url, 'HTTP Error 403: Forbidden'), __name__, level=log_utils.LOGDEBUG) return None except: log_utils.error() elif 'cf-browser-verification' in cf_result: netloc = '%s://%s' % (urlparse(url).scheme, urlparse(url).netloc) ua = headers['User-Agent'] cf = cache.get(cfcookie().get, 168, netloc, ua, timeout) headers['Cookie'] = cf req = urllib2.Request(url, data=post) _add_request_header(req, headers) response = urllib2.urlopen(req, timeout=int(timeout)) else: if error is False: log_utils.error('Request-Error url=(%s)' % url) return None else: if error is False: log_utils.error('Request-Error url=(%s)' % url) return None elif error is True and response.code in [ 401, 404, 405 ]: # no point in continuing after this exception runs with these response.code's try: response_headers = dict( [(item[0].title(), item[1]) for item in list(response.info().items())] ) # behaves differently 18 to 19. 18 I had 3 "Set-Cookie:" it combined all 3 values into 1 key. In 19 only the last keys value was present. except: log_utils.error() response_headers = response.headers return (str(response), str(response.code), response_headers) if output == 'cookie': try: result = '; '.join( ['%s=%s' % (i.name, i.value) for i in cookies]) except: pass try: result = cf except: pass if close is True: response.close() return result elif output == 'geturl': result = response.geturl() if close is True: response.close() return result elif output == 'headers': result = response.headers if close is True: response.close() return result elif output == 'chunk': try: content = int(response.headers['Content-Length']) except: content = (2049 * 1024) if content < (2048 * 1024): return try: result = response.read(16 * 1024) except: result = response # testing if close is True: response.close() return result elif output == 'file_size': try: content = int(response.headers['Content-Length']) except: content = '0' if close is True: response.close() return content if flare != 'cloudflare': if limit == '0': result = response.read(224 * 1024) elif limit is not None: result = response.read(int(limit) * 1024) else: result = response.read(5242880) try: encoding = response.headers["Content-Encoding"] except: encoding = None if encoding == 'gzip': result = gzip.GzipFile(fileobj=StringIO(result)).read() if not as_bytes: result = py_tools.ensure_text(result, errors='ignore') if not as_bytes and 'sucuri_cloudproxy_js' in result: # who da f**k? su = sucuri().get(result) headers['Cookie'] = su req = urllib2.Request(url, data=post) _add_request_header(req, headers) response = urllib2.urlopen(req, timeout=int(timeout)) if limit == '0': result = response.read(224 * 1024) elif limit is not None: result = response.read(int(limit) * 1024) else: result = response.read(5242880) try: encoding = response.headers["Content-Encoding"] except: encoding = None if encoding == 'gzip': result = gzip.GzipFile(fileobj=StringIO(result)).read() if not as_bytes and 'Blazingfast.io' in result and 'xhr.open' in result: # who da f**k? netloc = '%s://%s' % (urlparse(url).scheme, urlparse(url).netloc) ua = headers['User-Agent'] headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua, timeout) result = _basic_request(url, headers=headers, post=post, timeout=timeout, limit=limit) if output == 'extended': try: response_headers = dict( [(item[0].title(), item[1]) for item in list(response.info().items())] ) # behaves differently 18 to 19. 18 I had 3 "Set-Cookie:" it combined all 3 values into 1 key. In 19 only the last keys value was present. except: log_utils.error() response_headers = response.headers try: response_code = str(response.code) except: response_code = str(response.status_code ) # object from CFScrape Requests object. try: cookie = '; '.join( ['%s=%s' % (i.name, i.value) for i in cookies]) except: pass try: cookie = cf except: pass if close is True: response.close() return (result, response_code, response_headers, headers, cookie) else: if close is True: response.close() return result except: log_utils.error('Request-Error url=(%s)' % url) return None