def normalize(title):
try:
if py_tools.isPY2:
try:
return py_tools.ensure_str(
py_tools.ensure_text(title, encoding='ascii'))
except:
pass
# return str(''.join(c for c in unicodedata.normalize('NFKD', unicode(title.decode('utf-8'))) if unicodedata.category(c) != 'Mn'))
return ''.join(c for c in unicodedata.normalize(
'NFKD', py_tools.ensure_text(py_tools.ensure_str(title)))
if unicodedata.category(c) != 'Mn')
except:
log_utils.error()
return title
def normalize(title):
try:
return ''.join(c for c in unicodedata.normalize(
'NFKD', py_tools.ensure_text(py_tools.ensure_str(title)))
if unicodedata.category(c) != 'Mn')
except:
error()
return title
def playListItems():
rpc = '{"jsonrpc": "2.0", "method": "Playlist.GetItems", "params": {"playlistid" : %s}, "id": 1 }' % Id
result = control.jsonrpc(rpc)
limits =jsloads(result)['result']['limits']
total = limits['total']
if int(total) <= 0: return []
result = py_tools.ensure_text(result, errors='ignore')
result = jsloads(result)['result']['items']
try: return [i['label'] for i in result]
except: return []
def openMenuFile(self, menuFile):
try:
req = Request(menuFile)
req.add_header('User-Agent', self.agent)
response = urlopen(req)
link = response.read()
link = py_tools.ensure_text(link, errors='ignore')
response.close()
return link
except:
log_utils.error()
def log(msg, caller=None, level=LOGNOTICE):
debug_enabled = control.setting('debug.enabled') == 'true'
if not debug_enabled: return
debug_level = control.setting('debug.level')
if level == LOGDEBUG and debug_level != '1': return
debug_location = control.setting('debug.location')
if isinstance(msg, int):
msg = control.lang(msg) # for strings.po translations
try:
if py_tools.isPY3:
if not msg.isprintable(
): # ex. "\n" is not a printable character so returns False on those sort of cases
msg = '%s (NORMALIZED by log_utils.log())' % normalize(msg)
if isinstance(msg, py_tools.binary_type):
msg = '%s (ENCODED by log_utils.log())' % (py_tools.ensure_str(
msg, errors='replace'))
else:
if not is_printable(
msg
): # if not all(c in printable for c in msg): # .isprintable() not available in py2
msg = normalize(msg)
if isinstance(msg, py_tools.binary_type):
msg = '%s (ENCODED by log_utils.log())' % (
py_tools.ensure_text(msg))
if caller is not None and level != LOGERROR:
func = inspect.currentframe().f_back.f_code
line_number = inspect.currentframe().f_back.f_lineno
caller = "%s.%s()" % (caller, func.co_name)
msg = 'From func name: %s Line # :%s\n msg : %s' % (
caller, line_number, msg)
elif caller is not None and level == LOGERROR:
msg = 'From func name: %s.%s() Line # :%s\n msg : %s' % (
caller[0], caller[1], caller[2], msg)
if debug_location == '1':
log_file = control.joinPath(LOGPATH, 'venom.log')
if not control.existsPath(log_file):
f = open(log_file, 'w')
f.close()
with open(log_file, 'a', encoding='utf-8') as f:
line = '[%s %s] %s: %s' % (
datetime.now().date(), str(datetime.now().time())[:8],
DEBUGPREFIX % debug_list[level], msg)
f.write(line.rstrip('\r\n') + '\n')
# f.writelines([line1, line2]) ## maybe an option for the 2 lines without using "\n"
else:
xbmc.log('%s: %s' % (DEBUGPREFIX % debug_list[level], msg, level))
except Exception as e:
import traceback
traceback.print_exc()
xbmc.log(
'[ plugin.video.venom ] log_utils.log() Logging Failure: %s' % (e),
LOGERROR)
def ckKodiSources(self, paths=None):
contains = False
try:
if paths is None:
paths = []
movie_LibraryFolder = control.joinPath(control.transPath(control.setting('library.movie')), '')
special_movie_LibraryFolder = control.joinPath(control.setting('library.movie'), '')
paths.append(movie_LibraryFolder)
paths.append(special_movie_LibraryFolder)
tvShows_LibraryFolder = control.joinPath(control.transPath(control.setting('library.tv')),'')
speical_tvShows_LibraryFolder = control.joinPath(control.setting('library.tv'),'')
paths.append(tvShows_LibraryFolder)
paths.append(speical_tvShows_LibraryFolder)
paths = [i.rstrip('/').rstrip('\\') for i in paths]
result = control.jsonrpc('{"jsonrpc": "2.0", "method": "Files.GetSources", "params": {"media" : "video"}, "id": 1}')
result = py_tools.ensure_text(result, errors='ignore')
result = jsloads(result)['result']['sources']
for i in result:
if i['file'].rstrip('/').rstrip('\\') in paths:
contains = True
break
except:
log_utils.error()
if not contains:
try:
global folder_setup
global service_update
if control.setting('library.service.update') == 'false' or service_update is False:
return contains
if folder_setup:
contains = True
return contains
msg = 'Your Library Folders do not exist in Kodi Sources. Would you like to run full setup of Library Folders to Kodi Sources now?'
if control.yesnoDialog(msg, '', ''):
lib_tools.total_setup()
folder_setup = True
contains = True
else:
msg = 'Would you like to turn off Library Auto Update Service?'
if control.yesnoDialog(msg, '', ''):
service_update = False
control.setSetting('library.service.update', 'false')
contains = False
control.notification(message=32112)
# control.refresh()
except:
log_utils.error()
return contains
def add(self, name, title, year, imdb, tmdb, range=False):
try:
contains = lib_tools().ckKodiSources()
if general_notification:
if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'):
control.notification(title=name, message=32552)
try:
if not self.dupe_chk == 'true': raise Exception()
id = [imdb, tmdb] if tmdb != '0' else [imdb]
lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovies", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties" : ["imdbnumber", "title", "originaltitle", "year"]}, "id": 1}' % (year, str(int(year)+1), str(int(year)-1)))
lib = py_tools.ensure_text(lib, errors='ignore')
lib = jsloads(lib)['result']['movies']
# lib = [i for i in lib if str(i['imdbnumber']) in id or (cleantitle.get(title) in [cleantitle.get(i['title'].encode('utf-8')), cleantitle.get(i['originaltitle'].encode('utf-8'))] and str(i['year']) == year)]
lib = [i for i in lib if str(i['imdbnumber']) in id or (cleantitle.get(title) in [cleantitle.get(i['title']), cleantitle.get(i['originaltitle'])] and str(i['year']) == year)]
except:
lib = []
files_added = 0
try:
if lib != []: raise Exception()
if self.check_setting == 'true':
src = lib_tools.check_sources(title, year, imdb, None, None, None, None, None)
if not src: raise Exception()
self.strmFile({'name': name, 'title': title, 'year': year, 'imdb': imdb, 'tmdb': tmdb})
files_added += 1
except: pass
if files_added == 0 and general_notification:
control.notification(title=name, message=32652)
if range: return files_added
if self.library_update == 'true' and not control.condVisibility('Library.IsScanningVideo') and files_added > 0:
if contains:
if general_notification: control.notification(title=name, message=32554)
control.sleep(10000)
control.execute('UpdateLibrary(video)')
elif general_notification: control.notification(title=name, message=32104)
except: pass
def getMeta(self, meta):
try:
if not meta: raise Exception()
poster = meta.get('poster3') or meta.get('poster2') or meta.get(
'poster') #poster2 and poster3 may not be passed anymore
thumb = meta.get('thumb')
thumb = thumb or poster or control.addonThumb()
season_poster = meta.get('season_poster') or poster
fanart = meta.get('fanart')
banner = meta.get('banner')
clearart = meta.get('clearart')
clearlogo = meta.get('clearlogo')
discart = meta.get('discart')
if 'mediatype' not in meta:
meta.update(
{'mediatype': 'episode' if self.episode else 'movie'})
if self.episode:
meta.update({'season': self.season})
meta.update({'episode': self.episode})
meta.update({'tvshowtitle': self.title})
return (poster, thumb, season_poster, fanart, banner, clearart,
clearlogo, discart, meta)
except:
log_utils.error()
try:
if self.media_type != 'movie': raise Exception()
meta = control.jsonrpc(
'{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovies", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties" : ["title", "originaltitle", "year", "genre", "studio", "country", "runtime", "rating", "votes", "mpaa", "director", "writer", "plot", "plotoutline", "tagline", "thumbnail", "file"]}, "id": 1}'
%
(self.year, str(int(self.year) + 1), str(int(self.year) - 1)))
meta = py_tools.ensure_text(meta, errors='ignore')
meta = jsloads(meta)['result']['movies']
t = cleantitle.get(self.title)
meta = [
i for i in meta
if self.year == str(i['year']) and (t == cleantitle.get(
i['title']) or t == cleantitle.get(i['originaltitle']))
][0]
if 'mediatype' not in meta:
meta.update({'mediatype': 'movie'})
if 'duration' not in meta:
meta.update({'duration': meta.get('runtime') / 60
}) # this doesn't make sense
for k, v in control.iteritems(meta):
if type(v) == list:
try:
meta[k] = str(' / '.join([i for i in v]))
except:
meta[k] = ''
else:
try:
meta[k] = str(v)
except:
meta[k] = str(v)
if 'plugin' not in control.infoLabel('Container.PluginName'):
self.DBID = meta.get('movieid')
poster = thumb = meta.get('thumbnail')
return (poster, thumb, '', '', '', '', '', '', meta)
except:
log_utils.error()
try:
if self.media_type != 'episode': raise Exception()
meta = control.jsonrpc(
'{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties" : ["title", "year", "thumbnail", "file"]}, "id": 1}'
%
(self.year, str(int(self.year) + 1), str(int(self.year) - 1)))
meta = py_tools.ensure_text(meta, errors='ignore')
meta = jsloads(meta)['result']['tvshows']
t = cleantitle.get(self.title)
meta = [
i for i in meta if self.year == str(i['year'])
and t == cleantitle.get(i['title'])
][0]
tvshowid = meta['tvshowid']
poster = meta['thumbnail']
meta = control.jsonrpc(
'{"jsonrpc": "2.0", "method": "VideoLibrary.GetEpisodes", "params":{ "tvshowid": %d, "filter":{"and": [{"field": "season", "operator": "is", "value": "%s"}, {"field": "episode", "operator": "is", "value": "%s"}]}, "properties": ["title", "season", "episode", "showtitle", "firstaired", "runtime", "rating", "director", "writer", "plot", "thumbnail", "file"]}, "id": 1}'
% (tvshowid, self.season, self.episode))
meta = py_tools.ensure_text(meta, errors='ignore')
meta = jsloads(meta)['result']['episodes'][0]
if 'mediatype' not in meta:
meta.update({'mediatype': 'episode'})
if 'duration' not in meta:
meta.update({'duration': meta.get('runtime') / 60
}) # this doesn't make sense
for k, v in control.iteritems(meta):
if type(v) == list:
try:
meta[k] = str(' / '.join([i for i in v]))
except:
meta[k] = ''
else:
try:
meta[k] = str(v)
except:
meta[k] = str(v)
if 'plugin' not in control.infoLabel('Container.PluginName'):
self.DBID = meta.get('episodeid')
thumb = meta['thumbnail']
return (poster, thumb, '', '', '', '', '', '', meta)
except:
log_utils.error()
poster, thumb, season_poster, fanart, banner, clearart, clearlogo, discart, meta = '', '', '', '', '', '', '', '', {
'title': self.name
}
return (poster, thumb, season_poster, fanart, banner, clearart,
clearlogo, discart, meta)
def update(self):
if control.setting('library.service.update') == 'false': control.notification(message=32106)
contains = lib_tools().ckKodiSources()
if not contains:
control.notification(message=32107)
return
try:
items, season, episode = [], [], []
show = [control.joinPath(self.library_folder, i) for i in control.listDir(self.library_folder)[0]]
if show == []:
control.notification(message=32108)
return
for s in show:
try: season += [control.joinPath(s, i) for i in control.listDir(s)[0]]
except: pass
for s in season:
try: episode.append([control.joinPath(s, i) for i in control.listDir(s)[1] if i.endswith('.strm')][-1])
except: pass
for file in episode:
try:
file = control.openFile(file)
read = file.read()
# read = read.encode('utf-8')
file.close()
if not read.startswith(sys.argv[0]): continue
params = dict(parse_qsl(read.replace('?','')))
try: tvshowtitle = params['tvshowtitle']
except: tvshowtitle = None
try: tvshowtitle = params['show']
except: pass
if tvshowtitle is None or tvshowtitle == '': continue
year, imdb, tvdb = params['year'], params['imdb'], params['tvdb']
imdb = 'tt' + re.sub(r'[^0-9]', '', str(imdb))
tmdb = params.get('tmdb', '0')
items.append({'tvshowtitle': tvshowtitle, 'year': year, 'imdb': imdb, 'tmdb': tmdb, 'tvdb': tvdb})
except: pass
items = [i for x, i in enumerate(items) if i not in items[x + 1:]]
if len(items) == 0: return
except:
log_utils.error()
return
try:
lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"properties": ["imdbnumber", "title", "year"]}, "id": 1 }')
# lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties": ["imdbnumber", "title", "year"]}, "id": 1}' % (year, str(int(year)+1), str(int(year)-1)))
# lib = unicode(lib, 'utf-8', errors='ignore')
lib = py_tools.ensure_text(lib, errors='ignore')
lib = jsloads(lib)['result']['tvshows']
except:
log_utils.error()
return
if service_notification and not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'):
control.notification(message=32553)
try:
control.makeFile(control.dataPath)
dbcon = database.connect(control.libcacheFile)
dbcur = dbcon.cursor()
dbcur.execute('''CREATE TABLE IF NOT EXISTS tvshows (id TEXT, items TEXT, UNIQUE(id));''')
dbcur.connection.commit()
# from resources.lib.menus import episodes
from resources.lib.menus import seasons
except:
log_utils.error()
try: dbcur.close() ; dbcon.close()
except: pass
return
files_added = 0
# __init__ doesn't get called from services so self.date never gets updated and new episodes are not added to the library
# self.date_time = (datetime.utcnow() - timedelta(hours=5))
self.date_time = datetime.utcnow()
if control.setting('library.importdelay') != 'true': self.date = self.date_time.strftime('%Y%m%d')
else: self.date = (self.date_time - timedelta(hours=24)).strftime('%Y%m%d')
for item in items:
it = None
if control.monitor.abortRequested():
try: dbcur.close() ; dbcon.close()
except: pass
return sys.exit()
try:
fetch = dbcur.execute("SELECT * FROM tvshows WHERE id = '%s'" % item['tvdb']).fetchone()
# if fetch: it = eval(fetch[1].encode('utf-8'))
if fetch: it = eval(fetch[1])
except:
log_utils.error()
try:
if it: raise Exception()
# it = episodes.Episodes().get(item['tvshowtitle'], item['year'], item['imdb'], item['tmdb'], item['tvdb'], idx=False)
it = seasons.Seasons().tvdb_list(item['tvshowtitle'], item['year'], item['imdb'], item['tmdb'], item['tvdb'], control.apiLanguage()['tvdb'], '-1') # fetch fresh meta (uncached)
if not it: continue
status = it[0]['status'].lower()
it = [{'title': i['title'], 'year': i['year'], 'imdb': i['imdb'], 'tmdb': i['tmdb'], 'tvdb': i['tvdb'], 'season': i['season'], 'episode': i['episode'], 'tvshowtitle': i['tvshowtitle'], 'premiered': i['premiered']} for i in it]
if status == 'continuing': raise Exception()
dbcur.execute("INSERT INTO tvshows Values (?, ?)", (item['tvdb'], repr(it)))
dbcur.connection.commit()
except:
log_utils.error()
try:
id = [item['imdb'], item['tvdb']]
if item['tmdb'] != '0':
id += [item['tmdb']]
# ep = [x['title'].encode('utf-8') for x in lib if str(x['imdbnumber']) in id or (x['title'].encode('utf-8') == item['tvshowtitle'] and str(x['year']) == item['year'])][0]
ep = [x['title'] for x in lib if str(x['imdbnumber']) in id or (x['title'] == item['tvshowtitle'] and str(x['year']) == item['year'])][0]
ep = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetEpisodes", "params": {"filter":{"and": [{"field": "tvshow", "operator": "is", "value": "%s"}]}, "properties": ["season", "episode"]}, "id": 1}' % ep)
ep = py_tools.ensure_text(ep, errors='ignore')
ep = jsloads(ep).get('result', {}).get('episodes', {})
ep = [{'season': int(i['season']), 'episode': int(i['episode'])} for i in ep]
ep = sorted(ep, key = lambda x: (x['season'], x['episode']))[-1]
num = [x for x,y in enumerate(it) if str(y['season']) == str(ep['season']) and str(y['episode']) == str(ep['episode'])][-1]
it = [y for x,y in enumerate(it) if x > num]
if len(it) == 0: continue
except:
# log_utils.error()
continue
for i in it:
if control.monitor.abortRequested(): return sys.exit()
try:
# Show Season Special(Season0).
if str(i.get('season')) == '0' and control.setting('tv.specials') == 'false': continue
# Show Unaired items.
premiered = i.get('premiered', '0')
if premiered == '0' and self.include_unknown == 'false': continue
elif int(re.sub(r'[^0-9]', '', str(premiered))) > int(re.sub(r'[^0-9]', '', str(self.date))):
if self.showunaired != 'true': continue
libtvshows().strmFile(i)
files_added += 1
if service_notification :
control.notification(title=item['tvshowtitle'], message=32678)
except:
log_utils.error()
try: dbcur.close() ; dbcon.close()
except: pass
if files_added == 0 and service_notification :
control.notification(message=32109)
if self.library_update == 'true' and not control.condVisibility('Library.IsScanningVideo') and files_added > 0:
if contains:
if service_notification:
control.notification(message=32554)
control.sleep(10000)
control.execute('UpdateLibrary(video)')
elif service_notification:
control.notification(message=32103)
def add(self, tvshowtitle, year, imdb, tmdb, tvdb, range=False):
try:
contains = lib_tools().ckKodiSources()
if general_notification:
if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'):
control.notification(title=tvshowtitle, message=32552)
try:
# from resources.lib.menus import episodes
# items = episodes.Episodes().get(tvshowtitle, year, imdb, tmdb, tvdb, idx=False)
from resources.lib.menus import seasons
items = seasons.Seasons().tvdb_list(tvshowtitle, year, imdb, tmdb, tvdb, control.apiLanguage()['tvdb'], '-1') # fetch fresh meta (uncached)
except:
log_utils.error()
return
status = items[0]['status'].lower()
try: items = [{'title': i['title'], 'year': i['year'], 'imdb': i['imdb'], 'tmdb': i['tmdb'], 'tvdb': i['tvdb'], 'season': i['season'], 'episode': i['episode'], 'tvshowtitle': i['tvshowtitle'], 'premiered': i['premiered']} for i in items]
except: items = []
if items == []: return
try:
if self.dupe_chk != 'true': raise Exception()
id = [items[0]['imdb'], items[0]['tvdb']]
# lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"properties" : ["imdbnumber", "title", "year"]}, "id": 1}')
lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": {"filter":{"or": [{"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}, {"field": "year", "operator": "is", "value": "%s"}]}, "properties": ["imdbnumber", "title", "year"]}, "id": 1}' % (year, str(int(year)+1), str(int(year)-1)))
lib = py_tools.ensure_text(lib, errors='ignore')
lib = jsloads(lib)['result']['tvshows']
# lib = [i['title'].encode('utf-8') for i in lib if str(i['imdbnumber']) in id or (i['title'].encode('utf-8') == items[0]['tvshowtitle'] and str(i['year']) == items[0]['year'])][0]
lib = [i['title'] for i in lib if str(i['imdbnumber']) in id or (i['title'] == items[0]['tvshowtitle'] and str(i['year']) == items[0]['year'])][0]
lib = control.jsonrpc('{"jsonrpc": "2.0", "method": "VideoLibrary.GetEpisodes", "params": {"filter":{"and": [{"field": "tvshow", "operator": "is", "value": "%s"}]}, "properties": ["season", "episode"]}, "id": 1}' % lib)
lib = py_tools.ensure_text(lib, errors='ignore')
lib = jsloads(lib)['result']['episodes']
lib = ['S%02dE%02d' % (int(i['season']), int(i['episode'])) for i in lib]
items = [i for i in items if not 'S%02dE%02d' % (int(i['season']), int(i['episode'])) in lib]
except:
lib = []
files_added = 0
for i in items:
if lib != []: continue
if control.monitor.abortRequested(): return sys.exit()
try:
if self.check_setting == 'true':
if i['episode'] == '1':
self.block = True
src = lib_tools.check_sources(i['title'], i['year'], i['imdb'], i['tvdb'], i['season'], i['episode'], i['tvshowtitle'], i['premiered'])
if src: self.block = False
if self.block: continue
# Show Season Special(Season0).
if str(i.get('season')) == '0' and control.setting('tv.specials') == 'false': continue
# Show Unaired or Unknown items.
premiered = i.get('premiered', '0')
if premiered == '0' and self.include_unknown == 'false': continue
elif int(re.sub(r'[^0-9]', '', str(premiered))) > int(re.sub(r'[^0-9]', '', str(self.date))):
if self.showunaired != 'true': continue
self.strmFile(i)
files_added += 1
except:
log_utils.error()
if files_added == 0 and general_notification: control.notification(title=tvshowtitle, message=32652)
if range: return files_added
if self.library_update == 'true' and not control.condVisibility('Library.IsScanningVideo') and files_added > 0:
if contains:
if general_notification: control.notification(title=tvshowtitle, message=32554)
control.execute('UpdateLibrary(video)')
elif general_notification: control.notification(title=tvshowtitle, message=32104)
except: pass
def request(url,
close=True,
redirect=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
XHR=False,
limit=None,
referer=None,
cookie=None,
compression=True,
output='',
timeout='30',
verifySsl=True,
flare=True,
ignoreErrors=None,
as_bytes=False):
try:
if not url: return None
if url.startswith('//'): url = 'http:' + url
try:
url = py_tools.ensure_text(url, errors='ignore')
except:
pass
if isinstance(post, dict):
post = bytes(urlencode(post), encoding='utf-8')
elif isinstance(post, str) and py_tools.isPY3:
post = bytes(post, encoding='utf-8')
handlers = []
if proxy is not None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or close is not True:
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
if not verifySsl and sys.version_info >= (2, 7, 12):
try:
import ssl
ssl_context = ssl._create_unverified_context()
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
except:
log_utils.error()
if verifySsl and ((2, 7, 8) < sys.version_info < (2, 7, 12)):
# try:
# import ssl
# ssl_context = ssl.create_default_context()
# ssl_context.check_hostname = False
# ssl_context.verify_mode = ssl.CERT_NONE
# handlers += [urllib2.HTTPSHandler(context=ssl_context)]
# opener = urllib2.build_opener(*handlers)
# urllib2.install_opener(opener)
# except:
# log_utils.error()
try:
import ssl
try:
import _ssl
CERT_NONE = _ssl.CERT_NONE
except Exception:
CERT_NONE = ssl.CERT_NONE
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
except:
log_utils.error()
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers: pass
elif mobile is not True:
headers['User-Agent'] = cache.get(randomagent, 12)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers: pass
elif referer is not None: headers['Referer'] = referer
if 'Accept-Language' not in headers:
headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in headers: pass
elif XHR: headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in headers: pass
elif cookie: headers['Cookie'] = cookie
if 'Accept-Encoding' in headers: pass
elif compression and limit is None: headers['Accept-Encoding'] = 'gzip'
# if redirect is False:
# class NoRedirection(urllib2.HTTPErrorProcessor):
# def http_response(self, request, response):
# return response
# opener = urllib2.build_opener(NoRedirection)
# urllib2.install_opener(opener)
# try: del headers['Referer']
# except: pass
if redirect is False:
class NoRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_302(self, reqst, fp, code, msg, head):
infourl = addinfourl(fp, head, reqst.get_full_url())
infourl.status = code
infourl.code = code
return infourl
http_error_300 = http_error_302
http_error_301 = http_error_302
http_error_303 = http_error_302
http_error_307 = http_error_302
opener = urllib2.build_opener(NoRedirectHandler())
urllib2.install_opener(opener)
try:
del headers['Referer']
except:
pass
req = urllib2.Request(url, data=post)
_add_request_header(req, headers)
try:
response = urllib2.urlopen(req, timeout=int(timeout))
except HTTPError as error_response: # if HTTPError, using "as response" will be reset after entire Exception code runs and throws error around line 247 as "local variable 'response' referenced before assignment", re-assign it
response = error_response
try:
ignore = ignoreErrors and (int(response.code) == ignoreErrors
or int(
response.code) in ignoreErrors)
except:
ignore = False
if not ignore:
if response.code in [
301, 307, 308, 503, 403
]: # 403:Forbidden added 3/3/21 for cloudflare, fails on bad User-Agent
cf_result = response.read(5242880)
log_utils.log('cf_result = %s' % str(cf_result),
level=log_utils.LOGDEBUG)
try:
encoding = response.headers["Content-Encoding"]
except:
encoding = None
if encoding == 'gzip':
cf_result = gzip.GzipFile(
fileobj=StringIO(cf_result)).read()
if flare and 'cloudflare' in str(response.info()).lower():
log_utils.log(
'client module calling cfscrape: url=%s' % url,
level=log_utils.LOGDEBUG)
try:
from fenomscrapers.modules import cfscrape
if isinstance(post, dict): data = post
else:
try:
data = parse_qs(post)
except:
data = None
scraper = cfscrape.CloudScraper()
if response.code == 403: # possible bad User-Agent in headers, let cfscrape assign
response = scraper.request(
method='GET' if post is None else 'POST',
url=url,
data=data,
timeout=int(timeout))
else:
response = scraper.request(
method='GET' if post is None else 'POST',
url=url,
headers=headers,
data=data,
timeout=int(timeout))
result = response.content
flare = 'cloudflare' # Used below
try:
cookies = response.request._cookies
except:
log_utils.error()
if response.status_code == 403: # if cfscrape server still responds with 403
log_utils.log(
'cfscrape-Error url=(%s): %s' %
(url, 'HTTP Error 403: Forbidden'),
__name__,
level=log_utils.LOGDEBUG)
return None
except:
log_utils.error()
elif 'cf-browser-verification' in cf_result:
netloc = '%s://%s' % (urlparse(url).scheme,
urlparse(url).netloc)
ua = headers['User-Agent']
cf = cache.get(cfcookie().get, 168, netloc, ua,
timeout)
headers['Cookie'] = cf
req = urllib2.Request(url, data=post)
_add_request_header(req, headers)
response = urllib2.urlopen(req, timeout=int(timeout))
else:
if error is False:
log_utils.error('Request-Error url=(%s)' % url)
return None
else:
if error is False:
log_utils.error('Request-Error url=(%s)' % url)
return None
elif error is True and response.code in [
401, 404, 405
]: # no point in continuing after this exception runs with these response.code's
try:
response_headers = dict(
[(item[0].title(), item[1])
for item in list(response.info().items())]
) # behaves differently 18 to 19. 18 I had 3 "Set-Cookie:" it combined all 3 values into 1 key. In 19 only the last keys value was present.
except:
log_utils.error()
response_headers = response.headers
return (str(response), str(response.code),
response_headers)
if output == 'cookie':
try:
result = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
result = cf
except:
pass
if close is True: response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close is True: response.close()
return result
elif output == 'headers':
result = response.headers
if close is True: response.close()
return result
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024): return
try:
result = response.read(16 * 1024)
except:
result = response # testing
if close is True: response.close()
return result
elif output == 'file_size':
try:
content = int(response.headers['Content-Length'])
except:
content = '0'
if close is True: response.close()
return content
if flare != 'cloudflare':
if limit == '0': result = response.read(224 * 1024)
elif limit is not None: result = response.read(int(limit) * 1024)
else: result = response.read(5242880)
try:
encoding = response.headers["Content-Encoding"]
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO(result)).read()
if not as_bytes:
result = py_tools.ensure_text(result, errors='ignore')
if not as_bytes and 'sucuri_cloudproxy_js' in result: # who da f**k?
su = sucuri().get(result)
headers['Cookie'] = su
req = urllib2.Request(url, data=post)
_add_request_header(req, headers)
response = urllib2.urlopen(req, timeout=int(timeout))
if limit == '0': result = response.read(224 * 1024)
elif limit is not None: result = response.read(int(limit) * 1024)
else: result = response.read(5242880)
try:
encoding = response.headers["Content-Encoding"]
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO(result)).read()
if not as_bytes and 'Blazingfast.io' in result and 'xhr.open' in result: # who da f**k?
netloc = '%s://%s' % (urlparse(url).scheme, urlparse(url).netloc)
ua = headers['User-Agent']
headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua,
timeout)
result = _basic_request(url,
headers=headers,
post=post,
timeout=timeout,
limit=limit)
if output == 'extended':
try:
response_headers = dict(
[(item[0].title(), item[1])
for item in list(response.info().items())]
) # behaves differently 18 to 19. 18 I had 3 "Set-Cookie:" it combined all 3 values into 1 key. In 19 only the last keys value was present.
except:
log_utils.error()
response_headers = response.headers
try:
response_code = str(response.code)
except:
response_code = str(response.status_code
) # object from CFScrape Requests object.
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
cookie = cf
except:
pass
if close is True: response.close()
return (result, response_code, response_headers, headers, cookie)
else:
if close is True: response.close()
return result
except:
log_utils.error('Request-Error url=(%s)' % url)
return None