def _get_episode_url(self, show_url, video):
episode_pattern = 'href="([^"]+-s0*%se0*%s(?!\d)[^"]*)' % (video.season, video.episode)
result = self._default_get_episode_url(show_url, video, episode_pattern)
if result:
return result
url = urlparse.urljoin(self.base_url, show_url)
html = self._http_get(url, cache_limit=2)
fragment = dom_parser.parse_dom(html, "ul", {"class": "episode_list"})
if fragment:
ep_urls = dom_parser.parse_dom(fragment[0], "a", ret="href")
ep_dates = dom_parser.parse_dom(fragment[0], "span", {"class": "episode_air_d"})
ep_titles = dom_parser.parse_dom(fragment[0], "span", {"class": "episode_name"})
force_title = scraper_utils.force_title(video)
if not force_title and kodi.get_setting("airdate-fallback") == "true" and video.ep_airdate:
for ep_url, ep_date in zip(ep_urls, ep_dates):
log_utils.log(
"Quikr Ep Airdate Matching: %s - %s - %s" % (ep_url, ep_date, video.ep_airdate),
log_utils.LOGDEBUG,
)
if video.ep_airdate == scraper_utils.to_datetime(ep_date, "%Y-%m-%d").date():
return scraper_utils.pathify_url(ep_url)
if force_title or kodi.get_setting("title-fallback") == "true":
norm_title = scraper_utils.normalize_title(video.ep_title)
for ep_url, ep_title in zip(ep_urls, ep_titles):
ep_title = re.sub("<span>.*?</span>\s*", "", ep_title)
log_utils.log(
"Quikr Ep Title Matching: %s - %s - %s" % (ep_url, norm_title, video.ep_title),
log_utils.LOGDEBUG,
)
if norm_title == scraper_utils.normalize_title(ep_title):
return scraper_utils.pathify_url(ep_url)
def getChannelGuideUrl(tvchannel):
url = 'https://www.cinemagia.ro/program-tv/'
try:
req = urllib2.Request(url)
req.add_header('User-Agent', common.HEADERS['User-Agent'])
conn = urllib2.urlopen(req, timeout=5)
html = conn.read()
conn.close()
urls = parseDOM(html, 'a', attrs={'class': 'station-link'}, ret='href')
names = parseDOM(html, 'a', attrs={'class': 'station-link'})
seq2 = tvchannel.lower().replace(' ', '').strip()
seqm = SequenceMatcher()
seqm.set_seq2(seq2)
ratio_list = []
for name in names:
seq1 = name.lower().replace(' ', '').strip()
seqm.set_seq1(seq1)
ratio = seqm.ratio()
ratio_list.append(ratio)
ratio_max_index = max(xrange(len(ratio_list)), key=ratio_list.__getitem__)
return urls[ratio_max_index]
except:
log_utils.log(traceback.print_exc())
return None
def download_subtitle(self, url):
url = BASE_URL + url
(response, srt) = self.__get_url(url)
if "Content-Disposition" not in response.info():
return
cd = response.info()["Content-Disposition"]
r = re.search('filename="(.*)"', cd)
if r:
filename = r.group(1)
else:
filename = "addic7ed_subtitle.srt"
final_path = os.path.join(BASE_PATH, filename)
final_path = xbmc.translatePath(final_path)
if not xbmcvfs.exists(os.path.dirname(final_path)):
try:
try:
xbmcvfs.mkdirs(os.path.dirname(final_path))
except:
os.mkdir(os.path.dirname(final_path))
except:
log_utils.log("Failed to create directory %s" % os.path.dirname(final_path), log_utils.LOGERROR)
raise
with open(final_path, "w") as f:
f.write(srt)
return final_path
def __get_cached_url(self, url, cache=8):
log_utils.log("Fetching Cached URL: %s" % url, log_utils.LOGDEBUG)
before = time.time()
_, html = db_connection.get_cached_url(url, cache)
if html:
log_utils.log("Returning cached result for: %s" % (url), log_utils.LOGDEBUG)
return html
log_utils.log("No cached url found for: %s" % url, log_utils.LOGDEBUG)
req = urllib2.Request(url)
host = BASE_URL.replace("http://", "")
req.add_header("User-Agent", USER_AGENT)
req.add_header("Host", host)
req.add_header("Referer", BASE_URL)
try:
body = self.__http_get_with_retry(url, req)
body = body.decode("utf-8")
parser = HTMLParser.HTMLParser()
body = parser.unescape(body)
except Exception as e:
kodi.notify(msg="Failed to connect to URL: %s" % (url), duration=5000)
log_utils.log("Failed to connect to URL %s: (%s)" % (url, e), log_utils.LOGERROR)
return ""
db_connection.cache_url(url, body)
after = time.time()
log_utils.log("Cached Url Fetch took: %.2f secs" % (after - before), log_utils.LOGDEBUG)
return body
def __get_links_from_playlist(self, grab_url, headers):
sources = {}
grab_url = grab_url.replace('\\', '')
grab_html = self._http_get(grab_url, headers=headers, cache_limit=.5)
js_data = scraper_utils.parse_json(grab_html, grab_url)
try: playlist = js_data['playlist'][0]['sources']
except: playlist = []
for item in playlist:
stream_url = item.get('file')
if stream_url:
if stream_url.startswith('/'):
stream_url = urlparse.urljoin(self.base_url, stream_url)
redir_url = self._http_get(stream_url, headers=headers, allow_redirect=False, method='HEAD')
if redir_url.startswith('http'):
stream_url = redir_url
if self._get_direct_hostname(stream_url) == 'gvideo':
quality = scraper_utils.gv_get_quality(stream_url)
elif 'label' in item:
quality = scraper_utils.height_get_quality(item['label'])
else:
quality = QUALITIES.HIGH
log_utils.log('Adding stream: %s Quality: %s' % (stream_url, quality), log_utils.LOGDEBUG)
sources[stream_url] = {'quality': quality, 'direct': True}
if not kodi.get_setting('scraper_url'): break
return sources
def __get_cached_url(self, url, cache=8):
log_utils.log('Fetching Cached URL: %s' % url, xbmc.LOGDEBUG)
before = time.time()
_, html = db_connection.get_cached_url(url, cache)
if html:
log_utils.log('Returning cached result for: %s' % (url), xbmc.LOGDEBUG)
return html
log_utils.log('No cached url found for: %s' % url, xbmc.LOGDEBUG)
req = urllib2.Request(url)
host = BASE_URL.replace('http://', '')
req.add_header('User-Agent', USER_AGENT)
req.add_header('Host', host)
req.add_header('Referer', BASE_URL)
try:
body = self.__http_get_with_retry(url, req)
body = body.decode('utf-8')
parser = HTMLParser.HTMLParser()
body = parser.unescape(body)
except Exception as e:
builtin = 'XBMC.Notification(PrimeWire, Failed to connect to URL: %s, 5000, %s)'
xbmc.executebuiltin(builtin % (url, ICON_PATH))
log_utils.log('Failed to connect to URL %s: (%s)' % (url, e), xbmc.LOGERROR)
return ''
db_connection.cache_url(url, body)
after = time.time()
log_utils.log('Cached Url Fetch took: %.2f secs' % (after - before), xbmc.LOGDEBUG)
return body
def _get_episode_url(self, show_url, video):
query = urlparse.parse_qs(urlparse.urlparse(show_url).query)
if 'id' in query:
url = urlparse.urljoin(self.base_url, '/api/v2/shows/%s' % (query['id'][0]))
js_data = self._http_get(url, cache_limit=.5)
if 'episodes' in js_data:
force_title = scraper_utils.force_title(video)
if not force_title:
for episode in js_data['episodes']:
if int(video.season) == int(episode['season']) and int(video.episode) == int(episode['number']):
return scraper_utils.pathify_url('?id=%s' % (episode['id']))
if kodi.get_setting('airdate-fallback') == 'true' and video.ep_airdate:
for episode in js_data['episodes']:
if 'airdate' in episode:
ep_airdate = scraper_utils.to_datetime(episode['airdate'], "%Y-%m-%d").date()
if video.ep_airdate == (ep_airdate - datetime.timedelta(days=1)):
return scraper_utils.pathify_url('?id=%s' % (episode['id']))
else:
log_utils.log('Skipping S&E matching as title search is forced on: %s' % (video.trakt_id), log_utils.LOGDEBUG)
if (force_title or kodi.get_setting('title-fallback') == 'true') and video.ep_title:
norm_title = scraper_utils.normalize_title(video.ep_title)
for episode in js_data['episodes']:
if 'name' in episode and norm_title in scraper_utils.normalize_title(episode['name']):
return scraper_utils.pathify_url('?id=%s' % (episode['id']))
def parse_link(link, item, patterns):
delim = '[._ -]'
link = urllib.unquote(link)
file_name = link.split('/')[-1]
for pattern in patterns:
pattern = pattern.format(delim=delim)
match = re.search(pattern, file_name, re.I)
if match:
match = dict((k, v) for k, v in match.groupdict().iteritems() if v is not None)
item.update(match)
break
else:
log_utils.log('No Regex Match: |%s|%s|' % (item, link), log_utils.LOGDEBUG)
extra = item['extra'].upper()
if 'X265' in extra or 'HEVC' in extra:
item['format'] = 'x265'
item['dubbed'] = True if 'DUBBED' in extra else False
if 'airdate' in item and item['airdate']:
pattern = '{delim}+'.format(delim=delim)
item['airdate'] = re.sub(pattern, '-', item['airdate'])
item['airdate'] = utils2.to_datetime(item['airdate'], "%Y-%m-%d").date()
return item
def resolve_link(self, link):
try:
xbmcvfs.delete(M3U8_PATH)
query = urlparse.parse_qs(link)
query = dict([(key, query[key][0]) if query[key] else (key, '') for key in query])
if 'vid_id' in query and 'stream_id' in query and 'height' in query:
auth_url = PL_URL % (query['vid_id'], query['stream_id'])
result = self.__get_playlist_with_token(auth_url)
if not result:
if int(query['height']) > 720:
if self.auth_torba():
result = self.__get_playlist_with_token(auth_url)
else:
result = self.__authorize_ip(auth_url)
if result:
key = '%sp' % (query['height'])
if key in result:
if 'audio' in result:
streams = {'audio_stream': result['audio'], 'stream_name': key, 'video_stream': result[key]}
f = xbmcvfs.File(M3U8_PATH, 'w')
for line in M3U8_TEMPLATE:
line = line.format(**streams)
f.write(line + '\n')
return M3U8_PATH
else:
return result[key]
except Exception as e:
log_utils.log('Failure during torba resolver: %s' % (e), log_utils.LOGWARNING)
def __init__(self, timeout=scraper.DEFAULT_TIMEOUT):
self.timeout = timeout
self.__scraper = None
try:
self.__scraper = self.real_scraper(timeout)
except Exception as e:
log_utils.log('Failure during %s scraper creation: %s' % (self.get_name(), e), log_utils.LOGDEBUG)
def do_startup_task(task):
run_on_startup=ADDON.get_setting('auto-%s' % task)=='true' and ADDON.get_setting('%s-during-startup' % task) == 'true'
if run_on_startup and not xbmc.abortRequested:
log_utils.log('Service: Running startup task [%s]' % (task))
now = datetime.datetime.now()
xbmc.executebuiltin('RunPlugin(plugin://%s/?mode=%s)' % (ADDON.get_id(), task))
db_connection.set_setting('%s-last_run' % (task), now.strftime("%Y-%m-%d %H:%M:%S.%f"))
def _getHtml():
global LOGIN_COOKIE_FILE
global __email
global __password
html = ''
tries = 2
while (tries > 0):
try:
cj = mechanize.LWPCookieJar()
cj.load(LOGIN_COOKIE_FILE)
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cj))
req = mechanize.Request(common.URLS['login'])
for k, v in common.HEADERS.items():
req.add_header(k, v)
conn = opener.open(req)
cj.save(LOGIN_COOKIE_FILE)
html = conn.read()
conn.close()
except:
log_utils.log(traceback.print_exc())
if html and __email in html:
break
else:
doLogin(__email, __password)
tries -= 1
time.sleep(1)
return html
def finder1(html,url):
global limit
ref=url
try:
urls = re.findall('<i?frame.+?src=(?:\'|\")(.+?)(?:\'|\")',html)
try:
urls.append(re.findall("playStream\('iframe', '(.+?)'\)",html)[0])
except: pass
urls += re.findall('<a.+?href=[\'\"](/live-.+?stream.+?)[\'\"]',html)
from random import shuffle
shuffle(urls)
for url in urls:
if 'c4.zedo' in url:
continue
if "micast" in url or 'turbocast' in url:
return finder47(html,ref)
rr = resolve_it(url)
if rr:
return rr
uri = manual_fix(url,ref)
if limit>=25:
log("Exiting - iframe visit limit reached")
return
resolved = find_link(uri)
if resolved:
break
headers = {'User-Agent': client.agent(), 'Referer': ref}
if '.m3u8' in resolved and '|' not in resolved:
headers.update({'X-Requested-With':'ShockwaveFlash/20.0.0.286', 'Host':urlparse.urlparse(resolved).netloc, 'Connection':'keep-alive'})
resolved += '|%s' % urllib.urlencode(headers)
return resolved
except:
return
def __execute(self, sql, params=None):
if params is None:
params = []
rows = None
sql = self.__format(sql)
tries = 1
while True:
try:
cur = self.db.cursor()
# log_utils.log('Running: %s with %s' % (sql, params), log_utils.LOGDEBUG)
cur.execute(sql, params)
if sql[:6].upper() == 'SELECT' or sql[:4].upper() == 'SHOW':
rows = cur.fetchall()
cur.close()
self.db.commit()
return rows
except OperationalError as e:
if tries < MAX_TRIES:
tries += 1
log_utils.log('Retrying (%s/%s) SQL: %s Error: %s' % (tries, MAX_TRIES, sql, e), log_utils.LOGWARNING)
self.db = None
self.__connect_to_db()
elif any(s for s in ['no such table', 'no such column'] if s in str(e)):
self.db.rollback()
raise DatabaseRecoveryError(e)
else:
raise
except DatabaseError as e:
self.db.rollback()
raise DatabaseRecoveryError(e)
def parallel_get_progress(q, trakt_id, cached):
worker = threading.current_thread()
log_utils.log("Worker: %s (%s) for %s progress" % (worker.name, worker, trakt_id), log_utils.LOGDEBUG)
progress = trakt_api.get_show_progress(trakt_id, full=True, cached=cached)
progress["trakt"] = trakt_id # add in a hacked show_id to be used to match progress up to the show its for
log_utils.log("Got progress for %s from %s" % (trakt_id, worker), log_utils.LOGDEBUG)
q.put(progress)
def __get_cached_url(self, url, cache=8):
log_utils.log('Fetching Cached URL: %s' % url, log_utils.LOGDEBUG)
before = time.time()
_created, _res_header, html = self.db_connection.get_cached_url(url, cache_limit=cache)
if html:
log_utils.log('Returning cached result for: %s' % (url), log_utils.LOGDEBUG)
return html
log_utils.log('No cached url found for: %s' % url, log_utils.LOGDEBUG)
req = urllib2.Request(url)
host = BASE_URL.replace('http://', '')
req.add_header('User-Agent', USER_AGENT)
req.add_header('Host', host)
req.add_header('Referer', BASE_URL)
try:
body = self.__http_get_with_retry(url, req)
body = body.decode('utf-8')
parser = HTMLParser.HTMLParser()
body = parser.unescape(body)
except Exception as e:
kodi.notify(msg='Failed to connect to URL: %s' % (url), duration=5000)
log_utils.log('Failed to connect to URL %s: (%s)' % (url, e), log_utils.LOGERROR)
return ''
self.db_connection.cache_url(url, body)
after = time.time()
log_utils.log('Cached Url Fetch took: %.2f secs' % (after - before), log_utils.LOGDEBUG)
return body
def export_from_db(self, full_path):
temp_path = os.path.join(kodi.translate_path("special://profile"), 'temp_export_%s.csv' % (int(time.time())))
with open(temp_path, 'w') as f:
writer = csv.writer(f)
f.write('***VERSION: %s***\n' % self.get_db_version())
if self.__table_exists('rel_url'):
f.write(CSV_MARKERS.REL_URL + '\n')
for fav in self.get_all_rel_urls():
writer.writerow(self.__utf8_encode(fav))
if self.__table_exists('other_lists'):
f.write(CSV_MARKERS.OTHER_LISTS + '\n')
for sub in self.get_all_other_lists():
writer.writerow(self.__utf8_encode(sub))
if self.__table_exists('saved_searches'):
f.write(CSV_MARKERS.SAVED_SEARCHES + '\n')
for sub in self.get_all_searches():
writer.writerow(self.__utf8_encode(sub))
if self.__table_exists('bookmark'):
f.write(CSV_MARKERS.BOOKMARKS + '\n')
for sub in self.get_bookmarks():
writer.writerow(self.__utf8_encode(sub))
log_utils.log('Copying export file from: |%s| to |%s|' % (temp_path, full_path), log_utils.LOGDEBUG)
if not xbmcvfs.copy(temp_path, full_path):
raise Exception('Export: Copy from |%s| to |%s| failed' % (temp_path, full_path))
if not xbmcvfs.delete(temp_path):
raise Exception('Export: Delete of %s failed.' % (temp_path))
def search(self, video_type, title, year, season=''):
search_url = urlparse.urljoin(self.base_url, '/index.php?search_keywords=')
search_url += urllib.quote_plus(title)
search_url += '&year=' + urllib.quote_plus(str(year))
if video_type in [VIDEO_TYPES.TVSHOW, VIDEO_TYPES.EPISODE]:
search_url += '&search_section=2'
else:
search_url += '&search_section=1'
results = []
html = self. _http_get(self.base_url, cache_limit=0)
match = re.search('input type="hidden" name="key" value="([0-9a-f]*)"', html)
if match:
key = match.group(1)
search_url += '&key=' + key
html = self._http_get(search_url, cache_limit=.25)
pattern = r'class="index_item.+?href="(.+?)" title="Watch (.+?)"?\(?([0-9]{4})?\)?"?>'
for match in re.finditer(pattern, html):
url, title, year = match.groups('')
result = {'url': scraper_utils.pathify_url(url), 'title': scraper_utils.cleanse_title(title), 'year': year}
results.append(result)
else:
log_utils.log('Unable to locate PW search key', log_utils.LOGWARNING)
return results
def download_subtitle(self, url):
url = BASE_URL + url
(response, srt) = self.__get_url(url)
if not hasattr(response, 'info') or 'Content-Disposition' not in response.info():
return
cd = response.info()['Content-Disposition']
r = re.search('filename="(.*)"', cd)
if r:
filename = r.group(1)
else:
filename = 'addic7ed_subtitle.srt'
filename = re.sub('[^\x00-\x7F]', '', filename)
final_path = os.path.join(kodi.get_setting('subtitle-folder'), filename)
final_path = kodi.translate_path(final_path)
if not xbmcvfs.exists(os.path.dirname(final_path)):
try:
try: xbmcvfs.mkdirs(os.path.dirname(final_path))
except: os.mkdir(os.path.dirname(final_path))
except:
log_utils.log('Failed to create directory %s' % os.path.dirname(final_path), log_utils.LOGERROR)
raise
with open(final_path, 'w') as f:
f.write(srt)
return final_path
def __get_links(self, url, video):
hosters = []
seen_urls = set()
for search_type in SEARCH_TYPES:
search_url, params = self.__translate_search(url, search_type)
if search_url:
html = self._http_get(search_url, params=params, cache_limit=.5)
js_result = scraper_utils.parse_json(html, search_url)
if js_result.get('status') == 'success':
for result in js_result['result']:
if len(result['hosterurls']) > 1: continue
if result['extension'] == 'rar': continue
stream_url = result['hosterurls'][0]['url']
if stream_url not in seen_urls:
if scraper_utils.release_check(video, result['title']):
host = urlparse.urlsplit(stream_url).hostname
quality = scraper_utils.get_quality(video, host, self._get_title_quality(result['title']))
hoster = {'multi-part': False, 'class': self, 'views': None, 'url': stream_url, 'rating': None, 'host': host, 'quality': quality, 'direct': False}
hoster['extra'] = result['title']
hosters.append(hoster)
seen_urls.add(stream_url)
else:
log_utils.log('Alluc API Error: |%s|%s|: %s' % (search_url, params, js_result.get('message', 'Unknown Error')), log_utils.LOGWARNING)
return hosters
def __execute(self, sql, params=None):
if params is None:
params = []
rows = None
sql = self.__format(sql)
tries = 1
while True:
try:
cur = self.db.cursor()
# log_utils.log('Running: %s with %s' % (sql, params), log_utils.LOGDEBUG)
cur.execute(sql, params)
if sql[:6].upper() == 'SELECT' or sql[:4].upper() == 'SHOW':
rows = cur.fetchall()
cur.close()
self.db.commit()
return rows
except OperationalError:
if tries < MAX_TRIES:
tries += 1
log_utils.log('Retrying (%s/%s) SQL: %s' % (tries, MAX_TRIES, sql), log_utils.LOGWARNING)
self.db = None
self.__connect_to_db()
else:
raise
def get_url(self, video):
url = None
self.create_db_connection()
result = self.db_connection.get_related_url(video.video_type, video.title, video.year, self.get_name(), video.season, video.episode)
if result:
url = result[0][0]
log_utils.log('Got local related url: |%s|%s|%s|%s|%s|' % (video.video_type, video.title, video.year, self.get_name(), url), log_utils.LOGDEBUG)
else:
date_match = False
search_title = '%s S%02dE%02d' % (video.title, int(video.season), int(video.episode))
results = self.search(video.video_type, search_title, '')
if not results and video.ep_airdate is not None:
search_title = '%s %s' % (video.title, video.ep_airdate.strftime('%Y.%m.%d'))
results = self.search(video.video_type, search_title, '')
date_match = True
best_q_index = -1
for result in results:
if date_match and video.ep_airdate.strftime('%Y.%m.%d') not in result['title']:
continue
if Q_DICT[result['quality']] > best_q_index:
best_q_index = Q_DICT[result['quality']]
url = result['url']
self.db_connection.set_related_url(video.video_type, video.title, video.year, self.get_name(), url, video.season, video.episode)
return url
def processCaptcha(self, key, lang):
headers = {'Referer': 'https://www.google.com/recaptcha/api2/demo', 'Accept-Language': lang}
html = get_url('http://www.google.com/recaptcha/api/fallback?k=%s' % (key), headers=headers)
token = ''
iteration = 0
while True:
payload = re.findall('"(/recaptcha/api2/payload[^"]+)', html)
iteration += 1
message = re.findall('<label[^>]+class="fbc-imageselect-message-text"[^>]*>(.*?)</label>', html)
if not message:
message = re.findall('<div[^>]+class="fbc-imageselect-message-error">(.*?)</div>', html)
if not message:
token = re.findall('"this\.select\(\)">(.*?)</textarea>', html)[0]
if token:
log_utils.log('Captcha Success: %s' % (token), log_utils.LOGDEBUG)
else:
log_utils.log('Captcha Failed', log_utils.LOGDEBUG)
break
else:
message = message[0]
payload = payload[0]
cval = re.findall('name="c"\s+value="([^"]+)', html)[0]
captcha_imgurl = 'https://www.google.com%s' % (payload.replace('&', '&'))
message = re.sub('</?strong>', '', message)
oSolver = cInputWindow(captcha=captcha_imgurl, msg=message, iteration=iteration)
captcha_response = oSolver.get()
if not captcha_response:
break
data = {'c': cval, 'response': captcha_response}
html = get_url("http://www.google.com/recaptcha/api/fallback?k=%s" % (key), data=data, headers=headers)
return token
def update_all_scrapers():
try: last_check = int(kodi.get_setting('last_list_check'))
except: last_check = 0
now = int(time.time())
list_url = kodi.get_setting('scraper_url')
scraper_password = kodi.get_setting('scraper_password')
list_path = os.path.join(kodi.translate_path(kodi.get_profile()), 'scraper_list.txt')
exists = os.path.exists(list_path)
if list_url and scraper_password and (not exists or last_check < (now - (24 * 60 * 60))):
scraper_list = utils2.get_and_decrypt(list_url, scraper_password)
if scraper_list:
try:
with open(list_path, 'w') as f:
f.write(scraper_list)
kodi.set_setting('last_list_check', str(now))
kodi.set_setting('scraper_last_update', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now)))
for line in scraper_list.split('\n'):
line = line.replace(' ', '')
if line:
scraper_url, filename = line.split(',')
if scraper_url.startswith('http'):
update_scraper(filename, scraper_url)
except Exception as e:
log_utils.log('Exception during scraper update: %s' % (e), log_utils.LOGWARNING)
def get_sources(self, video):
source_url = self.get_url(video)
hosters = []
if source_url:
url = urlparse.urljoin(self.base_url, source_url)
html = self._http_get(url, cache_limit=.5)
js_result = scraper_utils.parse_json(html, url)
if 'error' in js_result:
log_utils.log('DD.tv API error: "%s" @ %s' % (js_result['error'], url), log_utils.LOGWARNING)
return hosters
for result in js_result:
if not scraper_utils.release_check(video, result['release'], require_title=False): continue
if result['quality'] in self.q_order:
for key in result['links']:
url = result['links'][key][0]
if re.search('\.rar(\.|$)', url):
continue
hostname = urlparse.urlparse(url).hostname
hoster = {'multi-part': False, 'class': self, 'views': None, 'url': url, 'rating': None, 'host': hostname, 'quality': QUALITY_MAP[result['quality']], 'direct': False}
hoster['format'] = result['quality']
if 'x265' in result['release'] and result['quality'] != '1080P-X265': hoster['dd_qual'] += '-x265'
hosters.append(hoster)
return hosters
def get_sources(self, video):
source_url = self.get_url(video)
hosters = []
if source_url and source_url != FORCE_NO_MATCH:
params = urlparse.parse_qs(source_url)
if video.video_type == VIDEO_TYPES.MOVIE:
cmd = '{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovieDetails", "params": {"movieid": %s, "properties" : ["file", "playcount", "streamdetails"]}, "id": "libMovies"}'
result_key = 'moviedetails'
else:
cmd = '{"jsonrpc": "2.0", "method": "VideoLibrary.GetEpisodeDetails", "params": {"episodeid": %s, "properties" : ["file", "playcount", "streamdetails"]}, "id": "libTvShows"}'
result_key = 'episodedetails'
run = cmd % (params['id'][0])
meta = xbmc.executeJSONRPC(run)
meta = scraper_utils.parse_json(meta)
log_utils.log('Source Meta: %s' % (meta), log_utils.LOGDEBUG)
if 'result' in meta and result_key in meta['result']:
details = meta['result'][result_key]
def_quality = [item[0] for item in sorted(SORT_KEYS['quality'].items(), key=lambda x:x[1])][self.def_quality]
host = {'multi-part': False, 'class': self, 'url': details['file'], 'host': 'XBMC Library', 'quality': def_quality, 'views': details['playcount'], 'rating': None, 'direct': True}
stream_details = details['streamdetails']
if len(stream_details['video']) > 0 and 'width' in stream_details['video'][0]:
host['quality'] = scraper_utils.width_get_quality(stream_details['video'][0]['width'])
hosters.append(host)
return hosters
def update_settings():
full_path = os.path.join(kodi.get_path(), 'resources', 'settings.xml')
try:
# open for append; skip update if it fails
with open(full_path, 'a') as f:
pass
except Exception as e:
log_utils.log('Dynamic settings update skipped: %s' % (e), log_utils.LOGWARNING)
else:
with open(full_path, 'r') as f:
xml = f.read()
new_settings = []
cat_count = 1
old_xml = xml
classes = scraper.Scraper.__class__.__subclasses__(scraper.Scraper) # @UndefinedVariable
classes += proxy.Proxy.__class__.__subclasses__(proxy.Proxy) # @UndefinedVariable
for cls in sorted(classes, key=lambda x: x.get_name().upper()):
if not cls.get_name() or cls.has_proxy(): continue
new_settings += cls.get_settings()
if len(new_settings) > 90:
xml = update_xml(xml, new_settings, cat_count)
new_settings = []
cat_count += 1
if new_settings:
xml = update_xml(xml, new_settings, cat_count)
if xml != old_xml:
with open(full_path, 'w') as f:
f.write(xml)
else:
log_utils.log('No Settings Update Needed', log_utils.LOGDEBUG)
def reset_cache():
try:
shutil.rmtree(cache_path)
return True
except Exception as e:
log_utils.log('Failed to Reset Cache: %s' % (e), log_utils.LOGWARNING)
return False
def doLogin(email, password):
global LOGIN_COOKIE_FILE
try:
if os.path.isfile(LOGIN_COOKIE_FILE):
os.remove(LOGIN_COOKIE_FILE)
data = {
'email': email,
'password': password,
'remember': '1',
'Submit': 'Login',
'action': 'process'
}
conn = http_req(common.URLS['login'], data, common.HEADERS, LOGIN_COOKIE_FILE)
ret_code = conn.code
conn.close()
if ret_code == 302:
return True
except:
log_utils.log(traceback.print_exc())
return False
def find_link(url, html=''):
global limit
limit+=1
log('Finding in : %s'%url)
try: referer = urlparse.parse_qs(urlparse.urlparse(url).query)['referer'][0]
except: referer = 'http://' + urlparse.urlparse(url).netloc
host = urlparse.urlparse(url).netloc
headers = {'Referer':referer, 'Host':host, 'User-Agent' : client.agent(), 'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language' : 'en-US,en;q=0.5'}
if html=='':
url = manual_url_fix(url)
html = client.request(url, headers=headers)
html = manual_html_fix(url,html,headers)
ref=url
fs=list(globals().copy())
for f in fs:
if 'finder' in f:
resolved = eval (f+"(html,ref)")
if resolved:
log('Resolved with %s: %s'%(f,resolved))
return resolved
break
return
def fit_transform(self, table):
todo_cols = [table.label]
if not todo_cols:
return
df = table.train_X
# log(f'before fit_trans df')
# print(df.head())
col2type = {}
opt = n_lag
opt = partial(opt, todo_cols=todo_cols, windows=self.windows)
if table.is_multivar:
#if not CONSTANT.USE_TIME_STAMP_MODE:
if table.key_num < CONSTANT.KEY_NUM:
log('use order')
groups = df[[table.key_time_col] + [table.key_col] +
todo_cols].groupby(table.key_col)
groups = [group[1] for group in groups]
#res = Parallel(n_jobs=CONSTANT.JOBS, require='sharedmem')(delayed(opt)(group) for group in groups)
res = []
for group in groups:
tmp = opt(group)
res.append(tmp)
if res:
tmp = pd.concat(res)
tmp = tmp.drop(todo_cols, axis=1)
for col in tmp.columns:
if col != table.key_col and col != table.key_time_col:
col2type[col] = CONSTANT.NUMERICAL_TYPE
print(f'before merge shape: {df.shape}')
df = pd.merge(df,
tmp,
how='left',
on=[table.key_time_col, table.key_col])
print(f'after merge shape: {df.shape}')
del tmp
gc.collect()
else:
time_col = table.key_time_col
cat_col = table.key_col
new_cols = []
for window in self.windows:
for col in todo_cols:
series_matrix = pd.pivot_table(
df,
index=[time_col],
values=[col],
columns=cat_col,
) # aggfunc=[np.mean])
series_matrix = series_matrix.fillna(method="ffill")
series_matrix_shift1 = series_matrix.shift(1)
lag_mean = series_matrix_shift1.rolling(window).mean()
lag_std = series_matrix_shift1.rolling(window).std()
lag_max = series_matrix_shift1.rolling(window).max()
lag_min = series_matrix_shift1.rolling(window).min()
new_col = f'{col}_lag_{window}_mean'
col2type[new_col] = CONSTANT.NUMERICAL_TYPE
new_cols.append(new_col)
df = revert_pivot_feat_join(df, lag_mean, new_col)
new_col = f'{col}_lag_{window}_std'
col2type[new_col] = CONSTANT.NUMERICAL_TYPE
new_cols.append(new_col)
df = revert_pivot_feat_join(df, lag_std, new_col)
new_col = f'{col}_lag_{window}_max'
col2type[new_col] = CONSTANT.NUMERICAL_TYPE
new_cols.append(new_col)
df = revert_pivot_feat_join(df, lag_max, new_col)
new_col = f'{col}_lag_{window}_min'
col2type[new_col] = CONSTANT.NUMERICAL_TYPE
new_cols.append(new_col)
df = revert_pivot_feat_join(df, lag_min, new_col)
else:
tmp = opt(df[[table.key_time_col] + todo_cols])
tmp.drop([table.key_time_col] + todo_cols, axis=1, inplace=True)
for col in tmp.columns:
col2type[col] = CONSTANT.NUMERICAL_TYPE
df[col] = tmp[col]
del tmp
gc.collect()
table.update_data(df, col2type, mode='train')
def content(url, searched=False):
try:
c = client.request(url)
r = dom_parser2.parse_dom(c, 'div',
{'class': ['video-box', 'four-column']})
r = [(dom_parser2.parse_dom(i, 'a', req='href'), \
dom_parser2.parse_dom(i, 'div', {'class': 'video-box-title'}), \
dom_parser2.parse_dom(i, 'div', {'class': 'video-duration'}), \
dom_parser2.parse_dom(i, 'img', req='data-original')) for i in r]
r = [(urlparse.urljoin(base_domain, i[0][0].attrs['href']),
i[1][0].content, i[2][0].content, i[3][0].attrs['data-original'])
for i in r if i]
if (not r) and (not searched):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for i in r:
try:
name = '%s - [ %s ]' % (kodi.sortX(i[1].encode('utf-8')).title(),
kodi.sortX(i[2].encode('utf-8')))
if searched:
description = 'Result provided by %s' % base_name.title()
else:
description = name
content_url = i[0] + '|SPLIT|%s' % base_name
fanarts = xbmc.translatePath(
os.path.join('special://home/addons/script.wankbank.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': content_url,
'mode': player_mode,
'icon': i[3],
'fanart': fanarts,
'description': description,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
try:
search_pattern = '''\<link\s*rel\=['"]next['"]\s*href\=['"]([^'"]+)'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename)
except Exception as e:
log_utils.log(
'Error getting next page for %s :: Error: %s' %
(base_name.title(), str(e)), log_utils.LOGERROR)
def record_timeouts(fails):
for key in fails:
if fails[key] == True:
log_utils.log('Recording Timeout of %s' % (key),
log_utils.LOGWARNING)
increment_setting('%s_fail' % key)
def download_media(url, path, file_name):
try:
progress = int(kodi.get_setting('down_progress'))
import urllib2
request = urllib2.Request(url)
request.add_header('User-Agent', USER_AGENT)
request.add_unredirected_header('Host', request.get_host())
response = urllib2.urlopen(request)
content_length = 0
if 'Content-Length' in response.info():
content_length = int(response.info()['Content-Length'])
file_name = file_name.replace('.strm', get_extension(url, response))
full_path = os.path.join(path, file_name)
log_utils.log('Downloading: %s -> %s' % (url, full_path),
log_utils.LOGDEBUG)
path = xbmc.makeLegalFilename(path)
if not xbmcvfs.exists(path):
try:
try:
xbmcvfs.mkdirs(path)
except:
os.mkdir(path)
except Exception as e:
raise Exception(i18n('failed_create_dir'))
file_desc = xbmcvfs.File(full_path, 'w')
total_len = 0
if progress:
if progress == PROGRESS.WINDOW:
dialog = xbmcgui.DialogProgress()
else:
dialog = xbmcgui.DialogProgressBG()
dialog.create('Stream All The Sources',
i18n('downloading') % (file_name))
dialog.update(0)
while True:
data = response.read(CHUNK_SIZE)
if not data:
break
if progress == PROGRESS.WINDOW and dialog.iscanceled():
break
total_len += len(data)
if not file_desc.write(data):
raise Exception('failed_write_file')
percent_progress = (
total_len) * 100 / content_length if content_length > 0 else 0
log_utils.log(
'Position : %s / %s = %s%%' %
(total_len, content_length, percent_progress),
log_utils.LOGDEBUG)
if progress == PROGRESS.WINDOW:
dialog.update(percent_progress)
elif progress == PROGRESS.BACKGROUND:
dialog.update(percent_progress, 'Stream All The Sources')
else:
kodi.notify(msg=i18n('download_complete') % (file_name),
duration=5000)
log_utils.log('Download Complete: %s -> %s' % (url, full_path),
log_utils.LOGDEBUG)
file_desc.close()
if progress:
dialog.close()
except Exception as e:
log_utils.log(
'Error (%s) during download: %s -> %s' % (str(e), url, file_name),
log_utils.LOGERROR)
kodi.notify(msg=i18n('download_error') % (str(e), file_name),
duration=5000)
def content(url, searched=False):
import xbmcgui
dialog = xbmcgui.Dialog()
try:
c = client.request(url)
soup = BeautifulSoup(c, 'html.parser')
r = soup.find_all('content')
if (not r) and (not searched):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for i in r:
try:
title = i.find('title').text
url2 = i.find('media').text
icon = i.find('icon').text
fanarts = i.find('fanart').text
description = i.find('desc').text
dirlst.append({
'name': title,
'url': url2,
'mode': player_mode,
'icon': icon,
'fanart': fanarts,
'description': description,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s Error: %s' % (title, str(e)),
log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
try:
search_pattern = '''\<link\s*rel\=['"]next['"]\s*href\=['"]([^'"]+)'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename)
except Exception as e:
log_utils.log(
'Error getting next page for %s :: Error: %s' %
(base_name.title(), str(e)), log_utils.LOGERROR)
def content(url, searched=False):
try:
c = client.request(url)
r = dom_parser2.parse_dom(c, 'div', {'class': 'plugcontainer'})
r = [(dom_parser2.parse_dom(i, 'a', req=['href','title']), \
dom_parser2.parse_dom(i, 'img', req='src'), \
dom_parser2.parse_dom(i, 'p', {'class': 'txt'})) \
for i in r if i]
r = [(i[0][0].attrs['href'], i[0][0].attrs['title'], \
i[1][0].attrs['src'], i[2][1].content if i[2][1] else 'Unknown') \
for i in r if 'link.php' not in i[0][0].attrs['href']]
if (not r) and (not searched):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for i in r:
try:
name = kodi.sortX(i[1].encode('utf-8')).title()
if searched:
description = 'Result provided by %s' % base_name.title()
else:
description = i[3]
content_url = i[0] + '|SPLIT|%s' % base_name
fanarts = xbmc.translatePath(
os.path.join('special://home/addons/script.wankbank.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': content_url,
'mode': player_mode,
'icon': i[2],
'fanart': fanarts,
'description': description,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
try:
search_pattern = '''<a\s*href=['"]([^'"]+)['"]\s*class=['"]plugurl['"]\s*title=['"]next\s*page['"]>'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename, parse)
except Exception as e:
log_utils.log(
'Error getting next page for %s :: Error: %s' %
(base_name.title(), str(e)), log_utils.LOGERROR)
def __call_trakt(self,
url,
data=None,
params=None,
auth=True,
cache_limit=.25,
cached=True):
if not cached: cache_limit = 0
db_cache_limit = cache_limit if cache_limit > 8 else 8
json_data = json.dumps(data) if data else None
headers = {
'Content-Type': 'application/json',
'trakt-api-key': V2_API_KEY,
'trakt-api-version': 2
}
url = '%s%s%s' % (self.protocol, BASE_URL, url)
if params: url = url + '?' + urllib.urlencode(params)
db_connection = DB_Connection()
created, cached_result = db_connection.get_cached_url(
url, db_cache_limit)
if cached_result and (time.time() - created) < (60 * 60 * cache_limit):
result = cached_result
log_utils.log('Returning cached result for: %s' % (url),
log_utils.LOGDEBUG)
else:
auth_retry = False
while True:
try:
if auth:
headers.update(
{'Authorization': 'Bearer %s' % (self.token)})
log_utils.log(
'Trakt Call: %s, header: %s, data: %s' %
(url, headers, data), log_utils.LOGDEBUG)
request = urllib2.Request(url,
data=json_data,
headers=headers)
f = urllib2.urlopen(request, timeout=self.timeout)
result = ''
while True:
data = f.read()
if not data: break
result += data
db_connection.cache_url(url, result)
break
except (ssl.SSLError, socket.timeout) as e:
if cached_result:
result = cached_result
log_utils.log(
'Temporary Trakt Error (%s). Using Cached Page Instead.'
% (str(e)), log_utils.LOGWARNING)
else:
raise TransientTraktError('Temporary Trakt Error: ' +
str(e))
except urllib2.URLError as e:
if isinstance(e, urllib2.HTTPError):
if e.code in TEMP_ERRORS:
if cached_result:
result = cached_result
log_utils.log(
'Temporary Trakt Error (%s). Using Cached Page Instead.'
% (str(e)), log_utils.LOGWARNING)
break
else:
raise TransientTraktError(
'Temporary Trakt Error: ' + str(e))
elif e.code == 401 or e.code == 405:
if auth_retry or url.endswith('/token'):
self.token = None
kodi.set_setting('trakt_oauth_token', '')
kodi.set_setting('trakt_refresh_token', '')
raise TraktError(
'Trakt Call Authentication Failed (%s)' %
(e.code))
else:
result = self.get_token()
self.token = result['access_token']
kodi.set_setting('trakt_oauth_token',
result['access_token'])
kodi.set_setting('trakt_refresh_token',
result['refresh_token'])
auth_retry = True
elif e.code == 404:
raise TraktNotFoundError()
else:
raise
elif isinstance(e.reason, socket.timeout) or isinstance(
e.reason, ssl.SSLError):
if cached_result:
result = cached_result
log_utils.log(
'Temporary Trakt Error (%s). Using Cached Page Instead'
% (str(e)), log_utils.LOGWARNING)
break
else:
raise TransientTraktError(
'Temporary Trakt Error: ' + str(e))
else:
raise TraktError('Trakt Error: ' + str(e))
except:
raise
response = json.loads(result)
if 'status' in response and response['status'] == 'failure':
if 'message' in response: raise TraktError(response['message'])
if 'error' in response: raise TraktError(response['error'])
else: raise TraktError()
else:
# log_utils.log('Trakt Response: %s' % (response), xbmc.LOGDEBUG)
return response
def content(url, searched=False):
if not base_domain in url: url = base_domain + url
try:
c = client.request(url)
r = re.findall(
'<div class="preloadLine">(.*?)<span class="video_count">',
c,
flags=re.DOTALL)
if (not r) and (not searched):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for i in r:
try:
name = re.findall('alt="(.*?)"', i, flags=re.DOTALL)[0]
url2 = re.findall('<a.+?href="(.*?)"', i, flags=re.DOTALL)[0]
if not base_domain in url2: url2 = base_domain + url2
icon = re.findall('data-thumb_url="(.*?)"', i, flags=re.DOTALL)[0]
#desc = re.findall('<span class="duration">(.*?)</span>',i, flags=re.DOTALL)[0].strip()
fanarts = xbmc.translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': url2,
'mode': player_mode,
'icon': icon,
'fanart': fanarts,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
try:
search_pattern = '''<link rel="next"\s+href=['"]([^'"]+)['"]'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename)
except Exception as e:
log_utils.log(
'Error getting next page for %s :: Error: %s' %
(base_name.title(), str(e)), log_utils.LOGERROR)
def fit_transform(self, table):
cat_cols = table.id_cols
todo_cols = table.init_num_cols + [table.label]
key_time_col = table.key_time_col
df = table.train_X
col2type = {}
# def window_encode(df, cat, time, todo_cols, windows):
# groups = df[[time] + [cat] + todo_cols].groupby(cat)
# groups = [group[1] for group in groups]
# res = []
# for group_df in groups:
# pre_lag = 1
# for window in windows:
# rolled = group_df[todo_cols].shift(pre_lag).rolling(window=window)
# group_df = group_df.join(rolled.mean().add_suffix(f'_{cat}_lag_{window}_mean'))
# group_df = group_df.join(rolled.max().add_suffix(f'_{cat}_lag_{window}_max'))
# group_df = group_df.join(rolled.min().add_suffix(f'_{cat}_lag_{window}_min'))
# res.append(group_df)
# tmp = pd.concat(res)
# #tmp.fillna(method='bfill', inplace=True)
# tmp.drop(todo_cols+[time, cat], axis=1, inplace=True)
# return tmp
def window_encode(df, cat, time, todo_cols, windows):
group_df = df[[time] + [cat] + todo_cols].groupby([cat,
time]).mean()
res = []
pre_lag = 1
for window in windows:
rolled = group_df[todo_cols].shift(pre_lag).rolling(
window=window)
group_df = group_df.join(
rolled.mean().add_suffix(f'_{cat}_lag_{window}_mean'))
group_df = group_df.join(
rolled.max().add_suffix(f'_{cat}_lag_{window}_max'))
group_df = group_df.join(
rolled.min().add_suffix(f'_{cat}_lag_{window}_min'))
res.append(group_df)
tmp = pd.concat(res, axis=1)
tmp.reset_index(drop=False, inplace=True)
tmp.drop(todo_cols, axis=1, inplace=True)
print(f'shape before merge{df.shape}')
tmp = pd.merge(df[[time, cat]], tmp, how='left', on=[time, cat])
print(f'shape after merge{tmp.shape}')
#tmp.fillna(method='bfill', inplace=True)
tmp.drop([time, cat], axis=1, inplace=True)
return tmp
opt = window_encode
opt = partial(opt,
time=key_time_col,
todo_cols=todo_cols,
windows=self.windows)
# for cat in cat_cols:
# tmp = opt(df, cat)
res = Parallel(n_jobs=CONSTANT.JOBS,
require='sharedmem')(delayed(opt)(df, cat)
for cat in cat_cols)
if res:
tmp = pd.concat(res, axis=1)
for col in tmp.columns:
col2type[col] = CONSTANT.NUMERICAL_TYPE
df[col] = tmp[col]
del tmp
gc.collect()
table.update_data(df, col2type, 'train')
log(f'{self.__class__.__name__} produce {len(col2type)} features')
def find_link(url, name, iconimage, downloadableLink=False):
xbmc.executebuiltin("ActivateWindow(busydialog)")
if '|SPLIT|' in url: url = url.split('|SPLIT|')[0]
if 'site=' in url: url = url.split('site=')[0]
if '|' in url: url = url.split('|User-Agent')[0]
c = client.request(url, output='headers')
checks = ['video', 'mpegurl']
exts = ['.mp4', '.flv', '.m3u8']
try:
if any(f for f in checks if f in c['Content-Type']):
downloadableLink = True
except:
if any(f for f in exts if f in url):
downloadableLink = True
else:
xbmc.executebuiltin("Dialog.Close(busydialog)")
kodi.notify(msg='Error downloading video.')
quit()
name = kodi.stripColor(name)
if '] -' in name: name = name.split('] -')[1]
if downloadableLink:
dest = getDest()
dest = os.path.join(dest, '%s.mp4' % urllib.quote_plus(name))
download(url, name, iconimage, dest)
else:
u = None
log_utils.log('Sending %s to XXX Resolver' % (url),
log_utils.LOGNOTICE)
if urlresolver.HostedMediaFile(url, include_xxx=True).valid_url():
log_utils.log(
'%s is a valid SMU resolvable URL. Attempting to resolve.' %
(url), log_utils.LOGNOTICE)
try:
u = urlresolver.HostedMediaFile(url,
include_xxx=True).resolve()
except Exception as e:
log_utils.log(
'Error getting valid link from SMU :: %s :: %s' %
(url, str(e)), log_utils.LOGERROR)
kodi.idle()
kodi.notify(msg='Something went wrong! | %s' % str(e),
duration=8000,
sound=True)
quit()
log_utils.log('Link returned by XXX Resolver :: %s' % (u),
log_utils.LOGNOTICE)
else:
log_utils.log(
'%s is not a valid SMU resolvable link. Attempting to resolve by XXXODUS backup resolver.'
% (url), log_utils.LOGNOTICE)
try:
u = adultresolver.resolve(url)
except Exception as e:
log_utils.log(
'Error getting valid link from SMU :: %s :: %s' %
(url, str(e)), log_utils.LOGERROR)
kodi.idle()
kodi.notify(msg='Something went wrong! | %s' % str(e),
duration=8000,
sound=True)
quit()
log_utils.log('%s returned by XXX-O-DUS backup resolver.' % (u),
log_utils.LOGNOTICE)
if (not isinstance(u, str)):
try:
u = multilinkselector(u)
except:
pass
if u == 'quit':
xbmc.executebuiltin("Dialog.Close(busydialog)")
quit()
if u:
dest = getDest()
dest = os.path.join(dest, '%s.tmp_mp4' % urllib.quote_plus(name))
download(u, name, iconimage, dest)
else:
xbmc.executebuiltin("Dialog.Close(busydialog)")
kodi.notify('No Downloadable Link Found.')
quit()
def content(url, searched=False):
try:
c = client.request(url)
match = re.findall('<div class="6u">(.*?)</section>',
c,
flags=re.DOTALL)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for items in match:
try:
name = re.findall('alt="(.*?)"', items, flags=re.DOTALL)[0]
name = name.title()
url2 = re.findall('<a href="(.*?)"', items, flags=re.DOTALL)[0]
icon = re.findall('''<div.*?onmouseleave=.*?\(['"](.*?)['"]''',
items,
flags=re.DOTALL)[0]
length = re.findall(
'<span class="icon fa-clock-o meta-data">(.*?)</span>',
items,
flags=re.DOTALL)[0]
if not 'https:' in url2: url2 = 'https://hqporner.com' + url2
if not 'https:' in icon: icon = 'https:' + icon
#icon = ''
desc = '[COLOR yellow]Video Length :: [/COLOR]' + length
fanarts = xbmc.translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': url2,
'mode': player_mode,
'icon': icon,
'fanart': fanarts,
'description': desc,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item. %s:: Error: %s' %
(base_name.title(), str(e)), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
search_pattern = '''<a\s*href=['"]([^'"]+)['"]\s*class=['"]button\s*mobile-pagi pagi-btn['"]>Next<\/a>'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename, parse)
def mainSearch(url):
if '|SPLIT|' in url: url, site = url.split('|SPLIT|')
term = url
if term == "null": term = kodi.get_keyboard('Search %s' % kodi.get_name())
if term:
search_on_off = kodi.get_setting("search_setting")
if search_on_off == "true":
delTerm(term)
addTerm(term)
display_term = term
term = urllib.quote_plus(term)
term = term.lower()
if site == 'all':
sources = __all__
search_sources = []
for i in sources:
try:
if eval(i + ".search_tag") == 1: search_sources.append(i)
except:
pass
if search_sources:
i = 0
source_num = 0
failed_list = ''
line1 = kodi.giveColor('Searching: ',
'white') + kodi.giveColor(
'%s', 'orangered')
line2 = kodi.giveColor('Found: %s videos', 'white')
line3 = kodi.giveColor(
'Source: %s of ' + str(len(search_sources)), 'white')
kodi.dp.create(kodi.get_name(), '', line2, '')
xbmc.executebuiltin('Dialog.Close(busydialog)')
for u in sorted(search_sources):
if kodi.dp.iscanceled(): break
try:
i += 1
progress = 100 * int(i) / len(search_sources)
kodi.dp.update(progress, line1 % u.title(),
line2 % str(source_num), line3 % str(i))
search_url = eval(u + ".search_base") % term
source_n = eval(u + ".content('%s',True)" % search_url)
try:
source_n = int(source_n)
except:
source_n = 0
if (not source_n):
if failed_list == '': failed_list += str(u).title()
else: failed_list += ', %s' % str(u).title()
else: source_num += int(source_n)
except Exception as e:
log_utils.log(
'Error searching %s :: Error: %s' %
(u.title(), str(e)), log_utils.LOGERROR)
pass
kodi.dp.close()
if failed_list != '':
kodi.notify(msg='%s failed to return results.' %
failed_list,
duration=4000,
sound=True)
log_utils.log(
'Scrapers failing to return search results are :: : %s'
% failed_list, log_utils.LOGERROR)
else:
kodi.notify(msg='%s results found.' % str(source_num),
duration=4000,
sound=True)
xbmcplugin.setContent(kodi.syshandle, 'movies')
xbmcplugin.endOfDirectory(kodi.syshandle, cacheToDisc=True)
utils.setView('search')
else:
search_url = eval(site + ".search_base") % term
eval(site + ".content('%s')" % search_url)
else:
kodi.notify(msg='Blank searches are not allowed.')
quit()
def download_media(url, path, file_name, translations, progress=None):
try:
if progress is None:
progress = int(kodi.get_setting('down_progress'))
i18n = translations.i18n
active = not progress == PROGRESS.OFF
background = progress == PROGRESS.BACKGROUND
with kodi.ProgressDialog(kodi.get_name(), i18n('downloading') % (file_name), background=background, active=active) as pd:
try:
headers = dict([item.split('=') for item in (url.split('|')[1]).split('&')])
for key in headers: headers[key] = urllib.unquote(headers[key])
except:
headers = {}
if 'User-Agent' not in headers: headers['User-Agent'] = BROWSER_UA
request = urllib2.Request(url.split('|')[0], headers=headers)
response = urllib2.urlopen(request)
if 'Content-Length' in response.info():
content_length = int(response.info()['Content-Length'])
else:
content_length = 0
file_name += '.' + get_extension(url, response)
full_path = os.path.join(path, file_name)
log_utils.log('Downloading: %s -> %s' % (url, full_path), log_utils.LOGDEBUG)
path = kodi.translate_path(xbmc.makeLegalFilename(path))
try:
try: xbmcvfs.mkdirs(path)
except: os.makedirs(path)
except Exception as e:
log_utils.log('Path Create Failed: %s (%s)' % (e, path), log_utils.LOGDEBUG)
if not path.endswith(os.sep): path += os.sep
if not xbmcvfs.exists(path):
raise Exception(i18n('failed_create_dir'))
file_desc = xbmcvfs.File(full_path, 'w')
total_len = 0
cancel = False
while True:
data = response.read(CHUNK_SIZE)
if not data:
break
if pd.is_canceled():
cancel = True
break
total_len += len(data)
if not file_desc.write(data):
raise Exception(i18n('failed_write_file'))
percent_progress = (total_len) * 100 / content_length if content_length > 0 else 0
log_utils.log('Position : %s / %s = %s%%' % (total_len, content_length, percent_progress), log_utils.LOGDEBUG)
pd.update(percent_progress)
file_desc.close()
if not cancel:
kodi.notify(msg=i18n('download_complete') % (file_name), duration=5000)
log_utils.log('Download Complete: %s -> %s' % (url, full_path), log_utils.LOGDEBUG)
except Exception as e:
log_utils.log('Error (%s) during download: %s -> %s' % (str(e), url, file_name), log_utils.LOGERROR)
kodi.notify(msg=i18n('download_error') % (str(e), file_name), duration=5000)
def content(url, searched=False):
try:
c = client.request(url)
r = dom_parser2.parse_dom(c, 'a')
r = [i for i in r if '<em class="time_thumb">' in i.content]
r = [(i.attrs['href'], \
dom_parser2.parse_dom(i.content, 'img', req=['src','alt']), \
dom_parser2.parse_dom(i.content, 'em', {'class': 'time_thumb'})) \
for i in r]
r = [(urljoin(base_domain, i[0]), i[1][0].attrs['alt'],
re.sub('<.+?>', '', i[2][0].content), i[1][0].attrs['src'])
for i in r]
if (not r) and (not searched):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for i in r:
try:
if PY2:
name = '%s - [ %s ]' % (kodi.sortX(
i[1].encode('utf-8')).title(),
kodi.sortX(i[2].encode('utf-8')))
else:
name = '%s - [ %s ]' % (kodi.sortX(
i[1]).title(), kodi.sortX(i[2]))
if searched:
description = 'Result provided by %s' % base_name.title()
else:
description = name
content_url = i[0] + '|SPLIT|%s' % base_name
fanarts = translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': content_url,
'mode': player_mode,
'icon': i[3],
'fanart': fanarts,
'description': description,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
try:
search_pattern = '''<li\s*class=['"]next['"]><a\s*href=['"]([^'"]+)'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename, parse)
except Exception as e:
log_utils.log(
'Error getting next page for %s :: Error: %s' %
(base_name.title(), str(e)), log_utils.LOGERROR)
def request(url,
close=True,
redirect=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
XHR=False,
limit=None,
referer=None,
cookie=None,
compression=True,
output='',
timeout='30'):
try:
if not url:
return
handlers = []
if not proxy == None:
handlers += [
urllib2.ProxyHandler({'http': '%s' % (proxy)}),
urllib2.HTTPHandler
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close == True:
cookies = cookielib.LWPCookieJar()
handlers += [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if (2, 7, 8) < sys.version_info < (2, 7, 12):
try:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
if url.startswith('//'): url = 'http:' + url
_headers = {}
try:
_headers.update(headers)
except:
pass
if 'User-Agent' in _headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
_headers['User-Agent'] = cache.get(randomagent, 1)
else:
_headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in _headers:
pass
elif referer is not None:
_headers['Referer'] = referer
if not 'Accept-Language' in _headers:
_headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in _headers:
pass
elif XHR == True:
_headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in _headers:
pass
elif not cookie == None:
_headers['Cookie'] = cookie
if 'Accept-Encoding' in _headers:
pass
elif compression and limit is None:
_headers['Accept-Encoding'] = 'gzip'
if redirect == False:
#old implementation
#class NoRedirection(urllib2.HTTPErrorProcessor):
# def http_response(self, request, response): return response
#opener = urllib2.build_opener(NoRedirection)
#opener = urllib2.install_opener(opener)
class NoRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_302(self, req, fp, code, msg, headers):
infourl = urllib.addinfourl(fp, headers,
req.get_full_url())
infourl.status = code
infourl.code = code
return infourl
http_error_300 = http_error_302
http_error_301 = http_error_302
http_error_303 = http_error_302
http_error_307 = http_error_302
opener = urllib2.build_opener(NoRedirectHandler())
urllib2.install_opener(opener)
try:
del _headers['Referer']
except:
pass
if isinstance(post, dict):
post = utils.byteify(post)
post = urllib.urlencode(post)
url = utils.byteify(url)
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
if response.code == 503:
cf_result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
cf_result = gzip.GzipFile(
fileobj=StringIO.StringIO(cf_result)).read()
if 'cf-browser-verification' in cf_result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
if not netloc.endswith('/'): netloc += '/'
ua = _headers['User-Agent']
cf = cache.get(cfcookie().get, 168, netloc, ua, timeout)
_headers['Cookie'] = cf
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
response = urllib2.urlopen(request, timeout=int(timeout))
else:
log_utils.log(
'Request-Error (%s): %s' % (str(response.code), url),
log_utils.LOGDEBUG)
if error == False: return
else:
log_utils.log(
'Request-Error (%s): %s' % (str(response.code), url),
log_utils.LOGDEBUG)
if error == False: return
if output == 'cookie':
try:
result = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
result = cf
except:
pass
if close == True: response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close == True: response.close()
return result
elif output == 'headers':
result = response.headers
if close == True: response.close()
return result
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
if close == True: response.close()
return result
elif output == 'file_size':
try:
content = int(response.headers['Content-Length'])
except:
content = '0'
response.close()
return content
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'sucuri_cloudproxy_js' in result:
su = sucuri().get(result)
_headers['Cookie'] = su
request = urllib2.Request(url, data=post)
_add_request_header(request, _headers)
response = urllib2.urlopen(request, timeout=int(timeout))
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(
fileobj=StringIO.StringIO(result)).read()
if 'Blazingfast.io' in result and 'xhr.open' in result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme,
urlparse.urlparse(url).netloc)
ua = _headers['User-Agent']
_headers['Cookie'] = cache.get(bfcookie().get, 168, netloc, ua,
timeout)
result = _basic_request(url,
headers=_headers,
post=post,
timeout=timeout,
limit=limit)
if output == 'extended':
try:
response_headers = dict([(item[0].title(), item[1])
for item in response.info().items()])
except:
response_headers = response.headers
response_code = str(response.code)
try:
cookie = '; '.join(
['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
cookie = cf
except:
pass
if close == True: response.close()
return (result, response_code, response_headers, _headers, cookie)
else:
if close == True: response.close()
return result
except Exception as e:
log_utils.log('Request-Error: (%s) => %s' % (str(e), url),
log_utils.LOGDEBUG)
return
import email_utils as mail
import time
import requests
import json
import sys
import log_utils as log
REQ_URL = "https://act.cmbchina.com/ActShipMobile/api/actshipprd/ACT20191009111228Y6pWLw23"
log.init(cfg)
if not mail.init(cfg):
log.error("[-] Mail system init faild, please check the config.py")
sys.exit(0)
log.log("[+] OK!")
if cfg.SEND_EMAIL_ON_STARTUP:
mail.send_email("Congratulations! Your server started successfully!")
query_times = 0
while True:
error_msg = None
try:
resp = requests.get(REQ_URL)
if resp.status_code != 200:
raise Exception("Got a wrong code: %d" % resp.status_code)
result_obj = json.loads(resp.text)
stock = result_obj[0]["prdstock"]
if stock != "0":
def sources(self, url, hostDict, hostprDict):
try:
sources = []
if url == None: return sources
html = client.request(url)
try:
iframe = client.parseDOM(
html,
'iframe',
attrs={'class': 'embed-responsive-item'},
ret='src')[0]
host = iframe.split('//')[1].replace('www.', '')
host = host.split('/')[0].split('.')[0].title()
sources.append({
'source': host,
'quality': 'SD',
'language': 'en',
'url': iframe,
'direct': False,
'debridonly': False
})
except:
flashvar = client.parseDOM(html,
'param',
attrs={'name': 'flashvars'},
ret='value')[0]
link = flashvar.split('file=')[1]
host = link.split('//')[1].replace('www.', '')
host = host.split('/')[0].split('.')[0].title()
sources.append({
'source': host,
'quality': 'SD',
'language': 'en',
'url': link,
'direct': False,
'debridonly': False
})
containers = client.parseDOM(html,
'div',
attrs={'class': 'dwn-box'})
for list in containers:
link = client.parseDOM(list,
'a',
attrs={'rel': 'nofollow'},
ret='href')[0]
redirect = client.request(link, output='geturl')
quality, info = source_utils.get_release_quality(redirect)
sources.append({
'source': 'DirectLink',
'quality': quality,
'language': 'en',
'url': redirect,
'info': info,
'direct': True,
'debridonly': False
})
return sources
except:
failure = traceback.format_exc()
log_utils.log('CoolTV - Exception: \n' + str(failure))
return
def resolve_url(url, name=None, iconimage=None, pattern=None):
dialog = xbmcgui.Dialog()
kodi.busy()
try: url,site = url.split('|SPLIT|')
except:
site = 'Unknown'
log_utils.log('Error getting site information from :: %s' % (url), log_utils.LOGERROR)
if not name: name = 'Unknown'
if not iconimage: iconimage = kodi.addonicon
name = re.sub(r'(\[.+?\])','',name); name = name.lstrip()
if '] - ' in name: name = name.split('] - ')[-1]
if 'site=' in url: url,site = url.split('site=')
if '|CHAT|' in url:
url,site,name = url.split('|CHAT|')
if '- [' in name:
name = name.split('- [')[0]
u = None
url2 = url
log_utils.log('Sending %s to XXX Resolver' % (url), log_utils.LOGNOTICE)
blacklist.Blacklistcheck(url)
if resolveurl.HostedMediaFile(url).valid_url():
log_utils.log('%s is a valid SMU resolvable URL. Attempting to resolve.' % (url), log_utils.LOGNOTICE)
try:
u = resolveurl.HostedMediaFile(url).resolve()
if u == None: u = adultresolver.resolve(url2)
except Exception as e:
log_utils.log('Error getting valid link from SMU :: %s :: %s' % (url, str(e)), log_utils.LOGERROR)
try:
u = adultresolver.resolve(url2)
except:
kodi.idle()
kodi.notify(msg='Something went wrong! | %s' % str(e), duration=8000, sound=True)
quit()
log_utils.log('Link returned by XXX Resolver :: %s' % (u), log_utils.LOGNOTICE)
else:
log_utils.log('%s is not a valid SMU resolvable link. Attempting to resolve by XXXODUS backup resolver.' % (url), log_utils.LOGNOTICE)
try:
u = adultresolver.resolve(url)
except Exception as e:
log_utils.log('Error getting valid link from XXXODUS backup resolver. :: %s :: %s' % (url, str(e)), log_utils.LOGERROR)
kodi.idle()
kodi.notify(msg='Something went wrong! | %s' % str(e), duration=8000, sound=True)
quit()
log_utils.log('%s returned by XXX-O-DUS backup resolver.' % (u), log_utils.LOGNOTICE)
if u == 'offline':
kodi.idle()
kodi.notify(msg='This performer is offline.', duration = 5000, sound = True)
quit()
if u:
kodi.idle()
play(u,name,iconimage,url,site)
else:
try:
liz=xbmcgui.ListItem(name)
liz.setArt({"thumb": iconimage})
liz.setProperty('IsPlayable', 'true')
xbmc.executebuiltin("Dialog.Close(busydialog)")
xbmc.Player().play(url, liz, False)
except:
kodi.idle()
log_utils.log('Failed to get any playable link for :: %s' % (url), log_utils.LOGERROR)
kodi.notify(msg='Failed to get any playable link.', duration=7500, sound=True)
quit()
def find(url, name=None, iconimage=None, pattern=None):
kodi.busy()
try:
url, site = url.split('|SPLIT|')
except:
site = 'Unknown'
log_utils.log('Error getting site information from :: %s' % (url),
log_utils.LOGERROR)
try:
if 'streamingporn.xyz' in url:
c = client.request(url)
r = dom_parser2.parse_dom(c,
'a',
req=['href', 'class', 'rel', 'target'])
r = [i for i in r if i.attrs['class'] == 'external']
r = [client.request(i.attrs['href'], output='geturl') for i in r]
r = [i for i in r if resolveurl.HostedMediaFile(i).valid_url()]
url = multi(r)
elif 'spreadporn.org' in url:
c = client.request(url)
r = dom_parser2.parse_dom(c, 'li', req=['data-show', 'data-link'])
r = [(i.attrs['data-link']) for i in r]
url = multi(r)
elif 'pandamovie.co' in url:
c = client.request(url)
r = dom_parser2.parse_dom(c, 'a', req='id')
r = [(i.attrs['href']) for i in r]
url = multi(r)
elif 'xxxmoviestream.com' in url:
c = client.request(url)
pattern = '''<iframe src="(.+?)" scrolling="no" frameborder="0" width="700"'''
r = re.findall(pattern, c)
url = multi(r)
elif 'sexkino.to' in url:
c = client.request(url)
u = dom_parser2.parse_dom(c, 'iframe',
{'class': ['metaframe', 'rptss']})
r = dom_parser2.parse_dom(c, 'tr')
r = [dom_parser2.parse_dom(i, 'a', req='href') for i in r]
r = [client.request(i[0].attrs['href']) for i in r if i]
r = [i.attrs['src'] for i in u] + [
re.findall("window.location.href='([^']+)", i)[0] for i in r
]
url = multi(r)
elif 'watchxxxfree.tv' in url:
r = client.request(url)
pattern = r"""<iframe.+?src=['"]([^'"]+)"""
r = re.findall(pattern, r)
url = multi(r)
except:
kodi.idle()
kodi.notify(msg='Error getting link for (Link Finder) %s' % name)
kodi.idle()
quit()
url += '|SPLIT|%s' % site
kodi.idle()
player.resolve_url(url, name, iconimage)
def init_database(self):
cur_version = kodi.get_version()
db_version = self.__get_db_version()
if not TRIG_DB_UPG:
db_version = cur_version
if db_version is not None and cur_version != db_version:
log_utils.log('DB Upgrade from %s to %s detected.' %
(db_version, cur_version))
self.progress = xbmcgui.DialogProgress()
self.progress.create('SALTS',
line1='Migrating from %s to %s' %
(db_version, cur_version),
line2='Saving current data.')
self.progress.update(0)
self.__prep_for_reinit()
log_utils.log('Building SALTS Database', log_utils.LOGDEBUG)
if self.db_type == DB_TYPES.MYSQL:
self.__execute(
'CREATE TABLE IF NOT EXISTS url_cache (url VARCHAR(255) NOT NULL, response MEDIUMBLOB, timestamp TEXT, PRIMARY KEY(url))'
)
self.__execute(
'CREATE TABLE IF NOT EXISTS db_info (setting VARCHAR(255) NOT NULL, value TEXT, PRIMARY KEY(setting))'
)
self.__execute('CREATE TABLE IF NOT EXISTS rel_url \
(video_type VARCHAR(15) NOT NULL, title VARCHAR(255) NOT NULL, year VARCHAR(4) NOT NULL, season VARCHAR(5) NOT NULL, episode VARCHAR(5) NOT NULL, source VARCHAR(50) NOT NULL, rel_url VARCHAR(255), \
PRIMARY KEY(video_type, title, year, season, episode, source))')
self.__execute(
'CREATE TABLE IF NOT EXISTS other_lists (section VARCHAR(10) NOT NULL, username VARCHAR(255) NOT NULL, slug VARCHAR(255) NOT NULL, name VARCHAR(255), \
PRIMARY KEY(section, username, slug))')
self.__execute(
'CREATE TABLE IF NOT EXISTS saved_searches (id INTEGER NOT NULL AUTO_INCREMENT, section VARCHAR(10) NOT NULL, added DOUBLE NOT NULL,query VARCHAR(255) NOT NULL, \
PRIMARY KEY(id))')
self.__execute(
'CREATE TABLE IF NOT EXISTS bookmark (slug VARCHAR(255) NOT NULL, season VARCHAR(5) NOT NULL, episode VARCHAR(5) NOT NULL, resumepoint DOUBLE NOT NULL, \
PRIMARY KEY(slug, season, episode))')
else:
self.__create_sqlite_db()
self.__execute(
'CREATE TABLE IF NOT EXISTS url_cache (url VARCHAR(255) NOT NULL, response, timestamp, PRIMARY KEY(url))'
)
self.__execute(
'CREATE TABLE IF NOT EXISTS db_info (setting VARCHAR(255), value TEXT, PRIMARY KEY(setting))'
)
self.__execute('CREATE TABLE IF NOT EXISTS rel_url \
(video_type TEXT NOT NULL, title TEXT NOT NULL, year TEXT NOT NULL, season TEXT NOT NULL, episode TEXT NOT NULL, source TEXT NOT NULL, rel_url TEXT, \
PRIMARY KEY(video_type, title, year, season, episode, source))')
self.__execute(
'CREATE TABLE IF NOT EXISTS other_lists (section TEXT NOT NULL, username TEXT NOT NULL, slug TEXT NOT NULL, name TEXT, PRIMARY KEY(section, username, slug))'
)
self.__execute(
'CREATE TABLE IF NOT EXISTS saved_searches (id INTEGER PRIMARY KEY, section TEXT NOT NULL, added DOUBLE NOT NULL,query TEXT NOT NULL)'
)
self.__execute(
'CREATE TABLE IF NOT EXISTS bookmark (slug TEXT NOT NULL, season TEXT NOT NULL, episode TEXT NOT NULL, resumepoint DOUBLE NOT NULL, \
PRIMARY KEY(slug, season, episode))')
# reload the previously saved backup export
if db_version is not None and cur_version != db_version:
log_utils.log('Restoring DB from backup at %s' % (self.mig_path),
log_utils.LOGDEBUG)
self.import_into_db(self.mig_path)
log_utils.log('DB restored from %s' % (self.mig_path))
sql = 'REPLACE INTO db_info (setting, value) VALUES(?,?)'
self.__execute(sql, ('version', kodi.get_version()))
def content(url, searched=False):
try:
c = client.request(url)
soup = BeautifulSoup(c, 'html5lib')
r = soup.find_all('div', class_={'video-box-card item'})
if (not r) and (not searched):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for i in r:
try:
name = i.img['alt']
url2 = i.a['href']
icon = i.img['data-src']
time = i.find('div', class_={'time'}).text
name = ('%s | [COLOR yellow]%s[/COLOR]' % (name, time))
fanarts = xbmc.translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': url2,
'mode': player_mode,
'icon': icon,
'fanart': fanarts,
'description': name,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
search_pattern = '''<link\s*href=['"]([^'"]+)['"]\s*rel=['"]next['"]'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename)
def execute_rpc(self, command):
if not self.has_connection_details:
log_utils.log(
'JSON-RPC Unable to complete request. %s' %
self.connection_details_error, log_utils.LOGINFO)
return {'error': self.connection_details_error}
log_utils.log('JSON-RPC request |%s|' % command, log_utils.LOGDEBUG)
null_response = None
data = json.dumps(command)
request = urllib2.Request(self.url, headers=self.headers, data=data)
method = 'POST'
request.get_method = lambda: method
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError as e:
error = 'JSON-RPC received HTTPError |[Code %s] %s|' % (e.code,
e.msg)
log_utils.log(error, log_utils.LOGINFO)
return {'error': 'HTTPError |[Code %s] %s|' % (e.code, e.msg)}
except urllib2.URLError as e:
error = 'JSON-RPC received URLError |%s|' % e.args
log_utils.log(error, log_utils.LOGINFO)
return {'error': 'URLError |%s|' % e.args}
except socket.timeout as e:
response = None
null_response = {
'result': 'No response/Timed out'
} # some requests do not respond timely. (ie. Player.Open + picture)
if not null_response and response:
contents = response.read()
log_utils.log('JSON-RPC response |%s|' % contents,
log_utils.LOGDEBUG)
json_response = json.loads(contents)
response.close()
else:
json_response = null_response
log_utils.log('JSON-RPC response |%s|' % null_response,
log_utils.LOGDEBUG)
return self._eval_response(json_response)
def solve(url, cj, user_agent=None, wait=True):
if user_agent is None: user_agent = USER_AGENT
headers = {'User-Agent': user_agent, 'Referer': url}
if cj is not None:
try:
cj.load(ignore_discard=True)
except:
pass
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
request = urllib2.Request(url)
for key in headers:
request.add_header(key, headers[key])
try:
response = urllib2.urlopen(request)
html = response.read()
except urllib2.HTTPError as e:
html = e.read()
tries = 0
while tries < MAX_TRIES:
solver_pattern = 'var t,r,a,f,\s*([^=]+)={"([^"]+)":([^}]+)};.+challenge-form\'\);.*?\n.*?;(.*?);a\.value'
vc_pattern = 'input type="hidden" name="jschl_vc" value="([^"]+)'
pass_pattern = 'input type="hidden" name="pass" value="([^"]+)'
init_match = re.search(solver_pattern, html, re.DOTALL)
vc_match = re.search(vc_pattern, html)
pass_match = re.search(pass_pattern, html)
if not init_match or not vc_match or not pass_match:
log_utils.log(
"Couldn't find attribute: init: |%s| vc: |%s| pass: |%s| No cloudflare check?"
% (init_match, vc_match, pass_match), log_utils.LOGWARNING)
return False
init_dict, init_var, init_equation, equations = init_match.groups()
vc = vc_match.group(1)
password = pass_match.group(1)
# log_utils.log("VC is: %s" % (vc), xbmc.LOGDEBUG)
varname = (init_dict, init_var)
result = int(solve_equation(init_equation.rstrip()))
log_utils.log(
'Initial value: |%s| Result: |%s|' % (init_equation, result),
log_utils.LOGDEBUG)
for equation in equations.split(';'):
equation = equation.rstrip()
if equation[:len('.'.join(varname))] != '.'.join(varname):
log_utils.log(
'Equation does not start with varname |%s|' % (equation),
log_utils.LOGDEBUG)
else:
equation = equation[len('.'.join(varname)):]
expression = equation[2:]
operator = equation[0]
if operator not in ['+', '-', '*', '/']:
log_utils.log('Unknown operator: |%s|' % (equation),
log_utils.LOGWARNING)
continue
result = int(
str(
eval(
str(result) + operator +
str(solve_equation(expression)))))
log_utils.log('intermediate: %s = %s' % (equation, result),
log_utils.LOGDEBUG)
scheme = urlparse.urlparse(url).scheme
domain = urlparse.urlparse(url).hostname
result += len(domain)
log_utils.log('Final Result: |%s|' % (result), log_utils.LOGDEBUG)
if wait:
log_utils.log('Sleeping for 5 Seconds', log_utils.LOGDEBUG)
xbmc.sleep(5000)
url = '%s://%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s&pass=%s' % (
scheme, domain, vc, result, urllib.quote(password))
log_utils.log('url: %s' % (url), log_utils.LOGDEBUG)
request = urllib2.Request(url)
for key in headers:
request.add_header(key, headers[key])
try:
opener = urllib2.build_opener(NoRedirection)
urllib2.install_opener(opener)
response = urllib2.urlopen(request)
while response.getcode() in [301, 302, 303, 307]:
if cj is not None:
cj.extract_cookies(response, request)
request = urllib2.Request(
response.info().getheader('location'))
for key in headers:
request.add_header(key, headers[key])
if cj is not None:
cj.add_cookie_header(request)
response = urllib2.urlopen(request)
final = response.read()
if 'cf-browser-verification' in final:
log_utils.log('CF Failure: html: %s url: %s' % (html, url),
log_utils.LOGWARNING)
tries += 1
html = final
else:
break
except urllib2.HTTPError as e:
log_utils.log('CloudFlare Error: %s on url: %s' % (e.code, url),
log_utils.LOGWARNING)
return False
if cj is not None:
cj.save()
return final
def comics(url=None):
reload = False
try:
if (not url): url = urlparse.urljoin(base_domain, 'comix/')
c = client.request(url)
try:
r = dom_parser2.parse_dom(c, 'div', {'class': 'gallery'})
r = dom_parser2.parse_dom(r, 'a', req='href')
r = [i for i in r if 'login' not in i.attrs['href']]
r = [(i.attrs['href'], \
dom_parser2.parse_dom(i, 'img', req='data-src'), \
dom_parser2.parse_dom(i, 'span', {'class': 'title-text'})) \
for i in r if i]
r = [(urlparse.urljoin(base_domain, i[0]), i[2][0].content,
i[1][0].attrs['data-src']) for i in r if i]
reload = True
except:
r = dom_parser2.parse_dom(c, 'div', {'class': 'gallery'})
r = dom_parser2.parse_dom(r, 'a', req='href')
r = [i for i in r if 'login' not in i.attrs['href']]
r = [(i.attrs['href'], \
dom_parser2.parse_dom(i, 'img', req='data-src')) \
for i in r if i]
r = [(urlparse.urljoin(base_domain,
i[0]), i[1][0].attrs['data-src']) for i in r
if i]
reload = False
if (not r):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
quit()
except Exception as e:
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
dirlst = []
if reload:
for i in r:
try:
name = kodi.sortX(i[1].encode('utf-8')).title()
fanarts = xbmc.translatePath(
os.path.join(
'special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': i[0],
'mode': comics_mode,
'icon': i[2],
'fanart': fanarts,
'folder': True
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)),
log_utils.LOGERROR)
else:
num = 1
for i in r:
try:
name = 'Page %s' % str(num)
fanarts = xbmc.translatePath(
os.path.join(
'special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': i[0],
'mode': pic_v_mode,
'icon': i[1],
'fanart': fanarts,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)),
log_utils.LOGERROR)
num += 1
if dirlst:
buildDirectory(dirlst, stopend=True, pictures=True)
search_pattern = '''<span\s*class=['"]next['"]>\s*<a\s*href=['"]([^'"]+)['"]>'''
parse = base_domain
helper.scraper().get_next_page(pic_men_mode,
url,
search_pattern,
filename,
parse,
pictures=True)
else:
kodi.notify(msg='No Menu Items Found')
quit()
def menu():
lover.checkupdates()
url = base_domain
r = requests.get(url, headers=headers).text
r = dom_parser2.parse_dom(r, 'dd')
r = dom_parser2.parse_dom(r, 'a', req='href')
r = [i for i in r if 'private-cams' not in i.attrs['href']]
r = [(urljoin(base_domain, i.attrs['href']), i.content) for i in r if i]
dirlst = []
icon = translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/main/%s.png' % filename))
fanarts = translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': 'Monitored Performers',
'url': 'none',
'mode': 30,
'icon': icon,
'fanart': fanarts,
'folder': True
})
dirlst.append({
'name': 'Search By Username',
'url': 'none',
'mode': 32,
'icon': icon,
'fanart': fanarts,
'folder': False
})
dirlst.append({
'name': 'Rooms By Tag',
'url': 'tags',
'mode': 302,
'icon': icon,
'fanart': fanarts,
'folder': True
})
for i in r:
try:
if PY2: name = kodi.sortX(i[1].encode('utf-8')).title()
else: name = kodi.sortX(i[1]).title()
dirlst.append({
'name': name,
'url': i[0],
'mode': content_mode,
'icon': icon,
'fanart': fanarts,
'folder': True
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)), log_utils.LOGERROR)
#dialog.ok("DIRLIST",str(dirlst))
if dirlst: buildDirectory(dirlst)
else:
kodi.notify(msg='No Menu Items Found')
quit()
def generic(self, url, pattern=None):
if 'youporn.com' in url: u = self.youporn(url)
try:
r = client.request(url)
if pattern: s=re.findall(r'%s' % pattern, r)
else:
patterns = [
r'''\s*=\s*[\'\"](http.+?)[\'\"]''', \
r'''\s*=\s*['"](http.+?)['"]''', \
r'''['"][0-9_'"]+:\s[\'\"]([^'"]+)''', \
r'''\(\w+\([\'\"]([^\'\"]*)''', \
r'''[\'\"]\w+[\'\"]:['"]([^'"]*)''', \
r'''\s*=\s*[\'\"](http.+?)[\'\"]''', \
r'''\s*:\s*[\'\"](//.+?)[\'\"]''', \
r'''\:[\'\"](\.+?)[\'\"]''', \
r'''\s*\(\s*[\'\"](http.+?)[\'\"]''', \
r'''\s*=\s*[\'\"](//.+?)[\'\"]''', \
r'''\w*:\s*[\'\"](http.+?)[\'\"]''', \
r'''\w*=[\'\"]([^\'\"]*)''', \
r'''\w*\s*=\s*[\'\"]([^\'\"]*)''', \
r'''(?s)<file>([^<]*)''', \
]
s = []
for pattern in patterns:
l = re.findall(pattern, r)
s += [i for i in l if (urlparse.urlparse(i).path).strip('/').split('/')[-1].split('.')[-1] in ['mp4', 'flv', 'm3u8']]
if s: s = [i for i in s if (urlparse.urlparse(i).path).strip('/').split('/')[-1].split('.')[-1] in ['mp4', 'flv', 'm3u8']]
else: s = client.parseDOM(r, 'source', ret='src', attrs = {'type': 'video.+?'})
if not s:
log_utils.log('Error resolving %s :: Error: %s' % (url,str(e)), log_utils.LOGERROR)
return
s = ['http:' + i if i.startswith('//') else i for i in s]
s = [urlparse.urljoin(url, i) if not i.startswith('http') else i for i in s]
s = [x for y,x in enumerate(s) if x not in s[:y]]
self.u = []
def request(i):
try:
i = i.replace(' ','%20')
c = client.request(i, output='headers', referer=url)
checks = ['video','mpegurl','html']
if any(f for f in checks if f in c['Content-Type']): self.u.append((i, int(c['Content-Length'])))
except:
pass
threads = []
for i in s: threads.append(workers.Thread(request, i))
[i.start() for i in threads] ; [i.join() for i in threads]
u = sorted(self.u, key=lambda x: x[1])[::-1]
mobile_mode = kodi.get_setting('mobile_mode')
if mobile_mode == 'true': u = client.request(u[-1][0], output='geturl', referer=url)
else: u = client.request(u[0][0], output='geturl', referer=url)
log_utils.log('Returning %s from XXX-O-DUS Resolver' % str(u), log_utils.LOGNOTICE)
return u
except Exception as e:
log_utils.log('Error resolving %s :: Error: %s' % (url,str(e)), log_utils.LOGERROR)
def train(
train_image0_path,
train_image1_path,
train_camera_path,
# Batch settings
n_batch=settings.N_BATCH,
n_height=settings.N_HEIGHT,
n_width=settings.N_WIDTH,
encoder_type=settings.ENCODER_TYPE,
decoder_type=settings.DECODER_TYPE,
activation_func=settings.ACTIVATION_FUNC,
n_pyramid=settings.N_PYRAMID,
# Training settings
n_epoch=settings.N_EPOCH,
learning_rates=settings.LEARNING_RATES,
learning_schedule=settings.LEARNING_SCHEDULE,
use_augment=settings.USE_AUGMENT,
w_color=settings.W_COLOR,
w_ssim=settings.W_SSIM,
w_smoothness=settings.W_SMOOTHNESS,
w_left_right=settings.W_LEFT_RIGHT,
# Depth range settings
scale_factor=settings.SCALE_FACTOR,
# Checkpoint settings
n_summary=settings.N_SUMMARY,
n_checkpoint=settings.N_CHECKPOINT,
checkpoint_path=settings.CHECKPOINT_PATH,
# Hardware settings
device=settings.DEVICE,
n_thread=settings.N_THREAD):
if device == settings.CUDA or device == settings.GPU:
device = torch.device(settings.CUDA)
else:
device = torch.device(settings.CPU)
if not os.path.exists(checkpoint_path):
os.makedirs(checkpoint_path)
# Set up checkpoint and event paths
encoder_checkpoint_path = os.path.join(checkpoint_path, 'encoder-{}.pth')
decoder_checkpoint_path = os.path.join(checkpoint_path, 'decoder-{}.pth')
log_path = os.path.join(checkpoint_path, 'results.txt')
event_path = os.path.join(checkpoint_path, 'events')
# Read paths for training
train_image0_paths = data_utils.read_paths(train_image0_path)
train_image1_paths = data_utils.read_paths(train_image1_path)
train_camera_paths = data_utils.read_paths(train_camera_path)
assert len(train_image0_paths) == len(train_image1_paths)
assert len(train_image0_paths) == len(train_camera_paths)
n_train_sample = len(train_image0_paths)
n_train_step = n_epoch * np.ceil(n_train_sample / n_batch).astype(np.int32)
train_dataloader = torch.utils.data.DataLoader(
datasets.ImagePairCameraDataset(train_image0_paths,
train_image1_paths,
train_camera_paths,
shape=(n_height, n_width),
augment=use_augment),
batch_size=n_batch,
shuffle=True,
num_workers=n_thread,
drop_last=False)
# Build network
model = MonodepthModel(encoder_type=encoder_type,
decoder_type=decoder_type,
activation_func=activation_func,
n_pyramid=n_pyramid,
scale_factor=scale_factor,
device=device)
train_summary = SummaryWriter(event_path)
parameters = model.parameters()
n_param = sum(p.numel() for p in parameters)
# Start training
model.train()
log('Network settings:', log_path)
log(
'n_batch=%d n_height=%d n_width=%d n_param=%d' %
(n_batch, n_height, n_width, n_param), log_path)
log(
'encoder_type=%s decoder_type=%s activation_func=%s n_pyramid=%d' %
(encoder_type, decoder_type, activation_func, n_pyramid), log_path)
log('Training settings:', log_path)
log(
'n_sample=%d n_epoch=%d n_step=%d' %
(n_train_sample, n_epoch, n_train_step), log_path)
log(
'learning_schedule=[%s]' %
', '.join('{}:{}'.format(l * (n_train_sample // n_batch), v, log_path)
for l, v in zip([0] + learning_schedule, learning_rates)),
log_path)
log('use_augment=%s' % use_augment, log_path)
log(
'w_color=%.2f w_ssim=%.2f w_smoothness=%.2f w_left_right=%.2f' %
(w_color, w_ssim, w_smoothness, w_left_right), log_path)
log('Depth range settings:', log_path)
log('scale_factor=%.2f' % (scale_factor), log_path)
log('Checkpoint settings:', log_path)
log('depth_model_checkpoint_path=%s' % checkpoint_path, log_path)
learning_schedule.append(n_epoch)
schedule_pos = 0
train_step = 0
time_start = time.time()
log('Begin training...', log_path)
for epoch in range(1, n_epoch + 1):
# Set learning rate schedule
if epoch > learning_schedule[schedule_pos]:
schedule_pos = schedule_pos + 1
learning_rate = learning_rates[schedule_pos]
optimizer = torch.optim.Adam(parameters, lr=learning_rate)
for train_image0, train_image1, train_camera in train_dataloader:
train_step = train_step + 1
# Fetch data
if device.type == settings.CUDA:
train_image0 = train_image0.cuda()
train_image1 = train_image1.cuda()
train_camera = train_camera.cuda()
# Forward through the network
model.forward(train_image0, train_camera)
# Compute loss function
loss = model.compute_loss(train_image0,
train_image1,
w_color=w_color,
w_ssim=w_ssim,
w_smoothness=w_smoothness,
w_left_right=w_left_right)
# Compute gradient and backpropagate
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (train_step % n_summary) == 0:
model.log_summary(summary_writer=train_summary,
step=train_step)
# Log results and save checkpoints
if (train_step % n_checkpoint) == 0:
time_elapse = (time.time() - time_start) / 3600
time_remain = (n_train_step -
train_step) * time_elapse / train_step
log(
'Step={:6}/{} Loss={:.5f} Time Elapsed={:.2f}h Time Remaining={:.2f}h'
.format(train_step, n_train_step, loss.item(), time_elapse,
time_remain), log_path)
# Save checkpoints
torch.save(
{
'train_step': train_step,
'model_state_dict': model.encoder.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, encoder_checkpoint_path.format(train_step))
torch.save(
{
'train_step': train_step,
'model_state_dict': model.decoder.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, decoder_checkpoint_path.format(train_step))
# Save checkpoints and close summary
train_summary.close()
torch.save(
{
'train_step': train_step,
'model_state_dict': model.encoder.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, encoder_checkpoint_path.format(train_step))
torch.save(
{
'train_step': train_step,
'model_state_dict': model.decoder.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, decoder_checkpoint_path.format(train_step))
def http_response(self, request, response):
log_utils.log('Stopping Redirect', log_utils.LOGDEBUG)
return response
def content(url, searched=False):
try:
c = requests.get(url, headers=headers).content
soup = BeautifulSoup(c, 'html5lib')
r = soup.find_all('li', class_={'masonry-item'})
if (not r) and (not searched):
log_utils.log(
'Scraping Error in %s:: Content of request: %s' %
(base_name.title(), str(c)), log_utils.LOGERROR)
kodi.notify(msg='Scraping Error: Info Added To Log File',
duration=6000,
sound=True)
except Exception as e:
if (not searched):
log_utils.log(
'Fatal Error in %s:: Error: %s' % (base_name.title(), str(e)),
log_utils.LOGERROR)
kodi.notify(msg='Fatal Error', duration=4000, sound=True)
quit()
else:
pass
dirlst = []
for i in r:
try:
name = i.a['title']
url2 = i.a['href']
icon = i.img['src']
fanarts = xbmc.translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': url2,
'mode': player_mode,
'icon': icon,
'fanart': fanarts,
'description': name,
'folder': False
})
except Exception as e:
log_utils.log('Error: %s' % str(e), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst, stopend=True, isVideo=True, isDownloadable=True)
else:
if (not searched):
kodi.notify(msg='No Content Found')
quit()
if searched: return str(len(r))
if not searched:
try:
search_pattern = '''href\=['"]([^'"]+)['"]\s+rel=['"]next'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern,
filename)
except Exception as e:
log_utils.log(
'Error getting next page for %s :: Error: %s' %
(base_name.title(), str(e)), log_utils.LOGERROR)
def content(url, searched=False):
r = requests.get(url, headers=headers).text
r = dom_parser2.parse_dom(r, 'li')
r = [(dom_parser2.parse_dom(i, 'div', {'class': 'title'}), \
dom_parser2.parse_dom(i, 'img', req='src'), \
dom_parser2.parse_dom(i, 'div', {'class': re.compile('thumbnail_label.+?')}), \
dom_parser2.parse_dom(i, 'li', {'title': re.compile('.+?')}), \
dom_parser2.parse_dom(i, 'li', {'class': 'location'}), \
dom_parser2.parse_dom(i, 'li', {'class': 'cams'}) \
) for i in r if '<div class="title">' in i.content]
r = [(dom_parser2.parse_dom(i[0], 'a'), \
dom_parser2.parse_dom(i[0], 'span'), \
i[2][0].content, \
i[1][0].attrs['src'], \
i[3][0].content if i[3] else 'Unknown', \
i[4][0].content, \
i[5][0].content, \
) for i in r]
r = [(urljoin(base_domain, i[0][0].attrs['href']), i[0][0].content,
i[1][0].content, i[2], i[3], i[6], i[5], i[4]) for i in r]
dirlst = []
for i in r:
try:
if PY2:
name = '%s - [ %s ]' % (kodi.sortX(
i[1].encode('utf-8')).title(),
kodi.sortX(i[3].encode('utf-8')))
else:
name = '%s - [ %s ]' % (kodi.sortX(
i[1]).title(), kodi.sortX(i[3]))
if PY2: description = 'Name: %s \nAge: %s \nLocation: %s \nStats: %s \n\nDescription: %s' % \
(kodi.sortX(i[1].encode('utf-8')),i[2],kodi.sortX(i[6].encode('utf-8')),kodi.sortX(i[5].encode('utf-8')),kodi.sortX(i[7].encode('utf-8')))
else: description = 'Name: %s \nAge: %s \nLocation: %s \nStats: %s \n\nDescription: %s' % \
(kodi.sortX(i[1]),i[2],kodi.sortX(i[6]),kodi.sortX(i[5]),kodi.sortX(i[7]))
content_url = i[0] + '|SPLIT|%s' % base_name
fanarts = translatePath(
os.path.join('special://home/addons/script.xxxodus.artwork',
'resources/art/%s/fanart.jpg' % filename))
dirlst.append({
'name': name,
'url': content_url,
'mode': player_mode,
'icon': i[4],
'fanart': fanarts,
'description': description,
'folder': False
})
except Exception as e:
log_utils.log(
'Error adding menu item %s in %s:: Error: %s' %
(i[1].title(), base_name.title(), str(e)), log_utils.LOGERROR)
if dirlst:
buildDirectory(dirlst,
stopend=True,
isVideo=False,
isDownloadable=False,
chaturbate=True)
else:
kodi.notify(msg='No Content Found')
quit()
search_pattern = '''<li><a\s*href=['"]([^'"]+)['"]\s*class=['"]next endless_page_link'''
parse = base_domain
helper.scraper().get_next_page(content_mode, url, search_pattern, filename,
parse)
