def fetch_stream_url(self, stream_page):
selectors = self.selectors
driver = self.driver
driver.get(stream_page)
self._execute_js_scripts()
# Choose yourupload as streaming server
yourupload_button = driver.find_element_by_css_selector(
selectors.YOURUPLOAD)
yourupload_button.click()
yourupload_button.click()
player = driver.find_element_by_css_selector(selectors.PLAYER)
res, calls, success = call_till_true(is_iframe_loaded,
self.episode_fetch_timeout,
driver)
printd("outside wait loop ;", "success:", success, "calls:", calls)
# No need to click. YourUpload initiates the request during page load.
# video = player.find_element_by_css_selector("video")
# video.click()
return self.search_url_in_perflogs(self.regex_pattern_objects)
def fetch_stream_url(self, stream_page):
selectors = self.selectors
driver = self.driver
driver.get(stream_page)
self._execute_js_scripts()
# Choose mp4upload as streaming server
# if self.selectors != KickassAnimeSelectors:
# # Kickassanime uses MP4Upload by default
# # print("clicking")
mp4upload_button = driver.find_element_by_css_selector(
selectors.MP4UPLOAD)
mp4upload_button.click()
mp4upload_button.click()
player = driver.find_element_by_css_selector(selectors.PLAYER)
# if self.selectors != KickassAnimeSelectors:
# res, calls, success = call_till_true(is_iframe_loaded, self.episode_fetch_timeout, driver)
# else:
# res, calls, success = call_till_true(is_document_loaded, self.episode_fetch_timeout, driver)
res, calls, success = call_till_true(is_iframe_loaded,
self.episode_fetch_timeout,
driver)
printd("outside wait loop ;", "success:", success, "calls:", calls)
iframe = player.find_element_by_css_selector("iframe")
iframe.click()
return self.search_url_in_perflogs(self.regex_pattern_objects)
def add_to_queue(self, directory, filename, url):
"""
Uses the IDM comand line interface to add files to the main download queue
"""
command = 'idman /d "{0}" /p "{1}" /f "{2}" /a'.format(url, directory, filename)
printd(command)
os.system(command)
def add_to_queue(self, directory, filename, url):
"""
Uses the uGet comand line interface to add files to the main download queue
"""
command = 'nohup uget-gtk --quiet --folder="{1}" --filename="{2}" "{0}" &'.format(url, directory, filename)
printd(command)
os.system(command)
def is_document_loaded(webdriver):
status_raw = webdriver.find_element_by_css_selector(LOAD_STATUS_SELECTOR).text
status = json.loads(status_raw)
printd("waiting")
if status["document_loaded"]:
printd("document loaded")
return True
return False
def fetch_stream_url(self, stream_page):
selectors = self.selectors
driver = self.driver
driver.get(stream_page)
self._execute_js_scripts()
# Choose openupload as streaming server
# GoGoAnime has 2 open upload buttons. First one's broken.
openupload_buttons = driver.find_elements_by_css_selector(
selectors.OPENUPLOAD)
openupload_button = openupload_buttons[-1]
if selectors == GoGoAnimeSelectors:
openupload_button.click()
else:
real_btn = openupload_button.find_element_by_tag_name("a")
time.sleep(3)
real_btn.click()
player = driver.find_element_by_css_selector(selectors.PLAYER)
res, calls, success = call_till_true(is_iframe_loaded,
self.episode_fetch_timeout,
driver)
printd("outside wait loop ;", "success:", success, "calls:", calls)
frame = player.find_element_by_tag_name("iframe")
if selectors == GoGoAnimeSelectors:
frame.click() # First click opens a pop up
time.sleep(2)
frame.click() # Second click triggers stream
else:
# Kickassanime is weird. They have a fake player on top of the real one.
time.sleep(2)
frame.click()
time.sleep(7)
frame.click()
time.sleep(3)
stream_url = self.search_url_in_perflogs(self.regex_pattern_objects)
if not stream_url:
player.click()
time.sleep(2)
stream_url = self.search_url_in_perflogs(
self.regex_pattern_objects)
return stream_url
def fetch_stream_url(self, stream_page):
selectors = self.selectors
driver = self.driver
driver.get(stream_page)
self._execute_js_scripts()
# Choose openupload as streaming server
# GoGoAnime has 2 open upload buttons. First one's broken.
openupload_buttons = driver.find_elements_by_css_selector(selectors.OPENUPLOAD)
openupload_button = openupload_buttons[1] if len(openupload_buttons) > 1 else openupload_buttons[0]
openupload_button.click()
openupload_button.click() # First click doesn't register in some cases
player = driver.find_element_by_css_selector(selectors.PLAYER)
'''
while True:
try:
status_raw = driver.find_element_by_css_selector(LOAD_STATUS_SELECTOR).text
status = json.loads(status_raw)
print("in loop")
if status["iframe_loaded"]:
print("iframe loaded")
break
except NoSuchElementException as err:
print("not there yet " + err.msg)
'''
res, calls, success = call_till_true(is_iframe_loaded, self.episode_fetch_timeout, driver)
printd("outside wait loop ;", "success:", success, "calls:", calls)
frame = player.find_element_by_tag_name("iframe")
# TODO: Find alternative to use time.sleep()
# First click opens a pop up
frame.click()
# [BAD] Waiting to make sure media player is accessible in the iframe
# (because I have a wooden PC from the stone age -_-)
time.sleep(2)
# Second click triggers stream
frame.click()
# time.sleep(3) # [BAD] Waiting to make sure HTTP request to the stream file is made
return self.search_url_in_perflogs(self.regex_pattern_objects)
def search_url_in_perflogs(self, regex_objects):
"""
Access driver performance logs and find the stream URL by matching with the
regular expression for the server.
"""
proxy = self.proxy.proxy
for entry in reversed(proxy.har['log']['entries']):
url = entry['request']['url']
printd(url, self.previos_urls)
if url not in self.previos_urls:
for regex in regex_objects:
if regex.match(url):
self.previos_urls.append(url)
return url
return ""
def fetch_episode_list(self):
# -> { 'Episode 1': 'https://www.kickassanime.ru/anime/gintama/episode-1', ... }
if self.episodes_dict:
return self.episodes_dict
printd("fetching episode list")
printing.fetching_list(self.anime_url)
driver = self.driver
# Execute JS to load the entire episode list
self._execute_js_scripts()
ep_list_container = driver.find_element_by_css_selector(
GoGoAnimeSelectors.EPISODE_LIST)
def fetch_ep_list(container):
return container.find_elements_by_css_selector(
GoGoAnimeSelectors.EPISODE)
# Sometimes the episode list takes a while to load and fetch_ep_list gets 0 episodes
# call_till_true will keep trying for n seconds till we get >0 episodes
ep_list, calls, success = call_till_true(
fetch_ep_list, TimeoutConfig.FETCHING_EPISODE_LIST,
ep_list_container)
if not success:
# TODO: Change error raised
raise ValueError("Failed to fetch episode list")
printd("calls", calls)
# print(ep_list_container.text)
# print(len(ep_list))
ep_dict = {}
for ep in ep_list:
if ep.text:
episode_name = re.search(r"EP ([\d\-\.]+)",
ep.text).group().replace(
"EP", "Episode")
ep_dict[episode_name] = ep.get_attribute("href")
# print(ep_dict)
return ep_dict
def fetch_episode_list(self):
# -> { 'Episode 1': 'https://www.kickassanime.ru/anime/gintama/episode-1', ... }
if self.episodes_dict:
return self.episodes_dict
printd("fetching episode list")
printing.fetching_list(self.anime_url)
driver = self.driver
# ep_list_container = driver.find_element_by_css_selector(KickassAnimeSelectors.EPISODE_LIST)
# def fetch_ep_list(container):
# return container.find_elements_by_css_selector(KickassAnimeSelectors.EPISODE)
# # Sometimes the episode list takes a while to load and we fetch_ep_list gets 0 episodes
# # call_till_true will keep trying for n seconds till we get >0 episodes
# ep_list, calls, success = call_till_true(fetch_ep_list, TimeoutConfig.FETCHING_EPISODE_LIST, ep_list_container)
# printd(ep_list)
# if not success:
# # TODO: Change error raised
# raise ValueError("Failed to fetch episode list")
# printd("calls", calls)
# print(ep_list_container.text)
# print(len(ep_list))
source = self.cfscraper.get(self.anime_url).content
soup = bs(source, "lxml")
ep_list = soup.select(KickassAnimeSelectors.EPISODE_LIST)[0].select(
KickassAnimeSelectors.EPISODE)
ep_dict = {}
for ep in ep_list:
if ep.text and ep.text not in ep_dict:
ep_name = ep.text
ep_page = ep["href"]
ep_dict[ep_name] = ep_page
printd(ep_dict)
return ep_dict
def fetch_episode(self, episode_name):
# -> { stream_page: http://.../watch/episode-01, stream_url: http://.../file.mp4 }
if episode_name in self.episodes_dict:
stream_page = self.episodes_dict[episode_name]
printd("Fetching", episode_name)
printing.fetching_episode(episode_name, stream_page)
# stream_url = self.server_scraper.fetch_stream_url(stream_page)
try:
stream_url = self.server_scraper.fetch_stream_url(stream_page)
except:
stream_url = ""
result = {"stream_page": stream_page, "stream_url": stream_url}
printd(result)
printing.fetched_episode(episode_name, stream_url,
True if stream_url else False)
return result
raise ValueError("%s does not exist" % episode_name)