def get_seasonal_shows(self, year=None, season=None, **kwargs):
#TODO: use year and season if provided
debug("Getting season shows: year={}, season={}".format(year, season))
# Request season page from MAL
response = self._mal_request(self._season_show_url, **kwargs)
if response is None:
error("Cannot get show list")
return list()
# Parse page (ugh, HTML parsing. Where's the useful API, MAL?)
lists = response.find_all(class_="seasonal-anime-list")
if len(lists) == 0:
error("Invalid page? Lists not found")
return list()
new_list = lists[0].find_all(class_="seasonal-anime")
if len(new_list) == 0:
error("Invalid page? Shows not found in list")
return list()
new_shows = list()
episode_count_regex = re.compile("(\d+|\?) eps?")
for show in new_list:
show_key = show.find(class_="genres")["id"]
title = str(show.find("a", class_="link-title").string)
title = _normalize_title(title)
more_names = [title[:-11]] if title.lower().endswith("2nd season") else list()
show_type = ShowType.TV #TODO, changes based on section/list
episode_count = episode_count_regex.search(show.find(class_="eps").find(string=episode_count_regex)).group(1)
episode_count = None if episode_count == "?" else int(episode_count)
has_source = show.find(class_="source").string != "Original"
new_shows.append(UnprocessedShow(self.key, show_key, title, more_names, show_type, episode_count, has_source))
return new_shows
def get_seasonal_shows(self, year=None, season=None, **kwargs):
return []
#TODO: use year and season if provided
debug("Getting season shows: year={}, season={}".format(year, season))
# Request season page from AniDB
response = self._site_request(self._season_url, **kwargs)
if response is None:
error("Cannot get show list")
return list()
# Parse page
shows_list = response.select(
".calendar_all .g_section.middle .content .box")
new_shows = list()
for show in shows_list:
top = show.find(class_="top")
title_e = top.find("a")
title = str(title_e.string)
title = _normalize_title(title)
show_link = title_e["href"]
key = re.search("aid=([0-9]+)", show_link).group(1)
data = show.find(class_="data")
more_names = list()
show_info_str = data.find(class_="series").string.strip()
debug("Show info: {}".format(show_info_str))
show_info = show_info_str.split(", ")
show_type = _convert_show_type(show_info[0])
if len(show_info) == 1:
episode_count = 1
else:
ec_match = re.match("([0-9]+) eps", show_info[1])
episode_count = int(ec_match.group(1)) if ec_match else None
tags = data.find(class_="tags")
has_source = tags.find(
"a", string=re.compile("manga|novel|visual novel")) is not None
new_shows.append(
UnprocessedShow(self.key, key, title, more_names, show_type,
episode_count, has_source))
return new_shows
def find_show(self, show_name, **kwargs):
url = self._api_search_base.format(q=show_name)
result = self._mal_api_request(url, **kwargs)
if result is None:
error("Failed to find show")
return list()
assert result.tag == "anime"
shows = list()
for child in result:
print(child)
assert child.tag == "entry"
id = child.find("id").text
name = child.find("title").text
more_names = [child.find("english").text]
show = UnprocessedShow(self.key, id, name, more_names, ShowType.UNKNOWN, 0, False)
shows.append(show)
return shows
def find_show_info(self, show_id, **kwargs):
debug("Getting show info for {}".format(show_id))
# Request show page from MAL
url = self._show_link_base.format(id=show_id)
response = self._mal_request(url, **kwargs)
if response is None:
error("Cannot get show page")
return None
# Parse show page
names_sib = response.find("h2", string="Alternative Titles")
# English
name_elem = names_sib.find_next_sibling("div")
if name_elem is None:
warning(" Name elem not found")
return None
name_english = name_elem.string
info(" English: {}".format(name_english))
names = [name_english]
return UnprocessedShow(self.key, id, None, names, ShowType.UNKNOWN, 0, False)
def _edit_with_file(db, edit_file):
import yaml
info("Parsing show edit file \"{}\"".format(edit_file))
try:
with open(edit_file, "r", encoding="UTF-8") as f:
parsed = list(yaml.full_load_all(f))
except yaml.YAMLError:
exception("Failed to parse edit file")
return
debug(" num shows={}".format(len(parsed)))
for doc in parsed:
name = doc["title"]
stype = str_to_showtype(doc.get("type", "tv")) # convert to enum?
length = doc.get("length", 0)
has_source = doc.get("has_source", False)
is_nsfw = doc.get("is_nsfw", False)
info("Adding show \"{}\" ({})".format(name, stype))
debug(" has_source={}".format(has_source))
debug(" is_nsfw={}".format(is_nsfw))
if stype == ShowType.UNKNOWN:
error("Invalid show type \"{}\"".format(stype))
return False
show = UnprocessedShow(None, None, name, [], stype, length, has_source,
is_nsfw)
found_ids = db.search_show_ids_by_names(name, exact=True)
debug("Found ids: {found_ids}")
if len(found_ids) == 0:
show_id = db.add_show(show, commit=False)
elif len(found_ids) == 1:
show_id = found_ids.pop()
db.update_show(show_id, show, commit=False)
else:
error("More than one ID found for show")
return False
# Info
if "info" in doc:
infos = doc["info"]
for info_key in infos:
url = infos[info_key]
if not url:
continue
debug(" Info {}: {}".format(info_key, url))
info_handler = services.get_link_handler(key=info_key)
if info_handler:
info_id = info_handler.extract_show_id(url)
debug(" id={}".format(info_id))
if not db.has_link(info_key, info_id):
show.site_key = info_key
show.show_key = info_id
db.add_link(show, show_id, commit=False)
else:
error(" Info handler not installed")
# Streams
if "streams" in doc:
streams = doc["streams"]
for service_key in streams:
url = streams[service_key]
if not url:
continue
remote_offset = 0
try:
roi = url.rfind("|")
if roi > 0:
if roi + 1 < len(url):
remote_offset = int(url[roi + 1:])
url = url[:roi]
except:
exception(
"Improperly formatted stream URL \"{}\"".format(url))
continue
info(" Stream {}: {}".format(service_key, url))
service_id = service_key.split('|')[0]
stream_handler = services.get_service_handler(key=service_id)
if stream_handler:
show_key = stream_handler.extract_show_key(url)
debug(" id={}".format(show_key))
if not db.has_stream(service_id, show_key):
s = UnprocessedStream(service_id, show_key, None, "",
remote_offset, 0)
db.add_stream(s, show_id, commit=False)
else:
service = db.get_service(key=service_id)
s = db.get_stream(service_tuple=(service, show_key))
db.update_stream(s,
show_key=show_key,
remote_offset=remote_offset,
commit=False)
elif "|" in service_key:
# Lite stream
service, service_name = service_key.split("|", maxsplit=1)
db.add_lite_stream(show_id, service, service_name, url)
else:
error(" Stream handler not installed")
# Aliases
if "alias" in doc:
aliases = doc["alias"]
for alias in aliases:
db.add_alias(show_id, alias)
info(
f"Added {len(aliases)} alias{'es' if len(aliases) > 1 else ''}"
)
return True
def _edit_with_file(db, edit_file):
import yaml
info("Parsing show edit file \"{}\"".format(edit_file))
try:
with open(edit_file, "r", encoding="UTF-8") as f:
parsed = list(yaml.load_all(f))
except yaml.YAMLError:
exception("Failed to parse edit file")
return
debug(" num shows={}".format(len(parsed)))
for doc in parsed:
name = doc["title"]
stype = str_to_showtype(doc["type"]) # convert to enum?
length = doc["length"] if "length" in doc else 0
has_source = doc["has_source"]
info("Adding show \"{}\" ({})".format(name, stype))
debug(" has_source={}".format(has_source))
if stype == ShowType.UNKNOWN:
error("Invalid show type \"{}\"".format(stype))
return False
show = UnprocessedShow(None, None, name, [], stype, length, has_source)
found_ids = db.search_show_ids_by_names(name, exact=True)
if len(found_ids) == 0:
show_id = db.add_show(show, commit=False)
elif len(found_ids) == 1:
show_id = found_ids.pop()
db.update_show(show_id, show, commit=False)
else:
error("More than one ID found for show")
return False
# Info
if "info" in doc:
infos = doc["info"]
for info_key in infos:
url = infos[info_key]
if not url:
continue
debug(" Info {}: {}".format(info_key, url))
info_handler = services.get_link_handler(key=info_key)
if info_handler:
info_id = info_handler.extract_show_id(url)
debug(" id={}".format(info_id))
if not db.has_link(info_key, info_id):
show.site_key = info_key
show.show_key = info_id
db.add_link(show, show_id, commit=False)
else:
error(" Info handler not installed")
# Streams
if "streams" in doc:
streams = doc["streams"]
for service_key in streams:
url = streams[service_key]
if not url:
continue
remote_offset = 0
try:
roi = url.rfind("|")
if roi > 0:
if roi+1 < len(url):
remote_offset = int(url[roi+1:])
url = url[:roi]
except:
exception("Improperly formatted stream URL \"{}\"".format(url))
continue
info(" Stream {}: {}".format(service_key, url))
stream_handler = services.get_service_handler(key=service_key)
if stream_handler:
show_key = stream_handler.extract_show_key(url)
debug(" id={}".format(show_key))
if not db.has_stream(service_key, show_key):
s = UnprocessedStream(service_key, show_key, None, "", remote_offset, 0)
db.add_stream(s, show_id, commit=False)
else:
service = db.get_service(key=service_key)
s = db.get_stream(service_tuple=(service, show_key))
db.update_stream(s, show=show_id, show_key=show_key, remote_offset=remote_offset, commit=False)
else:
error(" Stream handler not installed")
return True