def __init__(self):
self.return_sites = {
'mal': {'notify': 'myanimelist/anime', 'hato': 'mal_id'},
'kitsu': {'notify': 'kitsu/anime', 'hato': 'kitsu_id'},
'anilist': {'notify': 'anilist/anime', 'hato': 'anilist_id'}}
self.thread_list = ThreadPool()
self.remaining_threads = []
self.id_list = []
class Mappings:
def __init__(self):
self.return_sites = {
'mal': {'notify': 'myanimelist/anime', 'hato': 'mal_id'},
'kitsu': {'notify': 'kitsu/anime', 'hato': 'kitsu_id'},
'anilist': {'notify': 'anilist/anime', 'hato': 'anilist_id'}}
self.thread_list = ThreadPool()
self.remaining_threads = []
self.id_list = []
def get(self, site, id, return_site):
try:
resp = requests.get('https://arm.now.sh/api/v1/search?type=%s&id=%s' % (site, id))
load = json.loads(resp.content)
return load['services'][return_site]
except:
#Try Hato then Notify then just take the data
resp = requests.get('https://hato.malupdaterosx.moe/api/mappings/%s/anime/%s' % (site, id), headers={'User-Agent': tools.get_random_ua()})
load = json.loads(resp.content)
data = load['data']
try:
notify = data['notify_id']
resp = requests.get('https://notify.moe/api/anime/%s' % notify)
load = json.loads(resp.content)
mappings = load['mappings']
for a in mappings:
if a['service'] == self.return_sites[return_site]['notify']:
return a['serviceId']
except:
return data[self.return_sites[return_site]['hato']]
def get_thread(self, site, id, return_site):
new_id = cache.hummingCache().cacheCheck(self.get, 8760, site, id, return_site)
self.id_list.append({site: id, return_site: new_id})
self.remaining_threads.remove(id)
def get_list(self, site, list, return_site):
for a in list:
self.remaining_threads.append(a)
self.thread_list.put(self.get_thread, site, a, return_site)
while len(self.remaining_threads) > 0:
time.sleep(1)
sorted_list = []
for a in list:
for b in self.id_list:
if b[site] == a:
sorted_list.append(b[return_site])
return sorted_list
def __init__(self):
# Base Variables
self.hosters = providers.get_hosters()
self.torrents = providers.get_torrents()
self.remaining_providers = []
self.sources = {}
self.data = {}
self.task_queue = ThreadPool()
# Scraping Strings
self.heading = tools.addonName + ': Scraping'
self.startingLineOne = 'Starting Scraping...'
self.hosterLineOne = 'Hosters Scraping...'
self.torrentLineOne = 'Torrents Scraping...'
self.episodeLineOne = 'Scraping %s - Episode %s'
self.movieLineOne = 'Scraping %s'
self.LineTwo = 'Remaining Providers: %s'
self.LineThree = 'Sources Found: %s'
# Pre-emptive Settings
self.preemp_minimum_sources = int(tools.getSetting('preemptive.limit'))
self.preemp_resolution = int(tools.getSetting('preemptive.resolution'))
self.preemptive_source_type = tools.getSetting('preemptive.sourcetype')
self.preemptive_audiotype = tools.getSetting('preemptive.audiotype')
def __init__(self):
self.thread_list = ThreadPool()
self.remaining_threads = []
self.episodeList = []
self.extractedItems = []
class KitsuBrowser:
def __init__(self):
self.thread_list = ThreadPool()
self.remaining_threads = []
self.episodeList = []
self.extractedItems = []
def trending(self):
anime = []
resp = cache.hummingCache().cacheCheck(Backend().request, 24,
'/trending/anime')
load = json.loads(resp)
data = load['data']
for a in data:
self.remaining_threads.append(a['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, a)
while len(self.remaining_threads) > 0:
time.sleep(1)
sortedItems = tools.sort_anime_by_json(self.extractedItems, data)
return sortedItems
def top_airing(self, page):
anime = []
sort_status = tools.getSetting('airing.sort')
sort = tools.menu_sort[sort_status]
resp = cache.hummingCache().cacheCheck(
Backend().request,
24,
'/anime?filter[status]=current&sort=%s' % (sort),
page=page)
load = json.loads(resp)
data = load['data']
for a in data:
self.remaining_threads.append(a['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, a)
while len(self.remaining_threads) > 0:
time.sleep(1)
sortedItems = tools.sort_anime_by_json(self.extractedItems, data)
return sortedItems
def top_upcoming(self, page):
anime = []
sort_status = tools.getSetting('upcoming.sort')
sort = tools.menu_sort[sort_status]
resp = cache.hummingCache().cacheCheck(
Backend().request,
24,
'/anime?filter[status]=upcoming&sort=%s' % sort,
page=page)
load = json.loads(resp)
data = load['data']
for a in data:
self.remaining_threads.append(a['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, a)
while len(self.remaining_threads) > 0:
time.sleep(1)
sortedItems = tools.sort_anime_by_json(self.extractedItems, data)
return sortedItems
def most_popular(self, page):
anime = []
resp = cache.hummingCache().cacheCheck(Backend().request,
24,
'/anime?sort=popularityRank',
page=page)
load = json.loads(resp)
data = load['data']
for a in data:
self.remaining_threads.append(a['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, a)
while len(self.remaining_threads) > 0:
time.sleep(1)
sortedItems = tools.sort_anime_by_json(self.extractedItems, data)
return sortedItems
def highest_rated(self, page):
anime = []
resp = cache.hummingCache().cacheCheck(Backend().request,
24,
'/anime?sort=ratingRank',
page=page)
load = json.loads(resp)
data = load['data']
for a in data:
self.remaining_threads.append(a['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, a)
while len(self.remaining_threads) > 0:
time.sleep(1)
sortedItems = tools.sort_anime_by_json(self.extractedItems, data)
return sortedItems
def search(self,
title=None,
year=None,
rating=None,
season=None,
subtype=None,
genre=None,
page=1):
filters = []
if title != None and title != '':
filters.append('filter[text]=%s' % title)
if season == None or season == '':
if year != None and year != '':
filters.append(
'filter[season]=winter,spring,summer,fall&filter[seasonYear]=%s'
% year)
if season != None and season != '':
if year == None or year == '':
filters.append('filter[season]=%s' % season)
if year != None and year != '':
filters.append('filter[season]=%s&filter[seasonYear]=%s' %
(season, year))
if rating != None and rating != '':
filters.append('filter[ageRating]=%s' % rating)
if subtype != None and subtype != '':
filters.append('filter[subtype]=%s' % subtype)
if genre != None and genre != '':
filters.append('filter[categories]=%s' % genre)
#for a in filters:
#if a == 'filter[text]=None':
#filters.remove(a)
filter_string = '&'.join(filters)
sort_status = tools.getSetting('search.sort')
sort_string = tools.menu_sort[sort_status]
if title == None or title == '':
request = '/anime?' + filter_string + '&sort=%s' % sort_string
else:
request = '/anime?' + filter_string
anime = []
resp = cache.hummingCache().cacheCheck(Backend().request,
24,
request,
page=page)
load = json.loads(resp)
data = load['data']
for a in data:
self.remaining_threads.append(a['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, a)
while len(self.remaining_threads) > 0:
time.sleep(1)
sortedItems = tools.sort_anime_by_json(self.extractedItems, data)
return sortedItems
def all_categories(self):
categories = []
resp = cache.hummingCache().cacheCheck(Backend().request,
24 * 7,
'/categories',
limit=217)
load = json.loads(resp)
data = load['data']
for a in data:
attributes = a['attributes']
categories.append({
'name': attributes['title'],
'description': attributes['description'],
'slug': attributes['slug'],
'child_count': attributes['childCount']
})
return categories
def episodes(self, id, page):
resp = cache.hummingCache().cacheCheck(Backend().request,
24,
'/anime/%s/episodes' % id,
page=page)
load = json.loads(resp)
data = load['data']
for a in data:
attributes = a['attributes']
episode = {}
try:
episode['episode_title'] = attributes['canonicalTitle']
except:
episode['episode_title'] = None
episode['alt_titles'] = {}
try:
episode['alt_titles']['romaji'] = attributes['titles']['en_jp']
except:
episode['alt_titles']['romaji'] = None
try:
episode['alt_titles']['kanji'] = attributes['titles']['ja_jp']
except:
episode['alt_titles']['kanji'] = None
try:
episode['alt_titles']['english'] = attributes['titles']['en']
except:
try:
episode['alt_titles']['english'] = attributes['titles'][
'en_us']
except:
episode['alt_titles']['english'] = None
try:
episode['seasonNumber'] = attributes['seasonNumber']
except:
episode['seasonNumber'] = None
try:
episode['episodeNumber'] = attributes['number']
except:
episode['episodeNumber'] = None
try:
episode['relativeNumber'] = attributes['relativeNumber']
except:
episode['relativeNumber'] = None
try:
episode['episodePlot'] = attributes['synopsis']
except:
episode['episodePlot'] = None
try:
episode['year'] = attributes['airdate'][:4]
except:
episode['year'] = None
try:
episode['airdate'] = attributes['airdate']
except:
episode['airdate'] = None
episode['episodeLength'] = attributes['length']
try:
episode['thumbnail'] = attributes['thumbnail']['original']
except:
episode['thumbnail'] = None
self.episodeList.append(episode)
self.remaining_threads.remove(page)
def all_episodes(self, id):
resp = cache.hummingCache().cacheCheck(Backend().request,
24,
'/anime/%s/episodes' % id,
page=1)
load = json.loads(resp)
count = load['meta']['count']
pages = int(math.ceil(float(int(count)) / 20))
for a in range(pages):
self.remaining_threads.append(a + 1)
self.thread_list.put(self.episodes, id, a + 1)
while len(self.remaining_threads) > 0:
time.sleep(1)
sortedEps = sorted(self.episodeList,
key=lambda x: int(x['episodeNumber']),
reverse=False)
return sortedEps
def getShow(self, id):
resp = cache.hummingCache().cacheCheck(Backend().request, 24,
'/anime/%s' % id)
load = json.loads(resp)
data = load['data']
self.remaining_threads.append(data['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, data)
while len(self.remaining_threads) > 0:
time.sleep(1)
show = self.extractedItems[0]
return show
def getListById(self, list):
items = []
for a in list:
resp = cache.hummingCache().cacheCheck(Backend().request, 24,
'/anime/%s' % a)
data = json.loads(resp)
data = data['data']
items.append(data)
extracted_items = self.extract_items(items)
return extracted_items
def getListbyLink(self, list):
installments = []
for a in list:
resp = cache.hummingCache().cacheCheck(Backend().request, 24, a)
load = json.loads(resp)
data = load['data']
installments.append(data)
extracted_items = self.extract_items(installments)
return extracted_items
def extract_items(self, list):
for a in list:
self.remaining_threads.append(a['attributes']['canonicalTitle'])
self.thread_list.put(self.extract, a)
while len(self.remaining_threads) > 0:
time.sleep(1)
return self.extractedItems
def extract(self, item):
dict = {}
attributes = item['attributes']
dict['id'] = item['id']
dict['titles'] = {}
try:
dict['titles']['canon'] = attributes['canonicalTitle']
except:
dict['titles']['canon'] = None
try:
dict['titles']['english'] = attributes['titles']['en']
except:
try:
dict['titles']['english'] = attributes['titles']['en_us']
except:
dict['titles']['english'] = None
try:
dict['titles']['romaji'] = attributes['titles']['en_jp']
except:
dict['titles']['romaji'] = None
try:
dict['titles']['kanji'] = attributes['titles']['ja_jp']
except:
dict['titles']['kanji'] = None
try:
dict['abbreviated_titles'] = attributes['abbreviatedTitles']
except:
dict['abbreviated_titles'] = None
year = ''
try:
year = attributes['startDate'][:4]
except:
year = attributes['tba']
dict['mappings'] = cache.hummingCache().cacheCheck(
mappings.Mappings().get, 24, dict['id'], attributes['subtype'],
year)
try:
dict['mal_titles'] = cache.hummingCache().cacheCheck(
Other().getMALTitles, 24, dict['mappings'])
except:
dict['mal_titles'] = {}
try:
dict['plot'] = attributes['synopsis']
except:
sict['plot'] = None
try:
dict['year'] = attributes['startDate'][:4]
except:
dict['year'] = None
try:
dict['start_date'] = attributes['startDate']
except:
dict['start_date'] = None
try:
dict['end_date'] = attributes['endDate']
except:
dict['end_date'] = None
try:
dict['popularity_rank'] = attributes['popularityRank']
except:
dict['popularity_rank'] = None
try:
dict['rating_rank'] = attributes['ratingRank']
except:
dict['rating_rank'] = None
try:
dict['average_rating'] = attributes['averageRating']
except:
dict['average_rating'] = None
try:
dict['age_rating'] = attributes['ageRating']
except:
dict['age_rating'] = None
try:
dict['age_guide'] = attributes['ageRatingGuide']
except:
dict['age_guide'] = None
try:
dict['subtype'] = attributes['subtype']
except:
dict['subtype'] = None
try:
dict['status'] = attributes['status']
except:
dict['status'] = None
try:
dict['episode_count'] = attributes['episodeCount']
except:
dict['episode_count'] = None
try:
dict['episode_length'] = attributes['episodeLength']
except:
dict['episode_length'] = None
try:
dict['youtube_trailer'] = attributes['youtubeVideoId']
except:
dict['youtube_trailer'] = None
try:
dict['nsfw'] = attributes['nsfw']
except:
dict['nsfw'] = None
try:
dict['genres'] = Other().get_genres(item['id'])
except:
dict['genres'] = []
try:
dict['franchise_name'] = tools.get_franchise_name(dict['titles'])
except:
dict['franchise_name'] = None
try:
dict['season'] = tools.get_season_number(dict['titles'])
except:
dict['season'] = 1
qualities = ['original', 'large', 'medium', 'small', 'tiny']
posterCheck = attributes['posterImage']
fanartCheck = attributes['coverImage']
poster = ''
fanart = ''
if posterCheck is not None:
for a in qualities:
if poster == '':
try:
poster = posterCheck[a]
except:
poster = ''
if fanartCheck is not None:
for a in qualities:
if fanart == '':
try:
fanart = fanartCheck[a]
except:
fanart = ''
dict['art'] = {}
if not tools.getSetting('key.fanart') == '' and tools.getSetting(
'art.fanart') == 'true':
from resources.lib.modules import fanarttv
if dict['subtype'] == 'movie':
try:
dict['art'] = cache.hummingCache().cacheCheck(
fanarttv.FanartTV().get, 24, dict['mappings'],
dict['subtype'], tools.getSetting('fanart.lang'))
except:
pass
else:
try:
dict['art'] = cache.hummingCache().cacheCheck(
fanarttv.FanartTV().get,
24,
dict['mappings'],
dict['subtype'],
tools.getSetting('fanart.lang'),
season=dict['season'])
except:
pass
dict['art']['poster'] = dict['art'].get('poster', poster)
dict['art']['fanart'] = dict['art'].get('fanart', fanart)
else:
try:
dict['art']['poster'] = poster
except:
dict['art']['poster'] = None
try:
dict['art']['fanart'] = fanart
except:
dict['art']['fanart'] = None
self.remaining_threads.remove(item['attributes']['canonicalTitle'])
self.extractedItems.append(dict)
def franchise(self, id):
link = '/anime/%s/installments' % id
resp = cache.hummingCache().cacheCheck(Backend().request, 24 * 7, link)
load = json.loads(resp)
installment_id = []
franchise_id = []
franchise_installments = []
data = load['data']
for a in data:
installment_id.append(a['id'])
for a in installment_id:
link = '/installments/%s/franchise' % a
resp = cache.hummingCache().cacheCheck(Backend().request, 24 * 7,
link)
load = json.loads(resp)
data = load['data']
franchise_id.append(data['id'])
for a in franchise_id:
link = '/franchises/%s/installments' % a
resp = cache.hummingCache().cacheCheck(Backend().request, 24 * 7,
link)
load = json.loads(resp)
data = load['data']
for b in data:
franchise_installments.append('/installments/%s/media' %
b['id'])
anime = self.getListbyLink(franchise_installments)
return anime
class NectarScraper:
def __init__(self):
# Base Variables
self.hosters = providers.get_hosters()
self.torrents = providers.get_torrents()
self.remaining_providers = []
self.sources = {}
self.data = {}
self.task_queue = ThreadPool()
# Scraping Strings
self.heading = tools.addonName + ': Scraping'
self.startingLineOne = 'Starting Scraping...'
self.hosterLineOne = 'Hosters Scraping...'
self.torrentLineOne = 'Torrents Scraping...'
self.episodeLineOne = 'Scraping %s - Episode %s'
self.movieLineOne = 'Scraping %s'
self.LineTwo = 'Remaining Providers: %s'
self.LineThree = 'Sources Found: %s'
# Pre-emptive Settings
self.preemp_minimum_sources = int(tools.getSetting('preemptive.limit'))
self.preemp_resolution = int(tools.getSetting('preemptive.resolution'))
self.preemptive_source_type = tools.getSetting('preemptive.sourcetype')
self.preemptive_audiotype = tools.getSetting('preemptive.audiotype')
def scrape(self, data):
tools.progressDialog.create(heading=self.heading,
line1=self.startingLineOne)
self.data = data
if 'episode' in self.data:
tools.progressDialog.update(
0,
line1=self.episodeLineOne %
(self.data['titles']['canon'], self.data['episode']))
else:
tools.progressDialog.update(0,
line1=self.movieLineOne %
self.data['titles']['canon'])
for provider in self.torrents:
if tools.getSetting(provider) == 'true':
self.remaining_providers.append(provider)
self.task_queue.put(self._scrape_torrent_provider, provider)
for provider in self.hosters:
if tools.getSetting(provider) == 'true':
self.remaining_providers.append(provider)
self.task_queue.put(self._scrape_hoster_provider, provider)
run_time = 0
max_time = int(tools.getSetting('scraping.timeout'))
start_time = time.time()
while len(self.remaining_providers
) > 0 and not tools.progressDialog.iscanceled(
) and run_time < max_time:
tools.progressDialog.update(
self._get_progress_percentage(start_time),
line2=self.LineTwo % self._get_provider_display_string(),
line3=self.LineThree % len(self.sources))
run_time += int(time.time() - start_time)
time.sleep(1)
# Check to see if we should break early if preemptive is on
if tools.getSetting('preemptive.enable') == 'true':
if self._check_preemptive():
break
tools.progressDialog.close()
if tools.getSetting('general.enablememes') == 'true':
self._run_memes()
self.sources = [value for key, value in self.sources.items()]
return self.sources
def _check_preemptive(self):
try:
selected_sources = [value for key, value in self.sources.items()]
if self.preemptive_source_type == 'torrent':
selected_sources += [
i for i in selected_sources
if i.get('source_type') == 'torrent'
]
elif self.preemptive_source_type == 'hoster':
selected_sources += [
i for i in selected_sources
if i.get('source_type') == 'hoster'
]
if self.preemptive_audiotype != 'Any':
selected_sources = [
i for i in selected_sources
if i.get('audio_type') == self.preemptive_audiotype
]
selected_sources = [
i for i in selected_sources
if i['quality'] >= self.preemp_resolution
]
if len(selected_sources) >= self.preemp_minimum_sources:
return True
else:
return False
except:
import traceback
traceback.print_exc()
return False
def _run_memes(self):
# Memes
memeLine1 = "Adding Wilson's Special Sauce ( ͡° ͜ʖ ͡°)"
memeElipsis = ''
memeProgress = 0
tools.progressDialog.create(heading=self.heading,
line1=memeLine1,
line2=memeLine1,
line3=memeLine1)
for a in range(20):
memeElipsis = memeElipsis + '.'
tools.progressDialog.update(memeProgress,
line1=memeLine1 + memeElipsis,
line2=memeLine1 + memeElipsis,
line3=memeLine1 + memeElipsis)
memeProgress += 5
if len(memeElipsis) == 3:
memeElipsis = ''
time.sleep(0.1)
tools.progressDialog.close()
def _get_provider_display_string(self):
return ', '.join([i.upper() for i in self.remaining_providers])
def _get_progress_percentage(self, start_time):
return int(float(time.time()) - int(start_time))
def _remove_provider(self, provider_name):
try:
self.remaining_providers.remove(provider_name)
except:
pass
def _scrape_torrent_provider(self, provider_name):
try:
i = __import__('providers.torrents.%s' % provider_name,
fromlist=['sub'],
globals=globals())
mod = i.source()
if 'episode' in self.data:
link = mod.tvshow(self.data)
else:
link = mod.movie(self.data)
source = mod.sources(link)
self._store_and_de_dup_sources(source, 'torrent')
except Exception as e:
tools.log('Error with %s:/n%s' % (provider_name, e), 'error')
finally:
self._remove_provider(provider_name)
def _scrape_hoster_provider(self, provider_name):
try:
i = __import__('providers.hosters.%s' % provider_name,
fromlist=['sub'],
globals=globals())
mod = i.source()
if 'episode' in self.data:
results = mod.tvshow(self.data)
else:
results = mod.movie(self.data)
source = mod.sources(results)
self._store_and_de_dup_sources(source, 'hoster')
except Exception as e:
tools.log('Error with %s:/n%s' % (provider_name, e), 'error')
finally:
self._remove_provider(provider_name)
def _store_and_de_dup_sources(self, sources, type):
if type == 'hoster':
for source in sources:
source['source_type'] = 'hoster'
self.sources[source['link']] = source
else:
for source in sources:
source['source_type'] = 'torrent'
self.sources[source['hash']] = source