def should_update_media(type, plex_object):
"""
Whether given plex media object rating should be updated
:param type: the type of media
:param plex_object: the plex object containing rating and ratingKey
:return: True if should be updated, False if not
"""
if type is "movie":
db_movie = Movie.select().where(Movie.plex_id == plex_object.ratingKey)
if db_movie.exists():
if util.check_media_needs_update(db_movie, plex_object):
return True
else:
return True
elif type is "show":
# getting show and episodes from show
db_show = Show.select().where(Show.plex_id == plex_object.ratingKey)
db_episodes = Episode.select().where(Episode.parent_plex_id == plex_object.ratingKey)
if db_show.exists():
if util.check_media_needs_update(db_show, plex_object):
return True
# if show doesn't need update, maybe a single episode need one
for episode in db_episodes:
if util.check_media_needs_update(episode, plex_object, check_rating=False):
return True
else:
return True
elif type is "episode":
db_episode = Episode.select().where(Episode.plex_id == plex_object.ratingKey)
if db_episode.exists():
if util.check_media_needs_update(db_episode, plex_object):
return True
else:
return True
return False
def parse_tvdb_xml(xmlfile):
FIELDS_LIST = [
('blurb', 'Overview', 'str'),
('run_date', 'FirstAired', 'date'),
('season', 'Combined_season', 'int'),
('production_code', 'ProductionCode', 'str'),
('episode', 'EpisodeNumber', 'int'),
('season', 'Season', 'int')
]
soup = BeautifulSoup(xmlfile, "xml")
for episode in soup.findAll('Episode'):
season = int(episode.find('Combined_season').text)
if season > 0:
episode_dict = {}
for model_field, xml_field, data_type in FIELDS_LIST:
try:
if episode.find(xml_field).text:
episode_dict[model_field] = episode.find(xml_field).text
if data_type == 'int':
episode_dict[model_field] = int(episode_dict[model_field])
if data_type == 'date':
d = parse(episode_dict[model_field])
episode_dict[model_field] = datetime.date(d.year, d.month, d.day)
except AttributeError:
pass
if episode.find('filename').text:
episode_dict['tvdb_image'] = 'http://thetvdb.com/banners/_cache/%s' % episode.find('filename').text
episode_dict['code'] = 's%se%s' % (
str(episode_dict['season']).zfill(2),
str(episode_dict['episode']).zfill(2))
try:
Episode.get(code=episode_dict['code'])
Episode.update(**episode_dict).where(Episode.code == episode_dict['code']).execute()
except Episode.DoesNotExist:
pass
def _admin_episodes(episode_code):
from flask import request
if request.method == 'GET':
context = {}
context['episode'] = Episode.get(code=episode_code)
context['episodejokes'] = EpisodeJoke.select().join(Episode).where(Episode.code == episode_code)
context['jokes'] = Joke.select().order_by(Joke.primary_character)
context['seasons'] = _all_seasons()
try:
context['next'] = Episode.get(number=context['episode'].number + 1)
except Episode.DoesNotExist:
context['next'] = None
try:
context['prev'] = Episode.get(number=context['episode'].number - 1)
except Episode.DoesNotExist:
context['prev'] = None
return render_template('admin_episode_detail.html', **context)
if request.method == 'PUT':
e = Episode.get(code=episode_code)
e.blurb = request.form.get('blurb', None)
e.save()
return '%s' % e.id
def _episode_detail(episode_code):
context = make_context()
context['episode'] = Episode.get(Episode.code == episode_code)
context['jokes'] = {}
context['joke_count'] = 0
for joke in EpisodeJoke.select().where(
EpisodeJoke.episode == context['episode']):
group = joke.joke.primary_character
if group not in app_config.PRIMARY_CHARACTER_LIST:
group = 'Miscellaneous'
if group not in context['jokes']:
context['jokes'][group] = []
context['jokes'][group].append(joke)
context['joke_count'] += 1
context['seasons'] = _all_seasons()
context['group_order'] = [
g for g in app_config.PRIMARY_CHARACTER_LIST if g in context['jokes']
]
try:
context['next'] = Episode.get(number=context['episode'].number + 1)
except Episode.DoesNotExist:
context['next'] = None
try:
context['prev'] = Episode.get(number=context['episode'].number - 1)
except Episode.DoesNotExist:
context['prev'] = None
return render_template('episode_detail.html', **context)
def _admin_episodes(episode_code):
from flask import request
if request.method == 'GET':
context = {}
context['episode'] = Episode.get(code=episode_code)
context['episodejokes'] = EpisodeJoke.select().join(Episode).where(
Episode.code == episode_code)
context['jokes'] = Joke.select().order_by(Joke.primary_character)
context['seasons'] = _all_seasons()
try:
context['next'] = Episode.get(number=context['episode'].number + 1)
except Episode.DoesNotExist:
context['next'] = None
try:
context['prev'] = Episode.get(number=context['episode'].number - 1)
except Episode.DoesNotExist:
context['prev'] = None
return render_template('admin_episode_detail.html', **context)
if request.method == 'PUT':
e = Episode.get(code=episode_code)
e.blurb = request.form.get('blurb', None)
e.save()
return '%s' % e.id
def _parse_episode(self, episode, season, show):
episode_number_total = get_int( episode, "epnum")
if season.tag=="Special":
season_number = get_int(episode, "season")
episode_number = -1
else:
season_number = int(season.attrib["no"])
episode_number = get_int(episode, "seasonnum")
rating = get_float(episode, "rating")
airdate = get_date(episode, "airdate")
title = get_str(episode, "title")
epnum = get_int(episode, "epnum")
summary = get_str(episode, "summary") or ""
screenshot_link = get_str(episode, "screencap")
if episode_number!=-1:
episode_obj, created = Episode.get_or_create(
show=show, season_number=season_number, episode_number=episode_number)
else:
episode_obj, created = Episode.get_or_create(
show=show, season_number=season_number, special=True, title=title)
episode_obj.summary = summary
episode_obj.screenshot_link = screenshot_link
episode_obj.airdate = airdate
episode_obj.rating = rating
episode_obj.put()
def _episode_detail(episode_code):
context = make_context()
context['episode'] = Episode.get(Episode.code == episode_code)
context['jokes'] = {}
context['joke_count'] = 0
for joke in EpisodeJoke.select().where(EpisodeJoke.episode == context['episode']):
group = joke.joke.primary_character
if group not in app_config.PRIMARY_CHARACTER_LIST:
group = 'Miscellaneous'
if group not in context['jokes']:
context['jokes'][group] = []
context['jokes'][group].append(joke)
context['joke_count'] += 1
context['seasons'] = _all_seasons()
context['group_order'] = [g for g in app_config.PRIMARY_CHARACTER_LIST if g in context['jokes']]
try:
context['next'] = Episode.get(number=context['episode'].number + 1)
except Episode.DoesNotExist:
context['next'] = None
try:
context['prev'] = Episode.get(number=context['episode'].number - 1)
except Episode.DoesNotExist:
context['prev'] = None
return render_template('episode_detail.html', **context)
def create_database(destroy_existing=False):
""" Create db and tables if it doesn't exist """
if not os.path.exists(DB_NAME):
logger.info('Create database: {0}'.format(DB_NAME))
open(DB_NAME, 'a').close()
Show.create_table()
Episode.create_table()
Setting.create_table()
def create_database(destroy_existing=False):
""" Create db and tables if it doesn't exist """
if not os.path.exists(DB_NAME):
logger.info('Create database: {0}'.format(DB_NAME))
open(DB_NAME, 'a').close()
Show.create_table()
Episode.create_table()
Setting.create_table()
def setup_tables():
"""
Creates the tables for the Arrested Development database.
"""
db.connect()
Joke.create_table()
Episode.create_table()
EpisodeJoke.create_table()
JokeConnection.create_table()
def setup_tables():
"""
Creates the tables for the Arrested Development database.
"""
db.connect()
Joke.create_table()
Episode.create_table()
EpisodeJoke.create_table()
JokeConnection.create_table()
def importNewEpisodes(session, series, currentTime, tvdb):
indexURL = series.index_url
parser = None
if series.channel == ChannelEnum.KanalD:
parser = KanalDParser(indexURL)
# TODO: Add other channels?
episodes = parser.getEpisodes()
# Iterate episodes to see if we're missing any
for episodeNumber, (episodeURL, datePublished) in episodes.iteritems():
query = session.query(Episode).filter(Episode.season.series == series,
Episode.number == episodeNumber)
if query.count() == 0:
# Create the episode
episode = Episode(number=episodeNumber,
media_link=episodeURL,
date_aired=datePublished,
date_found=currentTime,
series_id=series.id)
# Find which season to put this shit in
assignEpisodeToSeason(series, episode)
# Also attempt to find TVDB release date
date_aired = retrieveTvdbReleaseDate(episode, tvdb)
if date_aired is not None:
episode.date_aired = date_aired
session.add(episode)
else:
# We want to check if the URL has changed
episode = query.first()
if episode.media_link != episodeURL:
# TODO: The media link has changed. Should we log this?
episode.media_link = episodeURL
# Also check if a tvdb release date has appeared
date_aired = retrieveTvdbReleaseDate(episode, tvdb)
if date_aired is not None:
episode.date_aired = date_aired
session.commit()
def _parse_episodejokes(sheet):
"""
Parses joke sheet.
Imports episodejokes.
Will not update.
"""
episodes = list(Episode.select())
for row in sheet:
for episode in episodes:
joke = Joke.get(Joke.code == row['code'])
if episode.code == 's02e03':
episode.title = '\xc2\xa1Amigos!'
if episode.code == 's01e13':
episode.title = 'Beef Consomm\xc3\xa9'
if row[episode.title] in ['1', 'f', 'b']:
ej_dict = {}
ej_dict['joke'] = joke
ej_dict['episode'] = episode
ej_dict['joke_type'] = row[episode.title]
ej_dict['code'] = '%sj%s' % (ej_dict['episode'].code, ej_dict['joke'].code)
try:
EpisodeJoke.get(EpisodeJoke.code == ej_dict['code'])
except EpisodeJoke.DoesNotExist:
EpisodeJoke(**ej_dict).save()
def get_recently_updated():
global episodes
request = urllib2.urlopen(API_BASE_URL +
"/recent/episodes?start=0&limit=48")
response = json.loads(request.read())
for load in response['result']:
ep = Episode(load)
episodes.append(ep)
for episode in episodes:
url = 'plugin://%s/?mode=episode_mirror&episode=%s&show=%s' % (
_plugId, episode.episode_number, episode.show_id)
li = list_item(title="[COLOR red]" + episode.title + "[/COLOR] Ep " +
episode.episode_number + " - [COLOR yellow]" +
episode.get_relative_time() + "[/COLOR]",
icon=MEDIUM_THUMB_URL_BASE + "/" + episode.show_id +
"_mth.jpg",
url=None)
li.setProperty("summary", episode.synopsis)
fanart = get_show_meta(episode.title)
if fanart is None:
li.setProperty('fanart_image',
ADDON.getAddonInfo('path') + '/fanart.jpg')
else:
if fanart['fanart']:
li.setProperty('fanart_image', fanart['fanart'])
if fanart['poster']:
li.setThumbnailImage(fanart['poster'])
add_directory_item(url=url, li=li, isFolder=True)
xbmcplugin.endOfDirectory(addon_handle)
def _parse_episodejokes(sheet):
"""
Parses joke sheet.
Imports episodejokes.
Will not update.
"""
episodes = list(Episode.select())
for row in sheet:
for episode in episodes:
joke = Joke.get(Joke.code == row['code'])
if episode.code == 's02e03':
episode.title = '\xc2\xa1Amigos!'
if episode.code == 's01e13':
episode.title = 'Beef Consomm\xc3\xa9'
if row[episode.title] in ['1', 'f', 'b']:
ej_dict = {}
ej_dict['joke'] = joke
ej_dict['episode'] = episode
ej_dict['joke_type'] = row[episode.title]
ej_dict['code'] = '%sj%s' % (ej_dict['episode'].code,
ej_dict['joke'].code)
try:
EpisodeJoke.get(EpisodeJoke.code == ej_dict['code'])
except EpisodeJoke.DoesNotExist:
EpisodeJoke(**ej_dict).save()
def process_search_torrent():
res = []
eps = Episode.select().join(Show).where(
Show.is_active == True, Episode.status == models.WANTED,
Episode.firstaired < datetime.datetime.now())
for e in eps:
c = search.get_search_client(settings.search_client)
torrents = c.search(e.show.seriesname, e.get_code())
# Create default filters for episode code
fns = [{'key': 'name', 'value': e.get_code(), 'operator': 'in'}]
# Add filters from show settings
fns.extend(json.loads(e.show.filters))
filters = [
build_filter(**f) for f in fns if f['key'] in c.valid_filters
]
for fl in filters:
torrents = filter(fl, torrents)
msg = {'msg': 'No hash found for {e}'.format(e=e), 'stat': 'fail'}
if len(torrents) > 0:
e.magnet_hash = torrents[0]['hash']
e.status = models.FOUND
e.save()
msg['msg'] = 'Saved hash for {e}'.format(e=e)
msg['stat'] = 'ok'
res.append(msg)
return res
def _admin_output():
output = {}
output['joke_main'] = ''
output['joke_details'] = ''
output['joke_connections'] = ''
for joke in Joke.select():
for episode in Episode.select().where(Episode.season == 4).order_by(
Episode.number):
try:
ej = EpisodeJoke.get(episode=episode, joke=joke)
output['joke_main'] += '%s\t' % ej.joke_type
output['joke_details'] += '\'%s\t' % ej.details
if ej.connections():
output['joke_connections'] += '\'%s\t' % ej.connections(
)[0]['text']
else:
output['joke_connections'] += '\t'
except EpisodeJoke.DoesNotExist:
output['joke_main'] += '\t'
output['joke_details'] += '\t'
output['joke_connections'] += '\t'
output['joke_main'] += '\n'
output['joke_details'] += '\n'
output['joke_connections'] += '\n'
return render_template('_output.html', **output)
def _new_episode(self, show, episode):
if type(episode.get('title') is unicode):
episode['title'] = episode['title'].encode('utf-8')
is_audio = True
media_link = next((link for link in episode.links if link.get('type', '').startswith('audio/')), {})
if not media_link:
is_audio = False
media_link = next((link for link in episode.links if link.get('type', '').startswith('video/')), {})
description = episode.get('subtitle') or episode.get('summary')
media_encoding = media_link['type'].split('/')[1] if media_link.get('type', '').split('/')[1:2] else None
media_url = self._sanitize_url(media_link.get('href'))
if not media_url:
print('Warning! Episode has no link to media file.')
return None
episode_dict = {
'show': show,
'title': self._sanitize_title(episode.get('title')),
'description': self._sanitize_text(description),
'media_encoding': self._sanitize_word(media_encoding),
'media_length': self._sanitize_int(media_link.get('itunes_duration')),
'is_audio': is_audio,
'image_url': self._sanitize_url(episode.get('image', {}).get('href')),
'episode_url': self._sanitize_url(episode.get('link')),
'explicit': self._sanitize_bool(episode.get('itunes_explicit'))
}
episode_dict = {key: value for key, value in episode_dict.iteritems() if value is not None}
episode, is_new = Episode.get_or_create(media_url=media_url, defaults=episode_dict)
print('Saved episode "{}": new={}'.format(episode_dict['title'], is_new))
return show
def _admin_episodejokes(episode_code):
from flask import request
details = request.form.get('details', None)
if request.method == 'POST':
episode_joke_id = request.form.get('episode_joke_id', None)
ej = EpisodeJoke.get(id=int(episode_joke_id))
ej.details = details
ej.save()
return '%s' % ej.id
if request.method == 'PUT':
joke_code = request.form.get('joke_code', None)
joke_type = request.form.get('type', None)
joke = Joke.get(code=int(joke_code))
episode = Episode.get(code=episode_code)
code = 's%se%sj%s' % (
str(episode.season).zfill(2),
str(episode.episode).zfill(2),
joke.code
)
context = {}
context['ej'] = EpisodeJoke(joke=joke, episode=episode, joke_type=joke_type, details=details, code=code)
context['ej'].save()
return render_template('_episodejoke_form_row.html', **context)
def _parse_episodejoke_details(sheet, sheet_num):
"""
Parses the details, origin and connection sheets.
Adds data to existing episodejokes.
"""
FIELDS = [None, None, None, 'details', None, 'connection']
field = FIELDS[int(sheet_num)]
episodes = list(Episode.select())
for row in sheet:
for episode in episodes:
joke = Joke.get(Joke.code == row['code'])
if episode.code == 's02e03':
episode.title = '\xc2\xa1Amigos!'
if episode.code == 's01e13':
episode.title = 'Beef Consomm\xc3\xa9'
if row[episode.title]:
if field:
ej_code = '%sj%s' % (episode.code, joke.code)
payload = {}
payload[field] = row[episode.title]
EpisodeJoke.update(**payload).where(
EpisodeJoke.code == ej_code).execute()
def _admin_episodejokes(episode_code):
from flask import request
details = request.form.get('details', None)
if request.method == 'POST':
episode_joke_id = request.form.get('episode_joke_id', None)
ej = EpisodeJoke.get(id=int(episode_joke_id))
ej.details = details
ej.save()
return '%s' % ej.id
if request.method == 'PUT':
joke_code = request.form.get('joke_code', None)
joke_type = request.form.get('type', None)
joke = Joke.get(code=int(joke_code))
episode = Episode.get(code=episode_code)
code = 's%se%sj%s' % (str(episode.season).zfill(2), str(
episode.episode).zfill(2), joke.code)
context = {}
context['ej'] = EpisodeJoke(joke=joke,
episode=episode,
joke_type=joke_type,
details=details,
code=code)
context['ej'].save()
return render_template('_episodejoke_form_row.html', **context)
def UpdateSeriesStatus(cursor = None, num_updated = 0):
query = Episode.all().filter('status =', Episode.status.NEW)
if cursor:
query.with_cursor(cursor)
seriesMap = models.makeSeriesMap()
episodes = []
resultCount = 0
for episode in query.fetch(limit=BATCH_SIZE):
resultCount = resultCount + 1
if episode.seriesName in seriesMap:
episodes.append(episode)
if seriesMap[episode.seriesName]:
episode.status = 'ignore'
else:
episode.status = 'download'
if resultCount > 0:
db.put(episodes)
num_updated += len(episodes)
logging.debug(
'Put %d Episode entities to Datastore for a total of %d',
len(episodes), num_updated)
deferred.defer(
UpdateSeriesStatus, cursor = query.cursor(), num_updated = num_updated)
else:
logging.debug(
'UpdateSeriesStatus complete with %d updates!', num_updated)
def process_search_torrent():
res = []
eps = Episode.select().join(Show).where(
Show.is_active == True,
Episode.status == models.WANTED,
Episode.firstaired < datetime.datetime.now()
)
for e in eps:
c = search.get_search_client(settings.search_client)
torrents = c.search(e.show.seriesname, e.get_code())
# Create default filters for episode code
fns = [{'key': 'name', 'value': e.get_code(), 'operator': 'in'}]
# Add filters from show settings
fns.extend(json.loads(e.show.filters))
filters = [build_filter(**f) for f in fns
if f['key'] in c.valid_filters]
for fl in filters:
torrents = filter(fl, torrents)
msg = {'msg': 'No hash found for {e}'.format(e=e), 'stat': 'fail'}
if len(torrents) > 0:
e.magnet_hash = torrents[0]['hash']
e.status = models.FOUND
e.save()
msg['msg'] = 'Saved hash for {e}'.format(e=e)
msg['stat'] = 'ok'
res.append(msg)
return res
def POST(self):
f = submit_form()
if not f.validates():
return env.get_template('submit.html').render(form=f.render())
else:
episode = Episode(
title=f.d.title,
number=int(f.d.number),
href=db.Link(f.d.href),
thumb=db.Link(f.d.thumb if f.d.thumb else BLANK),
writeup=markdown(f.d.writeup),
)
episode.put()
mdel(key='4_posts', namespace='posts')
mdel(key='10_posts', namespace='posts')
raise web.seeother(web.ctx.homedomain)
def get_posts(limit=4):
post_data = mget(
key='_'.join([str(limit), 'posts']),
namespace='posts'
)
if post_data is not None:
return post_data
else:
query = Episode.all()
query.order('-number')
results = query.fetch(limit)
post_data = [{
'key': str(r.key()),
'title': r.title,
'number': r.number,
'href': r.href,
'thumb': r.thumb,
'writeup': r.writeup,
} for r in results]
web.debug(post_data)
if not madd(
key='_'.join([str(limit), 'posts']),
value=post_data,
namespace="posts"
):
logging.error('Memcache add failed!')
return post_data
def get_feed_info(podcast_id):
logging.toaster()
podcast = Podcast.query.filter_by(id=podcast_id).first()
feed = feedparser.parse(podcast.url)
podcast.set_data(feed['feed'])
db.session.add(podcast)
db.session.commit()
# add episodes too while we have the feed pulled:
# only add new episodes. maybe per-show uniqueness eventually?
episode_ids = [e.episode_id for e in podcast.episodes]
for episode_data in feed['entries']:
if episode_data['id'] in episode_ids:
# update existing entry
pass
else:
episode = Episode()
episode.title = episode_data['title']
episode.episode_id = episode_data['id']
episode.set_data(episode_data)
episode.podcast_id = podcast.id
db.session.add(episode)
cache_episodes.delay(podcast_id)
def createEp(data):
ep = Episode()
ep.name = data.get('name')
ep.date = convStrToDate(data.get('date'))
ep.link = data.get('link')
ep.thumbnail_url = data.get('thumbnail_url')
ep.save()
def _parse_episodes(sheet):
"""
Parses episode sheet.
Imports episodes.
Will not update.
"""
episodes = []
seasons = []
ratings = []
names = []
indexes = []
zip_list = [seasons, episodes, ratings, None, names]
counter = 0
for row in sheet:
if counter != 3:
zip_list[counter] += row.values()
counter += 1
indexes = row.keys()
output = []
for episode in zip(episodes, seasons, ratings, names, indexes):
if episode[0] == 'EPISODE':
pass
else:
episode_dict = {}
episode_dict['episode'] = int(episode[0])
episode_dict['season'] = int(episode[1])
episode_dict['title'] = episode[3].decode('utf-8')
episode_dict['rating'] = episode[2]
episode_dict['code'] = 's%se%s' % (episode[1].zfill(2),
episode[0].zfill(2))
episode_dict['number'] = int(episode[4])
output.append(episode_dict)
for row in output:
try:
Episode.get(Episode.code == row['code'])
except Episode.DoesNotExist:
Episode.create(**row).save()
def _parse_episodes(sheet):
"""
Parses episode sheet.
Imports episodes.
Will not update.
"""
episodes = []
seasons = []
ratings = []
names = []
indexes = []
zip_list = [seasons, episodes, ratings, None, names]
counter = 0
for row in sheet:
if counter != 3:
zip_list[counter] += row.values()
counter += 1
indexes = row.keys()
output = []
for episode in zip(episodes, seasons, ratings, names, indexes):
if episode[0] == 'EPISODE':
pass
else:
episode_dict = {}
episode_dict['episode'] = int(episode[0])
episode_dict['season'] = int(episode[1])
episode_dict['title'] = episode[3].decode('utf-8')
episode_dict['rating'] = episode[2]
episode_dict['code'] = 's%se%s' % (episode[1].zfill(2), episode[0].zfill(2))
episode_dict['number'] = int(episode[4])
output.append(episode_dict)
for row in output:
try:
Episode.get(Episode.code == row['code'])
except Episode.DoesNotExist:
Episode.create(**row).save()
def run_episode(self, verbose=False, mode='explore', update=True):
if mode == 'replay':
calculate_reward = False
else:
calculate_reward = True
done = False
# total_reward, reward = 0, 0
q_target_sequence = []
# state_sequence = []
self.env.reset()
step = 0
if self.network_type == 'LSTM':
self.model.reset()
while not done:
action = self.choose_action(mode, step)
if self.network_type == 'LSTM':
self.model.update_network_input(self.env.s, action, step)
# state_sequence.append(self.env.s)
next_state, reward, done, _ = self.env.step(
action, calculate_reward)
# if done and mode == 'replay':
# reward = self.best_encountered_reward
# total_reward += reward
step += 1
if not done:
q_target_sequence.append(
self.model.get_best_action(next_state,
step=step,
use_target=True)[1])
if calculate_reward:
reward_sequence = self.env.reward_sequence
else:
reward_sequence = self.best_encountered_rewards
if verbose:
print(f'\n----------Total Reward: {reward_sequence[-1]:.2f}')
# state_sequence = self.env.get_states_from_action_sequence()
episode = Episode(self.env.action_sequence, self.env.state_sequence,
reward_sequence, np.array(q_target_sequence))
n_pushes = 1
if mode == 'replay':
n_pushes = self.n_replays
self.memory.push(episode, n_pushes)
if update:
self.model.update(
memory=self.memory,
sampling_size=self.sampling_size,
# batch_size=self.batch_size,
epochs=self.n_epochs)
return reward_sequence
def api(**kwargs):
args = []
action = request.args.get('action', '')
def scan_show(show):
return show.check_download_status()
def delete_show(show):
show.delete_instance(recursive=True)
def update_show(show):
return utils.save_show(tvdb_id=show.tvdb_id)
def episode_mark_skipped(episode):
episode.status = models.SKIPPED
episode.save()
def episode_mark_wanted(episode):
episode.status = models.WANTED
episode.save()
def episode_mark_downloaded(episode, location):
episode.status = models.DOWNLOADED
episode.location = location
episode.save()
actions = {
'scan': scan_show,
'delete': delete_show,
'update': update_show,
'episode_mark_wanted': episode_mark_wanted,
'episode_mark_skipped': episode_mark_skipped,
'episode_mark_downloaded': episode_mark_downloaded,
'background_search': utils.process_search_torrent,
'background_download': utils.process_download_torrent,
'background_update': utils.process_check_new_episodes,
'background_status': utils.process_check_downloaded
}
if not action in actions:
msg = 'No action named {}'.format(action)
return jsonify(stat='fail', msg=msg, result=[])
if 'show_id' in request.args:
args.append(Show.get(id=request.args['show_id']))
if 'episode_id' in request.args:
args.append(Episode.get(id=request.args['episode_id']))
if 'location' in request.args:
args.append(request.args['location'])
res = actions[action](*args)
return jsonify(stat='ok', result=res)
def api(**kwargs):
args = []
action = request.args.get('action', '')
def scan_show(show):
return show.check_download_status()
def delete_show(show):
show.delete_instance(recursive=True)
def update_show(show):
return utils.save_show(tvdb_id=show.tvdb_id)
def episode_mark_skipped(episode):
episode.status = models.SKIPPED
episode.save()
def episode_mark_wanted(episode):
episode.status = models.WANTED
episode.save()
def episode_mark_downloaded(episode, location):
episode.status = models.DOWNLOADED
episode.location = location
episode.save()
actions = {
'scan': scan_show,
'delete': delete_show,
'update': update_show,
'episode_mark_wanted': episode_mark_wanted,
'episode_mark_skipped': episode_mark_skipped,
'episode_mark_downloaded': episode_mark_downloaded,
'background_search': utils.process_search_torrent,
'background_download': utils.process_download_torrent,
'background_update': utils.process_check_new_episodes,
'background_status': utils.process_check_downloaded
}
if not action in actions:
msg = 'No action named {}'.format(action)
return jsonify(stat='fail', msg=msg, result=[])
if 'show_id' in request.args:
args.append(Show.get(id=request.args['show_id']))
if 'episode_id' in request.args:
args.append(Episode.get(id=request.args['episode_id']))
if 'location' in request.args:
args.append(request.args['location'])
res = actions[action](*args)
return jsonify(stat='ok', result=res)
def parse_tvdb_xml(xmlfile):
FIELDS_LIST = [('blurb', 'Overview', 'str'),
('run_date', 'FirstAired', 'date'),
('season', 'Combined_season', 'int'),
('production_code', 'ProductionCode', 'str'),
('episode', 'EpisodeNumber', 'int'),
('season', 'Season', 'int')]
soup = BeautifulSoup(xmlfile, "xml")
for episode in soup.findAll('Episode'):
season = int(episode.find('Combined_season').text)
if season > 0:
episode_dict = {}
for model_field, xml_field, data_type in FIELDS_LIST:
try:
if episode.find(xml_field).text:
episode_dict[model_field] = episode.find(
xml_field).text
if data_type == 'int':
episode_dict[model_field] = int(
episode_dict[model_field])
if data_type == 'date':
d = parse(episode_dict[model_field])
episode_dict[model_field] = datetime.date(
d.year, d.month, d.day)
except AttributeError:
pass
if episode.find('filename').text:
episode_dict[
'tvdb_image'] = 'http://thetvdb.com/banners/_cache/%s' % episode.find(
'filename').text
episode_dict['code'] = 's%se%s' % (str(
episode_dict['season']).zfill(2), str(
episode_dict['episode']).zfill(2))
try:
Episode.get(code=episode_dict['code'])
Episode.update(**episode_dict).where(
Episode.code == episode_dict['code']).execute()
except Episode.DoesNotExist:
pass
def build_regression_csv():
"""
Builds an episode-based CSV for @stiles to do our regression.
Outputs a list of episodes with joke counts and ratings.
"""
with open('data/regression.csv', 'wb') as csvfile:
regressionwriter = csv.DictWriter(csvfile, ['episode_name', 'episode_id', 'jokes', 'rating'])
regressionwriter.writerow({'episode_name': 'episode_name', 'episode_id': 'episode_id', 'jokes': 'jokes', 'rating': 'rating'})
for episode in Episode.select():
episode_dict = {}
episode_dict['episode_name'] = episode.title.encode('utf-8')
episode_dict['episode_id'] = episode.number
episode_dict['rating'] = episode.rating
episode_dict['jokes'] = EpisodeJoke.select().where(EpisodeJoke.episode == episode).count()
regressionwriter.writerow(episode_dict)
def _series_to_list(json_data, serie_id):
response = json.loads(_strip_bom(json_data))
last_season = None
seasons = []
current_episodes = []
for item in sorted(response['data'], key=lambda x: natural_sort_key(x['season'])):
episode = Episode(item['id'], item['season'], item['episode'], item['url'])
if last_season and episode.season != last_season:
seasons.append(_make_season(serie_id, current_episodes))
current_episodes = []
current_episodes.append(episode)
last_season = episode.season
seasons.append(_make_season(serie_id, current_episodes))
return seasons
def _new_episode(self, show, episode):
if type(episode.get('title') is unicode):
episode['title'] = episode['title'].encode('utf-8')
is_audio = True
media_link = next((link for link in episode.links
if link.get('type', '').startswith('audio/')), {})
if not media_link:
is_audio = False
media_link = next((link for link in episode.links
if link.get('type', '').startswith('video/')),
{})
description = episode.get('subtitle') or episode.get('summary')
media_encoding = media_link['type'].split('/')[1] if media_link.get(
'type', '').split('/')[1:2] else None
media_url = self._sanitize_url(media_link.get('href'))
if not media_url:
print('Warning! Episode has no link to media file.')
return None
episode_dict = {
'show': show,
'title': self._sanitize_title(episode.get('title')),
'description': self._sanitize_text(description),
'media_encoding': self._sanitize_word(media_encoding),
'media_length':
self._sanitize_int(media_link.get('itunes_duration')),
'is_audio': is_audio,
'image_url':
self._sanitize_url(episode.get('image', {}).get('href')),
'episode_url': self._sanitize_url(episode.get('link')),
'explicit': self._sanitize_bool(episode.get('itunes_explicit'))
}
episode_dict = {
key: value
for key, value in episode_dict.iteritems() if value is not None
}
episode, is_new = Episode.get_or_create(media_url=media_url,
defaults=episode_dict)
print('Saved episode "{}": new={}'.format(episode_dict['title'],
is_new))
return show
def _all_seasons():
output = []
SEASONS = [1, 2, 3]
if app_config.IMPORT_NEW_SEASON is True:
SEASONS.append(4)
for season in SEASONS:
season_dict = {}
season_dict['season'] = season
season_dict['episodes'] = []
for episode in Episode.select().where(Episode.season == season):
season_dict['episodes'].append({
'url': 'episode-%s.html' % episode.code,
'text': '%s: %s' % (episode.episode, episode.title),
'episode': episode.episode,
'code': episode.code
})
season_dict['episodes'] = sorted(season_dict['episodes'], key=lambda episode: episode['episode'])
output.append(season_dict)
return output
def update_episode_rating(database, episode, imdb_episodes, imdb_id, plex_object, season):
"""
Update the episode rating if it is outdated
:param database: connection to the Plex DB
:param episode: the episode object from Plex
:param imdb_episodes: the IMDB episodes object containing the rating
:param imdb_id: the IMDB id from the parent show
:param plex_object: the parent plex object
:param season: the season from which this episode is beloning to
:return: True if updated, False if not
"""
db_episode = Episode.select().where(Episode.plex_id == episode.ratingKey)
if not db_episode.exists():
if imdb_episodes is None:
imdb_episodes = imdb.get_season_from_imdb(imdb_id, season.index)
if update_imdb_episode_rating(database, episode,
imdb_episodes, plex_object, season):
logger.debug("Created episode '{e.title}' '{e.index}' "
"with new ratings".format(e=episode))
return True
else:
return False
else:
# check if we need to update this item
need_update = False
if db_episode.get().rating is not episode.rating:
need_update = True
elif db_episode.get().last_update > datetime.now() - timedelta(days=-7):
need_update = True
if need_update:
if imdb_episodes is None:
imdb_episodes = imdb.get_season_from_imdb(imdb_id, season.index)
if update_imdb_episode_rating(database, episode,
imdb_episodes, plex_object,
season, db_episode):
logger.debug("Update episode '{e.title}' '{e.index}' "
"with new ratings".format(e=episode))
return True
else:
return False
return False
def process_download_torrent():
res = []
eps = Episode.select().join(Show).where(Show.is_active == True,
Episode.status == models.FOUND)
tc = clients.get_torrent_client(settings.client)
for e in eps:
try:
tc.add_magnet_hash(e.magnet_hash, e.get_download_dir())
e.status = models.DOWNLOADING
e.save()
res.append({
'msg': 'Started downloading {e}'.format(e=e),
'stat': 'ok'
})
except Exception, ex:
res.append({
'msg':
'Failed to add {e} to client: {ex}'.format(e=e, ex=ex),
'stat':
'fail'
})
def process_download_torrent():
res = []
eps = Episode.select().join(Show).where(
Show.is_active == True,
Episode.status == models.FOUND
)
tc = clients.get_torrent_client(settings.client)
for e in eps:
try:
tc.add_magnet_hash(e.magnet_hash, e.get_download_dir())
e.status = models.DOWNLOADING
e.save()
res.append({
'msg': 'Started downloading {e}'.format(e=e),
'stat': 'ok'
})
except Exception, ex:
res.append({
'msg': 'Failed to add {e} to client: {ex}'.format(e=e, ex=ex),
'stat': 'fail'
})
def putText():
book = request.json.get('book', None)
no = request.json.get('no', None)
data = request.json.get('data', None)
if not book or not no or not data:
return json.dumps({'error': u'无书名或无数据'})
try:
episode = Episode.query.filter_by(name=book.strip(),
no = int(no), user_id=g.user.get_id()).one_or_none()
except MultipleResultsFound:
return json.dumps({'error': u'重复数据异常'})
if episode is None:
new_word_user = Episode(g.user.get_id(), book.strip(), int(no), data)
db.session.add(new_word_user)
else:
episode.data = data
db.session.commit()
return json.dumps({})
def refresh():
for rss in rss_links:
parsed = podcastparser.parse(rss, urllib.request.urlopen(rss), 10)
title = parsed.get('title')
description = remove_html(parsed.get('description'))
image = parsed.get('cover_url')
link = parsed.get('link')
podcast = Podcast.query.filter_by(link=parsed.get('link')).first()
if (podcast is None):
podcast = Podcast(title=title,
description=description,
image=image,
link=link)
db.session.add(podcast)
db.session.commit()
for episode in parsed.get('episodes'):
episode_title = episode.get('title')
episode_link = episode.get('link')
episode_audio_url = episode.get('enclosures')[0]['url']
episode_time_published = episode.get('published')
episode_length = episode.get('total_time')
episode_podcast = podcast
episode = Episode.query.filter_by(
audio_url=episode_audio_url).first()
if (episode is None):
episode = Episode(title=episode_title,
link=episode_link,
audio_url=episode_audio_url,
time_published=episode_time_published,
length=episode_length,
podcast=podcast)
db.session.add(episode)
db.session.commit()
def build_regression_csv():
"""
Builds an episode-based CSV for @stiles to do our regression.
Outputs a list of episodes with joke counts and ratings.
"""
with open('data/regression.csv', 'wb') as csvfile:
regressionwriter = csv.DictWriter(
csvfile, ['episode_name', 'episode_id', 'jokes', 'rating'])
regressionwriter.writerow({
'episode_name': 'episode_name',
'episode_id': 'episode_id',
'jokes': 'jokes',
'rating': 'rating'
})
for episode in Episode.select():
episode_dict = {}
episode_dict['episode_name'] = episode.title.encode('utf-8')
episode_dict['episode_id'] = episode.number
episode_dict['rating'] = episode.rating
episode_dict['jokes'] = EpisodeJoke.select().where(
EpisodeJoke.episode == episode).count()
regressionwriter.writerow(episode_dict)
def AddStatusToEpisode(cursor = None, num_updated = 0):
query = Episode.all()
if cursor:
query.with_cursor(cursor)
episodes = []
for episode in query.fetch(limit=BATCH_SIZE):
episode.status = 'new'
episodes.append(episode)
if episodes:
# Default value for status is fine, so just write them out.
db.put(episodes)
num_updated += len(episodes)
logging.debug(
'Put %d Episode entities to Datastore for a total of %d',
len(episodes), num_updated)
deferred.defer(
AddStatusToEpisode, cursor = query.cursor(), num_updated = num_updated)
else:
logging.debug(
'AddStatusToEpisode complete with %d updates!', num_updated)
def _admin_output():
output = {}
output['joke_main'] = ''
output['joke_details'] = ''
output['joke_connections'] = ''
for joke in Joke.select():
for episode in Episode.select().where(Episode.season == 4).order_by(Episode.number):
try:
ej = EpisodeJoke.get(episode=episode, joke=joke)
output['joke_main'] += '%s\t' % ej.joke_type
output['joke_details'] += '\'%s\t' % ej.details
if ej.connections():
output['joke_connections'] += '\'%s\t' % ej.connections()[0]['text']
else:
output['joke_connections'] += '\t'
except EpisodeJoke.DoesNotExist:
output['joke_main'] += '\t'
output['joke_details'] += '\t'
output['joke_connections'] += '\t'
output['joke_main'] += '\n'
output['joke_details'] += '\n'
output['joke_connections'] += '\n'
return render_template('_output.html', **output)
def _parse_episodejoke_details(sheet, sheet_num):
"""
Parses the details, origin and connection sheets.
Adds data to existing episodejokes.
"""
FIELDS = [None, None, None, 'details', None, 'connection']
field = FIELDS[int(sheet_num)]
episodes = list(Episode.select())
for row in sheet:
for episode in episodes:
joke = Joke.get(Joke.code == row['code'])
if episode.code == 's02e03':
episode.title = '\xc2\xa1Amigos!'
if episode.code == 's01e13':
episode.title = 'Beef Consomm\xc3\xa9'
if row[episode.title]:
if field:
ej_code = '%sj%s' % (episode.code, joke.code)
payload = {}
payload[field] = row[episode.title]
EpisodeJoke.update(**payload).where(EpisodeJoke.code == ej_code).execute()
def _all_seasons():
output = []
SEASONS = [1, 2, 3]
if app_config.IMPORT_NEW_SEASON is True:
SEASONS.append(4)
for season in SEASONS:
season_dict = {}
season_dict['season'] = season
season_dict['episodes'] = []
for episode in Episode.select().where(Episode.season == season):
season_dict['episodes'].append({
'url':
'episode-%s.html' % episode.code,
'text':
'%s: %s' % (episode.episode, episode.title),
'episode':
episode.episode,
'code':
episode.code
})
season_dict['episodes'] = sorted(
season_dict['episodes'], key=lambda episode: episode['episode'])
output.append(season_dict)
return output
def add_eps(username,pod_id):
user = User.query.filter_by(username=username).first_or_404()
podcast = Podcast.query.filter_by(id=pod_id).first_or_404()
podcast.buildtime = datetime.utcnow()
form = AddEpisodeForm()
if form.validate_on_submit():
epaudio = form.epaudio.data
filename = secure_filename(epaudio.filename)
if os.path.exists(os.path.join(path, 'static/'+str(user.id)+'/'+str(podcast.id))):
epaudio.save(os.path.join(path, 'static/'+str(user.id)+'/'+str(podcast.id), filename))
else:
os.mkdir(os.path.join(path, 'static/'+str(user.id)+'/'+str(podcast.id)))
epaudio.save(os.path.join(path, 'static/'+str(user.id)+'/'+str(podcast.id), filename))
###############################################################
# need to also construct and insert link to rss feed
###############################################################
ep = Episode(epname=form.epname.data, epnum=form.epnum.data, epdesc=form.epdesc.data, eplink=form.eplink.data, pod_id=pod_id, audio_path=filename, it_sum=form.it_sum.data, it_subtitle=form.it_subtitle.data)
######################################################################
# Add RSS episode
rss = rss_lib.rss_add_ep(podcast,ep)
######################################################################
db.session.add(ep)
db.session.commit()
return redirect(url_for('podcast', pod_id=podcast.id, user_id=user.id))
return render_template('add_eps.html', title='Add Episode', form=form)
def update_imdb_episode_rating(database, episode, imdb_episodes, plex_object, season, db_episode=None):
"""
Update episode rating from IMDB
:param database: connection to the database
:param episode: the episode object from plex
:param imdb_episodes: the episode ratings from imdb
:param plex_object: the plex object from the parent show
:param season: the season of this episode
:param exists: whether the media already exists in local db
:return: true if update succeeds, false if not
"""
if episode.index in imdb_episodes:
if imdb_episodes[episode.index] == 'N/A':
if not DRY_RUN:
db.reset_rating(database, episode)
db.set_locked_fields(database, episode)
logger.debug("Episode '{e.title}' '{e.index}' has no rating available".format(
e=episode))
return False
else:
if not DRY_RUN:
db.set_rating_and_imdb_image(database, episode,
imdb_episodes[episode.index]["rating"])
db.set_locked_fields(database, episode)
# create episode in database
if db_episode is None:
Episode.create(
parent_plex_id=plex_object.ratingKey,
plex_id=episode.ratingKey,
imdb_id=imdb_episodes[episode.index]["imdb_id"],
title=episode.title,
season=season.index,
episode=episode.index,
release_date=episode.originallyAvailableAt,
rating=imdb_episodes[episode.index]["rating"]
)
else:
db.update_db_rating(db_episode.get(), imdb_episodes[episode.index]["rating"])
return True
else:
if not DRY_RUN:
db.reset_rating(database, episode)
db.set_locked_fields(database, episode)
logger.debug("Episode '{e.title}' '{e.index}' not found. Cannot update".format(
e=episode))
if db_episode is None:
Episode.create(
parent_plex_id=plex_object.ratingKey,
plex_id=episode.ratingKey,
title=episode.title,
season=season.index,
episode=episode.index,
release_date=episode.originallyAvailableAt
)
else:
if episode.index in imdb_episodes:
db.update_db_rating(db_episode.get(), imdb_episodes[episode.index]["rating"])
else:
logger.debug("Episode '{e.title}' '{e.index}' not found. Cannot update".format(
e=episode))
return False
return False
def render_pages():
render()
local('rm -rf www/episode*.html')
local('rm -rf www/joke*.html')
_render_iterable(Joke.select(), 'joke', 'code')
_render_iterable(Episode.select(), 'episode', 'code')
def render_pages():
render()
local('rm -rf www/episode*.html')
local('rm -rf www/joke*.html')
_render_iterable(Joke.select(), 'joke', 'code')
_render_iterable(Episode.select(), 'episode', 'code')
def write_jokes_json():
"""
Writes the JSON for @onyxfish and @alykat's viz.
Produces a list of jokes and within that a list of episodejokes
where the joke appears, sorted by episode index number.
"""
payload = {
'group_order': app_config.PRIMARY_CHARACTER_LIST,
'jokes': {},
'connections': [],
'episodes': {}
}
for joke in Joke.select().order_by(Joke.primary_character):
if joke.primary_character not in payload['group_order']:
joke.primary_character = 'Miscellaneous'
if joke.primary_character not in payload['jokes']:
payload['jokes'][joke.primary_character] = []
joke_dict = joke.__dict__['_data']
del joke_dict['id']
del joke_dict['blurb']
joke_dict['episodejokes'] = []
for ej in EpisodeJoke.select().join(Joke).where(
Joke.code == joke.code):
episode_dict = ej.__dict__['_data']
episode_dict['episode_number'] = ej.episode.number
for k in ['episode', 'joke', 'id', 'origin']:
del episode_dict[k]
joke_dict['episodejokes'].append(episode_dict)
joke_dict['episodejokes'] = sorted(joke_dict['episodejokes'],
key=lambda ej: ej['episode_number'])
payload['jokes'][joke.primary_character].append(joke_dict)
if app_config.IMPORT_NEW_SEASON is True:
episodes = Episode().select().order_by(Episode.number)
else:
episodes = Episode().select().where(Episode.season != 4).order_by(
Episode.number)
for episode in episodes:
episode_dict = episode.__dict__['_data']
episode_dict['run_date'] = episode_dict['run_date'].strftime(
'%Y-%m-%d')
for k in ['blurb', 'id', 'production_code', 'rating', 'tvdb_image']:
del episode_dict[k]
payload['episodes'][episode.number] = episode_dict
for primary_character, jokes in payload['jokes'].items():
payload['jokes'][primary_character] = sorted(
jokes, key=lambda j: j['episodejokes'][0]['episode_number'])
for connection in JokeConnection.select():
payload['connections'].append({
'joke1_code':
connection.joke1.code,
'joke2_code':
connection.joke2.code,
'episode_number':
connection.episode.number
})
payload['connections'] = sorted(payload['connections'],
key=lambda c: c['joke1_code'])
with open('www/live-data/jokes.json', 'wb') as jokefile:
jokefile.write(json.dumps(payload))
def get_show(show_id):
c = {
'show': Show.get(id=show_id),
'episodes': Episode.select().join(Show).where(Show.id == show_id)
}
return render_template('show.html', **c)
def get_show(show_id):
c = {
'show': Show.get(id=show_id),
'episodes': Episode.select().join(Show).where(Show.id == show_id)
}
return render_template('show.html', **c)
