def SearchDbForShow(list_of_shows):
db = models.connect()
source = db.query(ScanURL).filter(ScanURL.media_type == "index").first()
if source:
browser = LinkRetrieve.source_login(source)
if browser is None:
ActionLog.log('%s could not logon' % source.login_page)
else:
for show_searcher in [l for l in list_of_shows if not l.retrieved]:
config = db.query(models.Config).first()
matching_indexes = []
for search_text in show_searcher.search_list: # all potential links to list
matching_indexes.extend(db.query(LinkIndex).filter(LinkIndex.link_text.like('%' + search_text + '%')).filter(LinkIndex.link_text.like('%' + show_searcher.episode_code + '%')).all())
if len(matching_indexes) > 0:
for match in matching_indexes:
tv_response = browser.get(match.link_url)
if tv_response.status_code == 200:
episode_soup = tv_response.soup
episode_links = LinkRetrieve.get_download_links(episode_soup, config, source.domain, config.hd_format)
# check to make sure links are active
for l in episode_links:
link_response = browser.get(l)
if link_response.status_code == 404:
episode_links = None
break
if episode_links:
LinkRetrieve.process_tv_link(db, config, show_searcher, episode_links)
break # since we got the download, we can break out of the loop
def search(self, show_search):
try:
t = tvdb_api.Tvdb()
search_results = t.search(show_search)
ActionLog.log('Search for "%s".' % show_search)
except Exception as ex:
search_results = "{error: %s}" % Exception
return json.dumps(search_results)
def update_episode(self):
status = 'success'
try:
data = cherrypy.request.json
episode_id = data['episodeid']
change_to_value = data['changeto']
if (change_to_value == 'search'):
new_status = LinkRetrieve.show_search(episode_id, cherrypy.request.db)
status = 'success' if new_status == 'Retrieved' else 'failed'
else:
change_to_value = change_to_value.title()
e = cherrypy.request.db.query(Episode).filter(Episode.id == episode_id).first()
e.status = change_to_value
cherrypy.request.db.commit()
except Exception as ex:
ActionLog.log(ex)
status = 'error'
return json.dumps(status)
def refresh(self):
status = 'success'
try:
data = cherrypy.request.json
is_show_refresh = data['isshowrefresh']
is_scan = data['isscan']
if is_scan:
LinkRetrieve.handle_downloads()
if is_show_refresh:
Utils.update_all()
except Exception as ex:
ActionLog.log(ex)
status = 'error'
return json.dumps(status)
def handle_movie(self):
ar = AjaxResponse('Movie downloading...')
try:
data = cherrypy.request.json
movie_id = data['movieid']
is_ignore = data['isignore']
is_cleanup = data['iscleanup']
config = cherrypy.request.db.query(Config).first()
if is_cleanup:
cherrypy.request.db.query(Movie).filter(Movie.status == 'Ignored').delete()
cherrypy.request.db.commit()
ActionLog.log("DB cleanup completed")
else:
m = cherrypy.request.db.query(Movie).filter(Movie.id == movie_id).first()
if is_ignore:
m.status = 'Ignored'
else:
jdownloader_string = ''
for l in m.movieurls.all():
jdownloader_string += l.url + ' '
LinkRetrieve.write_crawljob_file(m.name, config.movies_directory, jdownloader_string,
config.crawljob_directory)
ActionLog.log('"%s\'s" .crawljob file created.' % m.name)
m.status = 'Retrieved'
cherrypy.request.db.commit()
except Exception as ex:
ActionLog.log('error - ' + ex)
ar.status = 'error'
ar.message = ex
return ar.to_JSON()
def update_show(self):
status = 'success'
try:
data = cherrypy.request.json
show_id = data['showid']
action = data['action']
if action == 'refresh':
Utils.add_episodes(show_id)
if action == 'remove':
s = cherrypy.request.db.query(Show).filter(Show.show_id == show_id).first()
ActionLog.log('"%s" removed.' % s.show_name)
cherrypy.request.db.delete(s)
cherrypy.request.db.commit()
except Exception as ex:
# logger.exception(ex)
status = 'error'
return json.dumps(status)
def add_show(self):
status = 'success'
try:
data = cherrypy.request.json
series_id = data['seriesid']
t = tvdb_api.Tvdb()
s = t[series_id]
# db = models.connect()
if cherrypy.request.db.query(Show).filter(Show.show_id == series_id).first() is None:
# save new show to db
first_aired_date = datetime.strptime(s['firstaired'], "%Y-%m-%d")
new_show = Show(show_id=series_id, show_name=s['seriesname'], first_aired=first_aired_date,
is_active=s.data['status'] == 'Continuing', banner=s['banner'])
if new_show.banner == None:
new_show.banner = ''
# create folder based on show name:
new_show.show_directory = '/' + new_show.show_name.replace('.', '').strip()
phys_directory = cherrypy.request.db.query(Config).first().tv_parent_directory + new_show.show_directory
if not os.path.exists(phys_directory):
os.makedirs(phys_directory)
cherrypy.request.db.add(new_show)
cherrypy.request.db.commit()
ActionLog.log('"%s" added.' % new_show.show_name)
Utils.add_episodes(series_id, t, cherrypy.request.db)
else:
status = 'duplicate'
# http://stackoverflow.com/questions/7753073/jquery-ajax-post-to-django-view
except Exception as ex:
# logger.exception(ex)
ActionLog.log(ex)
status = 'error'
return json.dumps(status)
def CreateIndexes(is_full_index = False):
db = models.connect()
# check if index is empty to make the default index
indexes = db.query(LinkIndex).all()
#if len(indexes) == 0:
#start new indexing thingy
source = db.query(ScanURL).filter(ScanURL.media_type == "index").first()
if (source):
browser = source_login(source)
if browser is None:
ActionLog.log('%s could not logon' % source.login_page)
else:
# this is the first page, we have to index by 1 for each subsequent page
i = 1
is_indexed = False
duplicates_encountered = 0
try:
while is_indexed == False:
link = source.url
if i > 1:
link = '%sindex%s.html' % (source.url, i)
if i%100 == 0:
browser = source_login(source) # reset the browser session every 100 pages
soup = browser.get(link).soup
all_rows_soup = soup.select(source.link_select) # get all rows $("#threadbits_forum_73 tr").soup
page_adds = 0
for row in all_rows_soup:
if 'STICKY' in row.text.upper(): # skip sticky links
continue
else:
row_links = row.find_all('a')
links_with_text = [t for t in row_links if t.text != '']
index_link = links_with_text[0]
l = LinkIndex()
l.link_text = index_link.text
l.link_url = index_link.attrs['href']
l.id = int(index_link.attrs['id'].split('_')[2])
if l.link_text == '':
continue
existing_link_index = db.query(LinkIndex).filter(LinkIndex.id == l.id).first()
if existing_link_index:
duplicates_encountered += 1
if duplicates_encountered > 35 and not is_full_index: #this should be 1 plus pages
is_indexed = True
break
else:
continue
else:
page_adds += 1
db.add(l)
if page_adds > 0:
db.commit()
if i == 100000:
is_indexed = True
else:
print(i)
i += 1
except Exception as ex:
print(ex)
