def process_tv_link(db, config, episode, episode_links):
save_directory = config.tv_parent_directory + episode.show.show_directory
write_crawljob_file(str(episode), save_directory, ' '.join(episode_links),
config.crawljob_directory)
episode.is_downloaded = True
episode.status = 'Retrieved'
episode.retrieved_on = datetime.date.today()
episode.download_time = datetime.datetime.now()
db.commit()
ActionLog.log('"%s\'s" .crawljob file created.' % str(episode), db)
def search(self, show_search):
try:
c = TvdbInteraction.Contentor()
search_results = c.search_show(show_search)
#t = tvdb_api.Tvdb()
#search_results_old = t.search(show_search)
ActionLog.log('Search for "%s".' % show_search)
except Exception as ex:
search_results = "{error: %s}" % Exception
return jsonpickle.encode(search_results) # json.dumps(search_results)
def j_downloader_check(config):
try:
if len(config.jd_path) > 0:
start_command = 'open "%s"' % config.jd_path
# start_command = 'open "/Applications/JDownloader"'
kill_command = 'killall JavaApplicationStub'
os.system(kill_command)
time.sleep(10)
os.system(start_command)
except Exception as ex:
ActionLog.log(
'%s is not a valid directory for JDownloader in OSX. JDownloader has not restarted.' % config.jd_path)
def handle_movie(self):
ar = AjaxResponse('Movie downloading...')
try:
data = cherrypy.request.json
movie_id = data['movie_id']
action = data['movie_action']
config = cherrypy.request.db.query(Config).first()
if action == 'cleanup':
cherrypy.request.db.query(Movie).filter(Movie.status == 'Ignored').delete()
cherrypy.request.db.commit()
ActionLog.log("DB cleanup completed")
else:
m = cherrypy.request.db.query(Movie).filter(Movie.id == movie_id).first()
if action == 'ignore':
m.status = 'Ignored'
else:
LinkInteraction.write_crawljob_file(m.title, config.movies_directory, m.links,
config.crawljob_directory)
ActionLog.log('"%s\'s" .crawljob file created.' % m.name)
m.status = 'Retrieved'
cherrypy.request.db.commit()
except FileExistsError as no_file_ex:
ActionLog.log('error - ' + str(no_file_ex))
ar.status = 'error'
ar.message = 'Could not save to "%s" - Check your config.' % config.movies_directory
except Exception as ex:
ActionLog.log('error - ' + str(ex))
ar.status = 'error'
ar.message = str(ex)
return ar.to_JSON()
def add_show(show_id, db):
c = TvdbInteraction.Contentor()
series = c.get_show(show_id)
# episodes = series.episodes()
# artwork = c.get_show_images(show_id)
new_show = Show(show_id=series['id'],
show_name=series['name'],
first_aired=datetime.datetime.strptime(series['first_air_date'], '%Y-%m-%d'),
is_active=series['status'] != 'Ended',
overview=series['overview'],
banner="",
poster='static/tmdb-stacked.png' if series['poster_path'] is None else 'https://image.tmdb.org/t/p/w185' + series['poster_path'],
thumb="",
background= '' if series['backdrop_path'] is None else 'https://image.tmdb.org/t/p/original' + series['backdrop_path'],
large_image='')
if new_show.background == '':
new_show.background = new_show.poster
new_show.make_regex()
# create folder based on show name:
new_show.show_directory = '/' + new_show.show_name.replace('.', '').strip()
phys_directory = db.query(Config).first().tv_parent_directory + new_show.show_directory
if not os.path.exists(phys_directory):
os.makedirs(phys_directory)
# add show before adding all the episodes
db.add(new_show)
db.commit()
ActionLog.log('"%s" added.' % new_show.show_name)
#add episodes
for s in series['seasons']:
season_number = s['season_number']
if season_number == 0:
continue
else:
season_detail = c.tmdb.TV_Seasons(series['id'], season_number).info()
for e in season_detail['episodes']:
add_episode(e, new_show, db, s)
# for e in episodes:
# add_episode(e, new_show, db, c, artwork)
db.commit()
def update_episode(self):
status = 'success'
try:
data = cherrypy.request.json
episode_id = data['id']
e = cherrypy.request.db.query(Episode).filter(Episode.id == episode_id).first()
e.status = 'Pending' if e.status == 'Retrieved' else 'Retrieved'
if e.status == 'Pending':
e.reset()
cherrypy.request.db.commit()
status = e.status
except Exception as ex:
ActionLog.log(ex)
status = 'error'
return json.dumps(status)
def open_links(self, browser, config, source):
for episode in [s for s in self.shows_to_download if not s.is_downloaded]:
if not episode.is_found:
ActionLog.log("%s not found in soup" % str(episode), self.db)
continue
else:
tv_response = browser.get(episode.url_download_source)
if tv_response.status_code == 200:
episode_soup = tv_response.soup
episode_links = LinkInteraction.get_download_links(episode_soup, config, source.domain)
if LinkInteraction.is_valid_links(episode_links, browser, episode):
episode.is_found = True
episode.download_links = ' | '.join(episode_links)
LinkInteraction.process_tv_link(self.db, config, episode, episode_links)
else:
episode.is_found = False
def refresh(self):
status = 'success'
try:
data = cherrypy.request.json
action = data['action']
if action == 'scan':
LinkRetrieve.search_all(cherrypy.request.db)
if action == 'refresh':
MediaInteraction.update_all(cherrypy.request.db)
except Exception as ex:
ActionLog.log(str(ex))
status = 'error'
return json.dumps(status)
def process_search_result(self, links, episode, browser, source, config):
for l in links:
individual_page = browser.get(urljoin(source.domain, l.get('href')))
if individual_page.status_code == 200:
episode_soup = individual_page.soup
episode_links = LinkInteraction.get_download_links(episode_soup, config, source.domain)
ActionLog.log('%s found at %s. Processing links...' % ((episode), l.get('href')), self.db)
if LinkInteraction.is_valid_links(episode_links, browser, episode):
episode.is_found = True
episode.download_links = '\r'.join(episode_links)
episode.parent_download_page = source.url
LinkInteraction.process_tv_link(self.db, config, episode, episode_links)
break
else:
episode.is_found = False
ActionLog.log('%s had invalid or unavailable links at %s.' % ((episode), l.get('href')), self.db)
def add_show(self):
status = 'success'
try:
data = cherrypy.request.json
series_id = data['series_id']
# db = models.connect()
if cherrypy.request.db.query(Show).filter(Show.show_id == series_id).first() is None:
# save new show to db
MediaInteraction.add_show(series_id, cherrypy.request.db)
else:
status = 'duplicate'
# http://stackoverflow.com/questions/7753073/jquery-ajax-post-to-django-view
except Exception as ex:
# logger.exception(ex)
ActionLog.log(ex)
status = 'error'
return json.dumps(status)
def search_sites(browser, db=None):
# This is to search pages. They can either be long pages like cardman's or search results
# also this can be used to search a dynamic results page or
if db is None:
db = Models.connect()
config = db.query(Config).first()
search = Search.Search(db)
search.j_downloader_check(config)
for source in db.query(ScanURL).filter(
ScanURL.scan_type == 'static').order_by(ScanURL.priority).all():
tv_is_completed = search.is_completed()
browser_status = WebInteraction.source_login(source, browser)
# get browser and login to the source
if browser_status is False:
ActionLog.log('%s could not logon' % source.login_page)
continue
else:
ActionLog.log(
'Scanning %s for the STATIC page %s.' %
(source.domain, source.url), db)
# if you can login, start checking for content
try:
soup = browser.get(source.url).soup
except Exception as ex:
continue
# soup = soup.select(source.link_select)[:1000] # get dem links
soup = soup.select('a')[:2000] # get dem links
# let's try doing this with out the link selection, since we can parse them fast with
# the regex, we don't necessarily need it.
for link in soup:
check_result = search.check_link(link, source)
if check_result is False: # move onto new link if it doesn't match
continue
# open links and get download links for TV
search.open_links(browser, config, source)
LinkInteraction.scan_movie_links(db, browser, source, config)
def get_episode_list(self):
list_of_shows = []
for s in self.db.query(Show).all():
episodes = s.episodes.filter(Episode.air_date <= datetime.date.today() - datetime.timedelta(days=1)).filter(
Episode.status == 'Pending').all()
if len(episodes) > 0:
list_of_shows.extend(episodes)
if len(list_of_shows) > 0:
all_tv = ', '.join(str(s) for s in list_of_shows)
else:
all_tv = 'no shows'
self.db.commit()
ActionLog.log('Searching for: %s.' % all_tv, self.db)
self.shows_to_download = list_of_shows
def scan_movie_links(db, browser, source, config):
c = TvdbInteraction.Contentor()
for movie in db.query(Movie).all():
# only movies without a movie_link set
if movie.links is None or movie.links.strip() == '':
movie_link = urljoin(source.domain, movie.link_text)
movie_response = browser.get(movie_link)
if movie_response.status_code == 200:
movie_soup = movie_response.soup
movie_links = get_download_links(movie_soup, config,
source.domain, '1080p')
movie.links = ' '.join(movie_links)
if len(movie_links) == 0 or movie.name.strip() == '':
db.query(Movie).filter(Movie.id == movie.id).delete()
else:
search_details = movie.get_name_and_year()
tmdb_movie = c.get_movie_details(search_details[0],
search_details[1])
if tmdb_movie is None:
na_message = '"%s" not added. Found at %s' % (
movie.name, movie_link)
ActionLog.log(na_message, db)
continue
else:
movie.tmdb_id = tmdb_movie.movie['id']
movie.title = tmdb_movie.movie['title']
movie.tmdb_rating = tmdb_movie.movie['vote_average']
movie.poster = 'https://image.tmdb.org/t/p/w185' + tmdb_movie.movie[
'poster_path']
movie.overview = tmdb_movie.movie['overview']
movie.actors = ', '.join(tmdb_movie.cast)
movie.status = "Ready"
ActionLog.log(
'"%s" added to downloadable movies' % movie.name,
db)
db.commit()
def update_show(self):
status = 'success'
try:
data = cherrypy.request.json
show_id = data['id']
action = data['action']
if action == 'refresh':
MediaInteraction.update_one(show_id, cherrypy.request.db)
if action == 'remove':
s = cherrypy.request.db.query(Show).filter(Show.show_id == show_id).first()
ActionLog.log('"%s" removed.' % s.show_name)
cherrypy.request.db.delete(s)
cherrypy.request.db.commit()
except Exception as ex:
# logger.exception(ex)
status = 'error'
return json.dumps(status)
def search_forums(browser, db=None):
if db is None:
db = Models.connect()
config = db.query(Config).first()
search = Search.Search(db)
# search.j_downloader_check(config) prob dont' need this since it will be run after 'search_sites'
for source in db.query(ScanURL).filter(
ScanURL.scan_type == 'search').order_by(ScanURL.priority).all():
tv_is_completed = search.is_completed()
if tv_is_completed:
break
browser_status = WebInteraction.source_login(source, browser)
if browser_status is False:
ActionLog.log('%s could not logon' % source.login_page)
continue
else:
ActionLog.log(
'Searching via the search form on %s.' % source.domain, db)
# we invert the search format and check for each show, not each link in the page
for s in search.shows_to_download:
ActionLog.log('Searching for %s.' % str(s), db)
response_links = WebInteraction.source_search(
source, str(s), browser)
correct_links = [
l for l in response_links if s.episode_in_link(l.text.lower())
]
ActionLog.log(
'Found %s links for %s on %s' %
(str(len(correct_links)), str(s), source.domain), db)
search.process_search_result(correct_links, s, browser, source,
config)
time.sleep(
15) # wait five seconds between searches for warez-bb.org
def is_valid_links(episode_links, browser, episode):
links_valid = True
for file_share_link in episode_links:
try:
test_response = browser.get(file_share_link)
if test_response.status_code != 200 or 'File not found' in str(
test_response.soup):
ActionLog.log('Just kidding, "%s" had a bad link or links :(' %
episode)
links_valid = False
break
except Exception as ex:
ActionLog.log('Just kidding, "%s" had a bad link or links :(' %
episode)
ActionLog.log('Link check error: %s.' % str(ex))
links_valid = False
break
if len(episode_links) == 0:
links_valid = False
return links_valid