class Command(BaseCommand):
help = 'Run cron tasks for the given category'
option_list = BaseCommand.option_list + (
make_option('-n', '--no-repeat', action="store_false", dest='repeat', default=True,
help='Do not repeat every few seconds for one minute, execute just once'),
) + (
make_option('-f', '--forever', action="store_true", dest='forever', default=False,
help='Keep iterating infinitely (and keep objects running, like the bittorrent client)'),
)
def __init__(self):
super(self.__class__,self).__init__()
self.dl_manager = DownloadManager()
self.start = time.time()
def handle(self, *args, **options):
if len(args) != 1 or args[0] not in self.dl_manager.get_actions_list():
raise CommandError('You must specify one valid command (%s)' % repr(self.dl_manager.get_actions_list()))
command = args[0]
repeat = options.get('repeat')
forever = options.get('forever')
# Only allow one cron process per command to run at a single time
lock_path = os.path.join(settings.LOCK_PATH, '.%s.pid' % command)
try:
l = lock.lock(lock_path, timeout=MAX_RUN_TIME) # wait at most 50s
self.do(command, repeat, forever)
except error.LockHeld:
log.debug("Active process for command '%s', aborting.", command)
else:
l.release()
def do(self, command, repeat, forever):
'''Run the specified action
- once or repeat=True for a time-limited loop
- for MAX_RUN_TIME or forever=True for forever'''
log.info("Running download manager for command '%s' (repeat=%d, forever=%d)", command, repeat, forever)
if not repeat:
self.dl_manager.do(command)
else:
# Runs the specified action, every DELAY seconds for X minutes
stop_time = self.start + MAX_RUN_TIME - DELAY
while forever or time.time() <= stop_time:
self.dl_manager.do(command)
time.sleep(DELAY)
def __init__(self):
super(self.__class__,self).__init__()
self.dl_manager = DownloadManager()
self.start = time.time()
def test_series_processing(self, mock_video_transcoder):
'''Fake download of a list of test series to process and get success rates on'''
import wall.plugins, wall.thetvdbapi, wall.helpers
# Don't pollute the real download dir
settings.DOWNLOAD_DIR = settings.TEST_DOWNLOAD_DIR
default_open_url = None
default_update_queued_torrents = None
default_get_url = None
default_get_torrent_info = None
default_add_magnet = None
# FIXME: Tests should cover all plugins
self.select_plugin(TorrentSearcher, 'torrentz-searcher')
# Don't let the other tests continue without reverting to the default methods/objects
# we override in this test
try:
#### Retreive TVDB data for test strings ####
# Cache it to allow to run this test multiple consecutive times quickly, without
# having to query the db engine every time
def open_url(url):
url_id = url.replace('/', '_')
cache_path = os.path.join(settings.CACHE_DIR, url_id)
# TVDB's parsing really wants a fp to an actual file
# Instead of using get/set_cache, prepare the file directly
mkdir_p(settings.CACHE_DIR)
if not os.path.isfile(cache_path):
cache_fp = default_open_url(url)
with open(cache_path, 'w') as f:
f.write(cache_fp.read())
return open(cache_path)
default_open_url = wall.helpers.open_url
wall.helpers.open_url = open_url
for series_name in settings.TEST_SERIES_LIST:
# Retreive series info
Series.objects.add_by_search(series_name)
# Make series active (scheduled for download)
series = Series.objects.get(name=series_name)
series.update_from_tvdb()
#### Torrent search ####
from wall.downloadmanager import DownloadManager
def get_url(url):
url_id = url.replace('/', '_')
cached_content = get_cache(url_id)
if cached_content:
return cached_content
else:
content = default_get_url(url)
set_cache(url_id, content)
return content
default_get_url = wall.helpers.get_url
wall.helpers.get_url = get_url
download_manager = DownloadManager()
download_manager.do('torrent_search')
#### Torrent download ####
# Don't actually download the torrents, create fake torrent files
# based on the retreived torrent info
def update_queued_torrents(self):
for torrent in Torrent.objects.filter(status='Queued'):
# Mark directly as completed only if enough seeds
if torrent.seeds < 1:
torrent.status = 'Error'
torrent.save()
torrent.status = 'Completed'
torrent.save()
# Iterate over each of the torrent files
for torrent_file_info in json.loads(torrent.file_list):
torrent_file = os.path.join(settings.TEST_DOWNLOAD_DIR, torrent_file_info['path'])
mkdir_p(os.path.split(torrent_file)[0])
# Fill the files with fake data, to keep file sizes poportional
with open(torrent_file, 'w') as f:
f.write('#' * (torrent_file_info['size']/1000))
import wall.torrentdownloader
default_update_queued_torrents = wall.torrentdownloader.TorrentDownloadManager.update_queued_torrents
wall.torrentdownloader.TorrentDownloadManager.update_queued_torrents = update_queued_torrents
# Caching of torrent info #
def get_torrent_info(self, torrent):
# Dirty hack - make torrent timeout if we have seen it timeout in the past (cache)
torrent_timeout_id = '%s_timeout' % torrent.hash
if get_cache(torrent_timeout_id):
from datetime import timedelta
log.info('Torrent timeout found in cache for %s', torrent.hash)
torrent.last_status_change -= timedelta(weeks=100)
torrent.save()
# torrent_info retrieval/cache
cached_content = get_cache(torrent.hash)
if cached_content:
log.info('Torrent info found in cache for %s', torrent.hash)
return cached_content
else:
torrent_bt = default_get_torrent_info(self, torrent)
# Only cache once we've got the metadata
if torrent_bt.has_metadata:
log.info('Adding torrent info in cache for %s', torrent.hash)
set_cache(torrent.hash, torrent_bt)
# Or if the torrent is timeout
elif torrent.is_timeout(settings.BITTORRENT_METADATA_TIMEOUT):
log.info('Recording torrent timeout in cache for %s', torrent.hash)
set_cache(torrent_timeout_id, True)
return torrent_bt
default_get_torrent_info = wall.torrentdownloader.Bittorrent.get_torrent_info
wall.torrentdownloader.Bittorrent.get_torrent_info = get_torrent_info
def add_magnet(self, magnet_uri):
hash = magnet_uri[20:60]
if get_cache(hash) is None:
default_add_magnet(self, magnet_uri)
else:
log.info('Not adding magnet, torrent info found in cache for %s', hash)
default_add_magnet = wall.torrentdownloader.Bittorrent.add_magnet
wall.torrentdownloader.Bittorrent.add_magnet = add_magnet
# Wait until all torrents metadata have been downloaded or cancelled #
while Torrent.objects.filter(Q(status="New")|Q(status='Downloading metadata')|Q(status='Queued')).count() > 0:
# The actual processing of the torrents objects
download_manager.do('torrent_download')
time.sleep(1)
#### Package management ####
download_manager.do('package_management')
#### Video transcoding ####
# Don't actually transcode anything
vt = mock_video_transcoder.return_value
# Never delay any transcoding processing
vt.has_free_slot.return_value = True
# Transcoding completes immediately
vt.is_running.return_value = False
# Run twice (first to start transcoding videos, second to mark transcoding as completed
download_manager.do('video_transcoding')
download_manager.do('video_transcoding')
#### Report ####
self.dump_test_db()
self.dump_processing_stats()
#### Cleanup ####
except:
raise
finally:
if default_open_url is not None:
wall.helpers.open_url = default_open_url
if default_get_url is not None:
wall.helpers.get_url = default_get_url
if default_update_queued_torrents is not None:
wall.torrentdownloader.TorrentDownloadManager.update_queued_torrents = default_update_queued_torrents
if default_get_torrent_info is not None:
wall.torrentdownloader.Bittorrent.get_torrent_info = default_get_torrent_info
if default_add_magnet is not None:
wall.torrentdownloader.Bittorrent.add_magnet = default_add_magnet