예제 #1
0
파일: cron.py 프로젝트: antoviaque/plebia
class Command(BaseCommand):
    help = 'Run cron tasks for the given category'
    option_list = BaseCommand.option_list + (
        make_option('-n', '--no-repeat', action="store_false", dest='repeat', default=True,
            help='Do not repeat every few seconds for one minute, execute just once'),
        ) + (
        make_option('-f', '--forever', action="store_true", dest='forever', default=False,
            help='Keep iterating infinitely (and keep objects running, like the bittorrent client)'),
        )

    def __init__(self):
        super(self.__class__,self).__init__()

        self.dl_manager = DownloadManager()
        self.start = time.time()

    def handle(self, *args, **options):

        if len(args) != 1 or args[0] not in self.dl_manager.get_actions_list():
            raise CommandError('You must specify one valid command (%s)' % repr(self.dl_manager.get_actions_list()))

        command = args[0]
        repeat = options.get('repeat')
        forever = options.get('forever')

        # Only allow one cron process per command to run at a single time
        lock_path = os.path.join(settings.LOCK_PATH, '.%s.pid' % command)
        try:
            l = lock.lock(lock_path, timeout=MAX_RUN_TIME) # wait at most 50s
            self.do(command, repeat, forever)
        except error.LockHeld:
            log.debug("Active process for command '%s', aborting.", command)
        else:
            l.release()

    def do(self, command, repeat, forever):
        '''Run the specified action 
        - once or repeat=True for a time-limited loop
        - for MAX_RUN_TIME or forever=True for forever'''

        log.info("Running download manager for command '%s' (repeat=%d, forever=%d)", command, repeat, forever)

        if not repeat:
            self.dl_manager.do(command)
        else:
            # Runs the specified action, every DELAY seconds for X minutes
            stop_time = self.start + MAX_RUN_TIME - DELAY
            while forever or time.time() <= stop_time:
                self.dl_manager.do(command)
                time.sleep(DELAY)
예제 #2
0
파일: cron.py 프로젝트: antoviaque/plebia
    def __init__(self):
        super(self.__class__,self).__init__()

        self.dl_manager = DownloadManager()
        self.start = time.time()
예제 #3
0
파일: tests.py 프로젝트: antoviaque/plebia
    def test_series_processing(self, mock_video_transcoder):
        '''Fake download of a list of test series to process and get success rates on'''

        import wall.plugins, wall.thetvdbapi, wall.helpers

        # Don't pollute the real download dir
        settings.DOWNLOAD_DIR = settings.TEST_DOWNLOAD_DIR

        default_open_url = None
        default_update_queued_torrents = None
        default_get_url = None
        default_get_torrent_info = None
        default_add_magnet = None

        # FIXME: Tests should cover all plugins
        self.select_plugin(TorrentSearcher, 'torrentz-searcher')

        # Don't let the other tests continue without reverting to the default methods/objects
        # we override in this test
        try:
            #### Retreive TVDB data for test strings ####

            # Cache it to allow to run this test multiple consecutive times quickly, without 
            # having to query the db engine every time
            def open_url(url):
                url_id = url.replace('/', '_')
                cache_path = os.path.join(settings.CACHE_DIR, url_id)

                # TVDB's parsing really wants a fp to an actual file
                # Instead of using get/set_cache, prepare the file directly
                mkdir_p(settings.CACHE_DIR)
                if not os.path.isfile(cache_path):
                    cache_fp = default_open_url(url)
                    with open(cache_path, 'w') as f:
                        f.write(cache_fp.read())

                return open(cache_path)

            default_open_url = wall.helpers.open_url
            wall.helpers.open_url = open_url

            for series_name in settings.TEST_SERIES_LIST:
                # Retreive series info
                Series.objects.add_by_search(series_name)
                
                # Make series active (scheduled for download)
                series = Series.objects.get(name=series_name)
                series.update_from_tvdb()


            #### Torrent search ####

            from wall.downloadmanager import DownloadManager

            def get_url(url):
                url_id = url.replace('/', '_')
                cached_content = get_cache(url_id)
                if cached_content:
                    return cached_content
                else:
                    content = default_get_url(url)
                    set_cache(url_id, content)
                    return content

            default_get_url = wall.helpers.get_url
            wall.helpers.get_url = get_url

            download_manager = DownloadManager()
            download_manager.do('torrent_search')


            #### Torrent download ####

            # Don't actually download the torrents, create fake torrent files
            # based on the retreived torrent info
            def update_queued_torrents(self):
                for torrent in Torrent.objects.filter(status='Queued'):
                    # Mark directly as completed only if enough seeds
                    if torrent.seeds < 1:
                        torrent.status = 'Error'
                        torrent.save()
                    
                    torrent.status = 'Completed'
                    torrent.save()

                    # Iterate over each of the torrent files
                    for torrent_file_info in json.loads(torrent.file_list):
                        torrent_file = os.path.join(settings.TEST_DOWNLOAD_DIR, torrent_file_info['path'])
                        mkdir_p(os.path.split(torrent_file)[0])
                        
                        # Fill the files with fake data, to keep file sizes poportional
                        with open(torrent_file, 'w') as f: 
                            f.write('#' * (torrent_file_info['size']/1000)) 

            import wall.torrentdownloader
            default_update_queued_torrents = wall.torrentdownloader.TorrentDownloadManager.update_queued_torrents
            wall.torrentdownloader.TorrentDownloadManager.update_queued_torrents = update_queued_torrents

            # Caching of torrent info #

            def get_torrent_info(self, torrent):
                # Dirty hack - make torrent timeout if we have seen it timeout in the past (cache)
                torrent_timeout_id = '%s_timeout' % torrent.hash
                if get_cache(torrent_timeout_id):
                    from datetime import timedelta
                    log.info('Torrent timeout found in cache for %s', torrent.hash)
                    torrent.last_status_change -= timedelta(weeks=100)
                    torrent.save()

                # torrent_info retrieval/cache
                cached_content = get_cache(torrent.hash)
                if cached_content:
                    log.info('Torrent info found in cache for %s', torrent.hash)
                    return cached_content
                else:
                    torrent_bt = default_get_torrent_info(self, torrent)
                    # Only cache once we've got the metadata
                    if torrent_bt.has_metadata:
                        log.info('Adding torrent info in cache for %s', torrent.hash)
                        set_cache(torrent.hash, torrent_bt)
                    # Or if the torrent is timeout
                    elif torrent.is_timeout(settings.BITTORRENT_METADATA_TIMEOUT):
                        log.info('Recording torrent timeout in cache for %s', torrent.hash)
                        set_cache(torrent_timeout_id, True)
                    return torrent_bt
            
            default_get_torrent_info = wall.torrentdownloader.Bittorrent.get_torrent_info
            wall.torrentdownloader.Bittorrent.get_torrent_info = get_torrent_info

            def add_magnet(self, magnet_uri):
                hash = magnet_uri[20:60]
                if get_cache(hash) is None:
                    default_add_magnet(self, magnet_uri)
                else:
                    log.info('Not adding magnet, torrent info found in cache for %s', hash)

            default_add_magnet = wall.torrentdownloader.Bittorrent.add_magnet
            wall.torrentdownloader.Bittorrent.add_magnet = add_magnet

            # Wait until all torrents metadata have been downloaded or cancelled #

            while Torrent.objects.filter(Q(status="New")|Q(status='Downloading metadata')|Q(status='Queued')).count() > 0:
                # The actual processing of the torrents objects
                download_manager.do('torrent_download')
                time.sleep(1)
            

            #### Package management ####

            download_manager.do('package_management')


            #### Video transcoding ####

            # Don't actually transcode anything
            vt = mock_video_transcoder.return_value

            # Never delay any transcoding processing
            vt.has_free_slot.return_value = True

            # Transcoding completes immediately
            vt.is_running.return_value = False

            # Run twice (first to start transcoding videos, second to mark transcoding as completed
            download_manager.do('video_transcoding')
            download_manager.do('video_transcoding')


            #### Report ####

            self.dump_test_db()
            self.dump_processing_stats()
            
        #### Cleanup ####

        except:
            raise
        finally:
            if default_open_url is not None:
                wall.helpers.open_url = default_open_url
            if default_get_url is not None:
                wall.helpers.get_url = default_get_url
            if default_update_queued_torrents is not None:
                wall.torrentdownloader.TorrentDownloadManager.update_queued_torrents = default_update_queued_torrents
            if default_get_torrent_info is not None:
                wall.torrentdownloader.Bittorrent.get_torrent_info = default_get_torrent_info
            if default_add_magnet is not None:
                wall.torrentdownloader.Bittorrent.add_magnet = default_add_magnet