def test_space_around(self): # We don't want folder or file names to end or start with spaces on any platform space_paths = {" / aoeu /aoeu ": "/aoeu/aoeu", "/ a/a ": "/a/a", "/a /": "/a/"} for platform in ["windows", "linux", "mac"]: for test in space_paths: result = pathscrub(test, filename=False) assert result == space_paths[test], "%s != %s (%s)" % (result, space_paths[test], platform) # Windows only should also use backslashes as dir separators test = ["c:\\ aoeu \\aoeu /aoeu ", "c:\\aoeu\\aoeu/aoeu"] result = pathscrub(test[0], os="windows", filename=False) assert result == test[1], "%s != %s" % (result, test[1])
def on_get_session_state(torrent_ids): """Gets called with a list of torrent_ids loaded in the deluge session. Adds new torrents and modifies the settings for ones already in the session.""" dlist = [] # add the torrents for entry in task.accepted: def add_entry(entry, opts): """Adds an entry to the deluge session""" magnet, filedump = None, None if entry.get('url', '').startswith('magnet:'): magnet = entry['url'] else: if not os.path.exists(entry['file']): task.fail(entry, 'Downloaded temp file \'%s\' doesn\'t exist!' % entry['file']) del(entry['file']) return try: f = open(entry['file'], 'rb') filedump = base64.encodestring(f.read()) finally: f.close() log.verbose('Adding %s to deluge.' % entry['title']) if magnet: return client.core.add_torrent_magnet(magnet, opts) else: return client.core.add_torrent_file(entry['title'], filedump, opts) # Generate deluge options dict for torrent add add_opts = {} try: path = entry.render(entry.get('path', config['path'])) if path: add_opts['download_location'] = pathscrub(os.path.expanduser(path)) except RenderError, e: log.error('Could not set path for %s: %s' % (entry['title'], e)) for fopt, dopt in self.options.iteritems(): value = entry.get(fopt, config.get(fopt)) if value is not None: add_opts[dopt] = value if fopt == 'ratio': add_opts['stop_at_ratio'] = True # Make another set of options, that get set after the torrent has been added modify_opts = {'label': format_label(entry.get('label', config['label'])), 'queuetotop': entry.get('queuetotop', config.get('queuetotop')), 'main_file_only': entry.get('main_file_only', config.get('main_file_only', False))} try: movedone = entry.render(entry.get('movedone', config['movedone'])) modify_opts['movedone'] = pathscrub(os.path.expanduser(movedone)) except RenderError, e: log.error('Error setting movedone for %s: %s' % (entry['title'], e))
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout): import libtorrent params = libtorrent.parse_magnet_uri(magnet_uri) session = libtorrent.session() lt_version = [int(v) for v in libtorrent.version.split('.')] if lt_version > [0, 16, 13, 0] and lt_version < [1, 1, 3, 0]: # for some reason the info_hash needs to be bytes but it's a struct called sha1_hash params['info_hash'] = params['info_hash'].to_bytes() params.url = magnet_uri handle = session.add_torrent(params) logger.debug('Acquiring torrent metadata for magnet {}', magnet_uri) timeout_value = timeout while not handle.has_metadata(): time.sleep(0.1) timeout_value -= 0.1 if timeout_value <= 0: raise plugin.PluginError( 'Timed out after {} seconds trying to magnetize'.format( timeout)) logger.debug('Metadata acquired') torrent_info = handle.get_torrent_info() torrent_file = libtorrent.create_torrent(torrent_info) torrent_path = pathscrub( os.path.join(destination_folder, torrent_info.name() + ".torrent")) with open(torrent_path, "wb") as f: f.write(libtorrent.bencode(torrent_file.generate())) logger.debug('Torrent file wrote to {}', torrent_path) return torrent_path
def test_space_around(self): # We don't want folder or file names to end or start with spaces on any platform space_paths = { ' / aoeu /aoeu ': '/aoeu/aoeu', '/ a/a ': '/a/a', '/a /': '/a/' } for platform in ['windows', 'linux', 'mac']: for test in space_paths: result = pathscrub(test, filename=False) assert result == space_paths[test], '%s != %s (%s)' % (result, space_paths[test], platform) # Windows only should also use backslashes as dir separators test = ['c:\\ aoeu \\aoeu /aoeu ', 'c:\\aoeu\\aoeu/aoeu'] result = pathscrub(test[0], os='windows', filename=False) assert result == test[1], '%s != %s' % (result, test[1])
def _build_options(self, config, entry, entry_first=True): options = {} for opt_key in ("path", "message", "priority", "custom1", "custom2", "custom3", "custom4", "custom5"): # Values do not merge config with task # Task takes priority then config is used entry_value = entry.get(opt_key) config_value = config.get(opt_key) if entry_first: if entry_value: options[opt_key] = entry.render(entry_value) elif config_value: options[opt_key] = entry.render(config_value) else: if config_value: options[opt_key] = entry.render(config_value) elif entry_value: options[opt_key] = entry.render(entry_value) # Convert priority from string to int priority = options.get("priority") if priority and priority in self.priority_map: options["priority"] = self.priority_map[priority] # Map Flexget path to directory in rTorrent if options.get("path"): options["directory"] = options["path"] del options["path"] if "directory" in options: options["directory"] = pathscrub(options["directory"]) return options
def test_space_around(self): # We don't want folder or file names to end or start with spaces on any platform space_paths = {' / aoeu /aoeu ': '/aoeu/aoeu', '/ a/a ': '/a/a', '/a /': '/a/'} for platform in ['windows', 'linux', 'mac']: for test in space_paths: result = pathscrub(test, filename=False) assert result == space_paths[test], '%s != %s (%s)' % ( result, space_paths[test], platform, ) # Windows only should also use backslashes as dir separators test = ['c:\\ aoeu \\aoeu /aoeu ', 'c:\\aoeu\\aoeu/aoeu'] result = pathscrub(test[0], os='windows', filename=False) assert result == test[1], '%s != %s' % (result, test[1])
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout=60): import libtorrent params = libtorrent.parse_magnet_uri(magnet_uri) session = libtorrent.session() handle = session.add_torrent(params) log.debug( 'Acquiring torrent metadata for magnet {}'.format(magnet_uri)) timeout_value = timeout while not handle.has_metadata(): time.sleep(0.1) timeout_value -= 0.1 if timeout_value <= 0: raise Exception( 'Timed out after {} seconds acquiring torrent metadata from the DHT/trackers.' .format(timeout)) log.debug('Metadata acquired') torrent_info = handle.get_torrent_info() torrent_file = libtorrent.create_torrent(torrent_info) torrent_path = pathscrub( os.path.join(destination_folder, torrent_info.name() + ".torrent")) with open(torrent_path, "wb") as f: f.write(libtorrent.bencode(torrent_file.generate())) log.debug('Torrent file wrote to {}'.format(torrent_path)) return torrent_path
def _build_options(self, config, entry, entry_first=True): options = {} for opt_key in ('path', 'message', 'priority', 'custom1', 'custom2', 'custom3', 'custom4', 'custom5'): # Values do not merge config with task # Task takes priority then config is used entry_value = entry.get(opt_key) config_value = config.get(opt_key) if entry_first: if entry_value: options[opt_key] = entry.render(entry_value) elif config_value: options[opt_key] = entry.render(config_value) else: if config_value: options[opt_key] = entry.render(config_value) elif entry_value: options[opt_key] = entry.render(entry_value) # Convert priority from string to int priority = options.get('priority') if priority and priority in self.priority_map: options['priority'] = self.priority_map[priority] # Map Flexget path to directory in rTorrent if options.get('path'): options['directory'] = options['path'] del options['path'] if 'directory' in options: options['directory'] = pathscrub(options['directory']) return options
def process_invalid_content(self, task, data, url): """If feedparser reports error, save the received data and log error.""" if data is None: log.critical("Received empty page - no content") return ext = "xml" if "<html>" in data.lower(): log.critical("Received content is HTML page, not an RSS feed") ext = "html" if "login" in data.lower() or "username" in data.lower(): log.critical("Received content looks a bit like login page") if "error" in data.lower(): log.critical("Received content looks a bit like error page") received = os.path.join(task.manager.config_base, "received") if not os.path.isdir(received): os.mkdir(received) filename = task.name sourcename = urlparse.urlparse(url).netloc if sourcename: filename += "-" + sourcename filename = pathscrub(filename, filename=True) filepath = os.path.join(received, "%s.%s" % (filename, ext)) with open(filepath, "w") as f: f.write(data) log.critical("I have saved the invalid content to %s for you to view", filepath)
def process_invalid_content(self, task, data, url): """If feedparser reports error, save the received data and log error.""" if data is None: log.critical('Received empty page - no content') return else: data = tobytes(data) ext = 'xml' if b'<html>' in data.lower(): log.critical('Received content is HTML page, not an RSS feed') ext = 'html' if b'login' in data.lower() or b'username' in data.lower(): log.critical('Received content looks a bit like login page') if b'error' in data.lower(): log.critical('Received content looks a bit like error page') received = os.path.join(task.manager.config_base, 'received') if not os.path.isdir(received): os.mkdir(received) filename = task.name sourcename = urlparse(url).netloc if sourcename: filename += '-' + sourcename filename = pathscrub(filename, filename=True) filepath = os.path.join(received, '%s.%s' % (filename, ext)) with open(filepath, 'wb') as f: f.write(data) log.critical('I have saved the invalid content to %s for you to view', filepath)
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout): import libtorrent params = libtorrent.parse_magnet_uri(magnet_uri) session = libtorrent.session() # for some reason the info_hash needs to be bytes but it's a struct called sha1_hash params['info_hash'] = bytes(params['info_hash']) handle = libtorrent.add_magnet_uri(session, magnet_uri, params) log.debug('Acquiring torrent metadata for magnet %s', magnet_uri) timeout_value = timeout while not handle.has_metadata(): time.sleep(0.1) timeout_value -= 0.1 if timeout_value <= 0: raise plugin.PluginError( 'Timed out after {} seconds trying to magnetize'.format( timeout)) log.debug('Metadata acquired') torrent_info = handle.get_torrent_info() torrent_file = libtorrent.create_torrent(torrent_info) torrent_path = pathscrub( os.path.join(destination_folder, torrent_info.name() + ".torrent")) with open(torrent_path, "wb") as f: f.write(libtorrent.bencode(torrent_file.generate())) log.debug('Torrent file wrote to %s', torrent_path) return torrent_path
def save_error_page(self, entry, task, page): received = os.path.join(task.manager.config_base, 'received', task.name) if not os.path.isdir(received): os.makedirs(received) filename = os.path.join(received, pathscrub('%s.error' % entry['title'], filename=True)) log.error('Error retrieving %s, the error page has been saved to %s', entry['title'], filename) with io.open(filename, 'wb') as outfile: outfile.write(page)
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority', 'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio', 'main_file_only', 'include_subs'): if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {'add': {}, 'change': {}, 'post': {}} add = options['add'] if opt_dic.get('path'): try: path = os.path.expanduser(entry.render(opt_dic['path'])) add['download_dir'] = pathscrub(path).encode('utf-8') except RenderError as e: log.error('Error setting path for %s: %s' % (entry['title'], e)) if 'bandwidthpriority' in opt_dic: add['bandwidthPriority'] = opt_dic['bandwidthpriority'] if 'maxconnections' in opt_dic: add['peer_limit'] = opt_dic['maxconnections'] # make sure we add it paused, will modify status after adding add['paused'] = True change = options['change'] if 'honourlimits' in opt_dic and not opt_dic['honourlimits']: change['honorsSessionLimits'] = False if 'maxupspeed' in opt_dic: change['uploadLimit'] = opt_dic['maxupspeed'] change['uploadLimited'] = True if 'maxdownspeed' in opt_dic: change['downloadLimit'] = opt_dic['maxdownspeed'] change['downloadLimited'] = True if 'ratio' in opt_dic: change['seedRatioLimit'] = opt_dic['ratio'] if opt_dic['ratio'] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio change['seedRatioMode'] = 2 else: change['seedRatioMode'] = 1 post = options['post'] # set to modify paused status after if 'addpaused' in opt_dic: post['paused'] = opt_dic['addpaused'] if 'main_file_only' in opt_dic: post['main_file_only'] = opt_dic['main_file_only'] if 'include_subs' in opt_dic: post['include_subs'] = opt_dic['include_subs'] return options
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ( "path", "addpaused", "honourlimits", "bandwidthpriority", "maxconnections", "maxupspeed", "maxdownspeed", "ratio", ): if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {"add": {}, "change": {}} add = options["add"] if opt_dic.get("path"): try: path = os.path.expanduser(entry.render(opt_dic["path"])) add["download_dir"] = pathscrub(path).encode("utf-8") except RenderError as e: log.error("Error setting path for %s: %s" % (entry["title"], e)) if "addpaused" in opt_dic: add["paused"] = opt_dic["addpaused"] if "bandwidthpriority" in opt_dic: add["bandwidthPriority"] = opt_dic["bandwidthpriority"] if "maxconnections" in opt_dic: add["peer_limit"] = opt_dic["maxconnections"] change = options["change"] if "honourlimits" in opt_dic and not opt_dic["honourlimits"]: change["honorsSessionLimits"] = False if "maxupspeed" in opt_dic: change["uploadLimit"] = opt_dic["maxupspeed"] change["uploadLimited"] = True if "maxdownspeed" in opt_dic: change["downloadLimit"] = opt_dic["maxdownspeed"] change["downloadLimited"] = True if "ratio" in opt_dic: change["seedRatioLimit"] = opt_dic["ratio"] if opt_dic["ratio"] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio change["seedRatioMode"] = 2 else: change["seedRatioMode"] = 1 return options
def test_windows_paths(self): win_path = { 'aoeu/aoeu': 'aoeu/aoeu', # Don't strip slashes in path mode 'aoeu\\aoeu': 'aoeu\\aoeu', # Or backslashes 'aoeu / aoeu ': 'aoeu/aoeu', # Don't leave spaces at the begin or end of folder names 'aoeu \\aoeu ': 'aoeu\\aoeu', 'aoeu./aoeu.\\aoeu.': 'aoeu/aoeu\\aoeu', # Or dots } for test in win_path: result = pathscrub(test, os='windows', filename=False) assert result == win_path[test], '%s != %s' % (result, win_path[test])
def test_windows_paths(self): win_path = { 'aoeu/aoeu': 'aoeu/aoeu', # Don't strip slashes in path mode 'aoeu\\aoeu': 'aoeu\\aoeu', # Or backslashes 'aoeu / aoeu ': 'aoeu/aoeu', # Don't leave spaces at the begin or end of folder names 'aoeu \\aoeu ': 'aoeu\\aoeu', 'aoeu./aoeu.\\aoeu.': 'aoeu/aoeu\\aoeu' # Or dots } for test in win_path: result = pathscrub(test, os='windows', filename=False) assert result == win_path[test], '%s != %s' % (result, win_path[test])
def test_windows_paths(self): win_path = { "aoeu/aoeu": "aoeu/aoeu", # Don't strip slashes in path mode "aoeu\\aoeu": "aoeu\\aoeu", # Or backslashes "aoeu / aoeu ": "aoeu/aoeu", # Don't leave spaces at the begin or end of folder names "aoeu \\aoeu ": "aoeu\\aoeu", "aoeu./aoeu.\\aoeu.": "aoeu/aoeu\\aoeu", # Or dots } for test in win_path: result = pathscrub(test, os="windows", filename=False) assert result == win_path[test], "%s != %s" % (result, win_path[test])
def process(self, entry, destination_folder, timeout): import libtorrent magnet_uri = entry['url'] params = libtorrent.parse_magnet_uri(magnet_uri) session = libtorrent.session() lt_version = [int(v) for v in libtorrent.version.split('.')] if lt_version > [0, 16, 13, 0] and lt_version < [1, 1, 3, 0]: # for some reason the info_hash needs to be bytes but it's a struct called sha1_hash params['info_hash'] = params['info_hash'].to_bytes() params.url = magnet_uri handle = session.add_torrent(params) log.debug('Acquiring torrent metadata for magnet %s', magnet_uri) handle.force_dht_announce() timeout_value = timeout while not handle.has_metadata(): time.sleep(0.1) timeout_value -= 0.1 if timeout_value <= 0: raise plugin.PluginError( 'Timed out after {} seconds trying to magnetize'.format( timeout)) log.debug('Metadata acquired') torrent_info = handle.get_torrent_info() torrent_file = libtorrent.create_torrent(torrent_info) torrent_path = pathscrub( os.path.join(destination_folder, torrent_info.name() + ".torrent")) with open(torrent_path, "wb") as f: f.write(libtorrent.bencode(torrent_file.generate())) log.debug('Torrent file wrote to %s', torrent_path) # Windows paths need an extra / prepended to them for url if not torrent_path.startswith('/'): torrent_path = '/' + torrent_path entry['url'] = torrent_path entry['file'] = torrent_path # make sure it's first in the list because of how download plugin works entry['urls'].insert(0, 'file://{}'.format(torrent_path)) entry['content_size'] = torrent_info.total_size() / 1024 / 1024 # Might as well get some more info while handle.status(0).num_complete < 0: time.sleep(0.1) timeout_value -= 0.1 if timeout_value <= 0: log.debug('Timed out after {} seconds trying to get peer info'. format(timeout)) return log.debug('Peer info acquired') torrent_status = handle.status(0) entry['torrent_seeds'] = torrent_status.num_complete entry['torrent_leeches'] = torrent_status.num_incomplete
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority', 'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio'): if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {'add': {}, 'change': {}} add = options['add'] if opt_dic.get('path'): try: path = os.path.expanduser(entry.render(opt_dic['path'])) add['download_dir'] = pathscrub(path).encode('utf-8') except RenderError as e: log.error('Error setting path for %s: %s' % (entry['title'], e)) if 'addpaused' in opt_dic: add['paused'] = opt_dic['addpaused'] if 'bandwidthpriority' in opt_dic: add['bandwidthPriority'] = opt_dic['bandwidthpriority'] if 'maxconnections' in opt_dic: add['peer_limit'] = opt_dic['maxconnections'] change = options['change'] if 'honourlimits' in opt_dic and not opt_dic['honourlimits']: change['honorsSessionLimits'] = False if 'maxupspeed' in opt_dic: change['uploadLimit'] = opt_dic['maxupspeed'] change['uploadLimited'] = True if 'maxdownspeed' in opt_dic: change['downloadLimit'] = opt_dic['maxdownspeed'] change['downloadLimited'] = True if 'ratio' in opt_dic: change['seedRatioLimit'] = opt_dic['ratio'] if opt_dic['ratio'] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio change['seedRatioMode'] = 2 else: change['seedRatioMode'] = 1 return options
def on_task_download(self, task, config): config = self.prepare_config(config, task) for entry in task.accepted: ftp_url = urlparse(entry.get('url')) ftp_url = ftp_url._replace(path=unquote(ftp_url.path)) current_path = os.path.dirname(ftp_url.path) try: ftp = self.ftp_connect(config, ftp_url, current_path) except ftplib.all_errors as e: entry.fail("Unable to connect to server : %s" % (e)) break to_path = config['ftp_tmp_path'] try: to_path = entry.render(to_path) except RenderError as err: raise plugin.PluginError( "Path value replacement `%s` failed: %s" % (to_path, err.args[0])) # Clean invalid characters with pathscrub plugin to_path = pathscrub(to_path) if not os.path.exists(to_path): log.debug("Creating base path: %s" % to_path) os.makedirs(to_path) if not os.path.isdir(to_path): raise plugin.PluginWarning( "Destination `%s` is not a directory." % to_path) file_name = os.path.basename(ftp_url.path) try: # Directory ftp = self.check_connection(ftp, config, ftp_url, current_path) ftp.cwd(file_name) self.ftp_walk(ftp, os.path.join(to_path, file_name), config, ftp_url, ftp_url.path) ftp = self.check_connection(ftp, config, ftp_url, current_path) ftp.cwd('..') if config['delete_origin']: ftp.rmd(file_name) except ftplib.error_perm: # File self.ftp_down(ftp, file_name, to_path, config, ftp_url, current_path) ftp.close()
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority', 'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio'): if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {'add': {}, 'change': {}} if opt_dic.get('path'): try: path = os.path.expanduser(entry.render(opt_dic['path'])).encode('utf-8') options['add']['download_dir'] = pathscrub(path) except RenderError as e: log.error('Error setting path for %s: %s' % (entry['title'], e)) if 'addpaused' in opt_dic: options['add']['paused'] = opt_dic['addpaused'] if 'bandwidthpriority' in opt_dic: options['add']['bandwidthPriority'] = opt_dic['bandwidthpriority'] if 'maxconnections' in opt_dic: options['add']['peer_limit'] = opt_dic['maxconnections'] if 'honourlimits' in opt_dic and not opt_dic['honourlimits']: options['change']['honorsSessionLimits'] = False if 'maxupspeed' in opt_dic: options['change']['uploadLimit'] = opt_dic['maxupspeed'] options['change']['uploadLimited'] = True if 'maxdownspeed' in opt_dic: options['change']['downloadLimit'] = opt_dic['maxdownspeed'] options['change']['downloadLimited'] = True if 'ratio' in opt_dic: options['change']['seedRatioLimit'] = opt_dic['ratio'] if opt_dic['ratio'] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio options['change']['seedRatioMode'] = 2 else: options['change']['seedRatioMode'] = 1 return options
def on_task_output(self, task, config): import youtube_dl.YoutubeDL from youtube_dl.utils import ExtractorError class YoutubeDL(youtube_dl.YoutubeDL): def __init__(self, *args, **kwargs): self.to_stderr = self.to_screen self.processed_info_dicts = [] super(YoutubeDL, self).__init__(*args, **kwargs) def report_warning(self, message): raise ExtractorError(message) def process_info(self, info_dict): self.processed_info_dicts.append(info_dict) return super(YoutubeDL, self).process_info(info_dict) for entry in task.accepted: if task.options.test: log.info('Would download %s' % entry['title']) else: try: outtmpl = entry.render(config['path']) + '/' + pathscrub(entry.render(config['template']) + '.%(ext)s', filename=True) log.info("Output file: %s" % outtmpl) except RenderError as e: log.error('Error setting output file: %s' % e) entry.fail('Error setting output file: %s' % e) params = {'quiet': True, 'outtmpl': outtmpl} if 'username' in config and 'password' in config: params.update({'username': config['username'], 'password': config['password']}) elif 'username' in config or 'password' in config: log.error('Both username and password is required') if 'videopassword' in config: params.update({'videopassword': config['videopassword']}) if 'title' in config: params.update({'title': config['title']}) ydl = YoutubeDL(params) ydl.add_default_info_extractors() log.info('Downloading %s' % entry['title']) try: ydl.download([entry['url']]) except ExtractorError as e: log.error('Youtube-DL was unable to download the video. Error message %s' % e.message) entry.fail('Youtube-DL was unable to download the video. Error message %s' % e.message) except Exception as e: log.error('Youtube-DL failed. Error message %s' % e.message) entry.fail('Youtube-DL failed. Error message %s' % e.message)
def test_windows_filenames(self): # Windows filename tests # 'None' indicates there should be no changes after path scrub win_fn = { 'afilename': 'afilename', 'filename/with/slash': 'filename with slash', 'filename\\with\\backslash': 'filename with backslash', 'afilename.': 'afilename', # filenames can't end in dot 'a<b>c:d"e/f\\g|h?i*j': 'a b c d e f g h i j', # Can't contain invalid characters 'a<<b?*?c: d': 'a b c d', # try with some repeated bad characters 'something.>': 'something', # Don't leave dots at the end 'something *': 'something', # Don't leave spaces at the end 'aoeu. > * . * <': 'aoeu' # Really don't leave spaces or dots at the end } for test in win_fn: result = pathscrub(test, os='windows', filename=True) assert result == win_fn[test], '%s != %s' % (result, win_fn[test])
def test_windows_filenames(self): # Windows filename tests # 'None' indicates there should be no changes after path scrub win_fn = { 'afilename': 'afilename', 'filename/with/slash': 'filename with slash', 'filename\\with\\backslash': 'filename with backslash', 'afilename.': 'afilename', # filenames can't end in dot 'a<b>c:d"e/f\\g|h?i*j': 'a b c d e f g h i j', # Can't contain invalid characters 'a<<b?*?c: d': 'a b c d', # try with some repeated bad characters 'something.>': 'something', # Don't leave dots at the end 'something *': 'something', # Don't leave spaces at the end 'aoeu. > * . * <': 'aoeu', # Really don't leave spaces or dots at the end } for test in win_fn: result = pathscrub(test, os='windows', filename=True) assert result == win_fn[test], '%s != %s' % (result, win_fn[test])
def test_windows_filenames(self): # Windows filename tests # 'None' indicates there should be no changes after path scrub win_fn = { "afilename": "afilename", "filename/with/slash": "filename with slash", "filename\\with\\backslash": "filename with backslash", "afilename.": "afilename", # filenames can't end in dot 'a<b>c:d"e/f\\g|h?i*j': "a b c d e f g h i j", # Can't contain invalid characters "a<<b?*?c: d": "a b c d", # try with some repeated bad characters "something.>": "something", # Don't leave dots at the end "something *": "something", # Don't leave spaces at the end "aoeu. > * . * <": "aoeu", # Really don't leave spaces or dots at the end } for test in win_fn: result = pathscrub(test, os="windows", filename=True) assert result == win_fn[test], "%s != %s" % (result, win_fn[test])
def on_task_download(self, task, config): config = self.prepare_config(config, task) for entry in task.accepted: ftp_url = urlparse(entry.get('url')) ftp_url = ftp_url._replace(path=unquote(ftp_url.path)) current_path = os.path.dirname(ftp_url.path) try: ftp = self.ftp_connect(config, ftp_url, current_path) except ftplib.all_errors as e: entry.fail("Unable to connect to server : %s" % (e)) break to_path = config['ftp_tmp_path'] try: to_path = entry.render(to_path) except RenderError as err: raise plugin.PluginError("Path value replacement `%s` failed: %s" % (to_path, err.args[0])) # Clean invalid characters with pathscrub plugin to_path = pathscrub(to_path) if not os.path.exists(to_path): log.debug("Creating base path: %s" % to_path) os.makedirs(to_path) if not os.path.isdir(to_path): raise plugin.PluginWarning("Destination `%s` is not a directory." % to_path) file_name = os.path.basename(ftp_url.path) try: # Directory ftp = self.check_connection(ftp, config, ftp_url, current_path) ftp.cwd(file_name) self.ftp_walk(ftp, os.path.join(to_path, file_name), config, ftp_url, ftp_url.path) ftp = self.check_connection(ftp, config, ftp_url, current_path) ftp.cwd('..') if config['delete_origin']: ftp.rmd(file_name) except ftplib.error_perm: # File self.ftp_down(ftp, file_name, to_path, config, ftp_url, current_path) ftp.close()
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority', 'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio'): if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {'add': {}, 'change': {}} if opt_dic.get('path'): try: path = os.path.expanduser(entry.render(opt_dic['path'])).encode('utf-8') options['add']['download_dir'] = pathscrub(path) except RenderError, e: log.error('Error setting path for %s: %s' % (entry['title'], e))
def on_task_output(self, task, config): for entry in task.accepted: path = self.prepare_path(entry, config) try: outtmpl = os.path.join( path, pathscrub(entry.render(config['template']))) logger.debug("Output template: %s" % outtmpl) except RenderError as e: logger.error('Error setting output file: %s' % e) entry.fail('Error setting output file: %s' % e) # ytdl options by default params = { 'quiet': True, 'outtmpl': outtmpl, 'logger': logger, 'noprogress': True } # add config to params if 'format' in config: if config['format']: params.update({'format': config['format']}) if config.get('other_options'): params.update(config['other_options']) logger.debug(params) if task.options.test: logger.info('Would download `{}` in `{}`', entry['title'], path) else: logger.info('Downloading `{}` in `{}`', entry['title'], path) try: with self.ytdl_module.YoutubeDL(params) as ydl: ydl.download([entry['url']]) except (self.ytdl_module.utils.ExtractorError, self.ytdl_module.utils.DownloadError) as e: entry.fail('Youtube downloader error: %s' % e) except Exception as e: entry.fail('Youtube downloader failed. Error message: %s' % e)
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout=60): import libtorrent params = libtorrent.parse_magnet_uri(magnet_uri) session = libtorrent.session() handle = session.add_torrent(params) log.debug('Acquiring torrent metadata for magnet {}'.format(magnet_uri)) timeout_value = timeout while not handle.has_metadata(): time.sleep(0.1) timeout_value -= 0.1 if timeout_value <= 0: raise Exception('Timed out after {} seconds acquiring torrent metadata from the DHT/trackers.'.format( timeout )) log.debug('Metadata acquired') torrent_info = handle.get_torrent_info() torrent_file = libtorrent.create_torrent(torrent_info) torrent_path = pathscrub(os.path.join(destination_folder, torrent_info.name() + ".torrent")) with open(torrent_path, "wb") as f: f.write(libtorrent.bencode(torrent_file.generate())) log.debug('Torrent file wrote to {}'.format(torrent_path)) return torrent_path
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout): import libtorrent params = libtorrent.parse_magnet_uri(magnet_uri) session = libtorrent.session() # for some reason the info_hash needs to be bytes but it's a struct called sha1_hash params["info_hash"] = bytes(params["info_hash"]) handle = libtorrent.add_magnet_uri(session, magnet_uri, params) log.debug("Acquiring torrent metadata for magnet %s", magnet_uri) timeout_value = timeout while not handle.has_metadata(): time.sleep(0.1) timeout_value -= 0.1 if timeout_value <= 0: raise plugin.PluginError("Timed out after {} seconds trying to magnetize".format(timeout)) log.debug("Metadata acquired") torrent_info = handle.get_torrent_info() torrent_file = libtorrent.create_torrent(torrent_info) torrent_path = pathscrub(os.path.join(destination_folder, torrent_info.name() + ".torrent")) with open(torrent_path, "wb") as f: f.write(libtorrent.bencode(torrent_file.generate())) log.debug("Torrent file wrote to %s", torrent_path) return torrent_path
def _set_torrent_options(self, client, torrent_id, entry, opts): """Gets called when a torrent was added to the daemon.""" log.info('%s successfully added to deluge.', entry['title']) entry['deluge_id'] = torrent_id if opts.get('move_completed_path'): client.call('core.set_torrent_move_completed', torrent_id, True) client.call('core.set_torrent_move_completed_path', torrent_id, opts['move_completed_path']) log.debug('%s move on complete set to %s', entry['title'], opts['move_completed_path']) if opts.get('label'): client.call('label.set_torrent', torrent_id, opts['label']) if opts.get('queue_to_top') is not None: if opts['queue_to_top']: client.call('core.queue_top', [torrent_id]) log.debug('%s moved to top of queue', entry['title']) else: client.call('core.queue_bottom', [torrent_id]) log.debug('%s moved to bottom of queue', entry['title']) status_keys = [ 'files', 'total_size', 'save_path', 'move_on_completed_path', 'move_on_completed', 'progress', ] status = client.call('core.get_torrent_status', torrent_id, status_keys) # Determine where the file should be move_now_path = None if opts.get('move_completed_path'): if status['progress'] == 100: move_now_path = opts['move_completed_path'] else: # Deluge will unset the move completed option if we move the storage, forgo setting proper # path, in favor of leaving proper final location. log.debug( 'Not moving storage for %s, as this will prevent move_completed_path.', entry['title'], ) elif opts.get('path'): move_now_path = opts['path'] if move_now_path and os.path.normpath( move_now_path) != os.path.normpath(status['save_path']): log.debug('Moving storage for %s to %s', entry['title'], move_now_path) client.call('core.move_storage', [torrent_id], move_now_path) big_file_name = '' if opts.get('content_filename') or opts.get('main_file_only'): # find a file that makes up more than main_file_ratio (default: 90%) of the total size main_file = None for file in status['files']: if file['size'] > (status['total_size'] * opts.get('main_file_ratio')): main_file = file break def file_exists(filename): # Checks the download path as well as the move completed path for existence of the file if os.path.exists(os.path.join(status['save_path'], filename)): return True elif status.get('move_on_completed') and status.get( 'move_on_completed_path'): if os.path.exists( os.path.join(status['move_on_completed_path'], filename)): return True else: return False def unused_name(name): # If on local computer, tries appending a (#) suffix until a unique filename is found if client.host in ['127.0.0.1', 'localhost']: counter = 2 while file_exists(name): name = ''.join([ os.path.splitext(name)[0], " (", str(counter), ')', os.path.splitext(name)[1], ]) counter += 1 else: log.debug( 'Cannot ensure content_filename is unique when adding to a remote deluge daemon.' ) return name def rename(file, new_name): # Renames a file in torrent client.call('core.rename_files', torrent_id, [(file['index'], new_name)]) log.debug('File %s in %s renamed to %s', file['path'], entry['title'], new_name) if main_file is not None: # proceed with renaming only if such a big file is found # find the subtitle file keep_subs = opts.get('keep_subs') sub_file = None if keep_subs: sub_exts = [".srt", ".sub"] for file in status['files']: ext = os.path.splitext(file['path'])[1] if ext in sub_exts: sub_file = file break # check for single file torrents so we dont add unnecessary folders top_files_dir = "/" if os.path.dirname(main_file['path']) is not ("" or "/"): # check for top folder in user config if (opts.get('content_filename') and os.path.dirname( opts['content_filename']) is not ""): top_files_dir = os.path.dirname( opts['content_filename']) + "/" else: top_files_dir = os.path.dirname( main_file['path']) + "/" if opts.get('content_filename'): # rename the main file big_file_name = ( top_files_dir + os.path.basename(opts['content_filename']) + os.path.splitext(main_file['path'])[1]) big_file_name = unused_name(big_file_name) rename(main_file, big_file_name) # rename subs along with the main file if sub_file is not None and keep_subs: sub_file_name = (os.path.splitext(big_file_name)[0] + os.path.splitext(sub_file['path'])[1]) rename(sub_file, sub_file_name) if opts.get('main_file_only'): # download only the main file (and subs) file_priorities = [ 1 if f == main_file or f == sub_file and keep_subs else 0 for f in status['files'] ] client.call('core.set_torrent_file_priorities', torrent_id, file_priorities) if opts.get('hide_sparse_files'): # hide the other sparse files that are not supposed to download but are created anyway # http://dev.deluge-torrent.org/ticket/1827 # Made sparse files behave better with deluge http://flexget.com/ticket/2881 sparse_files = [ f for f in status['files'] if f != main_file and ( f != sub_file or not keep_subs) ] rename_pairs = [( f['index'], top_files_dir + ".sparse_files/" + os.path.basename(f['path']), ) for f in sparse_files] client.call('core.rename_files', torrent_id, rename_pairs) else: log.warning( 'No files in "%s" are > %d%% of content size, no files renamed.', entry['title'], opts.get('main_file_ratio') * 100, ) container_directory = pathscrub( entry.render( entry.get('container_directory', opts.get('container_directory', '')))) if container_directory: if big_file_name: folder_structure = big_file_name.split(os.sep) elif len(status['files']) > 0: folder_structure = status['files'][0]['path'].split(os.sep) else: folder_structure = [] if len(folder_structure) > 1: log.verbose('Renaming Folder %s to %s', folder_structure[0], container_directory) client.call('core.rename_folder', torrent_id, folder_structure[0], container_directory) else: log.debug( 'container_directory specified however the torrent %s does not have a directory structure; ' 'skipping folder rename', entry['title'], )
def on_task_output(self, task, config): """Add torrents to deluge at exit.""" config = self.prepare_config(config) client = self.setup_client(config) # don't add when learning if task.options.learn: return if not config['enabled'] or not (task.accepted or task.options.test): return client.connect() if task.options.test: log.debug('Test connection to deluge daemon successful.') client.disconnect() return # loop through entries to get a list of labels to add labels = set() for entry in task.accepted: label = entry.get('label', config.get('label')) if label and label.lower() != 'no label': try: label = self._format_label( entry.render(entry.get('label', config.get('label')))) log.debug('Rendered label: %s', label) except RenderError as e: log.error('Error rendering label `%s`: %s', label, e) continue labels.add(label) if labels: # Make sure the label plugin is available and enabled, then add appropriate labels enabled_plugins = client.call('core.get_enabled_plugins') label_enabled = 'Label' in enabled_plugins if not label_enabled: available_plugins = client.call('core.get_available_plugins') if 'Label' in available_plugins: log.debug('Enabling label plugin in deluge') label_enabled = client.call('core.enable_plugin', 'Label') else: log.error('Label plugin is not installed in deluge') if label_enabled: d_labels = client.call('label.get_labels') for label in labels: if label not in d_labels: log.debug('Adding the label `%s` to deluge', label) client.call('label.add', label) # add the torrents torrent_ids = client.call('core.get_session_state') for entry in task.accepted: # Generate deluge options dict for torrent add add_opts = {} try: path = entry.render(entry.get('path', config['path'])) if path: add_opts['download_location'] = pathscrub( os.path.expanduser(path)) except RenderError as e: log.error('Could not set path for %s: %s', entry['title'], e) for fopt, dopt in self.options.items(): value = entry.get(fopt, config.get(fopt)) if value is not None: add_opts[dopt] = value if fopt == 'ratio': add_opts['stop_at_ratio'] = True # Make another set of options, that get set after the torrent has been added modify_opts = { 'queue_to_top': entry.get('queue_to_top', config.get('queue_to_top')), 'main_file_only': entry.get('main_file_only', config.get('main_file_only', False)), 'main_file_ratio': entry.get('main_file_ratio', config.get('main_file_ratio')), 'hide_sparse_files': entry.get('hide_sparse_files', config.get('hide_sparse_files', True)), 'keep_subs': entry.get('keep_subs', config.get('keep_subs', True)), 'container_directory': config.get('container_directory', ''), } try: label = entry.render(entry.get('label', config['label'])) modify_opts['label'] = self._format_label(label) except RenderError as e: log.error('Error setting label for `%s`: %s', entry['title'], e) try: move_completed_path = entry.render( entry.get('move_completed_path', config['move_completed_path'])) modify_opts['move_completed_path'] = pathscrub( os.path.expanduser(move_completed_path)) except RenderError as e: log.error('Error setting move_completed_path for %s: %s', entry['title'], e) try: content_filename = entry.get( 'content_filename', config.get('content_filename', '')) modify_opts['content_filename'] = pathscrub( entry.render(content_filename)) except RenderError as e: log.error('Error setting content_filename for %s: %s', entry['title'], e) torrent_id = entry.get('deluge_id') or entry.get( 'torrent_info_hash') torrent_id = torrent_id and torrent_id.lower() if torrent_id in torrent_ids: log.info('%s is already loaded in deluge, setting options', entry['title']) # Entry has a deluge id, verify the torrent is still in the deluge session and apply options # Since this is already loaded in deluge, we may also need to change the path modify_opts['path'] = add_opts.pop('download_location', None) client.call('core.set_torrent_options', [torrent_id], add_opts) self._set_torrent_options(client, torrent_id, entry, modify_opts) elif config['action'] != 'add': log.warning( 'Cannot %s %s, because it is not loaded in deluge.', config['action'], entry['title'], ) continue else: magnet, filedump = None, None if entry.get('url', '').startswith('magnet:'): magnet = entry['url'] else: if not os.path.exists(entry['file']): entry.fail( 'Downloaded temp file \'%s\' doesn\'t exist!' % entry['file']) del (entry['file']) return with open(entry['file'], 'rb') as f: filedump = base64.encodestring(f.read()) log.verbose('Adding %s to deluge.', entry['title']) added_torrent = None if magnet: added_torrent = client.call('core.add_torrent_magnet', magnet, add_opts) if config.get('magnetization_timeout'): timeout = config['magnetization_timeout'] log.verbose('Waiting %d seconds for "%s" to magnetize', timeout, entry['title']) for _ in range(timeout): time.sleep(1) try: status = client.call('core.get_torrent_status', torrent_id, ['files']) except Exception as err: log.error('wait_for_metadata Error: %s', err) break if status.get('files'): log.info('"%s" magnetization successful', entry['title']) break else: log.warning( '"%s" did not magnetize before the timeout elapsed, ' 'file list unavailable for processing.', entry['title'], ) else: try: added_torrent = client.call('core.add_torrent_file', entry['title'], filedump, add_opts) except Exception as e: log.info('%s was not added to deluge! %s', entry['title'], e) entry.fail('Could not be added to deluge') if not added_torrent: log.error('There was an error adding %s to deluge.' % entry['title']) else: self._set_torrent_options(client, added_torrent, entry, modify_opts) if config['action'] in ('remove', 'purge'): client.call('core.remove_torrent', torrent_id, config['action'] == 'purge') elif config['action'] == 'pause': client.call('core.pause_torrent', [torrent_id]) elif config['action'] == 'resume': client.call('core.resume_torrent', [torrent_id]) client.disconnect()
def add_to_transmission(self, cli, task, config): """Adds accepted entries to transmission """ for entry in task.accepted: if task.options.test: log.info("Would add %s to transmission" % entry["url"]) continue # Compile user options into appripriate dict options = self._make_torrent_options_dict(config, entry) downloaded = not entry["url"].startswith("magnet:") # Check that file is downloaded if downloaded and "file" not in entry: entry.fail("file missing?") continue # Verify the temp file exists if downloaded and not os.path.exists(entry["file"]): tmp_path = os.path.join(task.manager.config_base, "temp") log.debug("entry: %s", entry) log.debug("temp: %s", ", ".join(os.listdir(tmp_path))) entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry["file"]) continue try: if downloaded: with open(entry["file"], "rb") as f: filedump = base64.b64encode(f.read()).decode("utf-8") r = cli.add_torrent(filedump, 30, **options["add"]) else: # we need to set paused to false so the magnetization begins immediately options["add"]["paused"] = False r = cli.add_torrent(entry["url"], timeout=30, **options["add"]) log.info('"%s" torrent added to transmission', entry["title"]) total_size = cli.get_torrent(r.id, ["id", "totalSize"]).totalSize def _filter_list(list): for item in list: if not isinstance(item, basestring): list.remove(item) return list def _find_matches(name, list): for mask in list: if fnmatch(name, mask): return True return False def _wait_for_files(cli, r, timeout): from time import sleep while timeout > 0: sleep(1) fl = cli.get_files(r.id) if len(fl[r.id]) > 0: return fl else: timeout -= 1 return fl skip_files = False # Filter list because "set" plugin doesn't validate based on schema # Skip files only used if we have no main file if "skip_files" in options["post"]: skip_files = True options["post"]["skip_files"] = _filter_list(options["post"]["skip_files"]) main_id = None find_main_file = options["post"].get("main_file_only") or "content_filename" in options["post"] # We need to index the files if any of the following are defined if find_main_file or skip_files: fl = cli.get_files(r.id) if ( "magnetization_timeout" in options["post"] and options["post"]["magnetization_timeout"] > 0 and not downloaded and len(fl[r.id]) == 0 ): log.debug( 'Waiting %d seconds for "%s" to magnetize', options["post"]["magnetization_timeout"], entry["title"], ) fl = _wait_for_files(cli, r, options["post"]["magnetization_timeout"]) if len(fl[r.id]) == 0: log.warning( '"%s" did not magnetize before the timeout elapsed, ' "file list unavailable for processing.", entry["title"], ) else: total_size = cli.get_torrent(r.id, ["id", "totalSize"]).totalSize # Find files based on config dl_list = [] skip_list = [] main_list = [] full_list = [] ext_list = ["*.srt", "*.sub", "*.idx", "*.ssa", "*.ass"] main_ratio = config["main_file_ratio"] if "main_file_ratio" in options["post"]: main_ratio = options["post"]["main_file_ratio"] if "include_files" in options["post"]: options["post"]["include_files"] = _filter_list(options["post"]["include_files"]) for f in fl[r.id]: full_list.append(f) # No need to set main_id if we're not going to need it if find_main_file and fl[r.id][f]["size"] > total_size * main_ratio: main_id = f if "include_files" in options["post"]: if _find_matches(fl[r.id][f]["name"], options["post"]["include_files"]): dl_list.append(f) elif options["post"].get("include_subs") and _find_matches(fl[r.id][f]["name"], ext_list): dl_list.append(f) if skip_files: if _find_matches(fl[r.id][f]["name"], options["post"]["skip_files"]): skip_list.append(f) if main_id is not None: # Look for files matching main ID title but with a different extension if options["post"].get("rename_like_files"): for f in fl[r.id]: # if this filename matches main filename we want to rename it as well fs = os.path.splitext(fl[r.id][f]["name"]) if fs[0] == os.path.splitext(fl[r.id][main_id]["name"])[0]: main_list.append(f) else: main_list = [main_id] if main_id not in dl_list: dl_list.append(main_id) elif find_main_file: log.warning( 'No files in "%s" are > %d%% of content size, no files renamed.', entry["title"], main_ratio * 100, ) # If we have a main file and want to rename it and associated files if "content_filename" in options["post"] and main_id is not None: if "download_dir" not in options["add"]: download_dir = cli.get_session().download_dir else: download_dir = options["add"]["download_dir"] # Get new filename without ext file_ext = os.path.splitext(fl[r.id][main_id]["name"])[1] file_path = os.path.dirname(os.path.join(download_dir, fl[r.id][main_id]["name"])) filename = options["post"]["content_filename"] if config["host"] == "localhost" or config["host"] == "127.0.0.1": counter = 1 while os.path.exists(os.path.join(file_path, filename + file_ext)): # Try appending a (#) suffix till a unique filename is found filename = "%s(%s)" % (options["post"]["content_filename"], counter) counter += 1 else: log.debug( "Cannot ensure content_filename is unique " "when adding to a remote transmission daemon." ) for index in main_list: file_ext = os.path.splitext(fl[r.id][index]["name"])[1] log.debug("File %s renamed to %s" % (fl[r.id][index]["name"], filename + file_ext)) # change to below when set_files will allow setting name, more efficient to have one call # fl[r.id][index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8')) try: cli.rename_torrent_path( r.id, fl[r.id][index]["name"], os.path.basename(str(pathscrub(filename + file_ext))) ) except TransmissionError: log.error("content_filename only supported with transmission 2.8+") if options["post"].get("main_file_only") and main_id is not None: # Set Unwanted Files options["change"]["files_unwanted"] = [x for x in full_list if x not in dl_list] options["change"]["files_wanted"] = dl_list log.debug( "Downloading %s of %s files in torrent.", len(options["change"]["files_wanted"]), len(full_list), ) elif (not options["post"].get("main_file_only") or main_id is None) and skip_files: # If no main file and we want to skip files if len(skip_list) >= len(full_list): log.debug( "skip_files filter would cause no files to be downloaded; " "including all files in torrent." ) else: options["change"]["files_unwanted"] = skip_list options["change"]["files_wanted"] = [x for x in full_list if x not in skip_list] log.debug( "Downloading %s of %s files in torrent.", len(options["change"]["files_wanted"]), len(full_list), ) # Set any changed file properties if list(options["change"].keys()): cli.change_torrent(r.id, 30, **options["change"]) # if addpaused was defined and set to False start the torrent; # prevents downloading data before we set what files we want if ( "paused" in options["post"] and not options["post"]["paused"] or "paused" not in options["post"] and cli.get_session().start_added_torrents ): cli.start_torrent(r.id) elif options["post"].get("paused"): log.debug("sleeping 5s to stop the torrent...") time.sleep(5) cli.stop_torrent(r.id) log.info('Torrent "%s" stopped because of addpaused=yes', entry["title"]) except TransmissionError as e: log.debug("TransmissionError", exc_info=True) log.debug("Failed options dict: %s", options) msg = "TransmissionError: %s" % e.message or "N/A" log.error(msg) entry.fail(msg)
def on_task_output(self, task, config): if not config: return config = self.prepare_config(config) existing = config['existing'] for entry in task.accepted: if 'location' not in entry: entry.fail('Does not have location field for symlinking') continue linkfrom = entry['location'] linkfrom_path, linkfrom_name = os.path.split(linkfrom) # get the proper path and name in order of: entry, config, above split linkto_path = entry.get('link_to', config.get('to', linkfrom_path)) if config.get('rename'): linkto_name = config['rename'] elif entry.get('filename') and entry['filename'] != linkfrom_name: # entry specifies different filename than what was split from the path # since some inputs fill in filename it must be different in order to be used linkto_name = entry['filename'] else: linkto_name = linkfrom_name try: linkto_path = entry.render(linkto_path) except RenderError as err: raise plugin.PluginError( 'Path value replacement `%s` failed: %s' % (linkto_path, err.args[0])) try: linkto_name = entry.render(linkto_name) except RenderError as err: raise plugin.PluginError( 'Filename value replacement `%s` failed: %s' % (linkto_name, err.args[0])) # Clean invalid characters with pathscrub plugin linkto_path = pathscrub(os.path.expanduser(linkto_path)) linkto_name = pathscrub(linkto_name, filename=True) # Join path and filename linkto = os.path.join(linkto_path, linkto_name) if linkto == entry['location']: raise plugin.PluginWarning( 'source and destination are the same.') # Hardlinks for dirs will not be failed here if os.path.exists(linkto) and (config['link_type'] == 'soft' or os.path.isfile(linkfrom)): msg = 'Symlink destination %s already exists' % linkto if existing == 'ignore': logger.verbose(msg) else: entry.fail(msg) continue logger.verbose('{}link `{}` to `{}`', config['link_type'], linkfrom, linkto) try: if config['link_type'] == 'soft': os.symlink(linkfrom, linkto) else: if os.path.isdir(linkfrom): self.hard_link_dir(linkfrom, linkto, existing) else: dirname = os.path.dirname(linkto) if not os.path.exists(dirname): os.makedirs(dirname) os.link(linkfrom, linkto) except OSError as e: entry.fail('Failed to create %slink, %s' % (config['link_type'], e))
def on_get_torrent_status(status): """Gets called with torrent status, including file info. Sets the torrent options which require knowledge of the current status of the torrent.""" main_file_dlist = [] # Determine where the file should be move_now_path = None if opts.get('movedone'): if status['progress'] == 100: move_now_path = opts['movedone'] else: # Deluge will unset the move completed option if we move the storage, forgo setting proper # path, in favor of leaving proper final location. log.debug( 'Not moving storage for %s, as this will prevent movedone.' % entry['title']) elif opts.get('path'): move_now_path = opts['path'] if move_now_path and os.path.normpath( move_now_path) != os.path.normpath( status['save_path']): main_file_dlist.append( version_deferred.addCallback(create_path, move_now_path)) log.debug('Moving storage for %s to %s' % (entry['title'], move_now_path)) main_file_dlist.append( client.core.move_storage([torrent_id], move_now_path)) big_file_name = '' if opts.get('content_filename') or opts.get('main_file_only'): def file_exists(filename): # Checks the download path as well as the move completed path for existence of the file if os.path.exists( os.path.join(status['save_path'], filename)): return True elif status.get('move_on_completed') and status.get( 'move_on_completed_path'): if os.path.exists( os.path.join( status['move_on_completed_path'], filename)): return True else: return False def unused_name(name): # If on local computer, tries appending a (#) suffix until a unique filename is found if client.is_localhost(): counter = 2 while file_exists(name): name = ''.join([ os.path.splitext(name)[0], " (", str(counter), ')', os.path.splitext(name)[1] ]) counter += 1 else: log.debug( 'Cannot ensure content_filename is unique ' 'when adding to a remote deluge daemon.') return name def rename(file, new_name): # Renames a file in torrent main_file_dlist.append( client.core.rename_files( torrent_id, [(file['index'], new_name)])) log.debug('File %s in %s renamed to %s' % (file['path'], entry['title'], new_name)) # find a file that makes up more than main_file_ratio (default: 90%) of the total size main_file = None for file in status['files']: if file['size'] > (status['total_size'] * opts.get('main_file_ratio')): main_file = file break if main_file is not None: # proceed with renaming only if such a big file is found # find the subtitle file keep_subs = opts.get('keep_subs') sub_file = None if keep_subs: sub_exts = [".srt", ".sub"] for file in status['files']: ext = os.path.splitext(file['path'])[1] if ext in sub_exts: sub_file = file break # check for single file torrents so we dont add unnecessary folders if (os.path.dirname(main_file['path']) is not ("" or "/")): # check for top folder in user config if (opts.get('content_filename') and os.path.dirname( opts['content_filename']) is not ""): top_files_dir = os.path.dirname( opts['content_filename']) + "/" else: top_files_dir = os.path.dirname( main_file['path']) + "/" else: top_files_dir = "/" if opts.get('content_filename'): # rename the main file big_file_name = ( top_files_dir + os.path.basename(opts['content_filename']) + os.path.splitext(main_file['path'])[1]) big_file_name = unused_name(big_file_name) rename(main_file, big_file_name) # rename subs along with the main file if sub_file is not None and keep_subs: sub_file_name = ( os.path.splitext(big_file_name)[0] + os.path.splitext(sub_file['path'])[1]) rename(sub_file, sub_file_name) if opts.get('main_file_only'): # download only the main file (and subs) file_priorities = [ 1 if f == main_file or (f == sub_file and keep_subs) else 0 for f in status['files'] ] main_file_dlist.append( client.core.set_torrent_file_priorities( torrent_id, file_priorities)) if opts.get('hide_sparse_files'): # hide the other sparse files that are not supposed to download but are created anyway # http://dev.deluge-torrent.org/ticket/1827 # Made sparse files behave better with deluge http://flexget.com/ticket/2881 sparse_files = [ f for f in status['files'] if f != main_file and ( f != sub_file or (not keep_subs)) ] rename_pairs = [ (f['index'], top_files_dir + ".sparse_files/" + os.path.basename(f['path'])) for f in sparse_files ] main_file_dlist.append( client.core.rename_files( torrent_id, rename_pairs)) else: log.warning( 'No files in "%s" are > %d%% of content size, no files renamed.' % (entry['title'], opts.get('main_file_ratio') * 100)) container_directory = pathscrub( entry.render( entry.get('container_directory', config.get('container_directory', '')))) if container_directory: if big_file_name: folder_structure = big_file_name.split(os.sep) elif len(status['files']) > 0: folder_structure = status['files'][0]['path'].split( os.sep) else: folder_structure = [] if len(folder_structure) > 1: log.verbose('Renaming Folder %s to %s', folder_structure[0], container_directory) main_file_dlist.append( client.core.rename_folder(torrent_id, folder_structure[0], container_directory)) else: log.debug( 'container_directory specified however the torrent %s does not have a directory structure; skipping folder rename', entry['title']) return defer.DeferredList(main_file_dlist)
def handle_entry(self, task, config, entry, siblings): src = entry['location'] src_isdir = os.path.isdir(src) src_path, src_name = os.path.split(src) # get proper value in order of: entry, config, above split dst_path = entry.get('path', config.get('to', src_path)) dst_name = entry.get('filename', config.get('filename', src_name)) try: dst_path = entry.render(dst_path) except RenderError as err: raise plugin.PluginWarning('Path value replacement `%s` failed: %s' % (dst_path, err.args[0])) try: dst_name = entry.render(dst_name) except RenderError as err: raise plugin.PluginWarning('Filename value replacement `%s` failed: %s' % (dst_name, err.args[0])) # Clean invalid characters with pathscrub plugin dst_path = pathscrub(os.path.expanduser(dst_path)) dst_name = pathscrub(dst_name, filename=True) # Join path and filename dst = os.path.join(dst_path, dst_name) if dst == entry['location']: raise plugin.PluginWarning('source and destination are the same.') if not os.path.exists(dst_path): if task.options.test: self.log.info('Would create `%s`' % dst_path) else: self.log.info('Creating destination directory `%s`' % dst_path) os.makedirs(dst_path) if not os.path.isdir(dst_path) and not task.options.test: raise plugin.PluginWarning('destination `%s` is not a directory.' % dst_path) # unpack_safety if config.get('unpack_safety', entry.get('unpack_safety', True)): count = 0 while True: if count > 60 * 30: raise plugin.PluginWarning('The task has been waiting unpacking for 30 minutes') size = os.path.getsize(src) time.sleep(1) new_size = os.path.getsize(src) if size != new_size: if not count % 10: self.log.verbose('File `%s` is possibly being unpacked, waiting ...' % src_name) else: break count += 1 src_file, src_ext = os.path.splitext(src) dst_file, dst_ext = os.path.splitext(dst) # Check dst contains src_ext if dst_ext != src_ext: self.log.verbose('Adding extension `%s` to dst `%s`' % (src_ext, dst)) dst += src_ext funct_name = 'move' if self.move else 'copy' funct_done = 'moved' if self.move else 'copied' if task.options.test: self.log.info('Would %s `%s` to `%s`' % (funct_name, src, dst)) for s in siblings: # we cannot rely on splitext for extensions here (subtitles may have the language code) d = dst_file + s[len(src_file):] self.log.info('Would also %s `%s` to `%s`' % (funct_name, s, d)) else: # IO errors will have the entry mark failed in the base class if self.move: shutil.move(src, dst) elif src_isdir: shutil.copytree(src, dst) else: shutil.copy(src, dst) self.log.info('`%s` has been %s to `%s`' % (src, funct_done, dst)) # further errors will not have any effect (the entry has been successfully moved or copied out) for s in siblings: # we cannot rely on splitext for extensions here (subtitles may have the language code) d = dst_file + s[len(src_file):] try: if self.move: shutil.move(s, d) else: shutil.copy(s, d) self.log.info('`%s` has been %s to `%s` as well.' % (s, funct_done, d)) except Exception as err: self.log.warning(str(err)) entry['output'] = dst if self.move and not src_isdir: self.clean_source(task, config, entry)
def output(self, task, entry, config): """Moves temp-file into final destination Raises: PluginError if operation fails """ if 'file' not in entry and not task.options.test: log.debug('file missing, entry: %s' % entry) raise plugin.PluginError('Entry `%s` has no temp file associated with' % entry['title']) try: # use path from entry if has one, otherwise use from download definition parameter path = entry.get('path', config.get('path')) if not isinstance(path, basestring): raise plugin.PluginError('Invalid `path` in entry `%s`' % entry['title']) # override path from command line parameter if task.options.dl_path: path = task.options.dl_path # expand variables in path try: path = os.path.expanduser(entry.render(path)) except RenderError as e: entry.fail('Could not set path. Error during string replacement: %s' % e) return # Clean illegal characters from path name path = pathscrub(path) # If we are in test mode, report and return if task.options.test: log.info('Would write `%s` to `%s`' % (entry['title'], path)) # Set a fake location, so the exec plugin can do string replacement during --test #1015 entry['output'] = os.path.join(path, 'TEST_MODE_NO_OUTPUT') return # make path if not os.path.isdir(path): log.debug('Creating directory %s' % path) try: os.makedirs(path) except: raise plugin.PluginError('Cannot create path %s' % path, log) # check that temp file is present if not os.path.exists(entry['file']): log.debug('entry: %s' % entry) raise plugin.PluginWarning('Downloaded temp file `%s` doesn\'t exist!?' % entry['file']) # if we still don't have a filename, try making one from title (last resort) if not entry.get('filename'): entry['filename'] = entry['title'] log.debug('set filename from title %s' % entry['filename']) if 'mime-type' not in entry: log.warning('Unable to figure proper filename for %s. Using title.' % entry['title']) else: guess = mimetypes.guess_extension(entry['mime-type']) if not guess: log.warning('Unable to guess extension with mime-type %s' % guess) else: self.filename_ext_from_mime(entry) name = entry.get('filename', entry['title']) # Remove illegal characters from filename #325, #353 name = pathscrub(name) # Remove directory separators from filename #208 name = name.replace('/', ' ') if sys.platform.startswith('win'): name = name.replace('\\', ' ') # remove duplicate spaces name = ' '.join(name.split()) # combine to full path + filename destfile = os.path.join(path, name) log.debug('destfile: %s' % destfile) if os.path.exists(destfile): import filecmp if filecmp.cmp(entry['file'], destfile): log.debug("Identical destination file '%s' already exists", destfile) elif config.get('overwrite'): log.debug("Overwriting already existing file %s" % destfile) else: log.info('File `%s` already exists and is not identical, download failed.' % destfile) entry.fail('File `%s` already exists and is not identical.' % destfile) return else: # move temp file log.debug('moving %s to %s' % (entry['file'], destfile)) try: shutil.move(entry['file'], destfile) except OSError as err: # ignore permission errors, see ticket #555 import errno if not os.path.exists(destfile): raise plugin.PluginError('Unable to write %s' % destfile) if err.errno != errno.EPERM: raise # store final destination as output key entry['output'] = destfile finally: self.cleanup_temp_file(entry)
def add_to_transmission(self, cli, task, config): """Adds accepted entries to transmission """ from transmissionrpc import TransmissionError for entry in task.accepted: if task.options.test: log.info('Would add %s to transmission' % entry['url']) continue # Compile user options into appripriate dict options = self._make_torrent_options_dict(config, entry) downloaded = not entry['url'].startswith('magnet:') # Check that file is downloaded if downloaded and 'file' not in entry: entry.fail('file missing?') continue # Verify the temp file exists if downloaded and not os.path.exists(entry['file']): tmp_path = os.path.join(task.manager.config_base, 'temp') log.debug('entry: %s' % entry) log.debug('temp: %s' % ', '.join(os.listdir(tmp_path))) entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry['file']) continue try: if downloaded: with open(entry['file'], 'rb') as f: filedump = base64.b64encode(f.read()).encode('utf-8') r = cli.add_torrent(filedump, 30, **options['add']) else: r = cli.add_torrent(entry['url'], timeout=30, **options['add']) if r: torrent = r log.info('"%s" torrent added to transmission' % (entry['title'])) total_size = cli.get_torrent(r.id, ['id', 'totalSize']).totalSize def _filter_list(list): for item in list: if not isinstance(item, basestring): list.remove(item) return list def _find_matches(name, list): for mask in list: if fnmatch(name, mask): return True return False skip_files = False # Filter list because "set" plugin doesn't validate based on schema # Skip files only used if we have no main file if 'skip_files' in options['post']: skip_files = True options['post']['skip_files'] = _filter_list(options['post']['skip_files']) main_id = None # We need to index the files if any of the following are defined if ('main_file_only' in options['post'] and options['post']['main_file_only'] == True or 'content_filename' in options['post'] or skip_files): fl = cli.get_files(r.id) # Find files based on config dl_list = [] skip_list = [] main_list = [] full_list = [] ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass'] if 'include_files' in options['post']: include_files = True options['post']['include_files'] = _filter_list(options['post']['include_files']) for f in fl[r.id]: full_list.append(f) if fl[r.id][f]['size'] > total_size * 0.90: main_id = f if 'include_files' in options['post']: if _find_matches(fl[r.id][f]['name'], options['post']['include_files']): dl_list.append(f) elif ('include_subs' in options['post'] and options['post']['include_subs'] == True and _find_matches(fl[r.id][f]['name'], ext_list)): dl_list.append(f) if skip_files: if _find_matches(fl[r.id][f]['name'], options['post']['skip_files']): skip_list.append(f) if main_id is not None: # Look for files matching main ID title but with a different extension if 'rename_like_files' in options['post'] and options['post']['rename_like_files'] == True: for f in fl[r.id]: # if this filename matches main filename we want to rename it as well fs = os.path.splitext(fl[r.id][f]['name']) if fs[0] == os.path.splitext(fl[r.id][main_id]['name'])[0]: main_list.append(f) else: main_list = [main_id] if main_id not in dl_list: dl_list.append(main_id) # If we have a main file and want to rename it and associated files if 'content_filename' in options['post'] and main_id is not None: download_dir = "" if 'download_dir' not in options['add']: download_dir = cli.get_session().download_dir else: download_dir = options['add']['download_dir'] # Get new filename without ext file_ext = os.path.splitext(fl[r.id][main_id]['name'])[1] file_path = os.path.dirname(os.path.join(download_dir, fl[r.id][main_id]['name'])) filename = options['post']['content_filename'] if config['host'] == 'localhost' or config['host'] == '127.0.0.1': counter = 1 while os.path.exists(os.path.join(file_path, filename + file_ext)): # Try appending a (#) suffix till a unique filename is found filename = ''.join(options['post']['content_filename'], '(', str(counter), ')') counter += 1 else: log.debug('Cannot ensure content_filename is unique ' 'when adding to a remote transmission daemon.') for index in main_list: file_ext = os.path.splitext(fl[r.id][index]['name'])[1] log.debug('File %s renamed to %s' % (fl[r.id][index]['name'], filename + file_ext)) # change to below when set_files will allow setting name, more efficient to have one call # fl[r.id][index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8')) cli.rename_torrent_path(r.id, fl[r.id][index]['name'], os.path.basename( pathscrub(filename + file_ext).encode('utf-8')) ) if ('main_file_only' in options['post'] and options['post']['main_file_only'] == True and main_id is not None): # Set Unwanted Files options['change']['files_unwanted'] = [x for x in full_list if x not in dl_list] options['change']['files_wanted'] = dl_list log.debug('Downloading %s of %s files in torrent.' % (len(options['change']['files_wanted']), len(full_list))) elif(('main_file_only' not in options['post'] or options['post']['main_file_only'] == False or main_id is None) and skip_files): # If no main file and we want to skip files if len(skip_list) >= len(full_list): log.debug('skip_files filter would cause no files to be downloaded; ' 'including all files in torrent.') else: options['change']['files_unwanted'] = skip_list options['change']['files_wanted'] = [x for x in full_list if x not in skip_list] log.debug('Downloading %s of %s files in torrent.' % (len(options['change']['files_wanted']), len(full_list))) # Set any changed file properties if options['change'].keys(): cli.change_torrent(r.id, 30, **options['change']) # if addpaused was defined and set to False start the torrent; # prevents downloading data before we set what files we want if ('paused' in options['post'] and options['post']['paused'] == False or 'paused' not in options['post'] and cli.get_session().start_added_torrents == True): cli.start_torrent(r.id) except TransmissionError as e: log.debug('TransmissionError', exc_info=True) log.debug('Failed options dict: %s' % options) msg = 'TransmissionError: %s' % e.message or 'N/A' log.error(msg) entry.fail(msg)
def _set_torrent_options(self, client, torrent_id, entry, opts): """Gets called when a torrent was added to the daemon.""" log.info('%s successfully added to deluge.', entry['title']) entry['deluge_id'] = torrent_id if opts.get('move_completed_path'): client.call('core.set_torrent_move_completed', torrent_id, True) client.call( 'core.set_torrent_move_completed_path', torrent_id, opts['move_completed_path'] ) log.debug('%s move on complete set to %s', entry['title'], opts['move_completed_path']) if opts.get('label'): client.call('label.set_torrent', torrent_id, opts['label']) if opts.get('queue_to_top') is not None: if opts['queue_to_top']: client.call('core.queue_top', [torrent_id]) log.debug('%s moved to top of queue', entry['title']) else: client.call('core.queue_bottom', [torrent_id]) log.debug('%s moved to bottom of queue', entry['title']) status_keys = [ 'files', 'total_size', 'save_path', 'move_on_completed_path', 'move_on_completed', 'progress', ] status = client.call('core.get_torrent_status', torrent_id, status_keys) # Determine where the file should be move_now_path = None if opts.get('move_completed_path'): if status['progress'] == 100: move_now_path = opts['move_completed_path'] else: # Deluge will unset the move completed option if we move the storage, forgo setting proper # path, in favor of leaving proper final location. log.debug( 'Not moving storage for %s, as this will prevent move_completed_path.', entry['title'], ) elif opts.get('path'): move_now_path = opts['path'] if move_now_path and os.path.normpath(move_now_path) != os.path.normpath( status['save_path'] ): log.debug('Moving storage for %s to %s', entry['title'], move_now_path) client.call('core.move_storage', [torrent_id], move_now_path) big_file_name = '' if opts.get('content_filename') or opts.get('main_file_only'): # find a file that makes up more than main_file_ratio (default: 90%) of the total size main_file = None for file in status['files']: if file['size'] > (status['total_size'] * opts.get('main_file_ratio')): main_file = file break def file_exists(filename): # Checks the download path as well as the move completed path for existence of the file if os.path.exists(os.path.join(status['save_path'], filename)): return True elif status.get('move_on_completed') and status.get('move_on_completed_path'): if os.path.exists(os.path.join(status['move_on_completed_path'], filename)): return True else: return False def unused_name(name): # If on local computer, tries appending a (#) suffix until a unique filename is found if client.host in ['127.0.0.1', 'localhost']: counter = 2 while file_exists(name): name = ''.join( [ os.path.splitext(name)[0], " (", str(counter), ')', os.path.splitext(name)[1], ] ) counter += 1 else: log.debug( 'Cannot ensure content_filename is unique when adding to a remote deluge daemon.' ) return name def rename(file, new_name): # Renames a file in torrent client.call('core.rename_files', torrent_id, [(file['index'], new_name)]) log.debug('File %s in %s renamed to %s', file['path'], entry['title'], new_name) if main_file is not None: # proceed with renaming only if such a big file is found # find the subtitle file keep_subs = opts.get('keep_subs') sub_file = None if keep_subs: sub_exts = [".srt", ".sub"] for file in status['files']: ext = os.path.splitext(file['path'])[1] if ext in sub_exts: sub_file = file break # check for single file torrents so we dont add unnecessary folders top_files_dir = "/" if os.path.dirname(main_file['path']) is not ("" or "/"): # check for top folder in user config if ( opts.get('content_filename') and os.path.dirname(opts['content_filename']) is not "" ): top_files_dir = os.path.dirname(opts['content_filename']) + "/" else: top_files_dir = os.path.dirname(main_file['path']) + "/" if opts.get('content_filename'): # rename the main file big_file_name = ( top_files_dir + os.path.basename(opts['content_filename']) + os.path.splitext(main_file['path'])[1] ) big_file_name = unused_name(big_file_name) rename(main_file, big_file_name) # rename subs along with the main file if sub_file is not None and keep_subs: sub_file_name = ( os.path.splitext(big_file_name)[0] + os.path.splitext(sub_file['path'])[1] ) rename(sub_file, sub_file_name) if opts.get('main_file_only'): # download only the main file (and subs) file_priorities = [ 1 if f == main_file or f == sub_file and keep_subs else 0 for f in status['files'] ] client.call('core.set_torrent_file_priorities', torrent_id, file_priorities) if opts.get('hide_sparse_files'): # hide the other sparse files that are not supposed to download but are created anyway # http://dev.deluge-torrent.org/ticket/1827 # Made sparse files behave better with deluge http://flexget.com/ticket/2881 sparse_files = [ f for f in status['files'] if f != main_file and (f != sub_file or not keep_subs) ] rename_pairs = [ ( f['index'], top_files_dir + ".sparse_files/" + os.path.basename(f['path']), ) for f in sparse_files ] client.call('core.rename_files', torrent_id, rename_pairs) else: log.warning( 'No files in "%s" are > %d%% of content size, no files renamed.', entry['title'], opts.get('main_file_ratio') * 100, ) container_directory = pathscrub( entry.render(entry.get('container_directory', opts.get('container_directory', ''))) ) if container_directory: if big_file_name: folder_structure = big_file_name.split(os.sep) elif len(status['files']) > 0: folder_structure = status['files'][0]['path'].split(os.sep) else: folder_structure = [] if len(folder_structure) > 1: log.verbose('Renaming Folder %s to %s', folder_structure[0], container_directory) client.call( 'core.rename_folder', torrent_id, folder_structure[0], container_directory ) else: log.debug( 'container_directory specified however the torrent %s does not have a directory structure; ' 'skipping folder rename', entry['title'], )
def on_task_output(self, task, config): """Add torrents to deluge at exit.""" config = self.prepare_config(config) client = self.setup_client(config) # don't add when learning if task.options.learn: return if not config['enabled'] or not (task.accepted or task.options.test): return client.connect() if task.options.test: log.debug('Test connection to deluge daemon successful.') client.disconnect() return # loop through entries to get a list of labels to add labels = set() for entry in task.accepted: label = entry.get('label', config.get('label')) if label and label.lower() != 'no label': try: label = self._format_label( entry.render(entry.get('label', config.get('label'))) ) log.debug('Rendered label: %s', label) except RenderError as e: log.error('Error rendering label `%s`: %s', label, e) continue labels.add(label) if labels: # Make sure the label plugin is available and enabled, then add appropriate labels enabled_plugins = client.call('core.get_enabled_plugins') label_enabled = 'Label' in enabled_plugins if not label_enabled: available_plugins = client.call('core.get_available_plugins') if 'Label' in available_plugins: log.debug('Enabling label plugin in deluge') label_enabled = client.call('core.enable_plugin', 'Label') else: log.error('Label plugin is not installed in deluge') if label_enabled: d_labels = client.call('label.get_labels') for label in labels: if label not in d_labels: log.debug('Adding the label `%s` to deluge', label) client.call('label.add', label) # add the torrents torrent_ids = client.call('core.get_session_state') for entry in task.accepted: # Generate deluge options dict for torrent add add_opts = {} try: path = entry.render(entry.get('path', config['path'])) if path: add_opts['download_location'] = pathscrub(os.path.expanduser(path)) except RenderError as e: log.error('Could not set path for %s: %s', entry['title'], e) for fopt, dopt in self.options.items(): value = entry.get(fopt, config.get(fopt)) if value is not None: add_opts[dopt] = value if fopt == 'ratio': add_opts['stop_at_ratio'] = True # Make another set of options, that get set after the torrent has been added modify_opts = { 'queue_to_top': entry.get('queue_to_top', config.get('queue_to_top')), 'main_file_only': entry.get('main_file_only', config.get('main_file_only', False)), 'main_file_ratio': entry.get('main_file_ratio', config.get('main_file_ratio')), 'hide_sparse_files': entry.get( 'hide_sparse_files', config.get('hide_sparse_files', True) ), 'keep_subs': entry.get('keep_subs', config.get('keep_subs', True)), 'container_directory': config.get('container_directory', ''), } try: label = entry.render(entry.get('label', config['label'])) modify_opts['label'] = self._format_label(label) except RenderError as e: log.error('Error setting label for `%s`: %s', entry['title'], e) try: move_completed_path = entry.render( entry.get('move_completed_path', config['move_completed_path']) ) modify_opts['move_completed_path'] = pathscrub( os.path.expanduser(move_completed_path) ) except RenderError as e: log.error('Error setting move_completed_path for %s: %s', entry['title'], e) try: content_filename = entry.get( 'content_filename', config.get('content_filename', '') ) modify_opts['content_filename'] = pathscrub(entry.render(content_filename)) except RenderError as e: log.error('Error setting content_filename for %s: %s', entry['title'], e) torrent_id = entry.get('deluge_id') or entry.get('torrent_info_hash') torrent_id = torrent_id and torrent_id.lower() if torrent_id in torrent_ids: log.info('%s is already loaded in deluge, setting options', entry['title']) # Entry has a deluge id, verify the torrent is still in the deluge session and apply options # Since this is already loaded in deluge, we may also need to change the path modify_opts['path'] = add_opts.pop('download_location', None) client.call('core.set_torrent_options', [torrent_id], add_opts) self._set_torrent_options(client, torrent_id, entry, modify_opts) elif config['action'] != 'add': log.warning( 'Cannot %s %s, because it is not loaded in deluge.', config['action'], entry['title'], ) continue else: magnet, filedump = None, None if entry.get('url', '').startswith('magnet:'): magnet = entry['url'] else: if not os.path.exists(entry['file']): entry.fail('Downloaded temp file \'%s\' doesn\'t exist!' % entry['file']) del (entry['file']) return with open(entry['file'], 'rb') as f: filedump = base64.encodestring(f.read()) log.verbose('Adding %s to deluge.', entry['title']) added_torrent = None if magnet: added_torrent = client.call('core.add_torrent_magnet', magnet, add_opts) if config.get('magnetization_timeout'): timeout = config['magnetization_timeout'] log.verbose( 'Waiting %d seconds for "%s" to magnetize', timeout, entry['title'] ) for _ in range(timeout): time.sleep(1) try: status = client.call( 'core.get_torrent_status', torrent_id, ['files'] ) except Exception as err: log.error('wait_for_metadata Error: %s', err) break if status.get('files'): log.info('"%s" magnetization successful', entry['title']) break else: log.warning( '"%s" did not magnetize before the timeout elapsed, ' 'file list unavailable for processing.', entry['title'], ) else: try: added_torrent = client.call( 'core.add_torrent_file', entry['title'], filedump, add_opts ) except Exception as e: log.info('%s was not added to deluge! %s', entry['title'], e) entry.fail('Could not be added to deluge') if not added_torrent: log.error('There was an error adding %s to deluge.' % entry['title']) else: self._set_torrent_options(client, added_torrent, entry, modify_opts) if config['action'] in ('remove', 'purge'): client.call('core.remove_torrent', torrent_id, config['action'] == 'purge') elif config['action'] == 'pause': client.call('core.pause_torrent', [torrent_id]) elif config['action'] == 'resume': client.call('core.resume_torrent', [torrent_id]) client.disconnect()
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ( "path", "addpaused", "honourlimits", "bandwidthpriority", "maxconnections", "maxupspeed", "maxdownspeed", "ratio", "main_file_only", "main_file_ratio", "magnetization_timeout", "include_subs", "content_filename", "include_files", "skip_files", "rename_like_files", "queue_position", ): # Values do not merge config with task # Task takes priority then config is used if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {"add": {}, "change": {}, "post": {}} add = options["add"] if opt_dic.get("path"): try: path = os.path.expanduser(entry.render(opt_dic["path"])) add["download_dir"] = text_to_native_str(pathscrub(path), "utf-8") except RenderError as e: log.error("Error setting path for %s: %s" % (entry["title"], e)) if "bandwidthpriority" in opt_dic: add["bandwidthPriority"] = opt_dic["bandwidthpriority"] if "maxconnections" in opt_dic: add["peer_limit"] = opt_dic["maxconnections"] # make sure we add it paused, will modify status after adding add["paused"] = True change = options["change"] if "honourlimits" in opt_dic and not opt_dic["honourlimits"]: change["honorsSessionLimits"] = False if "maxupspeed" in opt_dic: change["uploadLimit"] = opt_dic["maxupspeed"] change["uploadLimited"] = True if "maxdownspeed" in opt_dic: change["downloadLimit"] = opt_dic["maxdownspeed"] change["downloadLimited"] = True if "ratio" in opt_dic: change["seedRatioLimit"] = opt_dic["ratio"] if opt_dic["ratio"] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio change["seedRatioMode"] = 2 else: change["seedRatioMode"] = 1 if "queue_position" in opt_dic: change["queuePosition"] = opt_dic["queue_position"] post = options["post"] # set to modify paused status after if "addpaused" in opt_dic: post["paused"] = opt_dic["addpaused"] if "main_file_only" in opt_dic: post["main_file_only"] = opt_dic["main_file_only"] if "main_file_ratio" in opt_dic: post["main_file_ratio"] = opt_dic["main_file_ratio"] if "magnetization_timeout" in opt_dic: post["magnetization_timeout"] = opt_dic["magnetization_timeout"] if "include_subs" in opt_dic: post["include_subs"] = opt_dic["include_subs"] if "content_filename" in opt_dic: try: post["content_filename"] = entry.render(opt_dic["content_filename"]) except RenderError as e: log.error("Unable to render content_filename %s: %s" % (entry["title"], e)) if "skip_files" in opt_dic: post["skip_files"] = opt_dic["skip_files"] if "include_files" in opt_dic: post["include_files"] = opt_dic["include_files"] if "rename_like_files" in opt_dic: post["rename_like_files"] = opt_dic["rename_like_files"] return options
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority', 'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio', 'main_file_only', 'main_file_ratio', 'magnetization_timeout', 'include_subs', 'content_filename', 'include_files', 'skip_files', 'rename_like_files', 'queue_position'): # Values do not merge config with task # Task takes priority then config is used if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {'add': {}, 'change': {}, 'post': {}} add = options['add'] if opt_dic.get('path'): try: path = os.path.expanduser(entry.render(opt_dic['path'])) add['download_dir'] = text_to_native_str( pathscrub(path), 'utf-8') except RenderError as e: log.error('Error setting path for %s: %s' % (entry['title'], e)) if 'bandwidthpriority' in opt_dic: add['bandwidthPriority'] = opt_dic['bandwidthpriority'] if 'maxconnections' in opt_dic: add['peer_limit'] = opt_dic['maxconnections'] # make sure we add it paused, will modify status after adding add['paused'] = True change = options['change'] if 'honourlimits' in opt_dic and not opt_dic['honourlimits']: change['honorsSessionLimits'] = False if 'maxupspeed' in opt_dic: change['uploadLimit'] = opt_dic['maxupspeed'] change['uploadLimited'] = True if 'maxdownspeed' in opt_dic: change['downloadLimit'] = opt_dic['maxdownspeed'] change['downloadLimited'] = True if 'ratio' in opt_dic: change['seedRatioLimit'] = opt_dic['ratio'] if opt_dic['ratio'] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio change['seedRatioMode'] = 2 else: change['seedRatioMode'] = 1 if 'queue_position' in opt_dic: change['queuePosition'] = opt_dic['queue_position'] post = options['post'] # set to modify paused status after if 'addpaused' in opt_dic: post['paused'] = opt_dic['addpaused'] if 'main_file_only' in opt_dic: post['main_file_only'] = opt_dic['main_file_only'] if 'main_file_ratio' in opt_dic: post['main_file_ratio'] = opt_dic['main_file_ratio'] if 'magnetization_timeout' in opt_dic: post['magnetization_timeout'] = opt_dic['magnetization_timeout'] if 'include_subs' in opt_dic: post['include_subs'] = opt_dic['include_subs'] if 'content_filename' in opt_dic: try: post['content_filename'] = entry.render( opt_dic['content_filename']) except RenderError as e: log.error('Unable to render content_filename %s: %s' % (entry['title'], e)) if 'skip_files' in opt_dic: post['skip_files'] = opt_dic['skip_files'] if 'include_files' in opt_dic: post['include_files'] = opt_dic['include_files'] if 'rename_like_files' in opt_dic: post['rename_like_files'] = opt_dic['rename_like_files'] return options
def handle_entry(self, task, config, entry, siblings): src = entry['location'] src_isdir = os.path.isdir(src) src_path, src_name = os.path.split(src) # get the proper path and name in order of: entry, config, above split dst_path = entry.get(self.destination_field, config.get('to', src_path)) if config.get('rename'): dst_name = config['rename'] elif entry.get('filename') and entry['filename'] != src_name: # entry specifies different filename than what was split from the path # since some inputs fill in filename it must be different in order to be used dst_name = entry['filename'] else: dst_name = src_name try: dst_path = entry.render(dst_path) except RenderError as err: raise plugin.PluginError('Path value replacement `%s` failed: %s' % (dst_path, err.args[0])) try: dst_name = entry.render(dst_name) except RenderError as err: raise plugin.PluginError( 'Filename value replacement `%s` failed: %s' % (dst_name, err.args[0])) # Clean invalid characters with pathscrub plugin dst_path = pathscrub(os.path.expanduser(dst_path)) dst_name = pathscrub(dst_name, filename=True) # Join path and filename dst = os.path.join(dst_path, dst_name) if dst == entry['location']: raise plugin.PluginWarning('source and destination are the same.') if not os.path.exists(dst_path): if task.options.test: self.log.info('Would create `%s`', dst_path) else: self.log.info('Creating destination directory `%s`', dst_path) os.makedirs(dst_path) if not os.path.isdir(dst_path) and not task.options.test: raise plugin.PluginWarning('destination `%s` is not a directory.' % dst_path) # unpack_safety if config.get('unpack_safety', entry.get('unpack_safety', True)): count = 0 while True: if count > 60 * 30: raise plugin.PluginWarning( 'The task has been waiting unpacking for 30 minutes') size = os.path.getsize(src) time.sleep(1) new_size = os.path.getsize(src) if size != new_size: if not count % 10: self.log.verbose( 'File `%s` is possibly being unpacked, waiting ...', src_name) else: break count += 1 src_file, src_ext = os.path.splitext(src) dst_file, dst_ext = os.path.splitext(dst) # Check dst contains src_ext if config.get('keep_extension', entry.get('keep_extension', True)): if not src_isdir and dst_ext != src_ext: self.log.verbose('Adding extension `%s` to dst `%s`', src_ext, dst) dst += src_ext dst_file += dst_ext # this is used for sibling files. dst_ext turns out not to be an extension! funct_name = 'move' if self.move else 'copy' funct_done = 'moved' if self.move else 'copied' if task.options.test: self.log.info('Would %s `%s` to `%s`', funct_name, src, dst) for s, ext in siblings.items(): # we cannot rely on splitext for extensions here (subtitles may have the language code) d = dst_file + ext self.log.info('Would also %s `%s` to `%s`', funct_name, s, d) else: # IO errors will have the entry mark failed in the base class if self.move: shutil.move(src, dst) elif src_isdir: shutil.copytree(src, dst) else: shutil.copy(src, dst) self.log.info('`%s` has been %s to `%s`', src, funct_done, dst) # further errors will not have any effect (the entry has been successfully moved or copied out) for s, ext in siblings.items(): # we cannot rely on splitext for extensions here (subtitles may have the language code) d = dst_file + ext try: if self.move: shutil.move(s, d) else: shutil.copy(s, d) self.log.info('`%s` has been %s to `%s` as well.', s, funct_done, d) except Exception as err: self.log.warning(str(err)) entry['old_location'] = entry['location'] entry['location'] = dst if self.move and not src_isdir: self.clean_source(task, config, entry)
def add_to_transmission(self, cli, task, config): """Adds accepted entries to transmission """ for entry in task.accepted: if task.options.test: log.info('Would add %s to transmission' % entry['url']) continue # Compile user options into appripriate dict options = self._make_torrent_options_dict(config, entry) downloaded = not entry['url'].startswith('magnet:') # Check that file is downloaded if downloaded and 'file' not in entry: entry.fail('file missing?') continue # Verify the temp file exists if downloaded and not os.path.exists(entry['file']): tmp_path = os.path.join(task.manager.config_base, 'temp') log.debug('entry: %s', entry) log.debug('temp: %s', ', '.join(os.listdir(tmp_path))) entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry['file']) continue try: if downloaded: with open(entry['file'], 'rb') as f: filedump = base64.b64encode(f.read()).decode('utf-8') r = cli.add_torrent(filedump, 30, **options['add']) else: # we need to set paused to false so the magnetization begins immediately options['add']['paused'] = False r = cli.add_torrent(entry['url'], timeout=30, **options['add']) log.info('"%s" torrent added to transmission', entry['title']) total_size = cli.get_torrent(r.id, ['id', 'totalSize']).totalSize def _filter_list(list): for item in list: if not isinstance(item, basestring): list.remove(item) return list def _find_matches(name, list): for mask in list: if fnmatch(name, mask): return True return False def _wait_for_files(cli, r, timeout): from time import sleep while timeout > 0: sleep(1) fl = cli.get_files(r.id) if len(fl[r.id]) > 0: return fl else: timeout -= 1 return fl skip_files = False # Filter list because "set" plugin doesn't validate based on schema # Skip files only used if we have no main file if 'skip_files' in options['post']: skip_files = True options['post']['skip_files'] = _filter_list( options['post']['skip_files']) main_id = None find_main_file = options['post'].get( 'main_file_only') or 'content_filename' in options['post'] # We need to index the files if any of the following are defined if find_main_file or skip_files: fl = cli.get_files(r.id) if ('magnetization_timeout' in options['post'] and options['post']['magnetization_timeout'] > 0 and not downloaded and len(fl[r.id]) == 0): log.debug('Waiting %d seconds for "%s" to magnetize', options['post']['magnetization_timeout'], entry['title']) fl = _wait_for_files( cli, r, options['post']['magnetization_timeout']) if len(fl[r.id]) == 0: log.warning( '"%s" did not magnetize before the timeout elapsed, ' 'file list unavailable for processing.', entry['title']) else: total_size = cli.get_torrent( r.id, ['id', 'totalSize']).totalSize # Find files based on config dl_list = [] skip_list = [] main_list = [] full_list = [] ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass'] main_ratio = config['main_file_ratio'] if 'main_file_ratio' in options['post']: main_ratio = options['post']['main_file_ratio'] if 'include_files' in options['post']: options['post']['include_files'] = _filter_list( options['post']['include_files']) for f in fl[r.id]: full_list.append(f) # No need to set main_id if we're not going to need it if find_main_file and fl[ r.id][f]['size'] > total_size * main_ratio: main_id = f if 'include_files' in options['post']: if _find_matches(fl[r.id][f]['name'], options['post']['include_files']): dl_list.append(f) elif options['post'].get( 'include_subs') and _find_matches( fl[r.id][f]['name'], ext_list): dl_list.append(f) if skip_files: if _find_matches(fl[r.id][f]['name'], options['post']['skip_files']): skip_list.append(f) if main_id is not None: # Look for files matching main ID title but with a different extension if options['post'].get('rename_like_files'): for f in fl[r.id]: # if this filename matches main filename we want to rename it as well fs = os.path.splitext(fl[r.id][f]['name']) if fs[0] == os.path.splitext( fl[r.id][main_id]['name'])[0]: main_list.append(f) else: main_list = [main_id] if main_id not in dl_list: dl_list.append(main_id) elif find_main_file: log.warning( 'No files in "%s" are > %d%% of content size, no files renamed.', entry['title'], main_ratio * 100) # If we have a main file and want to rename it and associated files if 'content_filename' in options[ 'post'] and main_id is not None: if 'download_dir' not in options['add']: download_dir = cli.get_session().download_dir else: download_dir = options['add']['download_dir'] # Get new filename without ext file_ext = os.path.splitext( fl[r.id][main_id]['name'])[1] file_path = os.path.dirname( os.path.join(download_dir, fl[r.id][main_id]['name'])) filename = options['post']['content_filename'] if config['host'] == 'localhost' or config[ 'host'] == '127.0.0.1': counter = 1 while os.path.exists( os.path.join(file_path, filename + file_ext)): # Try appending a (#) suffix till a unique filename is found filename = '%s(%s)' % ( options['post']['content_filename'], counter) counter += 1 else: log.debug( 'Cannot ensure content_filename is unique ' 'when adding to a remote transmission daemon.') for index in main_list: file_ext = os.path.splitext( fl[r.id][index]['name'])[1] log.debug( 'File %s renamed to %s' % (fl[r.id][index]['name'], filename + file_ext)) # change to below when set_files will allow setting name, more efficient to have one call # fl[r.id][index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8')) try: cli.rename_torrent_path( r.id, fl[r.id][index]['name'], os.path.basename( str(pathscrub(filename + file_ext)))) except TransmissionError: log.error( 'content_filename only supported with transmission 2.8+' ) if options['post'].get( 'main_file_only') and main_id is not None: # Set Unwanted Files options['change']['files_unwanted'] = [ x for x in full_list if x not in dl_list ] options['change']['files_wanted'] = dl_list log.debug('Downloading %s of %s files in torrent.', len(options['change']['files_wanted']), len(full_list)) elif (not options['post'].get('main_file_only') or main_id is None) and skip_files: # If no main file and we want to skip files if len(skip_list) >= len(full_list): log.debug( 'skip_files filter would cause no files to be downloaded; ' 'including all files in torrent.') else: options['change']['files_unwanted'] = skip_list options['change']['files_wanted'] = [ x for x in full_list if x not in skip_list ] log.debug('Downloading %s of %s files in torrent.', len(options['change']['files_wanted']), len(full_list)) # Set any changed file properties if list(options['change'].keys()): cli.change_torrent(r.id, 30, **options['change']) # if addpaused was defined and set to False start the torrent; # prevents downloading data before we set what files we want if ('paused' in options['post'] and not options['post']['paused'] or 'paused' not in options['post'] and cli.get_session().start_added_torrents): cli.start_torrent(r.id) elif options['post'].get('paused'): log.debug('sleeping 5s to stop the torrent...') time.sleep(5) cli.stop_torrent(r.id) log.info('Torrent "%s" stopped because of addpaused=yes', entry['title']) except TransmissionError as e: log.debug('TransmissionError', exc_info=True) log.debug('Failed options dict: %s', options) msg = 'TransmissionError: %s' % e.message or 'N/A' log.error(msg) entry.fail(msg)
def on_get_session_state(torrent_ids): """Gets called with a list of torrent_ids loaded in the deluge session. Adds new torrents and modifies the settings for ones already in the session.""" dlist = [] # add the torrents for entry in task.accepted: @defer.inlineCallbacks def _wait_for_metadata(torrent_id, timeout): log.verbose('Waiting %d seconds for "%s" to magnetize' % (timeout, entry['title'])) for _ in range(timeout): time.sleep(1) try: status = yield client.core.get_torrent_status( torrent_id, ['files']) except Exception as err: log.error('wait_for_metadata Error: %s' % err) break if status.get('files'): log.info('"%s" magnetization successful' % (entry['title'])) break else: log.warning( '"%s" did not magnetize before the timeout elapsed, ' 'file list unavailable for processing.' % entry['title']) defer.returnValue(torrent_id) def add_entry(entry, opts): """Adds an entry to the deluge session""" magnet, filedump = None, None if entry.get('url', '').startswith('magnet:'): magnet = entry['url'] else: if not os.path.exists(entry['file']): entry.fail( 'Downloaded temp file \'%s\' doesn\'t exist!' % entry['file']) del (entry['file']) return with open(entry['file'], 'rb') as f: filedump = base64.encodestring(f.read()) log.verbose('Adding %s to deluge.' % entry['title']) if magnet: d = client.core.add_torrent_magnet(magnet, opts) if config.get('magnetization_timeout'): d.addCallback(_wait_for_metadata, config['magnetization_timeout']) return d else: return client.core.add_torrent_file( entry['title'], filedump, opts) # Generate deluge options dict for torrent add add_opts = {} try: path = entry.render(entry.get('path', config['path'])) if path: add_opts['download_location'] = pathscrub( os.path.expanduser(path)) except RenderError as e: log.error('Could not set path for %s: %s' % (entry['title'], e)) for fopt, dopt in self.options.items(): value = entry.get(fopt, config.get(fopt)) if value is not None: add_opts[dopt] = value if fopt == 'ratio': add_opts['stop_at_ratio'] = True # Make another set of options, that get set after the torrent has been added modify_opts = { 'queuetotop': entry.get('queuetotop', config.get('queuetotop')), 'main_file_only': entry.get('main_file_only', config.get('main_file_only', False)), 'main_file_ratio': entry.get('main_file_ratio', config.get('main_file_ratio')), 'hide_sparse_files': entry.get('hide_sparse_files', config.get('hide_sparse_files', True)), 'keep_subs': entry.get('keep_subs', config.get('keep_subs', True)) } try: label = entry.render(entry.get('label', config['label'])) modify_opts['label'] = format_label(label) except RenderError as e: log.error('Error setting label for `%s`: %s', entry['title'], e) try: movedone = entry.render( entry.get('movedone', config['movedone'])) modify_opts['movedone'] = pathscrub( os.path.expanduser(movedone)) except RenderError as e: log.error('Error setting movedone for %s: %s' % (entry['title'], e)) try: content_filename = entry.get( 'content_filename', config.get('content_filename', '')) modify_opts['content_filename'] = pathscrub( entry.render(content_filename)) except RenderError as e: log.error('Error setting content_filename for %s: %s' % (entry['title'], e)) torrent_id = entry.get('deluge_id') or entry.get( 'torrent_info_hash') torrent_id = torrent_id and torrent_id.lower() if torrent_id in torrent_ids: log.info( '%s is already loaded in deluge, setting options' % entry['title']) # Entry has a deluge id, verify the torrent is still in the deluge session and apply options # Since this is already loaded in deluge, we may also need to change the path modify_opts['path'] = add_opts.pop('download_location', None) dlist.extend([ set_torrent_options(torrent_id, entry, modify_opts), client.core.set_torrent_options([torrent_id], add_opts) ]) else: dlist.append( add_entry(entry, add_opts).addCallbacks( set_torrent_options, on_fail, callbackArgs=(entry, modify_opts), errbackArgs=(task, entry))) return defer.DeferredList(dlist)
def on_task_output(self, task, config): config = self.prepare_config(config) # don't add when learning if task.options.learn: return if not config['enabled']: return # Do not run if there is nothing to do if not task.accepted: return if self.client is None: self.client = self.create_rpc_client(config) if self.client: logger.debug('Successfully connected to transmission.') else: raise plugin.PluginError("Couldn't connect to transmission.") session_torrents = self.client.get_torrents() for entry in task.accepted: if task.options.test: logger.info('Would {} {} in transmission.', config['action'], entry['title']) continue # Compile user options into appropriate dict options = self._make_torrent_options_dict(config, entry) torrent_info = None for t in session_torrents: if t.hashString.lower() == entry.get( 'torrent_info_hash', '').lower() or t.id == entry.get('transmission_id'): torrent_info = t logger.debug( 'Found {} already loaded in transmission as {}', entry['title'], torrent_info.name, ) break if not torrent_info: if config['action'] != 'add': logger.warning( 'Cannot {} {} because it is not loaded in transmission.', config['action'], entry['title'], ) continue downloaded = not entry['url'].startswith('magnet:') # Check that file is downloaded if downloaded and 'file' not in entry: entry.fail('`file` field missing?') continue # Verify the temp file exists if downloaded and not os.path.exists(entry['file']): tmp_path = os.path.join(task.manager.config_base, 'temp') logger.debug('entry: {}', entry) logger.debug('temp: {}', ', '.join(os.listdir(tmp_path))) entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry['file']) continue try: if downloaded: with open(entry['file'], 'rb') as f: filedump = base64.b64encode( f.read()).decode('utf-8') torrent_info = self.client.add_torrent( filedump, 30, **options['add']) else: if options['post'].get('magnetization_timeout', 0) > 0: options['add']['paused'] = False torrent_info = self.client.add_torrent( entry['url'], timeout=30, **options['add']) except TransmissionError as e: logger.opt(exception=True).debug('TransmissionError') logger.debug('Failed options dict: {}', options['add']) msg = 'Error adding {} to transmission. TransmissionError: {}'.format( entry['title'], e.message or 'N/A') logger.error(msg) entry.fail(msg) continue logger.info('"{}" torrent added to transmission', entry['title']) # The info returned by the add call is incomplete, refresh it torrent_info = self.client.get_torrent(torrent_info.id) else: # Torrent already loaded in transmission if options['add'].get('download_dir'): logger.verbose('Moving {} to "{}"', torrent_info.name, options['add']['download_dir']) # Move data even if current reported torrent location matches new location # as transmission may fail to automatically move completed file to final # location but continue reporting final location instead of real location. # In such case this will kick transmission to really move data. # If data is already located at new location then transmission just ignore # this command. self.client.move_torrent_data( torrent_info.id, options['add']['download_dir'], 120) try: total_size = torrent_info.totalSize main_id = None find_main_file = (options['post'].get('main_file_only') or 'content_filename' in options['post']) skip_files = options['post'].get('skip_files') # We need to index the files if any of the following are defined if find_main_file or skip_files: file_list = self.client.get_files( torrent_info.id)[torrent_info.id] if options['post'].get('magnetization_timeout', 0) > 0 and not file_list: logger.debug( 'Waiting {} seconds for "{}" to magnetize', options['post']['magnetization_timeout'], entry['title'], ) for _ in range( options['post']['magnetization_timeout']): sleep(1) file_list = self.client.get_files( torrent_info.id)[torrent_info.id] if file_list: total_size = self.client.get_torrent( torrent_info.id, ['id', 'totalSize']).totalSize break else: logger.warning( '"{}" did not magnetize before the timeout elapsed, file list unavailable for processing.', entry['title'], ) # Find files based on config dl_list = [] skip_list = [] main_list = [] ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass'] main_ratio = config['main_file_ratio'] if 'main_file_ratio' in options['post']: main_ratio = options['post']['main_file_ratio'] for file_id, file in enumerate(file_list): # No need to set main_id if we're not going to need it if find_main_file and file.size > total_size * main_ratio: main_id = file_id if 'include_files' in options['post']: if any( fnmatch(file.name, mask) for mask in options['post']['include_files']): dl_list.append(file_id) elif options['post'].get('include_subs') and any( fnmatch(file.name, mask) for mask in ext_list): dl_list.append(file_id) if skip_files: if any( fnmatch(file.name, mask) for mask in skip_files): skip_list.append(file_id) if main_id is not None: # Look for files matching main ID title but with a different extension if options['post'].get('rename_like_files'): for file_id, file in enumerate(file_list): # if this filename matches main filename we want to rename it as well fs = os.path.splitext(file.name) if fs[0] == os.path.splitext( file_list[main_id].name)[0]: main_list.append(file_id) else: main_list = [main_id] if main_id not in dl_list: dl_list.append(main_id) elif find_main_file: logger.warning( 'No files in "{}" are > {:.0f}% of content size, no files renamed.', entry['title'], main_ratio * 100, ) # If we have a main file and want to rename it and associated files if 'content_filename' in options[ 'post'] and main_id is not None: if 'download_dir' not in options['add']: download_dir = self.client.get_session( ).download_dir else: download_dir = options['add']['download_dir'] # Get new filename without ext file_ext = os.path.splitext(file_list[main_id].name)[1] file_path = os.path.dirname( os.path.join(download_dir, file_list[main_id].name)) filename = options['post']['content_filename'] if config['host'] == 'localhost' or config[ 'host'] == '127.0.0.1': counter = 1 while os.path.exists( os.path.join(file_path, filename + file_ext)): # Try appending a (#) suffix till a unique filename is found filename = f'{options["post"]["content_filename"]}({counter})' counter += 1 else: logger.debug( 'Cannot ensure content_filename is unique ' 'when adding to a remote transmission daemon.') for file_id in main_list: file_ext = os.path.splitext( file_list[file_id].name)[1] logger.debug( 'File {} renamed to {}', file_list[file_id].name, filename + file_ext, ) # change to below when set_files will allow setting name, more efficient to have one call # fl[index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8')) try: self.client.rename_torrent_path( torrent_info.id, file_list[file_id].name, os.path.basename( str(pathscrub(filename + file_ext))), ) except TransmissionError: logger.error( 'content_filename only supported with transmission 2.8+' ) if options['post'].get( 'main_file_only') and main_id is not None: # Set Unwanted Files options['change']['files_unwanted'] = [ x for x in file_list if x not in dl_list ] options['change']['files_wanted'] = dl_list logger.debug( 'Downloading {} of {} files in torrent.', len(options['change']['files_wanted']), len(file_list), ) elif (not options['post'].get('main_file_only') or main_id is None) and skip_files: # If no main file and we want to skip files if len(skip_list) >= len(file_list): logger.debug( 'skip_files filter would cause no files to be downloaded; ' 'including all files in torrent.') else: options['change']['files_unwanted'] = skip_list options['change']['files_wanted'] = [ x for x in file_list if x not in skip_list ] logger.debug( 'Downloading {} of {} files in torrent.', len(options['change']['files_wanted']), len(file_list), ) # Set any changed file properties if list(options['change'].keys()): self.client.change_torrent(torrent_info.id, 30, **options['change']) start_torrent = partial(self.client.start_torrent, [torrent_info.id]) if config['action'] == 'add': # if add_paused was defined and set to False start the torrent; # prevents downloading data before we set what files we want start_paused = ( options['post']['paused'] if 'paused' in options['post'] else not self.client.get_session().start_added_torrents) if start_paused: self.client.stop_torrent(torrent_info.id) else: self.client.start_torrent(torrent_info.id) elif config['action'] in ('remove', 'purge'): self.client.remove_torrent( [torrent_info.id], delete_data=config['action'] == 'purge') logger.info('{}d {} from transmission', config['action'], torrent_info.name) elif config['action'] == 'pause': self.client.stop_torrent([torrent_info.id]) logger.info('paused {} in transmission', torrent_info.name) elif config['action'] == 'resume': start_torrent() logger.info('resumed {} in transmission', torrent_info.name) elif config['action'] == 'bypass_queue': start_torrent(bypass_queue=True) logger.info('resumed (bypass queue) {} in transmission', torrent_info.name) except TransmissionError as e: logger.opt(exception=True).debug('TransmissionError') logger.debug('Failed options dict: {}', options) msg = 'Error trying to {} {}, TransmissionError: {}'.format( config['action'], entry['title'], e.message or 'N/A') logger.error(msg) continue
def filter_pathscrub(val, os_mode=None): """Replace problematic characters in a path.""" return pathscrub(val, os_mode)
def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ( 'path', 'add_paused', 'honor_limits', 'bandwidth_priority', 'max_connections', 'max_up_speed', 'max_down_speed', 'ratio', 'main_file_only', 'main_file_ratio', 'magnetization_timeout', 'include_subs', 'content_filename', 'include_files', 'skip_files', 'rename_like_files', 'queue_position', ): # Values do not merge config with task # Task takes priority then config is used if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {'add': {}, 'change': {}, 'post': {}} add = options['add'] if opt_dic.get('path'): try: path = os.path.expanduser(entry.render(opt_dic['path'])) except RenderError as e: logger.error('Error setting path for {}: {}', entry['title'], e) else: # Transmission doesn't like it when paths end in a separator path = path.rstrip('\\/') add['download_dir'] = pathscrub(path) # make sure we add it paused, will modify status after adding add['paused'] = True change = options['change'] if 'bandwidth_priority' in opt_dic: change['bandwidthPriority'] = opt_dic['bandwidth_priority'] if 'honor_limits' in opt_dic and not opt_dic['honor_limits']: change['honorsSessionLimits'] = False if 'max_up_speed' in opt_dic: change['uploadLimit'] = opt_dic['max_up_speed'] change['uploadLimited'] = True if 'max_down_speed' in opt_dic: change['downloadLimit'] = opt_dic['max_down_speed'] change['downloadLimited'] = True if 'max_connections' in opt_dic: change['peer_limit'] = opt_dic['max_connections'] if 'ratio' in opt_dic: change['seedRatioLimit'] = opt_dic['ratio'] if opt_dic['ratio'] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio change['seedRatioMode'] = 2 else: change['seedRatioMode'] = 1 if 'queue_position' in opt_dic: change['queuePosition'] = opt_dic['queue_position'] post = options['post'] # set to modify paused status after if 'add_paused' in opt_dic: post['paused'] = opt_dic['add_paused'] if 'main_file_only' in opt_dic: post['main_file_only'] = opt_dic['main_file_only'] if 'main_file_ratio' in opt_dic: post['main_file_ratio'] = opt_dic['main_file_ratio'] if 'magnetization_timeout' in opt_dic: post['magnetization_timeout'] = opt_dic['magnetization_timeout'] if 'include_subs' in opt_dic: post['include_subs'] = opt_dic['include_subs'] if 'content_filename' in opt_dic: try: post['content_filename'] = entry.render( opt_dic['content_filename']) except RenderError as e: logger.error('Unable to render content_filename {}: {}', entry['title'], e) if 'skip_files' in opt_dic: post['skip_files'] = opt_dic['skip_files'] if not isinstance(post['skip_files'], list): post['skip_files'] = [post['skip_files']] if 'include_files' in opt_dic: post['include_files'] = opt_dic['include_files'] if not isinstance(post['include_files'], list): post['include_files'] = [post['include_files']] if 'rename_like_files' in opt_dic: post['rename_like_files'] = opt_dic['rename_like_files'] return options
def filter_pathscrub(val: str, os_mode: str = None) -> str: """Replace problematic characters in a path.""" return pathscrub(val, os_mode)
def download_entry(self, task, entry, url, tmp_path): """Downloads `entry` by using `url`. :raises: Several types of exceptions ... :raises: PluginWarning """ log.debug('Downloading url \'%s\'', url) # get content auth = None if 'download_auth' in entry: auth = entry['download_auth'] log.debug('Custom auth enabled for %s download: %s', entry['title'], entry['download_auth']) try: response = task.requests.get(url, auth=auth, raise_status=False) except UnicodeError: log.error('Unicode error while encoding url %s', url) return if response.status_code != 200: log.debug('Got %s response from server. Saving error page.', response.status_code) # Save the error page if response.content: self.save_error_page(entry, task, response.content) # Raise the error response.raise_for_status() return # expand ~ in temp path # TODO jinja? try: tmp_path = os.path.expanduser(tmp_path) except RenderError as e: entry.fail( 'Could not set temp path. Error during string replacement: %s' % e) return # Clean illegal characters from temp path name tmp_path = pathscrub(tmp_path) # create if missing if not os.path.isdir(tmp_path): log.debug('creating tmp_path %s' % tmp_path) os.mkdir(tmp_path) # check for write-access if not os.access(tmp_path, os.W_OK): raise plugin.PluginError( 'Not allowed to write to temp directory `%s`' % tmp_path) # download and write data into a temp file tmp_dir = tempfile.mkdtemp(dir=tmp_path) fname = hashlib.md5(url.encode('utf-8', 'replace')).hexdigest() datafile = os.path.join(tmp_dir, fname) outfile = io.open(datafile, 'wb') try: for chunk in response.iter_content(chunk_size=150 * 1024, decode_unicode=False): outfile.write(chunk) except Exception as e: # don't leave futile files behind # outfile has to be closed before we can delete it on Windows outfile.close() log.debug('Download interrupted, removing datafile') os.remove(datafile) if isinstance(e, socket.timeout): log.error('Timeout while downloading file') else: raise else: outfile.close() # Do a sanity check on downloaded file if os.path.getsize(datafile) == 0: entry.fail('File %s is 0 bytes in size' % datafile) os.remove(datafile) return # store temp filename into entry so other plugins may read and modify content # temp file is moved into final destination at self.output entry['file'] = datafile log.debug('%s field file set to: %s', entry['title'], entry['file']) if 'content-type' in response.headers: entry['mime-type'] = str( parse_header(response.headers['content-type'])[0]) else: entry['mime-type'] = "unknown/unknown" content_encoding = response.headers.get('content-encoding', '') decompress = 'gzip' in content_encoding or 'deflate' in content_encoding if 'content-length' in response.headers and not decompress: entry['content-length'] = int(response.headers['content-length']) # prefer content-disposition naming, note: content-disposition can be disabled completely # by setting entry field `content-disposition` to False if entry.get('content-disposition', True): self.filename_from_headers(entry, response) else: log.info('Content-disposition disabled for %s', entry['title']) self.filename_ext_from_mime(entry) if not entry.get('filename'): filename = unquote(url.rsplit('/', 1)[1]) log.debug('No filename - setting from url: %s', filename) entry['filename'] = filename log.debug('Finishing download_entry() with filename %s', entry.get('filename'))
def on_get_session_state(torrent_ids): """Gets called with a list of torrent_ids loaded in the deluge session. Adds new torrents and modifies the settings for ones already in the session.""" dlist = [] # add the torrents for entry in task.accepted: def add_entry(entry, opts): """Adds an entry to the deluge session""" magnet, filedump = None, None if entry.get('url', '').startswith('magnet:'): magnet = entry['url'] else: if not os.path.exists(entry['file']): entry.fail('Downloaded temp file \'%s\' doesn\'t exist!' % entry['file']) del(entry['file']) return with open(entry['file'], 'rb') as f: filedump = base64.encodestring(f.read()) log.verbose('Adding %s to deluge.' % entry['title']) if magnet: return client.core.add_torrent_magnet(magnet, opts) else: return client.core.add_torrent_file(entry['title'], filedump, opts) # Generate deluge options dict for torrent add add_opts = {} try: path = entry.render(entry.get('path', config['path'])) if path: add_opts['download_location'] = pathscrub(os.path.expanduser(path)) except RenderError as e: log.error('Could not set path for %s: %s' % (entry['title'], e)) for fopt, dopt in self.options.iteritems(): value = entry.get(fopt, config.get(fopt)) if value is not None: add_opts[dopt] = value if fopt == 'ratio': add_opts['stop_at_ratio'] = True # Make another set of options, that get set after the torrent has been added modify_opts = {'label': format_label(entry.get('label', config['label'])), 'queuetotop': entry.get('queuetotop', config.get('queuetotop')), 'main_file_only': entry.get('main_file_only', config.get('main_file_only', False))} try: movedone = entry.render(entry.get('movedone', config['movedone'])) modify_opts['movedone'] = pathscrub(os.path.expanduser(movedone)) except RenderError as e: log.error('Error setting movedone for %s: %s' % (entry['title'], e)) try: content_filename = entry.get('content_filename', config.get('content_filename', '')) modify_opts['content_filename'] = pathscrub(entry.render(content_filename)) except RenderError as e: log.error('Error setting content_filename for %s: %s' % (entry['title'], e)) torrent_id = entry.get('deluge_id') or entry.get('torrent_info_hash') torrent_id = torrent_id and torrent_id.lower() if torrent_id in torrent_ids: log.info('%s is already loaded in deluge, setting options' % entry['title']) # Entry has a deluge id, verify the torrent is still in the deluge session and apply options # Since this is already loaded in deluge, we may also need to change the path modify_opts['path'] = add_opts.pop('download_location', None) dlist.extend([set_torrent_options(torrent_id, entry, modify_opts), client.core.set_torrent_options([torrent_id], add_opts)]) else: dlist.append(add_entry(entry, add_opts).addCallbacks( set_torrent_options, on_fail, callbackArgs=(entry, modify_opts), errbackArgs=(task, entry))) return defer.DeferredList(dlist)
def on_connect_success(self, result, task, config): """Gets called when successfully connected to a daemon.""" from deluge.ui.client import client from twisted.internet import reactor, defer if not result: log.debug('on_connect_success returned a failed result. BUG?') if task.manager.options.test: log.debug('Test connection to deluge daemon successful.') client.disconnect() return def format_label(label): """Makes a string compliant with deluge label naming rules""" return re.sub('[^\w-]+', '_', label.lower()) def set_torrent_options(torrent_id, entry, opts): """Gets called when a torrent was added to the daemon.""" dlist = [] if not torrent_id: log.error('There was an error adding %s to deluge.' % entry['title']) # TODO: Fail entry? How can this happen still now? return log.info('%s successfully added to deluge.' % entry['title']) entry['deluge_id'] = torrent_id def create_path(result, path): """Creates the specified path if deluge is older than 1.3""" from deluge.common import VersionSplit # Before 1.3, deluge would not create a non-existent move directory, so we need to. if VersionSplit('1.3.0') > VersionSplit(self.deluge_version): if client.is_localhost(): if not os.path.isdir(path): log.debug('path %s doesn\'t exist, creating' % path) os.makedirs(path) else: log.warning( 'If path does not exist on the machine running the daemon, move will fail.' ) if opts.get('movedone'): dlist.append( version_deferred.addCallback(create_path, opts['movedone'])) dlist.append( client.core.set_torrent_move_completed(torrent_id, True)) dlist.append( client.core.set_torrent_move_completed_path( torrent_id, opts['movedone'])) log.debug('%s move on complete set to %s' % (entry['title'], opts['movedone'])) if opts.get('label'): def apply_label(result, torrent_id, label): """Gets called after labels and torrent were added to deluge.""" return client.label.set_torrent(torrent_id, label) dlist.append( label_deferred.addCallback(apply_label, torrent_id, opts['label'])) if opts.get('queuetotop') is not None: if opts['queuetotop']: dlist.append(client.core.queue_top([torrent_id])) log.debug('%s moved to top of queue' % entry['title']) else: dlist.append(client.core.queue_bottom([torrent_id])) log.debug('%s moved to bottom of queue' % entry['title']) def on_get_torrent_status(status): """Gets called with torrent status, including file info. Sets the torrent options which require knowledge of the current status of the torrent.""" main_file_dlist = [] # Determine where the file should be move_now_path = None if opts.get('movedone'): if status['progress'] == 100: move_now_path = opts['movedone'] else: # Deluge will unset the move completed option if we move the storage, forgo setting proper # path, in favor of leaving proper final location. log.debug( 'Not moving storage for %s, as this will prevent movedone.' % entry['title']) elif opts.get('path'): move_now_path = opts['path'] if move_now_path and os.path.normpath( move_now_path) != os.path.normpath( status['save_path']): main_file_dlist.append( version_deferred.addCallback(create_path, move_now_path)) log.debug('Moving storage for %s to %s' % (entry['title'], move_now_path)) main_file_dlist.append( client.core.move_storage([torrent_id], move_now_path)) if opts.get('content_filename') or opts.get('main_file_only'): def file_exists(): # Checks the download path as well as the move completed path for existence of the file if os.path.exists( os.path.join(status['save_path'], filename)): return True elif status.get('move_on_completed') and status.get( 'move_on_completed_path'): if os.path.exists( os.path.join( status['move_on_completed_path'], filename)): return True else: return False for file in status['files']: # Only rename file if it is > 90% of the content if file['size'] > (status['total_size'] * 0.9): if opts.get('content_filename'): filename = opts[ 'content_filename'] + os.path.splitext( file['path'])[1] counter = 1 if client.is_localhost(): while file_exists(): # Try appending a (#) suffix till a unique filename is found filename = ''.join([ opts['content_filename'], '(', str(counter), ')', os.path.splitext(file['path'])[1] ]) counter += 1 else: log.debug( 'Cannot ensure content_filename is unique when adding to a remote deluge daemon.' ) log.debug( 'File %s in %s renamed to %s' % (file['path'], entry['title'], filename)) main_file_dlist.append( client.core.rename_files( torrent_id, [(file['index'], filename)])) if opts.get('main_file_only'): file_priorities = [ 1 if f['index'] == file['index'] else 0 for f in status['files'] ] main_file_dlist.append( client.core.set_torrent_file_priorities( torrent_id, file_priorities)) break else: log.warning( 'No files in %s are > 90%% of content size, no files renamed.' % entry['title']) return defer.DeferredList(main_file_dlist) status_keys = [ 'files', 'total_size', 'save_path', 'move_on_completed_path', 'move_on_completed', 'progress' ] dlist.append( client.core.get_torrent_status( torrent_id, status_keys).addCallback(on_get_torrent_status)) return defer.DeferredList(dlist) def on_fail(result, task, entry): """Gets called when daemon reports a failure adding the torrent.""" log.info('%s was not added to deluge! %s' % (entry['title'], result)) task.fail(entry, 'Could not be added to deluge') # dlist is a list of deferreds that must complete before we exit dlist = [] # loop through entries to get a list of labels to add labels = set([ format_label(entry['label']) for entry in task.accepted if entry.get('label') ]) if config.get('label'): labels.add(format_label(config['label'])) label_deferred = defer.succeed(True) if labels: # Make sure the label plugin is available and enabled, then add appropriate labels def on_get_enabled_plugins(plugins): """Gets called with the list of enabled deluge plugins.""" def on_label_enabled(result): """ This runs when we verify the label plugin is enabled. """ def on_get_labels(d_labels): """Gets available labels from deluge, and adds any new labels we need.""" dlist = [] for label in labels: if not label in d_labels: log.debug('Adding the label %s to deluge' % label) dlist.append(client.label.add(label)) return defer.DeferredList(dlist) return client.label.get_labels().addCallback(on_get_labels) if 'Label' in plugins: return on_label_enabled(True) else: # Label plugin isn't enabled, so we check if it's available and enable it. def on_get_available_plugins(plugins): """Gets plugins available to deluge, enables Label plugin if available.""" if 'Label' in plugins: log.debug('Enabling label plugin in deluge') return client.core.enable_plugin( 'Label').addCallback(on_label_enabled) else: log.error( 'Label plugin is not installed in deluge') return client.core.get_available_plugins().addCallback( on_get_available_plugins) label_deferred = client.core.get_enabled_plugins().addCallback( on_get_enabled_plugins) dlist.append(label_deferred) def on_get_daemon_info(ver): """Gets called with the daemon version info, stores it in self.""" log.debug('deluge version %s' % ver) self.deluge_version = ver version_deferred = client.daemon.info().addCallback(on_get_daemon_info) dlist.append(version_deferred) def on_get_session_state(torrent_ids): """Gets called with a list of torrent_ids loaded in the deluge session. Adds new torrents and modifies the settings for ones already in the session.""" dlist = [] # add the torrents for entry in task.accepted: def add_entry(entry, opts): """Adds an entry to the deluge session""" magnet, filedump = None, None if entry.get('url', '').startswith('magnet:'): magnet = entry['url'] else: if not os.path.exists(entry['file']): task.fail( entry, 'Downloaded temp file \'%s\' doesn\'t exist!' % entry['file']) del (entry['file']) return try: f = open(entry['file'], 'rb') filedump = base64.encodestring(f.read()) finally: f.close() log.verbose('Adding %s to deluge.' % entry['title']) if magnet: return client.core.add_torrent_magnet(magnet, opts) else: return client.core.add_torrent_file( entry['title'], filedump, opts) # Generate deluge options dict for torrent add add_opts = {} try: path = entry.render(entry.get('path', config['path'])) if path: add_opts['download_location'] = pathscrub( os.path.expanduser(path)) except RenderError, e: log.error('Could not set path for %s: %s' % (entry['title'], e)) for fopt, dopt in self.options.iteritems(): value = entry.get(fopt, config.get(fopt)) if value is not None: add_opts[dopt] = value if fopt == 'ratio': add_opts['stop_at_ratio'] = True # Make another set of options, that get set after the torrent has been added modify_opts = { 'label': format_label(entry.get('label', config['label'])), 'queuetotop': entry.get('queuetotop', config.get('queuetotop')), 'main_file_only': entry.get('main_file_only', config.get('main_file_only', False)) } try: movedone = entry.render( entry.get('movedone', config['movedone'])) modify_opts['movedone'] = pathscrub( os.path.expanduser(movedone)) except RenderError, e: log.error('Error setting movedone for %s: %s' % (entry['title'], e)) try: content_filename = entry.get( 'content_filename', config.get('content_filename', '')) modify_opts['content_filename'] = pathscrub( entry.render(content_filename)) except RenderError, e: log.error('Error setting content_filename for %s: %s' % (entry['title'], e))
def output(self, task, entry, config): """Moves temp-file into final destination Raises: PluginError if operation fails """ if 'file' not in entry and not task.options.test: log.debug('file missing, entry: %s', entry) raise plugin.PluginError( 'Entry `%s` has no temp file associated with' % entry['title']) try: # use path from entry if has one, otherwise use from download definition parameter path = entry.get('path', config.get('path')) if not isinstance(path, str): raise plugin.PluginError('Invalid `path` in entry `%s`' % entry['title']) # override path from command line parameter if task.options.dl_path: path = task.options.dl_path # expand variables in path try: path = os.path.expanduser(entry.render(path)) except RenderError as e: entry.fail( 'Could not set path. Error during string replacement: %s' % e) return # Clean illegal characters from path name path = pathscrub(path) # If we are in test mode, report and return if task.options.test: log.info('Would write `%s` to `%s`', entry['title'], path) # Set a fake location, so the exec plugin can do string replacement during --test #1015 entry['location'] = os.path.join(path, 'TEST_MODE_NO_OUTPUT') return # make path if not os.path.isdir(path): log.debug('Creating directory %s', path) try: os.makedirs(path) except: raise plugin.PluginError('Cannot create path %s' % path, log) # check that temp file is present if not os.path.exists(entry['file']): log.debug('entry: %s', entry) raise plugin.PluginWarning( 'Downloaded temp file `%s` doesn\'t exist!?' % entry['file']) if config.get('filename'): try: entry['filename'] = entry.render(config['filename']) log.debug('set filename from config %s' % entry['filename']) except RenderError as e: entry.fail( 'Could not set filename. Error during string replacement: %s' % e) return # if we still don't have a filename, try making one from title (last resort) elif not entry.get('filename'): entry['filename'] = entry['title'] log.debug('set filename from title %s', entry['filename']) if 'mime-type' not in entry: log.warning( 'Unable to figure proper filename for %s. Using title.', entry['title']) else: guess = mimetypes.guess_extension(entry['mime-type']) if not guess: log.warning( 'Unable to guess extension with mime-type %s', guess) else: self.filename_ext_from_mime(entry) name = entry.get('filename', entry['title']) # Remove illegal characters from filename #325, #353 name = pathscrub(name) # Remove directory separators from filename #208 name = name.replace('/', ' ') if sys.platform.startswith('win'): name = name.replace('\\', ' ') # remove duplicate spaces name = ' '.join(name.split()) # combine to full path + filename destfile = os.path.join(path, name) log.debug('destfile: %s', destfile) if os.path.exists(destfile): import filecmp if filecmp.cmp(entry['file'], destfile): log.debug("Identical destination file '%s' already exists", destfile) elif config.get('overwrite'): log.debug("Overwriting already existing file %s", destfile) else: log.info( 'File `%s` already exists and is not identical, download failed.', destfile) entry.fail( 'File `%s` already exists and is not identical.' % destfile) return else: # move temp file log.debug('moving %s to %s', entry['file'], destfile) try: shutil.move(entry['file'], destfile) except (IOError, OSError) as err: # ignore permission errors, see ticket #555 import errno if not os.path.exists(destfile): raise plugin.PluginError('Unable to write %s: %s' % (destfile, err)) if err.errno != errno.EPERM and err.errno != errno.EACCES: raise # store final destination as output key entry['location'] = destfile finally: self.cleanup_temp_file(entry)
def test_degenerate(self): # If path is reduced to nothing, make sure it complains with pytest.raises(ValueError): pathscrub('<<<<:>>>>', os='windows', filename=True)