def test_space_around(self):
# We don't want folder or file names to end or start with spaces on any platform
space_paths = {" / aoeu /aoeu ": "/aoeu/aoeu", "/ a/a ": "/a/a", "/a /": "/a/"}
for platform in ["windows", "linux", "mac"]:
for test in space_paths:
result = pathscrub(test, filename=False)
assert result == space_paths[test], "%s != %s (%s)" % (result, space_paths[test], platform)
# Windows only should also use backslashes as dir separators
test = ["c:\\ aoeu \\aoeu /aoeu ", "c:\\aoeu\\aoeu/aoeu"]
result = pathscrub(test[0], os="windows", filename=False)
assert result == test[1], "%s != %s" % (result, test[1])
def on_get_session_state(torrent_ids):
"""Gets called with a list of torrent_ids loaded in the deluge session.
Adds new torrents and modifies the settings for ones already in the session."""
dlist = []
# add the torrents
for entry in task.accepted:
def add_entry(entry, opts):
"""Adds an entry to the deluge session"""
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
task.fail(entry, 'Downloaded temp file \'%s\' doesn\'t exist!' % entry['file'])
del(entry['file'])
return
try:
f = open(entry['file'], 'rb')
filedump = base64.encodestring(f.read())
finally:
f.close()
log.verbose('Adding %s to deluge.' % entry['title'])
if magnet:
return client.core.add_torrent_magnet(magnet, opts)
else:
return client.core.add_torrent_file(entry['title'], filedump, opts)
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(os.path.expanduser(path))
except RenderError, e:
log.error('Could not set path for %s: %s' % (entry['title'], e))
for fopt, dopt in self.options.iteritems():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {'label': format_label(entry.get('label', config['label'])),
'queuetotop': entry.get('queuetotop', config.get('queuetotop')),
'main_file_only': entry.get('main_file_only', config.get('main_file_only', False))}
try:
movedone = entry.render(entry.get('movedone', config['movedone']))
modify_opts['movedone'] = pathscrub(os.path.expanduser(movedone))
except RenderError, e:
log.error('Error setting movedone for %s: %s' % (entry['title'], e))
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout):
import libtorrent
params = libtorrent.parse_magnet_uri(magnet_uri)
session = libtorrent.session()
lt_version = [int(v) for v in libtorrent.version.split('.')]
if lt_version > [0, 16, 13, 0] and lt_version < [1, 1, 3, 0]:
# for some reason the info_hash needs to be bytes but it's a struct called sha1_hash
params['info_hash'] = params['info_hash'].to_bytes()
params.url = magnet_uri
handle = session.add_torrent(params)
logger.debug('Acquiring torrent metadata for magnet {}', magnet_uri)
timeout_value = timeout
while not handle.has_metadata():
time.sleep(0.1)
timeout_value -= 0.1
if timeout_value <= 0:
raise plugin.PluginError(
'Timed out after {} seconds trying to magnetize'.format(
timeout))
logger.debug('Metadata acquired')
torrent_info = handle.get_torrent_info()
torrent_file = libtorrent.create_torrent(torrent_info)
torrent_path = pathscrub(
os.path.join(destination_folder,
torrent_info.name() + ".torrent"))
with open(torrent_path, "wb") as f:
f.write(libtorrent.bencode(torrent_file.generate()))
logger.debug('Torrent file wrote to {}', torrent_path)
return torrent_path
def test_space_around(self):
# We don't want folder or file names to end or start with spaces on any platform
space_paths = {
' / aoeu /aoeu ': '/aoeu/aoeu',
'/ a/a ': '/a/a',
'/a /': '/a/'
}
for platform in ['windows', 'linux', 'mac']:
for test in space_paths:
result = pathscrub(test, filename=False)
assert result == space_paths[test], '%s != %s (%s)' % (result, space_paths[test], platform)
# Windows only should also use backslashes as dir separators
test = ['c:\\ aoeu \\aoeu /aoeu ', 'c:\\aoeu\\aoeu/aoeu']
result = pathscrub(test[0], os='windows', filename=False)
assert result == test[1], '%s != %s' % (result, test[1])
def _build_options(self, config, entry, entry_first=True):
options = {}
for opt_key in ("path", "message", "priority", "custom1", "custom2", "custom3", "custom4", "custom5"):
# Values do not merge config with task
# Task takes priority then config is used
entry_value = entry.get(opt_key)
config_value = config.get(opt_key)
if entry_first:
if entry_value:
options[opt_key] = entry.render(entry_value)
elif config_value:
options[opt_key] = entry.render(config_value)
else:
if config_value:
options[opt_key] = entry.render(config_value)
elif entry_value:
options[opt_key] = entry.render(entry_value)
# Convert priority from string to int
priority = options.get("priority")
if priority and priority in self.priority_map:
options["priority"] = self.priority_map[priority]
# Map Flexget path to directory in rTorrent
if options.get("path"):
options["directory"] = options["path"]
del options["path"]
if "directory" in options:
options["directory"] = pathscrub(options["directory"])
return options
def test_space_around(self):
# We don't want folder or file names to end or start with spaces on any platform
space_paths = {' / aoeu /aoeu ': '/aoeu/aoeu', '/ a/a ': '/a/a', '/a /': '/a/'}
for platform in ['windows', 'linux', 'mac']:
for test in space_paths:
result = pathscrub(test, filename=False)
assert result == space_paths[test], '%s != %s (%s)' % (
result,
space_paths[test],
platform,
)
# Windows only should also use backslashes as dir separators
test = ['c:\\ aoeu \\aoeu /aoeu ', 'c:\\aoeu\\aoeu/aoeu']
result = pathscrub(test[0], os='windows', filename=False)
assert result == test[1], '%s != %s' % (result, test[1])
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout=60):
import libtorrent
params = libtorrent.parse_magnet_uri(magnet_uri)
session = libtorrent.session()
handle = session.add_torrent(params)
log.debug(
'Acquiring torrent metadata for magnet {}'.format(magnet_uri))
timeout_value = timeout
while not handle.has_metadata():
time.sleep(0.1)
timeout_value -= 0.1
if timeout_value <= 0:
raise Exception(
'Timed out after {} seconds acquiring torrent metadata from the DHT/trackers.'
.format(timeout))
log.debug('Metadata acquired')
torrent_info = handle.get_torrent_info()
torrent_file = libtorrent.create_torrent(torrent_info)
torrent_path = pathscrub(
os.path.join(destination_folder,
torrent_info.name() + ".torrent"))
with open(torrent_path, "wb") as f:
f.write(libtorrent.bencode(torrent_file.generate()))
log.debug('Torrent file wrote to {}'.format(torrent_path))
return torrent_path
def _build_options(self, config, entry, entry_first=True):
options = {}
for opt_key in ('path', 'message', 'priority', 'custom1', 'custom2',
'custom3', 'custom4', 'custom5'):
# Values do not merge config with task
# Task takes priority then config is used
entry_value = entry.get(opt_key)
config_value = config.get(opt_key)
if entry_first:
if entry_value:
options[opt_key] = entry.render(entry_value)
elif config_value:
options[opt_key] = entry.render(config_value)
else:
if config_value:
options[opt_key] = entry.render(config_value)
elif entry_value:
options[opt_key] = entry.render(entry_value)
# Convert priority from string to int
priority = options.get('priority')
if priority and priority in self.priority_map:
options['priority'] = self.priority_map[priority]
# Map Flexget path to directory in rTorrent
if options.get('path'):
options['directory'] = options['path']
del options['path']
if 'directory' in options:
options['directory'] = pathscrub(options['directory'])
return options
def _build_options(self, config, entry, entry_first=True):
options = {}
for opt_key in ('path', 'message', 'priority',
'custom1', 'custom2', 'custom3', 'custom4', 'custom5'):
# Values do not merge config with task
# Task takes priority then config is used
entry_value = entry.get(opt_key)
config_value = config.get(opt_key)
if entry_first:
if entry_value:
options[opt_key] = entry.render(entry_value)
elif config_value:
options[opt_key] = entry.render(config_value)
else:
if config_value:
options[opt_key] = entry.render(config_value)
elif entry_value:
options[opt_key] = entry.render(entry_value)
# Convert priority from string to int
priority = options.get('priority')
if priority and priority in self.priority_map:
options['priority'] = self.priority_map[priority]
# Map Flexget path to directory in rTorrent
if options.get('path'):
options['directory'] = options['path']
del options['path']
if 'directory' in options:
options['directory'] = pathscrub(options['directory'])
return options
def process_invalid_content(self, task, data, url):
"""If feedparser reports error, save the received data and log error."""
if data is None:
log.critical("Received empty page - no content")
return
ext = "xml"
if "<html>" in data.lower():
log.critical("Received content is HTML page, not an RSS feed")
ext = "html"
if "login" in data.lower() or "username" in data.lower():
log.critical("Received content looks a bit like login page")
if "error" in data.lower():
log.critical("Received content looks a bit like error page")
received = os.path.join(task.manager.config_base, "received")
if not os.path.isdir(received):
os.mkdir(received)
filename = task.name
sourcename = urlparse.urlparse(url).netloc
if sourcename:
filename += "-" + sourcename
filename = pathscrub(filename, filename=True)
filepath = os.path.join(received, "%s.%s" % (filename, ext))
with open(filepath, "w") as f:
f.write(data)
log.critical("I have saved the invalid content to %s for you to view", filepath)
def process_invalid_content(self, task, data, url):
"""If feedparser reports error, save the received data and log error."""
if data is None:
log.critical('Received empty page - no content')
return
else:
data = tobytes(data)
ext = 'xml'
if b'<html>' in data.lower():
log.critical('Received content is HTML page, not an RSS feed')
ext = 'html'
if b'login' in data.lower() or b'username' in data.lower():
log.critical('Received content looks a bit like login page')
if b'error' in data.lower():
log.critical('Received content looks a bit like error page')
received = os.path.join(task.manager.config_base, 'received')
if not os.path.isdir(received):
os.mkdir(received)
filename = task.name
sourcename = urlparse(url).netloc
if sourcename:
filename += '-' + sourcename
filename = pathscrub(filename, filename=True)
filepath = os.path.join(received, '%s.%s' % (filename, ext))
with open(filepath, 'wb') as f:
f.write(data)
log.critical('I have saved the invalid content to %s for you to view',
filepath)
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout):
import libtorrent
params = libtorrent.parse_magnet_uri(magnet_uri)
session = libtorrent.session()
# for some reason the info_hash needs to be bytes but it's a struct called sha1_hash
params['info_hash'] = bytes(params['info_hash'])
handle = libtorrent.add_magnet_uri(session, magnet_uri, params)
log.debug('Acquiring torrent metadata for magnet %s', magnet_uri)
timeout_value = timeout
while not handle.has_metadata():
time.sleep(0.1)
timeout_value -= 0.1
if timeout_value <= 0:
raise plugin.PluginError(
'Timed out after {} seconds trying to magnetize'.format(
timeout))
log.debug('Metadata acquired')
torrent_info = handle.get_torrent_info()
torrent_file = libtorrent.create_torrent(torrent_info)
torrent_path = pathscrub(
os.path.join(destination_folder,
torrent_info.name() + ".torrent"))
with open(torrent_path, "wb") as f:
f.write(libtorrent.bencode(torrent_file.generate()))
log.debug('Torrent file wrote to %s', torrent_path)
return torrent_path
def process_invalid_content(self, task, data, url):
"""If feedparser reports error, save the received data and log error."""
if data is None:
log.critical('Received empty page - no content')
return
else:
data = tobytes(data)
ext = 'xml'
if b'<html>' in data.lower():
log.critical('Received content is HTML page, not an RSS feed')
ext = 'html'
if b'login' in data.lower() or b'username' in data.lower():
log.critical('Received content looks a bit like login page')
if b'error' in data.lower():
log.critical('Received content looks a bit like error page')
received = os.path.join(task.manager.config_base, 'received')
if not os.path.isdir(received):
os.mkdir(received)
filename = task.name
sourcename = urlparse(url).netloc
if sourcename:
filename += '-' + sourcename
filename = pathscrub(filename, filename=True)
filepath = os.path.join(received, '%s.%s' % (filename, ext))
with open(filepath, 'wb') as f:
f.write(data)
log.critical('I have saved the invalid content to %s for you to view', filepath)
def save_error_page(self, entry, task, page):
received = os.path.join(task.manager.config_base, 'received', task.name)
if not os.path.isdir(received):
os.makedirs(received)
filename = os.path.join(received, pathscrub('%s.error' % entry['title'], filename=True))
log.error('Error retrieving %s, the error page has been saved to %s', entry['title'], filename)
with io.open(filename, 'wb') as outfile:
outfile.write(page)
def save_error_page(self, entry, task, page):
received = os.path.join(task.manager.config_base, 'received', task.name)
if not os.path.isdir(received):
os.makedirs(received)
filename = os.path.join(received, pathscrub('%s.error' % entry['title'], filename=True))
log.error('Error retrieving %s, the error page has been saved to %s', entry['title'], filename)
with io.open(filename, 'wb') as outfile:
outfile.write(page)
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority',
'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio', 'main_file_only', 'include_subs'):
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}, 'post': {}}
add = options['add']
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path']))
add['download_dir'] = pathscrub(path).encode('utf-8')
except RenderError as e:
log.error('Error setting path for %s: %s' % (entry['title'], e))
if 'bandwidthpriority' in opt_dic:
add['bandwidthPriority'] = opt_dic['bandwidthpriority']
if 'maxconnections' in opt_dic:
add['peer_limit'] = opt_dic['maxconnections']
# make sure we add it paused, will modify status after adding
add['paused'] = True
change = options['change']
if 'honourlimits' in opt_dic and not opt_dic['honourlimits']:
change['honorsSessionLimits'] = False
if 'maxupspeed' in opt_dic:
change['uploadLimit'] = opt_dic['maxupspeed']
change['uploadLimited'] = True
if 'maxdownspeed' in opt_dic:
change['downloadLimit'] = opt_dic['maxdownspeed']
change['downloadLimited'] = True
if 'ratio' in opt_dic:
change['seedRatioLimit'] = opt_dic['ratio']
if opt_dic['ratio'] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change['seedRatioMode'] = 2
else:
change['seedRatioMode'] = 1
post = options['post']
# set to modify paused status after
if 'addpaused' in opt_dic:
post['paused'] = opt_dic['addpaused']
if 'main_file_only' in opt_dic:
post['main_file_only'] = opt_dic['main_file_only']
if 'include_subs' in opt_dic:
post['include_subs'] = opt_dic['include_subs']
return options
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in (
"path",
"addpaused",
"honourlimits",
"bandwidthpriority",
"maxconnections",
"maxupspeed",
"maxdownspeed",
"ratio",
):
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {"add": {}, "change": {}}
add = options["add"]
if opt_dic.get("path"):
try:
path = os.path.expanduser(entry.render(opt_dic["path"]))
add["download_dir"] = pathscrub(path).encode("utf-8")
except RenderError as e:
log.error("Error setting path for %s: %s" % (entry["title"], e))
if "addpaused" in opt_dic:
add["paused"] = opt_dic["addpaused"]
if "bandwidthpriority" in opt_dic:
add["bandwidthPriority"] = opt_dic["bandwidthpriority"]
if "maxconnections" in opt_dic:
add["peer_limit"] = opt_dic["maxconnections"]
change = options["change"]
if "honourlimits" in opt_dic and not opt_dic["honourlimits"]:
change["honorsSessionLimits"] = False
if "maxupspeed" in opt_dic:
change["uploadLimit"] = opt_dic["maxupspeed"]
change["uploadLimited"] = True
if "maxdownspeed" in opt_dic:
change["downloadLimit"] = opt_dic["maxdownspeed"]
change["downloadLimited"] = True
if "ratio" in opt_dic:
change["seedRatioLimit"] = opt_dic["ratio"]
if opt_dic["ratio"] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change["seedRatioMode"] = 2
else:
change["seedRatioMode"] = 1
return options
def test_windows_paths(self):
win_path = {
'aoeu/aoeu': 'aoeu/aoeu', # Don't strip slashes in path mode
'aoeu\\aoeu': 'aoeu\\aoeu', # Or backslashes
'aoeu / aoeu ': 'aoeu/aoeu', # Don't leave spaces at the begin or end of folder names
'aoeu \\aoeu ': 'aoeu\\aoeu',
'aoeu./aoeu.\\aoeu.': 'aoeu/aoeu\\aoeu', # Or dots
}
for test in win_path:
result = pathscrub(test, os='windows', filename=False)
assert result == win_path[test], '%s != %s' % (result, win_path[test])
def test_windows_paths(self):
win_path = {
'aoeu/aoeu': 'aoeu/aoeu', # Don't strip slashes in path mode
'aoeu\\aoeu': 'aoeu\\aoeu', # Or backslashes
'aoeu / aoeu ': 'aoeu/aoeu', # Don't leave spaces at the begin or end of folder names
'aoeu \\aoeu ': 'aoeu\\aoeu',
'aoeu./aoeu.\\aoeu.': 'aoeu/aoeu\\aoeu' # Or dots
}
for test in win_path:
result = pathscrub(test, os='windows', filename=False)
assert result == win_path[test], '%s != %s' % (result, win_path[test])
def test_windows_paths(self):
win_path = {
"aoeu/aoeu": "aoeu/aoeu", # Don't strip slashes in path mode
"aoeu\\aoeu": "aoeu\\aoeu", # Or backslashes
"aoeu / aoeu ": "aoeu/aoeu", # Don't leave spaces at the begin or end of folder names
"aoeu \\aoeu ": "aoeu\\aoeu",
"aoeu./aoeu.\\aoeu.": "aoeu/aoeu\\aoeu", # Or dots
}
for test in win_path:
result = pathscrub(test, os="windows", filename=False)
assert result == win_path[test], "%s != %s" % (result, win_path[test])
def process(self, entry, destination_folder, timeout):
import libtorrent
magnet_uri = entry['url']
params = libtorrent.parse_magnet_uri(magnet_uri)
session = libtorrent.session()
lt_version = [int(v) for v in libtorrent.version.split('.')]
if lt_version > [0, 16, 13, 0] and lt_version < [1, 1, 3, 0]:
# for some reason the info_hash needs to be bytes but it's a struct called sha1_hash
params['info_hash'] = params['info_hash'].to_bytes()
params.url = magnet_uri
handle = session.add_torrent(params)
log.debug('Acquiring torrent metadata for magnet %s', magnet_uri)
handle.force_dht_announce()
timeout_value = timeout
while not handle.has_metadata():
time.sleep(0.1)
timeout_value -= 0.1
if timeout_value <= 0:
raise plugin.PluginError(
'Timed out after {} seconds trying to magnetize'.format(
timeout))
log.debug('Metadata acquired')
torrent_info = handle.get_torrent_info()
torrent_file = libtorrent.create_torrent(torrent_info)
torrent_path = pathscrub(
os.path.join(destination_folder,
torrent_info.name() + ".torrent"))
with open(torrent_path, "wb") as f:
f.write(libtorrent.bencode(torrent_file.generate()))
log.debug('Torrent file wrote to %s', torrent_path)
# Windows paths need an extra / prepended to them for url
if not torrent_path.startswith('/'):
torrent_path = '/' + torrent_path
entry['url'] = torrent_path
entry['file'] = torrent_path
# make sure it's first in the list because of how download plugin works
entry['urls'].insert(0, 'file://{}'.format(torrent_path))
entry['content_size'] = torrent_info.total_size() / 1024 / 1024
# Might as well get some more info
while handle.status(0).num_complete < 0:
time.sleep(0.1)
timeout_value -= 0.1
if timeout_value <= 0:
log.debug('Timed out after {} seconds trying to get peer info'.
format(timeout))
return
log.debug('Peer info acquired')
torrent_status = handle.status(0)
entry['torrent_seeds'] = torrent_status.num_complete
entry['torrent_leeches'] = torrent_status.num_incomplete
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in ('path', 'addpaused', 'honourlimits',
'bandwidthpriority', 'maxconnections', 'maxupspeed',
'maxdownspeed', 'ratio'):
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}}
add = options['add']
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path']))
add['download_dir'] = pathscrub(path).encode('utf-8')
except RenderError as e:
log.error('Error setting path for %s: %s' %
(entry['title'], e))
if 'addpaused' in opt_dic:
add['paused'] = opt_dic['addpaused']
if 'bandwidthpriority' in opt_dic:
add['bandwidthPriority'] = opt_dic['bandwidthpriority']
if 'maxconnections' in opt_dic:
add['peer_limit'] = opt_dic['maxconnections']
change = options['change']
if 'honourlimits' in opt_dic and not opt_dic['honourlimits']:
change['honorsSessionLimits'] = False
if 'maxupspeed' in opt_dic:
change['uploadLimit'] = opt_dic['maxupspeed']
change['uploadLimited'] = True
if 'maxdownspeed' in opt_dic:
change['downloadLimit'] = opt_dic['maxdownspeed']
change['downloadLimited'] = True
if 'ratio' in opt_dic:
change['seedRatioLimit'] = opt_dic['ratio']
if opt_dic['ratio'] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change['seedRatioMode'] = 2
else:
change['seedRatioMode'] = 1
return options
def on_task_download(self, task, config):
config = self.prepare_config(config, task)
for entry in task.accepted:
ftp_url = urlparse(entry.get('url'))
ftp_url = ftp_url._replace(path=unquote(ftp_url.path))
current_path = os.path.dirname(ftp_url.path)
try:
ftp = self.ftp_connect(config, ftp_url, current_path)
except ftplib.all_errors as e:
entry.fail("Unable to connect to server : %s" % (e))
break
to_path = config['ftp_tmp_path']
try:
to_path = entry.render(to_path)
except RenderError as err:
raise plugin.PluginError(
"Path value replacement `%s` failed: %s" %
(to_path, err.args[0]))
# Clean invalid characters with pathscrub plugin
to_path = pathscrub(to_path)
if not os.path.exists(to_path):
log.debug("Creating base path: %s" % to_path)
os.makedirs(to_path)
if not os.path.isdir(to_path):
raise plugin.PluginWarning(
"Destination `%s` is not a directory." % to_path)
file_name = os.path.basename(ftp_url.path)
try:
# Directory
ftp = self.check_connection(ftp, config, ftp_url, current_path)
ftp.cwd(file_name)
self.ftp_walk(ftp, os.path.join(to_path, file_name), config,
ftp_url, ftp_url.path)
ftp = self.check_connection(ftp, config, ftp_url, current_path)
ftp.cwd('..')
if config['delete_origin']:
ftp.rmd(file_name)
except ftplib.error_perm:
# File
self.ftp_down(ftp, file_name, to_path, config, ftp_url,
current_path)
ftp.close()
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority',
'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio'):
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}}
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path'])).encode('utf-8')
options['add']['download_dir'] = pathscrub(path)
except RenderError as e:
log.error('Error setting path for %s: %s' % (entry['title'], e))
if 'addpaused' in opt_dic:
options['add']['paused'] = opt_dic['addpaused']
if 'bandwidthpriority' in opt_dic:
options['add']['bandwidthPriority'] = opt_dic['bandwidthpriority']
if 'maxconnections' in opt_dic:
options['add']['peer_limit'] = opt_dic['maxconnections']
if 'honourlimits' in opt_dic and not opt_dic['honourlimits']:
options['change']['honorsSessionLimits'] = False
if 'maxupspeed' in opt_dic:
options['change']['uploadLimit'] = opt_dic['maxupspeed']
options['change']['uploadLimited'] = True
if 'maxdownspeed' in opt_dic:
options['change']['downloadLimit'] = opt_dic['maxdownspeed']
options['change']['downloadLimited'] = True
if 'ratio' in opt_dic:
options['change']['seedRatioLimit'] = opt_dic['ratio']
if opt_dic['ratio'] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
options['change']['seedRatioMode'] = 2
else:
options['change']['seedRatioMode'] = 1
return options
def on_task_output(self, task, config):
import youtube_dl.YoutubeDL
from youtube_dl.utils import ExtractorError
class YoutubeDL(youtube_dl.YoutubeDL):
def __init__(self, *args, **kwargs):
self.to_stderr = self.to_screen
self.processed_info_dicts = []
super(YoutubeDL, self).__init__(*args, **kwargs)
def report_warning(self, message):
raise ExtractorError(message)
def process_info(self, info_dict):
self.processed_info_dicts.append(info_dict)
return super(YoutubeDL, self).process_info(info_dict)
for entry in task.accepted:
if task.options.test:
log.info('Would download %s' % entry['title'])
else:
try:
outtmpl = entry.render(config['path']) + '/' + pathscrub(entry.render(config['template']) + '.%(ext)s', filename=True)
log.info("Output file: %s" % outtmpl)
except RenderError as e:
log.error('Error setting output file: %s' % e)
entry.fail('Error setting output file: %s' % e)
params = {'quiet': True, 'outtmpl': outtmpl}
if 'username' in config and 'password' in config:
params.update({'username': config['username'], 'password': config['password']})
elif 'username' in config or 'password' in config:
log.error('Both username and password is required')
if 'videopassword' in config:
params.update({'videopassword': config['videopassword']})
if 'title' in config:
params.update({'title': config['title']})
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
log.info('Downloading %s' % entry['title'])
try:
ydl.download([entry['url']])
except ExtractorError as e:
log.error('Youtube-DL was unable to download the video. Error message %s' % e.message)
entry.fail('Youtube-DL was unable to download the video. Error message %s' % e.message)
except Exception as e:
log.error('Youtube-DL failed. Error message %s' % e.message)
entry.fail('Youtube-DL failed. Error message %s' % e.message)
def test_windows_filenames(self):
# Windows filename tests
# 'None' indicates there should be no changes after path scrub
win_fn = {
'afilename': 'afilename',
'filename/with/slash': 'filename with slash',
'filename\\with\\backslash': 'filename with backslash',
'afilename.': 'afilename', # filenames can't end in dot
'a<b>c:d"e/f\\g|h?i*j': 'a b c d e f g h i j', # Can't contain invalid characters
'a<<b?*?c: d': 'a b c d', # try with some repeated bad characters
'something.>': 'something', # Don't leave dots at the end
'something *': 'something', # Don't leave spaces at the end
'aoeu. > * . * <': 'aoeu' # Really don't leave spaces or dots at the end
}
for test in win_fn:
result = pathscrub(test, os='windows', filename=True)
assert result == win_fn[test], '%s != %s' % (result, win_fn[test])
def test_windows_filenames(self):
# Windows filename tests
# 'None' indicates there should be no changes after path scrub
win_fn = {
'afilename': 'afilename',
'filename/with/slash': 'filename with slash',
'filename\\with\\backslash': 'filename with backslash',
'afilename.': 'afilename', # filenames can't end in dot
'a<b>c:d"e/f\\g|h?i*j': 'a b c d e f g h i j', # Can't contain invalid characters
'a<<b?*?c: d': 'a b c d', # try with some repeated bad characters
'something.>': 'something', # Don't leave dots at the end
'something *': 'something', # Don't leave spaces at the end
'aoeu. > * . * <': 'aoeu', # Really don't leave spaces or dots at the end
}
for test in win_fn:
result = pathscrub(test, os='windows', filename=True)
assert result == win_fn[test], '%s != %s' % (result, win_fn[test])
def test_windows_filenames(self):
# Windows filename tests
# 'None' indicates there should be no changes after path scrub
win_fn = {
"afilename": "afilename",
"filename/with/slash": "filename with slash",
"filename\\with\\backslash": "filename with backslash",
"afilename.": "afilename", # filenames can't end in dot
'a<b>c:d"e/f\\g|h?i*j': "a b c d e f g h i j", # Can't contain invalid characters
"a<<b?*?c: d": "a b c d", # try with some repeated bad characters
"something.>": "something", # Don't leave dots at the end
"something *": "something", # Don't leave spaces at the end
"aoeu. > * . * <": "aoeu", # Really don't leave spaces or dots at the end
}
for test in win_fn:
result = pathscrub(test, os="windows", filename=True)
assert result == win_fn[test], "%s != %s" % (result, win_fn[test])
def on_task_download(self, task, config):
config = self.prepare_config(config, task)
for entry in task.accepted:
ftp_url = urlparse(entry.get('url'))
ftp_url = ftp_url._replace(path=unquote(ftp_url.path))
current_path = os.path.dirname(ftp_url.path)
try:
ftp = self.ftp_connect(config, ftp_url, current_path)
except ftplib.all_errors as e:
entry.fail("Unable to connect to server : %s" % (e))
break
to_path = config['ftp_tmp_path']
try:
to_path = entry.render(to_path)
except RenderError as err:
raise plugin.PluginError("Path value replacement `%s` failed: %s" % (to_path, err.args[0]))
# Clean invalid characters with pathscrub plugin
to_path = pathscrub(to_path)
if not os.path.exists(to_path):
log.debug("Creating base path: %s" % to_path)
os.makedirs(to_path)
if not os.path.isdir(to_path):
raise plugin.PluginWarning("Destination `%s` is not a directory." % to_path)
file_name = os.path.basename(ftp_url.path)
try:
# Directory
ftp = self.check_connection(ftp, config, ftp_url, current_path)
ftp.cwd(file_name)
self.ftp_walk(ftp, os.path.join(to_path, file_name), config, ftp_url, ftp_url.path)
ftp = self.check_connection(ftp, config, ftp_url, current_path)
ftp.cwd('..')
if config['delete_origin']:
ftp.rmd(file_name)
except ftplib.error_perm:
# File
self.ftp_down(ftp, file_name, to_path, config, ftp_url, current_path)
ftp.close()
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority',
'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio'):
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}}
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path'])).encode('utf-8')
options['add']['download_dir'] = pathscrub(path)
except RenderError, e:
log.error('Error setting path for %s: %s' % (entry['title'], e))
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in ('path', 'addpaused', 'honourlimits', 'bandwidthpriority',
'maxconnections', 'maxupspeed', 'maxdownspeed', 'ratio'):
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}}
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path'])).encode('utf-8')
options['add']['download_dir'] = pathscrub(path)
except RenderError, e:
log.error('Error setting path for %s: %s' % (entry['title'], e))
def on_task_output(self, task, config):
for entry in task.accepted:
path = self.prepare_path(entry, config)
try:
outtmpl = os.path.join(
path, pathscrub(entry.render(config['template'])))
logger.debug("Output template: %s" % outtmpl)
except RenderError as e:
logger.error('Error setting output file: %s' % e)
entry.fail('Error setting output file: %s' % e)
# ytdl options by default
params = {
'quiet': True,
'outtmpl': outtmpl,
'logger': logger,
'noprogress': True
}
# add config to params
if 'format' in config:
if config['format']:
params.update({'format': config['format']})
if config.get('other_options'):
params.update(config['other_options'])
logger.debug(params)
if task.options.test:
logger.info('Would download `{}` in `{}`', entry['title'],
path)
else:
logger.info('Downloading `{}` in `{}`', entry['title'], path)
try:
with self.ytdl_module.YoutubeDL(params) as ydl:
ydl.download([entry['url']])
except (self.ytdl_module.utils.ExtractorError,
self.ytdl_module.utils.DownloadError) as e:
entry.fail('Youtube downloader error: %s' % e)
except Exception as e:
entry.fail('Youtube downloader failed. Error message: %s' %
e)
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout=60):
import libtorrent
params = libtorrent.parse_magnet_uri(magnet_uri)
session = libtorrent.session()
handle = session.add_torrent(params)
log.debug('Acquiring torrent metadata for magnet {}'.format(magnet_uri))
timeout_value = timeout
while not handle.has_metadata():
time.sleep(0.1)
timeout_value -= 0.1
if timeout_value <= 0:
raise Exception('Timed out after {} seconds acquiring torrent metadata from the DHT/trackers.'.format(
timeout
))
log.debug('Metadata acquired')
torrent_info = handle.get_torrent_info()
torrent_file = libtorrent.create_torrent(torrent_info)
torrent_path = pathscrub(os.path.join(destination_folder, torrent_info.name() + ".torrent"))
with open(torrent_path, "wb") as f:
f.write(libtorrent.bencode(torrent_file.generate()))
log.debug('Torrent file wrote to {}'.format(torrent_path))
return torrent_path
def magnet_to_torrent(self, magnet_uri, destination_folder, timeout):
import libtorrent
params = libtorrent.parse_magnet_uri(magnet_uri)
session = libtorrent.session()
# for some reason the info_hash needs to be bytes but it's a struct called sha1_hash
params["info_hash"] = bytes(params["info_hash"])
handle = libtorrent.add_magnet_uri(session, magnet_uri, params)
log.debug("Acquiring torrent metadata for magnet %s", magnet_uri)
timeout_value = timeout
while not handle.has_metadata():
time.sleep(0.1)
timeout_value -= 0.1
if timeout_value <= 0:
raise plugin.PluginError("Timed out after {} seconds trying to magnetize".format(timeout))
log.debug("Metadata acquired")
torrent_info = handle.get_torrent_info()
torrent_file = libtorrent.create_torrent(torrent_info)
torrent_path = pathscrub(os.path.join(destination_folder, torrent_info.name() + ".torrent"))
with open(torrent_path, "wb") as f:
f.write(libtorrent.bencode(torrent_file.generate()))
log.debug("Torrent file wrote to %s", torrent_path)
return torrent_path
def _set_torrent_options(self, client, torrent_id, entry, opts):
"""Gets called when a torrent was added to the daemon."""
log.info('%s successfully added to deluge.', entry['title'])
entry['deluge_id'] = torrent_id
if opts.get('move_completed_path'):
client.call('core.set_torrent_move_completed', torrent_id, True)
client.call('core.set_torrent_move_completed_path', torrent_id,
opts['move_completed_path'])
log.debug('%s move on complete set to %s', entry['title'],
opts['move_completed_path'])
if opts.get('label'):
client.call('label.set_torrent', torrent_id, opts['label'])
if opts.get('queue_to_top') is not None:
if opts['queue_to_top']:
client.call('core.queue_top', [torrent_id])
log.debug('%s moved to top of queue', entry['title'])
else:
client.call('core.queue_bottom', [torrent_id])
log.debug('%s moved to bottom of queue', entry['title'])
status_keys = [
'files',
'total_size',
'save_path',
'move_on_completed_path',
'move_on_completed',
'progress',
]
status = client.call('core.get_torrent_status', torrent_id,
status_keys)
# Determine where the file should be
move_now_path = None
if opts.get('move_completed_path'):
if status['progress'] == 100:
move_now_path = opts['move_completed_path']
else:
# Deluge will unset the move completed option if we move the storage, forgo setting proper
# path, in favor of leaving proper final location.
log.debug(
'Not moving storage for %s, as this will prevent move_completed_path.',
entry['title'],
)
elif opts.get('path'):
move_now_path = opts['path']
if move_now_path and os.path.normpath(
move_now_path) != os.path.normpath(status['save_path']):
log.debug('Moving storage for %s to %s', entry['title'],
move_now_path)
client.call('core.move_storage', [torrent_id], move_now_path)
big_file_name = ''
if opts.get('content_filename') or opts.get('main_file_only'):
# find a file that makes up more than main_file_ratio (default: 90%) of the total size
main_file = None
for file in status['files']:
if file['size'] > (status['total_size'] *
opts.get('main_file_ratio')):
main_file = file
break
def file_exists(filename):
# Checks the download path as well as the move completed path for existence of the file
if os.path.exists(os.path.join(status['save_path'], filename)):
return True
elif status.get('move_on_completed') and status.get(
'move_on_completed_path'):
if os.path.exists(
os.path.join(status['move_on_completed_path'],
filename)):
return True
else:
return False
def unused_name(name):
# If on local computer, tries appending a (#) suffix until a unique filename is found
if client.host in ['127.0.0.1', 'localhost']:
counter = 2
while file_exists(name):
name = ''.join([
os.path.splitext(name)[0],
" (",
str(counter),
')',
os.path.splitext(name)[1],
])
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique when adding to a remote deluge daemon.'
)
return name
def rename(file, new_name):
# Renames a file in torrent
client.call('core.rename_files', torrent_id,
[(file['index'], new_name)])
log.debug('File %s in %s renamed to %s', file['path'],
entry['title'], new_name)
if main_file is not None:
# proceed with renaming only if such a big file is found
# find the subtitle file
keep_subs = opts.get('keep_subs')
sub_file = None
if keep_subs:
sub_exts = [".srt", ".sub"]
for file in status['files']:
ext = os.path.splitext(file['path'])[1]
if ext in sub_exts:
sub_file = file
break
# check for single file torrents so we dont add unnecessary folders
top_files_dir = "/"
if os.path.dirname(main_file['path']) is not ("" or "/"):
# check for top folder in user config
if (opts.get('content_filename') and os.path.dirname(
opts['content_filename']) is not ""):
top_files_dir = os.path.dirname(
opts['content_filename']) + "/"
else:
top_files_dir = os.path.dirname(
main_file['path']) + "/"
if opts.get('content_filename'):
# rename the main file
big_file_name = (
top_files_dir +
os.path.basename(opts['content_filename']) +
os.path.splitext(main_file['path'])[1])
big_file_name = unused_name(big_file_name)
rename(main_file, big_file_name)
# rename subs along with the main file
if sub_file is not None and keep_subs:
sub_file_name = (os.path.splitext(big_file_name)[0] +
os.path.splitext(sub_file['path'])[1])
rename(sub_file, sub_file_name)
if opts.get('main_file_only'):
# download only the main file (and subs)
file_priorities = [
1
if f == main_file or f == sub_file and keep_subs else 0
for f in status['files']
]
client.call('core.set_torrent_file_priorities', torrent_id,
file_priorities)
if opts.get('hide_sparse_files'):
# hide the other sparse files that are not supposed to download but are created anyway
# http://dev.deluge-torrent.org/ticket/1827
# Made sparse files behave better with deluge http://flexget.com/ticket/2881
sparse_files = [
f for f in status['files'] if f != main_file and (
f != sub_file or not keep_subs)
]
rename_pairs = [(
f['index'],
top_files_dir + ".sparse_files/" +
os.path.basename(f['path']),
) for f in sparse_files]
client.call('core.rename_files', torrent_id,
rename_pairs)
else:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.',
entry['title'],
opts.get('main_file_ratio') * 100,
)
container_directory = pathscrub(
entry.render(
entry.get('container_directory',
opts.get('container_directory', ''))))
if container_directory:
if big_file_name:
folder_structure = big_file_name.split(os.sep)
elif len(status['files']) > 0:
folder_structure = status['files'][0]['path'].split(os.sep)
else:
folder_structure = []
if len(folder_structure) > 1:
log.verbose('Renaming Folder %s to %s', folder_structure[0],
container_directory)
client.call('core.rename_folder', torrent_id,
folder_structure[0], container_directory)
else:
log.debug(
'container_directory specified however the torrent %s does not have a directory structure; '
'skipping folder rename',
entry['title'],
)
def on_task_output(self, task, config):
"""Add torrents to deluge at exit."""
config = self.prepare_config(config)
client = self.setup_client(config)
# don't add when learning
if task.options.learn:
return
if not config['enabled'] or not (task.accepted or task.options.test):
return
client.connect()
if task.options.test:
log.debug('Test connection to deluge daemon successful.')
client.disconnect()
return
# loop through entries to get a list of labels to add
labels = set()
for entry in task.accepted:
label = entry.get('label', config.get('label'))
if label and label.lower() != 'no label':
try:
label = self._format_label(
entry.render(entry.get('label', config.get('label'))))
log.debug('Rendered label: %s', label)
except RenderError as e:
log.error('Error rendering label `%s`: %s', label, e)
continue
labels.add(label)
if labels:
# Make sure the label plugin is available and enabled, then add appropriate labels
enabled_plugins = client.call('core.get_enabled_plugins')
label_enabled = 'Label' in enabled_plugins
if not label_enabled:
available_plugins = client.call('core.get_available_plugins')
if 'Label' in available_plugins:
log.debug('Enabling label plugin in deluge')
label_enabled = client.call('core.enable_plugin', 'Label')
else:
log.error('Label plugin is not installed in deluge')
if label_enabled:
d_labels = client.call('label.get_labels')
for label in labels:
if label not in d_labels:
log.debug('Adding the label `%s` to deluge', label)
client.call('label.add', label)
# add the torrents
torrent_ids = client.call('core.get_session_state')
for entry in task.accepted:
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(
os.path.expanduser(path))
except RenderError as e:
log.error('Could not set path for %s: %s', entry['title'], e)
for fopt, dopt in self.options.items():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {
'queue_to_top':
entry.get('queue_to_top', config.get('queue_to_top')),
'main_file_only':
entry.get('main_file_only', config.get('main_file_only',
False)),
'main_file_ratio':
entry.get('main_file_ratio', config.get('main_file_ratio')),
'hide_sparse_files':
entry.get('hide_sparse_files',
config.get('hide_sparse_files', True)),
'keep_subs':
entry.get('keep_subs', config.get('keep_subs', True)),
'container_directory':
config.get('container_directory', ''),
}
try:
label = entry.render(entry.get('label', config['label']))
modify_opts['label'] = self._format_label(label)
except RenderError as e:
log.error('Error setting label for `%s`: %s', entry['title'],
e)
try:
move_completed_path = entry.render(
entry.get('move_completed_path',
config['move_completed_path']))
modify_opts['move_completed_path'] = pathscrub(
os.path.expanduser(move_completed_path))
except RenderError as e:
log.error('Error setting move_completed_path for %s: %s',
entry['title'], e)
try:
content_filename = entry.get(
'content_filename', config.get('content_filename', ''))
modify_opts['content_filename'] = pathscrub(
entry.render(content_filename))
except RenderError as e:
log.error('Error setting content_filename for %s: %s',
entry['title'], e)
torrent_id = entry.get('deluge_id') or entry.get(
'torrent_info_hash')
torrent_id = torrent_id and torrent_id.lower()
if torrent_id in torrent_ids:
log.info('%s is already loaded in deluge, setting options',
entry['title'])
# Entry has a deluge id, verify the torrent is still in the deluge session and apply options
# Since this is already loaded in deluge, we may also need to change the path
modify_opts['path'] = add_opts.pop('download_location', None)
client.call('core.set_torrent_options', [torrent_id], add_opts)
self._set_torrent_options(client, torrent_id, entry,
modify_opts)
elif config['action'] != 'add':
log.warning(
'Cannot %s %s, because it is not loaded in deluge.',
config['action'],
entry['title'],
)
continue
else:
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
entry.fail(
'Downloaded temp file \'%s\' doesn\'t exist!' %
entry['file'])
del (entry['file'])
return
with open(entry['file'], 'rb') as f:
filedump = base64.encodestring(f.read())
log.verbose('Adding %s to deluge.', entry['title'])
added_torrent = None
if magnet:
added_torrent = client.call('core.add_torrent_magnet',
magnet, add_opts)
if config.get('magnetization_timeout'):
timeout = config['magnetization_timeout']
log.verbose('Waiting %d seconds for "%s" to magnetize',
timeout, entry['title'])
for _ in range(timeout):
time.sleep(1)
try:
status = client.call('core.get_torrent_status',
torrent_id, ['files'])
except Exception as err:
log.error('wait_for_metadata Error: %s', err)
break
if status.get('files'):
log.info('"%s" magnetization successful',
entry['title'])
break
else:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
'file list unavailable for processing.',
entry['title'],
)
else:
try:
added_torrent = client.call('core.add_torrent_file',
entry['title'], filedump,
add_opts)
except Exception as e:
log.info('%s was not added to deluge! %s',
entry['title'], e)
entry.fail('Could not be added to deluge')
if not added_torrent:
log.error('There was an error adding %s to deluge.' %
entry['title'])
else:
self._set_torrent_options(client, added_torrent, entry,
modify_opts)
if config['action'] in ('remove', 'purge'):
client.call('core.remove_torrent', torrent_id,
config['action'] == 'purge')
elif config['action'] == 'pause':
client.call('core.pause_torrent', [torrent_id])
elif config['action'] == 'resume':
client.call('core.resume_torrent', [torrent_id])
client.disconnect()
def add_to_transmission(self, cli, task, config):
"""Adds accepted entries to transmission """
for entry in task.accepted:
if task.options.test:
log.info("Would add %s to transmission" % entry["url"])
continue
# Compile user options into appripriate dict
options = self._make_torrent_options_dict(config, entry)
downloaded = not entry["url"].startswith("magnet:")
# Check that file is downloaded
if downloaded and "file" not in entry:
entry.fail("file missing?")
continue
# Verify the temp file exists
if downloaded and not os.path.exists(entry["file"]):
tmp_path = os.path.join(task.manager.config_base, "temp")
log.debug("entry: %s", entry)
log.debug("temp: %s", ", ".join(os.listdir(tmp_path)))
entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry["file"])
continue
try:
if downloaded:
with open(entry["file"], "rb") as f:
filedump = base64.b64encode(f.read()).decode("utf-8")
r = cli.add_torrent(filedump, 30, **options["add"])
else:
# we need to set paused to false so the magnetization begins immediately
options["add"]["paused"] = False
r = cli.add_torrent(entry["url"], timeout=30, **options["add"])
log.info('"%s" torrent added to transmission', entry["title"])
total_size = cli.get_torrent(r.id, ["id", "totalSize"]).totalSize
def _filter_list(list):
for item in list:
if not isinstance(item, basestring):
list.remove(item)
return list
def _find_matches(name, list):
for mask in list:
if fnmatch(name, mask):
return True
return False
def _wait_for_files(cli, r, timeout):
from time import sleep
while timeout > 0:
sleep(1)
fl = cli.get_files(r.id)
if len(fl[r.id]) > 0:
return fl
else:
timeout -= 1
return fl
skip_files = False
# Filter list because "set" plugin doesn't validate based on schema
# Skip files only used if we have no main file
if "skip_files" in options["post"]:
skip_files = True
options["post"]["skip_files"] = _filter_list(options["post"]["skip_files"])
main_id = None
find_main_file = options["post"].get("main_file_only") or "content_filename" in options["post"]
# We need to index the files if any of the following are defined
if find_main_file or skip_files:
fl = cli.get_files(r.id)
if (
"magnetization_timeout" in options["post"]
and options["post"]["magnetization_timeout"] > 0
and not downloaded
and len(fl[r.id]) == 0
):
log.debug(
'Waiting %d seconds for "%s" to magnetize',
options["post"]["magnetization_timeout"],
entry["title"],
)
fl = _wait_for_files(cli, r, options["post"]["magnetization_timeout"])
if len(fl[r.id]) == 0:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
"file list unavailable for processing.",
entry["title"],
)
else:
total_size = cli.get_torrent(r.id, ["id", "totalSize"]).totalSize
# Find files based on config
dl_list = []
skip_list = []
main_list = []
full_list = []
ext_list = ["*.srt", "*.sub", "*.idx", "*.ssa", "*.ass"]
main_ratio = config["main_file_ratio"]
if "main_file_ratio" in options["post"]:
main_ratio = options["post"]["main_file_ratio"]
if "include_files" in options["post"]:
options["post"]["include_files"] = _filter_list(options["post"]["include_files"])
for f in fl[r.id]:
full_list.append(f)
# No need to set main_id if we're not going to need it
if find_main_file and fl[r.id][f]["size"] > total_size * main_ratio:
main_id = f
if "include_files" in options["post"]:
if _find_matches(fl[r.id][f]["name"], options["post"]["include_files"]):
dl_list.append(f)
elif options["post"].get("include_subs") and _find_matches(fl[r.id][f]["name"], ext_list):
dl_list.append(f)
if skip_files:
if _find_matches(fl[r.id][f]["name"], options["post"]["skip_files"]):
skip_list.append(f)
if main_id is not None:
# Look for files matching main ID title but with a different extension
if options["post"].get("rename_like_files"):
for f in fl[r.id]:
# if this filename matches main filename we want to rename it as well
fs = os.path.splitext(fl[r.id][f]["name"])
if fs[0] == os.path.splitext(fl[r.id][main_id]["name"])[0]:
main_list.append(f)
else:
main_list = [main_id]
if main_id not in dl_list:
dl_list.append(main_id)
elif find_main_file:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.',
entry["title"],
main_ratio * 100,
)
# If we have a main file and want to rename it and associated files
if "content_filename" in options["post"] and main_id is not None:
if "download_dir" not in options["add"]:
download_dir = cli.get_session().download_dir
else:
download_dir = options["add"]["download_dir"]
# Get new filename without ext
file_ext = os.path.splitext(fl[r.id][main_id]["name"])[1]
file_path = os.path.dirname(os.path.join(download_dir, fl[r.id][main_id]["name"]))
filename = options["post"]["content_filename"]
if config["host"] == "localhost" or config["host"] == "127.0.0.1":
counter = 1
while os.path.exists(os.path.join(file_path, filename + file_ext)):
# Try appending a (#) suffix till a unique filename is found
filename = "%s(%s)" % (options["post"]["content_filename"], counter)
counter += 1
else:
log.debug(
"Cannot ensure content_filename is unique "
"when adding to a remote transmission daemon."
)
for index in main_list:
file_ext = os.path.splitext(fl[r.id][index]["name"])[1]
log.debug("File %s renamed to %s" % (fl[r.id][index]["name"], filename + file_ext))
# change to below when set_files will allow setting name, more efficient to have one call
# fl[r.id][index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8'))
try:
cli.rename_torrent_path(
r.id, fl[r.id][index]["name"], os.path.basename(str(pathscrub(filename + file_ext)))
)
except TransmissionError:
log.error("content_filename only supported with transmission 2.8+")
if options["post"].get("main_file_only") and main_id is not None:
# Set Unwanted Files
options["change"]["files_unwanted"] = [x for x in full_list if x not in dl_list]
options["change"]["files_wanted"] = dl_list
log.debug(
"Downloading %s of %s files in torrent.",
len(options["change"]["files_wanted"]),
len(full_list),
)
elif (not options["post"].get("main_file_only") or main_id is None) and skip_files:
# If no main file and we want to skip files
if len(skip_list) >= len(full_list):
log.debug(
"skip_files filter would cause no files to be downloaded; "
"including all files in torrent."
)
else:
options["change"]["files_unwanted"] = skip_list
options["change"]["files_wanted"] = [x for x in full_list if x not in skip_list]
log.debug(
"Downloading %s of %s files in torrent.",
len(options["change"]["files_wanted"]),
len(full_list),
)
# Set any changed file properties
if list(options["change"].keys()):
cli.change_torrent(r.id, 30, **options["change"])
# if addpaused was defined and set to False start the torrent;
# prevents downloading data before we set what files we want
if (
"paused" in options["post"]
and not options["post"]["paused"]
or "paused" not in options["post"]
and cli.get_session().start_added_torrents
):
cli.start_torrent(r.id)
elif options["post"].get("paused"):
log.debug("sleeping 5s to stop the torrent...")
time.sleep(5)
cli.stop_torrent(r.id)
log.info('Torrent "%s" stopped because of addpaused=yes', entry["title"])
except TransmissionError as e:
log.debug("TransmissionError", exc_info=True)
log.debug("Failed options dict: %s", options)
msg = "TransmissionError: %s" % e.message or "N/A"
log.error(msg)
entry.fail(msg)
def on_task_output(self, task, config):
if not config:
return
config = self.prepare_config(config)
existing = config['existing']
for entry in task.accepted:
if 'location' not in entry:
entry.fail('Does not have location field for symlinking')
continue
linkfrom = entry['location']
linkfrom_path, linkfrom_name = os.path.split(linkfrom)
# get the proper path and name in order of: entry, config, above split
linkto_path = entry.get('link_to', config.get('to', linkfrom_path))
if config.get('rename'):
linkto_name = config['rename']
elif entry.get('filename') and entry['filename'] != linkfrom_name:
# entry specifies different filename than what was split from the path
# since some inputs fill in filename it must be different in order to be used
linkto_name = entry['filename']
else:
linkto_name = linkfrom_name
try:
linkto_path = entry.render(linkto_path)
except RenderError as err:
raise plugin.PluginError(
'Path value replacement `%s` failed: %s' %
(linkto_path, err.args[0]))
try:
linkto_name = entry.render(linkto_name)
except RenderError as err:
raise plugin.PluginError(
'Filename value replacement `%s` failed: %s' %
(linkto_name, err.args[0]))
# Clean invalid characters with pathscrub plugin
linkto_path = pathscrub(os.path.expanduser(linkto_path))
linkto_name = pathscrub(linkto_name, filename=True)
# Join path and filename
linkto = os.path.join(linkto_path, linkto_name)
if linkto == entry['location']:
raise plugin.PluginWarning(
'source and destination are the same.')
# Hardlinks for dirs will not be failed here
if os.path.exists(linkto) and (config['link_type'] == 'soft'
or os.path.isfile(linkfrom)):
msg = 'Symlink destination %s already exists' % linkto
if existing == 'ignore':
logger.verbose(msg)
else:
entry.fail(msg)
continue
logger.verbose('{}link `{}` to `{}`', config['link_type'],
linkfrom, linkto)
try:
if config['link_type'] == 'soft':
os.symlink(linkfrom, linkto)
else:
if os.path.isdir(linkfrom):
self.hard_link_dir(linkfrom, linkto, existing)
else:
dirname = os.path.dirname(linkto)
if not os.path.exists(dirname):
os.makedirs(dirname)
os.link(linkfrom, linkto)
except OSError as e:
entry.fail('Failed to create %slink, %s' %
(config['link_type'], e))
def on_get_torrent_status(status):
"""Gets called with torrent status, including file info.
Sets the torrent options which require knowledge of the current status of the torrent."""
main_file_dlist = []
# Determine where the file should be
move_now_path = None
if opts.get('movedone'):
if status['progress'] == 100:
move_now_path = opts['movedone']
else:
# Deluge will unset the move completed option if we move the storage, forgo setting proper
# path, in favor of leaving proper final location.
log.debug(
'Not moving storage for %s, as this will prevent movedone.'
% entry['title'])
elif opts.get('path'):
move_now_path = opts['path']
if move_now_path and os.path.normpath(
move_now_path) != os.path.normpath(
status['save_path']):
main_file_dlist.append(
version_deferred.addCallback(create_path,
move_now_path))
log.debug('Moving storage for %s to %s' %
(entry['title'], move_now_path))
main_file_dlist.append(
client.core.move_storage([torrent_id], move_now_path))
big_file_name = ''
if opts.get('content_filename') or opts.get('main_file_only'):
def file_exists(filename):
# Checks the download path as well as the move completed path for existence of the file
if os.path.exists(
os.path.join(status['save_path'], filename)):
return True
elif status.get('move_on_completed') and status.get(
'move_on_completed_path'):
if os.path.exists(
os.path.join(
status['move_on_completed_path'],
filename)):
return True
else:
return False
def unused_name(name):
# If on local computer, tries appending a (#) suffix until a unique filename is found
if client.is_localhost():
counter = 2
while file_exists(name):
name = ''.join([
os.path.splitext(name)[0], " (",
str(counter), ')',
os.path.splitext(name)[1]
])
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote deluge daemon.')
return name
def rename(file, new_name):
# Renames a file in torrent
main_file_dlist.append(
client.core.rename_files(
torrent_id, [(file['index'], new_name)]))
log.debug('File %s in %s renamed to %s' %
(file['path'], entry['title'], new_name))
# find a file that makes up more than main_file_ratio (default: 90%) of the total size
main_file = None
for file in status['files']:
if file['size'] > (status['total_size'] *
opts.get('main_file_ratio')):
main_file = file
break
if main_file is not None:
# proceed with renaming only if such a big file is found
# find the subtitle file
keep_subs = opts.get('keep_subs')
sub_file = None
if keep_subs:
sub_exts = [".srt", ".sub"]
for file in status['files']:
ext = os.path.splitext(file['path'])[1]
if ext in sub_exts:
sub_file = file
break
# check for single file torrents so we dont add unnecessary folders
if (os.path.dirname(main_file['path'])
is not ("" or "/")):
# check for top folder in user config
if (opts.get('content_filename')
and os.path.dirname(
opts['content_filename']) is not ""):
top_files_dir = os.path.dirname(
opts['content_filename']) + "/"
else:
top_files_dir = os.path.dirname(
main_file['path']) + "/"
else:
top_files_dir = "/"
if opts.get('content_filename'):
# rename the main file
big_file_name = (
top_files_dir +
os.path.basename(opts['content_filename']) +
os.path.splitext(main_file['path'])[1])
big_file_name = unused_name(big_file_name)
rename(main_file, big_file_name)
# rename subs along with the main file
if sub_file is not None and keep_subs:
sub_file_name = (
os.path.splitext(big_file_name)[0] +
os.path.splitext(sub_file['path'])[1])
rename(sub_file, sub_file_name)
if opts.get('main_file_only'):
# download only the main file (and subs)
file_priorities = [
1 if f == main_file or
(f == sub_file and keep_subs) else 0
for f in status['files']
]
main_file_dlist.append(
client.core.set_torrent_file_priorities(
torrent_id, file_priorities))
if opts.get('hide_sparse_files'):
# hide the other sparse files that are not supposed to download but are created anyway
# http://dev.deluge-torrent.org/ticket/1827
# Made sparse files behave better with deluge http://flexget.com/ticket/2881
sparse_files = [
f for f in status['files']
if f != main_file and (
f != sub_file or (not keep_subs))
]
rename_pairs = [
(f['index'],
top_files_dir + ".sparse_files/" +
os.path.basename(f['path']))
for f in sparse_files
]
main_file_dlist.append(
client.core.rename_files(
torrent_id, rename_pairs))
else:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.'
% (entry['title'],
opts.get('main_file_ratio') * 100))
container_directory = pathscrub(
entry.render(
entry.get('container_directory',
config.get('container_directory', ''))))
if container_directory:
if big_file_name:
folder_structure = big_file_name.split(os.sep)
elif len(status['files']) > 0:
folder_structure = status['files'][0]['path'].split(
os.sep)
else:
folder_structure = []
if len(folder_structure) > 1:
log.verbose('Renaming Folder %s to %s',
folder_structure[0], container_directory)
main_file_dlist.append(
client.core.rename_folder(torrent_id,
folder_structure[0],
container_directory))
else:
log.debug(
'container_directory specified however the torrent %s does not have a directory structure; skipping folder rename',
entry['title'])
return defer.DeferredList(main_file_dlist)
def handle_entry(self, task, config, entry, siblings):
src = entry['location']
src_isdir = os.path.isdir(src)
src_path, src_name = os.path.split(src)
# get proper value in order of: entry, config, above split
dst_path = entry.get('path', config.get('to', src_path))
dst_name = entry.get('filename', config.get('filename', src_name))
try:
dst_path = entry.render(dst_path)
except RenderError as err:
raise plugin.PluginWarning('Path value replacement `%s` failed: %s' % (dst_path, err.args[0]))
try:
dst_name = entry.render(dst_name)
except RenderError as err:
raise plugin.PluginWarning('Filename value replacement `%s` failed: %s' % (dst_name, err.args[0]))
# Clean invalid characters with pathscrub plugin
dst_path = pathscrub(os.path.expanduser(dst_path))
dst_name = pathscrub(dst_name, filename=True)
# Join path and filename
dst = os.path.join(dst_path, dst_name)
if dst == entry['location']:
raise plugin.PluginWarning('source and destination are the same.')
if not os.path.exists(dst_path):
if task.options.test:
self.log.info('Would create `%s`' % dst_path)
else:
self.log.info('Creating destination directory `%s`' % dst_path)
os.makedirs(dst_path)
if not os.path.isdir(dst_path) and not task.options.test:
raise plugin.PluginWarning('destination `%s` is not a directory.' % dst_path)
# unpack_safety
if config.get('unpack_safety', entry.get('unpack_safety', True)):
count = 0
while True:
if count > 60 * 30:
raise plugin.PluginWarning('The task has been waiting unpacking for 30 minutes')
size = os.path.getsize(src)
time.sleep(1)
new_size = os.path.getsize(src)
if size != new_size:
if not count % 10:
self.log.verbose('File `%s` is possibly being unpacked, waiting ...' % src_name)
else:
break
count += 1
src_file, src_ext = os.path.splitext(src)
dst_file, dst_ext = os.path.splitext(dst)
# Check dst contains src_ext
if dst_ext != src_ext:
self.log.verbose('Adding extension `%s` to dst `%s`' % (src_ext, dst))
dst += src_ext
funct_name = 'move' if self.move else 'copy'
funct_done = 'moved' if self.move else 'copied'
if task.options.test:
self.log.info('Would %s `%s` to `%s`' % (funct_name, src, dst))
for s in siblings:
# we cannot rely on splitext for extensions here (subtitles may have the language code)
d = dst_file + s[len(src_file):]
self.log.info('Would also %s `%s` to `%s`' % (funct_name, s, d))
else:
# IO errors will have the entry mark failed in the base class
if self.move:
shutil.move(src, dst)
elif src_isdir:
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
self.log.info('`%s` has been %s to `%s`' % (src, funct_done, dst))
# further errors will not have any effect (the entry has been successfully moved or copied out)
for s in siblings:
# we cannot rely on splitext for extensions here (subtitles may have the language code)
d = dst_file + s[len(src_file):]
try:
if self.move:
shutil.move(s, d)
else:
shutil.copy(s, d)
self.log.info('`%s` has been %s to `%s` as well.' % (s, funct_done, d))
except Exception as err:
self.log.warning(str(err))
entry['output'] = dst
if self.move and not src_isdir:
self.clean_source(task, config, entry)
def output(self, task, entry, config):
"""Moves temp-file into final destination
Raises:
PluginError if operation fails
"""
if 'file' not in entry and not task.options.test:
log.debug('file missing, entry: %s' % entry)
raise plugin.PluginError('Entry `%s` has no temp file associated with' % entry['title'])
try:
# use path from entry if has one, otherwise use from download definition parameter
path = entry.get('path', config.get('path'))
if not isinstance(path, basestring):
raise plugin.PluginError('Invalid `path` in entry `%s`' % entry['title'])
# override path from command line parameter
if task.options.dl_path:
path = task.options.dl_path
# expand variables in path
try:
path = os.path.expanduser(entry.render(path))
except RenderError as e:
entry.fail('Could not set path. Error during string replacement: %s' % e)
return
# Clean illegal characters from path name
path = pathscrub(path)
# If we are in test mode, report and return
if task.options.test:
log.info('Would write `%s` to `%s`' % (entry['title'], path))
# Set a fake location, so the exec plugin can do string replacement during --test #1015
entry['output'] = os.path.join(path, 'TEST_MODE_NO_OUTPUT')
return
# make path
if not os.path.isdir(path):
log.debug('Creating directory %s' % path)
try:
os.makedirs(path)
except:
raise plugin.PluginError('Cannot create path %s' % path, log)
# check that temp file is present
if not os.path.exists(entry['file']):
log.debug('entry: %s' % entry)
raise plugin.PluginWarning('Downloaded temp file `%s` doesn\'t exist!?' % entry['file'])
# if we still don't have a filename, try making one from title (last resort)
if not entry.get('filename'):
entry['filename'] = entry['title']
log.debug('set filename from title %s' % entry['filename'])
if 'mime-type' not in entry:
log.warning('Unable to figure proper filename for %s. Using title.' % entry['title'])
else:
guess = mimetypes.guess_extension(entry['mime-type'])
if not guess:
log.warning('Unable to guess extension with mime-type %s' % guess)
else:
self.filename_ext_from_mime(entry)
name = entry.get('filename', entry['title'])
# Remove illegal characters from filename #325, #353
name = pathscrub(name)
# Remove directory separators from filename #208
name = name.replace('/', ' ')
if sys.platform.startswith('win'):
name = name.replace('\\', ' ')
# remove duplicate spaces
name = ' '.join(name.split())
# combine to full path + filename
destfile = os.path.join(path, name)
log.debug('destfile: %s' % destfile)
if os.path.exists(destfile):
import filecmp
if filecmp.cmp(entry['file'], destfile):
log.debug("Identical destination file '%s' already exists", destfile)
elif config.get('overwrite'):
log.debug("Overwriting already existing file %s" % destfile)
else:
log.info('File `%s` already exists and is not identical, download failed.' % destfile)
entry.fail('File `%s` already exists and is not identical.' % destfile)
return
else:
# move temp file
log.debug('moving %s to %s' % (entry['file'], destfile))
try:
shutil.move(entry['file'], destfile)
except OSError as err:
# ignore permission errors, see ticket #555
import errno
if not os.path.exists(destfile):
raise plugin.PluginError('Unable to write %s' % destfile)
if err.errno != errno.EPERM:
raise
# store final destination as output key
entry['output'] = destfile
finally:
self.cleanup_temp_file(entry)
def add_to_transmission(self, cli, task, config):
"""Adds accepted entries to transmission """
from transmissionrpc import TransmissionError
for entry in task.accepted:
if task.options.test:
log.info('Would add %s to transmission' % entry['url'])
continue
# Compile user options into appripriate dict
options = self._make_torrent_options_dict(config, entry)
downloaded = not entry['url'].startswith('magnet:')
# Check that file is downloaded
if downloaded and 'file' not in entry:
entry.fail('file missing?')
continue
# Verify the temp file exists
if downloaded and not os.path.exists(entry['file']):
tmp_path = os.path.join(task.manager.config_base, 'temp')
log.debug('entry: %s' % entry)
log.debug('temp: %s' % ', '.join(os.listdir(tmp_path)))
entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry['file'])
continue
try:
if downloaded:
with open(entry['file'], 'rb') as f:
filedump = base64.b64encode(f.read()).encode('utf-8')
r = cli.add_torrent(filedump, 30, **options['add'])
else:
r = cli.add_torrent(entry['url'], timeout=30, **options['add'])
if r:
torrent = r
log.info('"%s" torrent added to transmission' % (entry['title']))
total_size = cli.get_torrent(r.id, ['id', 'totalSize']).totalSize
def _filter_list(list):
for item in list:
if not isinstance(item, basestring):
list.remove(item)
return list
def _find_matches(name, list):
for mask in list:
if fnmatch(name, mask):
return True
return False
skip_files = False
# Filter list because "set" plugin doesn't validate based on schema
# Skip files only used if we have no main file
if 'skip_files' in options['post']:
skip_files = True
options['post']['skip_files'] = _filter_list(options['post']['skip_files'])
main_id = None
# We need to index the files if any of the following are defined
if ('main_file_only' in options['post'] and options['post']['main_file_only'] == True or
'content_filename' in options['post'] or skip_files):
fl = cli.get_files(r.id)
# Find files based on config
dl_list = []
skip_list = []
main_list = []
full_list = []
ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass']
if 'include_files' in options['post']:
include_files = True
options['post']['include_files'] = _filter_list(options['post']['include_files'])
for f in fl[r.id]:
full_list.append(f)
if fl[r.id][f]['size'] > total_size * 0.90:
main_id = f
if 'include_files' in options['post']:
if _find_matches(fl[r.id][f]['name'], options['post']['include_files']):
dl_list.append(f)
elif ('include_subs' in options['post'] and options['post']['include_subs'] == True and
_find_matches(fl[r.id][f]['name'], ext_list)):
dl_list.append(f)
if skip_files:
if _find_matches(fl[r.id][f]['name'], options['post']['skip_files']):
skip_list.append(f)
if main_id is not None:
# Look for files matching main ID title but with a different extension
if 'rename_like_files' in options['post'] and options['post']['rename_like_files'] == True:
for f in fl[r.id]:
# if this filename matches main filename we want to rename it as well
fs = os.path.splitext(fl[r.id][f]['name'])
if fs[0] == os.path.splitext(fl[r.id][main_id]['name'])[0]:
main_list.append(f)
else:
main_list = [main_id]
if main_id not in dl_list:
dl_list.append(main_id)
# If we have a main file and want to rename it and associated files
if 'content_filename' in options['post'] and main_id is not None:
download_dir = ""
if 'download_dir' not in options['add']:
download_dir = cli.get_session().download_dir
else:
download_dir = options['add']['download_dir']
# Get new filename without ext
file_ext = os.path.splitext(fl[r.id][main_id]['name'])[1]
file_path = os.path.dirname(os.path.join(download_dir, fl[r.id][main_id]['name']))
filename = options['post']['content_filename']
if config['host'] == 'localhost' or config['host'] == '127.0.0.1':
counter = 1
while os.path.exists(os.path.join(file_path, filename + file_ext)):
# Try appending a (#) suffix till a unique filename is found
filename = ''.join(options['post']['content_filename'], '(', str(counter), ')')
counter += 1
else:
log.debug('Cannot ensure content_filename is unique '
'when adding to a remote transmission daemon.')
for index in main_list:
file_ext = os.path.splitext(fl[r.id][index]['name'])[1]
log.debug('File %s renamed to %s' % (fl[r.id][index]['name'], filename + file_ext))
# change to below when set_files will allow setting name, more efficient to have one call
# fl[r.id][index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8'))
cli.rename_torrent_path(r.id, fl[r.id][index]['name'],
os.path.basename(
pathscrub(filename + file_ext).encode('utf-8'))
)
if ('main_file_only' in options['post'] and options['post']['main_file_only'] == True and
main_id is not None):
# Set Unwanted Files
options['change']['files_unwanted'] = [x for x in full_list if x not in dl_list]
options['change']['files_wanted'] = dl_list
log.debug('Downloading %s of %s files in torrent.' %
(len(options['change']['files_wanted']), len(full_list)))
elif(('main_file_only' not in options['post'] or
options['post']['main_file_only'] == False or
main_id is None) and
skip_files):
# If no main file and we want to skip files
if len(skip_list) >= len(full_list):
log.debug('skip_files filter would cause no files to be downloaded; '
'including all files in torrent.')
else:
options['change']['files_unwanted'] = skip_list
options['change']['files_wanted'] = [x for x in full_list if x not in skip_list]
log.debug('Downloading %s of %s files in torrent.'
% (len(options['change']['files_wanted']), len(full_list)))
# Set any changed file properties
if options['change'].keys():
cli.change_torrent(r.id, 30, **options['change'])
# if addpaused was defined and set to False start the torrent;
# prevents downloading data before we set what files we want
if ('paused' in options['post'] and options['post']['paused'] == False or
'paused' not in options['post'] and cli.get_session().start_added_torrents == True):
cli.start_torrent(r.id)
except TransmissionError as e:
log.debug('TransmissionError', exc_info=True)
log.debug('Failed options dict: %s' % options)
msg = 'TransmissionError: %s' % e.message or 'N/A'
log.error(msg)
entry.fail(msg)
def _set_torrent_options(self, client, torrent_id, entry, opts):
"""Gets called when a torrent was added to the daemon."""
log.info('%s successfully added to deluge.', entry['title'])
entry['deluge_id'] = torrent_id
if opts.get('move_completed_path'):
client.call('core.set_torrent_move_completed', torrent_id, True)
client.call(
'core.set_torrent_move_completed_path', torrent_id, opts['move_completed_path']
)
log.debug('%s move on complete set to %s', entry['title'], opts['move_completed_path'])
if opts.get('label'):
client.call('label.set_torrent', torrent_id, opts['label'])
if opts.get('queue_to_top') is not None:
if opts['queue_to_top']:
client.call('core.queue_top', [torrent_id])
log.debug('%s moved to top of queue', entry['title'])
else:
client.call('core.queue_bottom', [torrent_id])
log.debug('%s moved to bottom of queue', entry['title'])
status_keys = [
'files',
'total_size',
'save_path',
'move_on_completed_path',
'move_on_completed',
'progress',
]
status = client.call('core.get_torrent_status', torrent_id, status_keys)
# Determine where the file should be
move_now_path = None
if opts.get('move_completed_path'):
if status['progress'] == 100:
move_now_path = opts['move_completed_path']
else:
# Deluge will unset the move completed option if we move the storage, forgo setting proper
# path, in favor of leaving proper final location.
log.debug(
'Not moving storage for %s, as this will prevent move_completed_path.',
entry['title'],
)
elif opts.get('path'):
move_now_path = opts['path']
if move_now_path and os.path.normpath(move_now_path) != os.path.normpath(
status['save_path']
):
log.debug('Moving storage for %s to %s', entry['title'], move_now_path)
client.call('core.move_storage', [torrent_id], move_now_path)
big_file_name = ''
if opts.get('content_filename') or opts.get('main_file_only'):
# find a file that makes up more than main_file_ratio (default: 90%) of the total size
main_file = None
for file in status['files']:
if file['size'] > (status['total_size'] * opts.get('main_file_ratio')):
main_file = file
break
def file_exists(filename):
# Checks the download path as well as the move completed path for existence of the file
if os.path.exists(os.path.join(status['save_path'], filename)):
return True
elif status.get('move_on_completed') and status.get('move_on_completed_path'):
if os.path.exists(os.path.join(status['move_on_completed_path'], filename)):
return True
else:
return False
def unused_name(name):
# If on local computer, tries appending a (#) suffix until a unique filename is found
if client.host in ['127.0.0.1', 'localhost']:
counter = 2
while file_exists(name):
name = ''.join(
[
os.path.splitext(name)[0],
" (",
str(counter),
')',
os.path.splitext(name)[1],
]
)
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique when adding to a remote deluge daemon.'
)
return name
def rename(file, new_name):
# Renames a file in torrent
client.call('core.rename_files', torrent_id, [(file['index'], new_name)])
log.debug('File %s in %s renamed to %s', file['path'], entry['title'], new_name)
if main_file is not None:
# proceed with renaming only if such a big file is found
# find the subtitle file
keep_subs = opts.get('keep_subs')
sub_file = None
if keep_subs:
sub_exts = [".srt", ".sub"]
for file in status['files']:
ext = os.path.splitext(file['path'])[1]
if ext in sub_exts:
sub_file = file
break
# check for single file torrents so we dont add unnecessary folders
top_files_dir = "/"
if os.path.dirname(main_file['path']) is not ("" or "/"):
# check for top folder in user config
if (
opts.get('content_filename')
and os.path.dirname(opts['content_filename']) is not ""
):
top_files_dir = os.path.dirname(opts['content_filename']) + "/"
else:
top_files_dir = os.path.dirname(main_file['path']) + "/"
if opts.get('content_filename'):
# rename the main file
big_file_name = (
top_files_dir
+ os.path.basename(opts['content_filename'])
+ os.path.splitext(main_file['path'])[1]
)
big_file_name = unused_name(big_file_name)
rename(main_file, big_file_name)
# rename subs along with the main file
if sub_file is not None and keep_subs:
sub_file_name = (
os.path.splitext(big_file_name)[0]
+ os.path.splitext(sub_file['path'])[1]
)
rename(sub_file, sub_file_name)
if opts.get('main_file_only'):
# download only the main file (and subs)
file_priorities = [
1 if f == main_file or f == sub_file and keep_subs else 0
for f in status['files']
]
client.call('core.set_torrent_file_priorities', torrent_id, file_priorities)
if opts.get('hide_sparse_files'):
# hide the other sparse files that are not supposed to download but are created anyway
# http://dev.deluge-torrent.org/ticket/1827
# Made sparse files behave better with deluge http://flexget.com/ticket/2881
sparse_files = [
f
for f in status['files']
if f != main_file and (f != sub_file or not keep_subs)
]
rename_pairs = [
(
f['index'],
top_files_dir + ".sparse_files/" + os.path.basename(f['path']),
)
for f in sparse_files
]
client.call('core.rename_files', torrent_id, rename_pairs)
else:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.',
entry['title'],
opts.get('main_file_ratio') * 100,
)
container_directory = pathscrub(
entry.render(entry.get('container_directory', opts.get('container_directory', '')))
)
if container_directory:
if big_file_name:
folder_structure = big_file_name.split(os.sep)
elif len(status['files']) > 0:
folder_structure = status['files'][0]['path'].split(os.sep)
else:
folder_structure = []
if len(folder_structure) > 1:
log.verbose('Renaming Folder %s to %s', folder_structure[0], container_directory)
client.call(
'core.rename_folder', torrent_id, folder_structure[0], container_directory
)
else:
log.debug(
'container_directory specified however the torrent %s does not have a directory structure; '
'skipping folder rename',
entry['title'],
)
def on_task_output(self, task, config):
"""Add torrents to deluge at exit."""
config = self.prepare_config(config)
client = self.setup_client(config)
# don't add when learning
if task.options.learn:
return
if not config['enabled'] or not (task.accepted or task.options.test):
return
client.connect()
if task.options.test:
log.debug('Test connection to deluge daemon successful.')
client.disconnect()
return
# loop through entries to get a list of labels to add
labels = set()
for entry in task.accepted:
label = entry.get('label', config.get('label'))
if label and label.lower() != 'no label':
try:
label = self._format_label(
entry.render(entry.get('label', config.get('label')))
)
log.debug('Rendered label: %s', label)
except RenderError as e:
log.error('Error rendering label `%s`: %s', label, e)
continue
labels.add(label)
if labels:
# Make sure the label plugin is available and enabled, then add appropriate labels
enabled_plugins = client.call('core.get_enabled_plugins')
label_enabled = 'Label' in enabled_plugins
if not label_enabled:
available_plugins = client.call('core.get_available_plugins')
if 'Label' in available_plugins:
log.debug('Enabling label plugin in deluge')
label_enabled = client.call('core.enable_plugin', 'Label')
else:
log.error('Label plugin is not installed in deluge')
if label_enabled:
d_labels = client.call('label.get_labels')
for label in labels:
if label not in d_labels:
log.debug('Adding the label `%s` to deluge', label)
client.call('label.add', label)
# add the torrents
torrent_ids = client.call('core.get_session_state')
for entry in task.accepted:
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(os.path.expanduser(path))
except RenderError as e:
log.error('Could not set path for %s: %s', entry['title'], e)
for fopt, dopt in self.options.items():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {
'queue_to_top': entry.get('queue_to_top', config.get('queue_to_top')),
'main_file_only': entry.get('main_file_only', config.get('main_file_only', False)),
'main_file_ratio': entry.get('main_file_ratio', config.get('main_file_ratio')),
'hide_sparse_files': entry.get(
'hide_sparse_files', config.get('hide_sparse_files', True)
),
'keep_subs': entry.get('keep_subs', config.get('keep_subs', True)),
'container_directory': config.get('container_directory', ''),
}
try:
label = entry.render(entry.get('label', config['label']))
modify_opts['label'] = self._format_label(label)
except RenderError as e:
log.error('Error setting label for `%s`: %s', entry['title'], e)
try:
move_completed_path = entry.render(
entry.get('move_completed_path', config['move_completed_path'])
)
modify_opts['move_completed_path'] = pathscrub(
os.path.expanduser(move_completed_path)
)
except RenderError as e:
log.error('Error setting move_completed_path for %s: %s', entry['title'], e)
try:
content_filename = entry.get(
'content_filename', config.get('content_filename', '')
)
modify_opts['content_filename'] = pathscrub(entry.render(content_filename))
except RenderError as e:
log.error('Error setting content_filename for %s: %s', entry['title'], e)
torrent_id = entry.get('deluge_id') or entry.get('torrent_info_hash')
torrent_id = torrent_id and torrent_id.lower()
if torrent_id in torrent_ids:
log.info('%s is already loaded in deluge, setting options', entry['title'])
# Entry has a deluge id, verify the torrent is still in the deluge session and apply options
# Since this is already loaded in deluge, we may also need to change the path
modify_opts['path'] = add_opts.pop('download_location', None)
client.call('core.set_torrent_options', [torrent_id], add_opts)
self._set_torrent_options(client, torrent_id, entry, modify_opts)
elif config['action'] != 'add':
log.warning(
'Cannot %s %s, because it is not loaded in deluge.',
config['action'],
entry['title'],
)
continue
else:
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
entry.fail('Downloaded temp file \'%s\' doesn\'t exist!' % entry['file'])
del (entry['file'])
return
with open(entry['file'], 'rb') as f:
filedump = base64.encodestring(f.read())
log.verbose('Adding %s to deluge.', entry['title'])
added_torrent = None
if magnet:
added_torrent = client.call('core.add_torrent_magnet', magnet, add_opts)
if config.get('magnetization_timeout'):
timeout = config['magnetization_timeout']
log.verbose(
'Waiting %d seconds for "%s" to magnetize', timeout, entry['title']
)
for _ in range(timeout):
time.sleep(1)
try:
status = client.call(
'core.get_torrent_status', torrent_id, ['files']
)
except Exception as err:
log.error('wait_for_metadata Error: %s', err)
break
if status.get('files'):
log.info('"%s" magnetization successful', entry['title'])
break
else:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
'file list unavailable for processing.',
entry['title'],
)
else:
try:
added_torrent = client.call(
'core.add_torrent_file', entry['title'], filedump, add_opts
)
except Exception as e:
log.info('%s was not added to deluge! %s', entry['title'], e)
entry.fail('Could not be added to deluge')
if not added_torrent:
log.error('There was an error adding %s to deluge.' % entry['title'])
else:
self._set_torrent_options(client, added_torrent, entry, modify_opts)
if config['action'] in ('remove', 'purge'):
client.call('core.remove_torrent', torrent_id, config['action'] == 'purge')
elif config['action'] == 'pause':
client.call('core.pause_torrent', [torrent_id])
elif config['action'] == 'resume':
client.call('core.resume_torrent', [torrent_id])
client.disconnect()
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in (
"path",
"addpaused",
"honourlimits",
"bandwidthpriority",
"maxconnections",
"maxupspeed",
"maxdownspeed",
"ratio",
"main_file_only",
"main_file_ratio",
"magnetization_timeout",
"include_subs",
"content_filename",
"include_files",
"skip_files",
"rename_like_files",
"queue_position",
):
# Values do not merge config with task
# Task takes priority then config is used
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {"add": {}, "change": {}, "post": {}}
add = options["add"]
if opt_dic.get("path"):
try:
path = os.path.expanduser(entry.render(opt_dic["path"]))
add["download_dir"] = text_to_native_str(pathscrub(path), "utf-8")
except RenderError as e:
log.error("Error setting path for %s: %s" % (entry["title"], e))
if "bandwidthpriority" in opt_dic:
add["bandwidthPriority"] = opt_dic["bandwidthpriority"]
if "maxconnections" in opt_dic:
add["peer_limit"] = opt_dic["maxconnections"]
# make sure we add it paused, will modify status after adding
add["paused"] = True
change = options["change"]
if "honourlimits" in opt_dic and not opt_dic["honourlimits"]:
change["honorsSessionLimits"] = False
if "maxupspeed" in opt_dic:
change["uploadLimit"] = opt_dic["maxupspeed"]
change["uploadLimited"] = True
if "maxdownspeed" in opt_dic:
change["downloadLimit"] = opt_dic["maxdownspeed"]
change["downloadLimited"] = True
if "ratio" in opt_dic:
change["seedRatioLimit"] = opt_dic["ratio"]
if opt_dic["ratio"] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change["seedRatioMode"] = 2
else:
change["seedRatioMode"] = 1
if "queue_position" in opt_dic:
change["queuePosition"] = opt_dic["queue_position"]
post = options["post"]
# set to modify paused status after
if "addpaused" in opt_dic:
post["paused"] = opt_dic["addpaused"]
if "main_file_only" in opt_dic:
post["main_file_only"] = opt_dic["main_file_only"]
if "main_file_ratio" in opt_dic:
post["main_file_ratio"] = opt_dic["main_file_ratio"]
if "magnetization_timeout" in opt_dic:
post["magnetization_timeout"] = opt_dic["magnetization_timeout"]
if "include_subs" in opt_dic:
post["include_subs"] = opt_dic["include_subs"]
if "content_filename" in opt_dic:
try:
post["content_filename"] = entry.render(opt_dic["content_filename"])
except RenderError as e:
log.error("Unable to render content_filename %s: %s" % (entry["title"], e))
if "skip_files" in opt_dic:
post["skip_files"] = opt_dic["skip_files"]
if "include_files" in opt_dic:
post["include_files"] = opt_dic["include_files"]
if "rename_like_files" in opt_dic:
post["rename_like_files"] = opt_dic["rename_like_files"]
return options
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in ('path', 'addpaused', 'honourlimits',
'bandwidthpriority', 'maxconnections', 'maxupspeed',
'maxdownspeed', 'ratio', 'main_file_only',
'main_file_ratio', 'magnetization_timeout',
'include_subs', 'content_filename', 'include_files',
'skip_files', 'rename_like_files', 'queue_position'):
# Values do not merge config with task
# Task takes priority then config is used
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}, 'post': {}}
add = options['add']
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path']))
add['download_dir'] = text_to_native_str(
pathscrub(path), 'utf-8')
except RenderError as e:
log.error('Error setting path for %s: %s' %
(entry['title'], e))
if 'bandwidthpriority' in opt_dic:
add['bandwidthPriority'] = opt_dic['bandwidthpriority']
if 'maxconnections' in opt_dic:
add['peer_limit'] = opt_dic['maxconnections']
# make sure we add it paused, will modify status after adding
add['paused'] = True
change = options['change']
if 'honourlimits' in opt_dic and not opt_dic['honourlimits']:
change['honorsSessionLimits'] = False
if 'maxupspeed' in opt_dic:
change['uploadLimit'] = opt_dic['maxupspeed']
change['uploadLimited'] = True
if 'maxdownspeed' in opt_dic:
change['downloadLimit'] = opt_dic['maxdownspeed']
change['downloadLimited'] = True
if 'ratio' in opt_dic:
change['seedRatioLimit'] = opt_dic['ratio']
if opt_dic['ratio'] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change['seedRatioMode'] = 2
else:
change['seedRatioMode'] = 1
if 'queue_position' in opt_dic:
change['queuePosition'] = opt_dic['queue_position']
post = options['post']
# set to modify paused status after
if 'addpaused' in opt_dic:
post['paused'] = opt_dic['addpaused']
if 'main_file_only' in opt_dic:
post['main_file_only'] = opt_dic['main_file_only']
if 'main_file_ratio' in opt_dic:
post['main_file_ratio'] = opt_dic['main_file_ratio']
if 'magnetization_timeout' in opt_dic:
post['magnetization_timeout'] = opt_dic['magnetization_timeout']
if 'include_subs' in opt_dic:
post['include_subs'] = opt_dic['include_subs']
if 'content_filename' in opt_dic:
try:
post['content_filename'] = entry.render(
opt_dic['content_filename'])
except RenderError as e:
log.error('Unable to render content_filename %s: %s' %
(entry['title'], e))
if 'skip_files' in opt_dic:
post['skip_files'] = opt_dic['skip_files']
if 'include_files' in opt_dic:
post['include_files'] = opt_dic['include_files']
if 'rename_like_files' in opt_dic:
post['rename_like_files'] = opt_dic['rename_like_files']
return options
def handle_entry(self, task, config, entry, siblings):
src = entry['location']
src_isdir = os.path.isdir(src)
src_path, src_name = os.path.split(src)
# get the proper path and name in order of: entry, config, above split
dst_path = entry.get(self.destination_field,
config.get('to', src_path))
if config.get('rename'):
dst_name = config['rename']
elif entry.get('filename') and entry['filename'] != src_name:
# entry specifies different filename than what was split from the path
# since some inputs fill in filename it must be different in order to be used
dst_name = entry['filename']
else:
dst_name = src_name
try:
dst_path = entry.render(dst_path)
except RenderError as err:
raise plugin.PluginError('Path value replacement `%s` failed: %s' %
(dst_path, err.args[0]))
try:
dst_name = entry.render(dst_name)
except RenderError as err:
raise plugin.PluginError(
'Filename value replacement `%s` failed: %s' %
(dst_name, err.args[0]))
# Clean invalid characters with pathscrub plugin
dst_path = pathscrub(os.path.expanduser(dst_path))
dst_name = pathscrub(dst_name, filename=True)
# Join path and filename
dst = os.path.join(dst_path, dst_name)
if dst == entry['location']:
raise plugin.PluginWarning('source and destination are the same.')
if not os.path.exists(dst_path):
if task.options.test:
self.log.info('Would create `%s`', dst_path)
else:
self.log.info('Creating destination directory `%s`', dst_path)
os.makedirs(dst_path)
if not os.path.isdir(dst_path) and not task.options.test:
raise plugin.PluginWarning('destination `%s` is not a directory.' %
dst_path)
# unpack_safety
if config.get('unpack_safety', entry.get('unpack_safety', True)):
count = 0
while True:
if count > 60 * 30:
raise plugin.PluginWarning(
'The task has been waiting unpacking for 30 minutes')
size = os.path.getsize(src)
time.sleep(1)
new_size = os.path.getsize(src)
if size != new_size:
if not count % 10:
self.log.verbose(
'File `%s` is possibly being unpacked, waiting ...',
src_name)
else:
break
count += 1
src_file, src_ext = os.path.splitext(src)
dst_file, dst_ext = os.path.splitext(dst)
# Check dst contains src_ext
if config.get('keep_extension', entry.get('keep_extension', True)):
if not src_isdir and dst_ext != src_ext:
self.log.verbose('Adding extension `%s` to dst `%s`', src_ext,
dst)
dst += src_ext
dst_file += dst_ext # this is used for sibling files. dst_ext turns out not to be an extension!
funct_name = 'move' if self.move else 'copy'
funct_done = 'moved' if self.move else 'copied'
if task.options.test:
self.log.info('Would %s `%s` to `%s`', funct_name, src, dst)
for s, ext in siblings.items():
# we cannot rely on splitext for extensions here (subtitles may have the language code)
d = dst_file + ext
self.log.info('Would also %s `%s` to `%s`', funct_name, s, d)
else:
# IO errors will have the entry mark failed in the base class
if self.move:
shutil.move(src, dst)
elif src_isdir:
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
self.log.info('`%s` has been %s to `%s`', src, funct_done, dst)
# further errors will not have any effect (the entry has been successfully moved or copied out)
for s, ext in siblings.items():
# we cannot rely on splitext for extensions here (subtitles may have the language code)
d = dst_file + ext
try:
if self.move:
shutil.move(s, d)
else:
shutil.copy(s, d)
self.log.info('`%s` has been %s to `%s` as well.', s,
funct_done, d)
except Exception as err:
self.log.warning(str(err))
entry['old_location'] = entry['location']
entry['location'] = dst
if self.move and not src_isdir:
self.clean_source(task, config, entry)
def add_to_transmission(self, cli, task, config):
"""Adds accepted entries to transmission """
for entry in task.accepted:
if task.options.test:
log.info('Would add %s to transmission' % entry['url'])
continue
# Compile user options into appripriate dict
options = self._make_torrent_options_dict(config, entry)
downloaded = not entry['url'].startswith('magnet:')
# Check that file is downloaded
if downloaded and 'file' not in entry:
entry.fail('file missing?')
continue
# Verify the temp file exists
if downloaded and not os.path.exists(entry['file']):
tmp_path = os.path.join(task.manager.config_base, 'temp')
log.debug('entry: %s', entry)
log.debug('temp: %s', ', '.join(os.listdir(tmp_path)))
entry.fail("Downloaded temp file '%s' doesn't exist!?" %
entry['file'])
continue
try:
if downloaded:
with open(entry['file'], 'rb') as f:
filedump = base64.b64encode(f.read()).decode('utf-8')
r = cli.add_torrent(filedump, 30, **options['add'])
else:
# we need to set paused to false so the magnetization begins immediately
options['add']['paused'] = False
r = cli.add_torrent(entry['url'],
timeout=30,
**options['add'])
log.info('"%s" torrent added to transmission', entry['title'])
total_size = cli.get_torrent(r.id,
['id', 'totalSize']).totalSize
def _filter_list(list):
for item in list:
if not isinstance(item, basestring):
list.remove(item)
return list
def _find_matches(name, list):
for mask in list:
if fnmatch(name, mask):
return True
return False
def _wait_for_files(cli, r, timeout):
from time import sleep
while timeout > 0:
sleep(1)
fl = cli.get_files(r.id)
if len(fl[r.id]) > 0:
return fl
else:
timeout -= 1
return fl
skip_files = False
# Filter list because "set" plugin doesn't validate based on schema
# Skip files only used if we have no main file
if 'skip_files' in options['post']:
skip_files = True
options['post']['skip_files'] = _filter_list(
options['post']['skip_files'])
main_id = None
find_main_file = options['post'].get(
'main_file_only') or 'content_filename' in options['post']
# We need to index the files if any of the following are defined
if find_main_file or skip_files:
fl = cli.get_files(r.id)
if ('magnetization_timeout' in options['post']
and options['post']['magnetization_timeout'] > 0
and not downloaded and len(fl[r.id]) == 0):
log.debug('Waiting %d seconds for "%s" to magnetize',
options['post']['magnetization_timeout'],
entry['title'])
fl = _wait_for_files(
cli, r, options['post']['magnetization_timeout'])
if len(fl[r.id]) == 0:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
'file list unavailable for processing.',
entry['title'])
else:
total_size = cli.get_torrent(
r.id, ['id', 'totalSize']).totalSize
# Find files based on config
dl_list = []
skip_list = []
main_list = []
full_list = []
ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass']
main_ratio = config['main_file_ratio']
if 'main_file_ratio' in options['post']:
main_ratio = options['post']['main_file_ratio']
if 'include_files' in options['post']:
options['post']['include_files'] = _filter_list(
options['post']['include_files'])
for f in fl[r.id]:
full_list.append(f)
# No need to set main_id if we're not going to need it
if find_main_file and fl[
r.id][f]['size'] > total_size * main_ratio:
main_id = f
if 'include_files' in options['post']:
if _find_matches(fl[r.id][f]['name'],
options['post']['include_files']):
dl_list.append(f)
elif options['post'].get(
'include_subs') and _find_matches(
fl[r.id][f]['name'], ext_list):
dl_list.append(f)
if skip_files:
if _find_matches(fl[r.id][f]['name'],
options['post']['skip_files']):
skip_list.append(f)
if main_id is not None:
# Look for files matching main ID title but with a different extension
if options['post'].get('rename_like_files'):
for f in fl[r.id]:
# if this filename matches main filename we want to rename it as well
fs = os.path.splitext(fl[r.id][f]['name'])
if fs[0] == os.path.splitext(
fl[r.id][main_id]['name'])[0]:
main_list.append(f)
else:
main_list = [main_id]
if main_id not in dl_list:
dl_list.append(main_id)
elif find_main_file:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.',
entry['title'], main_ratio * 100)
# If we have a main file and want to rename it and associated files
if 'content_filename' in options[
'post'] and main_id is not None:
if 'download_dir' not in options['add']:
download_dir = cli.get_session().download_dir
else:
download_dir = options['add']['download_dir']
# Get new filename without ext
file_ext = os.path.splitext(
fl[r.id][main_id]['name'])[1]
file_path = os.path.dirname(
os.path.join(download_dir,
fl[r.id][main_id]['name']))
filename = options['post']['content_filename']
if config['host'] == 'localhost' or config[
'host'] == '127.0.0.1':
counter = 1
while os.path.exists(
os.path.join(file_path,
filename + file_ext)):
# Try appending a (#) suffix till a unique filename is found
filename = '%s(%s)' % (
options['post']['content_filename'],
counter)
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote transmission daemon.')
for index in main_list:
file_ext = os.path.splitext(
fl[r.id][index]['name'])[1]
log.debug(
'File %s renamed to %s' %
(fl[r.id][index]['name'], filename + file_ext))
# change to below when set_files will allow setting name, more efficient to have one call
# fl[r.id][index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8'))
try:
cli.rename_torrent_path(
r.id, fl[r.id][index]['name'],
os.path.basename(
str(pathscrub(filename + file_ext))))
except TransmissionError:
log.error(
'content_filename only supported with transmission 2.8+'
)
if options['post'].get(
'main_file_only') and main_id is not None:
# Set Unwanted Files
options['change']['files_unwanted'] = [
x for x in full_list if x not in dl_list
]
options['change']['files_wanted'] = dl_list
log.debug('Downloading %s of %s files in torrent.',
len(options['change']['files_wanted']),
len(full_list))
elif (not options['post'].get('main_file_only')
or main_id is None) and skip_files:
# If no main file and we want to skip files
if len(skip_list) >= len(full_list):
log.debug(
'skip_files filter would cause no files to be downloaded; '
'including all files in torrent.')
else:
options['change']['files_unwanted'] = skip_list
options['change']['files_wanted'] = [
x for x in full_list if x not in skip_list
]
log.debug('Downloading %s of %s files in torrent.',
len(options['change']['files_wanted']),
len(full_list))
# Set any changed file properties
if list(options['change'].keys()):
cli.change_torrent(r.id, 30, **options['change'])
# if addpaused was defined and set to False start the torrent;
# prevents downloading data before we set what files we want
if ('paused' in options['post']
and not options['post']['paused']
or 'paused' not in options['post']
and cli.get_session().start_added_torrents):
cli.start_torrent(r.id)
elif options['post'].get('paused'):
log.debug('sleeping 5s to stop the torrent...')
time.sleep(5)
cli.stop_torrent(r.id)
log.info('Torrent "%s" stopped because of addpaused=yes',
entry['title'])
except TransmissionError as e:
log.debug('TransmissionError', exc_info=True)
log.debug('Failed options dict: %s', options)
msg = 'TransmissionError: %s' % e.message or 'N/A'
log.error(msg)
entry.fail(msg)
def on_get_session_state(torrent_ids):
"""Gets called with a list of torrent_ids loaded in the deluge session.
Adds new torrents and modifies the settings for ones already in the session."""
dlist = []
# add the torrents
for entry in task.accepted:
@defer.inlineCallbacks
def _wait_for_metadata(torrent_id, timeout):
log.verbose('Waiting %d seconds for "%s" to magnetize' %
(timeout, entry['title']))
for _ in range(timeout):
time.sleep(1)
try:
status = yield client.core.get_torrent_status(
torrent_id, ['files'])
except Exception as err:
log.error('wait_for_metadata Error: %s' % err)
break
if status.get('files'):
log.info('"%s" magnetization successful' %
(entry['title']))
break
else:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
'file list unavailable for processing.' %
entry['title'])
defer.returnValue(torrent_id)
def add_entry(entry, opts):
"""Adds an entry to the deluge session"""
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
entry.fail(
'Downloaded temp file \'%s\' doesn\'t exist!' %
entry['file'])
del (entry['file'])
return
with open(entry['file'], 'rb') as f:
filedump = base64.encodestring(f.read())
log.verbose('Adding %s to deluge.' % entry['title'])
if magnet:
d = client.core.add_torrent_magnet(magnet, opts)
if config.get('magnetization_timeout'):
d.addCallback(_wait_for_metadata,
config['magnetization_timeout'])
return d
else:
return client.core.add_torrent_file(
entry['title'], filedump, opts)
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(
os.path.expanduser(path))
except RenderError as e:
log.error('Could not set path for %s: %s' %
(entry['title'], e))
for fopt, dopt in self.options.items():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {
'queuetotop':
entry.get('queuetotop', config.get('queuetotop')),
'main_file_only':
entry.get('main_file_only',
config.get('main_file_only', False)),
'main_file_ratio':
entry.get('main_file_ratio',
config.get('main_file_ratio')),
'hide_sparse_files':
entry.get('hide_sparse_files',
config.get('hide_sparse_files', True)),
'keep_subs':
entry.get('keep_subs', config.get('keep_subs', True))
}
try:
label = entry.render(entry.get('label', config['label']))
modify_opts['label'] = format_label(label)
except RenderError as e:
log.error('Error setting label for `%s`: %s',
entry['title'], e)
try:
movedone = entry.render(
entry.get('movedone', config['movedone']))
modify_opts['movedone'] = pathscrub(
os.path.expanduser(movedone))
except RenderError as e:
log.error('Error setting movedone for %s: %s' %
(entry['title'], e))
try:
content_filename = entry.get(
'content_filename', config.get('content_filename', ''))
modify_opts['content_filename'] = pathscrub(
entry.render(content_filename))
except RenderError as e:
log.error('Error setting content_filename for %s: %s' %
(entry['title'], e))
torrent_id = entry.get('deluge_id') or entry.get(
'torrent_info_hash')
torrent_id = torrent_id and torrent_id.lower()
if torrent_id in torrent_ids:
log.info(
'%s is already loaded in deluge, setting options' %
entry['title'])
# Entry has a deluge id, verify the torrent is still in the deluge session and apply options
# Since this is already loaded in deluge, we may also need to change the path
modify_opts['path'] = add_opts.pop('download_location',
None)
dlist.extend([
set_torrent_options(torrent_id, entry, modify_opts),
client.core.set_torrent_options([torrent_id], add_opts)
])
else:
dlist.append(
add_entry(entry, add_opts).addCallbacks(
set_torrent_options,
on_fail,
callbackArgs=(entry, modify_opts),
errbackArgs=(task, entry)))
return defer.DeferredList(dlist)
def on_task_output(self, task, config):
config = self.prepare_config(config)
# don't add when learning
if task.options.learn:
return
if not config['enabled']:
return
# Do not run if there is nothing to do
if not task.accepted:
return
if self.client is None:
self.client = self.create_rpc_client(config)
if self.client:
logger.debug('Successfully connected to transmission.')
else:
raise plugin.PluginError("Couldn't connect to transmission.")
session_torrents = self.client.get_torrents()
for entry in task.accepted:
if task.options.test:
logger.info('Would {} {} in transmission.', config['action'],
entry['title'])
continue
# Compile user options into appropriate dict
options = self._make_torrent_options_dict(config, entry)
torrent_info = None
for t in session_torrents:
if t.hashString.lower() == entry.get(
'torrent_info_hash',
'').lower() or t.id == entry.get('transmission_id'):
torrent_info = t
logger.debug(
'Found {} already loaded in transmission as {}',
entry['title'],
torrent_info.name,
)
break
if not torrent_info:
if config['action'] != 'add':
logger.warning(
'Cannot {} {} because it is not loaded in transmission.',
config['action'],
entry['title'],
)
continue
downloaded = not entry['url'].startswith('magnet:')
# Check that file is downloaded
if downloaded and 'file' not in entry:
entry.fail('`file` field missing?')
continue
# Verify the temp file exists
if downloaded and not os.path.exists(entry['file']):
tmp_path = os.path.join(task.manager.config_base, 'temp')
logger.debug('entry: {}', entry)
logger.debug('temp: {}', ', '.join(os.listdir(tmp_path)))
entry.fail("Downloaded temp file '%s' doesn't exist!?" %
entry['file'])
continue
try:
if downloaded:
with open(entry['file'], 'rb') as f:
filedump = base64.b64encode(
f.read()).decode('utf-8')
torrent_info = self.client.add_torrent(
filedump, 30, **options['add'])
else:
if options['post'].get('magnetization_timeout', 0) > 0:
options['add']['paused'] = False
torrent_info = self.client.add_torrent(
entry['url'], timeout=30, **options['add'])
except TransmissionError as e:
logger.opt(exception=True).debug('TransmissionError')
logger.debug('Failed options dict: {}', options['add'])
msg = 'Error adding {} to transmission. TransmissionError: {}'.format(
entry['title'], e.message or 'N/A')
logger.error(msg)
entry.fail(msg)
continue
logger.info('"{}" torrent added to transmission',
entry['title'])
# The info returned by the add call is incomplete, refresh it
torrent_info = self.client.get_torrent(torrent_info.id)
else:
# Torrent already loaded in transmission
if options['add'].get('download_dir'):
logger.verbose('Moving {} to "{}"', torrent_info.name,
options['add']['download_dir'])
# Move data even if current reported torrent location matches new location
# as transmission may fail to automatically move completed file to final
# location but continue reporting final location instead of real location.
# In such case this will kick transmission to really move data.
# If data is already located at new location then transmission just ignore
# this command.
self.client.move_torrent_data(
torrent_info.id, options['add']['download_dir'], 120)
try:
total_size = torrent_info.totalSize
main_id = None
find_main_file = (options['post'].get('main_file_only')
or 'content_filename' in options['post'])
skip_files = options['post'].get('skip_files')
# We need to index the files if any of the following are defined
if find_main_file or skip_files:
file_list = self.client.get_files(
torrent_info.id)[torrent_info.id]
if options['post'].get('magnetization_timeout',
0) > 0 and not file_list:
logger.debug(
'Waiting {} seconds for "{}" to magnetize',
options['post']['magnetization_timeout'],
entry['title'],
)
for _ in range(
options['post']['magnetization_timeout']):
sleep(1)
file_list = self.client.get_files(
torrent_info.id)[torrent_info.id]
if file_list:
total_size = self.client.get_torrent(
torrent_info.id,
['id', 'totalSize']).totalSize
break
else:
logger.warning(
'"{}" did not magnetize before the timeout elapsed, file list unavailable for processing.',
entry['title'],
)
# Find files based on config
dl_list = []
skip_list = []
main_list = []
ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass']
main_ratio = config['main_file_ratio']
if 'main_file_ratio' in options['post']:
main_ratio = options['post']['main_file_ratio']
for file_id, file in enumerate(file_list):
# No need to set main_id if we're not going to need it
if find_main_file and file.size > total_size * main_ratio:
main_id = file_id
if 'include_files' in options['post']:
if any(
fnmatch(file.name, mask) for mask in
options['post']['include_files']):
dl_list.append(file_id)
elif options['post'].get('include_subs') and any(
fnmatch(file.name, mask)
for mask in ext_list):
dl_list.append(file_id)
if skip_files:
if any(
fnmatch(file.name, mask)
for mask in skip_files):
skip_list.append(file_id)
if main_id is not None:
# Look for files matching main ID title but with a different extension
if options['post'].get('rename_like_files'):
for file_id, file in enumerate(file_list):
# if this filename matches main filename we want to rename it as well
fs = os.path.splitext(file.name)
if fs[0] == os.path.splitext(
file_list[main_id].name)[0]:
main_list.append(file_id)
else:
main_list = [main_id]
if main_id not in dl_list:
dl_list.append(main_id)
elif find_main_file:
logger.warning(
'No files in "{}" are > {:.0f}% of content size, no files renamed.',
entry['title'],
main_ratio * 100,
)
# If we have a main file and want to rename it and associated files
if 'content_filename' in options[
'post'] and main_id is not None:
if 'download_dir' not in options['add']:
download_dir = self.client.get_session(
).download_dir
else:
download_dir = options['add']['download_dir']
# Get new filename without ext
file_ext = os.path.splitext(file_list[main_id].name)[1]
file_path = os.path.dirname(
os.path.join(download_dir,
file_list[main_id].name))
filename = options['post']['content_filename']
if config['host'] == 'localhost' or config[
'host'] == '127.0.0.1':
counter = 1
while os.path.exists(
os.path.join(file_path,
filename + file_ext)):
# Try appending a (#) suffix till a unique filename is found
filename = f'{options["post"]["content_filename"]}({counter})'
counter += 1
else:
logger.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote transmission daemon.')
for file_id in main_list:
file_ext = os.path.splitext(
file_list[file_id].name)[1]
logger.debug(
'File {} renamed to {}',
file_list[file_id].name,
filename + file_ext,
)
# change to below when set_files will allow setting name, more efficient to have one call
# fl[index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8'))
try:
self.client.rename_torrent_path(
torrent_info.id,
file_list[file_id].name,
os.path.basename(
str(pathscrub(filename + file_ext))),
)
except TransmissionError:
logger.error(
'content_filename only supported with transmission 2.8+'
)
if options['post'].get(
'main_file_only') and main_id is not None:
# Set Unwanted Files
options['change']['files_unwanted'] = [
x for x in file_list if x not in dl_list
]
options['change']['files_wanted'] = dl_list
logger.debug(
'Downloading {} of {} files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
elif (not options['post'].get('main_file_only')
or main_id is None) and skip_files:
# If no main file and we want to skip files
if len(skip_list) >= len(file_list):
logger.debug(
'skip_files filter would cause no files to be downloaded; '
'including all files in torrent.')
else:
options['change']['files_unwanted'] = skip_list
options['change']['files_wanted'] = [
x for x in file_list if x not in skip_list
]
logger.debug(
'Downloading {} of {} files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
# Set any changed file properties
if list(options['change'].keys()):
self.client.change_torrent(torrent_info.id, 30,
**options['change'])
start_torrent = partial(self.client.start_torrent,
[torrent_info.id])
if config['action'] == 'add':
# if add_paused was defined and set to False start the torrent;
# prevents downloading data before we set what files we want
start_paused = (
options['post']['paused']
if 'paused' in options['post'] else
not self.client.get_session().start_added_torrents)
if start_paused:
self.client.stop_torrent(torrent_info.id)
else:
self.client.start_torrent(torrent_info.id)
elif config['action'] in ('remove', 'purge'):
self.client.remove_torrent(
[torrent_info.id],
delete_data=config['action'] == 'purge')
logger.info('{}d {} from transmission', config['action'],
torrent_info.name)
elif config['action'] == 'pause':
self.client.stop_torrent([torrent_info.id])
logger.info('paused {} in transmission', torrent_info.name)
elif config['action'] == 'resume':
start_torrent()
logger.info('resumed {} in transmission',
torrent_info.name)
elif config['action'] == 'bypass_queue':
start_torrent(bypass_queue=True)
logger.info('resumed (bypass queue) {} in transmission',
torrent_info.name)
except TransmissionError as e:
logger.opt(exception=True).debug('TransmissionError')
logger.debug('Failed options dict: {}', options)
msg = 'Error trying to {} {}, TransmissionError: {}'.format(
config['action'], entry['title'], e.message or 'N/A')
logger.error(msg)
continue
def filter_pathscrub(val, os_mode=None):
"""Replace problematic characters in a path."""
return pathscrub(val, os_mode)
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in (
'path',
'add_paused',
'honor_limits',
'bandwidth_priority',
'max_connections',
'max_up_speed',
'max_down_speed',
'ratio',
'main_file_only',
'main_file_ratio',
'magnetization_timeout',
'include_subs',
'content_filename',
'include_files',
'skip_files',
'rename_like_files',
'queue_position',
):
# Values do not merge config with task
# Task takes priority then config is used
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}, 'post': {}}
add = options['add']
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path']))
except RenderError as e:
logger.error('Error setting path for {}: {}', entry['title'],
e)
else:
# Transmission doesn't like it when paths end in a separator
path = path.rstrip('\\/')
add['download_dir'] = pathscrub(path)
# make sure we add it paused, will modify status after adding
add['paused'] = True
change = options['change']
if 'bandwidth_priority' in opt_dic:
change['bandwidthPriority'] = opt_dic['bandwidth_priority']
if 'honor_limits' in opt_dic and not opt_dic['honor_limits']:
change['honorsSessionLimits'] = False
if 'max_up_speed' in opt_dic:
change['uploadLimit'] = opt_dic['max_up_speed']
change['uploadLimited'] = True
if 'max_down_speed' in opt_dic:
change['downloadLimit'] = opt_dic['max_down_speed']
change['downloadLimited'] = True
if 'max_connections' in opt_dic:
change['peer_limit'] = opt_dic['max_connections']
if 'ratio' in opt_dic:
change['seedRatioLimit'] = opt_dic['ratio']
if opt_dic['ratio'] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change['seedRatioMode'] = 2
else:
change['seedRatioMode'] = 1
if 'queue_position' in opt_dic:
change['queuePosition'] = opt_dic['queue_position']
post = options['post']
# set to modify paused status after
if 'add_paused' in opt_dic:
post['paused'] = opt_dic['add_paused']
if 'main_file_only' in opt_dic:
post['main_file_only'] = opt_dic['main_file_only']
if 'main_file_ratio' in opt_dic:
post['main_file_ratio'] = opt_dic['main_file_ratio']
if 'magnetization_timeout' in opt_dic:
post['magnetization_timeout'] = opt_dic['magnetization_timeout']
if 'include_subs' in opt_dic:
post['include_subs'] = opt_dic['include_subs']
if 'content_filename' in opt_dic:
try:
post['content_filename'] = entry.render(
opt_dic['content_filename'])
except RenderError as e:
logger.error('Unable to render content_filename {}: {}',
entry['title'], e)
if 'skip_files' in opt_dic:
post['skip_files'] = opt_dic['skip_files']
if not isinstance(post['skip_files'], list):
post['skip_files'] = [post['skip_files']]
if 'include_files' in opt_dic:
post['include_files'] = opt_dic['include_files']
if not isinstance(post['include_files'], list):
post['include_files'] = [post['include_files']]
if 'rename_like_files' in opt_dic:
post['rename_like_files'] = opt_dic['rename_like_files']
return options
def filter_pathscrub(val: str, os_mode: str = None) -> str:
"""Replace problematic characters in a path."""
return pathscrub(val, os_mode)
def download_entry(self, task, entry, url, tmp_path):
"""Downloads `entry` by using `url`.
:raises: Several types of exceptions ...
:raises: PluginWarning
"""
log.debug('Downloading url \'%s\'', url)
# get content
auth = None
if 'download_auth' in entry:
auth = entry['download_auth']
log.debug('Custom auth enabled for %s download: %s',
entry['title'], entry['download_auth'])
try:
response = task.requests.get(url, auth=auth, raise_status=False)
except UnicodeError:
log.error('Unicode error while encoding url %s', url)
return
if response.status_code != 200:
log.debug('Got %s response from server. Saving error page.',
response.status_code)
# Save the error page
if response.content:
self.save_error_page(entry, task, response.content)
# Raise the error
response.raise_for_status()
return
# expand ~ in temp path
# TODO jinja?
try:
tmp_path = os.path.expanduser(tmp_path)
except RenderError as e:
entry.fail(
'Could not set temp path. Error during string replacement: %s'
% e)
return
# Clean illegal characters from temp path name
tmp_path = pathscrub(tmp_path)
# create if missing
if not os.path.isdir(tmp_path):
log.debug('creating tmp_path %s' % tmp_path)
os.mkdir(tmp_path)
# check for write-access
if not os.access(tmp_path, os.W_OK):
raise plugin.PluginError(
'Not allowed to write to temp directory `%s`' % tmp_path)
# download and write data into a temp file
tmp_dir = tempfile.mkdtemp(dir=tmp_path)
fname = hashlib.md5(url.encode('utf-8', 'replace')).hexdigest()
datafile = os.path.join(tmp_dir, fname)
outfile = io.open(datafile, 'wb')
try:
for chunk in response.iter_content(chunk_size=150 * 1024,
decode_unicode=False):
outfile.write(chunk)
except Exception as e:
# don't leave futile files behind
# outfile has to be closed before we can delete it on Windows
outfile.close()
log.debug('Download interrupted, removing datafile')
os.remove(datafile)
if isinstance(e, socket.timeout):
log.error('Timeout while downloading file')
else:
raise
else:
outfile.close()
# Do a sanity check on downloaded file
if os.path.getsize(datafile) == 0:
entry.fail('File %s is 0 bytes in size' % datafile)
os.remove(datafile)
return
# store temp filename into entry so other plugins may read and modify content
# temp file is moved into final destination at self.output
entry['file'] = datafile
log.debug('%s field file set to: %s', entry['title'],
entry['file'])
if 'content-type' in response.headers:
entry['mime-type'] = str(
parse_header(response.headers['content-type'])[0])
else:
entry['mime-type'] = "unknown/unknown"
content_encoding = response.headers.get('content-encoding', '')
decompress = 'gzip' in content_encoding or 'deflate' in content_encoding
if 'content-length' in response.headers and not decompress:
entry['content-length'] = int(response.headers['content-length'])
# prefer content-disposition naming, note: content-disposition can be disabled completely
# by setting entry field `content-disposition` to False
if entry.get('content-disposition', True):
self.filename_from_headers(entry, response)
else:
log.info('Content-disposition disabled for %s', entry['title'])
self.filename_ext_from_mime(entry)
if not entry.get('filename'):
filename = unquote(url.rsplit('/', 1)[1])
log.debug('No filename - setting from url: %s', filename)
entry['filename'] = filename
log.debug('Finishing download_entry() with filename %s',
entry.get('filename'))
def on_get_session_state(torrent_ids):
"""Gets called with a list of torrent_ids loaded in the deluge session.
Adds new torrents and modifies the settings for ones already in the session."""
dlist = []
# add the torrents
for entry in task.accepted:
def add_entry(entry, opts):
"""Adds an entry to the deluge session"""
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
entry.fail('Downloaded temp file \'%s\' doesn\'t exist!' % entry['file'])
del(entry['file'])
return
with open(entry['file'], 'rb') as f:
filedump = base64.encodestring(f.read())
log.verbose('Adding %s to deluge.' % entry['title'])
if magnet:
return client.core.add_torrent_magnet(magnet, opts)
else:
return client.core.add_torrent_file(entry['title'], filedump, opts)
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(os.path.expanduser(path))
except RenderError as e:
log.error('Could not set path for %s: %s' % (entry['title'], e))
for fopt, dopt in self.options.iteritems():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {'label': format_label(entry.get('label', config['label'])),
'queuetotop': entry.get('queuetotop', config.get('queuetotop')),
'main_file_only': entry.get('main_file_only', config.get('main_file_only', False))}
try:
movedone = entry.render(entry.get('movedone', config['movedone']))
modify_opts['movedone'] = pathscrub(os.path.expanduser(movedone))
except RenderError as e:
log.error('Error setting movedone for %s: %s' % (entry['title'], e))
try:
content_filename = entry.get('content_filename', config.get('content_filename', ''))
modify_opts['content_filename'] = pathscrub(entry.render(content_filename))
except RenderError as e:
log.error('Error setting content_filename for %s: %s' % (entry['title'], e))
torrent_id = entry.get('deluge_id') or entry.get('torrent_info_hash')
torrent_id = torrent_id and torrent_id.lower()
if torrent_id in torrent_ids:
log.info('%s is already loaded in deluge, setting options' % entry['title'])
# Entry has a deluge id, verify the torrent is still in the deluge session and apply options
# Since this is already loaded in deluge, we may also need to change the path
modify_opts['path'] = add_opts.pop('download_location', None)
dlist.extend([set_torrent_options(torrent_id, entry, modify_opts),
client.core.set_torrent_options([torrent_id], add_opts)])
else:
dlist.append(add_entry(entry, add_opts).addCallbacks(
set_torrent_options, on_fail, callbackArgs=(entry, modify_opts), errbackArgs=(task, entry)))
return defer.DeferredList(dlist)
def on_connect_success(self, result, task, config):
"""Gets called when successfully connected to a daemon."""
from deluge.ui.client import client
from twisted.internet import reactor, defer
if not result:
log.debug('on_connect_success returned a failed result. BUG?')
if task.manager.options.test:
log.debug('Test connection to deluge daemon successful.')
client.disconnect()
return
def format_label(label):
"""Makes a string compliant with deluge label naming rules"""
return re.sub('[^\w-]+', '_', label.lower())
def set_torrent_options(torrent_id, entry, opts):
"""Gets called when a torrent was added to the daemon."""
dlist = []
if not torrent_id:
log.error('There was an error adding %s to deluge.' %
entry['title'])
# TODO: Fail entry? How can this happen still now?
return
log.info('%s successfully added to deluge.' % entry['title'])
entry['deluge_id'] = torrent_id
def create_path(result, path):
"""Creates the specified path if deluge is older than 1.3"""
from deluge.common import VersionSplit
# Before 1.3, deluge would not create a non-existent move directory, so we need to.
if VersionSplit('1.3.0') > VersionSplit(self.deluge_version):
if client.is_localhost():
if not os.path.isdir(path):
log.debug('path %s doesn\'t exist, creating' %
path)
os.makedirs(path)
else:
log.warning(
'If path does not exist on the machine running the daemon, move will fail.'
)
if opts.get('movedone'):
dlist.append(
version_deferred.addCallback(create_path,
opts['movedone']))
dlist.append(
client.core.set_torrent_move_completed(torrent_id, True))
dlist.append(
client.core.set_torrent_move_completed_path(
torrent_id, opts['movedone']))
log.debug('%s move on complete set to %s' %
(entry['title'], opts['movedone']))
if opts.get('label'):
def apply_label(result, torrent_id, label):
"""Gets called after labels and torrent were added to deluge."""
return client.label.set_torrent(torrent_id, label)
dlist.append(
label_deferred.addCallback(apply_label, torrent_id,
opts['label']))
if opts.get('queuetotop') is not None:
if opts['queuetotop']:
dlist.append(client.core.queue_top([torrent_id]))
log.debug('%s moved to top of queue' % entry['title'])
else:
dlist.append(client.core.queue_bottom([torrent_id]))
log.debug('%s moved to bottom of queue' % entry['title'])
def on_get_torrent_status(status):
"""Gets called with torrent status, including file info.
Sets the torrent options which require knowledge of the current status of the torrent."""
main_file_dlist = []
# Determine where the file should be
move_now_path = None
if opts.get('movedone'):
if status['progress'] == 100:
move_now_path = opts['movedone']
else:
# Deluge will unset the move completed option if we move the storage, forgo setting proper
# path, in favor of leaving proper final location.
log.debug(
'Not moving storage for %s, as this will prevent movedone.'
% entry['title'])
elif opts.get('path'):
move_now_path = opts['path']
if move_now_path and os.path.normpath(
move_now_path) != os.path.normpath(
status['save_path']):
main_file_dlist.append(
version_deferred.addCallback(create_path,
move_now_path))
log.debug('Moving storage for %s to %s' %
(entry['title'], move_now_path))
main_file_dlist.append(
client.core.move_storage([torrent_id], move_now_path))
if opts.get('content_filename') or opts.get('main_file_only'):
def file_exists():
# Checks the download path as well as the move completed path for existence of the file
if os.path.exists(
os.path.join(status['save_path'], filename)):
return True
elif status.get('move_on_completed') and status.get(
'move_on_completed_path'):
if os.path.exists(
os.path.join(
status['move_on_completed_path'],
filename)):
return True
else:
return False
for file in status['files']:
# Only rename file if it is > 90% of the content
if file['size'] > (status['total_size'] * 0.9):
if opts.get('content_filename'):
filename = opts[
'content_filename'] + os.path.splitext(
file['path'])[1]
counter = 1
if client.is_localhost():
while file_exists():
# Try appending a (#) suffix till a unique filename is found
filename = ''.join([
opts['content_filename'], '(',
str(counter), ')',
os.path.splitext(file['path'])[1]
])
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique when adding to a remote deluge daemon.'
)
log.debug(
'File %s in %s renamed to %s' %
(file['path'], entry['title'], filename))
main_file_dlist.append(
client.core.rename_files(
torrent_id,
[(file['index'], filename)]))
if opts.get('main_file_only'):
file_priorities = [
1 if f['index'] == file['index'] else 0
for f in status['files']
]
main_file_dlist.append(
client.core.set_torrent_file_priorities(
torrent_id, file_priorities))
break
else:
log.warning(
'No files in %s are > 90%% of content size, no files renamed.'
% entry['title'])
return defer.DeferredList(main_file_dlist)
status_keys = [
'files', 'total_size', 'save_path', 'move_on_completed_path',
'move_on_completed', 'progress'
]
dlist.append(
client.core.get_torrent_status(
torrent_id,
status_keys).addCallback(on_get_torrent_status))
return defer.DeferredList(dlist)
def on_fail(result, task, entry):
"""Gets called when daemon reports a failure adding the torrent."""
log.info('%s was not added to deluge! %s' %
(entry['title'], result))
task.fail(entry, 'Could not be added to deluge')
# dlist is a list of deferreds that must complete before we exit
dlist = []
# loop through entries to get a list of labels to add
labels = set([
format_label(entry['label']) for entry in task.accepted
if entry.get('label')
])
if config.get('label'):
labels.add(format_label(config['label']))
label_deferred = defer.succeed(True)
if labels:
# Make sure the label plugin is available and enabled, then add appropriate labels
def on_get_enabled_plugins(plugins):
"""Gets called with the list of enabled deluge plugins."""
def on_label_enabled(result):
""" This runs when we verify the label plugin is enabled. """
def on_get_labels(d_labels):
"""Gets available labels from deluge, and adds any new labels we need."""
dlist = []
for label in labels:
if not label in d_labels:
log.debug('Adding the label %s to deluge' %
label)
dlist.append(client.label.add(label))
return defer.DeferredList(dlist)
return client.label.get_labels().addCallback(on_get_labels)
if 'Label' in plugins:
return on_label_enabled(True)
else:
# Label plugin isn't enabled, so we check if it's available and enable it.
def on_get_available_plugins(plugins):
"""Gets plugins available to deluge, enables Label plugin if available."""
if 'Label' in plugins:
log.debug('Enabling label plugin in deluge')
return client.core.enable_plugin(
'Label').addCallback(on_label_enabled)
else:
log.error(
'Label plugin is not installed in deluge')
return client.core.get_available_plugins().addCallback(
on_get_available_plugins)
label_deferred = client.core.get_enabled_plugins().addCallback(
on_get_enabled_plugins)
dlist.append(label_deferred)
def on_get_daemon_info(ver):
"""Gets called with the daemon version info, stores it in self."""
log.debug('deluge version %s' % ver)
self.deluge_version = ver
version_deferred = client.daemon.info().addCallback(on_get_daemon_info)
dlist.append(version_deferred)
def on_get_session_state(torrent_ids):
"""Gets called with a list of torrent_ids loaded in the deluge session.
Adds new torrents and modifies the settings for ones already in the session."""
dlist = []
# add the torrents
for entry in task.accepted:
def add_entry(entry, opts):
"""Adds an entry to the deluge session"""
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
task.fail(
entry,
'Downloaded temp file \'%s\' doesn\'t exist!' %
entry['file'])
del (entry['file'])
return
try:
f = open(entry['file'], 'rb')
filedump = base64.encodestring(f.read())
finally:
f.close()
log.verbose('Adding %s to deluge.' % entry['title'])
if magnet:
return client.core.add_torrent_magnet(magnet, opts)
else:
return client.core.add_torrent_file(
entry['title'], filedump, opts)
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(
os.path.expanduser(path))
except RenderError, e:
log.error('Could not set path for %s: %s' %
(entry['title'], e))
for fopt, dopt in self.options.iteritems():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {
'label':
format_label(entry.get('label', config['label'])),
'queuetotop':
entry.get('queuetotop', config.get('queuetotop')),
'main_file_only':
entry.get('main_file_only',
config.get('main_file_only', False))
}
try:
movedone = entry.render(
entry.get('movedone', config['movedone']))
modify_opts['movedone'] = pathscrub(
os.path.expanduser(movedone))
except RenderError, e:
log.error('Error setting movedone for %s: %s' %
(entry['title'], e))
try:
content_filename = entry.get(
'content_filename', config.get('content_filename', ''))
modify_opts['content_filename'] = pathscrub(
entry.render(content_filename))
except RenderError, e:
log.error('Error setting content_filename for %s: %s' %
(entry['title'], e))
def output(self, task, entry, config):
"""Moves temp-file into final destination
Raises:
PluginError if operation fails
"""
if 'file' not in entry and not task.options.test:
log.debug('file missing, entry: %s', entry)
raise plugin.PluginError(
'Entry `%s` has no temp file associated with' % entry['title'])
try:
# use path from entry if has one, otherwise use from download definition parameter
path = entry.get('path', config.get('path'))
if not isinstance(path, str):
raise plugin.PluginError('Invalid `path` in entry `%s`' %
entry['title'])
# override path from command line parameter
if task.options.dl_path:
path = task.options.dl_path
# expand variables in path
try:
path = os.path.expanduser(entry.render(path))
except RenderError as e:
entry.fail(
'Could not set path. Error during string replacement: %s' %
e)
return
# Clean illegal characters from path name
path = pathscrub(path)
# If we are in test mode, report and return
if task.options.test:
log.info('Would write `%s` to `%s`', entry['title'], path)
# Set a fake location, so the exec plugin can do string replacement during --test #1015
entry['location'] = os.path.join(path, 'TEST_MODE_NO_OUTPUT')
return
# make path
if not os.path.isdir(path):
log.debug('Creating directory %s', path)
try:
os.makedirs(path)
except:
raise plugin.PluginError('Cannot create path %s' % path,
log)
# check that temp file is present
if not os.path.exists(entry['file']):
log.debug('entry: %s', entry)
raise plugin.PluginWarning(
'Downloaded temp file `%s` doesn\'t exist!?' %
entry['file'])
if config.get('filename'):
try:
entry['filename'] = entry.render(config['filename'])
log.debug('set filename from config %s' %
entry['filename'])
except RenderError as e:
entry.fail(
'Could not set filename. Error during string replacement: %s'
% e)
return
# if we still don't have a filename, try making one from title (last resort)
elif not entry.get('filename'):
entry['filename'] = entry['title']
log.debug('set filename from title %s', entry['filename'])
if 'mime-type' not in entry:
log.warning(
'Unable to figure proper filename for %s. Using title.',
entry['title'])
else:
guess = mimetypes.guess_extension(entry['mime-type'])
if not guess:
log.warning(
'Unable to guess extension with mime-type %s',
guess)
else:
self.filename_ext_from_mime(entry)
name = entry.get('filename', entry['title'])
# Remove illegal characters from filename #325, #353
name = pathscrub(name)
# Remove directory separators from filename #208
name = name.replace('/', ' ')
if sys.platform.startswith('win'):
name = name.replace('\\', ' ')
# remove duplicate spaces
name = ' '.join(name.split())
# combine to full path + filename
destfile = os.path.join(path, name)
log.debug('destfile: %s', destfile)
if os.path.exists(destfile):
import filecmp
if filecmp.cmp(entry['file'], destfile):
log.debug("Identical destination file '%s' already exists",
destfile)
elif config.get('overwrite'):
log.debug("Overwriting already existing file %s", destfile)
else:
log.info(
'File `%s` already exists and is not identical, download failed.',
destfile)
entry.fail(
'File `%s` already exists and is not identical.' %
destfile)
return
else:
# move temp file
log.debug('moving %s to %s', entry['file'], destfile)
try:
shutil.move(entry['file'], destfile)
except (IOError, OSError) as err:
# ignore permission errors, see ticket #555
import errno
if not os.path.exists(destfile):
raise plugin.PluginError('Unable to write %s: %s' %
(destfile, err))
if err.errno != errno.EPERM and err.errno != errno.EACCES:
raise
# store final destination as output key
entry['location'] = destfile
finally:
self.cleanup_temp_file(entry)
def test_degenerate(self):
# If path is reduced to nothing, make sure it complains
with pytest.raises(ValueError):
pathscrub('<<<<:>>>>', os='windows', filename=True)
