def _expired(cls, file, config):
added_interval = datetime.combine(date.today(), time()) - file['added']
if file['remove_after'] and added_interval > parse_timedelta(file['remove_after']):
return True
elif config.get('remove_after') and added_interval > parse_timedelta(config['remove_after']):
return True
return False
def _expired(cls, file, config):
added_interval = datetime.combine(date.today(), time()) - file['added']
if file['remove_after'] and added_interval > parse_timedelta(file['remove_after']):
return True
elif config.get('remove_after') and added_interval > parse_timedelta(config['remove_after']):
return True
return False
def prepare_config(self, config):
settings = {}
for entry in config:
if isinstance(entry, dict):
if entry.get('field') and not entry.get('field').isspace():
key = entry.get('field')
settings[key] = entry
delta = settings[key].get('delta_distance')
if delta and isinstance(delta, str):
settings[key]['delta_distance'] = parse_timedelta(delta)
limit = settings[key].get('upper_limit')
if limit and isinstance(limit, str):
settings[key]['upper_limit'] = parse_timedelta(limit)
return settings
def prepare_config(self, config):
settings = {}
for entry in config:
if isinstance(entry, dict):
if entry.get('field') and not entry.get('field').isspace():
key = entry.get('field')
settings[key] = entry
delta = settings[key].get('delta_distance')
if delta and isinstance(delta, str):
settings[key]['delta_distance'] = parse_timedelta(
delta)
limit = settings[key].get('upper_limit')
if limit and isinstance(limit, str):
settings[key]['upper_limit'] = parse_timedelta(limit)
return settings
def consider_entry(_entry, _link):
try:
discount, seeders, leechers, hr, expired_time = NexusPHP._get_info(
task, _link, config)
except plugin.PluginError as e:
raise e
except Exception as e:
log.info('NexusPHP._get_info: ' + str(e))
return
remember = config['remember']
if config['discount']:
if discount not in config['discount']:
_entry.reject('%s does not match discount' % discount,
remember=remember) # 优惠信息不匹配
return
if config['left-time'] and expired_time:
left_time = expired_time - datetime.now()
# 实际剩余时间 < 'left-time'
if left_time < parse_timedelta(config['left-time']):
_entry.reject('its discount time only left [%s]' %
left_time,
remember=remember) # 剩余时间不足
return
if config['hr'] is False and hr:
_entry.reject('it is HR', remember=True) # 拒绝HR
if config['seeders']:
seeder_max = config['seeders']['max']
seeder_min = config['seeders']['min']
if len(seeders) not in range(seeder_min, seeder_max + 1):
_entry.reject('%d is out of range of seeder' %
len(seeders),
remember=True) # 做种人数不匹配
return
if config['leechers']:
leecher_max = config['leechers']['max']
leecher_min = config['leechers']['min']
if len(leechers) not in range(leecher_min, leecher_max + 1):
_entry.reject('%d is out of range of leecher' %
len(leechers),
remember=True) # 下载人数不匹配
return
if len(leechers) != 0:
max_complete = max(
leechers, key=lambda x: x['completed'])['completed']
else:
max_complete = 0
if max_complete > config['leechers']['max_complete']:
_entry.reject('%f is more than max_complete' %
max_complete,
remember=True) # 最大完成度不匹配
return
_entry.accept()
def on_task_urlrewrite(self, task, config):
if config is False:
return
# Create the conversion target directory
converted_path = os.path.join(task.manager.config_base, 'converted')
# Calculate the timeout config in seconds (Py2.6 compatible, can be replaced with total_seconds in 2.7)
timeout = 60
if config and isinstance(config, dict):
timeout_delta = parse_timedelta(config['timeout'])
timeout = (timeout_delta.seconds + timeout_delta.days * 24 * 3600)
if not os.path.isdir(converted_path):
os.mkdir(converted_path)
for entry in task.accepted:
if entry['url'].startswith('magnet:'):
entry.setdefault('urls', [entry['url']])
try:
log.info('Converting entry {} magnet URI to a torrent file'.format(entry['title']))
torrent_file = self.magnet_to_torrent(entry['url'], converted_path, timeout)
except BaseException as e:
message = 'Unable to convert Magnet URI for entry {}: {}'.format(entry['title'], e)
log.error(message)
continue
# Windows paths need an extra / prepended to them for url
if not torrent_file.startswith('/'):
torrent_file = '/' + torrent_file
entry['url'] = torrent_file
entry['file'] = torrent_file
entry['urls'].append('file://{}'.format(torrent_file))
def on_task_input(self, task, config):
entries = []
with Session() as session:
digest_entries = session.query(DigestEntry).filter(DigestEntry.list == config['list'])
# Remove any entries older than the expire time, if defined.
if isinstance(config.get('expire'), basestring):
expire_time = parse_timedelta(config['expire'])
digest_entries.filter(DigestEntry.added < datetime.now() - expire_time).delete()
for index, digest_entry in enumerate(
digest_entries.order_by(DigestEntry.added.desc()).all()
):
# Just remove any entries past the limit, if set.
if 0 < config.get('limit', -1) <= index:
session.delete(digest_entry)
continue
entry = digest_entry.entry
if config.get('restore_state') and entry.get('digest_state'):
# Not sure this is the best way, but we don't want hooks running on this task
# (like backlog hooking entry.fail)
entry._state = entry['digest_state']
entries.append(entry)
# If expire is 'True', we remove it after it is output once.
if config.get('expire', True) is True:
session.delete(digest_entry)
return entries
def retry_time(self, fail_count, config):
"""Return the timedelta an entry that has failed `fail_count` times before should wait before being retried."""
base_retry_time = parse_timedelta(config['retry_time'])
# Timedeltas do not allow floating point multiplication. Convert to seconds and then back to avoid this.
base_retry_secs = base_retry_time.days * 86400 + base_retry_time.seconds
retry_secs = base_retry_secs * (config['retry_time_multiplier'] ** fail_count)
return timedelta(seconds=retry_secs)
def db_cleanup(manager, session):
value = datetime.datetime.now() - parse_timedelta('7 days')
for discover_entry in (
session.query(DiscoverEntry).filter(DiscoverEntry.last_execution <= value).all()
):
log.debug('deleting %s', discover_entry)
session.delete(discover_entry)
def on_task_learn(self, task, config):
if config is False:
return
config = self.prepare_config(config)
base_retry_time = parse_timedelta(config['retry_time'])
retry_time_multiplier = config['retry_time_multiplier']
for entry in task.failed:
item = task.session.query(FailedEntry).filter(FailedEntry.title == entry['title']).\
filter(FailedEntry.url == entry['original_url']).first()
if item:
# Do not count the failure on this run when adding additional retry time
fail_count = item.count - 1
# Don't bother saving this if it has met max retries
if fail_count >= config['max_retries']:
continue
# Timedeltas do not allow floating point multiplication. Convert to seconds and then back to avoid this.
base_retry_secs = base_retry_time.days * 86400 + base_retry_time.seconds
retry_secs = base_retry_secs * (retry_time_multiplier ** fail_count)
retry_time = timedelta(seconds=retry_secs)
else:
retry_time = base_retry_time
if self.backlog:
self.backlog.instance.add_backlog(task, entry, amount=retry_time)
if retry_time:
fail_reason = item.reason if item else entry.get('reason', 'unknown')
entry.reject(reason='Waiting before trying failed entry again. (failure reason: %s)' %
fail_reason, remember_time=retry_time)
# Cause a task rerun, to look for alternate releases
task.rerun()
def on_task_download(self, task, config):
if config is False:
return
# Create the conversion target directory
converted_path = os.path.join(task.manager.config_base, 'converted')
# Calculate the timeout config in seconds (Py2.6 compatible, can be replaced with total_seconds in 2.7)
timeout = 60
if config and isinstance(config, dict):
timeout_delta = parse_timedelta(config['timeout'])
timeout = (timeout_delta.seconds + timeout_delta.days * 24 * 3600)
if not os.path.isdir(converted_path):
os.mkdir(converted_path)
for entry in task.accepted:
if entry['url'].startswith('magnet:'):
entry.setdefault('urls', [entry['url']])
try:
log.info(
'Converting entry {} magnet URI to a torrent file'.
format(entry['title']))
torrent_file = self.magnet_to_torrent(
entry['url'], converted_path, timeout)
except BaseException as e:
message = 'Unable to convert Magnet URI for entry {}: {}'.format(
entry['title'], e)
log.error(message)
continue
# Windows paths need an extra / prepended to them for url
if not torrent_file.startswith('/'):
torrent_file = '/' + torrent_file
entry['url'] = torrent_file
entry['file'] = torrent_file
# make sure it's first in the list because of how download plugin works
entry['urls'].insert(0, 'file://{}'.format(torrent_file))
def estimated(self, entries, estimation_mode):
"""
:param dict estimation_mode: mode -> loose, strict, ignore
:return: Entries that we have estimated to be available
"""
estimator = get_plugin_by_name('estimate_release').instance
result = []
for entry in entries:
est_date = estimator.estimate(entry)
if est_date is None:
log.debug('No release date could be determined for %s', entry['title'])
if estimation_mode['mode'] == 'strict':
entry.reject('has no release date')
entry.complete()
else:
result.append(entry)
continue
if isinstance(est_date, datetime.date):
# If we just got a date, add a time so we can compare it to now()
est_date = datetime.datetime.combine(est_date, datetime.time())
if datetime.datetime.now() >= est_date:
log.debug('%s has been released at %s', entry['title'], est_date)
result.append(entry)
elif datetime.datetime.now() >= est_date - parse_timedelta(estimation_mode['optimistic']):
log.debug('%s will be released at %s. Ignoring release estimation because estimated release date is '
'in less than %s', entry['title'], est_date, estimation_mode['optimistic'])
result.append(entry)
else:
entry.reject('has not been released')
entry.complete()
log.verbose("%s hasn't been released yet (Expected: %s)", entry['title'], est_date)
return result
def on_task_exit(self, task, config):
if config is False:
return
config = self.prepare_config(config)
base_retry_time = parse_timedelta(config['retry_time'])
retry_time_multiplier = config['retry_time_multiplier']
for entry in task.failed:
item = task.session.query(FailedEntry).filter(FailedEntry.title == entry['title']).\
filter(FailedEntry.url == entry['original_url']).first()
if item:
# Do not count the failure on this run when adding additional retry time
fail_count = item.count - 1
# Don't bother saving this if it has met max retries
if fail_count >= config['max_retries']:
continue
# Timedeltas do not allow floating point multiplication. Convert to seconds and then back to avoid this.
base_retry_secs = base_retry_time.days * 86400 + base_retry_time.seconds
retry_secs = base_retry_secs * (retry_time_multiplier ** fail_count)
retry_time = timedelta(seconds=retry_secs)
else:
retry_time = base_retry_time
if self.backlog:
self.backlog.add_backlog(task, entry, amount=retry_time)
if retry_time:
fail_reason = item.reason if item else entry.get('reason', 'unknown')
task.reject(entry, reason='Waiting before trying failed entry again. (failure reason: %s)' %
fail_reason, remember_time=retry_time)
# Cause a task rerun, to look for alternate releases
task.rerun()
def db_cleanup(manager, session):
value = datetime.datetime.now() - parse_timedelta('7 days')
for discover_entry in (
session.query(DiscoverEntry).filter(DiscoverEntry.last_execution <= value).all()
):
log.debug('deleting %s', discover_entry)
session.delete(discover_entry)
def on_task_urlrewrite(self, task, config):
if config is False:
return
config = self.prepare_config(config)
# Create the conversion target directory
converted_path = os.path.join(task.manager.config_base, 'converted')
timeout = parse_timedelta(config['timeout']).total_seconds()
if not os.path.isdir(converted_path):
os.mkdir(converted_path)
for entry in task.accepted:
if entry['url'].startswith('magnet:'):
entry.setdefault('urls', [entry['url']])
try:
log.info(
'Converting entry {} magnet URI to a torrent file'.
format(entry['title']))
self.process(entry, converted_path, timeout)
except (plugin.PluginError, TypeError) as e:
log.error('Unable to convert Magnet URI for entry %s: %s',
entry['title'], e)
if config['force']:
entry.fail('Magnet URI conversion failed')
continue
def parse_interval(interval_string):
"""Takes an interval string from the config and turns it into a :class:`datetime.timedelta` object."""
regexp = r'^\d+ (second|minute|hour|day|week)s?$'
if not re.match(regexp, interval_string):
raise ValueError(
"should be in format 'x (seconds|minutes|hours|days|weeks)'")
return parse_timedelta(interval_string)
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config['enabled'] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
nrat = float(config['min_ratio']) if 'min_ratio' in config else None
nfor = parse_timedelta(config['finished_for']) if 'finished_for' in config else None
delete_files = bool(config['delete_files']) if 'delete_files' in config else False
remove_ids = []
for torrent in self.client.get_torrents():
log.verbose('Torrent "%s": status: "%s" - ratio: %s - date done: %s' %
(torrent.name, torrent.status, torrent.ratio, torrent.date_done))
downloaded, dummy = self.torrent_info(torrent)
if (downloaded and ((nrat is None and nfor is None) or
(nrat and (nrat <= torrent.ratio)) or
(nfor and ((torrent.date_done + nfor) <= datetime.now())))):
if task.options.test:
log.info('Would remove finished torrent `%s` from transmission' % torrent.name)
continue
log.info('Removing finished torrent `%s` from transmission' % torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids, delete_files)
def on_task_download(self, task, config):
if config is False:
return
config = self.prepare_config(config)
# Create the conversion target directory
converted_path = os.path.join(task.manager.config_base, 'converted')
timeout = parse_timedelta(config['timeout']).total_seconds()
if not os.path.isdir(converted_path):
os.mkdir(converted_path)
for entry in task.accepted:
if entry['url'].startswith('magnet:'):
entry.setdefault('urls', [entry['url']])
try:
log.info(
'Converting entry {} magnet URI to a torrent file'.
format(entry['title']))
torrent_file = self.magnet_to_torrent(
entry['url'], converted_path, timeout)
except (plugin.PluginError, TypeError) as e:
log.error('Unable to convert Magnet URI for entry %s: %s',
entry['title'], e)
continue
# Windows paths need an extra / prepended to them for url
if not torrent_file.startswith('/'):
torrent_file = '/' + torrent_file
entry['url'] = torrent_file
entry['file'] = torrent_file
# make sure it's first in the list because of how download plugin works
entry['urls'].insert(0, 'file://{}'.format(torrent_file))
def on_task_filter(self, task, config):
for entry in task.entries:
field = config['field']
if field not in entry:
entry.fail('Field {0} does not exist'.format(field))
continue
field_value = entry[field]
if isinstance(field_value, datetime):
field_date = field_value
elif isinstance(field_value, float):
field_date = datetime.fromtimestamp(field_value)
elif isinstance(field_value, str):
try:
field_date = dateutil_parse(entry[field])
except ValueError:
log.warning('Entry %s ignored: %s is not a valid date', entry['title'], field_value)
continue
else:
log.warning('Entry %s ignored: %s is not a valid date', entry['title'], field_value)
continue
age_cutoff = datetime.now() - parse_timedelta(config['age'])
if field_date < age_cutoff:
info_string = 'Date in field `{0}` is older than {1}'.format(field, config['age'])
if config['action'] == 'accept':
entry.accept(info_string)
else:
entry.reject(info_string)
log.debug('Entry %s was %sed because date in field `%s` is older than %s', entry['title'],
config['action'], field, config['age'])
def on_entry_reject(self,
entry,
task=None,
remember=None,
remember_time=None,
**kwargs):
# We only remember rejections that specify the remember keyword argument
if not remember and not remember_time:
return
expires = None
if remember_time:
if isinstance(remember_time, basestring):
remember_time = parse_timedelta(remember_time)
expires = datetime.now() + remember_time
if not entry.get('title') or not entry.get('original_url'):
log.debug(
'Can\'t remember rejection for entry without title or url.')
return
message = 'Remembering rejection of `%s`' % entry['title']
if remember_time:
message += ' for %i minutes' % (remember_time.seconds / 60)
log.info(message)
(remember_task_id, ) = task.session.query(
RememberTask.id).filter(RememberTask.name == task.name).first()
task.session.add(
RememberEntry(title=entry['title'],
url=entry['original_url'],
task_id=remember_task_id,
rejected_by=task.current_plugin,
reason=kwargs.get('reason'),
expires=expires))
# The test stops passing when this is taken out for some reason...
task.session.flush()
def add_backlog(self, task, entry, amount='', session=None):
"""Add single entry to task backlog
If :amount: is not specified, entry will only be injected on next execution."""
snapshot = entry.snapshots.get('after_input')
if not snapshot:
if task.current_phase != 'input':
# Not having a snapshot is normal during input phase, don't display a warning
log.warning(
'No input snapshot available for `%s`, using current state'
% entry['title'])
snapshot = entry
expire_time = datetime.now() + parse_timedelta(amount)
backlog_entry = session.query(BacklogEntry).filter(BacklogEntry.title == entry['title']).\
filter(BacklogEntry.task == task.name).first()
if backlog_entry:
# If there is already a backlog entry for this, update the expiry time if necessary.
if backlog_entry.expire < expire_time:
log.debug('Updating expiry time for %s' % entry['title'])
backlog_entry.expire = expire_time
else:
log.debug('Saving %s' % entry['title'])
backlog_entry = BacklogEntry()
backlog_entry.title = entry['title']
backlog_entry.entry = snapshot
backlog_entry.task = task.name
backlog_entry.expire = expire_time
session.add(backlog_entry)
def on_task_input(self, task, config):
entries = []
with Session() as session:
digest_entries = session.query(DigestEntry).filter(
DigestEntry.list == config['list'])
# Remove any entries older than the expire time, if defined.
if isinstance(config.get('expire'), basestring):
expire_time = parse_timedelta(config['expire'])
digest_entries.filter(
DigestEntry.added < datetime.now() - expire_time).delete()
for index, digest_entry in enumerate(
digest_entries.order_by(DigestEntry.added.desc()).all()):
# Just remove any entries past the limit, if set.
if 0 < config.get('limit', -1) <= index:
session.delete(digest_entry)
continue
entry = digest_entry.entry
if config.get('restore_state') and entry.get('digest_state'):
# Not sure this is the best way, but we don't want hooks running on this task
# (like backlog hooking entry.fail)
entry._state = entry['digest_state']
entries.append(entry)
# If expire is 'True', we remove it after it is output once.
if config.get('expire', True) is True:
session.delete(digest_entry)
return entries
def lookup_series(name=None, session=None):
series = None
res = session.query(TVRageLookup).filter(TVRageLookup.name == name.lower()).first()
if res:
series = res.series
# if too old result, clean the db and refresh it
interval = parse_timedelta(update_interval)
if datetime.datetime.now() > series.last_update + interval:
log.debug('Refreshing tvrage info for %s', name)
else:
return series
log.debug('Fetching tvrage info for %s' % name)
try:
fetched = tvrage.api.Show(name.encode('utf-8'))
except tvrage.exceptions.ShowNotFound:
raise LookupError('Could not find show %s' % name)
except (timeout, AttributeError):
# AttributeError is due to a bug in tvrage package trying to access URLError.code
raise LookupError('Timed out while connecting to tvrage')
if not series:
series = session.query(TVRageSeries).filter(TVRageSeries.showid == fetched.showid).first()
if not series:
series = TVRageSeries(fetched)
session.add(series)
session.add(TVRageLookup(unicode(fetched.name), series))
else:
series.update(fetched)
if name.lower() != fetched.name.lower():
session.add(TVRageLookup(name, series))
return series
def estimated(self, entries, estimation_mode):
"""
:param dict estimation_mode: mode -> loose, strict, ignore
:return: Entries that we have estimated to be available
"""
estimator = get_plugin_by_name('estimate_release').instance
result = []
for entry in entries:
est_date = estimator.estimate(entry)
if est_date is None:
log.debug('No release date could be determined for %s', entry['title'])
if estimation_mode['mode'] == 'strict':
entry.reject('has no release date')
entry.complete()
else:
result.append(entry)
continue
if isinstance(est_date, datetime.date):
# If we just got a date, add a time so we can compare it to now()
est_date = datetime.datetime.combine(est_date, datetime.time())
if datetime.datetime.now() >= est_date:
log.debug('%s has been released at %s', entry['title'], est_date)
result.append(entry)
elif datetime.datetime.now() >= est_date - parse_timedelta(estimation_mode['optimistic']):
log.debug('%s will be released at %s. Ignoring release estimation because estimated release date is '
'in less than %s', entry['title'], est_date, estimation_mode['optimistic'])
result.append(entry)
else:
entry.reject('has not been released')
entry.complete()
log.verbose("%s hasn't been released yet (Expected: %s)", entry['title'], est_date)
return result
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config['enabled'] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
nrat = float(config['min_ratio']) if 'min_ratio' in config else None
nfor = parse_timedelta(config['finished_for']) if 'finished_for' in config else None
delete_files = bool(config['delete_files']) if 'delete_files' in config else False
trans_checks = bool(config['transmission_seed_limits']) if 'transmission_seed_limits' in config else False
session = self.client.get_session()
remove_ids = []
for torrent in self.client.get_torrents():
log.verbose('Torrent "%s": status: "%s" - ratio: %s - date done: %s' %
(torrent.name, torrent.status, torrent.ratio, torrent.date_done))
downloaded, dummy = self.torrent_info(torrent)
seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session)
if (downloaded and ((nrat is None and nfor is None and trans_checks is None) or
(trans_checks and ((seed_ratio_ok is None and idle_limit_ok is None) or
(seed_ratio_ok is True or idle_limit_ok is True))) or
(nrat and (nrat <= torrent.ratio)) or
(nfor and ((torrent.date_done + nfor) <= datetime.now())))):
if task.options.test:
log.info('Would remove finished torrent `%s` from transmission' % torrent.name)
continue
log.info('Removing finished torrent `%s` from transmission' % torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids, delete_files)
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config["enabled"] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
nrat = float(config["min_ratio"]) if "min_ratio" in config else None
nfor = parse_timedelta(config["finished_for"]) if "finished_for" in config else None
remove_ids = []
for torrent in self.client.get_torrents():
log.verbose(
'Torrent "%s": status: "%s" - ratio: %s - date done: %s'
% (torrent.name, torrent.status, torrent.ratio, torrent.date_done)
)
downloaded, dummy = self.torrent_info(torrent)
if downloaded and (
(nrat is None and nfor is None)
or (nrat and (nrat <= torrent.ratio))
or (nfor and ((torrent.date_done + nfor) <= datetime.now()))
):
if task.options.test:
log.info("Would remove finished torrent `%s` from transmission" % torrent.name)
continue
log.info("Removing finished torrent `%s` from transmission" % torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids)
def on_entry_reject(self, entry, task=None, remember=None, remember_time=None, **kwargs):
# We only remember rejections that specify the remember keyword argument
if not remember and not remember_time:
return
expires = None
if remember_time:
if isinstance(remember_time, basestring):
remember_time = parse_timedelta(remember_time)
expires = datetime.now() + remember_time
if not entry.get("title") or not entry.get("original_url"):
log.debug("Can't remember rejection for entry without title or url.")
return
message = "Remembering rejection of `%s`" % entry["title"]
if remember_time:
message += " for %i minutes" % (remember_time.seconds / 60)
log.info(message)
(remember_task_id,) = task.session.query(RememberTask.id).filter(RememberTask.name == task.name).first()
task.session.add(
RememberEntry(
title=entry["title"],
url=entry["original_url"],
task_id=remember_task_id,
rejected_by=task.current_plugin,
reason=kwargs.get("reason"),
expires=expires,
)
)
# The test stops passing when this is taken out for some reason...
task.session.flush()
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config['enabled'] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
tracker_re = re.compile(config['tracker'], re.IGNORECASE) if 'tracker' in config else None
preserve_tracker_re = (
re.compile(config['preserve_tracker'], re.IGNORECASE)
if 'preserve_tracker' in config
else None
)
session = self.client.get_session()
remove_ids = []
for torrent in self.client.get_torrents():
logger.verbose(
'Torrent "{}": status: "{}" - ratio: {} - date added: {}',
torrent.name,
torrent.status,
torrent.ratio,
torrent.date_added,
)
downloaded, dummy = self.torrent_info(torrent, config)
if not downloaded:
continue
if config.get('transmission_seed_limits'):
seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session)
if not seed_ratio_ok or not idle_limit_ok:
continue
if 'min_ratio' in config:
if torrent.ratio < config['min_ratio']:
continue
if 'finished_for' in config:
# done date might be invalid if this torrent was added to transmission when already completed
started_seeding = datetime.fromtimestamp(max(torrent.addedDate, torrent.doneDate))
if started_seeding + parse_timedelta(config['finished_for']) > datetime.now():
continue
tracker_hosts = (
urlparse(tracker['announce']).hostname for tracker in torrent.trackers
)
if 'tracker' in config:
if not any(tracker_re.search(tracker) for tracker in tracker_hosts):
continue
if 'preserve_tracker' in config:
if any(preserve_tracker_re.search(tracker) for tracker in tracker_hosts):
continue
if config.get('directories'):
if not any(
re.search(d, torrent.downloadDir, re.IGNORECASE) for d in config['directories']
):
continue
if task.options.test:
logger.info('Would remove finished torrent `{}` from transmission', torrent.name)
continue
logger.info('Removing finished torrent `{}` from transmission', torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids, config.get('delete_files'))
def retry_time(self, fail_count, config):
"""Return the timedelta an entry that has failed `fail_count` times before should wait before being retried."""
base_retry_time = parse_timedelta(config['retry_time'])
# Timedeltas do not allow floating point multiplication. Convert to seconds and then back to avoid this.
base_retry_secs = base_retry_time.days * 86400 + base_retry_time.seconds
retry_secs = base_retry_secs * (config['retry_time_multiplier'] ** fail_count)
return timedelta(seconds=retry_secs)
def add_backlog(self, task, entry, amount='', session=None):
"""Add single entry to task backlog
If :amount: is not specified, entry will only be injected on next execution."""
snapshot = entry.snapshots.get('after_input')
if not snapshot:
if task.current_phase != 'input':
# Not having a snapshot is normal during input phase, don't display a warning
log.warning('No input snapshot available for `%s`, using current state' % entry['title'])
snapshot = entry
expire_time = datetime.now() + parse_timedelta(amount)
backlog_entry = session.query(BacklogEntry).filter(BacklogEntry.title == entry['title']). \
filter(BacklogEntry.task == task.name).first()
if backlog_entry:
# If there is already a backlog entry for this, update the expiry time if necessary.
if backlog_entry.expire < expire_time:
log.debug('Updating expiry time for %s' % entry['title'])
backlog_entry.expire = expire_time
else:
log.debug('Saving %s' % entry['title'])
backlog_entry = BacklogEntry()
backlog_entry.title = entry['title']
backlog_entry.entry = snapshot
backlog_entry.task = task.name
backlog_entry.expire = expire_time
session.add(backlog_entry)
def on_task_download(self, task, config):
if config is False:
return
config = self.prepare_config(config)
# Create the conversion target directory
converted_path = os.path.join(task.manager.config_base, "converted")
timeout = parse_timedelta(config["timeout"]).total_seconds()
if not os.path.isdir(converted_path):
os.mkdir(converted_path)
for entry in task.accepted:
if entry["url"].startswith("magnet:"):
entry.setdefault("urls", [entry["url"]])
try:
log.info("Converting entry {} magnet URI to a torrent file".format(entry["title"]))
torrent_file = self.magnet_to_torrent(entry["url"], converted_path, timeout)
except (plugin.PluginError, TypeError) as e:
log.error("Unable to convert Magnet URI for entry %s: %s", entry["title"], e)
continue
# Windows paths need an extra / prepended to them for url
if not torrent_file.startswith("/"):
torrent_file = "/" + torrent_file
entry["url"] = torrent_file
entry["file"] = torrent_file
# make sure it's first in the list because of how download plugin works
entry["urls"].insert(0, "file://{}".format(torrent_file))
def on_task_start(self, task, config):
# Allow reruns
if task.is_rerun:
return
if task.manager.options.learn:
log.info('Ignoring task %s interval for --learn' % task.name)
return
last_time = task.simple_persistence.get('last_time')
if not last_time:
log.info('No previous run recorded, running now')
elif task.manager.options.interval_ignore:
log.info('Ignoring interval because of --now')
else:
log.debug('last_time: %r' % last_time)
log.debug('interval: %s' % config)
next_time = last_time + parse_timedelta(config)
log.debug('next_time: %r' % next_time)
if datetime.datetime.now() < next_time:
log.debug('interval not met')
log.verbose(
'Interval %s not met on task %s. Use --now to override.' %
(config, task.name))
task.abort('Interval not met', silent=True)
return
log.debug('interval passed')
task.simple_persistence['last_time'] = datetime.datetime.now()
def lookup_series(name=None, session=None):
series = None
res = session.query(TVRageLookup).filter(
TVRageLookup.name == name.lower()).first()
if res and not res.series:
# The lookup failed in the past for this series, retry every week
# TODO: 1.2 this should also retry with --retry or whatever flag imdb lookup is using for that
if res.failed_time and res.failed_time > datetime.datetime.now(
) - datetime.timedelta(days=7):
raise LookupError('Could not find show %s' % name)
elif res:
series = res.series
# if too old result, clean the db and refresh it
interval = parse_timedelta(UPDATE_INTERVAL)
if datetime.datetime.now() > series.last_update + interval:
log.debug('Refreshing tvrage info for %s', name)
else:
return series
def store_failed_lookup():
if res:
res.series = None
res.failed_time = datetime.datetime.now()
else:
session.add(
TVRageLookup(name, None, failed_time=datetime.datetime.now()))
session.commit()
log.debug('Fetching tvrage info for %s' % name)
try:
fetched = tvrage.api.Show(name.encode('utf-8'))
except tvrage.exceptions.ShowNotFound:
store_failed_lookup()
raise LookupError('Could not find show %s' % name)
except (timeout, AttributeError):
# AttributeError is due to a bug in tvrage package trying to access URLError.code
raise LookupError('Timed out while connecting to tvrage')
# Make sure the result is close enough to the search
if difflib.SequenceMatcher(a=name, b=fetched.name).ratio() < 0.7:
log.debug('Show result `%s` was not a close enough match for `%s`' %
(fetched.name, name))
store_failed_lookup()
raise LookupError('Could not find show %s' % name)
if not series:
series = session.query(TVRageSeries).filter(
TVRageSeries.showid == fetched.showid).first()
if not series:
series = TVRageSeries(fetched)
session.add(series)
session.add(TVRageLookup(unicode(fetched.name), series))
else:
series.update(fetched)
if name.lower() != fetched.name.lower():
if res:
res.series = series
else:
session.add(TVRageLookup(name, series))
return series
def lookup_series(name=None, session=None):
series = None
res = session.query(TVRageLookup).filter(TVRageLookup.name == name.lower()).first()
if res and not res.series:
# The lookup failed in the past for this series, retry every week
# TODO: 1.2 this should also retry with --retry or whatever flag imdb lookup is using for that
if res.failed_time and res.failed_time > datetime.datetime.now() - datetime.timedelta(days=7):
raise LookupError('Could not find show %s' % name)
elif res:
series = res.series
# if too old result, clean the db and refresh it
interval = parse_timedelta(UPDATE_INTERVAL)
if datetime.datetime.now() > series.last_update + interval:
log.debug('Refreshing tvrage info for %s', name)
else:
return series
def store_failed_lookup():
if res:
res.series = None
res.failed_time = datetime.datetime.now()
else:
session.add(TVRageLookup(name, None, failed_time=datetime.datetime.now()))
session.commit()
log.debug('Fetching tvrage info for %s' % name)
try:
fetched = tvrage.api.Show(name.encode('utf-8'))
except tvrage.exceptions.ShowNotFound:
store_failed_lookup()
raise LookupError('Could not find show %s' % name)
except (timeout, AttributeError):
# AttributeError is due to a bug in tvrage package trying to access URLError.code
raise LookupError('Timed out while connecting to tvrage')
except TypeError:
# TODO: There should be option to pass tvrage id directly from within series plugin via "set:" (like tvdb_id)
# and search directly for tvrage id. This is problematic, because 3rd party TVRage API does not support this.
raise LookupError('Returned invalid data for "%s". This is often caused when TVRage is missing episode info'
% name)
# Make sure the result is close enough to the search
if difflib.SequenceMatcher(a=name, b=fetched.name).ratio() < 0.7:
log.debug('Show result `%s` was not a close enough match for `%s`' % (fetched.name, name))
store_failed_lookup()
raise LookupError('Could not find show %s' % name)
if not series:
series = session.query(TVRageSeries).filter(TVRageSeries.showid == fetched.showid).first()
if not series:
series = TVRageSeries(fetched)
session.add(series)
session.add(TVRageLookup(unicode(fetched.name), series))
else:
series.update(fetched)
if name.lower() != fetched.name.lower():
if res:
res.series = series
else:
session.add(TVRageLookup(name, series))
return series
def test_rejected_pagination(self, api_client, link_headers):
base_reject_entry = dict(title='test_title_',
url='test_url_',
rejected_by='rejected_by_',
reason='reason_')
number_of_entries = 200
with Session() as session:
task = RememberTask(name='rejected API test')
session.add(task)
session.commit()
for i in range(number_of_entries):
r_entry = copy.deepcopy(base_reject_entry)
for key, value in r_entry.items():
r_entry[key] = value + str(i)
expires = datetime.now() + parse_timedelta('1 hours')
session.add(
RememberEntry(expires=expires, task_id=task.id, **r_entry))
# Default values
rsp = api_client.get('/rejected/')
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
data = json.loads(rsp.get_data(as_text=True))
assert len(data) == 50
assert int(rsp.headers['total-count']) == 200
assert int(rsp.headers['count']) == 50
links = link_headers(rsp)
assert links['last']['page'] == 4
assert links['next']['page'] == 2
# Change page size
rsp = api_client.get('/rejected/?per_page=100')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
assert len(data) == 100
assert int(rsp.headers['total-count']) == 200
assert int(rsp.headers['count']) == 100
links = link_headers(rsp)
assert links['last']['page'] == 2
assert links['next']['page'] == 2
# Get different page
rsp = api_client.get('/rejected/?page=2')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
assert len(data) == 50
assert int(rsp.headers['total-count']) == 200
assert int(rsp.headers['count']) == 50
links = link_headers(rsp)
assert links['last']['page'] == 4
assert links['next']['page'] == 3
assert links['prev']['page'] == 1
def set_domain_delay(self, domain, delay):
"""
Registers a minimum interval between requests to `domain`
:param domain: The domain to set the interval on
:param delay: The amount of time between requests, can be a timedelta or string like '3 seconds'
"""
self.domain_delay[domain] = {'delay': parse_timedelta(delay)}
def set_domain_delay(self, domain, delay):
"""
Registers a minimum interval between requests to `domain`
:param domain: The domain to set the interval on
:param delay: The amount of time between requests, can be a timedelta or string like '3 seconds'
"""
self.domain_delay[domain] = {'delay': parse_timedelta(delay)}
def __init__(self, name, persist=None):
# Cast name to unicode to prevent sqlalchemy warnings when filtering
self.name = str(name)
# Parse persist time
self.persist = persist and parse_timedelta(persist)
# Will be set when wrapped function is called
self.config_hash = None
self.cache_name = None
def __init__(self, name: str, persist: str = None) -> None:
# Cast name to unicode to prevent sqlalchemy warnings when filtering
self.name = str(name)
# Parse persist time
self.persist: Optional[timedelta] = parse_timedelta(
persist) if persist else None
# Will be set when wrapped function is called
self.config_hash = None
self.cache_name = None
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config['enabled'] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
tracker_re = re.compile(config['tracker'], re.IGNORECASE) if 'tracker' in config else None
preserve_tracker_re = (
re.compile(config['preserve_tracker'], re.IGNORECASE)
if 'preserve_tracker' in config
else None
)
session = self.client.get_session()
remove_ids = []
for torrent in self.client.get_torrents():
log.verbose(
'Torrent "%s": status: "%s" - ratio: %s - date added: %s'
% (torrent.name, torrent.status, torrent.ratio, torrent.date_added)
)
downloaded, dummy = self.torrent_info(torrent, config)
if not downloaded:
continue
if config.get('transmission_seed_limits'):
seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session)
if not seed_ratio_ok or not idle_limit_ok:
continue
if 'min_ratio' in config:
if torrent.ratio < config['min_ratio']:
continue
if 'finished_for' in config:
# done date might be invalid if this torrent was added to transmission when already completed
started_seeding = datetime.fromtimestamp(max(torrent.addedDate, torrent.doneDate))
if started_seeding + parse_timedelta(config['finished_for']) > datetime.now():
continue
tracker_hosts = (
urlparse(tracker['announce']).hostname for tracker in torrent.trackers
)
if 'tracker' in config:
if not any(tracker_re.search(tracker) for tracker in tracker_hosts):
continue
if 'preserve_tracker' in config:
if any(preserve_tracker_re.search(tracker) for tracker in tracker_hosts):
continue
if config.get('directories'):
if not any(
re.search(d, torrent.downloadDir, re.IGNORECASE) for d in config['directories']
):
continue
if task.options.test:
log.info('Would remove finished torrent `%s` from transmission', torrent.name)
continue
log.info('Removing finished torrent `%s` from transmission', torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids, config.get('delete_files'))
def on_task_filter(self, task, config):
config = self.prepare_config(config)
if not config or not config['target']:
return
identified_by = '{{ id }}' if config['identified_by'] == 'auto' else config['identified_by']
grouped_entries = group_entries(task.accepted + task.undecided, identified_by)
if not grouped_entries:
return
with Session() as session:
# Prefetch Data
existing_ids = session.query(EntryUpgrade).filter(EntryUpgrade.id.in_(grouped_entries.keys())).all()
existing_ids = {e.id: e for e in existing_ids}
for identifier, entries in grouped_entries.items():
if not entries:
continue
existing = existing_ids.get(identifier)
if not existing:
# No existing, do_nothing
continue
log.debug('Looking for upgrades for identifier %s (within %s entries)', identifier, len(entries))
# Check if passed allowed timeframe
if config['timeframe']:
expires = existing.first_seen + parse_timedelta(config['timeframe'])
if expires <= datetime.now():
# Timeframe reached, allow
log.debug('Skipping upgrade with identifier %s as timeframe reached', identifier)
continue
# Filter out lower quality and propers
action_on_lower = entry_actions[config['on_lower']] if config['on_lower'] != 'do_nothing' else None
upgradeable = self.filter_entries(entries, existing, config['target'], action_on_lower)
# Skip if we have no entries after filtering
if not upgradeable:
continue
# Sort entities in order of quality and best proper
upgradeable.sort(key=lambda e: (e['quality'], e.get('proper_count', 0)), reverse=True)
# First entry will be the best quality
best = upgradeable.pop(0)
best.accept('upgraded quality')
log.debug('Found %s as upgraded quality for identifier %s', best['title'], identifier)
# Process rest
for entry in upgradeable:
log.debug('Skipping %s as lower quality then best %s', entry['title'], best['title'])
if action_on_lower:
action_on_lower(entry, 'lower quality then best match')
def queue_get(session=None):
subs = session.query(QueuedSubtitle).filter(QueuedSubtitle.downloaded == False).all()
# remove any items that have expired
for sub_item in subs:
if sub_item.added + parse_timedelta(sub_item.stop_after) < datetime.combine(date.today(), time()):
log.debug('%s has expired. Removing.' % sub_item.title)
subs.remove(sub_item)
session.delete(sub_item)
return subs
def estimated(self, entries, estimation_mode):
"""
:param dict estimation_mode: mode -> loose, strict, ignore
:return: Entries that we have estimated to be available
"""
estimator = plugin.get('estimate_release', self)
result = []
for entry in entries:
estimation = estimator.estimate(entry)
est_date = estimation['entity_date']
data_exists = estimation['data_exists']
if est_date is None:
if estimation_mode['mode'] == 'strict':
logger.debug('No release date could be determined for {}',
entry['title'])
entry.reject('has no release date')
entry.complete()
elif estimation_mode['mode'] == 'smart' and data_exists:
logger.debug(
'No release date could be determined for {}, but exists data',
entry['title'],
)
entry.reject('exists but has no release date')
entry.complete()
elif estimation_mode['mode'] == 'smart' and not data_exists:
logger.debug(
'Discovering because mode is \'{}\' and no data is found for entry',
estimation_mode['mode'],
)
result.append(entry)
else:
result.append(entry)
continue
if isinstance(est_date, datetime.date):
# If we just got a date, add a time so we can compare it to now()
est_date = datetime.datetime.combine(est_date, datetime.time())
if datetime.datetime.now() >= est_date:
logger.debug('{} has been released at {}', entry['title'],
est_date)
result.append(entry)
elif datetime.datetime.now() >= est_date - parse_timedelta(
estimation_mode['optimistic']):
logger.debug(
'{} will be released at {}. Ignoring release estimation because estimated release date is in less than {}',
entry['title'],
est_date,
estimation_mode['optimistic'],
)
result.append(entry)
else:
entry.reject('has not been released')
entry.complete()
logger.verbose("{} hasn't been released yet (Expected: {})",
entry['title'], est_date)
return result
def test_rejected_pagination(self, api_client, link_headers):
base_reject_entry = dict(
title='test_title_', url='test_url_', rejected_by='rejected_by_', reason='reason_'
)
number_of_entries = 200
with Session() as session:
task = RememberTask(name='rejected API test')
session.add(task)
session.commit()
for i in range(number_of_entries):
r_entry = copy.deepcopy(base_reject_entry)
for key, value in r_entry.items():
r_entry[key] = value + str(i)
expires = datetime.now() + parse_timedelta('1 hours')
session.add(RememberEntry(expires=expires, task_id=task.id, **r_entry))
# Default values
rsp = api_client.get('/rejected/')
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
data = json.loads(rsp.get_data(as_text=True))
assert len(data) == 50
assert int(rsp.headers['total-count']) == 200
assert int(rsp.headers['count']) == 50
links = link_headers(rsp)
assert links['last']['page'] == 4
assert links['next']['page'] == 2
# Change page size
rsp = api_client.get('/rejected/?per_page=100')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
assert len(data) == 100
assert int(rsp.headers['total-count']) == 200
assert int(rsp.headers['count']) == 100
links = link_headers(rsp)
assert links['last']['page'] == 2
assert links['next']['page'] == 2
# Get different page
rsp = api_client.get('/rejected/?page=2')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
assert len(data) == 50
assert int(rsp.headers['total-count']) == 200
assert int(rsp.headers['count']) == 50
links = link_headers(rsp)
assert links['last']['page'] == 4
assert links['next']['page'] == 3
assert links['prev']['page'] == 1
def queue_get(session=None):
subs = session.query(QueuedSubtitle).filter(QueuedSubtitle.downloaded == False).all()
# remove any items that have expired
for sub_item in subs:
if sub_item.added + parse_timedelta(sub_item.stop_after) < datetime.combine(date.today(), time()):
log.debug('%s has expired. Removing.' % sub_item.title)
subs.remove(sub_item)
session.delete(sub_item)
return subs
def interval_expired(self, config, task, entries):
"""
Maintain some limit levels so that we don't hammer search
sites with unreasonable amount of queries.
:return: Entries that are up for ``config['interval']``
"""
config.setdefault('interval', '5 hour')
interval = parse_timedelta(config['interval'])
if task.options.discover_now:
log.info('Ignoring interval because of --discover-now')
result = []
interval_count = 0
with Session() as session:
for entry in entries:
discover_entry = (
session.query(DiscoverEntry)
.filter(DiscoverEntry.title == entry['title'])
.filter(DiscoverEntry.task == task.name)
.first()
)
if not discover_entry:
log.debug('%s -> No previous run recorded', entry['title'])
discover_entry = DiscoverEntry(entry['title'], task.name)
session.add(discover_entry)
if (
not task.is_rerun and task.options.discover_now
) or not discover_entry.last_execution:
# First time we execute (and on --discover-now) we randomize time to avoid clumping
delta = multiply_timedelta(interval, random.random())
discover_entry.last_execution = datetime.datetime.now() - delta
else:
next_time = discover_entry.last_execution + interval
log.debug(
'last_time: %r, interval: %s, next_time: %r, ',
discover_entry.last_execution,
config['interval'],
next_time,
)
if datetime.datetime.now() < next_time:
log.debug('interval not met')
interval_count += 1
entry.reject('discover interval not met')
entry.complete()
continue
discover_entry.last_execution = datetime.datetime.now()
log.trace('interval passed for %s', entry['title'])
result.append(entry)
if interval_count and not task.is_rerun:
log.verbose(
'Discover interval of %s not met for %s entries. Use --discover-now to override.',
config['interval'],
interval_count,
)
return result
def interval_expired(self, config, task, entries):
"""
Maintain some limit levels so that we don't hammer search
sites with unreasonable amount of queries.
:return: Entries that are up for ``config['interval']``
"""
config.setdefault('interval', '5 hour')
interval = parse_timedelta(config['interval'])
if task.options.discover_now:
log.info('Ignoring interval because of --discover-now')
result = []
interval_count = 0
with Session() as session:
for entry in entries:
discover_entry = (
session.query(DiscoverEntry)
.filter(DiscoverEntry.title == entry['title'])
.filter(DiscoverEntry.task == task.name)
.first()
)
if not discover_entry:
log.debug('%s -> No previous run recorded', entry['title'])
discover_entry = DiscoverEntry(entry['title'], task.name)
session.add(discover_entry)
if (
not task.is_rerun and task.options.discover_now
) or not discover_entry.last_execution:
# First time we execute (and on --discover-now) we randomize time to avoid clumping
delta = multiply_timedelta(interval, random.random())
discover_entry.last_execution = datetime.datetime.now() - delta
else:
next_time = discover_entry.last_execution + interval
log.debug(
'last_time: %r, interval: %s, next_time: %r, ',
discover_entry.last_execution,
config['interval'],
next_time,
)
if datetime.datetime.now() < next_time:
log.debug('interval not met')
interval_count += 1
entry.reject('discover interval not met')
entry.complete()
continue
discover_entry.last_execution = datetime.datetime.now()
log.trace('interval passed for %s', entry['title'])
result.append(entry)
if interval_count and not task.is_rerun:
log.verbose(
'Discover interval of %s not met for %s entries. Use --discover-now to override.',
config['interval'],
interval_count,
)
return result
def add_rejected_entry(entry):
with Session() as session:
task = RememberTask(name='rejected API test')
session.add(task)
session.commit()
expires = datetime.now() + parse_timedelta('1 hours')
session.add(
RememberEntry(title=entry['test_title'], url=entry['test_url'], task_id=task.id,
rejected_by=entry['rejected_by'], reason=entry['reason'], expires=expires))
session.commit()
def add_rejected_entry(entry):
with Session() as session:
task = RememberTask(name='rejected API test')
session.add(task)
session.commit()
expires = datetime.now() + parse_timedelta('1 hours')
session.add(
RememberEntry(title=entry['test_title'], url=entry['test_url'], task_id=task.id,
rejected_by=entry['rejected_by'], reason=entry['reason'], expires=expires))
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config['enabled'] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
nrat = float(config['min_ratio']) if 'min_ratio' in config else None
nfor = parse_timedelta(
config['finished_for']) if 'finished_for' in config else None
delete_files = bool(
config['delete_files']) if 'delete_files' in config else False
trans_checks = bool(
config['transmission_seed_limits']
) if 'transmission_seed_limits' in config else False
session = self.client.get_session()
remove_ids = []
for torrent in self.client.get_torrents():
log.verbose(
'Torrent "%s": status: "%s" - ratio: %s - date added: %s - date done: %s'
% (torrent.name, torrent.status, torrent.ratio,
torrent.date_added, torrent.date_done))
downloaded, dummy = self.torrent_info(torrent)
seed_ratio_ok, idle_limit_ok = self.check_seed_limits(
torrent, session)
is_clean_all = nrat is None and nfor is None and trans_checks is None
is_minratio_reached = nrat and (nrat <= torrent.ratio)
is_transmission_seedlimit_unset = trans_checks and seed_ratio_ok is None and idle_limit_ok is None
is_transmission_seedlimit_reached = trans_checks and seed_ratio_ok is True
is_transmission_idlelimit_reached = trans_checks and idle_limit_ok is True
is_torrent_seed_only = torrent.date_done <= torrent.date_added
is_torrent_idlelimit_since_added_reached = nfor and (
torrent.date_added + nfor) <= datetime.now()
is_torrent_idlelimit_since_finished_reached = nfor and (
torrent.date_done + nfor) <= datetime.now()
if (downloaded and
(is_clean_all or is_transmission_seedlimit_unset
or is_transmission_seedlimit_reached
or is_transmission_idlelimit_reached or is_minratio_reached or
(is_torrent_seed_only
and is_torrent_idlelimit_since_added_reached) or
(not is_torrent_seed_only
and is_torrent_idlelimit_since_finished_reached))):
if task.options.test:
log.info(
'Would remove finished torrent `%s` from transmission'
% torrent.name)
continue
log.info('Removing finished torrent `%s` from transmission' %
torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids, delete_files)
def retry_time(self, fail_count, config):
"""Return the timedelta an entry that has failed `fail_count` times before should wait before being retried."""
base_retry_time = parse_timedelta(config['retry_time'])
# Timedeltas do not allow floating point multiplication. Convert to seconds and then back to avoid this.
base_retry_secs = base_retry_time.days * 86400 + base_retry_time.seconds
retry_secs = base_retry_secs * (config['retry_time_multiplier'] ** fail_count)
# prevent OverflowError: date value out of range, cap to 30 days
max = 60 * 60 * 24 * 30
if retry_secs > max:
retry_secs = max
return timedelta(seconds=retry_secs)
def retry_time(self, fail_count, config):
"""Return the timedelta an entry that has failed `fail_count` times before should wait before being retried."""
base_retry_time = parse_timedelta(config['retry_time'])
# Timedeltas do not allow floating point multiplication. Convert to seconds and then back to avoid this.
base_retry_secs = base_retry_time.days * 86400 + base_retry_time.seconds
retry_secs = base_retry_secs * (config['retry_time_multiplier'] ** fail_count)
# prevent OverflowError: date value out of range, cap to 30 days
max = 60 * 60 * 24 * 30
if retry_secs > max:
retry_secs = max
return timedelta(seconds=retry_secs)
def __init__(self, domain, tokens, rate, wait=True):
"""
:param int tokens: Size of bucket
:param rate: Amount of time to accrue 1 token. Either `timedelta` or interval string.
:param bool wait: If true, will wait for a token to be available. If false, errors when token is not available.
"""
super(TokenBucketLimiter, self).__init__(domain)
self.max_tokens = tokens
self.rate = parse_timedelta(rate)
self.wait = wait
# Restore previous state for this domain, or establish new state cache
self.state = self.state_cache.setdefault(domain, {'tokens': self.max_tokens, 'last_update': datetime.now()})
def __init__(self, domain, tokens, rate, wait=True):
"""
:param int tokens: Size of bucket
:param rate: Amount of time to accrue 1 token. Either `timedelta` or interval string.
:param bool wait: If true, will wait for a token to be available. If false, errors when token is not available.
"""
super(TokenBucketLimiter, self).__init__(domain)
self.max_tokens = tokens
self.rate = parse_timedelta(rate)
self.wait = wait
# Restore previous state for this domain, or establish new state cache
self.state = self.state_cache.setdefault(domain, {"tokens": self.max_tokens, "last_update": datetime.now()})
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config['enabled'] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
nrat = float(config['min_ratio']) if 'min_ratio' in config else None
nfor = parse_timedelta(config['finished_for']) if 'finished_for' in config else None
delete_files = bool(config['delete_files']) if 'delete_files' in config else False
trans_checks = bool(config['transmission_seed_limits']) if 'transmission_seed_limits' in config else False
tracker_re = re.compile(config['tracker'], re.IGNORECASE) if 'tracker' in config else None
preserve_tracker_re = re.compile(config['preserve_tracker'], re.IGNORECASE) if 'preserve_tracker' in config else None
directories_re = config.get('directories')
session = self.client.get_session()
remove_ids = []
for torrent in self.client.get_torrents():
log.verbose('Torrent "%s": status: "%s" - ratio: %s - date added: %s - date done: %s' %
(torrent.name, torrent.status, torrent.ratio, torrent.date_added, torrent.date_done))
downloaded, dummy = self.torrent_info(torrent, config)
seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session)
tracker_hosts = (urlparse(tracker['announce']).hostname for tracker in torrent.trackers)
is_clean_all = nrat is None and nfor is None and trans_checks is False
is_minratio_reached = nrat and (nrat <= torrent.ratio)
is_transmission_seedlimit_unset = trans_checks and seed_ratio_ok is None and idle_limit_ok is None
is_transmission_seedlimit_reached = trans_checks and seed_ratio_ok is True
is_transmission_idlelimit_reached = trans_checks and idle_limit_ok is True
is_torrent_seed_only = torrent.date_done <= torrent.date_added
is_torrent_idlelimit_since_added_reached = nfor and (torrent.date_added + nfor) <= datetime.now()
is_torrent_idlelimit_since_finished_reached = nfor and (torrent.date_done + nfor) <= datetime.now()
is_tracker_matching = not tracker_re or any(tracker_re.search(host) for host in tracker_hosts)
is_preserve_tracker_matching = False
if preserve_tracker_re is not None:
is_preserve_tracker_matching = any(preserve_tracker_re.search(host) for host in tracker_hosts)
is_directories_matching = not directories_re or any(
re.compile(directory, re.IGNORECASE).search(torrent.downloadDir) for directory in directories_re)
if (downloaded and (is_clean_all or
is_transmission_seedlimit_unset or
is_transmission_seedlimit_reached or
is_transmission_idlelimit_reached or
is_minratio_reached or
(is_torrent_seed_only and is_torrent_idlelimit_since_added_reached) or
(not is_torrent_seed_only and is_torrent_idlelimit_since_finished_reached)) and
is_directories_matching and (not is_preserve_tracker_matching and is_tracker_matching)):
if task.options.test:
log.info('Would remove finished torrent `%s` from transmission', torrent.name)
continue
log.info('Removing finished torrent `%s` from transmission', torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids, delete_files)
def process_timeframe(self, feed, config, eps, series_name):
"""
Runs the timeframe logic to determine if we should wait for a better quality.
Saves current best to backlog if timeframe has not expired.
:returns: True - if we should keep the quality (or qualities) restriction
False - if the quality restriction should be released, due to timeframe expiring
"""
if 'timeframe' not in config:
return True
best = eps[0]
# parse options
log.debug('timeframe: %s' % config['timeframe'])
try:
timeframe = parse_timedelta(config['timeframe'])
except ValueError:
raise PluginWarning('Invalid time format', log)
# Make sure we only start timing from the first seen quality that matches min and max requirements.
min_quality = config.get('min_quality') and qualities.get(config['min_quality'])
max_quality = config.get('max_quality') and qualities.get(config['max_quality'])
first_seen = self.get_first_seen(feed.session, best, min_quality, max_quality)
expires = first_seen + timeframe
log.debug('timeframe: %s, first_seen: %s, expires: %s' % (timeframe, first_seen, expires))
stop = feed.manager.options.stop_waiting.lower() == series_name.lower()
if expires <= datetime.now() or stop:
# Expire timeframe, accept anything
log.info('Timeframe expired, releasing quality restriction.')
return False
else:
# verbose waiting, add to backlog
diff = expires - datetime.now()
hours, remainder = divmod(diff.seconds, 3600)
hours += diff.days * 24
minutes, seconds = divmod(remainder, 60)
entry = self.parser2entry[best]
log.info('Timeframe waiting %s for %sh:%smin, currently best is %s' % \
(series_name, hours, minutes, entry['title']))
# add best entry to backlog (backlog is able to handle duplicate adds)
if self.backlog:
self.backlog.add_backlog(feed, entry)
return True
def on_entry_reject(self, entry, remember=None, remember_time=None, **kwargs):
# We only remember rejections that specify the remember keyword argument
if not (remember or remember_time):
return
if not entry.get("title") or not entry.get("original_url"):
log.debug("Can't remember rejection for entry without title or url.")
return
if remember_time:
if isinstance(remember_time, basestring):
remember_time = parse_timedelta(remember_time)
message = "Remembering rejection of `%s`" % entry["title"]
if remember_time:
message += " for %i minutes" % (remember_time.seconds / 60)
log.info(message)
entry["remember_rejected"] = remember_time or remember
def on_entry_reject(self, entry, remember=None, remember_time=None, **kwargs):
# We only remember rejections that specify the remember keyword argument
if not (remember or remember_time):
return
if not entry.get('title') or not entry.get('original_url'):
log.debug('Can\'t remember rejection for entry without title or url.')
return
if remember_time:
if isinstance(remember_time, basestring):
remember_time = parse_timedelta(remember_time)
message = 'Remembering rejection of `%s`' % entry['title']
if remember_time:
message += ' for %i minutes' % (remember_time.seconds / 60)
log.info(message)
entry['remember_rejected'] = remember_time or remember
def on_task_input(self, task, config):
entries = []
with Session() as session:
digest_entries = session.query(DigestEntry).filter(DigestEntry.list == config["list"])
# Remove any entries older than the expire time, if defined.
if isinstance(config.get("expire"), basestring):
expire_time = parse_timedelta(config["expire"])
digest_entries.filter(DigestEntry.added < datetime.now() - expire_time).delete()
for index, digest_entry in enumerate(digest_entries.order_by(DigestEntry.added.desc()).all()):
# Just remove any entries past the limit, if set.
if 0 < config.get("limit", -1) <= index:
session.delete(digest_entry)
continue
entries.append(Entry(digest_entry.entry))
# If expire is 'True', we remove it after it is output once.
if config.get("expire", True) is True:
session.delete(digest_entry)
return entries
