def run():
Transfer.remove({'finished': {
'$lt': datetime.utcnow() - DELTA_OBSOLETE,
}}, safe=True)
if Settings.get_settings('transmission').get('active', True):
clean_torrents()
if Settings.get_settings('sabnzbd').get('active', True):
clean_nzbs()
def clean_nzbs():
try:
client = get_nzb_client()
nzbs = client.list_nzbs(history=False) + client.list_nzbs(history=True)
if nzbs:
return
config = client.get_config()
base_path = str(Settings.get_settings('sabnzbd')['base_path'])
for name in ('download_dir', 'complete_dir'):
path = config['misc'].get(name)
if not path:
logger.error('failed to get sabnzbd %s', name)
continue
path = str(os.path.join(base_path, path))
if not os.path.exists(path):
logger.error('sabnzbd %s path %s does not exist', name, path)
continue
for file in glob(path + '/*'):
if remove_file(file):
logger.info('removed obsolete sabnzbd path %s', file)
except SabnzbdError, e:
logger.error('nzb client error: %s', str(e))
def manage_torrent(client, hash, dst):
try:
torrent = client.get_torrent(hash=hash)
if not torrent:
return
if not client.check_torrent_files(torrent,
check_unfinished=settings.CHECK_UNFINISHED_TORRENTS):
invalid_dir = str(Settings.get_settings('paths')['invalid'])
if torrent.progress == 100 and not client.move_files(torrent, invalid_dir):
return
if client.remove_torrent(hash=hash, delete_data=True):
logger.info('removed invalid torrent "%s" (%s%% done)', torrent.name, int(torrent.progress))
Transfer.update({'info.hash': hash, 'finished': None},
{'$set': {'finished': datetime.utcnow()}}, safe=True)
return
if torrent.progress == 100:
destination = client.get_destination_dir(torrent, dst)
if not client.move_files(torrent, destination):
return
if client.remove_torrent(hash=hash):
logger.info('moved finished torrent "%s" to %s', torrent.name, dst)
Transfer.update({'info.hash': hash, 'finished': None},
{'$set': {'finished': datetime.utcnow()}}, safe=True)
return
torrent_settings = Settings.get_settings('torrent')
now = datetime.utcnow()
inactive_delta = torrent_settings['inactive_delta']
if inactive_delta:
date = torrent.date_active or torrent.date_added
if date < now - timedelta(hours=inactive_delta) \
and client.remove_torrent(hash=hash, delete_data=True):
logger.debug('removed inactive torrent "%s": no activity since %s', torrent.name, date)
return
added_delta = torrent_settings['added_delta']
if added_delta:
date = torrent.date_added
if date < now - timedelta(hours=added_delta) \
and client.remove_torrent(hash=hash, delete_data=True):
logger.debug('removed obsolete torrent "%s": added %s', torrent.name, date)
except TransmissionError, e:
logger.error('torrent client error: %s', str(e))
def manage_nzbs():
client = get_nzb_client()
for transfer in Transfer.find({
'started': {'$ne': None},
'finished': None,
'type': 'binsearch',
}):
nzb_id = transfer['info'].get('nzo_id')
if not nzb_id:
continue
info = client.get_nzb(nzb_id)
if not info:
Transfer.update({'_id': transfer['_id']},
{'$set': {'finished': datetime.utcnow()}},
safe=True)
else:
info['name'] = info.get('filename', transfer['info'].get('name'))
total = get_float(info.get('mb', 0)) * 1024 ** 2
Transfer.update({'_id': transfer['_id']},
{'$set': {
'total': total,
'transferred': total - get_float(info.get('mbleft', 0)) * 1024 ** 2,
'progress': get_float(info.get('percentage', 0)),
'info': info,
}}, safe=True)
paths = Settings.get_settings('paths')
# Manage queued nzbs
for nzb in client.list_nzbs():
transfer = get_nzb_transfer(nzb['nzo_id'])
if not transfer:
now = datetime.utcnow()
Transfer.add(nzb['filename'], str(paths['default']),
type='binsearch', added=now, started=now, queued=now,
info={'nzo_id': nzb['nzo_id']})
elif transfer['finished']:
client.remove_nzb(nzb['nzo_id'])
logger.info('removed finished nzb "%s" (%s)', nzb['filename'], nzb['nzo_id'])
# Manage finished nzbs
for nzb in client.list_nzbs(history=True):
transfer = get_nzb_transfer(nzb['nzo_id'])
if nzb['status'] == 'Completed':
dst = transfer['dst'] if transfer else str(paths['default'])
elif nzb['status'] == 'Failed':
dst = str(paths['invalid'])
else:
continue
manage_nzb(client, nzb_id=nzb['nzo_id'], dst=dst)
def run():
running = {}
for res in Transfer.find({
'started': {'$ne': None},
'finished': None,
}):
add_running(running, res['type'])
settings_ = Settings.get_settings('general')
retry_delta = timedelta(seconds=settings_['retry_delta'])
for transfer in Transfer.find({
'finished': None,
'$or': [
{'added': None},
{
'added': {'$lt': datetime.utcnow() - retry_delta},
'started': None,
'tries': {'$lt': settings_['max_tries']},
},
],
}):
limit = settings.WORKERS_LIMITS.get(transfer['type'])
if limit and running.get(transfer['type'], 0) >= limit:
continue
factory = get_factory()
target = '%s.workers.add.process' % settings.PACKAGE_NAME
if factory.get(target=target,
args=(transfer['_id'], transfer['type'])):
continue
factory.add(target=target,
args=(transfer['_id'], transfer['type']),
timeout=settings.PROCESS_TIMEOUT)
now = datetime.utcnow()
Transfer.update({'_id': transfer['_id']}, {
'$set': {'added': now, 'started': now},
'$inc': {'tries': 1},
}, safe=True)
add_running(running, transfer['type'])
count_str = ' (#%s)' % transfer['tries'] if transfer['tries'] > 1 else ''
logger.info('started%s %s transfer %s to %s', count_str, transfer['type'], transfer['src'], transfer['dst'])
def manage_torrents():
client = get_torrent_client()
for transfer in Transfer.find({
'started': {'$ne': None},
'finished': None,
'type': 'torrent',
}):
hash = transfer['info'].get('hash')
if not hash:
continue
try:
torrent = client.get_torrent(hash=hash)
except TorrentError:
continue
if not torrent:
transfer['finished'] = datetime.utcnow()
logger.debug('torrent %s is not queued', hash)
else:
transfer['info'] = torrent
transfer['transferred'] = torrent.transferred
transfer['total'] = torrent.size
transfer['progress'] = torrent.progress
Transfer.save(transfer, safe=True)
default_dir = str(Settings.get_settings('paths')['default'])
for torrent in client.iter_torrents():
transfer = Transfer.find_one({'info.hash': torrent.hash},
sort=[('created', DESCENDING)])
if not transfer or (transfer['finished'] \
and torrent.date_added > transfer['finished']):
now = datetime.utcnow()
Transfer.add(torrent.magnet_url, default_dir,
type='torrent', added=now, started=now,
info={'hash': torrent.hash})
logger.info('started torrent transfer %s to %s', torrent.magnet_url, default_dir)
elif transfer['finished']:
client.remove_torrent(hash=torrent.hash, delete_data=True)
logger.debug('removed finished torrent "%s" (%s)', torrent.name, torrent.hash)
else:
manage_torrent(client, hash=torrent.hash, dst=transfer['dst'])
def process_transfer(self, id):
self.transfer = Transfer.find_one({'_id': id})
if not self.transfer:
return
src = self.transfer['src']
dst = self.transfer['dst']
if not isinstance(src, (tuple, list)):
src = [src]
temp_dir = Settings.get_settings('paths')['tmp']
res = self.process(src, dst, temp_dir)
if res:
self.transfer['info']['files'] = res
self.transfer['transferred'] = self.total
self.transfer['progress'] = 100
logger.info('finished http transfer %s to %s', src, dst)
else:
self.transfer['started'] = None
logger.error('failed to process http transfer %s to %s', src, dst)
self.transfer['finished'] = datetime.utcnow()
Transfer.save(self.transfer, safe=True)
def run():
if Settings.get_settings('transmission').get('active', True):
try:
manage_torrents()
except TransmissionError, e:
logger.error('torrent client error: %s', str(e))
elif transfer['finished']:
client.remove_nzb(nzb['nzo_id'])
logger.info('removed finished nzb "%s" (%s)', nzb['filename'], nzb['nzo_id'])
# Manage finished nzbs
for nzb in client.list_nzbs(history=True):
transfer = get_nzb_transfer(nzb['nzo_id'])
if nzb['status'] == 'Completed':
dst = transfer['dst'] if transfer else str(paths['default'])
elif nzb['status'] == 'Failed':
dst = str(paths['invalid'])
else:
continue
manage_nzb(client, nzb_id=nzb['nzo_id'], dst=dst)
@loop(30)
@timeout(minutes=30)
@timer()
def run():
if Settings.get_settings('transmission').get('active', True):
try:
manage_torrents()
except TransmissionError, e:
logger.error('torrent client error: %s', str(e))
if Settings.get_settings('sabnzbd').get('active', True):
try:
manage_nzbs()
except SabnzbdError, e:
logger.error('nzb client error: %s', str(e))
def update_settings():
data = request.json
for section, settings in data.items():
Settings.set_settings(section, settings, overwrite=True)
return jsonify(result=True)
def list_settings():
settings = {}
for section in ('general', 'paths', 'transmission',
'torrent', 'sabnzbd', 'rsync'):
settings[section] = Settings.get_settings(section)
return serialize({'result': settings})
def __init__(self):
self.name = None
self.transferred = 0
self.last_callback = 0
self.default_args = Settings.get_settings('rsync')['default_args']
def get_torrent_client():
settings = Settings.get_settings('transmission')
return Transmission(host=settings['host'], port=settings['port'],
username=settings['username'], password=settings['password'])
def get_nzb_client():
settings = Settings.get_settings('sabnzbd')
return Sabnzbd(host=settings['host'], port=settings['port'],
api_key=settings['api_key'])
