def run(): Transfer.remove({'finished': { '$lt': datetime.utcnow() - DELTA_OBSOLETE, }}, safe=True) if Settings.get_settings('transmission').get('active', True): clean_torrents() if Settings.get_settings('sabnzbd').get('active', True): clean_nzbs()
def clean_nzbs(): try: client = get_nzb_client() nzbs = client.list_nzbs(history=False) + client.list_nzbs(history=True) if nzbs: return config = client.get_config() base_path = str(Settings.get_settings('sabnzbd')['base_path']) for name in ('download_dir', 'complete_dir'): path = config['misc'].get(name) if not path: logger.error('failed to get sabnzbd %s', name) continue path = str(os.path.join(base_path, path)) if not os.path.exists(path): logger.error('sabnzbd %s path %s does not exist', name, path) continue for file in glob(path + '/*'): if remove_file(file): logger.info('removed obsolete sabnzbd path %s', file) except SabnzbdError, e: logger.error('nzb client error: %s', str(e))
def manage_torrent(client, hash, dst): try: torrent = client.get_torrent(hash=hash) if not torrent: return if not client.check_torrent_files(torrent, check_unfinished=settings.CHECK_UNFINISHED_TORRENTS): invalid_dir = str(Settings.get_settings('paths')['invalid']) if torrent.progress == 100 and not client.move_files(torrent, invalid_dir): return if client.remove_torrent(hash=hash, delete_data=True): logger.info('removed invalid torrent "%s" (%s%% done)', torrent.name, int(torrent.progress)) Transfer.update({'info.hash': hash, 'finished': None}, {'$set': {'finished': datetime.utcnow()}}, safe=True) return if torrent.progress == 100: destination = client.get_destination_dir(torrent, dst) if not client.move_files(torrent, destination): return if client.remove_torrent(hash=hash): logger.info('moved finished torrent "%s" to %s', torrent.name, dst) Transfer.update({'info.hash': hash, 'finished': None}, {'$set': {'finished': datetime.utcnow()}}, safe=True) return torrent_settings = Settings.get_settings('torrent') now = datetime.utcnow() inactive_delta = torrent_settings['inactive_delta'] if inactive_delta: date = torrent.date_active or torrent.date_added if date < now - timedelta(hours=inactive_delta) \ and client.remove_torrent(hash=hash, delete_data=True): logger.debug('removed inactive torrent "%s": no activity since %s', torrent.name, date) return added_delta = torrent_settings['added_delta'] if added_delta: date = torrent.date_added if date < now - timedelta(hours=added_delta) \ and client.remove_torrent(hash=hash, delete_data=True): logger.debug('removed obsolete torrent "%s": added %s', torrent.name, date) except TransmissionError, e: logger.error('torrent client error: %s', str(e))
def manage_nzbs(): client = get_nzb_client() for transfer in Transfer.find({ 'started': {'$ne': None}, 'finished': None, 'type': 'binsearch', }): nzb_id = transfer['info'].get('nzo_id') if not nzb_id: continue info = client.get_nzb(nzb_id) if not info: Transfer.update({'_id': transfer['_id']}, {'$set': {'finished': datetime.utcnow()}}, safe=True) else: info['name'] = info.get('filename', transfer['info'].get('name')) total = get_float(info.get('mb', 0)) * 1024 ** 2 Transfer.update({'_id': transfer['_id']}, {'$set': { 'total': total, 'transferred': total - get_float(info.get('mbleft', 0)) * 1024 ** 2, 'progress': get_float(info.get('percentage', 0)), 'info': info, }}, safe=True) paths = Settings.get_settings('paths') # Manage queued nzbs for nzb in client.list_nzbs(): transfer = get_nzb_transfer(nzb['nzo_id']) if not transfer: now = datetime.utcnow() Transfer.add(nzb['filename'], str(paths['default']), type='binsearch', added=now, started=now, queued=now, info={'nzo_id': nzb['nzo_id']}) elif transfer['finished']: client.remove_nzb(nzb['nzo_id']) logger.info('removed finished nzb "%s" (%s)', nzb['filename'], nzb['nzo_id']) # Manage finished nzbs for nzb in client.list_nzbs(history=True): transfer = get_nzb_transfer(nzb['nzo_id']) if nzb['status'] == 'Completed': dst = transfer['dst'] if transfer else str(paths['default']) elif nzb['status'] == 'Failed': dst = str(paths['invalid']) else: continue manage_nzb(client, nzb_id=nzb['nzo_id'], dst=dst)
def run(): running = {} for res in Transfer.find({ 'started': {'$ne': None}, 'finished': None, }): add_running(running, res['type']) settings_ = Settings.get_settings('general') retry_delta = timedelta(seconds=settings_['retry_delta']) for transfer in Transfer.find({ 'finished': None, '$or': [ {'added': None}, { 'added': {'$lt': datetime.utcnow() - retry_delta}, 'started': None, 'tries': {'$lt': settings_['max_tries']}, }, ], }): limit = settings.WORKERS_LIMITS.get(transfer['type']) if limit and running.get(transfer['type'], 0) >= limit: continue factory = get_factory() target = '%s.workers.add.process' % settings.PACKAGE_NAME if factory.get(target=target, args=(transfer['_id'], transfer['type'])): continue factory.add(target=target, args=(transfer['_id'], transfer['type']), timeout=settings.PROCESS_TIMEOUT) now = datetime.utcnow() Transfer.update({'_id': transfer['_id']}, { '$set': {'added': now, 'started': now}, '$inc': {'tries': 1}, }, safe=True) add_running(running, transfer['type']) count_str = ' (#%s)' % transfer['tries'] if transfer['tries'] > 1 else '' logger.info('started%s %s transfer %s to %s', count_str, transfer['type'], transfer['src'], transfer['dst'])
def manage_torrents(): client = get_torrent_client() for transfer in Transfer.find({ 'started': {'$ne': None}, 'finished': None, 'type': 'torrent', }): hash = transfer['info'].get('hash') if not hash: continue try: torrent = client.get_torrent(hash=hash) except TorrentError: continue if not torrent: transfer['finished'] = datetime.utcnow() logger.debug('torrent %s is not queued', hash) else: transfer['info'] = torrent transfer['transferred'] = torrent.transferred transfer['total'] = torrent.size transfer['progress'] = torrent.progress Transfer.save(transfer, safe=True) default_dir = str(Settings.get_settings('paths')['default']) for torrent in client.iter_torrents(): transfer = Transfer.find_one({'info.hash': torrent.hash}, sort=[('created', DESCENDING)]) if not transfer or (transfer['finished'] \ and torrent.date_added > transfer['finished']): now = datetime.utcnow() Transfer.add(torrent.magnet_url, default_dir, type='torrent', added=now, started=now, info={'hash': torrent.hash}) logger.info('started torrent transfer %s to %s', torrent.magnet_url, default_dir) elif transfer['finished']: client.remove_torrent(hash=torrent.hash, delete_data=True) logger.debug('removed finished torrent "%s" (%s)', torrent.name, torrent.hash) else: manage_torrent(client, hash=torrent.hash, dst=transfer['dst'])
def process_transfer(self, id): self.transfer = Transfer.find_one({'_id': id}) if not self.transfer: return src = self.transfer['src'] dst = self.transfer['dst'] if not isinstance(src, (tuple, list)): src = [src] temp_dir = Settings.get_settings('paths')['tmp'] res = self.process(src, dst, temp_dir) if res: self.transfer['info']['files'] = res self.transfer['transferred'] = self.total self.transfer['progress'] = 100 logger.info('finished http transfer %s to %s', src, dst) else: self.transfer['started'] = None logger.error('failed to process http transfer %s to %s', src, dst) self.transfer['finished'] = datetime.utcnow() Transfer.save(self.transfer, safe=True)
def run(): if Settings.get_settings('transmission').get('active', True): try: manage_torrents() except TransmissionError, e: logger.error('torrent client error: %s', str(e))
elif transfer['finished']: client.remove_nzb(nzb['nzo_id']) logger.info('removed finished nzb "%s" (%s)', nzb['filename'], nzb['nzo_id']) # Manage finished nzbs for nzb in client.list_nzbs(history=True): transfer = get_nzb_transfer(nzb['nzo_id']) if nzb['status'] == 'Completed': dst = transfer['dst'] if transfer else str(paths['default']) elif nzb['status'] == 'Failed': dst = str(paths['invalid']) else: continue manage_nzb(client, nzb_id=nzb['nzo_id'], dst=dst) @loop(30) @timeout(minutes=30) @timer() def run(): if Settings.get_settings('transmission').get('active', True): try: manage_torrents() except TransmissionError, e: logger.error('torrent client error: %s', str(e)) if Settings.get_settings('sabnzbd').get('active', True): try: manage_nzbs() except SabnzbdError, e: logger.error('nzb client error: %s', str(e))
def update_settings(): data = request.json for section, settings in data.items(): Settings.set_settings(section, settings, overwrite=True) return jsonify(result=True)
def list_settings(): settings = {} for section in ('general', 'paths', 'transmission', 'torrent', 'sabnzbd', 'rsync'): settings[section] = Settings.get_settings(section) return serialize({'result': settings})
def __init__(self): self.name = None self.transferred = 0 self.last_callback = 0 self.default_args = Settings.get_settings('rsync')['default_args']
def get_torrent_client(): settings = Settings.get_settings('transmission') return Transmission(host=settings['host'], port=settings['port'], username=settings['username'], password=settings['password'])
def get_nzb_client(): settings = Settings.get_settings('sabnzbd') return Sabnzbd(host=settings['host'], port=settings['port'], api_key=settings['api_key'])