def run(): running = {} for res in Transfer.find({ 'started': {'$ne': None}, 'finished': None, }): add_running(running, res['type']) settings_ = Settings.get_settings('general') retry_delta = timedelta(seconds=settings_['retry_delta']) for transfer in Transfer.find({ 'finished': None, '$or': [ {'added': None}, { 'added': {'$lt': datetime.utcnow() - retry_delta}, 'started': None, 'tries': {'$lt': settings_['max_tries']}, }, ], }): limit = settings.WORKERS_LIMITS.get(transfer['type']) if limit and running.get(transfer['type'], 0) >= limit: continue factory = get_factory() target = '%s.workers.add.process' % settings.PACKAGE_NAME if factory.get(target=target, args=(transfer['_id'], transfer['type'])): continue factory.add(target=target, args=(transfer['_id'], transfer['type']), timeout=settings.PROCESS_TIMEOUT) now = datetime.utcnow() Transfer.update({'_id': transfer['_id']}, { '$set': {'added': now, 'started': now}, '$inc': {'tries': 1}, }, safe=True) add_running(running, transfer['type']) count_str = ' (#%s)' % transfer['tries'] if transfer['tries'] > 1 else '' logger.info('started%s %s transfer %s to %s', count_str, transfer['type'], transfer['src'], transfer['dst'])
def manage_nzbs(): client = get_nzb_client() for transfer in Transfer.find({ 'started': {'$ne': None}, 'finished': None, 'type': 'binsearch', }): nzb_id = transfer['info'].get('nzo_id') if not nzb_id: continue info = client.get_nzb(nzb_id) if not info: Transfer.update({'_id': transfer['_id']}, {'$set': {'finished': datetime.utcnow()}}, safe=True) else: info['name'] = info.get('filename', transfer['info'].get('name')) total = get_float(info.get('mb', 0)) * 1024 ** 2 Transfer.update({'_id': transfer['_id']}, {'$set': { 'total': total, 'transferred': total - get_float(info.get('mbleft', 0)) * 1024 ** 2, 'progress': get_float(info.get('percentage', 0)), 'info': info, }}, safe=True) paths = Settings.get_settings('paths') # Manage queued nzbs for nzb in client.list_nzbs(): transfer = get_nzb_transfer(nzb['nzo_id']) if not transfer: now = datetime.utcnow() Transfer.add(nzb['filename'], str(paths['default']), type='binsearch', added=now, started=now, queued=now, info={'nzo_id': nzb['nzo_id']}) elif transfer['finished']: client.remove_nzb(nzb['nzo_id']) logger.info('removed finished nzb "%s" (%s)', nzb['filename'], nzb['nzo_id']) # Manage finished nzbs for nzb in client.list_nzbs(history=True): transfer = get_nzb_transfer(nzb['nzo_id']) if nzb['status'] == 'Completed': dst = transfer['dst'] if transfer else str(paths['default']) elif nzb['status'] == 'Failed': dst = str(paths['invalid']) else: continue manage_nzb(client, nzb_id=nzb['nzo_id'], dst=dst)
def manage_torrents(): client = get_torrent_client() for transfer in Transfer.find({ 'started': {'$ne': None}, 'finished': None, 'type': 'torrent', }): hash = transfer['info'].get('hash') if not hash: continue try: torrent = client.get_torrent(hash=hash) except TorrentError: continue if not torrent: transfer['finished'] = datetime.utcnow() logger.debug('torrent %s is not queued', hash) else: transfer['info'] = torrent transfer['transferred'] = torrent.transferred transfer['total'] = torrent.size transfer['progress'] = torrent.progress Transfer.save(transfer, safe=True) default_dir = str(Settings.get_settings('paths')['default']) for torrent in client.iter_torrents(): transfer = Transfer.find_one({'info.hash': torrent.hash}, sort=[('created', DESCENDING)]) if not transfer or (transfer['finished'] \ and torrent.date_added > transfer['finished']): now = datetime.utcnow() Transfer.add(torrent.magnet_url, default_dir, type='torrent', added=now, started=now, info={'hash': torrent.hash}) logger.info('started torrent transfer %s to %s', torrent.magnet_url, default_dir) elif transfer['finished']: client.remove_torrent(hash=torrent.hash, delete_data=True) logger.debug('removed finished torrent "%s" (%s)', torrent.name, torrent.hash) else: manage_torrent(client, hash=torrent.hash, dst=transfer['dst'])
def list_transfers(): items = [_get_transfer(t) for t in Transfer.find({'finished': None})] return serialize({'result': items})