def __init__(self, response, f): self.response = response self.f = f self.status = Status.create(f.size, 'Downloading ' + os.path.basename(f.url))
def getDownload(request, response, start=None, end=None): try: nsp = Nsps.getByTitleId(request.bits[2]) response.attachFile(nsp.titleId + '.nsp') if len(request.bits) >= 5: start = int(request.bits[-2]) end = int(request.bits[-1]) #chunkSize = 0x1000000 chunkSize = 0x400000 with open(nsp.path, "rb") as f: f.seek(0, 2) size = f.tell() if 'Range' in request.headers: start, end = request.headers.get('Range').strip().strip( 'bytes=').split('-') if end == '': end = size - 1 else: end = int(end) + 1 if start == '': start = size - end else: start = int(start) if start >= size or start < 0 or end <= 0: return Server.Response400( request, response, 'Invalid range request %d - %d' % (start, end)) response.setStatus(206) else: if start == None: start = 0 if end == None: end = size if end >= size: end = size if end <= start: response.write(b'') return print('ranged request for %d - %d' % (start, end)) f.seek(start, 0) response.setMime(nsp.path) response.setHeader('Accept-Ranges', 'bytes') response.setHeader('Content-Range', 'bytes %s-%s/%s' % (start, end - 1, size)) response.setHeader('Content-Length', str(end - start)) #Print.info(response.headers['Content-Range']) response.sendHeader() if not response.head: size = end - start i = 0 status = Status.create( size, 'Downloading ' + os.path.basename(nsp.path)) while i < size: chunk = f.read(min(size - i, chunkSize)) i += len(chunk) status.add(len(chunk)) if chunk: pass response.write(chunk) else: break status.close() except BaseException as e: Print.error('NSP download exception: ' + str(e)) if response.bytesSent == 0: response.write(b'')
def serveFile(response, path, filename=None, start=None, end=None): try: if start is not None: start = int(start) if end is not None: end = int(end) if not filename: filename = os.path.basename(path) response.attachFile(filename) chunkSize = 0x400000 with open(path, "rb") as f: f.seek(0, 2) size = f.tell() if start and end: if end == None: end = size - 1 else: end = int(end) if start == None: start = size - end else: start = int(start) if start >= size or start < 0 or end <= 0: return Server.Response400( request, response, 'Invalid range request %d - %d' % (start, end)) response.setStatus(206) else: if start == None: start = 0 if end == None: end = size if end >= size: end = size if end <= start: response.write(b'') return print('ranged request for %d - %d' % (start, end)) f.seek(start, 0) response.setMime(path) response.setHeader('Accept-Ranges', 'bytes') response.setHeader('Content-Range', 'bytes %s-%s/%s' % (start, end - 1, size)) response.setHeader('Content-Length', str(end - start)) response.sendHeader() if not response.head: size = end - start i = 0 status = Status.create(size, 'Downloading ' + os.path.basename(path)) while i < size: chunk = f.read(min(size - i, chunkSize)) i += len(chunk) status.add(len(chunk)) if chunk: pass response.write(chunk) else: break status.close() except BaseException as e: Print.error('File download exception: ' + str(e)) if response.bytesSent == 0: response.write(b'')
def scan(base): i = 0 fileList = {} nspOut = os.path.abspath(Config.paths.nspOut) duplicatesFolder = os.path.abspath(Config.paths.duplicates) Print.info('scanning %s' % base) for root, _, _files in os.walk(base, topdown=False): for name in _files: if _is_file_hidden(name): continue suffix = pathlib.Path(name).suffix if suffix in ('.nsp', '.nsx', '.xci', '.nsz'): path = os.path.abspath(root + '/' + name) if not path.startswith(nspOut) and not path.startswith( duplicatesFolder): fileList[path] = name if len(fileList) == 0: save() return 0 status = Status.create(len(fileList), desc='Scanning files...') for file, items in list(files.items()): if (not os.path.isfile(file)): print('removed: ' + file) unregisterFile(file) try: for path, name in fileList.items(): try: status.add(1) if path not in files: Print.info('scanning ' + name) nsp = Fs.Nsp(path, None) nsp.timestamp = time.time() nsp.getFileSize() # cache file size files[nsp.path] = nsp i = i + 1 if i % 20 == 0: save() except KeyboardInterrupt: status.close() raise except BaseException as e: # pylint: disable=broad-except Print.info('An error occurred processing file: ' + str(e)) save() status.close() except BaseException as e: # pylint: disable=broad-except Print.info('An error occurred scanning files: ' + str(e)) return i
def __decompressNcz(nspf, f, statusReportInfo): UNCOMPRESSABLE_HEADER_SIZE = 0x4000 blockID = 0 nspf.seek(0) header = nspf.read(UNCOMPRESSABLE_HEADER_SIZE) if f != None: start = f.tell() magic = nspf.read(8) if not magic == b'NCZSECTN': raise ValueError("No NCZSECTN found! Is this really a .ncz file?") sectionCount = nspf.readInt64() sections = [Section(nspf) for _ in range(sectionCount)] if sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE > 0: fakeSection = FakeSection( UNCOMPRESSABLE_HEADER_SIZE, sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE) sections.insert(0, fakeSection) nca_size = UNCOMPRESSABLE_HEADER_SIZE for i in range(sectionCount): nca_size += sections[i].size decompressor = ZstdDecompressor().stream_reader(nspf) hash = sha256() bar = Status.create(nspf.size, desc=os.path.basename(nspf._path), unit='B') #if statusReportInfo == None: # BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' # bar = enlighten.Counter(total=nca_size//1048576, desc='Decompress', unit="MiB", color='red', bar_format=BAR_FMT) decompressedBytes = len(header) if f != None: f.write(header) bar.add(len(header)) hash.update(header) firstSection = True for s in sections: i = s.offset useCrypto = s.cryptoType in (3, 4) if useCrypto: crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter) end = s.offset + s.size if firstSection: firstSection = False uncompressedSize = UNCOMPRESSABLE_HEADER_SIZE - sections[0].offset if uncompressedSize > 0: i += uncompressedSize while i < end: if useCrypto: crypto.seek(i) chunkSz = 0x10000 if end - i > 0x10000 else end - i inputChunk = decompressor.read(chunkSz) decompressor.flush() if not len(inputChunk): break if useCrypto: inputChunk = crypto.encrypt(inputChunk) if f != None: f.write(inputChunk) bar.add(len(inputChunk)) hash.update(inputChunk) lenInputChunk = len(inputChunk) i += lenInputChunk decompressedBytes += lenInputChunk bar.add(lenInputChunk) bar.close() print() hexHash = hash.hexdigest() if f != None: end = f.tell() written = (end - start) return (written, hexHash) return (0, hexHash)
if Config.download.mtime_min and f.getFileModified( ) < Config.download.mtime_min: continue if Config.download.mtime_max and f.getFileModified( ) > Config.download.mtime_max: continue filesToScan[path] = nsp except: pass with open('file.verification.txt', 'w+', encoding="utf-8") as bf: s = Status.create(len(filesToScan), desc='Verifying files...', unit='B') for path, nsp in filesToScan.items(): try: f = nsp f.open(str(path), 'r+b') if not f.verifyNcaHeaders(): nsp.verified = False raise IOError('bad file') nsp.verified = True Print.info('good file: ' + str(path)) bf.write('good file: %s\n' % str(path))