def mmap_move(fileobj, dest, src, count): """Mmaps the file object if possible and moves 'count' data from 'src' to 'dest'. All data has to be inside the file size (enlarging the file through this function isn't possible) Will adjust the file offset. Args: fileobj (fileobj) dest (int): The destination offset src (int): The source offset count (int) The amount of data to move Raises: mmap.error: In case move failed IOError: In case an operation on the fileobj fails ValueError: In case invalid parameters were given """ assert mmap is not None, "no mmap support" if dest < 0 or src < 0 or count < 0: raise ValueError("Invalid parameters") try: fileno = fileobj.fileno() except (AttributeError, IOError): raise mmap.error( "File object does not expose/support a file descriptor") fileobj.seek(0, 2) filesize = fileobj.tell() length = max(dest, src) + count if length > filesize: raise ValueError("Not in file size boundary") offset = ((min(dest, src) // mmap.ALLOCATIONGRANULARITY) * mmap.ALLOCATIONGRANULARITY) assert dest >= offset assert src >= offset assert offset % mmap.ALLOCATIONGRANULARITY == 0 # Windows doesn't handle empty mappings, add a fast path here instead if count == 0: return # fast path if src == dest: return fileobj.flush() file_map = mmap.mmap(fileno, length - offset, offset=offset) try: file_map.move(dest - offset, src - offset, count) finally: file_map.close()
def mmapwrapper(*args, **kwargs): """ Python's mmap call sucks and ommitted the "offset" argument for no discernable reason. Replace this with a mmap module that has offset. """ offset = kwargs.get('offset', None) if offset in [None, 0]: if 'offset' in kwargs: del kwargs['offset'] else: raise mmap.error("mmap: Python sucks and does not support offset.") return mmap.mmap(*args, **kwargs)
def flush(self): raise mmap.error('flush is broken on this platform')
def __init__(self, filename, mode='r', first_byte=0, verbose=False, local_cache=None, use_memmap=True): """ local_cache : {None, str} Specify path to local cache. Local cache will be used to temporarily store files from external devises such as NFS. """ self.verbose = verbose self.first_byte = first_byte self.use_memmap = use_memmap try: if local_cache is not None: cache_filename = local_cache + '/' + filename if os.path.exists(cache_filename): filename = cache_filename elif not isindisk(filename): assert isindisk(local_cache), repr(local_cache) dirname = os.path.dirname(cache_filename) if not os.path.isdir(dirname): os.makedirs(dirname) shutil.copyfile(filename, cache_filename) filename = cache_filename if verbose: sys.stdout.write('Opening file %r\n' % (filename)) sys.stdout.flush() if mode != 'r': raise NotImplementedError(repr(mode)) if not os.path.isfile(filename): raise ValueError('file does not exists') if not os.stat(filename).st_size: raise ValueError('file has zero size') if use_memmap: self.data = numpy.memmap(filename, dtype=numpy.ubyte, mode=mode) else: assert mode == 'r', repr(mode) f = open(filename, 'rb') self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte) f.close() except IOError as msg: if 'Too many open files' in str(msg): raise IOError(IOError_too_many_open_files_hint % msg) if 'Operation not permitted' in str(msg): raise IOError(OSError_operation_not_permitted_hint % msg) raise except OSError as msg: if 'Operation not permitted' in str(msg): raise OSError(OSError_operation_not_permitted_hint % msg) raise except mmap.error as msg: if 'Too many open files' in str(msg): raise mmap.error(IOError_too_many_open_files_hint % msg) raise self.filename = filename self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')] byteorder = self.data[first_byte:first_byte + 2].view( dtype=numpy.uint16)[0] if byteorder == 0x4949: self.endian = 'little' self.dtypes = LittleEndianNumpyDTypes elif byteorder == 0x4d4d: self.endian = 'big' self.dtypes = BigEndianNumpyDTypes else: raise ValueError('unrecognized byteorder: %s' % (hex(byteorder))) magic = self.get_uint16(first_byte + 2) if magic != 42: raise ValueError('wrong magic number for TIFF file: %s' % (magic)) self.IFD0 = IFD0 = first_byte + self.get_uint32(first_byte + 4) self.memory_usage.append((first_byte, first_byte + 8, 'file header')) n = self.get_uint16(IFD0) IFD_list = [] IFD_offset = IFD0 while IFD_offset: n = self.get_uint16(IFD_offset) ifd = IFD(self) exif_offset = 0 for i in range(n): entry = IFDEntry(ifd, self, IFD_offset + 2 + i * 12) ifd.append(entry) if entry.tag == 0x8769: # TIFFTAG_EXIFIFD exif_offset = entry.value ifd.finalize() IFD_list.append(ifd) self.memory_usage.append((IFD_offset, IFD_offset + 2 + n * 12 + 4, 'IFD%s entries(%s)' % ( len(IFD_list), len(ifd)))) IFD_offset = self.get_uint32(IFD_offset + 2 + n * 12) if IFD_offset == 0 and exif_offset != 0: IFD_offset = exif_offset exif_offset = 0 if verbose: sys.stdout.write( '\rIFD information read: %s..' % (len(IFD_list))) sys.stdout.flush() self.IFD = IFD_list if verbose: sys.stdout.write(' done\n') sys.stdout.flush() self.time = None
f = open (filename, 'rb') self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte) f.close() except IOError, msg: if 'Too many open files' in str (msg): raise IOError(IOError_too_many_open_files_hint % msg) if 'Operation not permitted' in str (msg): raise IOError(OSError_operation_not_permitted_hint % msg) raise except OSError, msg: if 'Operation not permitted' in str (msg): raise OSError(OSError_operation_not_permitted_hint % msg) raise except mmap.error, msg: if 'Too many open files' in str (msg): raise mmap.error(IOError_too_many_open_files_hint % msg) raise self.filename = filename self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')] byteorder = self.data[first_byte:first_byte+2].view(dtype=numpy.uint16)[0] if byteorder==0x4949: self.endian = 'little' self.dtypes = LittleEndianNumpyDTypes elif byteorder==0x4d4d: self.endian = 'big' self.dtypes = BigEndianNumpyDTypes else:
f = open(filename, 'rb') self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte) f.close() except IOError, msg: if 'Too many open files' in str(msg): raise IOError(IOError_too_many_open_files_hint % msg) if 'Operation not permitted' in str(msg): raise IOError(OSError_operation_not_permitted_hint % msg) raise except OSError, msg: if 'Operation not permitted' in str(msg): raise OSError(OSError_operation_not_permitted_hint % msg) raise except mmap.error, msg: if 'Too many open files' in str(msg): raise mmap.error(IOError_too_many_open_files_hint % msg) raise self.filename = filename self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')] byteorder = self.data[first_byte:first_byte + 2].view(dtype=numpy.uint16)[0] if byteorder == 0x4949: self.endian = 'little' self.dtypes = LittleEndianNumpyDTypes elif byteorder == 0x4d4d: self.endian = 'big' self.dtypes = BigEndianNumpyDTypes
def __init__(self, filename, mode='r', first_byte=0, verbose=False, local_cache=None, use_memmap=True): """ local_cache : {None, str} Specify path to local cache. Local cache will be used to temporarily store files from external devises such as NFS. """ self.verbose = verbose self.first_byte = first_byte self.use_memmap = use_memmap try: if local_cache is not None: cache_filename = local_cache + '/' + filename if os.path.exists(cache_filename): filename = cache_filename elif not isindisk(filename): assert isindisk(local_cache), repr(local_cache) dirname = os.path.dirname(cache_filename) if not os.path.isdir(dirname): os.makedirs(dirname) shutil.copyfile(filename, cache_filename) filename = cache_filename if verbose: sys.stdout.write('Opening file %r\n' % (filename)); sys.stdout.flush() if mode != 'r': raise NotImplementedError(repr(mode)) if not os.path.isfile(filename): raise ValueError('file does not exists') if not os.stat(filename).st_size: raise ValueError('file has zero size') if use_memmap: self.data = numpy.memmap(filename, dtype=numpy.ubyte, mode=mode) else: assert mode == 'r', repr(mode) f = open(filename, 'rb') self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte) f.close() except IOError as msg: if 'Too many open files' in str(msg): raise IOError(IOError_too_many_open_files_hint % msg) if 'Operation not permitted' in str(msg): raise IOError(OSError_operation_not_permitted_hint % msg) raise except OSError as msg: if 'Operation not permitted' in str(msg): raise OSError(OSError_operation_not_permitted_hint % msg) raise except mmap.error as msg: if 'Too many open files' in str(msg): raise mmap.error(IOError_too_many_open_files_hint % msg) raise self.filename = filename self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')] byteorder = \ self.data[first_byte:first_byte + 2].view(dtype=numpy.uint16)[0] if byteorder == 0x4949: self.endian = 'little' self.dtypes = LittleEndianNumpyDTypes elif byteorder == 0x4d4d: self.endian = 'big' self.dtypes = BigEndianNumpyDTypes else: raise ValueError('unrecognized byteorder: %s' % (hex(byteorder))) magic = self.get_uint16(first_byte + 2) if magic != 42: raise ValueError('wrong magic number for TIFF file: %s' % (magic)) self.IFD0 = IFD0 = first_byte + self.get_uint32(first_byte + 4) self.memory_usage.append((first_byte, first_byte + 8, 'file header')) n = self.get_uint16(IFD0) IFD_list = [] IFD_offset = IFD0 while IFD_offset: n = self.get_uint16(IFD_offset) ifd = IFD(self) exif_offset = 0 for i in range(n): entry = IFDEntry(ifd, self, IFD_offset + 2 + i * 12) ifd.append(entry) if entry.tag == 0x8769: # TIFFTAG_EXIFIFD exif_offset = entry.value ifd.finalize() IFD_list.append(ifd) self.memory_usage.append((IFD_offset, IFD_offset + 2 + n * 12 + 4, 'IFD%s entries (%s)' % ( len(IFD_list), len(ifd)))) IFD_offset = self.get_uint32(IFD_offset + 2 + n * 12) if IFD_offset == 0 and exif_offset != 0: IFD_offset = exif_offset exif_offset = 0 if verbose: sys.stdout.write( '\rIFD information read: %s..' % (len(IFD_list))); sys.stdout.flush() self.IFD = IFD_list if verbose: sys.stdout.write(' done\n'); sys.stdout.flush() self.time = None