def mmap_move(fileobj, dest, src, count):
"""Mmaps the file object if possible and moves 'count' data
from 'src' to 'dest'. All data has to be inside the file size
(enlarging the file through this function isn't possible)
Will adjust the file offset.
Args:
fileobj (fileobj)
dest (int): The destination offset
src (int): The source offset
count (int) The amount of data to move
Raises:
mmap.error: In case move failed
IOError: In case an operation on the fileobj fails
ValueError: In case invalid parameters were given
"""
assert mmap is not None, "no mmap support"
if dest < 0 or src < 0 or count < 0:
raise ValueError("Invalid parameters")
try:
fileno = fileobj.fileno()
except (AttributeError, IOError):
raise mmap.error(
"File object does not expose/support a file descriptor")
fileobj.seek(0, 2)
filesize = fileobj.tell()
length = max(dest, src) + count
if length > filesize:
raise ValueError("Not in file size boundary")
offset = ((min(dest, src) // mmap.ALLOCATIONGRANULARITY) *
mmap.ALLOCATIONGRANULARITY)
assert dest >= offset
assert src >= offset
assert offset % mmap.ALLOCATIONGRANULARITY == 0
# Windows doesn't handle empty mappings, add a fast path here instead
if count == 0:
return
# fast path
if src == dest:
return
fileobj.flush()
file_map = mmap.mmap(fileno, length - offset, offset=offset)
try:
file_map.move(dest - offset, src - offset, count)
finally:
file_map.close()
def mmap_move(fileobj, dest, src, count):
"""Mmaps the file object if possible and moves 'count' data
from 'src' to 'dest'. All data has to be inside the file size
(enlarging the file through this function isn't possible)
Will adjust the file offset.
Args:
fileobj (fileobj)
dest (int): The destination offset
src (int): The source offset
count (int) The amount of data to move
Raises:
mmap.error: In case move failed
IOError: In case an operation on the fileobj fails
ValueError: In case invalid parameters were given
"""
assert mmap is not None, "no mmap support"
if dest < 0 or src < 0 or count < 0:
raise ValueError("Invalid parameters")
try:
fileno = fileobj.fileno()
except (AttributeError, IOError):
raise mmap.error(
"File object does not expose/support a file descriptor")
fileobj.seek(0, 2)
filesize = fileobj.tell()
length = max(dest, src) + count
if length > filesize:
raise ValueError("Not in file size boundary")
offset = ((min(dest, src) // mmap.ALLOCATIONGRANULARITY) *
mmap.ALLOCATIONGRANULARITY)
assert dest >= offset
assert src >= offset
assert offset % mmap.ALLOCATIONGRANULARITY == 0
# Windows doesn't handle empty mappings, add a fast path here instead
if count == 0:
return
# fast path
if src == dest:
return
fileobj.flush()
file_map = mmap.mmap(fileno, length - offset, offset=offset)
try:
file_map.move(dest - offset, src - offset, count)
finally:
file_map.close()
def mmapwrapper(*args, **kwargs):
"""
Python's mmap call sucks and ommitted the "offset" argument for no
discernable reason. Replace this with a mmap module that has offset.
"""
offset = kwargs.get('offset', None)
if offset in [None, 0]:
if 'offset' in kwargs:
del kwargs['offset']
else:
raise mmap.error("mmap: Python sucks and does not support offset.")
return mmap.mmap(*args, **kwargs)
def mmapwrapper(*args, **kwargs):
"""
Python's mmap call sucks and ommitted the "offset" argument for no
discernable reason. Replace this with a mmap module that has offset.
"""
offset = kwargs.get('offset', None)
if offset in [None, 0]:
if 'offset' in kwargs:
del kwargs['offset']
else:
raise mmap.error("mmap: Python sucks and does not support offset.")
return mmap.mmap(*args, **kwargs)
def flush(self):
raise mmap.error('flush is broken on this platform')
def flush(self):
raise mmap.error('flush is broken on this platform')
def __init__(self, filename, mode='r', first_byte=0, verbose=False,
local_cache=None, use_memmap=True):
"""
local_cache : {None, str}
Specify path to local cache. Local cache will be used to
temporarily store files from external devises such as NFS.
"""
self.verbose = verbose
self.first_byte = first_byte
self.use_memmap = use_memmap
try:
if local_cache is not None:
cache_filename = local_cache + '/' + filename
if os.path.exists(cache_filename):
filename = cache_filename
elif not isindisk(filename):
assert isindisk(local_cache), repr(local_cache)
dirname = os.path.dirname(cache_filename)
if not os.path.isdir(dirname):
os.makedirs(dirname)
shutil.copyfile(filename, cache_filename)
filename = cache_filename
if verbose:
sys.stdout.write('Opening file %r\n' % (filename))
sys.stdout.flush()
if mode != 'r':
raise NotImplementedError(repr(mode))
if not os.path.isfile(filename):
raise ValueError('file does not exists')
if not os.stat(filename).st_size:
raise ValueError('file has zero size')
if use_memmap:
self.data = numpy.memmap(filename, dtype=numpy.ubyte,
mode=mode)
else:
assert mode == 'r', repr(mode)
f = open(filename, 'rb')
self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte)
f.close()
except IOError as msg:
if 'Too many open files' in str(msg):
raise IOError(IOError_too_many_open_files_hint % msg)
if 'Operation not permitted' in str(msg):
raise IOError(OSError_operation_not_permitted_hint % msg)
raise
except OSError as msg:
if 'Operation not permitted' in str(msg):
raise OSError(OSError_operation_not_permitted_hint % msg)
raise
except mmap.error as msg:
if 'Too many open files' in str(msg):
raise mmap.error(IOError_too_many_open_files_hint % msg)
raise
self.filename = filename
self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')]
byteorder = self.data[first_byte:first_byte + 2].view(
dtype=numpy.uint16)[0]
if byteorder == 0x4949:
self.endian = 'little'
self.dtypes = LittleEndianNumpyDTypes
elif byteorder == 0x4d4d:
self.endian = 'big'
self.dtypes = BigEndianNumpyDTypes
else:
raise ValueError('unrecognized byteorder: %s' % (hex(byteorder)))
magic = self.get_uint16(first_byte + 2)
if magic != 42:
raise ValueError('wrong magic number for TIFF file: %s' % (magic))
self.IFD0 = IFD0 = first_byte + self.get_uint32(first_byte + 4)
self.memory_usage.append((first_byte, first_byte + 8, 'file header'))
n = self.get_uint16(IFD0)
IFD_list = []
IFD_offset = IFD0
while IFD_offset:
n = self.get_uint16(IFD_offset)
ifd = IFD(self)
exif_offset = 0
for i in range(n):
entry = IFDEntry(ifd, self, IFD_offset + 2 + i * 12)
ifd.append(entry)
if entry.tag == 0x8769: # TIFFTAG_EXIFIFD
exif_offset = entry.value
ifd.finalize()
IFD_list.append(ifd)
self.memory_usage.append((IFD_offset, IFD_offset + 2 + n * 12 + 4,
'IFD%s entries(%s)' % (
len(IFD_list), len(ifd))))
IFD_offset = self.get_uint32(IFD_offset + 2 + n * 12)
if IFD_offset == 0 and exif_offset != 0:
IFD_offset = exif_offset
exif_offset = 0
if verbose:
sys.stdout.write(
'\rIFD information read: %s..' % (len(IFD_list)))
sys.stdout.flush()
self.IFD = IFD_list
if verbose:
sys.stdout.write(' done\n')
sys.stdout.flush()
self.time = None
f = open (filename, 'rb')
self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte)
f.close()
except IOError, msg:
if 'Too many open files' in str (msg):
raise IOError(IOError_too_many_open_files_hint % msg)
if 'Operation not permitted' in str (msg):
raise IOError(OSError_operation_not_permitted_hint % msg)
raise
except OSError, msg:
if 'Operation not permitted' in str (msg):
raise OSError(OSError_operation_not_permitted_hint % msg)
raise
except mmap.error, msg:
if 'Too many open files' in str (msg):
raise mmap.error(IOError_too_many_open_files_hint % msg)
raise
self.filename = filename
self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')]
byteorder = self.data[first_byte:first_byte+2].view(dtype=numpy.uint16)[0]
if byteorder==0x4949:
self.endian = 'little'
self.dtypes = LittleEndianNumpyDTypes
elif byteorder==0x4d4d:
self.endian = 'big'
self.dtypes = BigEndianNumpyDTypes
else:
f = open(filename, 'rb')
self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte)
f.close()
except IOError, msg:
if 'Too many open files' in str(msg):
raise IOError(IOError_too_many_open_files_hint % msg)
if 'Operation not permitted' in str(msg):
raise IOError(OSError_operation_not_permitted_hint % msg)
raise
except OSError, msg:
if 'Operation not permitted' in str(msg):
raise OSError(OSError_operation_not_permitted_hint % msg)
raise
except mmap.error, msg:
if 'Too many open files' in str(msg):
raise mmap.error(IOError_too_many_open_files_hint % msg)
raise
self.filename = filename
self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')]
byteorder = self.data[first_byte:first_byte +
2].view(dtype=numpy.uint16)[0]
if byteorder == 0x4949:
self.endian = 'little'
self.dtypes = LittleEndianNumpyDTypes
elif byteorder == 0x4d4d:
self.endian = 'big'
self.dtypes = BigEndianNumpyDTypes
def __init__(self, filename, mode='r', first_byte=0, verbose=False,
local_cache=None, use_memmap=True):
"""
local_cache : {None, str}
Specify path to local cache. Local cache will be used to
temporarily store files from external devises such as NFS.
"""
self.verbose = verbose
self.first_byte = first_byte
self.use_memmap = use_memmap
try:
if local_cache is not None:
cache_filename = local_cache + '/' + filename
if os.path.exists(cache_filename):
filename = cache_filename
elif not isindisk(filename):
assert isindisk(local_cache), repr(local_cache)
dirname = os.path.dirname(cache_filename)
if not os.path.isdir(dirname):
os.makedirs(dirname)
shutil.copyfile(filename, cache_filename)
filename = cache_filename
if verbose:
sys.stdout.write('Opening file %r\n' % (filename));
sys.stdout.flush()
if mode != 'r':
raise NotImplementedError(repr(mode))
if not os.path.isfile(filename):
raise ValueError('file does not exists')
if not os.stat(filename).st_size:
raise ValueError('file has zero size')
if use_memmap:
self.data = numpy.memmap(filename, dtype=numpy.ubyte,
mode=mode)
else:
assert mode == 'r', repr(mode)
f = open(filename, 'rb')
self.data = numpy.frombuffer(f.read(), dtype=numpy.ubyte)
f.close()
except IOError as msg:
if 'Too many open files' in str(msg):
raise IOError(IOError_too_many_open_files_hint % msg)
if 'Operation not permitted' in str(msg):
raise IOError(OSError_operation_not_permitted_hint % msg)
raise
except OSError as msg:
if 'Operation not permitted' in str(msg):
raise OSError(OSError_operation_not_permitted_hint % msg)
raise
except mmap.error as msg:
if 'Too many open files' in str(msg):
raise mmap.error(IOError_too_many_open_files_hint % msg)
raise
self.filename = filename
self.memory_usage = [(self.data.nbytes, self.data.nbytes, 'eof')]
byteorder = \
self.data[first_byte:first_byte + 2].view(dtype=numpy.uint16)[0]
if byteorder == 0x4949:
self.endian = 'little'
self.dtypes = LittleEndianNumpyDTypes
elif byteorder == 0x4d4d:
self.endian = 'big'
self.dtypes = BigEndianNumpyDTypes
else:
raise ValueError('unrecognized byteorder: %s' % (hex(byteorder)))
magic = self.get_uint16(first_byte + 2)
if magic != 42:
raise ValueError('wrong magic number for TIFF file: %s' % (magic))
self.IFD0 = IFD0 = first_byte + self.get_uint32(first_byte + 4)
self.memory_usage.append((first_byte, first_byte + 8, 'file header'))
n = self.get_uint16(IFD0)
IFD_list = []
IFD_offset = IFD0
while IFD_offset:
n = self.get_uint16(IFD_offset)
ifd = IFD(self)
exif_offset = 0
for i in range(n):
entry = IFDEntry(ifd, self, IFD_offset + 2 + i * 12)
ifd.append(entry)
if entry.tag == 0x8769: # TIFFTAG_EXIFIFD
exif_offset = entry.value
ifd.finalize()
IFD_list.append(ifd)
self.memory_usage.append((IFD_offset, IFD_offset + 2 + n * 12 + 4,
'IFD%s entries (%s)' % (
len(IFD_list), len(ifd))))
IFD_offset = self.get_uint32(IFD_offset + 2 + n * 12)
if IFD_offset == 0 and exif_offset != 0:
IFD_offset = exif_offset
exif_offset = 0
if verbose:
sys.stdout.write(
'\rIFD information read: %s..' % (len(IFD_list)));
sys.stdout.flush()
self.IFD = IFD_list
if verbose:
sys.stdout.write(' done\n');
sys.stdout.flush()
self.time = None
