def calculate_nchunks(in_file_size, chunk_size=DEFAULT_CHUNK_SIZE):
""" Determine chunking for an input file.
Parameters
----------
in_file_size : int
the size of the input file
chunk_size : int or str
the desired chunk size
Returns
-------
nchunks, chunk_size, last_chunk_size
nchunks : int
the number of chunks
chunk_size : int
the size of each chunk in bytes
last_chunk_size : int
the size of the last chunk in bytes
Raises
------
ChunkingException
if the resulting nchunks is larger than MAX_CHUNKS
"""
if in_file_size <= 0:
raise ValueError("'in_file_size' must be strictly positive, not %d"
% in_file_size)
# convert a human readable description to an int
if isinstance(chunk_size, basestring):
chunk_size = reverse_pretty(chunk_size)
check_range('chunk_size', chunk_size, 1, blosc.BLOSC_MAX_BUFFERSIZE)
# downcast
if chunk_size > in_file_size:
log.verbose(
"Input was smaller than the given 'chunk_size': %s using: %s"
% (double_pretty_size(chunk_size),
double_pretty_size(in_file_size)))
chunk_size = in_file_size
quotient, remainder = divmod(in_file_size, chunk_size)
# the user wants a single chunk
if chunk_size == in_file_size:
nchunks = 1
chunk_size = in_file_size
last_chunk_size = in_file_size
# no remainder, perfect fit
elif remainder == 0:
nchunks = quotient
last_chunk_size = chunk_size
# with a remainder
else:
nchunks = quotient + 1
last_chunk_size = remainder
if nchunks > MAX_CHUNKS:
raise ChunkingException(
"nchunks: '%d' is greater than the MAX_CHUNKS: '%d'" %
(nchunks, MAX_CHUNKS))
log.verbose('nchunks: %d' % nchunks)
log.verbose('chunk_size: %s' % double_pretty_size(chunk_size))
log.verbose('last_chunk_size: %s' % double_pretty_size(last_chunk_size))
return nchunks, chunk_size, last_chunk_size
def test_verbose():
nt.assert_raises(TypeError, log.verbose, 'message', 'MAXIMUM')
log.set_level(log.DEBUG)
# should probably hijack the print statement
log.verbose('notification')
log.set_level(log.NORMAL)
def test_verbose():
nt.assert_raises(TypeError, log.verbose, 'message', 'MAXIMUM')
log.set_level(log.DEBUG)
# should probably hijack the print statement
log.verbose('notification')
log.set_level(log.NORMAL)