def _read_data_chunk(self):
"""
Reads a chunk of data for the intended computation into memory
"""
if self.__start_pos < self.__rank_end_pos:
self.__end_pos = int(
min(self.__rank_end_pos,
self.__start_pos + self._max_pos_per_read))
# DON'T DIRECTLY apply the start and end indices anymore to the h5 dataset. Find out what it means first
self.__pixels_in_batch = self.__compute_jobs[self.__start_pos:self.
__end_pos]
if self.verbose:
print('Rank {} will read positions: {}'.format(
self.mpi_rank, self.__pixels_in_batch))
bytes_this_read = self.__bytes_per_pos * len(
self.__pixels_in_batch)
print('Rank {} will read {} of the SOURCE dataset'
'.'.format(self.mpi_rank, format_size(bytes_this_read)))
if self.mpi_rank == self.__socket_master_rank:
tot_workers = self.__ranks_on_socket * self._cores
print('Rank: {} available memory: {}. '
'{} workers on this socket will in total read ~ {}'
'.'.format(
self.mpi_rank,
format_size(get_available_memory()), tot_workers,
format_size(bytes_this_read * tot_workers)))
# Reading as Dask array to minimize memory copies when restructuring in child classes
if self.__lazy:
main_dset = lazy_load_array(self.h5_main)
else:
main_dset = self.h5_main
self.data = main_dset[self.__pixels_in_batch, :]
# DON'T update the start position
else:
if self.verbose:
print('Rank {} - Finished reading all data!'.format(
self.mpi_rank))
self.data = None
def format_size(size_in_bytes, decimals=2):
"""
Formats the provided size in bytes to kB, MB, GB, TB etc.
Parameters
----------
size_in_bytes : number
size in bytes
decimals : uint, optional. default = 2
Number of decimal places to which the size needs to be formatted
Returns
-------
str
String with size formatted correctly
"""
warn(
'pyUSID.io.io_utils.format_size has been moved to '
'sidpy.base.string_utils.format_size. This copy in pyUSID will'
'be removed in future release. Please update your import statements',
FutureWarning)
return sut.format_size(size_in_bytes, decimals=decimals)
def compute(self, override=False, *args, **kwargs):
"""
Creates placeholders for the results, applies the :meth:`~pyUSID.processing.process.Process._unit_computation`
to chunks of the dataset
Parameters
----------
override : bool, optional. default = False
By default, compute will simply return duplicate results to avoid recomputing or resume computation on a
group with partial results. Set to True to force fresh computation.
args : list
arguments to the mapped function in the correct order
kwargs : dict
keyword arguments to the mapped function
Returns
-------
h5_results_grp : :class:`h5py.Group`
Group containing all the results
"""
class SimpleFIFO(object):
"""
Simple class that maintains a moving average of some numbers.
"""
def __init__(self, length=5):
"""
Create a SimpleFIFO object
Parameters
----------
length : unsigned integer
Number of values that need to be maintained for the moving average
"""
self.__queue = list()
if not isinstance(length, int):
raise TypeError('length must be a positive integer')
if length <= 0:
raise ValueError('length must be a positive integer')
self.__max_length = length
self.__count = 0
def put(self, item):
"""
Adds the item to the internal queue. If the size of the queue exceeds its capacity, the oldest
item is removed.
Parameters
----------
item : float or int
Any real valued number
"""
if (not isinstance(item, Number)) or isinstance(item, complex):
raise TypeError(
'Provided item: {} is not a Number'.format(item))
self.__queue.append(item)
self.__count += 1
if len(self.__queue) > self.__max_length:
_ = self.__queue.pop(0)
def get_mean(self):
"""
Returns the average of the elements within the queue
Returns
-------
avg : number.Number
Mean of all elements within the queue
"""
return np.mean(self.__queue)
def get_cycles(self):
"""
Returns the number of items that have been added to the queue in total
Returns
-------
count : int
number of items that have been added to the queue in total
"""
return self.__count
if not override:
if len(self.duplicate_h5_groups) > 0:
if self.mpi_rank == 0:
print('Returned previously computed results at ' +
self.duplicate_h5_groups[-1].name)
self.h5_results_grp = self.duplicate_h5_groups[-1]
return self.duplicate_h5_groups[-1]
elif len(self.partial_h5_groups
) > 0 and self.h5_results_grp is None:
if self.mpi_rank == 0:
print('Resuming computation in group: ' +
self.partial_h5_groups[-1].name)
self.use_partial_computation()
resuming = False
if self.h5_results_grp is None:
# starting fresh
if self.verbose and self.mpi_rank == 0:
print('Creating HDF5 group and datasets to hold results')
self._create_results_datasets()
self._write_source_dset_provenance()
else:
# resuming from previous checkpoint
resuming = True
self._get_existing_datasets()
self.__create_compute_status_dataset()
if resuming and self.mpi_rank == 0:
percent_complete = int(
100 * len(np.where(self._h5_status_dset[()] == 1)[0]) /
self._h5_status_dset.shape[0])
print('Resuming computation. {}% completed already'.format(
percent_complete))
self.__assign_job_indices()
# Not sure if this is necessary but I don't think it would hurt either
if self.mpi_comm is not None:
self.mpi_comm.barrier()
compute_times = SimpleFIFO(5)
write_times = SimpleFIFO(5)
orig_rank_start = self.__start_pos
if self.mpi_rank == 0 and self.mpi_size == 1:
if self.__resume_implemented:
print(
'\tThis class (likely) supports interruption and resuming of computations!\n'
'\tIf you are operating in a python console, press Ctrl+C or Cmd+C to abort\n'
'\tIf you are in a Jupyter notebook, click on "Kernel">>"Interrupt"\n'
'\tIf you are operating on a cluster and your job gets killed, re-run the job to resume\n'
)
else:
print(
'\tThis class does NOT support interruption and resuming of computations.\n'
'\tIn order to enable this feature, simply implement the _get_existing_datasets() function'
)
if self.verbose and self.mpi_rank == self.__socket_master_rank:
print('Rank: {} - with nothing loaded has {} free memory'
''.format(self.mpi_rank,
format_size(get_available_memory())))
self._read_data_chunk()
if self.mpi_comm is not None:
self.mpi_comm.barrier()
if self.verbose and self.mpi_rank == self.__socket_master_rank:
print('Rank: {} - with only raw data loaded has {} free memory'
''.format(self.mpi_rank,
format_size(get_available_memory())))
while self.data is not None:
num_jobs_in_batch = self.__end_pos - self.__start_pos
t_start_1 = tm.time()
self._unit_computation(*args, **kwargs)
comp_time = np.round(tm.time() - t_start_1,
decimals=2) # in seconds
time_per_pix = comp_time / num_jobs_in_batch
compute_times.put(time_per_pix)
if self.verbose:
print(
'Rank {} - computed chunk in {} or {} per pixel. Average: {} per pixel'
'.'.format(self.mpi_rank, format_time(comp_time),
format_time(time_per_pix),
format_time(compute_times.get_mean())))
# Ranks can become memory starved. Check memory usage - raw data + results in memory at this point
if self.verbose and self.mpi_rank == self.__socket_master_rank:
print(
'Rank: {} - now holding onto raw data + results has {} free memory'
''.format(self.mpi_rank,
format_size(get_available_memory())))
t_start_2 = tm.time()
self._write_results_chunk()
# NOW, update the positions. Users are NOT allowed to touch start and end pos
self.__start_pos = self.__end_pos
# Leaving in this provision that will allow restarting of processes
if self.mpi_size == 1:
self.h5_results_grp.attrs['last_pixel'] = self.__end_pos
# Child classes don't even have to worry about flushing. Process will do it.
self.h5_main.file.flush()
dump_time = np.round(tm.time() - t_start_2, decimals=2)
write_times.put(dump_time / num_jobs_in_batch)
if self.verbose:
print('Rank {} - wrote its {} pixel chunk in {}'.format(
self.mpi_rank, num_jobs_in_batch, format_time(dump_time)))
time_remaining = (self.__rank_end_pos - self.__end_pos) * \
(compute_times.get_mean() + write_times.get_mean())
if self.verbose or self.mpi_rank == 0:
percent_complete = int(100 *
(self.__end_pos - orig_rank_start) /
(self.__rank_end_pos - orig_rank_start))
print('Rank {} - {}% complete. Time remaining: {}'.format(
self.mpi_rank, percent_complete,
format_time(time_remaining)))
# All ranks should mark the pixels for this batch as completed. 'last_pixel' attribute will be updated later
# Setting each section to 1 independently
for curr_slice in integers_to_slices(self.__pixels_in_batch):
self._h5_status_dset[curr_slice] = 1
self._read_data_chunk()
if self.verbose:
print('Rank {} - Finished computing all jobs!'.format(
self.mpi_rank))
if self.mpi_comm is not None:
self.mpi_comm.barrier()
if self.mpi_rank == 0:
print('Finished processing the entire dataset!')
# Update the legacy 'last_pixel' attribute here:
if self.mpi_rank == 0:
self.h5_results_grp.attrs['last_pixel'] = self.h5_main.shape[0]
return self.h5_results_grp
def __set_memory(self, man_mem_limit=None, mem_multiplier=1.0):
"""
Checks memory capabilities of each node and sets the recommended data
chunk sizes to be used by analysis methods.
This function can work with clusters with heterogeneous memory sizes
(e.g. CADES SHPC Condo).
Parameters
----------
man_mem_limit : uint, optional, Default = None (all available memory)
The amount a memory in Mb to use in the computation
mem_multiplier : float, optional. Default = 1
mem_multiplier is the number that will be multiplied with the
(byte) size of a single position in the source dataset in order to
better estimate the number of positions that can be processed at
any given time (how many pixels of the source and results datasets
can be retained in memory). The default value of 1.0 only accounts
for the source dataset. A value greater than 1 would account for
the size of results datasets as well. For example, if the result
dataset is the same size and precision as the source dataset,
the multiplier will be 2 (1 for source, 1 for result)
"""
if not isinstance(mem_multiplier, float):
raise TypeError('mem_multiplier must be a floating point number')
mem_multiplier = abs(mem_multiplier)
if mem_multiplier < 1:
raise ValueError('mem_multiplier must be at least 1')
avail_mem_bytes = get_available_memory() # in bytes
if self.verbose and self.mpi_rank == self.__socket_master_rank:
# expected to be the same for all ranks so just use this.
print('Rank {} - on socket with {} cores and {} avail. RAM shared '
'by {} ranks each given {} cores'
'.'.format(self.__socket_master_rank, psutil.cpu_count(),
format_size(avail_mem_bytes),
self.__ranks_on_socket, self._cores))
if man_mem_limit is None:
man_mem_limit = avail_mem_bytes
else:
if not isinstance(man_mem_limit, int):
raise TypeError('man_mem_limit must be a whole number')
# Note that man_mem_limit is specified in mega bytes
man_mem_limit = abs(man_mem_limit) * 1024**2 # in bytes
if self.verbose and self.mpi_rank == 0:
print('User has requested to use no more than {} of memory'
'.'.format(format_size(man_mem_limit)))
max_mem_bytes = min(avail_mem_bytes, man_mem_limit)
# Remember that multiple processes (either via MPI or joblib) will share this socket
# This makes logical sense but there's always too much free memory and the
# cores are starved.
max_mem_per_worker = max_mem_bytes / (self._cores *
self.__ranks_on_socket)
if self.verbose and self.mpi_rank == self.__socket_master_rank:
print('Rank {}: Each of the {} workers on this socket are allowed '
'to use {} of RAM'
'.'.format(self.mpi_rank,
self._cores * self.__ranks_on_socket,
format_size(max_mem_per_worker)))
# Now calculate the number of positions OF RAW DATA ONLY that can be
# stored in memory in one go PER worker
self.__bytes_per_pos = self.h5_main.dtype.itemsize * self.h5_main.shape[
1]
if self.verbose and self.mpi_rank == 0:
print('Each position in the SOURCE dataset is {} large'
'.'.format(format_size(self.__bytes_per_pos)))
# Now multiply this with a factor that takes into account the expected
# sizes of the results (Final and intermediate) datasets.
self.__bytes_per_pos *= mem_multiplier
if self.verbose and self.mpi_rank == 0 and mem_multiplier > 1:
print('Each position of the source and results dataset(s) is {} '
'large.'.format(format_size(self.__bytes_per_pos)))
self._max_pos_per_read = int(
np.floor(max_mem_per_worker / self.__bytes_per_pos))
if self.verbose and self.mpi_rank == self.__socket_master_rank:
title = 'SOURCE dataset only'
if mem_multiplier > 1:
title = 'source and result(s) datasets'
# expected to be the same for all ranks so just use this.
print('Rank {}: Workers on this socket allowed to read {} '
'positions of the {} per chunk'
'.'.format(self.mpi_rank, self._max_pos_per_read, title))