def __next__(self):
if self._data_indices is None:
self._init_data_indices()
data_indices_this = self._data_indices[:self._chunk_size]
if not data_indices_this.size:
raise StopIteration
self._data_indices = self._data_indices[self._chunk_size:]
return dict_to_numpy_array({name: self._descqa_obj[self._column_map[name][0]][data_indices_this] for name in self._colnames})
def __next__(self):
descqa_catalog = self._descqa_obj._catalog
if self._data_indices is None:
if self._loaded_qties is not None:
raise RuntimeError(
"data_indices is None, but loaded_qties isn't "
"in DESCQAChunkIterator")
self._init_data_indices()
qty_name_list = [
self._column_map[name][0] for name in self._colnames
if descqa_catalog.has_quantity(self._column_map[name][0])
]
self._loaded_qties = {}
for name in qty_name_list:
raw_qties = descqa_catalog.get_quantities(
name, native_filters=self._native_filters)
self._loaded_qties[name] = raw_qties[name][self._data_indices]
# since we are only keeping the objects that will ultimately go into
# the catalog, we now change self._data_indices to range from 0
# to the length of the final catalog; the indices relative to the
# extragalactic catalog have been forgotten
self._data_indices = np.arange(0,
len(self._data_indices),
dtype=int)
data_indices_this = self._data_indices[:self._chunk_size]
if not data_indices_this.size:
self._loaded_qties = None
self._data_indices = None
raise StopIteration
self._data_indices = self._data_indices[self._chunk_size:]
# temporarily suppress divide by zero warnings
with np.errstate(divide='ignore', invalid='ignore'):
chunk = dict_to_numpy_array({
name: self._loaded_qties[self._column_map[name][0]]
[data_indices_this]
for name in self._colnames
if descqa_catalog.has_quantity(self._column_map[name][0])
})
return self._append_defaults(chunk)
def __next__(self):
if self._data_indices is None:
self._init_data_indices()
descqa_catalog = self._descqa_obj._catalog
data_indices_this = self._data_indices[:self._chunk_size]
if not data_indices_this.size:
raise StopIteration
self._data_indices = self._data_indices[self._chunk_size:]
chunk = dict_to_numpy_array({
name: self._descqa_obj._catalog[self._column_map[name][0]]
[data_indices_this]
for name in self._colnames
if descqa_catalog.has_quantity(self._column_map[name][0])
})
need_to_append_defaults = False
for name in self._colnames:
if not descqa_catalog.has_quantity(self._column_map[name][0]):
need_to_append_defaults = True
break
if need_to_append_defaults:
dtype_list = [(name, chunk.dtype[name])
for name in chunk.dtype.names]
for name in self._colnames:
if not descqa_catalog.has_quantity(self._column_map[name][0]):
dtype_list.append((name, self._default_values[name][1]))
new_dtype = np.dtype(dtype_list)
new_chunk = np.zeros(len(chunk), dtype=new_dtype)
for name in self._colnames:
if name in chunk.dtype.names:
new_chunk[name] = chunk[name]
else:
new_chunk[name] = self._default_values[name][0]
chunk = new_chunk
return self._descqa_obj._postprocess_results(chunk)
def __next__(self):
descqa_catalog = self._descqa_obj._catalog
if self._healpix_and_indices_list is None:
self._init_data_indices()
self._qty_name_list = [
self._column_map[name][0] for name in self._colnames
if descqa_catalog.has_quantity(self._column_map[name][0])
]
if self._loaded_qties is None or self._indices_to_load is None or len(
self._data_indices) == 0:
if self._indices_to_load is None or len(
self._indices_to_load) == 0:
try:
(self._healpix_loaded, self._healpix_filter,
self._indices_to_load
) = self._healpix_and_indices_list.pop()
except IndexError:
self._healpix_and_indices_list = None
self._loaded_qties = None
self._healpix_loaded = -1
self._data_indices = None
self._qty_name_list = None
self._descqa_obj._loaded_healpixel = -1
self._indices_to_load = None
self._healpix_filter = None
raise StopIteration
self._indices_to_load = np.sort(self._indices_to_load)
self._descqa_obj._loaded_healpixel = self._healpix_loaded
self._healpix_loaded = self._healpix_loaded
_DESCQAObject_metadata[
'loaded_healpixel'] = self._healpix_loaded
valid_indices = self._indices_to_load[:self._loader_chunk_size]
self._indices_to_load = self._indices_to_load[self.
_loader_chunk_size:]
self._loaded_qties = {}
for name in self._qty_name_list:
raw_qties = descqa_catalog.get_quantities(
name, native_filters=[self._healpix_filter])
self._loaded_qties[name] = raw_qties[name][valid_indices]
self._data_indices = np.arange(len(valid_indices), dtype=int)
if self._chunk_size is None:
data_indices_this = self._data_indices
else:
data_indices_this = self._data_indices[:self._chunk_size]
with np.errstate(divide='ignore', invalid='ignore'):
chunk = dict_to_numpy_array({
name: self._loaded_qties[self._column_map[name][0]]
[data_indices_this]
for name in self._colnames
if descqa_catalog.has_quantity(self._column_map[name][0])
})
self._data_indices = self._data_indices[len(data_indices_this):]
return self._append_defaults(chunk)