def __next__(self): if self._data_indices is None: self._init_data_indices() data_indices_this = self._data_indices[:self._chunk_size] if not data_indices_this.size: raise StopIteration self._data_indices = self._data_indices[self._chunk_size:] return dict_to_numpy_array({name: self._descqa_obj[self._column_map[name][0]][data_indices_this] for name in self._colnames})
def __next__(self): descqa_catalog = self._descqa_obj._catalog if self._data_indices is None: if self._loaded_qties is not None: raise RuntimeError( "data_indices is None, but loaded_qties isn't " "in DESCQAChunkIterator") self._init_data_indices() qty_name_list = [ self._column_map[name][0] for name in self._colnames if descqa_catalog.has_quantity(self._column_map[name][0]) ] self._loaded_qties = {} for name in qty_name_list: raw_qties = descqa_catalog.get_quantities( name, native_filters=self._native_filters) self._loaded_qties[name] = raw_qties[name][self._data_indices] # since we are only keeping the objects that will ultimately go into # the catalog, we now change self._data_indices to range from 0 # to the length of the final catalog; the indices relative to the # extragalactic catalog have been forgotten self._data_indices = np.arange(0, len(self._data_indices), dtype=int) data_indices_this = self._data_indices[:self._chunk_size] if not data_indices_this.size: self._loaded_qties = None self._data_indices = None raise StopIteration self._data_indices = self._data_indices[self._chunk_size:] # temporarily suppress divide by zero warnings with np.errstate(divide='ignore', invalid='ignore'): chunk = dict_to_numpy_array({ name: self._loaded_qties[self._column_map[name][0]] [data_indices_this] for name in self._colnames if descqa_catalog.has_quantity(self._column_map[name][0]) }) return self._append_defaults(chunk)
def __next__(self): if self._data_indices is None: self._init_data_indices() descqa_catalog = self._descqa_obj._catalog data_indices_this = self._data_indices[:self._chunk_size] if not data_indices_this.size: raise StopIteration self._data_indices = self._data_indices[self._chunk_size:] chunk = dict_to_numpy_array({ name: self._descqa_obj._catalog[self._column_map[name][0]] [data_indices_this] for name in self._colnames if descqa_catalog.has_quantity(self._column_map[name][0]) }) need_to_append_defaults = False for name in self._colnames: if not descqa_catalog.has_quantity(self._column_map[name][0]): need_to_append_defaults = True break if need_to_append_defaults: dtype_list = [(name, chunk.dtype[name]) for name in chunk.dtype.names] for name in self._colnames: if not descqa_catalog.has_quantity(self._column_map[name][0]): dtype_list.append((name, self._default_values[name][1])) new_dtype = np.dtype(dtype_list) new_chunk = np.zeros(len(chunk), dtype=new_dtype) for name in self._colnames: if name in chunk.dtype.names: new_chunk[name] = chunk[name] else: new_chunk[name] = self._default_values[name][0] chunk = new_chunk return self._descqa_obj._postprocess_results(chunk)
def __next__(self): descqa_catalog = self._descqa_obj._catalog if self._healpix_and_indices_list is None: self._init_data_indices() self._qty_name_list = [ self._column_map[name][0] for name in self._colnames if descqa_catalog.has_quantity(self._column_map[name][0]) ] if self._loaded_qties is None or self._indices_to_load is None or len( self._data_indices) == 0: if self._indices_to_load is None or len( self._indices_to_load) == 0: try: (self._healpix_loaded, self._healpix_filter, self._indices_to_load ) = self._healpix_and_indices_list.pop() except IndexError: self._healpix_and_indices_list = None self._loaded_qties = None self._healpix_loaded = -1 self._data_indices = None self._qty_name_list = None self._descqa_obj._loaded_healpixel = -1 self._indices_to_load = None self._healpix_filter = None raise StopIteration self._indices_to_load = np.sort(self._indices_to_load) self._descqa_obj._loaded_healpixel = self._healpix_loaded self._healpix_loaded = self._healpix_loaded _DESCQAObject_metadata[ 'loaded_healpixel'] = self._healpix_loaded valid_indices = self._indices_to_load[:self._loader_chunk_size] self._indices_to_load = self._indices_to_load[self. _loader_chunk_size:] self._loaded_qties = {} for name in self._qty_name_list: raw_qties = descqa_catalog.get_quantities( name, native_filters=[self._healpix_filter]) self._loaded_qties[name] = raw_qties[name][valid_indices] self._data_indices = np.arange(len(valid_indices), dtype=int) if self._chunk_size is None: data_indices_this = self._data_indices else: data_indices_this = self._data_indices[:self._chunk_size] with np.errstate(divide='ignore', invalid='ignore'): chunk = dict_to_numpy_array({ name: self._loaded_qties[self._column_map[name][0]] [data_indices_this] for name in self._colnames if descqa_catalog.has_quantity(self._column_map[name][0]) }) self._data_indices = self._data_indices[len(data_indices_this):] return self._append_defaults(chunk)