def readall(self): """ Read all available data, returning a dictionary This provides ``Position`` and optionally ``Velocity`` columns """ from nbodykit.ndarray import extend_dtype # read in the plain text file as a recarray kwargs = {} kwargs['comments'] = '#' kwargs['names'] = self.names kwargs['usecols'] = self.usecols data = numpy.recfromtxt(self.path, **kwargs) nobj = len(data) # copy the data new_dtypes = [('Position', ('f4', len(self.poscols)))] if self.velcols is not None or self.rsd is not None: new_dtypes += [('Velocity', ('f4', len(self.velcols)))] data = extend_dtype(data, new_dtypes) # get position and velocity, if we have it pos = numpy.vstack(data[k] for k in self.poscols).T.astype('f4') pos *= self.posf if self.velcols is not None or self.rsd is not None: vel = numpy.vstack(data[k] for k in self.velcols).T.astype('f4') vel *= self.velf data['Velocity'] = Velocity # do RSD if self.rsd is not None: dir = "xyz".index(self.rsd) pos[:, dir] += vel[:, dir] pos[:, dir] %= self.BoxSize[dir] data['Position'] = pos # select based on input conditions if self.select is not None: mask = self.select.get_mask(data) data = data[mask] self.logger.info("total number of objects selected is %d / %d" % (len(data), nobj)) toret = {} for name in data.dtype.names: toret[name] = data[name].copy() return toret
def readall(self): """ Read all available data, returning a dictionary This provides ``Position`` and optionally ``Velocity`` columns """ from nbodykit.ndarray import extend_dtype # read in the plain text file as a recarray kwargs = {} kwargs["comments"] = "#" kwargs["names"] = self.names kwargs["usecols"] = self.usecols data = numpy.recfromtxt(self.path, **kwargs) nobj = len(data) # copy the data new_dtypes = [("Position", ("f4", len(self.poscols)))] if self.velcols is not None or self.rsd is not None: new_dtypes += [("Velocity", ("f4", len(self.velcols)))] data = extend_dtype(data, new_dtypes) # get position and velocity, if we have it pos = numpy.vstack(data[k] for k in self.poscols).T.astype("f4") pos *= self.posf if self.velcols is not None or self.rsd is not None: vel = numpy.vstack(data[k] for k in self.velcols).T.astype("f4") vel *= self.velf data["Velocity"] = Velocity # do RSD if self.rsd is not None: dir = "xyz".index(self.rsd) pos[:, dir] += vel[:, dir] pos[:, dir] %= self.BoxSize[dir] data["Position"] = pos # select based on input conditions if self.select is not None: mask = self.select.get_mask(data) data = data[mask] self.logger.info("total number of objects selected is %d / %d" % (len(data), nobj)) toret = {} for name in data.dtype.names: toret[name] = data[name].copy() return toret
def readall(self): """ Read all available data, returning a dictionary This provides ``Position`` and optionally ``Velocity`` columns, as well as any columns listed in ``names`` """ from nbodykit.ndarray import extend_dtype try: import pandas as pd except: name = self.__class__.__name__ raise ImportError("pandas must be installed to use %s" % name) if self.ftype == 'auto': if self.path.endswith('.hdf5'): self.ftype = 'hdf5' else: self.ftype = 'text' # read in the hdf5 file using pandas if self.ftype == 'hdf5': data = pd.read_hdf(self.path, self.names[0], columns=self.usecols) # read in the plain text file using pandas elif self.ftype == 'text': kwargs = {} kwargs['comment'] = '#' kwargs['names'] = self.names kwargs['header'] = None kwargs['engine'] = 'c' kwargs['delim_whitespace'] = True kwargs['usecols'] = self.usecols data = pd.read_csv(self.path, **kwargs) # make sure 'Position' or 'Velocity' aren't columns already if 'Position' in data.columns: raise ValueError( "'Position' should not be a named column in input data") if 'Velocity' in data.columns: raise ValueError( "'Velocity' should not be a named column in input data") # objects read initially nobj = len(data) # copy the data new_dtypes = [('Position', ('f4', len(self.poscols)))] if self.velcols is not None or self.rsd is not None: new_dtypes += [('Velocity', ('f4', len(self.velcols)))] toret = extend_dtype(data.to_records(), new_dtypes) # get position and velocity, if we have it pos = data[self.poscols].values.astype('f4') pos *= self.posf if self.velcols is not None or self.rsd is not None: vel = data[self.velcols].values.astype('f4') vel *= self.velf toret['Velocity'] = vel del data # shift position by RSD if self.rsd is not None: dir = "xyz".index(self.rsd) pos[:, dir] += vel[:, dir] pos[:, dir] %= self.BoxSize[dir] toret['Position'] = pos # select based on input conditions if self.select is not None: mask = self.select.get_mask(toret) toret = toret[mask] self.logger.info("total number of objects selected is %d / %d" % (len(toret), nobj)) toret_dict = {} for name in toret.dtype.names: toret_dict[name] = toret[name].copy() return toret_dict
def readall(self): """ Read all available data, returning a dictionary This provides ``Position`` and optionally ``Velocity`` columns, as well as any columns listed in ``names`` """ from nbodykit.ndarray import extend_dtype try: import pandas as pd except: name = self.__class__.__name__ raise ImportError("pandas must be installed to use %s" %name) if self.ftype == 'auto': if self.path.endswith('.hdf5'): self.ftype = 'hdf5' else: self.ftype = 'text' # read in the hdf5 file using pandas if self.ftype == 'hdf5': data = pd.read_hdf(self.path, self.names[0], columns=self.usecols) # read in the plain text file using pandas elif self.ftype == 'text': kwargs = {} kwargs['comment'] = '#' kwargs['names'] = self.names kwargs['header'] = None kwargs['engine'] = 'c' kwargs['delim_whitespace'] = True kwargs['usecols'] = self.usecols data = pd.read_csv(self.path, **kwargs) # make sure 'Position' or 'Velocity' aren't columns already if 'Position' in data.columns: raise ValueError("'Position' should not be a named column in input data") if 'Velocity' in data.columns: raise ValueError("'Velocity' should not be a named column in input data") # objects read initially nobj = len(data) # copy the data new_dtypes = [('Position', ('f4', len(self.poscols)))] if self.velcols is not None or self.rsd is not None: new_dtypes += [('Velocity', ('f4', len(self.velcols)))] toret = extend_dtype(data.to_records(), new_dtypes) # get position and velocity, if we have it pos = data[self.poscols].values.astype('f4') pos *= self.posf if self.velcols is not None or self.rsd is not None: vel = data[self.velcols].values.astype('f4') vel *= self.velf toret['Velocity'] = vel del data # shift position by RSD if self.rsd is not None: dir = "xyz".index(self.rsd) pos[:, dir] += vel[:, dir] pos[:, dir] %= self.BoxSize[dir] toret['Position'] = pos # select based on input conditions if self.select is not None: mask = self.select.get_mask(toret) toret = toret[mask] self.logger.info("total number of objects selected is %d / %d" % (len(toret), nobj)) toret_dict = {} for name in toret.dtype.names: toret_dict[name] = toret[name].copy() return toret_dict