def test_set_get_fill_value_for_str_column(self): c = MaskedColumn(name='c', data=['xxxx', 'yyyy'], mask=[True, False]) # assert np.all(c.filled() == ['N/A', 'yyyy']) c.fill_value = 'ABCDEF' assert c.fill_value == 'ABCD' # string truncated to dtype length assert np.all(c.filled() == ['ABCD', 'yyyy']) assert np.all(c.filled('XY') == ['XY', 'yyyy'])
def test_mask_copy(): """Test that the mask is copied when copying a table (issue #7362).""" c = MaskedColumn([1, 2], mask=[False, True]) c2 = MaskedColumn(c, copy=True) c2.mask[0] = True assert np.all(c.mask == [False, True]) assert np.all(c2.mask == [True, True])
def test_remove_masked_column(self): t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1, 2, 3], mask=[0, 1, 0])) t['a'].fill_value = 42 t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1])) t.remove_column('b') assert t.masked assert np.all(t['a'] == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert t['a'].fill_value == 42 assert t.colnames == ['a']
def test_add_masked_row_to_masked_table_iterable(self): t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row([2, 5], mask=[1, 0]) t.add_row([3, 6], mask=[0, 1]) assert t.masked assert np.all(np.array(t['a']) == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert np.all(np.array(t['b']) == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping1(self): t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row({'b': 5, 'a': 2}, mask={'a': 1, 'b': 0}) t.add_row({'a': 3, 'b': 6}, mask={'b': 1, 'a': 0}) assert t.masked assert np.all(np.array(t['a']) == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert np.all(np.array(t['b']) == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def setup_method(self, method): self.a = MaskedColumn(name='a', data=[1, 2, 3], fill_value=1) self.b = MaskedColumn(name='b', data=[4, 5, 6], mask=True) self.c = MaskedColumn(name='c', data=[7, 8, 9], mask=False) self.d_mask = np.array([False, True, False]) self.d = MaskedColumn(name='d', data=[7, 8, 7], mask=self.d_mask) self.t = Table([self.a, self.b], masked=True) self.ca = Column(name='ca', data=[1, 2, 3]) self.sc = MaskedColumn(name='sc', data=[(1, 1.), (2, 2.), (3, 3.)], dtype='i8,f8', fill_value=(0, -1.))
def test_add_masked_column_to_masked_table(self): t = Table(masked=True) assert t.masked t.add_column(MaskedColumn(name='a', data=[1, 2, 3], mask=[0, 1, 0])) assert t.masked t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1])) assert t.masked assert isinstance(t['a'], MaskedColumn) assert isinstance(t['b'], MaskedColumn) assert np.all(t['a'] == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert np.all(t['b'] == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping2(self): # When adding values to a masked table, if the mask is specified as a # dict, then values not specified will have mask values set to True t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row({'b': 5}, mask={'b': 0}) t.add_row({'a': 3}, mask={'a': 0}) assert t.masked assert t['a'][0] == 1 and t['a'][2] == 3 assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert t['b'][1] == 5 assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping3(self): # When adding values to a masked table, if mask is not passed to # add_row, then the mask should be set to False if values are present # and True if not. t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row({'b': 5}) t.add_row({'a': 3}) assert t.masked assert t['a'][0] == 1 and t['a'][2] == 3 assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert t['b'][1] == 5 assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_masked_column_with_unit_in_qtable(): """Test that adding a MaskedColumn with a unit to QTable issues warning""" t = QTable() t['a'] = MaskedColumn([1, 2]) assert isinstance(t['a'], MaskedColumn) t['b'] = MaskedColumn([1, 2], unit=u.m) assert isinstance(t['b'], u.Quantity) with pytest.warns(UserWarning, match="dropping mask in Quantity column 'c'") as w: t['c'] = MaskedColumn([1, 2], unit=u.m, mask=[True, False]) assert len(w) == 1 assert isinstance(t['b'], u.Quantity)
def test_coercing_fill_value_type(): """ Test that masked column fill_value is coerced into the correct column type. """ # This is the original example posted on the astropy@scipy mailing list t = Table({'a': ['1']}, masked=True) t['a'].set_fill_value('0') t2 = Table(t, names=['a'], dtype=[np.int32]) assert isinstance(t2['a'].fill_value, np.int32) # Unit test the same thing. c = MaskedColumn(['1']) c.set_fill_value('0') c2 = MaskedColumn(c, dtype=np.int32) assert isinstance(c2.fill_value, np.int32)
def test_pickle_masked_column(protocol): c = MaskedColumn(data=[1, 2], name='a', format='%05d', description='col a', unit='cm', meta={'a': 1}) c.mask[1] = True c.fill_value = -99 cs = pickle.dumps(c) cp = pickle.loads(cs) assert np.all(cp._data == c._data) assert np.all(cp.mask == c.mask) assert cp.attrs_equal(c) assert cp.fill_value == -99 assert cp._parent_table is None assert repr(c) == repr(cp)
def test_masked_column_with_unit_in_qtable(): """Test that adding a MaskedColumn with a unit to QTable creates a MaskedQuantity.""" MaskedQuantity = Masked(u.Quantity) t = QTable() t['a'] = MaskedColumn([1, 2]) assert isinstance(t['a'], MaskedColumn) t['b'] = MaskedColumn([1, 2], unit=u.m) assert isinstance(t['b'], MaskedQuantity) assert np.all(t['b'].mask == False) # noqa t['c'] = MaskedColumn([1, 2], unit=u.m, mask=[True, False]) assert isinstance(t['c'], MaskedQuantity) assert np.all(t['c'].mask == [True, False])
def test_parsing_columns( columns_file ): """ Test parsing of COLUMN elements <COLUMN dmtype ref > <OPTIONMAPPING> <OPTION>"string"</OPTION> <ENUMLITERAL>"vodml-id"</ENUMLITERAL> or <SEMANTICCONCEPT> </OPTIONMAPPING> </COLUMN> The parser pushes the Table rows into value arrays """ luminosity = columns_file.find_instances( LuminosityMeasurement )[0] # COLUMN with dmtype, value (ivoa:string) assert len(luminosity.description) == 2 assert luminosity.description[0] == "some descriptive text" assert luminosity.description[1] == "more descriptive text" # COLUMN with dmtype, value, unit (ivoa:RealQuantity) # - uses astropy Quantity # o luminosity.value = Quantity with value array length 2 + unit # o luminosity.value[0] = Quantity with first value + unit assert len(luminosity.value) == 2 expected_lum = numpy.array([15.718, 14.847], dtype='float32') * u.Unit('mag') numpy.testing.assert_array_equal( expected_lum, luminosity.value ) # COLUMN with OPTIONMAPPING # MCD NOTE: TODO - enumeration type == 'string'.. OPTIONMAPPING not parsed/interpreted assert len(luminosity.type) == 2 assert luminosity.type[0] == "magnitude" assert luminosity.type[1] == "magnitude" #assert (luminosity.optionmapping) == 2 # Check multiplicity handling # o values dimension 1 == 'row' elem = columns_file.find_instances( MultiObj )[0] assert len(elem.a) == 2 assert isinstance(elem.a, MaskedColumn) assert elem.a.shape == (2,) numpy.testing.assert_array_equal( elem.a, MaskedColumn([100.0, 100.1], dtype='float32')) # MCD NOTE: TODO - check this, I do not think 'b' should be a List of MaskedColumn assert len(elem.b) == 1 # b is a list of length=1 assert isinstance(elem.b[0], MaskedColumn) # of MaskedColumn assert elem.b[0].shape == (2,2) # 2 rows x 2 values each numpy.testing.assert_array_equal( elem.b[0], MaskedColumn([[200.0, 201.0],[200.1,201.1]], dtype='float32'))
def setup_method(self, method): mask = [True, False, False] self.meta = {'a': 1, 'b': [2, 3]} a = self.a = MaskedColumn(name='a', data=[1, 2, 3], fill_value=10, mask=mask, meta={'a': 1}) b = self.b = MaskedColumn(name='b', data=[4.0, 5.0, 6.0], fill_value=10.0, mask=mask) c = self.c = MaskedColumn(name='c', data=['7', '8', '9'], fill_value='1', mask=mask)
def test_astropy(self): """test astropy table + test null_value + test masked column """ tablemaker = CDSTablesMaker() ntab = Table([(1, 2, None, 4, 999), (4.0, 1115.0, np.NaN, None, 999), (1.1, 2., 999, 12.3, 12.3), (-1.001, 2., 0, -99.12, np.NaN)], names=['a', 'b', 'col3', 'col4']) ntab["col4"] = MaskedColumn(ntab["col4"], mask=[(val > 0) for val in ntab["col4"]]) tablename = ReadMeCase.__result_file("astropy") table = tablemaker.addTable(ntab, name=tablename, description="test astropy table", nullvalue=999) col = table.get_column("col4") col.set_null_value(-1.001) tablemaker.writeCDSTables() self.assertTrue(self.__test_file(tablename, 3), "astropy file") readme = ReadMeCase.__readme_file("astropy") with open(readme, "w") as fd: tablemaker.makeReadMe(out=fd) self.assertTrue(self.__test_file(readme, 5), "Readme for astropy") sys.stderr.write("generate {0}/{1} {0}/{2}\n".format( RESULT_DIR, tablename, readme))
def _gemini_json_to_table(json): """ takes a JSON object as returned from the Gemini archive webserver and turns it into an `~astropy.table.Table` Parameters ---------- json : dict A JSON object from the Gemini archive webserver Returns ------- response : `~astropy.table.Table` """ data_table = Table(masked=True) for key in __keys__: col_data = np.array([obj.get(key, None) for obj in json]) atype = str col_mask = np.equal(col_data, None) data_table.add_column(MaskedColumn(col_data.astype(atype), name=key, mask=col_mask)) return data_table
def merge_gaia(tbl: Table) -> Table: """Merges Gaia data for non-Celestia stars.""" with gzip.open(GAIA_PATH, 'rb') as f: gaia = votable.parse_single_table(f).to_table() bp_rp = gaia['bp_rp'].filled(0) bp_rp2 = bp_rp * bp_rp gaia.add_column( MaskedColumn(data=gaia['phot_g_mean_mag'].filled(np.nan) + 0.01760 + bp_rp * 0.006860 + bp_rp2 * 0.1732, name='flux', mask=gaia['phot_g_mean_mag'].mask)) gaia.remove_columns(['phot_g_mean_mag', 'bp_rp']) gaia.rename_column('source_id', 'gaia') gaia.rename_column('r_est', 'dist') has_gaia = tbl[np.logical_not(tbl['gaia'].mask)] merged = join(has_gaia, gaia, keys=['gaia'], join_type='left', table_names=['cel', 'gaia']) merged['ra'] = np.where(merged['ra_gaia'].mask, merged['ra_cel'], merged['ra_gaia']) merged['dec'] = np.where(merged['dec_gaia'].mask, merged['dec_cel'], merged['dec_gaia']) merged.add_columns([ MaskedColumn(data=np.where(merged['dist_gaia'].mask, merged['dist_cel'], merged['dist_gaia']), name='dist', mask=np.logical_and(merged['dist_gaia'].mask, merged['dist_cel'].mask)), MaskedColumn(data=np.where(merged['flux_cel'].mask, merged['flux_gaia'], merged['flux_cel']), name='flux', mask=np.logical_and(merged['flux_cel'].mask, merged['flux_gaia'].mask)) ]) merged.remove_columns([ 'ra_cel', 'ra_gaia', 'dec_cel', 'dec_gaia', 'dist_cel', 'dist_gaia', 'flux_cel', 'flux_gaia' ]) return vstack([tbl[tbl['gaia'].mask], merged], join_type='exact')
def test_set_get_fill_value_for_structured_column(self): assert self.sc.fill_value == np.array((0, -1.), self.sc.dtype) sc = self.sc.copy() assert sc.fill_value.item() == (0, -1.) sc.fill_value = (-1, np.inf) assert sc.fill_value == np.array((-1, np.inf), self.sc.dtype) sc2 = MaskedColumn(sc, fill_value=(-2, -np.inf)) assert sc2.fill_value == np.array((-2, -np.inf), sc2.dtype)
def add_distance(self, verbose = True): """ Add "distance" column to the catalogue, where distance = 1000./parallaxes. """ ncol = MaskedColumn(data = 1./self.cat['parallax'] * 1000, name = 'distance', unit = u.parsec, format = '4.1F') self.cat.add_column(ncol) if verbose: print('Adding new column to Gaia DR3 dataset: Distance')
def _fromat_column_entry(self, t): for n in self._numeric_fields: if n in t.colnames: try: if None in t[n].data: t[n] = MaskedColumn(t[n].data, name=n, dtype=np.float, mask=t[n].data == None) else: t[n] = MaskedColumn(t[n].data, name=n, dtype=np.float) t[n].format = '%e' except: for ID, v in enumerate(t[n].data): try: c = ast.literal_eval(t[n].data[ID]) if type(c) == int: t[n].data[ID] = '%d' % c else: t[n].data[ID] = '%e' % c except: pass
def query(coords, catalog, cols, radius=6 * u.arcsec, fill_val=-99.99, full=False): results = Vizier.query_region(coords, catalog=catalog, radius=radius) if len(results) == 0: return None if full: return results results = results[0] # if dict, remap colnames if isinstance(cols, dict): for k, v in cols.iteritems(): results.rename_column(k, v) names = cols.values() else: names = cols # make new columns one-to-one with coords newtable = Table(masked=True) for col in names: oldcol = results[col] newcol = MaskedColumn(data=np.zeros(len(coords), dtype=oldcol.dtype), unit=oldcol.unit, name=col, mask=np.ones(len(coords), dtype=bool), fill_value=fill_val) # copy data from results for row in results: if not row[col]: continue # _q IS 1-BASED INDEXING?! newcol[row['_q'] - 1] = row[col] newcol.mask[row['_q'] - 1] = False newtable.add_column(newcol) return newtable
def _json_to_table(json_obj, col_config=None): """ Takes a JSON object as returned from a Mashup request and turns it into an `~astropy.table.Table`. Parameters ---------- json_obj : dict A Mashup response JSON object (python dictionary) col_config : dict, optional Dictionary that defines column properties, e.g. default value. Returns ------- response : `~astropy.table.Table` """ data_table = Table(masked=True) if not all(x in json_obj.keys() for x in ['fields', 'data']): raise KeyError("Missing required key(s) 'data' and/or 'fields.'") for col, atype in [(x['name'], x['type']) for x in json_obj['fields']]: # Removing "_selected_" column if col == "_selected_": continue # reading the colum config if given ignore_value = None if col_config: col_props = col_config.get(col, {}) ignore_value = col_props.get("ignoreValue", None) # regularlizing the type reg_type = utils.parse_type(atype) atype = reg_type[1] ignore_value = reg_type[2] if (ignore_value is None) else ignore_value # Make the column list (don't assign final type yet or there will be errors) col_data = np.array( [x.get(col, ignore_value) for x in json_obj['data']], dtype=object) if ignore_value is not None: col_data[np.where(np.equal(col_data, None))] = ignore_value # no consistant way to make the mask because np.equal fails on '' # and array == value fails with None if atype == 'str': col_mask = (col_data == ignore_value) else: col_mask = np.equal(col_data, ignore_value) # add the column data_table.add_column( MaskedColumn(col_data.astype(atype), name=col, mask=col_mask)) return data_table
def test_setting_from_masked_column(): """Test issue in #2997""" mask_b = np.array([True, True, False, False]) for select in (mask_b, slice(0, 2)): t = Table(masked=True) t['a'] = Column([1, 2, 3, 4]) t['b'] = MaskedColumn([11, 22, 33, 44], mask=mask_b) t['c'] = MaskedColumn([111, 222, 333, 444], mask=[True, False, True, False]) t['b'][select] = t['c'][select] assert t['b'][1] == t[1]['b'] assert t['b'][0] is np.ma.masked # Original state since t['c'][0] is masked assert t['b'][1] == 222 # New from t['c'] since t['c'][1] is unmasked assert t['b'][2] == 33 assert t['b'][3] == 44 assert np.all(t['b'].mask == t.mask['b']) # Avoid t.mask in general, this is for testing mask_before_add = t.mask.copy() t['d'] = np.arange(len(t)) assert np.all(t.mask['b'] == mask_before_add['b'])
def test_add_masked_column_to_non_masked_table(self): t = Table(masked=False) assert not t.masked t.add_column(Column(name='a', data=[1, 2, 3])) assert not t.masked t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1])) assert t.masked assert np.all(t['a'] == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 0, 0], bool)) assert np.all(t['b'] == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def match_truth_z(simz_tab, zb_tab, mini_read=False, outfil=None): """ Match truth and zbest tables :param simz_tab: astropy.Table; Either generated from load_z() or read from disk via 'truth.fits' :param zb_tab: astropy.Table; Either generated from load_z() or read from disk via 'zcatalog-mini.fits' :param mini_read: bool, optional; Tables were read from the summary tables written to disk :param outfil: str, optional :return: simz_tab: modified in place """ nsim = len(simz_tab) # Match up sim_id = np.array(simz_tab['TARGETID']) z_id = np.array(zb_tab['TARGETID']) inz = np.in1d(z_id, sim_id, assume_unique=True) ins = np.in1d(sim_id, z_id, assume_unique=True) z_idx = np.arange(z_id.shape[0])[inz] sim_idx = np.arange(sim_id.shape[0])[ins] assert np.array_equal(sim_id[sim_idx], z_id[z_idx]) # Fill up ztags = ['Z', 'ZERR', 'ZWARN', 'SPECTYPE'] # This is for truth and zcat tables read from disk as opposed to the fibermap files if mini_read: ztags += ['DESI_TARGET'] # And clean up the QSO names stypes = np.char.rstrip(simz_tab['TEMPLATETYPE']) qsol = np.where((stypes == 'QSO') & (simz_tab['TRUEZ'] >= 2.1))[0] simz_tab['TEMPLATETYPE'][qsol] = 'QSO_L' qsot = np.where((stypes == 'QSO') & (simz_tab['TRUEZ'] < 2.1))[0] simz_tab['TEMPLATETYPE'][qsot] = 'QSO_T' # Generate the new columns new_clms = [] mask = np.array([True] * nsim) mask[sim_idx] = False for kk, ztag in enumerate(ztags): # Generate a MaskedColumn new_clm = MaskedColumn([zb_tab[ztag][z_idx[0]]] * nsim, name=ztag, mask=mask) #name=new_tags[kk], mask=mask) # Fill new_clm[sim_idx] = zb_tab[ztag][z_idx] # Append new_clms.append(new_clm) # Add columns simz_tab.add_columns(new_clms) # Write? if outfil is not None: simz_tab.write(outfil, overwrite=True) return
def test_convert_to_masked_table_only_if_necessary(self): # Do not convert to masked table, if new column has no masked value. # See #1185 for details. t = Table(masked=False) assert not t.masked t.add_column(Column(name='a', data=[1, 2, 3])) assert not t.masked t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[0, 0, 0])) assert not t.masked assert np.all(t['a'] == np.array([1, 2, 3])) assert np.all(t['b'] == np.array([4, 5, 6]))
def set_last_version(self, update=True): r"""Set the `last_version` column to the `result_from_query` attribute `last_version` is a column of `bool` where `False` means that there is a more update version of a catalogue This works only if `result_from_query` contains the columns: `version` and `title`. In case the `last_version` column is already present, a warning is raised. Args: update (bool): in case the `last_version` column is already present the code will update the value only if `update` is set to `True` Returns: None """ # Require that title and version are present in result_from_query for check_column in ['title', 'version']: if check_column not in self.which_columns(): msgs.warning( '{} column not present, `last_version` will not be created' .format(check_column)) # Check that last_version is not present in result_from_query if 'last_version' in self.which_columns(): if update: msgs.warning('`last_version` column already present') else: msgs.warning( '`last_version` column already present and it will not be updated' ) return # Get last versions unique_titles = np.unique( self.result_from_query['title'].data).tolist() last_version = np.zeros_like(self.result_from_query['version'].data, dtype=bool) for unique_title in unique_titles: most_recent_version = np.nanmax( self.result_from_query['version'].data[( self.result_from_query['title'].data == unique_title)]) last_version[(self.result_from_query['title'].data == unique_title) & (self.result_from_query['version'].data == most_recent_version)] = True self.result_from_query.add_column( MaskedColumn( data=last_version, name='last_version', dtype=bool, description='True if this is the latest version of the catalog' )) return
def merge_tables() -> Table: """Merges the tables.""" data = join(load_gaia_tyc(), load_tyc_spec(), keys=['TYC'], join_type='left') data = join( data, load_ascc(), keys=['TYC'], join_type='left', table_names=('gaia', 'ascc'), metadata_conflicts='silent', ) data['SpType'] = MaskedColumn(data['SpType_gaia'].filled(data['SpType_ascc'].filled(''))) data['SpType'].mask = data['SpType'] == '' data.remove_columns(['SpType_gaia', 'SpType_ascc']) data = join(data, load_tyc_hd(), keys=['TYC'], join_type='left', metadata_conflicts='silent') data = join( data, load_tyc_teff(), keys=['TYC'], join_type='left', table_names=('gaia', 'tycteff'), ) data['teff_val'] = MaskedColumn( data['teff_val_gaia'].filled(data['teff_val_tycteff'].filled(np.nan)), ) data['teff_val'].mask = np.isnan(data['teff_val']) data.remove_columns(['teff_val_tycteff', 'teff_val_gaia']) data = join(data, load_sao(), keys=['TYC'], join_type='left') return data
def _add_blue_cols(self, tbl): """Add empty columns for the Ugr data.""" from astropy.table import MaskedColumn # Prepare the column arrays col_nan = np.array([np.nan for i in range(len(tbl))]) col_nullbyte = np.array(['\x00' for i in range(len(tbl))]) col_false = np.array([False for i in range(len(tbl))]) col_errormsg = np.array(['No_blue_data' for i in range(len(tbl))]) # Now add identical empty columns for each band for band in ['u', 'g', 'r2']: for colname in [ 'detectionID_', '', 'err_', 'chi_', 'sharpness_', 'sky_', 'error_', 'aperMag_', 'aperMagErr_', 'snr_', 'magLim_', 'psffwhm_', 'airmass_', 'mjd_', 'pixelShift_', 'clean_', 'offsetRa_', 'offsetDec_' ]: if colname == 'clean_': mydtype = 'bool' mydata = col_false elif colname == 'detectionID_': mydtype = '23a' mydata = col_nullbyte elif colname == 'error_': mydtype = '12a' mydata = col_errormsg elif colname == 'mjd_': mydtype = 'float64' mydata = col_nan else: mydtype = 'float32' mydata = col_nan # Note that mask=false, because we want to impose our own choice of missing values # to be written to the FITS files, rather than depending on astropy's behaviour here tbl[colname + band] = MaskedColumn(mydata, mask=col_false, dtype=mydtype) # Also add the composite colour columns tbl['u_g'] = MaskedColumn(col_nan, mask=col_false, dtype='float32') tbl['g_r2'] = MaskedColumn(col_nan, mask=col_false, dtype='float32') return tbl
def set_null_value(self, null_value): """Assign null value to the Column (create an astropy MaskedColumn) :param null_value: value """ mask = [] for value in self.__column: if isinstance(value, numpy.ma.core.MaskedConstant): mask.append(True) else: mask.append((value == null_value)) # mask = [(value==null_value) for value in col] self.__column = MaskedColumn(self.__column, mask=mask)
def test_add_masked_column_to_non_masked_table(self): t = Table(masked=False) assert not t.masked t.add_column(Column(name='a', data=[1, 2, 3])) assert not t.masked t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1])) assert not t.masked # Changed in 4.0, table no longer auto-upgrades assert isinstance(t['a'], Column) # Was MaskedColumn before 4.0 assert isinstance(t['b'], MaskedColumn) assert np.all(t['a'] == np.array([1, 2, 3])) assert not hasattr(t['a'], 'mask') assert np.all(t['b'] == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def query(coords,catalog,cols,radius=6*u.arcsec,fill_val=-99.99,full=False): results = Vizier.query_region(coords,catalog=catalog,radius=radius) if len(results) == 0: return None if full: return results results = results[0] # if dict, remap colnames if isinstance(cols,dict): for k,v in cols.iteritems(): results.rename_column(k,v) names = cols.values() else: names = cols # make new columns one-to-one with coords newtable = Table(masked=True) for col in names: oldcol = results[col] newcol = MaskedColumn(data=np.zeros(len(coords),dtype=oldcol.dtype),unit=oldcol.unit,name=col,mask=np.ones(len(coords),dtype=bool),fill_value=fill_val) # copy data from results for row in results: if not row[col]: continue # _q IS 1-BASED INDEXING?! newcol[row['_q']-1] = row[col] newcol.mask[row['_q']-1] = False newtable.add_column(newcol) return newtable
def parse_location(column): """ Given a column of location data in the form 'S10E10' convert to two columns of angles. """ latitude = MaskedColumn(name="Latitude", unit=u.deg) longitude = MaskedColumn(name="Longitude", unit=u.deg) for i, loc in enumerate(column): if loc: lati = parse_latitude(loc) longi = parse_longitude(loc) latitude = latitude.insert(i, lati) longitude = longitude.insert(i, longi) else: latitude = latitude.insert(i, None, mask=True) longitude = longitude.insert(i, None, mask=True) return latitude, longitude
def merge_lightcurve_fits(options): """Pythom program to merge the lightcurve fit results into a sigle format""" import numpy import astropy import os import JLA_library as JLA from astropy.table import Table, MaskedColumn, vstack from scipy.optimize import leastsq params = JLA.build_dictionary(options.config) # ---------------- JLA ------------------------ lightCurveFits = JLA.get_full_path(params['JLAlightCurveFits']) f=open(lightCurveFits) header=f.readlines() f.close() names=header[0].strip('#').split() # I imagine that the tables package in astropy could also be used to read the ascii input file SNeSpec = Table(numpy.genfromtxt(lightCurveFits, skip_header=1, dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8', names=names)) nSNeSpec=len(SNeSpec) print 'There are %d SNe from the spectrscopically confirmed sample' % (nSNeSpec) # Add an extra column to the table SNeSpec['source']=['JLA']*nSNeSpec # -------------- Malmquist bias fits malmBias={} if options.bias: # Compute the bias correction bias = numpy.genfromtxt(JLA.get_full_path(params['biasPolynomial']), skip_header=3, usecols=(0, 1, 2, 3), dtype='S10,f8,f8,f8', names=['sample', 'redshift', 'bias', 'e_bias']) for sample in numpy.unique(bias['sample']): selection=(bias['sample']==sample) guess=[0,0,0] plsq=leastsq(residuals, guess, args=(bias[selection]['bias'], bias[selection]['redshift'], bias[selection]['e_bias'], 'poly'), full_output=1) if plsq[4] in [1,2,3,4]: print 'Solution for %s found' % (sample) malmBias[sample]=plsq[0] # ---------------- Shuvo's sample aka JLA++ ------------------------ # Photometrically identified SNe in Shuvo's sample, if the parameter exists if params['photLightCurveFits']!='None': lightCurveFits = JLA.get_full_path(params['photLightCurveFits']) SNePhot=Table.read(lightCurveFits, format='fits') nSNePhot=len(SNePhot) print 'There are %d SNe from the photometric sample' % (nSNePhot) # Converting from Shuvo's names to thosed used by JLA conversion={'name':'name_adj', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':'col27', 'd3rdvar':'d3rdvar', 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':'col4', 'dec':'col5', 'biascor':None} # Add the uncertainty in the mass column SNePhot['d3rdvar']=(SNePhot['col29']+SNePhot['col28'])/2. - SNePhot['col27'] # Remove columns that are not listed in conversion for colname in SNePhot.colnames: if colname not in conversion.values(): SNePhot.remove_column(colname) for key in conversion.keys(): # Rename the column if it does not already exist if conversion[key]!=None and conversion[key]!=key: SNePhot.rename_column(conversion[key], key) elif conversion[key]==None: # Create it, mask it, and fill all values SNePhot[key]=MaskedColumn(numpy.zeros(nSNePhot), numpy.ones(nSNePhot,bool)) SNePhot[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line SNePhot[key]=-99.9 else: # Do nothing if the column already exists pass # Compute the bias correction for i,SN in enumerate(SNePhot): if 'SDSS' in SN['name']: SNePhot['biascor'][i]=poly(SN['zhel'],malmBias['SDSS']) else: SNePhot['biascor'][i]=poly(SN['zhel'],malmBias['SNLS']) # Add the source column SNePhot['source']="Phot_Uddin" # ---------------------- CfA4 ---------------------------------- if params['CfA4LightCurveFits']!='None': lightCurveFits = JLA.get_full_path(params['CfA4LightCurveFits']) f=open(lightCurveFits) header=f.readlines() f.close() names=header[0].strip('#').split(',') SNeCfA4=Table(numpy.genfromtxt(lightCurveFits, skip_header=1, dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8', names=names,delimiter=',')) nSNeCfA4=len(SNeCfA4) conversion={'name':'name', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':None, 'd3rdvar':None, 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':None, 'dec':None, 'biascor':None} # Remove columns that are not listed in conversion for colname in SNeCfA4.colnames: if colname not in conversion.values(): SNeCfA4.remove_column(colname) for key in conversion.keys(): # Rename the column if it does not already exist if conversion[key]!=None and conversion[key]!=key: SNeCfA4.rename_column(conversion[key], key) elif conversion[key]==None: # Create it, mask it, and fill all values SNeCfA4[key]=MaskedColumn(numpy.zeros(nSNeCfA4), numpy.ones(nSNeCfA4,bool)) SNeCfA4[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line SNeCfA4[key]=-99.9 else: # Do nothing if the column already exists pass # Add the source column SNeCfA4['source']="CfA4" # We also need to gather information on the host mass, the host mass uncertainty, CMB redshift and Malmquist bias CfA4lightcurves=[] CfA4_lcDirectories=params['CfA4MassesAndCMBz'].split(',') for lcDir in CfA4_lcDirectories: listing=os.listdir(JLA.get_full_path(lcDir)) for lc in listing: CfA4lightcurves.append(JLA.get_full_path(lcDir)+lc) for i,SN in enumerate(SNeCfA4): for lc in CfA4lightcurves: if SN['name'][2:] in lc: keywords=getKeywords(lc) SNeCfA4[i]['zcmb']=keywords['REDSHIFT_CMB'] SNeCfA4[i]['3rdvar']=keywords['HOSTGAL_LOGMASS'] SNeCfA4[i]['d3rdvar']=keywords['e_HOSTGAL_LOGMASS'] SNeCfA4[i]['ra']=keywords['RA'] SNeCfA4[i]['dec']=keywords['DECL'] if SNeCfA4[i]['3rdvar'] < 0: SNeCfA4[i]['3rdvar']=-99.9 SNeCfA4[i]['d3rdvar']=-99.9 # Compute the bias correction SNeCfA4['biascor']=poly(SNeCfA4['zcmb'],malmBias['nearby']) try: SNe=vstack([SNeSpec,SNePhot,SNeCfA4]) except: SNe=SNeSpec # print len(SNe),len(numpy.unique(SNe['name'])) # Write out the result as a FITS table date = JLA.get_date() SNe.write('%s_%s.fits' % (options.output, date), format='fits', overwrite=True) return
def _fits_summary(self, header_keywords): """ Generate a summary table of keywords from FITS headers. Parameters ---------- header_keywords : list of str or '*' Keywords whose value should be extracted from FITS headers or '*' to extract all. """ if not self.files: return None # Make sure we have a list...for example, in python 3, dict.keys() # is not a list. original_keywords = list(header_keywords) # Get rid of any duplicate keywords, also forces a copy. header_keys = set(original_keywords) header_keys.add('file') file_name_column = MaskedColumn(name='file', data=self.files) if not header_keys or (header_keys == {'file'}): summary_table = Table(masked=True) summary_table.add_column(file_name_column) return summary_table summary_dict = None missing_marker = None for file_name in file_name_column.tolist(): file_path = path.join(self.location, file_name) try: # Note: summary_dict is an OrderedDict, so should preserve # the order of the keywords in the FITS header. summary_dict = self._dict_from_fits_header( file_path, input_summary=summary_dict, missing_marker=missing_marker) except IOError as e: logger.warning('unable to get FITS header for file %s: %s.', file_path, e) continue summary_table = Table(summary_dict, masked=True) for column in summary_table.colnames: summary_table[column].mask = [ v is missing_marker for v in summary_table[column].tolist()] self._set_column_name_case_to_match_keywords(header_keys, summary_table) missing_columns = header_keys - set(summary_table.colnames) missing_columns -= {'*'} length = len(summary_table) for column in missing_columns: all_masked = MaskedColumn(name=column, data=np.zeros(length), mask=np.ones(length)) summary_table.add_column(all_masked) if '*' not in header_keys: # Rearrange table columns to match order of keywords. # File always comes first. header_keys -= {'file'} original_order = ['file'] + sorted(header_keys, key=original_keywords.index) summary_table = summary_table[original_order] if not summary_table.masked: summary_table = Table(summary_table, masked=True) return summary_table
def read_table_fits(input, hdu=None, astropy_native=False, memmap=False, character_as_bytes=True): """ Read a Table object from an FITS file If the ``astropy_native`` argument is ``True``, then input FITS columns which are representations of an astropy core object will be converted to that class and stored in the ``Table`` as "mixin columns". Currently this is limited to FITS columns which adhere to the FITS Time standard, in which case they will be converted to a `~astropy.time.Time` column in the output table. Parameters ---------- input : str or file-like object or compatible `astropy.io.fits` HDU object If a string, the filename to read the table from. If a file object, or a compatible HDU object, the object to extract the table from. The following `astropy.io.fits` HDU objects can be used as input: - :class:`~astropy.io.fits.hdu.table.TableHDU` - :class:`~astropy.io.fits.hdu.table.BinTableHDU` - :class:`~astropy.io.fits.hdu.table.GroupsHDU` - :class:`~astropy.io.fits.hdu.hdulist.HDUList` hdu : int or str, optional The HDU to read the table from. astropy_native : bool, optional Read in FITS columns as native astropy objects where possible instead of standard Table Column objects. Default is False. memmap : bool, optional Whether to use memory mapping, which accesses data on disk as needed. If you are only accessing part of the data, this is often more efficient. If you want to access all the values in the table, and you are able to fit the table in memory, you may be better off leaving memory mapping off. However, if your table would not fit in memory, you should set this to `True`. character_as_bytes : bool, optional If `True`, string columns are stored as Numpy byte arrays (dtype ``S``) and are converted on-the-fly to unicode strings when accessing individual elements. If you need to use Numpy unicode arrays (dtype ``U``) internally, you should set this to `False`, but note that this will use more memory. If set to `False`, string columns will not be memory-mapped even if ``memmap`` is `True`. """ if isinstance(input, HDUList): # Parse all table objects tables = OrderedDict() for ihdu, hdu_item in enumerate(input): if isinstance(hdu_item, (TableHDU, BinTableHDU, GroupsHDU)): tables[ihdu] = hdu_item if len(tables) > 1: if hdu is None: warnings.warn("hdu= was not specified but multiple tables" " are present, reading in first available" " table (hdu={0})".format(first(tables)), AstropyUserWarning) hdu = first(tables) # hdu might not be an integer, so we first need to convert it # to the correct HDU index hdu = input.index_of(hdu) if hdu in tables: table = tables[hdu] else: raise ValueError("No table found in hdu={0}".format(hdu)) elif len(tables) == 1: table = tables[first(tables)] else: raise ValueError("No table found") elif isinstance(input, (TableHDU, BinTableHDU, GroupsHDU)): table = input else: hdulist = fits_open(input, character_as_bytes=character_as_bytes, memmap=memmap) try: return read_table_fits(hdulist, hdu=hdu, astropy_native=astropy_native) finally: hdulist.close() # Check if table is masked masked = any(col.null is not None for col in table.columns) # TODO: in future, it may make more sense to do this column-by-column, # rather than via the structured array. # In the loop below we access the data using data[col.name] rather than # col.array to make sure that the data is scaled correctly if needed. data = table.data columns = [] for col in data.columns: # Set column data if masked: column = MaskedColumn(data=data[col.name], name=col.name, copy=False) if col.null is not None: column.set_fill_value(col.null) column.mask[column.data == col.null] = True else: column = Column(data=data[col.name], name=col.name, copy=False) # Copy over units if col.unit is not None: column.unit = u.Unit(col.unit, format='fits', parse_strict='silent') # Copy over display format if col.disp is not None: column.format = _fortran_to_python_format(col.disp) columns.append(column) # Create Table object t = Table(columns, masked=masked, copy=False) # TODO: deal properly with unsigned integers hdr = table.header if astropy_native: # Avoid circular imports, and also only import if necessary. from .fitstime import fits_to_time hdr = fits_to_time(hdr, t) for key, value, comment in hdr.cards: if key in ['COMMENT', 'HISTORY']: # Convert to io.ascii format if key == 'COMMENT': key = 'comments' if key in t.meta: t.meta[key].append(value) else: t.meta[key] = [value] elif key in t.meta: # key is duplicate if isinstance(t.meta[key], list): t.meta[key].append(value) else: t.meta[key] = [t.meta[key], value] elif is_column_keyword(key) or key in REMOVE_KEYWORDS: pass else: t.meta[key] = value # TODO: implement masking # Decode any mixin columns that have been stored as standard Columns. t = _decode_mixins(t) return t
def createTable(outlines, metaDict, colNames, colDefaults): """ Creates an astropy table from inputs. Parameters ---------- outlines : list of str Input lines metaDict : dict Input meta data colNames : list of str Input column names colDefaults : list Input column default values Returns ------- table : astropy.table.Table object """ # Before loading table into an astropy Table object, set lengths of Name, # Patch, and Type columns to 100 characters log = logging.getLogger('LSMTool.Load') converters = {} nameCol = 'col{0}'.format(colNames.index('Name')+1) converters[nameCol] = [ascii.convert_numpy('{}100'.format(numpy_type))] typeCol = 'col{0}'.format(colNames.index('Type')+1) converters[typeCol] = [ascii.convert_numpy('{}100'.format(numpy_type))] if 'Patch' in colNames: patchCol = 'col{0}'.format(colNames.index('Patch')+1) converters[patchCol] = [ascii.convert_numpy('{}100'.format(numpy_type))] log.debug('Creating table...') table = Table.read('\n'.join(outlines), guess=False, format='ascii.no_header', delimiter=',', names=colNames, comment='#', data_start=0, converters=converters) # Convert spectral index values from strings to arrays. if 'SpectralIndex' in table.keys(): log.debug('Converting spectral indices...') specOld = table['SpectralIndex'].data.tolist() specVec = [] maskVec = [] maxLen = 0 for l in specOld: try: if type(l) is float or type(l) is int: maxLen = 1 else: specEntry = [float(f) for f in l.split(';')] if len(specEntry) > maxLen: maxLen = len(specEntry) except: pass log.debug('Maximum number of spectral-index terms in model: {0}'.format(maxLen)) for l in specOld: try: if type(l) is float or type(l) is int: specEntry = [float(l)] specMask = [False] else: specEntry = [float(f) for f in l.split(';')] specMask = [False] * len(specEntry) while len(specEntry) < maxLen: specEntry.append(0.0) specMask.append(True) specVec.append(specEntry) maskVec.append(specMask) except: specVec.append([0.0]*maxLen) maskVec.append([True]*maxLen) specCol = MaskedColumn(name='SpectralIndex', data=np.array(specVec, dtype=np.float)) specCol.mask = maskVec specIndx = table.keys().index('SpectralIndex') table.remove_column('SpectralIndex') table.add_column(specCol, index=specIndx) # Convert RA and Dec to Angle objects log.debug('Converting RA...') RARaw = table['Ra'].data.tolist() RACol = Column(name='Ra', data=RA2Angle(RARaw)) def raformat(val): return Angle(val, unit='degree').to_string(unit='hourangle', sep=':') RACol.format = raformat RAIndx = table.keys().index('Ra') table.remove_column('Ra') table.add_column(RACol, index=RAIndx) log.debug('Converting Dec...') DecRaw = table['Dec'].data.tolist() DecCol = Column(name='Dec', data=Dec2Angle(DecRaw)) def decformat(val): return Angle(val, unit='degree').to_string(unit='degree', sep='.') DecCol.format = decformat DecIndx = table.keys().index('Dec') table.remove_column('Dec') table.add_column(DecCol, index=DecIndx) def fluxformat(val): return '{0:0.3f}'.format(val) table.columns['I'].format = fluxformat # Set column units and default values for i, colName in enumerate(colNames): log.debug("Setting units for column '{0}' to {1}".format( colName, allowedColumnUnits[colName.lower()])) table.columns[colName].unit = allowedColumnUnits[colName.lower()] if hasattr(table.columns[colName], 'filled') and colDefaults[i] is not None: fillVal = colDefaults[i] if colName == 'SpectralIndex': while len(fillVal) < maxLen: fillVal.append(0.0) log.debug("Setting default value for column '{0}' to {1}". format(colName, fillVal)) table.columns[colName].fill_value = fillVal table.meta = metaDict return table
def merge_lightcurve_fits(options): """Pythom program to merge the lightcurve fit results into a sigle format""" import numpy import astropy import JLA_library as JLA from astropy.table import Table, MaskedColumn, vstack params = JLA.build_dictionary(options.config) # ---------------- JLA ------------------------ lightCurveFits = JLA.get_full_path(params['JLAlightCurveFits']) f=open(lightCurveFits) header=f.readlines() f.close() names=header[0].strip('#').split() # I imagine that the tables package in astropy could also be used to read the ascii input file SNeSpec = Table(numpy.genfromtxt(lightCurveFits, skip_header=1, dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8', names=names)) nSNeSpec=len(SNeSpec) print 'There are %d SNe from the spectrscopically confirmed sample' % (nSNeSpec) # Add an extra column to the table SNeSpec['source']=['JLA']*nSNeSpec # ---------------- Shuvo's sample ------------------------ # Photometrically identified SNe in Shuvo's sample, if the parameter exists if params['photLightCurveFits']!='None': lightCurveFits = JLA.get_full_path(params['photLightCurveFits']) SNePhot=Table.read(lightCurveFits, format='fits') nSNePhot=len(SNePhot) print 'There are %d SNe from the photometric sample' % (nSNePhot) # Converting from Shuvo's names to thosed used by JLA conversion={'name':'name_adj', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':'col27', 'd3rdvar':'d3rdvar', 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':None, 'dec':None, 'biascor':None} # Add the uncertainty in the mass column SNePhot['d3rdvar']=(SNePhot['col29']+SNePhot['col28'])/2. - SNePhot['col27'] # Remove columns that are not listed in conversion for colname in SNePhot.colnames: if colname not in conversion.values(): SNePhot.remove_column(colname) for key in conversion.keys(): # Rename the column if it does not already exist if conversion[key]!=None and conversion[key]!=key: SNePhot.rename_column(conversion[key], key) elif conversion[key]==None: # Create it, mask it, and fill all values SNePhot[key]=MaskedColumn(numpy.zeros(nSNePhot), numpy.ones(nSNePhot,bool)) SNePhot[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line SNePhot[key]=-99.9 else: # Do nothing if the column already exists pass # Add the source column SNePhot['source']="Phot_Uddin" # ---------------------- CfA4 ---------------------------------- if params['CfA4LightCurveFits']!='None': lightCurveFits = JLA.get_full_path(params['CfA4LightCurveFits']) f=open(lightCurveFits) header=f.readlines() f.close() names=header[0].strip('#').split(',') SNeCfA4=Table(numpy.genfromtxt(lightCurveFits, skip_header=1, dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8', names=names,delimiter=',')) nSNeCfA4=len(SNeCfA4) conversion={'name':'name', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':None, 'd3rdvar':None, 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':None, 'dec':None, 'biascor':None} # Remove columns that are not listed in conversion for colname in SNeCfA4.colnames: if colname not in conversion.values(): SNeCfA4.remove_column(colname) for key in conversion.keys(): # Rename the column if it does not already exist if conversion[key]!=None and conversion[key]!=key: SNeCfA4.rename_column(conversion[key], key) elif conversion[key]==None: # Create it, mask it, and fill all values SNeCfA4[key]=MaskedColumn(numpy.zeros(nSNeCfA4), numpy.ones(nSNeCfA4,bool)) SNeCfA4[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line SNeCfA4[key]=-99.9 else: # Do nothing if the column already exists pass # Add the source column SNeCfA4['source']="CfA4" try: SNe=vstack([SNeSpec,SNePhot,SNeCfA4]) except: SNe=SNeSpec # Write out the result as a FITS table date = JLA.get_date() SNe.write('%s_%s.fits' % (options.output, date), format='fits') return