def jarnoTransf(x_p, y_p, x, y, ra, dec, log): """ Source: https://elonen.iki.fi/code/misc-notes/affine-fit/ Slightly modified to work better with (ra, dec) and simplified using np.linalg.solve() instead of the original Gauss-Jordan method. """ from_pt = np.array([x, y]).T to_pt = np.array([ra, dec]).T trn = Affine_Fit(from_pt, to_pt) print("\nTransformation is:") print(trn.To_Str()) print("\nTransformation is:", file=log) print(trn.To_Str(), file=log) ra_coeffs, dec_coeffs = trn.coeffs() # Transform cross-matched (x, y), for plotting ra_x, dec_y = trn.Transform(np.array([x, y])) # Fitting error as mean of distances. err = np.sqrt((ra - ra_x)**2 + (dec - dec_y)**2).mean() print("Fitting error = {:.9f}\n".format(err)) print("Fitting error = {:.9f}".format(err), file=log) ra_p, dec_p = trn.Transform(np.array([x_p, y_p])) ra_p = MaskedColumn(ra_p, name='ra') dec_p = MaskedColumn(dec_p, name='dec') return ra_p, dec_p, ra_x, dec_y, ra_coeffs, dec_coeffs, err
def merge_tables() -> Table: """Merges the tables.""" data = join(load_gaia_tyc(), load_tyc_spec(), keys=['TYC'], join_type='left') data = join( data, load_ascc(), keys=['TYC'], join_type='left', table_names=('gaia', 'ascc'), metadata_conflicts='silent', ) data['SpType'] = MaskedColumn(data['SpType_gaia'].filled(data['SpType_ascc'].filled(''))) data['SpType'].mask = data['SpType'] == '' data.remove_columns(['SpType_gaia', 'SpType_ascc']) data = join(data, load_tyc_hd(), keys=['TYC'], join_type='left', metadata_conflicts='silent') data = join( data, load_tyc_teff(), keys=['TYC'], join_type='left', table_names=('gaia', 'tycteff'), ) data['teff_val'] = MaskedColumn( data['teff_val_gaia'].filled(data['teff_val_tycteff'].filled(np.nan)), ) data['teff_val'].mask = np.isnan(data['teff_val']) data.remove_columns(['teff_val_tycteff', 'teff_val_gaia']) data = join(data, load_sao(), keys=['TYC'], join_type='left') return data
def px2Eq(x_p, y_p, cr_m_data): """ Transform pixels to (ra, dec) using the correlated astrometry.net file. """ ra, dec = cr_m_data['field_ra'], cr_m_data['field_dec'] x, y = cr_m_data['field_x'], cr_m_data['field_y'] def func(X, a, b, c, d, e, f): x, y = X return a + b * x + c * y + d * x * y + e * x**2 + f * y**2 # Find transformation from (x,y) to (ra,dec) p0 = (100., 0., 0., 0., 0., 0.) x2ra = curve_fit(func, (x, y), ra, p0)[0] y2de = curve_fit(func, (x, y), dec, p0)[0] print(("ra = {:.7f} + {:.7f} * x + {:.7f} * y + {:.7f} * xy +" " {:.7f} * x^2 + {:.7f} * y^2 +").format(*x2ra)) print(("dec = {:.7f} + {:.7f} * x + {:.7f} * y + {:.7f} * xy + " "{:.7f} * x^2 + {:.7f} * y^2 +").format(*y2de)) # Transform pixel (x,y) coordinates into the (ra,dec) system ra_p = func((x_p, y_p), *x2ra) dec_p = func((x_p, y_p), *y2de) ra_p = MaskedColumn(ra_p, name='ra') dec_p = MaskedColumn(dec_p, name='dec') return ra_p, dec_p
def nudgeTransf(x_p, y_p, cr_m_data): """ This works but gives *very* poor results (July 2019) """ ra, dec = cr_m_data['field_ra'], cr_m_data['field_dec'] x, y = cr_m_data['field_x'], cr_m_data['field_y'] import nudged dom = list(map(list, zip(x, y))) ran = list(map(list, zip(-ra, dec))) trans = nudged.estimate(dom, ran) cc = list(map(list, list(np.array([x_p, y_p]).T))) ra_p, dec_p = np.array(trans.transform(cc)).T ra_p = -ra_p print(trans.get_matrix()) print(trans.get_rotation()) print(trans.get_scale()) print(trans.get_translation()) # plt.subplot(221);plt.title("dom");plt.scatter(x, y) # plt.subplot(222);plt.title("ran");plt.scatter(ra, dec) # plt.subplot(223);plt.title("(x, y)");plt.scatter(x_p, y_p) # plt.subplot(224);plt.title("(x_t, y_t)");plt.scatter(ra_p, dec_p) # plt.show() ra_p = MaskedColumn(ra_p, name='ra') dec_p = MaskedColumn(dec_p, name='dec') return ra_p, dec_p
def test_coalesce_basic(): col1 = MaskedColumn(data=[1, 0], mask=[False, True]) col2 = MaskedColumn(data=[0, 2], mask=[True, False]) for result in [coalesce((col1, col2)), coalesce((col2, col1))]: assert result[0] == 1 assert result[1] == 2 assert result.mask.any() == False
def _map_ids(tbl: Table) -> dict[int, int]: gaia_ids = np.zeros(len(tbl), dtype=np.int64) cel_ids = np.zeros(len(tbl), dtype=np.uint32) id_idx = {} for i, ids in enumerate(tbl['ids']): for ident in ids.split('|'): if ident.startswith('HIP'): hip = int(ident[3:].strip()) assert hip not in id_idx id_idx[hip] = i cel_ids[i] = hip elif ident.startswith('TYC'): tycs = [int(t) for t in ident[3:].strip().split('-')] tyc = tycs[0] + tycs[1] * 10000 + tycs[2] * 1000000000 assert tyc not in id_idx id_idx[tyc] = i if cel_ids[i] == 0: cel_ids[i] = tyc elif ident.startswith('Gaia DR2'): gaia_ids[i] = int(ident[8:].strip()) tbl.add_columns([ MaskedColumn(data=gaia_ids, name='gaia'), MaskedColumn(data=cel_ids, name='hip'), ]) return id_idx
def _fromat_column_entry(self, t): for n in self._numeric_fields: if n in t.colnames: try: if None in t[n].data: t[n] = MaskedColumn(t[n].data, name=n, dtype=np.float, mask=t[n].data == None) else: t[n] = MaskedColumn(t[n].data, name=n, dtype=np.float) t[n].format = '%e' except: for ID, v in enumerate(t[n].data): try: c = ast.literal_eval(t[n].data[ID]) if type(c) == int: t[n].data[ID] = '%d' % c else: t[n].data[ID] = '%e' % c except: pass
def test_mask_copy(): """Test that the mask is copied when copying a table (issue #7362).""" c = MaskedColumn([1, 2], mask=[False, True]) c2 = MaskedColumn(c, copy=True) c2.mask[0] = True assert np.all(c.mask == [False, True]) assert np.all(c2.mask == [True, True])
def setup_method(self, method): self.a = MaskedColumn(name='a', data=[1, 2, 3], fill_value=1) self.b = MaskedColumn(name='b', data=[4, 5, 6], mask=True) self.c = MaskedColumn(name='c', data=[7, 8, 9], mask=False) self.d_mask = np.array([False, True, False]) self.d = MaskedColumn(name='d', data=[7, 8, 7], mask=self.d_mask) self.t = Table([self.a, self.b], masked=True) self.ca = Column(name='ca', data=[1, 2, 3])
def merge_all() -> Table: """Merges the HIP and TYC data.""" hip_data = process_hip() # extract the non-Gaia sources to make the merging easier non_gaia = hip_data[hip_data['source_id'].mask] # merge object data for objects in both catalogues hip_data = join(hip_data[np.logical_not(hip_data['source_id'].mask)], process_tyc(), keys=['source_id'], table_names=['hip', 'tyc'], join_type='outer') # Mask blank spectral type and component identifiers for str_col in (c for c in hip_data.colnames if hip_data[c].dtype.kind == 'U'): hip_data[str_col].mask = np.logical_or(hip_data[str_col].mask, hip_data[str_col] == '') prefer_tyc = {'HD', 'SAO', 'Comp'} for base_col in (c[:-4] for c in hip_data.colnames if c.endswith('_hip')): hip_col = base_col + '_hip' tyc_col = base_col + '_tyc' hip_data.rename_column(hip_col, base_col) if isinstance(hip_data[base_col], MaskedColumn): mask = np.logical_and(hip_data[base_col].mask, hip_data[tyc_col].mask) if base_col in prefer_tyc: base_data = hip_data[tyc_col].filled(hip_data[base_col]) else: base_data = hip_data[base_col].filled(hip_data[tyc_col]) hip_data[base_col] = MaskedColumn(base_data, mask=mask) hip_data.remove_column(tyc_col) hip_data['HIP'] = hip_data['HIP'].filled(hip_data['TYC']) hip_data.remove_columns('TYC') # Add the non-Gaia stars back into the dataset hip_data = vstack([hip_data, non_gaia], join_type='outer', metadata_conflicts='silent') # Merge SAO, preferring the values from the SAO catalogue sao = load_sao() sao = sao[np.isin(sao['HD'], hip_data[np.logical_not(hip_data['HD'].mask)]['HD'])] hip_data['SAO'].mask = np.logical_or(hip_data['SAO'].mask, np.isin(hip_data['SAO'], sao['SAO'])) hd_sao = join(hip_data[np.logical_not(hip_data['HD'].mask)], sao, keys=['HD'], table_names=['xref', 'sao'], join_type='left') hd_sao.rename_column('SAO_xref', 'SAO') hd_sao['SAO'] = MaskedColumn(hd_sao['SAO_sao'].filled(hd_sao['SAO']), mask=np.logical_and(hd_sao['SAO'].mask, hd_sao['SAO_sao'].mask)) hd_sao.remove_column('SAO_sao') return vstack([hip_data[hip_data['HD'].mask], hd_sao], join_type='exact')
def test_add_masked_row_to_masked_table_mapping4(self): # When adding values to a masked table, if the mask is specified as a # dict, then keys in values should match keys in mask t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) with pytest.raises(ValueError) as exc: t.add_row({'b': 5}, mask={'a': True}) assert exc.value.args[0] == 'keys in mask should match keys in vals'
def test_add_masked_row_to_masked_table_mismatch(self): t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) with pytest.raises(TypeError) as exc: t.add_row([2, 5], mask={'a': 1, 'b': 0}) assert exc.value.args[0] == "Mismatch between type of vals and mask" with pytest.raises(TypeError) as exc: t.add_row({'b': 5, 'a': 2}, mask=[1, 0]) assert exc.value.args[0] == "Mismatch between type of vals and mask"
def setup_method(self, method): self.c1 = MaskedColumn(name='col1', data=[1, 2, 3], mask=[False, False, False]) self.c2 = MaskedColumn(name='col2', data=[4, 5, 6], mask=[True, False, False]) self.c3 = MaskedColumn(name='col3', data=[7, 8, 9], mask=[False, True, False])
def setup_method(self, method): self.a = MaskedColumn(name='a', data=[1, 2, 3], fill_value=1) self.b = MaskedColumn(name='b', data=[4, 5, 6], mask=True) self.c = MaskedColumn(name='c', data=[7, 8, 9], mask=False) self.d_mask = np.array([False, True, False]) self.d = MaskedColumn(name='d', data=[7, 8, 7], mask=self.d_mask) self.t = Table([self.a, self.b], masked=True) self.ca = Column(name='ca', data=[1, 2, 3]) self.sc = MaskedColumn(name='sc', data=[(1, 1.), (2, 2.), (3, 3.)], dtype='i8,f8', fill_value=(0, -1.))
def test_remove_masked_column(self): t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1, 2, 3], mask=[0, 1, 0])) t['a'].fill_value = 42 t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1])) t.remove_column('b') assert t.masked assert np.all(t['a'] == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert t['a'].fill_value == 42 assert t.colnames == ['a']
def test_add_masked_row_to_masked_table_mapping1(self): t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row({'b': 5, 'a': 2}, mask={'a': 1, 'b': 0}) t.add_row({'a': 3, 'b': 6}, mask={'b': 1, 'a': 0}) assert t.masked assert np.all(np.array(t['a']) == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert np.all(np.array(t['b']) == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_column_to_masked_table(self): t = Table(masked=True) assert t.masked t.add_column(MaskedColumn(name='a', data=[1, 2, 3], mask=[0, 1, 0])) assert t.masked t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1])) assert t.masked assert np.all(t['a'] == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert np.all(t['b'] == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_iterable(self): t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row([2, 5], mask=[1, 0]) t.add_row([3, 6], mask=[0, 1]) assert t.masked assert np.all(np.array(t['a']) == np.array([1, 2, 3])) assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert np.all(np.array(t['b']) == np.array([4, 5, 6])) assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping2(self): # When adding values to a masked table, if the mask is specified as a # dict, then values not specified will have mask values set to True t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row({'b': 5}, mask={'b': 0}) t.add_row({'a': 3}, mask={'a': 0}) assert t.masked assert t['a'][0] == 1 and t['a'][2] == 3 assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert t['b'][1] == 5 assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping3(self): # When adding values to a masked table, if mask is not passed to # add_row, then the mask should be set to False if values are present # and True if not. t = Table(masked=True) t.add_column(MaskedColumn(name='a', data=[1], mask=[0])) t.add_column(MaskedColumn(name='b', data=[4], mask=[1])) t.add_row({'b': 5}) t.add_row({'a': 3}) assert t.masked assert t['a'][0] == 1 and t['a'][2] == 3 assert np.all(t['a'].mask == np.array([0, 1, 0], bool)) assert t['b'][1] == 5 assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_masked_column_with_unit_in_qtable(): """Test that adding a MaskedColumn with a unit to QTable issues warning""" t = QTable() t['a'] = MaskedColumn([1, 2]) assert isinstance(t['a'], MaskedColumn) t['b'] = MaskedColumn([1, 2], unit=u.m) assert isinstance(t['b'], u.Quantity) with pytest.warns(UserWarning, match="dropping mask in Quantity column 'c'") as w: t['c'] = MaskedColumn([1, 2], unit=u.m, mask=[True, False]) assert len(w) == 1 assert isinstance(t['b'], u.Quantity)
def test_masked_column_with_unit_in_qtable(): """Test that adding a MaskedColumn with a unit to QTable creates a MaskedQuantity.""" MaskedQuantity = Masked(u.Quantity) t = QTable() t['a'] = MaskedColumn([1, 2]) assert isinstance(t['a'], MaskedColumn) t['b'] = MaskedColumn([1, 2], unit=u.m) assert isinstance(t['b'], MaskedQuantity) assert np.all(t['b'].mask == False) # noqa t['c'] = MaskedColumn([1, 2], unit=u.m, mask=[True, False]) assert isinstance(t['c'], MaskedQuantity) assert np.all(t['c'].mask == [True, False])
def test_coercing_fill_value_type(): """ Test that masked column fill_value is coerced into the correct column type. """ # This is the original example posted on the astropy@scipy mailing list t = Table({'a': ['1']}, masked=True) t['a'].set_fill_value('0') t2 = Table(t, names=['a'], dtype=[np.int32]) assert isinstance(t2['a'].fill_value, np.int32) # Unit test the same thing. c = MaskedColumn(['1']) c.set_fill_value('0') c2 = MaskedColumn(c, dtype=np.int32) assert isinstance(c2.fill_value, np.int32)
def test_parsing_columns( columns_file ): """ Test parsing of COLUMN elements <COLUMN dmtype ref > <OPTIONMAPPING> <OPTION>"string"</OPTION> <ENUMLITERAL>"vodml-id"</ENUMLITERAL> or <SEMANTICCONCEPT> </OPTIONMAPPING> </COLUMN> The parser pushes the Table rows into value arrays """ luminosity = columns_file.find_instances( LuminosityMeasurement )[0] # COLUMN with dmtype, value (ivoa:string) assert len(luminosity.description) == 2 assert luminosity.description[0] == "some descriptive text" assert luminosity.description[1] == "more descriptive text" # COLUMN with dmtype, value, unit (ivoa:RealQuantity) # - uses astropy Quantity # o luminosity.value = Quantity with value array length 2 + unit # o luminosity.value[0] = Quantity with first value + unit assert len(luminosity.value) == 2 expected_lum = numpy.array([15.718, 14.847], dtype='float32') * u.Unit('mag') numpy.testing.assert_array_equal( expected_lum, luminosity.value ) # COLUMN with OPTIONMAPPING # MCD NOTE: TODO - enumeration type == 'string'.. OPTIONMAPPING not parsed/interpreted assert len(luminosity.type) == 2 assert luminosity.type[0] == "magnitude" assert luminosity.type[1] == "magnitude" #assert (luminosity.optionmapping) == 2 # Check multiplicity handling # o values dimension 1 == 'row' elem = columns_file.find_instances( MultiObj )[0] assert len(elem.a) == 2 assert isinstance(elem.a, MaskedColumn) assert elem.a.shape == (2,) numpy.testing.assert_array_equal( elem.a, MaskedColumn([100.0, 100.1], dtype='float32')) # MCD NOTE: TODO - check this, I do not think 'b' should be a List of MaskedColumn assert len(elem.b) == 1 # b is a list of length=1 assert isinstance(elem.b[0], MaskedColumn) # of MaskedColumn assert elem.b[0].shape == (2,2) # 2 rows x 2 values each numpy.testing.assert_array_equal( elem.b[0], MaskedColumn([[200.0, 201.0],[200.1,201.1]], dtype='float32'))
def test_set_get_fill_value_for_str_column(self): c = MaskedColumn(name='c', data=['xxxx', 'yyyy'], mask=[True, False]) # assert np.all(c.filled() == ['N/A', 'yyyy']) c.fill_value = 'ABCDEF' assert c.fill_value == 'ABCD' # string truncated to dtype length assert np.all(c.filled() == ['ABCD', 'yyyy']) assert np.all(c.filled('XY') == ['XY', 'yyyy'])
def test_astropy(self): """test astropy table + test null_value + test masked column """ tablemaker = CDSTablesMaker() ntab = Table([(1, 2, None, 4, 999), (4.0, 1115.0, np.NaN, None, 999), (1.1, 2., 999, 12.3, 12.3), (-1.001, 2., 0, -99.12, np.NaN)], names=['a', 'b', 'col3', 'col4']) ntab["col4"] = MaskedColumn(ntab["col4"], mask=[(val > 0) for val in ntab["col4"]]) tablename = ReadMeCase.__result_file("astropy") table = tablemaker.addTable(ntab, name=tablename, description="test astropy table", nullvalue=999) col = table.get_column("col4") col.set_null_value(-1.001) tablemaker.writeCDSTables() self.assertTrue(self.__test_file(tablename, 3), "astropy file") readme = ReadMeCase.__readme_file("astropy") with open(readme, "w") as fd: tablemaker.makeReadMe(out=fd) self.assertTrue(self.__test_file(readme, 5), "Readme for astropy") sys.stderr.write("generate {0}/{1} {0}/{2}\n".format( RESULT_DIR, tablename, readme))
def _gemini_json_to_table(json): """ takes a JSON object as returned from the Gemini archive webserver and turns it into an `~astropy.table.Table` Parameters ---------- json : dict A JSON object from the Gemini archive webserver Returns ------- response : `~astropy.table.Table` """ data_table = Table(masked=True) for key in __keys__: col_data = np.array([obj.get(key, None) for obj in json]) atype = str col_mask = np.equal(col_data, None) data_table.add_column(MaskedColumn(col_data.astype(atype), name=key, mask=col_mask)) return data_table
def setup_method(self, method): mask = [True, False, False] self.meta = {'a': 1, 'b': [2, 3]} self.a = MaskedColumn(name='a', data=[1, 2, 3], fill_value=10, mask=mask, meta={'a': 1}) self.b = MaskedColumn(name='b', data=[4.0, 5.0, 6.0], fill_value=10.0, mask=mask) self.c = MaskedColumn(name='c', data=['7', '8', '9'], fill_value='1', mask=mask)
def merge_gaia(tbl: Table) -> Table: """Merges Gaia data for non-Celestia stars.""" with gzip.open(GAIA_PATH, 'rb') as f: gaia = votable.parse_single_table(f).to_table() bp_rp = gaia['bp_rp'].filled(0) bp_rp2 = bp_rp * bp_rp gaia.add_column( MaskedColumn(data=gaia['phot_g_mean_mag'].filled(np.nan) + 0.01760 + bp_rp * 0.006860 + bp_rp2 * 0.1732, name='flux', mask=gaia['phot_g_mean_mag'].mask)) gaia.remove_columns(['phot_g_mean_mag', 'bp_rp']) gaia.rename_column('source_id', 'gaia') gaia.rename_column('r_est', 'dist') has_gaia = tbl[np.logical_not(tbl['gaia'].mask)] merged = join(has_gaia, gaia, keys=['gaia'], join_type='left', table_names=['cel', 'gaia']) merged['ra'] = np.where(merged['ra_gaia'].mask, merged['ra_cel'], merged['ra_gaia']) merged['dec'] = np.where(merged['dec_gaia'].mask, merged['dec_cel'], merged['dec_gaia']) merged.add_columns([ MaskedColumn(data=np.where(merged['dist_gaia'].mask, merged['dist_cel'], merged['dist_gaia']), name='dist', mask=np.logical_and(merged['dist_gaia'].mask, merged['dist_cel'].mask)), MaskedColumn(data=np.where(merged['flux_cel'].mask, merged['flux_gaia'], merged['flux_cel']), name='flux', mask=np.logical_and(merged['flux_cel'].mask, merged['flux_gaia'].mask)) ]) merged.remove_columns([ 'ra_cel', 'ra_gaia', 'dec_cel', 'dec_gaia', 'dist_cel', 'dist_gaia', 'flux_cel', 'flux_gaia' ]) return vstack([tbl[tbl['gaia'].mask], merged], join_type='exact')
def test_set_get_fill_value_for_structured_column(self): assert self.sc.fill_value == np.array((0, -1.), self.sc.dtype) sc = self.sc.copy() assert sc.fill_value.item() == (0, -1.) sc.fill_value = (-1, np.inf) assert sc.fill_value == np.array((-1, np.inf), self.sc.dtype) sc2 = MaskedColumn(sc, fill_value=(-2, -np.inf)) assert sc2.fill_value == np.array((-2, -np.inf), sc2.dtype)