def test_set_get_fill_value_for_str_column(self):
c = MaskedColumn(name='c', data=['xxxx', 'yyyy'], mask=[True, False])
# assert np.all(c.filled() == ['N/A', 'yyyy'])
c.fill_value = 'ABCDEF'
assert c.fill_value == 'ABCD' # string truncated to dtype length
assert np.all(c.filled() == ['ABCD', 'yyyy'])
assert np.all(c.filled('XY') == ['XY', 'yyyy'])
def test_mask_copy():
"""Test that the mask is copied when copying a table (issue #7362)."""
c = MaskedColumn([1, 2], mask=[False, True])
c2 = MaskedColumn(c, copy=True)
c2.mask[0] = True
assert np.all(c.mask == [False, True])
assert np.all(c2.mask == [True, True])
def test_remove_masked_column(self):
t = Table(masked=True)
t.add_column(MaskedColumn(name='a', data=[1, 2, 3], mask=[0, 1, 0]))
t['a'].fill_value = 42
t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1]))
t.remove_column('b')
assert t.masked
assert np.all(t['a'] == np.array([1, 2, 3]))
assert np.all(t['a'].mask == np.array([0, 1, 0], bool))
assert t['a'].fill_value == 42
assert t.colnames == ['a']
def test_add_masked_row_to_masked_table_iterable(self):
t = Table(masked=True)
t.add_column(MaskedColumn(name='a', data=[1], mask=[0]))
t.add_column(MaskedColumn(name='b', data=[4], mask=[1]))
t.add_row([2, 5], mask=[1, 0])
t.add_row([3, 6], mask=[0, 1])
assert t.masked
assert np.all(np.array(t['a']) == np.array([1, 2, 3]))
assert np.all(t['a'].mask == np.array([0, 1, 0], bool))
assert np.all(np.array(t['b']) == np.array([4, 5, 6]))
assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping1(self):
t = Table(masked=True)
t.add_column(MaskedColumn(name='a', data=[1], mask=[0]))
t.add_column(MaskedColumn(name='b', data=[4], mask=[1]))
t.add_row({'b': 5, 'a': 2}, mask={'a': 1, 'b': 0})
t.add_row({'a': 3, 'b': 6}, mask={'b': 1, 'a': 0})
assert t.masked
assert np.all(np.array(t['a']) == np.array([1, 2, 3]))
assert np.all(t['a'].mask == np.array([0, 1, 0], bool))
assert np.all(np.array(t['b']) == np.array([4, 5, 6]))
assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def setup_method(self, method):
self.a = MaskedColumn(name='a', data=[1, 2, 3], fill_value=1)
self.b = MaskedColumn(name='b', data=[4, 5, 6], mask=True)
self.c = MaskedColumn(name='c', data=[7, 8, 9], mask=False)
self.d_mask = np.array([False, True, False])
self.d = MaskedColumn(name='d', data=[7, 8, 7], mask=self.d_mask)
self.t = Table([self.a, self.b], masked=True)
self.ca = Column(name='ca', data=[1, 2, 3])
self.sc = MaskedColumn(name='sc',
data=[(1, 1.), (2, 2.), (3, 3.)],
dtype='i8,f8',
fill_value=(0, -1.))
def test_add_masked_column_to_masked_table(self):
t = Table(masked=True)
assert t.masked
t.add_column(MaskedColumn(name='a', data=[1, 2, 3], mask=[0, 1, 0]))
assert t.masked
t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1]))
assert t.masked
assert isinstance(t['a'], MaskedColumn)
assert isinstance(t['b'], MaskedColumn)
assert np.all(t['a'] == np.array([1, 2, 3]))
assert np.all(t['a'].mask == np.array([0, 1, 0], bool))
assert np.all(t['b'] == np.array([4, 5, 6]))
assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping2(self):
# When adding values to a masked table, if the mask is specified as a
# dict, then values not specified will have mask values set to True
t = Table(masked=True)
t.add_column(MaskedColumn(name='a', data=[1], mask=[0]))
t.add_column(MaskedColumn(name='b', data=[4], mask=[1]))
t.add_row({'b': 5}, mask={'b': 0})
t.add_row({'a': 3}, mask={'a': 0})
assert t.masked
assert t['a'][0] == 1 and t['a'][2] == 3
assert np.all(t['a'].mask == np.array([0, 1, 0], bool))
assert t['b'][1] == 5
assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_add_masked_row_to_masked_table_mapping3(self):
# When adding values to a masked table, if mask is not passed to
# add_row, then the mask should be set to False if values are present
# and True if not.
t = Table(masked=True)
t.add_column(MaskedColumn(name='a', data=[1], mask=[0]))
t.add_column(MaskedColumn(name='b', data=[4], mask=[1]))
t.add_row({'b': 5})
t.add_row({'a': 3})
assert t.masked
assert t['a'][0] == 1 and t['a'][2] == 3
assert np.all(t['a'].mask == np.array([0, 1, 0], bool))
assert t['b'][1] == 5
assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def test_masked_column_with_unit_in_qtable():
"""Test that adding a MaskedColumn with a unit to QTable issues warning"""
t = QTable()
t['a'] = MaskedColumn([1, 2])
assert isinstance(t['a'], MaskedColumn)
t['b'] = MaskedColumn([1, 2], unit=u.m)
assert isinstance(t['b'], u.Quantity)
with pytest.warns(UserWarning,
match="dropping mask in Quantity column 'c'") as w:
t['c'] = MaskedColumn([1, 2], unit=u.m, mask=[True, False])
assert len(w) == 1
assert isinstance(t['b'], u.Quantity)
def test_coercing_fill_value_type():
"""
Test that masked column fill_value is coerced into the correct column type.
"""
# This is the original example posted on the astropy@scipy mailing list
t = Table({'a': ['1']}, masked=True)
t['a'].set_fill_value('0')
t2 = Table(t, names=['a'], dtype=[np.int32])
assert isinstance(t2['a'].fill_value, np.int32)
# Unit test the same thing.
c = MaskedColumn(['1'])
c.set_fill_value('0')
c2 = MaskedColumn(c, dtype=np.int32)
assert isinstance(c2.fill_value, np.int32)
def test_pickle_masked_column(protocol):
c = MaskedColumn(data=[1, 2], name='a', format='%05d', description='col a', unit='cm',
meta={'a': 1})
c.mask[1] = True
c.fill_value = -99
cs = pickle.dumps(c)
cp = pickle.loads(cs)
assert np.all(cp._data == c._data)
assert np.all(cp.mask == c.mask)
assert cp.attrs_equal(c)
assert cp.fill_value == -99
assert cp._parent_table is None
assert repr(c) == repr(cp)
def test_coercing_fill_value_type():
"""
Test that masked column fill_value is coerced into the correct column type.
"""
# This is the original example posted on the astropy@scipy mailing list
t = Table({'a': ['1']}, masked=True)
t['a'].set_fill_value('0')
t2 = Table(t, names=['a'], dtype=[np.int32])
assert isinstance(t2['a'].fill_value, np.int32)
# Unit test the same thing.
c = MaskedColumn(['1'])
c.set_fill_value('0')
c2 = MaskedColumn(c, dtype=np.int32)
assert isinstance(c2.fill_value, np.int32)
def test_masked_column_with_unit_in_qtable():
"""Test that adding a MaskedColumn with a unit to QTable creates a MaskedQuantity."""
MaskedQuantity = Masked(u.Quantity)
t = QTable()
t['a'] = MaskedColumn([1, 2])
assert isinstance(t['a'], MaskedColumn)
t['b'] = MaskedColumn([1, 2], unit=u.m)
assert isinstance(t['b'], MaskedQuantity)
assert np.all(t['b'].mask == False) # noqa
t['c'] = MaskedColumn([1, 2], unit=u.m, mask=[True, False])
assert isinstance(t['c'], MaskedQuantity)
assert np.all(t['c'].mask == [True, False])
def test_parsing_columns( columns_file ):
"""
Test parsing of COLUMN elements
<COLUMN dmtype ref >
<OPTIONMAPPING>
<OPTION>"string"</OPTION>
<ENUMLITERAL>"vodml-id"</ENUMLITERAL> or <SEMANTICCONCEPT>
</OPTIONMAPPING>
</COLUMN>
The parser pushes the Table rows into value arrays
"""
luminosity = columns_file.find_instances( LuminosityMeasurement )[0]
# COLUMN with dmtype, value (ivoa:string)
assert len(luminosity.description) == 2
assert luminosity.description[0] == "some descriptive text"
assert luminosity.description[1] == "more descriptive text"
# COLUMN with dmtype, value, unit (ivoa:RealQuantity)
# - uses astropy Quantity
# o luminosity.value = Quantity with value array length 2 + unit
# o luminosity.value[0] = Quantity with first value + unit
assert len(luminosity.value) == 2
expected_lum = numpy.array([15.718, 14.847], dtype='float32') * u.Unit('mag')
numpy.testing.assert_array_equal( expected_lum, luminosity.value )
# COLUMN with OPTIONMAPPING
# MCD NOTE: TODO - enumeration type == 'string'.. OPTIONMAPPING not parsed/interpreted
assert len(luminosity.type) == 2
assert luminosity.type[0] == "magnitude"
assert luminosity.type[1] == "magnitude"
#assert (luminosity.optionmapping) == 2
# Check multiplicity handling
# o values dimension 1 == 'row'
elem = columns_file.find_instances( MultiObj )[0]
assert len(elem.a) == 2
assert isinstance(elem.a, MaskedColumn)
assert elem.a.shape == (2,)
numpy.testing.assert_array_equal( elem.a, MaskedColumn([100.0, 100.1], dtype='float32'))
# MCD NOTE: TODO - check this, I do not think 'b' should be a List of MaskedColumn
assert len(elem.b) == 1 # b is a list of length=1
assert isinstance(elem.b[0], MaskedColumn) # of MaskedColumn
assert elem.b[0].shape == (2,2) # 2 rows x 2 values each
numpy.testing.assert_array_equal( elem.b[0], MaskedColumn([[200.0, 201.0],[200.1,201.1]], dtype='float32'))
def setup_method(self, method):
mask = [True, False, False]
self.meta = {'a': 1, 'b': [2, 3]}
a = self.a = MaskedColumn(name='a',
data=[1, 2, 3],
fill_value=10,
mask=mask,
meta={'a': 1})
b = self.b = MaskedColumn(name='b',
data=[4.0, 5.0, 6.0],
fill_value=10.0,
mask=mask)
c = self.c = MaskedColumn(name='c',
data=['7', '8', '9'],
fill_value='1',
mask=mask)
def test_astropy(self):
"""test astropy table
+ test null_value
+ test masked column
"""
tablemaker = CDSTablesMaker()
ntab = Table([(1, 2, None, 4, 999), (4.0, 1115.0, np.NaN, None, 999),
(1.1, 2., 999, 12.3, 12.3),
(-1.001, 2., 0, -99.12, np.NaN)],
names=['a', 'b', 'col3', 'col4'])
ntab["col4"] = MaskedColumn(ntab["col4"],
mask=[(val > 0) for val in ntab["col4"]])
tablename = ReadMeCase.__result_file("astropy")
table = tablemaker.addTable(ntab,
name=tablename,
description="test astropy table",
nullvalue=999)
col = table.get_column("col4")
col.set_null_value(-1.001)
tablemaker.writeCDSTables()
self.assertTrue(self.__test_file(tablename, 3), "astropy file")
readme = ReadMeCase.__readme_file("astropy")
with open(readme, "w") as fd:
tablemaker.makeReadMe(out=fd)
self.assertTrue(self.__test_file(readme, 5), "Readme for astropy")
sys.stderr.write("generate {0}/{1} {0}/{2}\n".format(
RESULT_DIR, tablename, readme))
def _gemini_json_to_table(json):
"""
takes a JSON object as returned from the Gemini archive webserver and turns it into an `~astropy.table.Table`
Parameters
----------
json : dict
A JSON object from the Gemini archive webserver
Returns
-------
response : `~astropy.table.Table`
"""
data_table = Table(masked=True)
for key in __keys__:
col_data = np.array([obj.get(key, None) for obj in json])
atype = str
col_mask = np.equal(col_data, None)
data_table.add_column(MaskedColumn(col_data.astype(atype), name=key, mask=col_mask))
return data_table
def merge_gaia(tbl: Table) -> Table:
"""Merges Gaia data for non-Celestia stars."""
with gzip.open(GAIA_PATH, 'rb') as f:
gaia = votable.parse_single_table(f).to_table()
bp_rp = gaia['bp_rp'].filled(0)
bp_rp2 = bp_rp * bp_rp
gaia.add_column(
MaskedColumn(data=gaia['phot_g_mean_mag'].filled(np.nan) + 0.01760 +
bp_rp * 0.006860 + bp_rp2 * 0.1732,
name='flux',
mask=gaia['phot_g_mean_mag'].mask))
gaia.remove_columns(['phot_g_mean_mag', 'bp_rp'])
gaia.rename_column('source_id', 'gaia')
gaia.rename_column('r_est', 'dist')
has_gaia = tbl[np.logical_not(tbl['gaia'].mask)]
merged = join(has_gaia,
gaia,
keys=['gaia'],
join_type='left',
table_names=['cel', 'gaia'])
merged['ra'] = np.where(merged['ra_gaia'].mask, merged['ra_cel'],
merged['ra_gaia'])
merged['dec'] = np.where(merged['dec_gaia'].mask, merged['dec_cel'],
merged['dec_gaia'])
merged.add_columns([
MaskedColumn(data=np.where(merged['dist_gaia'].mask,
merged['dist_cel'], merged['dist_gaia']),
name='dist',
mask=np.logical_and(merged['dist_gaia'].mask,
merged['dist_cel'].mask)),
MaskedColumn(data=np.where(merged['flux_cel'].mask,
merged['flux_gaia'], merged['flux_cel']),
name='flux',
mask=np.logical_and(merged['flux_cel'].mask,
merged['flux_gaia'].mask))
])
merged.remove_columns([
'ra_cel', 'ra_gaia', 'dec_cel', 'dec_gaia', 'dist_cel', 'dist_gaia',
'flux_cel', 'flux_gaia'
])
return vstack([tbl[tbl['gaia'].mask], merged], join_type='exact')
def test_set_get_fill_value_for_structured_column(self):
assert self.sc.fill_value == np.array((0, -1.), self.sc.dtype)
sc = self.sc.copy()
assert sc.fill_value.item() == (0, -1.)
sc.fill_value = (-1, np.inf)
assert sc.fill_value == np.array((-1, np.inf), self.sc.dtype)
sc2 = MaskedColumn(sc, fill_value=(-2, -np.inf))
assert sc2.fill_value == np.array((-2, -np.inf), sc2.dtype)
def add_distance(self, verbose = True):
"""
Add "distance" column to the catalogue, where distance = 1000./parallaxes.
"""
ncol = MaskedColumn(data = 1./self.cat['parallax'] * 1000, name = 'distance', unit = u.parsec, format = '4.1F')
self.cat.add_column(ncol)
if verbose:
print('Adding new column to Gaia DR3 dataset: Distance')
def _fromat_column_entry(self, t):
for n in self._numeric_fields:
if n in t.colnames:
try:
if None in t[n].data:
t[n] = MaskedColumn(t[n].data,
name=n,
dtype=np.float,
mask=t[n].data == None)
else:
t[n] = MaskedColumn(t[n].data, name=n, dtype=np.float)
t[n].format = '%e'
except:
for ID, v in enumerate(t[n].data):
try:
c = ast.literal_eval(t[n].data[ID])
if type(c) == int:
t[n].data[ID] = '%d' % c
else:
t[n].data[ID] = '%e' % c
except:
pass
def query(coords,
catalog,
cols,
radius=6 * u.arcsec,
fill_val=-99.99,
full=False):
results = Vizier.query_region(coords, catalog=catalog, radius=radius)
if len(results) == 0:
return None
if full:
return results
results = results[0]
# if dict, remap colnames
if isinstance(cols, dict):
for k, v in cols.iteritems():
results.rename_column(k, v)
names = cols.values()
else:
names = cols
# make new columns one-to-one with coords
newtable = Table(masked=True)
for col in names:
oldcol = results[col]
newcol = MaskedColumn(data=np.zeros(len(coords), dtype=oldcol.dtype),
unit=oldcol.unit,
name=col,
mask=np.ones(len(coords), dtype=bool),
fill_value=fill_val)
# copy data from results
for row in results:
if not row[col]:
continue
# _q IS 1-BASED INDEXING?!
newcol[row['_q'] - 1] = row[col]
newcol.mask[row['_q'] - 1] = False
newtable.add_column(newcol)
return newtable
def _json_to_table(json_obj, col_config=None):
"""
Takes a JSON object as returned from a Mashup request and turns it into an `~astropy.table.Table`.
Parameters
----------
json_obj : dict
A Mashup response JSON object (python dictionary)
col_config : dict, optional
Dictionary that defines column properties, e.g. default value.
Returns
-------
response : `~astropy.table.Table`
"""
data_table = Table(masked=True)
if not all(x in json_obj.keys() for x in ['fields', 'data']):
raise KeyError("Missing required key(s) 'data' and/or 'fields.'")
for col, atype in [(x['name'], x['type']) for x in json_obj['fields']]:
# Removing "_selected_" column
if col == "_selected_":
continue
# reading the colum config if given
ignore_value = None
if col_config:
col_props = col_config.get(col, {})
ignore_value = col_props.get("ignoreValue", None)
# regularlizing the type
reg_type = utils.parse_type(atype)
atype = reg_type[1]
ignore_value = reg_type[2] if (ignore_value is None) else ignore_value
# Make the column list (don't assign final type yet or there will be errors)
col_data = np.array(
[x.get(col, ignore_value) for x in json_obj['data']], dtype=object)
if ignore_value is not None:
col_data[np.where(np.equal(col_data, None))] = ignore_value
# no consistant way to make the mask because np.equal fails on ''
# and array == value fails with None
if atype == 'str':
col_mask = (col_data == ignore_value)
else:
col_mask = np.equal(col_data, ignore_value)
# add the column
data_table.add_column(
MaskedColumn(col_data.astype(atype), name=col, mask=col_mask))
return data_table
def test_setting_from_masked_column():
"""Test issue in #2997"""
mask_b = np.array([True, True, False, False])
for select in (mask_b, slice(0, 2)):
t = Table(masked=True)
t['a'] = Column([1, 2, 3, 4])
t['b'] = MaskedColumn([11, 22, 33, 44], mask=mask_b)
t['c'] = MaskedColumn([111, 222, 333, 444], mask=[True, False, True, False])
t['b'][select] = t['c'][select]
assert t['b'][1] == t[1]['b']
assert t['b'][0] is np.ma.masked # Original state since t['c'][0] is masked
assert t['b'][1] == 222 # New from t['c'] since t['c'][1] is unmasked
assert t['b'][2] == 33
assert t['b'][3] == 44
assert np.all(t['b'].mask == t.mask['b']) # Avoid t.mask in general, this is for testing
mask_before_add = t.mask.copy()
t['d'] = np.arange(len(t))
assert np.all(t.mask['b'] == mask_before_add['b'])
def test_add_masked_column_to_non_masked_table(self):
t = Table(masked=False)
assert not t.masked
t.add_column(Column(name='a', data=[1, 2, 3]))
assert not t.masked
t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1]))
assert t.masked
assert np.all(t['a'] == np.array([1, 2, 3]))
assert np.all(t['a'].mask == np.array([0, 0, 0], bool))
assert np.all(t['b'] == np.array([4, 5, 6]))
assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def match_truth_z(simz_tab, zb_tab, mini_read=False, outfil=None):
""" Match truth and zbest tables
:param simz_tab: astropy.Table; Either generated from load_z() or read from disk via 'truth.fits'
:param zb_tab: astropy.Table; Either generated from load_z() or read from disk via 'zcatalog-mini.fits'
:param mini_read: bool, optional; Tables were read from the summary tables written to disk
:param outfil: str, optional
:return: simz_tab: modified in place
"""
nsim = len(simz_tab)
# Match up
sim_id = np.array(simz_tab['TARGETID'])
z_id = np.array(zb_tab['TARGETID'])
inz = np.in1d(z_id, sim_id, assume_unique=True)
ins = np.in1d(sim_id, z_id, assume_unique=True)
z_idx = np.arange(z_id.shape[0])[inz]
sim_idx = np.arange(sim_id.shape[0])[ins]
assert np.array_equal(sim_id[sim_idx], z_id[z_idx])
# Fill up
ztags = ['Z', 'ZERR', 'ZWARN', 'SPECTYPE']
# This is for truth and zcat tables read from disk as opposed to the fibermap files
if mini_read:
ztags += ['DESI_TARGET']
# And clean up the QSO names
stypes = np.char.rstrip(simz_tab['TEMPLATETYPE'])
qsol = np.where((stypes == 'QSO') & (simz_tab['TRUEZ'] >= 2.1))[0]
simz_tab['TEMPLATETYPE'][qsol] = 'QSO_L'
qsot = np.where((stypes == 'QSO') & (simz_tab['TRUEZ'] < 2.1))[0]
simz_tab['TEMPLATETYPE'][qsot] = 'QSO_T'
# Generate the new columns
new_clms = []
mask = np.array([True] * nsim)
mask[sim_idx] = False
for kk, ztag in enumerate(ztags):
# Generate a MaskedColumn
new_clm = MaskedColumn([zb_tab[ztag][z_idx[0]]] * nsim,
name=ztag,
mask=mask)
#name=new_tags[kk], mask=mask)
# Fill
new_clm[sim_idx] = zb_tab[ztag][z_idx]
# Append
new_clms.append(new_clm)
# Add columns
simz_tab.add_columns(new_clms)
# Write?
if outfil is not None:
simz_tab.write(outfil, overwrite=True)
return
def test_convert_to_masked_table_only_if_necessary(self):
# Do not convert to masked table, if new column has no masked value.
# See #1185 for details.
t = Table(masked=False)
assert not t.masked
t.add_column(Column(name='a', data=[1, 2, 3]))
assert not t.masked
t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[0, 0, 0]))
assert not t.masked
assert np.all(t['a'] == np.array([1, 2, 3]))
assert np.all(t['b'] == np.array([4, 5, 6]))
def set_last_version(self, update=True):
r"""Set the `last_version` column to the `result_from_query` attribute
`last_version` is a column of `bool` where `False` means that there is a more update version of a catalogue
This works only if `result_from_query` contains the columns: `version` and `title`. In case the `last_version`
column is already present, a warning is raised.
Args:
update (bool): in case the `last_version` column is already present the code will update the value only
if `update` is set to `True`
Returns:
None
"""
# Require that title and version are present in result_from_query
for check_column in ['title', 'version']:
if check_column not in self.which_columns():
msgs.warning(
'{} column not present, `last_version` will not be created'
.format(check_column))
# Check that last_version is not present in result_from_query
if 'last_version' in self.which_columns():
if update:
msgs.warning('`last_version` column already present')
else:
msgs.warning(
'`last_version` column already present and it will not be updated'
)
return
# Get last versions
unique_titles = np.unique(
self.result_from_query['title'].data).tolist()
last_version = np.zeros_like(self.result_from_query['version'].data,
dtype=bool)
for unique_title in unique_titles:
most_recent_version = np.nanmax(
self.result_from_query['version'].data[(
self.result_from_query['title'].data == unique_title)])
last_version[(self.result_from_query['title'].data == unique_title)
& (self.result_from_query['version'].data ==
most_recent_version)] = True
self.result_from_query.add_column(
MaskedColumn(
data=last_version,
name='last_version',
dtype=bool,
description='True if this is the latest version of the catalog'
))
return
def merge_tables() -> Table:
"""Merges the tables."""
data = join(load_gaia_tyc(), load_tyc_spec(), keys=['TYC'], join_type='left')
data = join(
data, load_ascc(),
keys=['TYC'],
join_type='left',
table_names=('gaia', 'ascc'),
metadata_conflicts='silent',
)
data['SpType'] = MaskedColumn(data['SpType_gaia'].filled(data['SpType_ascc'].filled('')))
data['SpType'].mask = data['SpType'] == ''
data.remove_columns(['SpType_gaia', 'SpType_ascc'])
data = join(data, load_tyc_hd(), keys=['TYC'], join_type='left', metadata_conflicts='silent')
data = join(
data,
load_tyc_teff(),
keys=['TYC'],
join_type='left',
table_names=('gaia', 'tycteff'),
)
data['teff_val'] = MaskedColumn(
data['teff_val_gaia'].filled(data['teff_val_tycteff'].filled(np.nan)),
)
data['teff_val'].mask = np.isnan(data['teff_val'])
data.remove_columns(['teff_val_tycteff', 'teff_val_gaia'])
data = join(data, load_sao(), keys=['TYC'], join_type='left')
return data
def _add_blue_cols(self, tbl):
"""Add empty columns for the Ugr data."""
from astropy.table import MaskedColumn
# Prepare the column arrays
col_nan = np.array([np.nan for i in range(len(tbl))])
col_nullbyte = np.array(['\x00' for i in range(len(tbl))])
col_false = np.array([False for i in range(len(tbl))])
col_errormsg = np.array(['No_blue_data' for i in range(len(tbl))])
# Now add identical empty columns for each band
for band in ['u', 'g', 'r2']:
for colname in [
'detectionID_', '', 'err_', 'chi_', 'sharpness_', 'sky_',
'error_', 'aperMag_', 'aperMagErr_', 'snr_', 'magLim_',
'psffwhm_', 'airmass_', 'mjd_', 'pixelShift_', 'clean_',
'offsetRa_', 'offsetDec_'
]:
if colname == 'clean_':
mydtype = 'bool'
mydata = col_false
elif colname == 'detectionID_':
mydtype = '23a'
mydata = col_nullbyte
elif colname == 'error_':
mydtype = '12a'
mydata = col_errormsg
elif colname == 'mjd_':
mydtype = 'float64'
mydata = col_nan
else:
mydtype = 'float32'
mydata = col_nan
# Note that mask=false, because we want to impose our own choice of missing values
# to be written to the FITS files, rather than depending on astropy's behaviour here
tbl[colname + band] = MaskedColumn(mydata,
mask=col_false,
dtype=mydtype)
# Also add the composite colour columns
tbl['u_g'] = MaskedColumn(col_nan, mask=col_false, dtype='float32')
tbl['g_r2'] = MaskedColumn(col_nan, mask=col_false, dtype='float32')
return tbl
def set_null_value(self, null_value):
"""Assign null value to the Column
(create an astropy MaskedColumn)
:param null_value: value
"""
mask = []
for value in self.__column:
if isinstance(value, numpy.ma.core.MaskedConstant):
mask.append(True)
else:
mask.append((value == null_value))
# mask = [(value==null_value) for value in col]
self.__column = MaskedColumn(self.__column, mask=mask)
def test_add_masked_column_to_non_masked_table(self):
t = Table(masked=False)
assert not t.masked
t.add_column(Column(name='a', data=[1, 2, 3]))
assert not t.masked
t.add_column(MaskedColumn(name='b', data=[4, 5, 6], mask=[1, 0, 1]))
assert not t.masked # Changed in 4.0, table no longer auto-upgrades
assert isinstance(t['a'], Column) # Was MaskedColumn before 4.0
assert isinstance(t['b'], MaskedColumn)
assert np.all(t['a'] == np.array([1, 2, 3]))
assert not hasattr(t['a'], 'mask')
assert np.all(t['b'] == np.array([4, 5, 6]))
assert np.all(t['b'].mask == np.array([1, 0, 1], bool))
def query(coords,catalog,cols,radius=6*u.arcsec,fill_val=-99.99,full=False):
results = Vizier.query_region(coords,catalog=catalog,radius=radius)
if len(results) == 0:
return None
if full:
return results
results = results[0]
# if dict, remap colnames
if isinstance(cols,dict):
for k,v in cols.iteritems():
results.rename_column(k,v)
names = cols.values()
else:
names = cols
# make new columns one-to-one with coords
newtable = Table(masked=True)
for col in names:
oldcol = results[col]
newcol = MaskedColumn(data=np.zeros(len(coords),dtype=oldcol.dtype),unit=oldcol.unit,name=col,mask=np.ones(len(coords),dtype=bool),fill_value=fill_val)
# copy data from results
for row in results:
if not row[col]:
continue
# _q IS 1-BASED INDEXING?!
newcol[row['_q']-1] = row[col]
newcol.mask[row['_q']-1] = False
newtable.add_column(newcol)
return newtable
def parse_location(column):
"""
Given a column of location data in the form 'S10E10' convert to two columns
of angles.
"""
latitude = MaskedColumn(name="Latitude", unit=u.deg)
longitude = MaskedColumn(name="Longitude", unit=u.deg)
for i, loc in enumerate(column):
if loc:
lati = parse_latitude(loc)
longi = parse_longitude(loc)
latitude = latitude.insert(i, lati)
longitude = longitude.insert(i, longi)
else:
latitude = latitude.insert(i, None, mask=True)
longitude = longitude.insert(i, None, mask=True)
return latitude, longitude
def merge_lightcurve_fits(options):
"""Pythom program to merge the lightcurve fit results into a sigle format"""
import numpy
import astropy
import os
import JLA_library as JLA
from astropy.table import Table, MaskedColumn, vstack
from scipy.optimize import leastsq
params = JLA.build_dictionary(options.config)
# ---------------- JLA ------------------------
lightCurveFits = JLA.get_full_path(params['JLAlightCurveFits'])
f=open(lightCurveFits)
header=f.readlines()
f.close()
names=header[0].strip('#').split()
# I imagine that the tables package in astropy could also be used to read the ascii input file
SNeSpec = Table(numpy.genfromtxt(lightCurveFits,
skip_header=1,
dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8',
names=names))
nSNeSpec=len(SNeSpec)
print 'There are %d SNe from the spectrscopically confirmed sample' % (nSNeSpec)
# Add an extra column to the table
SNeSpec['source']=['JLA']*nSNeSpec
# -------------- Malmquist bias fits
malmBias={}
if options.bias:
# Compute the bias correction
bias = numpy.genfromtxt(JLA.get_full_path(params['biasPolynomial']),
skip_header=3,
usecols=(0, 1, 2, 3),
dtype='S10,f8,f8,f8',
names=['sample', 'redshift', 'bias', 'e_bias'])
for sample in numpy.unique(bias['sample']):
selection=(bias['sample']==sample)
guess=[0,0,0]
plsq=leastsq(residuals, guess, args=(bias[selection]['bias'],
bias[selection]['redshift'],
bias[selection]['e_bias'],
'poly'), full_output=1)
if plsq[4] in [1,2,3,4]:
print 'Solution for %s found' % (sample)
malmBias[sample]=plsq[0]
# ---------------- Shuvo's sample aka JLA++ ------------------------
# Photometrically identified SNe in Shuvo's sample, if the parameter exists
if params['photLightCurveFits']!='None':
lightCurveFits = JLA.get_full_path(params['photLightCurveFits'])
SNePhot=Table.read(lightCurveFits, format='fits')
nSNePhot=len(SNePhot)
print 'There are %d SNe from the photometric sample' % (nSNePhot)
# Converting from Shuvo's names to thosed used by JLA
conversion={'name':'name_adj', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':'col27', 'd3rdvar':'d3rdvar', 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':'col4', 'dec':'col5', 'biascor':None}
# Add the uncertainty in the mass column
SNePhot['d3rdvar']=(SNePhot['col29']+SNePhot['col28'])/2. - SNePhot['col27']
# Remove columns that are not listed in conversion
for colname in SNePhot.colnames:
if colname not in conversion.values():
SNePhot.remove_column(colname)
for key in conversion.keys():
# Rename the column if it does not already exist
if conversion[key]!=None and conversion[key]!=key:
SNePhot.rename_column(conversion[key], key)
elif conversion[key]==None:
# Create it, mask it, and fill all values
SNePhot[key]=MaskedColumn(numpy.zeros(nSNePhot), numpy.ones(nSNePhot,bool))
SNePhot[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line
SNePhot[key]=-99.9
else:
# Do nothing if the column already exists
pass
# Compute the bias correction
for i,SN in enumerate(SNePhot):
if 'SDSS' in SN['name']:
SNePhot['biascor'][i]=poly(SN['zhel'],malmBias['SDSS'])
else:
SNePhot['biascor'][i]=poly(SN['zhel'],malmBias['SNLS'])
# Add the source column
SNePhot['source']="Phot_Uddin"
# ---------------------- CfA4 ----------------------------------
if params['CfA4LightCurveFits']!='None':
lightCurveFits = JLA.get_full_path(params['CfA4LightCurveFits'])
f=open(lightCurveFits)
header=f.readlines()
f.close()
names=header[0].strip('#').split(',')
SNeCfA4=Table(numpy.genfromtxt(lightCurveFits,
skip_header=1,
dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8',
names=names,delimiter=','))
nSNeCfA4=len(SNeCfA4)
conversion={'name':'name', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':None, 'd3rdvar':None, 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':None, 'dec':None, 'biascor':None}
# Remove columns that are not listed in conversion
for colname in SNeCfA4.colnames:
if colname not in conversion.values():
SNeCfA4.remove_column(colname)
for key in conversion.keys():
# Rename the column if it does not already exist
if conversion[key]!=None and conversion[key]!=key:
SNeCfA4.rename_column(conversion[key], key)
elif conversion[key]==None:
# Create it, mask it, and fill all values
SNeCfA4[key]=MaskedColumn(numpy.zeros(nSNeCfA4), numpy.ones(nSNeCfA4,bool))
SNeCfA4[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line
SNeCfA4[key]=-99.9
else:
# Do nothing if the column already exists
pass
# Add the source column
SNeCfA4['source']="CfA4"
# We also need to gather information on the host mass, the host mass uncertainty, CMB redshift and Malmquist bias
CfA4lightcurves=[]
CfA4_lcDirectories=params['CfA4MassesAndCMBz'].split(',')
for lcDir in CfA4_lcDirectories:
listing=os.listdir(JLA.get_full_path(lcDir))
for lc in listing:
CfA4lightcurves.append(JLA.get_full_path(lcDir)+lc)
for i,SN in enumerate(SNeCfA4):
for lc in CfA4lightcurves:
if SN['name'][2:] in lc:
keywords=getKeywords(lc)
SNeCfA4[i]['zcmb']=keywords['REDSHIFT_CMB']
SNeCfA4[i]['3rdvar']=keywords['HOSTGAL_LOGMASS']
SNeCfA4[i]['d3rdvar']=keywords['e_HOSTGAL_LOGMASS']
SNeCfA4[i]['ra']=keywords['RA']
SNeCfA4[i]['dec']=keywords['DECL']
if SNeCfA4[i]['3rdvar'] < 0:
SNeCfA4[i]['3rdvar']=-99.9
SNeCfA4[i]['d3rdvar']=-99.9
# Compute the bias correction
SNeCfA4['biascor']=poly(SNeCfA4['zcmb'],malmBias['nearby'])
try:
SNe=vstack([SNeSpec,SNePhot,SNeCfA4])
except:
SNe=SNeSpec
# print len(SNe),len(numpy.unique(SNe['name']))
# Write out the result as a FITS table
date = JLA.get_date()
SNe.write('%s_%s.fits' % (options.output, date), format='fits', overwrite=True)
return
def _fits_summary(self, header_keywords):
"""
Generate a summary table of keywords from FITS headers.
Parameters
----------
header_keywords : list of str or '*'
Keywords whose value should be extracted from FITS headers or '*'
to extract all.
"""
if not self.files:
return None
# Make sure we have a list...for example, in python 3, dict.keys()
# is not a list.
original_keywords = list(header_keywords)
# Get rid of any duplicate keywords, also forces a copy.
header_keys = set(original_keywords)
header_keys.add('file')
file_name_column = MaskedColumn(name='file', data=self.files)
if not header_keys or (header_keys == {'file'}):
summary_table = Table(masked=True)
summary_table.add_column(file_name_column)
return summary_table
summary_dict = None
missing_marker = None
for file_name in file_name_column.tolist():
file_path = path.join(self.location, file_name)
try:
# Note: summary_dict is an OrderedDict, so should preserve
# the order of the keywords in the FITS header.
summary_dict = self._dict_from_fits_header(
file_path, input_summary=summary_dict,
missing_marker=missing_marker)
except IOError as e:
logger.warning('unable to get FITS header for file %s: %s.',
file_path, e)
continue
summary_table = Table(summary_dict, masked=True)
for column in summary_table.colnames:
summary_table[column].mask = [
v is missing_marker for v in summary_table[column].tolist()]
self._set_column_name_case_to_match_keywords(header_keys,
summary_table)
missing_columns = header_keys - set(summary_table.colnames)
missing_columns -= {'*'}
length = len(summary_table)
for column in missing_columns:
all_masked = MaskedColumn(name=column, data=np.zeros(length),
mask=np.ones(length))
summary_table.add_column(all_masked)
if '*' not in header_keys:
# Rearrange table columns to match order of keywords.
# File always comes first.
header_keys -= {'file'}
original_order = ['file'] + sorted(header_keys,
key=original_keywords.index)
summary_table = summary_table[original_order]
if not summary_table.masked:
summary_table = Table(summary_table, masked=True)
return summary_table
def read_table_fits(input, hdu=None, astropy_native=False, memmap=False,
character_as_bytes=True):
"""
Read a Table object from an FITS file
If the ``astropy_native`` argument is ``True``, then input FITS columns
which are representations of an astropy core object will be converted to
that class and stored in the ``Table`` as "mixin columns". Currently this
is limited to FITS columns which adhere to the FITS Time standard, in which
case they will be converted to a `~astropy.time.Time` column in the output
table.
Parameters
----------
input : str or file-like object or compatible `astropy.io.fits` HDU object
If a string, the filename to read the table from. If a file object, or
a compatible HDU object, the object to extract the table from. The
following `astropy.io.fits` HDU objects can be used as input:
- :class:`~astropy.io.fits.hdu.table.TableHDU`
- :class:`~astropy.io.fits.hdu.table.BinTableHDU`
- :class:`~astropy.io.fits.hdu.table.GroupsHDU`
- :class:`~astropy.io.fits.hdu.hdulist.HDUList`
hdu : int or str, optional
The HDU to read the table from.
astropy_native : bool, optional
Read in FITS columns as native astropy objects where possible instead
of standard Table Column objects. Default is False.
memmap : bool, optional
Whether to use memory mapping, which accesses data on disk as needed. If
you are only accessing part of the data, this is often more efficient.
If you want to access all the values in the table, and you are able to
fit the table in memory, you may be better off leaving memory mapping
off. However, if your table would not fit in memory, you should set this
to `True`.
character_as_bytes : bool, optional
If `True`, string columns are stored as Numpy byte arrays (dtype ``S``)
and are converted on-the-fly to unicode strings when accessing
individual elements. If you need to use Numpy unicode arrays (dtype
``U``) internally, you should set this to `False`, but note that this
will use more memory. If set to `False`, string columns will not be
memory-mapped even if ``memmap`` is `True`.
"""
if isinstance(input, HDUList):
# Parse all table objects
tables = OrderedDict()
for ihdu, hdu_item in enumerate(input):
if isinstance(hdu_item, (TableHDU, BinTableHDU, GroupsHDU)):
tables[ihdu] = hdu_item
if len(tables) > 1:
if hdu is None:
warnings.warn("hdu= was not specified but multiple tables"
" are present, reading in first available"
" table (hdu={0})".format(first(tables)),
AstropyUserWarning)
hdu = first(tables)
# hdu might not be an integer, so we first need to convert it
# to the correct HDU index
hdu = input.index_of(hdu)
if hdu in tables:
table = tables[hdu]
else:
raise ValueError("No table found in hdu={0}".format(hdu))
elif len(tables) == 1:
table = tables[first(tables)]
else:
raise ValueError("No table found")
elif isinstance(input, (TableHDU, BinTableHDU, GroupsHDU)):
table = input
else:
hdulist = fits_open(input, character_as_bytes=character_as_bytes,
memmap=memmap)
try:
return read_table_fits(hdulist, hdu=hdu,
astropy_native=astropy_native)
finally:
hdulist.close()
# Check if table is masked
masked = any(col.null is not None for col in table.columns)
# TODO: in future, it may make more sense to do this column-by-column,
# rather than via the structured array.
# In the loop below we access the data using data[col.name] rather than
# col.array to make sure that the data is scaled correctly if needed.
data = table.data
columns = []
for col in data.columns:
# Set column data
if masked:
column = MaskedColumn(data=data[col.name], name=col.name, copy=False)
if col.null is not None:
column.set_fill_value(col.null)
column.mask[column.data == col.null] = True
else:
column = Column(data=data[col.name], name=col.name, copy=False)
# Copy over units
if col.unit is not None:
column.unit = u.Unit(col.unit, format='fits', parse_strict='silent')
# Copy over display format
if col.disp is not None:
column.format = _fortran_to_python_format(col.disp)
columns.append(column)
# Create Table object
t = Table(columns, masked=masked, copy=False)
# TODO: deal properly with unsigned integers
hdr = table.header
if astropy_native:
# Avoid circular imports, and also only import if necessary.
from .fitstime import fits_to_time
hdr = fits_to_time(hdr, t)
for key, value, comment in hdr.cards:
if key in ['COMMENT', 'HISTORY']:
# Convert to io.ascii format
if key == 'COMMENT':
key = 'comments'
if key in t.meta:
t.meta[key].append(value)
else:
t.meta[key] = [value]
elif key in t.meta: # key is duplicate
if isinstance(t.meta[key], list):
t.meta[key].append(value)
else:
t.meta[key] = [t.meta[key], value]
elif is_column_keyword(key) or key in REMOVE_KEYWORDS:
pass
else:
t.meta[key] = value
# TODO: implement masking
# Decode any mixin columns that have been stored as standard Columns.
t = _decode_mixins(t)
return t
def createTable(outlines, metaDict, colNames, colDefaults):
"""
Creates an astropy table from inputs.
Parameters
----------
outlines : list of str
Input lines
metaDict : dict
Input meta data
colNames : list of str
Input column names
colDefaults : list
Input column default values
Returns
-------
table : astropy.table.Table object
"""
# Before loading table into an astropy Table object, set lengths of Name,
# Patch, and Type columns to 100 characters
log = logging.getLogger('LSMTool.Load')
converters = {}
nameCol = 'col{0}'.format(colNames.index('Name')+1)
converters[nameCol] = [ascii.convert_numpy('{}100'.format(numpy_type))]
typeCol = 'col{0}'.format(colNames.index('Type')+1)
converters[typeCol] = [ascii.convert_numpy('{}100'.format(numpy_type))]
if 'Patch' in colNames:
patchCol = 'col{0}'.format(colNames.index('Patch')+1)
converters[patchCol] = [ascii.convert_numpy('{}100'.format(numpy_type))]
log.debug('Creating table...')
table = Table.read('\n'.join(outlines), guess=False, format='ascii.no_header', delimiter=',',
names=colNames, comment='#', data_start=0, converters=converters)
# Convert spectral index values from strings to arrays.
if 'SpectralIndex' in table.keys():
log.debug('Converting spectral indices...')
specOld = table['SpectralIndex'].data.tolist()
specVec = []
maskVec = []
maxLen = 0
for l in specOld:
try:
if type(l) is float or type(l) is int:
maxLen = 1
else:
specEntry = [float(f) for f in l.split(';')]
if len(specEntry) > maxLen:
maxLen = len(specEntry)
except:
pass
log.debug('Maximum number of spectral-index terms in model: {0}'.format(maxLen))
for l in specOld:
try:
if type(l) is float or type(l) is int:
specEntry = [float(l)]
specMask = [False]
else:
specEntry = [float(f) for f in l.split(';')]
specMask = [False] * len(specEntry)
while len(specEntry) < maxLen:
specEntry.append(0.0)
specMask.append(True)
specVec.append(specEntry)
maskVec.append(specMask)
except:
specVec.append([0.0]*maxLen)
maskVec.append([True]*maxLen)
specCol = MaskedColumn(name='SpectralIndex', data=np.array(specVec, dtype=np.float))
specCol.mask = maskVec
specIndx = table.keys().index('SpectralIndex')
table.remove_column('SpectralIndex')
table.add_column(specCol, index=specIndx)
# Convert RA and Dec to Angle objects
log.debug('Converting RA...')
RARaw = table['Ra'].data.tolist()
RACol = Column(name='Ra', data=RA2Angle(RARaw))
def raformat(val):
return Angle(val, unit='degree').to_string(unit='hourangle', sep=':')
RACol.format = raformat
RAIndx = table.keys().index('Ra')
table.remove_column('Ra')
table.add_column(RACol, index=RAIndx)
log.debug('Converting Dec...')
DecRaw = table['Dec'].data.tolist()
DecCol = Column(name='Dec', data=Dec2Angle(DecRaw))
def decformat(val):
return Angle(val, unit='degree').to_string(unit='degree', sep='.')
DecCol.format = decformat
DecIndx = table.keys().index('Dec')
table.remove_column('Dec')
table.add_column(DecCol, index=DecIndx)
def fluxformat(val):
return '{0:0.3f}'.format(val)
table.columns['I'].format = fluxformat
# Set column units and default values
for i, colName in enumerate(colNames):
log.debug("Setting units for column '{0}' to {1}".format(
colName, allowedColumnUnits[colName.lower()]))
table.columns[colName].unit = allowedColumnUnits[colName.lower()]
if hasattr(table.columns[colName], 'filled') and colDefaults[i] is not None:
fillVal = colDefaults[i]
if colName == 'SpectralIndex':
while len(fillVal) < maxLen:
fillVal.append(0.0)
log.debug("Setting default value for column '{0}' to {1}".
format(colName, fillVal))
table.columns[colName].fill_value = fillVal
table.meta = metaDict
return table
def merge_lightcurve_fits(options):
"""Pythom program to merge the lightcurve fit results into a sigle format"""
import numpy
import astropy
import JLA_library as JLA
from astropy.table import Table, MaskedColumn, vstack
params = JLA.build_dictionary(options.config)
# ---------------- JLA ------------------------
lightCurveFits = JLA.get_full_path(params['JLAlightCurveFits'])
f=open(lightCurveFits)
header=f.readlines()
f.close()
names=header[0].strip('#').split()
# I imagine that the tables package in astropy could also be used to read the ascii input file
SNeSpec = Table(numpy.genfromtxt(lightCurveFits,
skip_header=1,
dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8',
names=names))
nSNeSpec=len(SNeSpec)
print 'There are %d SNe from the spectrscopically confirmed sample' % (nSNeSpec)
# Add an extra column to the table
SNeSpec['source']=['JLA']*nSNeSpec
# ---------------- Shuvo's sample ------------------------
# Photometrically identified SNe in Shuvo's sample, if the parameter exists
if params['photLightCurveFits']!='None':
lightCurveFits = JLA.get_full_path(params['photLightCurveFits'])
SNePhot=Table.read(lightCurveFits, format='fits')
nSNePhot=len(SNePhot)
print 'There are %d SNe from the photometric sample' % (nSNePhot)
# Converting from Shuvo's names to thosed used by JLA
conversion={'name':'name_adj', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':'col27', 'd3rdvar':'d3rdvar', 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':None, 'dec':None, 'biascor':None}
# Add the uncertainty in the mass column
SNePhot['d3rdvar']=(SNePhot['col29']+SNePhot['col28'])/2. - SNePhot['col27']
# Remove columns that are not listed in conversion
for colname in SNePhot.colnames:
if colname not in conversion.values():
SNePhot.remove_column(colname)
for key in conversion.keys():
# Rename the column if it does not already exist
if conversion[key]!=None and conversion[key]!=key:
SNePhot.rename_column(conversion[key], key)
elif conversion[key]==None:
# Create it, mask it, and fill all values
SNePhot[key]=MaskedColumn(numpy.zeros(nSNePhot), numpy.ones(nSNePhot,bool))
SNePhot[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line
SNePhot[key]=-99.9
else:
# Do nothing if the column already exists
pass
# Add the source column
SNePhot['source']="Phot_Uddin"
# ---------------------- CfA4 ----------------------------------
if params['CfA4LightCurveFits']!='None':
lightCurveFits = JLA.get_full_path(params['CfA4LightCurveFits'])
f=open(lightCurveFits)
header=f.readlines()
f.close()
names=header[0].strip('#').split(',')
SNeCfA4=Table(numpy.genfromtxt(lightCurveFits,
skip_header=1,
dtype='S12,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8',
names=names,delimiter=','))
nSNeCfA4=len(SNeCfA4)
conversion={'name':'name', 'zcmb':None, 'zhel':'z', 'dz':None, 'mb':'mb', 'dmb':'emb', 'x1':'x1', 'dx1':'ex1', 'color':'c', 'dcolor':'ec', '3rdvar':None, 'd3rdvar':None, 'tmax':None, 'dtmax':None, 'cov_m_s':'cov_m_x1', 'cov_m_c':'cov_m_c', 'cov_s_c':'cov_x1_c', 'set':None, 'ra':None, 'dec':None, 'biascor':None}
# Remove columns that are not listed in conversion
for colname in SNeCfA4.colnames:
if colname not in conversion.values():
SNeCfA4.remove_column(colname)
for key in conversion.keys():
# Rename the column if it does not already exist
if conversion[key]!=None and conversion[key]!=key:
SNeCfA4.rename_column(conversion[key], key)
elif conversion[key]==None:
# Create it, mask it, and fill all values
SNeCfA4[key]=MaskedColumn(numpy.zeros(nSNeCfA4), numpy.ones(nSNeCfA4,bool))
SNeCfA4[key].fill_value=-99 # does not work as expected, so we set it explicitly in the next line
SNeCfA4[key]=-99.9
else:
# Do nothing if the column already exists
pass
# Add the source column
SNeCfA4['source']="CfA4"
try:
SNe=vstack([SNeSpec,SNePhot,SNeCfA4])
except:
SNe=SNeSpec
# Write out the result as a FITS table
date = JLA.get_date()
SNe.write('%s_%s.fits' % (options.output, date), format='fits')
return
