def test_parse_and_fit_to_line():
"""
Tests fitting a line to fake RA and DEC data which has errors calculated from the real intermediate data
from Hip1, Hip2, and GaiaDR2. This only fits a line to the first 11 points.
"""
stars = ['049699', '027321', '027321']
parsers = [GaiaeDR3, GaiaData, HipparcosOriginalData, HipparcosRereductionDVDBook]
subdirectories = ['GaiaeDR3', 'GaiaDR2', 'Hip1', 'Hip2']
base_directory = os.path.join(os.getcwd(), 'htof/test/data_for_tests')
for star, parser, subdirectory in zip(stars, parsers, subdirectories):
test_data_directory = os.path.join(base_directory, subdirectory)
data = parser()
data.parse(star_id=star,
intermediate_data_directory=test_data_directory)
data.calculate_inverse_covariance_matrices()
fitter = AstrometricFitter(inverse_covariance_matrices=data.inverse_covariance_matrix,
epoch_times=np.linspace(0, 10, num=11))
solution_vector = fitter.fit_line(ra_vs_epoch=np.linspace(30, 40, num=11),
dec_vs_epoch=np.linspace(20, 30, num=11))
ra0, dec0, mu_ra, mu_dec = solution_vector
assert np.isclose(ra0, 30)
assert np.isclose(dec0, 20)
assert np.isclose(mu_ra, 1)
assert np.isclose(mu_dec, 1)
def test_fitting_to_cubic_astrometric_data_with_parallax(self):
real_plx = 100
cntr_dec, cntr_ra = Angle(45, unit='degree'), Angle(45, unit='degree')
astrometric_data = generate_astrometric_data(acc=True, jerk=True)
jyear_epochs = Time(astrometric_data['epoch_delta_t'] + 2012,
format='decimalyear').jyear
ra_pert, dec_pert = parallactic_motion(jyear_epochs,
cntr_dec.mas,
cntr_ra.mas,
'mas',
2012,
parallax=1)
astrometric_data['dec'] += dec_pert * real_plx
astrometric_data['ra'] += ra_pert * real_plx
fitter = AstrometricFitter(
inverse_covariance_matrices=astrometric_data[
'inverse_covariance_matrix'],
epoch_times=astrometric_data['epoch_delta_t'],
use_parallax=True,
parallactic_pertubations={
'ra_plx': ra_pert,
'dec_plx': dec_pert
},
fit_degree=3)
solution, errors, chisq, resids = fitter.fit_line(
astrometric_data['ra'], astrometric_data['dec'], return_all=True)
assert np.allclose(solution[1:],
astrometric_data['nonlinear_solution'],
atol=0,
rtol=1E-4)
assert np.allclose(solution[0], real_plx)
def test_optimal_central_epoch_raises_error(self):
astrometric_data = generate_astrometric_data()
fitter = AstrometricFitter(
inverse_covariance_matrices=astrometric_data[
'inverse_covariance_matrix'],
epoch_times=astrometric_data['epoch_delta_t'])
with pytest.raises(ValueError):
fitter.find_optimal_central_epoch(
'incorrect_choice'), fitter.find_optimal_central_epoch('dec')
def test_errors_on_linear_astrometric_data(self):
astrometric_data = generate_astrometric_data()
fitter = AstrometricFitter(
inverse_covariance_matrices=astrometric_data[
'inverse_covariance_matrix'],
epoch_times=astrometric_data['epoch_delta_t'])
sol, errs, chisq, resids = fitter.fit_line(astrometric_data['ra'],
astrometric_data['dec'],
return_all=True)
assert errs.size == 4
assert np.isclose(0, chisq, atol=1e-7)
def test_optimal_central_epoch_on_linear_data(self):
astrometric_data = generate_astrometric_data()
fitter = AstrometricFitter(
inverse_covariance_matrices=astrometric_data[
'inverse_covariance_matrix'],
epoch_times=astrometric_data['epoch_delta_t'])
central_epoch_ra, central_epoch_dec = fitter.find_optimal_central_epoch(
'ra'), fitter.find_optimal_central_epoch('dec')
cov_matrix = fitter.evaluate_cov_matrix(central_epoch_ra,
central_epoch_dec)
assert np.allclose([cov_matrix[0, 2], cov_matrix[1, 3]], 0)
def test_optimal_central_epoch_on_cubic_data(self):
astrometric_data = generate_astrometric_data(acc=True, jerk=True)
fitter = AstrometricFitter(
inverse_covariance_matrices=astrometric_data[
'inverse_covariance_matrix'],
epoch_times=astrometric_data['epoch_delta_t'],
use_parallax=False,
fit_degree=3)
central_epoch_ra, central_epoch_dec = fitter.find_optimal_central_epoch(
'ra'), fitter.find_optimal_central_epoch('dec')
cov_matrix = fitter.evaluate_cov_matrix(central_epoch_ra,
central_epoch_dec)
assert np.allclose([cov_matrix[0, 2], cov_matrix[1, 3]], 0)
def test_chi2_matrix_many_epoch(self, fake_chi2_matrix_per_epoch):
ivar = np.ones((2, 2))
fitter = AstrometricFitter(
inverse_covariance_matrices=[ivar, ivar, ivar],
epoch_times=[1, 2, 3],
astrometric_solution_vector_components=[],
use_parallax=True,
fit_degree=3)
assert np.allclose(
np.ones((9, 9)) * 3,
fitter._init_astrometric_chi_squared_matrix(3)[0])
fake_chi2_matrix_per_epoch.return_value = np.ones((7, 7))
assert np.allclose(
np.ones((7, 7)) * 3,
fitter._init_astrometric_chi_squared_matrix(2)[0])
def test_chi2_vector(self, mock_ra_vec, mock_dec_vec):
covariance_matrix = np.array([[5, 1], [12, 2]])
expected_c = 4 * np.ones(4)
fitter = AstrometricFitter(inverse_covariance_matrices=np.array([
np.linalg.pinv(covariance_matrix),
np.linalg.pinv(covariance_matrix)
]),
epoch_times=np.array([1, 2]),
astrometric_chi_squared_matrices=[],
fit_degree=1,
use_parallax=False)
assert np.allclose(
expected_c,
fitter._chi2_vector(ra_vs_epoch=np.array([1, 1]),
dec_vs_epoch=np.array([1, 1])))
def test_fitting_with_nonzero_central_epoch(self):
ra_cnt = np.random.randint(1, 5)
dec_cnt = np.random.randint(1, 5)
astrometric_data = generate_astrometric_data()
expected_vec = astrometric_data['linear_solution']
expected_vec[0] += ra_cnt * expected_vec[
2] # r0 = ra_central_time * mu_ra
expected_vec[1] += dec_cnt * expected_vec[
3] # dec0 = dec_central_time * mu_dec
fitter = AstrometricFitter(
inverse_covariance_matrices=astrometric_data[
'inverse_covariance_matrix'],
epoch_times=astrometric_data['epoch_delta_t'],
central_epoch_dec=dec_cnt,
central_epoch_ra=ra_cnt)
assert np.allclose(
fitter.fit_line(astrometric_data['ra'], astrometric_data['dec']),
expected_vec)
def test_init_epochs(self):
fitter = AstrometricFitter(astrometric_solution_vector_components=[],
central_epoch_dec=1.5,
central_epoch_ra=1,
epoch_times=np.array([1, 2, 3]),
astrometric_chi_squared_matrices=[],
fit_degree=1,
use_parallax=False)
assert np.allclose(fitter.dec_epochs, [-0.5, 0.5, 1.5])
assert np.allclose(fitter.ra_epochs, [0, 1, 2])
def test_fitter_removes_parallax(self):
astrometric_data = generate_astrometric_data()
fitter = AstrometricFitter(
inverse_covariance_matrices=astrometric_data[
'inverse_covariance_matrix'],
epoch_times=astrometric_data['epoch_delta_t'],
use_parallax=False,
fit_degree=3)
assert fitter._chi2_matrix.shape == (8, 8)
assert fitter.astrometric_solution_vector_components['ra'][
0].shape == (8, )
assert fitter.astrometric_solution_vector_components['dec'][
0].shape == (8, )
def refit_hip_fromdata(data: DataParser, fit_degree, cntr_RA=Angle(0, unit='degree'), cntr_Dec=Angle(0, unit='degree'),
use_parallax=False):
data.calculate_inverse_covariance_matrices()
# generate parallax motion
jyear_epoch = time.Time(data.julian_day_epoch(), format='jd', scale='tcb').jyear
# note that ra_motion and dec_motion are in degrees here.
# generate sky path
year_epochs = jyear_epoch - time.Time(1991.25, format='decimalyear', scale='tcb').jyear
ra_motion, dec_motion = parallactic_motion(jyear_epoch, cntr_RA.degree, cntr_Dec.degree, 'degree',
time.Time(1991.25, format='decimalyear', scale='tcb').jyear,
ephemeris=earth_ephemeris) # Hipparcos was in a geostationary orbit.
ra_resid = Angle(data.residuals.values * np.sin(data.scan_angle.values), unit='mas')
dec_resid = Angle(data.residuals.values * np.cos(data.scan_angle.values), unit='mas')
# instantiate fitter
fitter = AstrometricFitter(data.inverse_covariance_matrix, year_epochs,
use_parallax=use_parallax, fit_degree=fit_degree,
parallactic_pertubations={'ra_plx': Angle(ra_motion, 'degree').mas,
'dec_plx': Angle(dec_motion, 'degree').mas})
fit_coeffs, errors, chisq, refit_residuals = fitter.fit_line(ra_resid.mas, dec_resid.mas, return_all=True)
parallax_factors = ra_motion * np.sin(data.scan_angle.values) + dec_motion * np.cos(data.scan_angle.values)
# technically this could be data.parallax_factors.values, but then we have to deal with
# getting the RA and Dec components of that.
if not use_parallax:
fit_coeffs = np.hstack([[0], fit_coeffs])
errors = np.hstack([[0], errors])
parallax_factors = np.zeros_like(parallax_factors)
# pad so coeffs and errors are 9 long.
errors = np.pad(errors, (0, 9 - len(fit_coeffs)))
fit_coeffs = np.pad(fit_coeffs, (0, 9 - len(fit_coeffs)))
# calculate the chisquared partials
sin_scan = np.sin(data.scan_angle.values)
cos_scan = np.cos(data.scan_angle.values)
dt = data.epoch - 1991.25
chi2_vector = (2 * data.residuals.values / data.along_scan_errs.values ** 2 * np.array(
[parallax_factors, sin_scan, cos_scan, dt * sin_scan, dt * cos_scan])).T
chi2_partials = np.sum(chi2_vector, axis=0) ** 2
return fit_coeffs, errors, chisq, chi2_partials
def __init__(self, data_choice, star_id, intermediate_data_directory, fitter=None, data=None,
central_epoch_ra=0, central_epoch_dec=0, format='jd', fit_degree=1,
use_parallax=False, central_ra=None, central_dec=None,
use_catalog_parallax_factors=False, along_scan_error_scaling=1.0, **kwargs):
self.along_scan_error_scaling = along_scan_error_scaling
if data_choice.lower() == 'hip2recalibrated':
warnings.warn(f'You have selected {data_choice}, the recalibrated Hipparcos 2 data. Note that for this,'
f' you should be feeding in the filepaths to the Hip21 (Hip2 java tool data), because'
f' htof applies the recalibration on-the-fly for each file. As well, be sure to read'
f' Brandt et al. 2022 to understand the limitations of using the recalibrated data. ') # pragma: no cover
if data is None:
ThisDataParser = self.parsers[data_choice.lower()]
data = ThisDataParser.parse_and_instantiate(star_id=star_id,
intermediate_data_directory=intermediate_data_directory)
data.scale_along_scan_errs(self.along_scan_error_scaling)
data.calculate_inverse_covariance_matrices()
parallactic_pertubations = None
if use_parallax:
if not use_catalog_parallax_factors:
# recompute the parallax factors at the new central_ra and central_ra epoch.
if not (isinstance(central_ra, Angle) and isinstance(central_dec, Angle)):
raise ValueError('Cannot compute parallax factors. central_ra and central_dec must be instances'
' of astropy.coordinates.Angle.') # pragma: no cover
if central_epoch_dec != central_epoch_ra:
warnings.warn('central_epoch_dec != central_epoch_ra. '
'Using central_epoch_ra as the central_epoch to compute the parallax motion.',
UserWarning) # pragma: no cover
ra_motion, dec_motion = parallactic_motion(Time(data.julian_day_epoch(), format='jd').jyear,
central_ra.mas, central_dec.mas, 'mas',
Time(central_epoch_ra, format=format).jyear,
ephemeris=self.ephemeri[data_choice.lower()])
else:
ra_motion, dec_motion = to_ra_dec_basis(data.parallax_factors.values, data.scan_angle.values)
parallactic_pertubations = {'ra_plx': ra_motion, 'dec_plx': dec_motion}
if fitter is None and data is not None:
fitter = AstrometricFitter(inverse_covariance_matrices=data.inverse_covariance_matrix,
epoch_times=Time(Time(data.julian_day_epoch(), format='jd'), format=format).value,
central_epoch_dec=Time(central_epoch_dec, format=format).value,
central_epoch_ra=Time(central_epoch_ra, format=format).value,
fit_degree=fit_degree,
use_parallax=use_parallax,
parallactic_pertubations=parallactic_pertubations)
self.data = data
self.fitter = fitter
def parse(self,
star_id,
intermediate_data_directory,
attempt_adhoc_rejection=True,
reject_known=True,
**kwargs):
# important that the 2007 error inflation is turned off (error_inflate=False)
header, raw_data = super(Hipparcos2Recalibrated, self).parse(
star_id,
intermediate_data_directory,
error_inflate=False,
attempt_adhoc_rejection=attempt_adhoc_rejection,
reject_known=reject_known,
**kwargs)
apply_calibrations = True
if not (self.meta['catalog_soltype'] == 5
or self.meta['catalog_soltype'] == 7
or self.meta['catalog_soltype'] == 9):
warnings.warn(
f'This source has a solution type of {self.meta["catalog_soltype"]}. '
f'htof will only recalibrate 5, 7, and 9 parameter solutions currently. '
f'No recalibration will be performed.')
apply_calibrations = False
if attempt_adhoc_rejection is False or reject_known is False:
warnings.warn(
'We have not tested recalibration without rejecting any of the flagged or bugged '
'observations. No recalibration will be performed.')
apply_calibrations = False
if apply_calibrations:
# apply the calibrations
self.residuals += self.residual_offset # note that this modifies the raw_data column also.
self.along_scan_errs = np.sqrt(self.along_scan_errs**2 +
self.cosmic_dispersion**2)
self.calculate_inverse_covariance_matrices()
# munge the parallax factors into the correct form. Note that we are using
# the parallax factors from the catalog here to keep everything consistent.
ra_motion, dec_motion = to_ra_dec_basis(
self.parallax_factors.values, self.scan_angle.values)
parallactic_perturbations = {
'ra_plx': ra_motion,
'dec_plx': dec_motion
}
# refit the data, calculate the new residuals, and parameters.
fit_degree = {5: 1, 7: 2, 9: 3}[int(self.meta['catalog_soltype'])]
fitter = AstrometricFitter(
inverse_covariance_matrices=self.inverse_covariance_matrix,
epoch_times=Time(Time(self.julian_day_epoch(), format='jd'),
format='jyear').value,
central_epoch_dec=1991.25,
central_epoch_ra=1991.25,
fit_degree=fit_degree,
use_parallax=True,
parallactic_pertubations=parallactic_perturbations)
# get residuals in ra and dec.
ra = Angle(self.residuals.values * np.sin(self.scan_angle.values),
unit='mas')
dec = Angle(self.residuals.values * np.cos(self.scan_angle.values),
unit='mas')
# fit the residuals
coeffs, errors, Q, new_residuals = fitter.fit_line(ra.mas,
dec.mas,
return_all=True)
# compute the along-scan residuals
new_residuals = to_along_scan_basis(new_residuals[:, 0],
new_residuals[:, 1],
self.scan_angle.values)
self.residuals = Series(new_residuals, index=self.residuals.index)
"""
update the header with the new statistics
"""
ntransits, nparam = len(self), int(self.meta['catalog_soltype'])
header['second']['NOB'] = len(self)
header['second'][
'NR'] = 0 # because we automatically remove any "flagged as rejected" observations.
header['first']['F1'] = 0
header['first']['NRES'] = len(self)
header['first']['F2'] = special.erfcinv(
stats.chi2.sf(Q, ntransits - nparam) * 2) * np.sqrt(2)
if np.isfinite(header['first']['F2']):
header['first']['F2'] = np.round(header['first']['F2'], 4)
# update the best fit parameters with the new values
# dpmRA dpmDE e_dpmRA e_dpmDE ddpmRA ddpmDE e_ddpmRA e_ddpmDE
for i, key, dp, in zip(np.arange(nparam), [
'Plx', 'RAdeg', 'DEdeg', 'pm_RA', 'pm_DE', 'dpmRA',
'dpmDE', 'ddpmRA', 'ddpmDE'
], [2, 8, 8, 2, 2, 2, 2, 2, 2]):
if key != 'RAdeg' and key != 'DEdeg':
header['third'][key] = np.round(
header['third'][key] + coeffs[i], dp)
else:
# convert the perturbation from mas to degrees.
header['third'][key] = np.round(
header['third'][key] +
(coeffs[i] * u.mas).to(u.degree).value, dp)
# update the errors with the new errors
for i, key, dp, in zip(np.arange(nparam), [
'e_Plx', 'e_RA', 'e_DE', 'e_pmRA', 'e_pmDE', 'e_dpmRA',
'e_dpmDE', 'e_ddpmRA', 'e_ddpmDE'
], [2, 2, 2, 2, 2, 2, 2, 2, 2]):
header['third'][key] = np.round(errors[i], dp)
# save the modified header to the class, because these will be
# used by self.write_as_javatool_format()
self.recalibrated_header = deepcopy(header)
"""
update the raw_data columns with the new data. Note that rejected/bugged epochs are already
taken care of.
"""
# data order in Java tool data: IORB EPOCH PARF CPSI SPSI RES SRES
recalibrated_data = DataFrame({
'1':
self._iorb,
'2':
self._epoch - 1991.25,
'3':
self.parallax_factors,
'4':
self._cpsi,
'5':
self._spsi,
'6':
np.round(self.residuals.values, 3),
'7':
np.round(self.along_scan_errs.values, 3)
})
self.recalibrated_data = recalibrated_data
header, raw_data = None, None # the raw header and raw data have been modified, so clear them.
return header, raw_data