def get_closest_stars(sds, star_catalog, chosen_record, resultdir, platesolved_file, starid):
sd_dict = utils.get_localid_to_sd_dict(sds)
chosen_star_sd = sd_dict[starid]
neighbours = []
for neigh in range(2, 22):
idx, d2d, _ = match_coordinates_sky(
chosen_star_sd.coords, star_catalog, nthneighbor=neigh
)
neighbours.append(sds[idx])
ucac4.add_sd_metadatas(neighbours)
ucac4.add_sd_metadatas([chosen_star_sd])
update_img(chosen_star_sd, chosen_record, neighbours, resultdir, platesolved_file)
def write_candidate_stars(resultdir, stars: List[StarDescription]):
candidates = utils.get_stars_with_metadata(stars,
"CANDIDATES",
exclude=["VSX"])
newname = f"{resultdir}candidate_stars.txt"
logging.info(f"Writing {newname}...")
total_found = 0
stardict = utils.get_localid_to_sd_dict(stars)
logging.debug(f"Receiving {len(stardict.keys())} as vsx input")
with open(newname, "wt") as fp:
for index, current_sd in enumerate(candidates):
found = False if current_sd.path == "" else True
total_found += 1 if found else 0
fp.write(f"{current_sd.local_id},,CANDIDATE-{index},,\n")
def test_get_calculated_compstars(self):
stars = [
self.stardesc(1, 1, 1, 10, 0.01, 10),
self.stardesc(4283, 3, 3, 12, 0.02, 10),
self.stardesc(132, 10.24496, 9.96736, 13, 0.02, 10),
self.stardesc(2, 10.24490, 9.96730, 12, 0.01, 10),
self.stardesc(3, 10.24490, 9.96730, 12, 0.01, 10),
] # not there
stardict = utils.get_localid_to_sd_dict(stars)
# def get_calculated_compstars(vastdir, stardict: StarDict, ref_jd, maglimit=15, starlimit=1000):
ids, sds = do_compstars.get_calculated_compstars(
test_file_path, stardict, ref_jd=1
)
self.assertEqual(4, len(ids))
def test_tag_selected(self):
# 3154, 515, 2269, 2635
stars = [
self.stardesc(515, 12.7725921, 12.0036631),
self.stardesc(2269, 12.2686600, 12.0382637),
self.stardesc(3154, 12.1520503, 12.0935881),
self.stardesc(2635, 10.24490, 9.96730),
self.stardesc(1, 10, 9),
]
main_vast.read_and_tag_localid(
PurePath(test_file_path, "wwcra2015_starlist.txt"),
utils.get_localid_to_sd_dict(stars),
)
test = utils.get_stars_with_metadata(stars, "SITE")
self.assertEqual(4, len(test))
test = utils.get_stars_with_metadata(stars, "SELECTEDTAG")
self.assertEqual(4, len(test))
def add_vsx_names_to_star_descriptions(
star_descriptions: List[StarDescription],
vsxcatalogdir: str,
max_separation=0.01):
result_ids = []
# copy.deepcopy(star_descriptions)
vsx_catalog, vsx_dict = create_vsx_astropy_catalog(vsxcatalogdir)
star_catalog = create_star_descriptions_catalog(star_descriptions)
# vsx catalog is bigger in this case than star_catalog, but if we switch then different stars can be
# matched with the same variable which is wrong.
#
# idx : integer array
# Indices into catalogcoord to get the matched points for each matchcoord. Shape matches matchcoord.
#
# sep2d : Angle
# The on-sky separation between the closest match for each matchcoord and the matchcoord. Shape matches matchcoord.
idx, d2d, _ = match_coordinates_sky(star_catalog, vsx_catalog)
logging.debug(f"length of idx: {len(idx)}")
results_dict = {} # have temp results_dict so we can remove duplicates
VsxInfo = namedtuple("VsxInfo", "starid_0 vsx_id sep")
for index_star_catalog, entry in enumerate(d2d):
if entry.value < max_separation:
index_vsx = idx[index_star_catalog]
# if it's a new vsx star or a better match than the last match, write into dict
if (index_vsx not in results_dict
or results_dict[index_vsx].sep > entry.value):
star_id = star_descriptions[index_star_catalog].local_id
results_dict[index_vsx] = VsxInfo(star_id, index_vsx,
entry.value)
logging.debug(
f"Adding {results_dict[index_vsx]} to VSX results")
cachedict = utils.get_localid_to_sd_dict(star_descriptions)
# loop over dict and add the new vsx matches to the star descriptions
for keys, vsxinfo in results_dict.items():
logging.debug(
f"len sd is {len(star_descriptions)}, vsxinfo.starid is {vsxinfo.starid_0}"
)
_add_vsx_metadata_to_star_description("VSX",
cachedict[vsxinfo.starid_0],
vsx_dict, vsxinfo.vsx_id,
vsxinfo.sep)
result_ids.append(vsxinfo.starid_0)
logging.debug(f"Added {len(results_dict)} vsx stars.")
return star_descriptions, result_ids
def write_vsx_stars(resultdir, results_ids, stars: List[StarDescription]):
newname = f"{resultdir}vsx_stars.txt"
selected_file = f"{resultdir}vsx_stars_selected.txt"
logging.info(f"Writing {newname}...")
total_found = 0
stardict = utils.get_localid_to_sd_dict(stars)
logging.debug(f"Receiving {len(stardict.keys())} as vsx input")
with open(newname, "wt") as fp, open(selected_file, "wt") as selected:
for number, vsx_id in enumerate(results_ids):
current_sd = stardict[vsx_id]
found = False if current_sd.path == "" else True
assert vsx_id == current_sd.local_id
total_found += 1 if found else 0
fp.write(
f"{vsx_id}{'' if found else '*'}:\t{current_sd.aavso_id}\t{utils.get_lesve_coords(current_sd.coords)}\n"
)
selected.write(f"{vsx_id},,VSX-{number},,\n")
fp.write(
f"# Total entries: {len(results_ids)}, found: {total_found}, not found: {len(results_ids) - total_found}\n"
)
def construct_star_descriptions(vastdir: str, resultdir: str, wcs: WCS, args):
star_descriptions = utils_sd.construct_raw_star_descriptions(
vastdir, wcs, None, STAR_KEEPER_PERCENTAGE)
logging.info(
f"Number of stars on more than {STAR_KEEPER_PERCENTAGE:.0%} of frames: {len(star_descriptions)}"
)
stardict = get_localid_to_sd_dict(star_descriptions)
# Add VSX information to SDs
star_descriptions, results_ids = do_calibration.add_vsx_names_to_star_descriptions(
star_descriptions, vsxcatalogdir, 0.01)
logging.debug(f"Identified {len(results_ids)} VSX stars")
vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
assert len(vsx_stars) == len(results_ids)
logging.debug(f"Test Tagged {len(vsx_stars)} stars as VSX.")
# write the vsx stars used into a file
results_ids.sort()
write_vsx_stars(resultdir, results_ids, star_descriptions)
# tag all candidates with a 'candidate' catalog
tag_candidates(vastdir, star_descriptions)
write_candidate_stars(resultdir, star_descriptions)
# adds sitedata to vsx stars
if args.selectvsx:
tag_vsx_as_selected(vsx_stars)
# adds sitedata to selected stars
if args.localidcatalog:
read_and_tag_localid(args.localidcatalog, stardict)
logging.debug(
f"Succesfully read {len(list(filter(lambda x: x.has_metadata('SELECTEDFILE'), star_descriptions)))} "
f"stars from file:"
f" {[x.local_id for x in list(filter(lambda x: x.has_metadata('SELECTEDFILE'), star_descriptions))]}"
)
if args.radeccatalog:
read_and_tag_radec(args.radeccatalog, star_descriptions)
return star_descriptions
def main(vastdir, star):
ucac4 = UCAC4()
# get UCAC4 sd for search later
ucac_details = ucac4.get_ucactuple_from_id(utils.get_full_ucac4_id(star))
ucacsd = ucac4.get_star_description_from_tuple(ucac_details)
logging.info(f"Found UCAC4: {ucacsd}")
# construct star descriptions
sds = utils_sd.construct_star_descriptions(vastdir, None)
star_catalog = do_calibration.create_star_descriptions_catalog(sds)
# Get the 10 closest neighbours
sd_dict = utils.get_localid_to_sd_dict(sds)
neighbours = []
for neigh in range(1, 22):
idx, d2d, _ = match_coordinates_sky(ucacsd.coords,
star_catalog,
nthneighbor=neigh)
neighbours.append(sds[idx])
ucac4.add_sd_metadatas(neighbours)
logging.info("\n" +
"\n".join([f"Star {x.local_id}: {x}" for x in neighbours]))
def run_do_rest(args):
thread_count = args.threads
vastdir = utils.add_trailing_slash(args.datadir)
resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
do_light = args.light
do_phase = args.phase
do_aavso = args.aavso
logging.info(
f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
# get wcs model from the reference header. Used in writing world positions and field charts (can fail)
wcs_file, wcs = reading.read_wcs_file(vastdir)
ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
_, _, _, first_frame = reading.extract_first_frame(vastdir)
reference_frame_path = Path(reference_frame)
reference_frame_filename = reference_frame_path.name
wcs_file, wcs = perform_astrometry_net(args, vastdir, wcs_file, wcs,
reference_frame_filename)
logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
logging.info(f"The first frame is '{first_frame}'")
logging.info(f"Reference header is '{wcs_file}'")
check_that_reference_image_not_within_jdfilter(ref_jd, args.jdfilter,
args.jdrefignore)
star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
args)
stardict = get_localid_to_sd_dict(star_descriptions)
logging.debug(
f"First (max) 10 star descriptions: "
f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
)
write_augmented_autocandidates(vastdir, resultdir, stardict)
write_augmented_all_stars(vastdir, resultdir, stardict)
candidate_stars = utils.get_stars_with_metadata(star_descriptions,
"CANDIDATE",
exclude=["VSX"])
if args.selectcandidates:
tag_candidates_as_selected(candidate_stars)
logging.info(f"There are {len(candidate_stars)} candidate stars")
vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
logging.info(f"There are {len(vsx_stars)} vsx stars")
if args.allstars:
tag_all_stars_as_selected(star_descriptions)
selected_stars = utils.get_stars_with_metadata(star_descriptions,
"SELECTEDTAG")
logging.info(f"There are {len(selected_stars)} selected stars")
compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
candidate_stars)
comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
args.checkstarfile, vastdir,
stardict, ref_jd)
# Set comp stars for all interesting stars (stars which are interesting enough to measure)
logging.info("Setting per star comparison stars...")
if args.checkstarfile:
utils.add_metadata(star_descriptions,
CompStarData(compstar_ids=comp_stars.ids))
else:
do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
10)
logging.info(
f"Using {thread_count} threads for phase plots, lightcurves, ...")
if args.allstars:
do_charts_vast.run(
star_descriptions,
comp_stars,
vastdir,
resultdir,
"phase_all/",
"light_all/",
"aavso_all/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of ALL stars",
)
else:
if args.vsx:
logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
do_charts_vast.run(
vsx_stars,
comp_stars,
vastdir,
resultdir,
"phase_vsx/",
"light_vsx/",
"aavso_vsx/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of VSX stars",
)
if args.radeccatalog or args.localidcatalog:
do_charts_vast.run(
selected_stars,
comp_stars,
vastdir,
resultdir,
"phase_selected/",
"light_selected/",
"aavso_selected",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of selected stars",
)
if args.candidates:
logging.info(f"Plotting {len(candidate_stars)} candidates...")
do_charts_vast.run(
candidate_stars,
comp_stars,
vastdir,
resultdir,
"phase_candidates/",
"light_candidates/",
"aavso_candidates/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of candidates",
)
# starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
ids = [x.local_id for x in selected_stars]
logging.info(
f"Writing selected files with {len(selected_stars)} selected stars: {ids}"
)
write_selected_files(resultdir, vastdir, selected_stars)
# only create fieldcharts dir if we use it
fieldchartsdir = resultdir + "fieldcharts/"
if args.field or args.stats:
trash_and_recreate_dir(fieldchartsdir)
if args.field:
do_charts_field.run_standard_field_charts(star_descriptions, wcs,
fieldchartsdir, wcs_file,
comp_stars)
if args.stats:
do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
stardict, args.jdfilter)
do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
args.jdfilter)
do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
args.jdfilter)
do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
args.jdfilter)
if args.site:
ids = [x.local_id for x in selected_stars]
logging.info(
f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
)
hugo_site.run(args.site, selected_stars, len(vsx_stars),
reference_frame_path, resultdir, args.explore)
def run_do_rest(args):
thread_count = args.threads
vastdir = utils.add_trailing_slash(args.datadir)
resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
fieldchartsdir = resultdir + "fieldcharts/"
do_light = args.light
do_phase = args.phase
do_aavso = args.aavso
logging.info(
f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
# get wcs model from the reference header. Used in writing world positions and field charts (can fail)
wcs_file, wcs = reading.read_wcs_file(vastdir)
ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
_, _, _, first_frame = reading.extract_first_frame(vastdir)
referene_frame_path = Path(reference_frame)
reference_frame_filename = referene_frame_path.name
logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
logging.info(f"The first frame is '{first_frame}'")
logging.info(f"Reference header is '{wcs_file}'")
# Log filtering settings + check that reference frame is not inside of filter
if args.jdfilter:
logging.info(f"Filtering JD's: {args.jdfilter}")
ref_inside_filter = (float(ref_jd) > args.jdfilter[0]
and float(ref_jd) > args.jdfilter[1])
if ref_inside_filter:
if not args.jdrefignore:
assert not ref_inside_filter, "Reference frame JD is filtered"
else:
logging.info(
"Reference frame JD is inside of the JD filter, but you indicated that's ok."
)
if not os.path.isfile(wcs_file):
full_ref_path = Path(args.fitsdir) / reference_frame_filename
if not args.fitsdir and args.apikey:
logging.error(
"There is no plate-solved reference frame {wcs_file}, please specify both --apikey "
"and --fitsdir.")
sys.exit(0)
rotation = reading.extract_reference_frame_rotation(
vastdir, reference_frame_filename)
assert (rotation == 0.0
), f"Error: rotation is {rotation} and should always be 0.0"
subprocess.Popen(
f"python3 ./src/astrometry_api.py --apikey={args.apikey} "
f"--upload={full_ref_path} --newfits={wcs_file} --private --no_commercial",
shell=True,
)
while not os.path.isfile(wcs_file):
logging.info(f"Waiting for the astrometry.net plate solve...")
time.sleep(10)
wcs_file, wcs = reading.read_wcs_file(vastdir)
star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
args)
stardict = get_localid_to_sd_dict(star_descriptions)
logging.debug(
f"First (max) 10 star descriptions: "
f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
)
write_augmented_autocandidates(vastdir, resultdir, stardict)
write_augmented_all_stars(vastdir, resultdir, stardict)
candidate_stars = utils.get_stars_with_metadata(star_descriptions,
"CANDIDATE",
exclude=["VSX"])
if args.selectcandidates:
tag_candidates_as_selected(candidate_stars)
logging.info(f"There are {len(candidate_stars)} candidate stars")
vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
logging.info(f"There are {len(vsx_stars)} vsx stars")
selected_stars = utils.get_stars_with_metadata(star_descriptions,
"SELECTEDTAG")
# if args.selectvsx:
# selected_stars = utils.concat_sd_lists(selected_stars, vsx_stars)
logging.info(f"There are {len(selected_stars)} selected stars")
compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
candidate_stars)
comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
args.checkstarfile, vastdir,
stardict, ref_jd)
# Set comp stars for all interesting stars (stars which are interesting enough to measure)
logging.info("Setting per star comparison stars...")
if args.checkstarfile:
utils.add_metadata(star_descriptions,
CompStarData(compstar_ids=comp_stars.ids))
else:
do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
10)
logging.info(
f"Using {thread_count} threads for phase plots, lightcurves, ...")
if args.allstars:
do_charts_vast.run(
star_descriptions,
comp_stars,
vastdir,
resultdir,
"phase_all/",
"light_all/",
"aavso_all/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of ALL stars",
)
else:
if args.vsx:
logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
do_charts_vast.run(
vsx_stars,
comp_stars,
vastdir,
resultdir,
"phase_vsx/",
"light_vsx/",
"aavso_vsx/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of VSX stars",
)
if args.radeccatalog or args.localidcatalog:
do_charts_vast.run(
selected_stars,
comp_stars,
vastdir,
resultdir,
"phase_selected/",
"light_selected/",
"aavso_selected",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of selected stars",
)
if args.candidates:
logging.info(f"Plotting {len(candidate_stars)} candidates...")
do_charts_vast.run(
candidate_stars,
comp_stars,
vastdir,
resultdir,
"phase_candidates/",
"light_candidates/",
"aavso_candidates/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of candidates",
)
# starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
ids = [x.local_id for x in selected_stars]
logging.info(
f"Writing selected files with {len(selected_stars)} selected stars: {ids}"
)
write_selected_files(resultdir, vastdir, selected_stars)
if args.field:
do_charts_field.run_standard_field_charts(star_descriptions, wcs,
fieldchartsdir, wcs_file,
comp_stars)
if args.stats:
do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
stardict, args.jdfilter)
do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
args.jdfilter)
do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
args.jdfilter)
do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
args.jdfilter)
if args.site:
ids = [x.local_id for x in selected_stars]
logging.info(
f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
)
hugo_site.run(args.site, selected_stars, len(vsx_stars),
referene_frame_path, resultdir, args.explore)