def get_closest_stars(sds, star_catalog, chosen_record, resultdir, platesolved_file, starid): sd_dict = utils.get_localid_to_sd_dict(sds) chosen_star_sd = sd_dict[starid] neighbours = [] for neigh in range(2, 22): idx, d2d, _ = match_coordinates_sky( chosen_star_sd.coords, star_catalog, nthneighbor=neigh ) neighbours.append(sds[idx]) ucac4.add_sd_metadatas(neighbours) ucac4.add_sd_metadatas([chosen_star_sd]) update_img(chosen_star_sd, chosen_record, neighbours, resultdir, platesolved_file)
def write_candidate_stars(resultdir, stars: List[StarDescription]): candidates = utils.get_stars_with_metadata(stars, "CANDIDATES", exclude=["VSX"]) newname = f"{resultdir}candidate_stars.txt" logging.info(f"Writing {newname}...") total_found = 0 stardict = utils.get_localid_to_sd_dict(stars) logging.debug(f"Receiving {len(stardict.keys())} as vsx input") with open(newname, "wt") as fp: for index, current_sd in enumerate(candidates): found = False if current_sd.path == "" else True total_found += 1 if found else 0 fp.write(f"{current_sd.local_id},,CANDIDATE-{index},,\n")
def test_get_calculated_compstars(self): stars = [ self.stardesc(1, 1, 1, 10, 0.01, 10), self.stardesc(4283, 3, 3, 12, 0.02, 10), self.stardesc(132, 10.24496, 9.96736, 13, 0.02, 10), self.stardesc(2, 10.24490, 9.96730, 12, 0.01, 10), self.stardesc(3, 10.24490, 9.96730, 12, 0.01, 10), ] # not there stardict = utils.get_localid_to_sd_dict(stars) # def get_calculated_compstars(vastdir, stardict: StarDict, ref_jd, maglimit=15, starlimit=1000): ids, sds = do_compstars.get_calculated_compstars( test_file_path, stardict, ref_jd=1 ) self.assertEqual(4, len(ids))
def test_tag_selected(self): # 3154, 515, 2269, 2635 stars = [ self.stardesc(515, 12.7725921, 12.0036631), self.stardesc(2269, 12.2686600, 12.0382637), self.stardesc(3154, 12.1520503, 12.0935881), self.stardesc(2635, 10.24490, 9.96730), self.stardesc(1, 10, 9), ] main_vast.read_and_tag_localid( PurePath(test_file_path, "wwcra2015_starlist.txt"), utils.get_localid_to_sd_dict(stars), ) test = utils.get_stars_with_metadata(stars, "SITE") self.assertEqual(4, len(test)) test = utils.get_stars_with_metadata(stars, "SELECTEDTAG") self.assertEqual(4, len(test))
def add_vsx_names_to_star_descriptions( star_descriptions: List[StarDescription], vsxcatalogdir: str, max_separation=0.01): result_ids = [] # copy.deepcopy(star_descriptions) vsx_catalog, vsx_dict = create_vsx_astropy_catalog(vsxcatalogdir) star_catalog = create_star_descriptions_catalog(star_descriptions) # vsx catalog is bigger in this case than star_catalog, but if we switch then different stars can be # matched with the same variable which is wrong. # # idx : integer array # Indices into catalogcoord to get the matched points for each matchcoord. Shape matches matchcoord. # # sep2d : Angle # The on-sky separation between the closest match for each matchcoord and the matchcoord. Shape matches matchcoord. idx, d2d, _ = match_coordinates_sky(star_catalog, vsx_catalog) logging.debug(f"length of idx: {len(idx)}") results_dict = {} # have temp results_dict so we can remove duplicates VsxInfo = namedtuple("VsxInfo", "starid_0 vsx_id sep") for index_star_catalog, entry in enumerate(d2d): if entry.value < max_separation: index_vsx = idx[index_star_catalog] # if it's a new vsx star or a better match than the last match, write into dict if (index_vsx not in results_dict or results_dict[index_vsx].sep > entry.value): star_id = star_descriptions[index_star_catalog].local_id results_dict[index_vsx] = VsxInfo(star_id, index_vsx, entry.value) logging.debug( f"Adding {results_dict[index_vsx]} to VSX results") cachedict = utils.get_localid_to_sd_dict(star_descriptions) # loop over dict and add the new vsx matches to the star descriptions for keys, vsxinfo in results_dict.items(): logging.debug( f"len sd is {len(star_descriptions)}, vsxinfo.starid is {vsxinfo.starid_0}" ) _add_vsx_metadata_to_star_description("VSX", cachedict[vsxinfo.starid_0], vsx_dict, vsxinfo.vsx_id, vsxinfo.sep) result_ids.append(vsxinfo.starid_0) logging.debug(f"Added {len(results_dict)} vsx stars.") return star_descriptions, result_ids
def write_vsx_stars(resultdir, results_ids, stars: List[StarDescription]): newname = f"{resultdir}vsx_stars.txt" selected_file = f"{resultdir}vsx_stars_selected.txt" logging.info(f"Writing {newname}...") total_found = 0 stardict = utils.get_localid_to_sd_dict(stars) logging.debug(f"Receiving {len(stardict.keys())} as vsx input") with open(newname, "wt") as fp, open(selected_file, "wt") as selected: for number, vsx_id in enumerate(results_ids): current_sd = stardict[vsx_id] found = False if current_sd.path == "" else True assert vsx_id == current_sd.local_id total_found += 1 if found else 0 fp.write( f"{vsx_id}{'' if found else '*'}:\t{current_sd.aavso_id}\t{utils.get_lesve_coords(current_sd.coords)}\n" ) selected.write(f"{vsx_id},,VSX-{number},,\n") fp.write( f"# Total entries: {len(results_ids)}, found: {total_found}, not found: {len(results_ids) - total_found}\n" )
def construct_star_descriptions(vastdir: str, resultdir: str, wcs: WCS, args): star_descriptions = utils_sd.construct_raw_star_descriptions( vastdir, wcs, None, STAR_KEEPER_PERCENTAGE) logging.info( f"Number of stars on more than {STAR_KEEPER_PERCENTAGE:.0%} of frames: {len(star_descriptions)}" ) stardict = get_localid_to_sd_dict(star_descriptions) # Add VSX information to SDs star_descriptions, results_ids = do_calibration.add_vsx_names_to_star_descriptions( star_descriptions, vsxcatalogdir, 0.01) logging.debug(f"Identified {len(results_ids)} VSX stars") vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX") assert len(vsx_stars) == len(results_ids) logging.debug(f"Test Tagged {len(vsx_stars)} stars as VSX.") # write the vsx stars used into a file results_ids.sort() write_vsx_stars(resultdir, results_ids, star_descriptions) # tag all candidates with a 'candidate' catalog tag_candidates(vastdir, star_descriptions) write_candidate_stars(resultdir, star_descriptions) # adds sitedata to vsx stars if args.selectvsx: tag_vsx_as_selected(vsx_stars) # adds sitedata to selected stars if args.localidcatalog: read_and_tag_localid(args.localidcatalog, stardict) logging.debug( f"Succesfully read {len(list(filter(lambda x: x.has_metadata('SELECTEDFILE'), star_descriptions)))} " f"stars from file:" f" {[x.local_id for x in list(filter(lambda x: x.has_metadata('SELECTEDFILE'), star_descriptions))]}" ) if args.radeccatalog: read_and_tag_radec(args.radeccatalog, star_descriptions) return star_descriptions
def main(vastdir, star): ucac4 = UCAC4() # get UCAC4 sd for search later ucac_details = ucac4.get_ucactuple_from_id(utils.get_full_ucac4_id(star)) ucacsd = ucac4.get_star_description_from_tuple(ucac_details) logging.info(f"Found UCAC4: {ucacsd}") # construct star descriptions sds = utils_sd.construct_star_descriptions(vastdir, None) star_catalog = do_calibration.create_star_descriptions_catalog(sds) # Get the 10 closest neighbours sd_dict = utils.get_localid_to_sd_dict(sds) neighbours = [] for neigh in range(1, 22): idx, d2d, _ = match_coordinates_sky(ucacsd.coords, star_catalog, nthneighbor=neigh) neighbours.append(sds[idx]) ucac4.add_sd_metadatas(neighbours) logging.info("\n" + "\n".join([f"Star {x.local_id}: {x}" for x in neighbours]))
def run_do_rest(args): thread_count = args.threads vastdir = utils.add_trailing_slash(args.datadir) resultdir = clean_and_create_resultdir(args.resultdir, vastdir) do_light = args.light do_phase = args.phase do_aavso = args.aavso logging.info( f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'") # get wcs model from the reference header. Used in writing world positions and field charts (can fail) wcs_file, wcs = reading.read_wcs_file(vastdir) ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir) _, _, _, first_frame = reading.extract_first_frame(vastdir) reference_frame_path = Path(reference_frame) reference_frame_filename = reference_frame_path.name wcs_file, wcs = perform_astrometry_net(args, vastdir, wcs_file, wcs, reference_frame_filename) logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}") logging.info(f"The first frame is '{first_frame}'") logging.info(f"Reference header is '{wcs_file}'") check_that_reference_image_not_within_jdfilter(ref_jd, args.jdfilter, args.jdrefignore) star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs, args) stardict = get_localid_to_sd_dict(star_descriptions) logging.debug( f"First (max) 10 star descriptions: " f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}" ) write_augmented_autocandidates(vastdir, resultdir, stardict) write_augmented_all_stars(vastdir, resultdir, stardict) candidate_stars = utils.get_stars_with_metadata(star_descriptions, "CANDIDATE", exclude=["VSX"]) if args.selectcandidates: tag_candidates_as_selected(candidate_stars) logging.info(f"There are {len(candidate_stars)} candidate stars") vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX") logging.info(f"There are {len(vsx_stars)} vsx stars") if args.allstars: tag_all_stars_as_selected(star_descriptions) selected_stars = utils.get_stars_with_metadata(star_descriptions, "SELECTEDTAG") logging.info(f"There are {len(selected_stars)} selected stars") compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars, candidate_stars) comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars, args.checkstarfile, vastdir, stardict, ref_jd) # Set comp stars for all interesting stars (stars which are interesting enough to measure) logging.info("Setting per star comparison stars...") if args.checkstarfile: utils.add_metadata(star_descriptions, CompStarData(compstar_ids=comp_stars.ids)) else: do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars, 10) logging.info( f"Using {thread_count} threads for phase plots, lightcurves, ...") if args.allstars: do_charts_vast.run( star_descriptions, comp_stars, vastdir, resultdir, "phase_all/", "light_all/", "aavso_all/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of ALL stars", ) else: if args.vsx: logging.info(f"Plotting {len(vsx_stars)} vsx stars...") do_charts_vast.run( vsx_stars, comp_stars, vastdir, resultdir, "phase_vsx/", "light_vsx/", "aavso_vsx/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of VSX stars", ) if args.radeccatalog or args.localidcatalog: do_charts_vast.run( selected_stars, comp_stars, vastdir, resultdir, "phase_selected/", "light_selected/", "aavso_selected", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of selected stars", ) if args.candidates: logging.info(f"Plotting {len(candidate_stars)} candidates...") do_charts_vast.run( candidate_stars, comp_stars, vastdir, resultdir, "phase_candidates/", "light_candidates/", "aavso_candidates/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of candidates", ) # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete ids = [x.local_id for x in selected_stars] logging.info( f"Writing selected files with {len(selected_stars)} selected stars: {ids}" ) write_selected_files(resultdir, vastdir, selected_stars) # only create fieldcharts dir if we use it fieldchartsdir = resultdir + "fieldcharts/" if args.field or args.stats: trash_and_recreate_dir(fieldchartsdir) if args.field: do_charts_field.run_standard_field_charts(star_descriptions, wcs, fieldchartsdir, wcs_file, comp_stars) if args.stats: do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars, stardict, args.jdfilter) do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir, args.jdfilter) do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs, args.jdfilter) do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars, args.jdfilter) if args.site: ids = [x.local_id for x in selected_stars] logging.info( f"Creating HTML site with {len(selected_stars)} selected stars: {ids}" ) hugo_site.run(args.site, selected_stars, len(vsx_stars), reference_frame_path, resultdir, args.explore)
def run_do_rest(args): thread_count = args.threads vastdir = utils.add_trailing_slash(args.datadir) resultdir = clean_and_create_resultdir(args.resultdir, vastdir) fieldchartsdir = resultdir + "fieldcharts/" do_light = args.light do_phase = args.phase do_aavso = args.aavso logging.info( f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'") # get wcs model from the reference header. Used in writing world positions and field charts (can fail) wcs_file, wcs = reading.read_wcs_file(vastdir) ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir) _, _, _, first_frame = reading.extract_first_frame(vastdir) referene_frame_path = Path(reference_frame) reference_frame_filename = referene_frame_path.name logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}") logging.info(f"The first frame is '{first_frame}'") logging.info(f"Reference header is '{wcs_file}'") # Log filtering settings + check that reference frame is not inside of filter if args.jdfilter: logging.info(f"Filtering JD's: {args.jdfilter}") ref_inside_filter = (float(ref_jd) > args.jdfilter[0] and float(ref_jd) > args.jdfilter[1]) if ref_inside_filter: if not args.jdrefignore: assert not ref_inside_filter, "Reference frame JD is filtered" else: logging.info( "Reference frame JD is inside of the JD filter, but you indicated that's ok." ) if not os.path.isfile(wcs_file): full_ref_path = Path(args.fitsdir) / reference_frame_filename if not args.fitsdir and args.apikey: logging.error( "There is no plate-solved reference frame {wcs_file}, please specify both --apikey " "and --fitsdir.") sys.exit(0) rotation = reading.extract_reference_frame_rotation( vastdir, reference_frame_filename) assert (rotation == 0.0 ), f"Error: rotation is {rotation} and should always be 0.0" subprocess.Popen( f"python3 ./src/astrometry_api.py --apikey={args.apikey} " f"--upload={full_ref_path} --newfits={wcs_file} --private --no_commercial", shell=True, ) while not os.path.isfile(wcs_file): logging.info(f"Waiting for the astrometry.net plate solve...") time.sleep(10) wcs_file, wcs = reading.read_wcs_file(vastdir) star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs, args) stardict = get_localid_to_sd_dict(star_descriptions) logging.debug( f"First (max) 10 star descriptions: " f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}" ) write_augmented_autocandidates(vastdir, resultdir, stardict) write_augmented_all_stars(vastdir, resultdir, stardict) candidate_stars = utils.get_stars_with_metadata(star_descriptions, "CANDIDATE", exclude=["VSX"]) if args.selectcandidates: tag_candidates_as_selected(candidate_stars) logging.info(f"There are {len(candidate_stars)} candidate stars") vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX") logging.info(f"There are {len(vsx_stars)} vsx stars") selected_stars = utils.get_stars_with_metadata(star_descriptions, "SELECTEDTAG") # if args.selectvsx: # selected_stars = utils.concat_sd_lists(selected_stars, vsx_stars) logging.info(f"There are {len(selected_stars)} selected stars") compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars, candidate_stars) comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars, args.checkstarfile, vastdir, stardict, ref_jd) # Set comp stars for all interesting stars (stars which are interesting enough to measure) logging.info("Setting per star comparison stars...") if args.checkstarfile: utils.add_metadata(star_descriptions, CompStarData(compstar_ids=comp_stars.ids)) else: do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars, 10) logging.info( f"Using {thread_count} threads for phase plots, lightcurves, ...") if args.allstars: do_charts_vast.run( star_descriptions, comp_stars, vastdir, resultdir, "phase_all/", "light_all/", "aavso_all/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of ALL stars", ) else: if args.vsx: logging.info(f"Plotting {len(vsx_stars)} vsx stars...") do_charts_vast.run( vsx_stars, comp_stars, vastdir, resultdir, "phase_vsx/", "light_vsx/", "aavso_vsx/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of VSX stars", ) if args.radeccatalog or args.localidcatalog: do_charts_vast.run( selected_stars, comp_stars, vastdir, resultdir, "phase_selected/", "light_selected/", "aavso_selected", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of selected stars", ) if args.candidates: logging.info(f"Plotting {len(candidate_stars)} candidates...") do_charts_vast.run( candidate_stars, comp_stars, vastdir, resultdir, "phase_candidates/", "light_candidates/", "aavso_candidates/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of candidates", ) # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete ids = [x.local_id for x in selected_stars] logging.info( f"Writing selected files with {len(selected_stars)} selected stars: {ids}" ) write_selected_files(resultdir, vastdir, selected_stars) if args.field: do_charts_field.run_standard_field_charts(star_descriptions, wcs, fieldchartsdir, wcs_file, comp_stars) if args.stats: do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars, stardict, args.jdfilter) do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir, args.jdfilter) do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs, args.jdfilter) do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars, args.jdfilter) if args.site: ids = [x.local_id for x in selected_stars] logging.info( f"Creating HTML site with {len(selected_stars)} selected stars: {ids}" ) hugo_site.run(args.site, selected_stars, len(vsx_stars), referene_frame_path, resultdir, args.explore)