def construct_star_descriptions(vastdir: str, stars: List[int] = None): vastdir = utils.add_trailing_slash(vastdir) wcs_file, wcs = reading.read_wcs_file(vastdir) all_files = utils.file_selector(the_dir=vastdir, match_pattern="*.dat") if stars is not None: all_files = [vastdir + reading.star_to_dat(int(x)) for x in stars] sds = construct_raw_star_descriptions(vastdir, wcs, all_files) return sds
def clean_and_create_resultdir(argsdir: str, vastdir: str): resultdir = utils.add_trailing_slash( argsdir) if argsdir is not None else vastdir # if resultdir does not exist, create it if not os.path.isdir(resultdir): logging.info( f"The resultdir '{resultdir}' does not exist, creating it...") reading.create_dir(resultdir) return resultdir
def meta_init(datadir: str): # make sure datadir has trailing slash datadir = utils.add_trailing_slash(datadir) global settings settings = Settings(datadir) global init # init = importlib.import_module(module_name, package=None) # print("datadir is", datadir+'init.py') init = imp.load_source('init', datadir+'init.py')
def home(): """Displays the FotoJazz home page.""" filebrowse_path = '' if len(sys.argv) > 1: filebrowse_path = add_trailing_slash(sys.argv[1]) # The filebrowse area has to be rendered by a separate function, # becase it's refreshed via AJAX. filebrowse_files_rendered = photos(filebrowse_path, check_all=True) return render_template('home.html', filebrowse_path=filebrowse_path, filebrowse_files_rendered=filebrowse_files_rendered)
def home(): """Displays the FotoJazz home page.""" filebrowse_path = '' if len(sys.argv) > 1: filebrowse_path = add_trailing_slash(sys.argv[1]) # The filebrowse area has to be rendered by a separate function, # becase it's refreshed via AJAX. filebrowse_files_rendered = photos(filebrowse_path, check_all=True) return render_template('home.html', filebrowse_path=filebrowse_path, filebrowse_files_rendered=filebrowse_files_rendered)
def photos(photos_path='', check_all=False): """Renders the photo browsing block. This view is often called by AJAX, as this block gets refreshed dynamically. This view expects the following GET params: - filenames_input: space-delimited string of absolute paths to the photos that should be rendered with their checkbox checked by default. - check_all: boolean indicating if all photos should be rendered with their checkbox checked by default. If true, filenames_input is ignored. - photos_path: absolute path to the directory from which to load photos.""" filenames_str = request.args.get('filenames_input', '', type=str) filenames_input = [] if filenames_str != '': filenames_input = [x.strip() for x in filenames_str.split(' ') if x != ''] if len(filenames_input): filenames_input.sort() check_all_arg = request.args.get('check_all', '', type=str) if check_all_arg != '': check_all = int(check_all_arg) and True or False filebrowse_files = [] filebrowse_path = '' filebrowse_error = '' if not photos_path: photos_path = request.args.get('photos_path', '', type=str) if not photos_path: filebrowse_error = 'No path specified.' else: filebrowse_path = add_trailing_slash(photos_path) thumb_resize_width = app.config['THUMB_RESIZE_WIDTH'] thumb_resize_height = app.config['THUMB_RESIZE_HEIGHT'] if not path.isdir(filebrowse_path): filebrowse_error = 'Specified path is not a valid directory.' else: glob_pattern = '%s%s' % (filebrowse_path, '*.[jJ][pP]*[gG]') filenames = glob(glob_pattern) filebrowse_files = [get_thumb_metadata( filename, thumb_resize_width, thumb_resize_height, checked=check_all or filename in filenames_input) for filename in filenames] if not filebrowse_files: filebrowse_error = 'No images in specified directory.' else: filebrowse_files.sort( key=itemgetter('date_taken_timestamp')) # Need to add timestamp to thumbnail img src's, as a unique url # value to ensure fresh thumbs get shown on ajax refresh. Doesn't # actually need to be passed as a get param to '/thumbs/' (although # it does get passed), not used for anything else. timestamp = int(time.time()) return render_template('fragments/photos.html', filebrowse_files=filebrowse_files, filenames_input=filenames_input, filebrowse_error=filebrowse_error, thumb_resize_width=thumb_resize_width, thumb_resize_height=thumb_resize_height, timestamp=timestamp)
def run_do_rest(args): thread_count = args.threads vastdir = utils.add_trailing_slash(args.datadir) resultdir = clean_and_create_resultdir(args.resultdir, vastdir) do_light = args.light do_phase = args.phase do_aavso = args.aavso logging.info( f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'") # get wcs model from the reference header. Used in writing world positions and field charts (can fail) wcs_file, wcs = reading.read_wcs_file(vastdir) ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir) _, _, _, first_frame = reading.extract_first_frame(vastdir) reference_frame_path = Path(reference_frame) reference_frame_filename = reference_frame_path.name wcs_file, wcs = perform_astrometry_net(args, vastdir, wcs_file, wcs, reference_frame_filename) logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}") logging.info(f"The first frame is '{first_frame}'") logging.info(f"Reference header is '{wcs_file}'") check_that_reference_image_not_within_jdfilter(ref_jd, args.jdfilter, args.jdrefignore) star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs, args) stardict = get_localid_to_sd_dict(star_descriptions) logging.debug( f"First (max) 10 star descriptions: " f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}" ) write_augmented_autocandidates(vastdir, resultdir, stardict) write_augmented_all_stars(vastdir, resultdir, stardict) candidate_stars = utils.get_stars_with_metadata(star_descriptions, "CANDIDATE", exclude=["VSX"]) if args.selectcandidates: tag_candidates_as_selected(candidate_stars) logging.info(f"There are {len(candidate_stars)} candidate stars") vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX") logging.info(f"There are {len(vsx_stars)} vsx stars") if args.allstars: tag_all_stars_as_selected(star_descriptions) selected_stars = utils.get_stars_with_metadata(star_descriptions, "SELECTEDTAG") logging.info(f"There are {len(selected_stars)} selected stars") compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars, candidate_stars) comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars, args.checkstarfile, vastdir, stardict, ref_jd) # Set comp stars for all interesting stars (stars which are interesting enough to measure) logging.info("Setting per star comparison stars...") if args.checkstarfile: utils.add_metadata(star_descriptions, CompStarData(compstar_ids=comp_stars.ids)) else: do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars, 10) logging.info( f"Using {thread_count} threads for phase plots, lightcurves, ...") if args.allstars: do_charts_vast.run( star_descriptions, comp_stars, vastdir, resultdir, "phase_all/", "light_all/", "aavso_all/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of ALL stars", ) else: if args.vsx: logging.info(f"Plotting {len(vsx_stars)} vsx stars...") do_charts_vast.run( vsx_stars, comp_stars, vastdir, resultdir, "phase_vsx/", "light_vsx/", "aavso_vsx/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of VSX stars", ) if args.radeccatalog or args.localidcatalog: do_charts_vast.run( selected_stars, comp_stars, vastdir, resultdir, "phase_selected/", "light_selected/", "aavso_selected", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of selected stars", ) if args.candidates: logging.info(f"Plotting {len(candidate_stars)} candidates...") do_charts_vast.run( candidate_stars, comp_stars, vastdir, resultdir, "phase_candidates/", "light_candidates/", "aavso_candidates/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of candidates", ) # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete ids = [x.local_id for x in selected_stars] logging.info( f"Writing selected files with {len(selected_stars)} selected stars: {ids}" ) write_selected_files(resultdir, vastdir, selected_stars) # only create fieldcharts dir if we use it fieldchartsdir = resultdir + "fieldcharts/" if args.field or args.stats: trash_and_recreate_dir(fieldchartsdir) if args.field: do_charts_field.run_standard_field_charts(star_descriptions, wcs, fieldchartsdir, wcs_file, comp_stars) if args.stats: do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars, stardict, args.jdfilter) do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir, args.jdfilter) do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs, args.jdfilter) do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars, args.jdfilter) if args.site: ids = [x.local_id for x in selected_stars] logging.info( f"Creating HTML site with {len(selected_stars)} selected stars: {ids}" ) hugo_site.run(args.site, selected_stars, len(vsx_stars), reference_frame_path, resultdir, args.explore)
help="Fps of the movie", nargs="?", required=False, default=30) # parser.add_argument('-s', '--start', # help="The start Julian Date", # nargs='?', required=True) # parser.add_argument('-e', '--end', # help="The end Julian Date", # nargs='?', required=True) parser.add_argument("-x", "--verbose", help="Set logging to debug mode", action="store_true") args = parser.parse_args() datadir = utils.add_trailing_slash(args.datadir) datenow = datetime.now() resultdir = Path(args.resultdir) filehandler = f"{datadir}vastlog-{datenow:%Y%M%d-%H_%M_%S}.log" fh = logging.FileHandler(filehandler) fh.setLevel(logging.INFO) # add the handlers to the logger logger.addHandler(fh) if args.verbose: logger.setLevel(logging.DEBUG) fh.setLevel(logging.DEBUG) assert os.path.exists(args.datadir), "datadir does not exist" # assert os.path.exists(args.resultdir), "resultdir does not exist" ==> this dir is created assert os.path.exists(args.resultdir), "resultdir does not exist" assert os.path.exists(args.fitsdir), "fitsdir does not exist"
def photos(photos_path='', check_all=False): """Renders the photo browsing block. This view is often called by AJAX, as this block gets refreshed dynamically. This view expects the following GET params: - filenames_input: space-delimited string of absolute paths to the photos that should be rendered with their checkbox checked by default. - check_all: boolean indicating if all photos should be rendered with their checkbox checked by default. If true, filenames_input is ignored. - photos_path: absolute path to the directory from which to load photos.""" filenames_str = request.args.get('filenames_input', '', type=str) filenames_input = [] if filenames_str != '': filenames_input = [ x.strip() for x in filenames_str.split(' ') if x != '' ] if len(filenames_input): filenames_input.sort() check_all_arg = request.args.get('check_all', '', type=str) if check_all_arg != '': check_all = int(check_all_arg) and True or False filebrowse_files = [] filebrowse_path = '' filebrowse_error = '' if not photos_path: photos_path = request.args.get('photos_path', '', type=str) if not photos_path: filebrowse_error = 'No path specified.' else: filebrowse_path = add_trailing_slash(photos_path) thumb_resize_width = app.config['THUMB_RESIZE_WIDTH'] thumb_resize_height = app.config['THUMB_RESIZE_HEIGHT'] if not path.isdir(filebrowse_path): filebrowse_error = 'Specified path is not a valid directory.' else: glob_pattern = '%s%s' % (filebrowse_path, '*.[jJ][pP]*[gG]') filenames = glob(glob_pattern) filebrowse_files = [ get_thumb_metadata(filename, thumb_resize_width, thumb_resize_height, checked=check_all or filename in filenames_input) for filename in filenames ] if not filebrowse_files: filebrowse_error = 'No images in specified directory.' else: filebrowse_files.sort(key=itemgetter('date_taken_timestamp')) # Need to add timestamp to thumbnail img src's, as a unique url # value to ensure fresh thumbs get shown on ajax refresh. Doesn't # actually need to be passed as a get param to '/thumbs/' (although # it does get passed), not used for anything else. timestamp = int(time.time()) return render_template('fragments/photos.html', filebrowse_files=filebrowse_files, filenames_input=filenames_input, filebrowse_error=filebrowse_error, thumb_resize_width=thumb_resize_width, thumb_resize_height=thumb_resize_height, timestamp=timestamp)
def run_do_rest(args): thread_count = args.threads vastdir = utils.add_trailing_slash(args.datadir) resultdir = clean_and_create_resultdir(args.resultdir, vastdir) fieldchartsdir = resultdir + "fieldcharts/" do_light = args.light do_phase = args.phase do_aavso = args.aavso logging.info( f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'") # get wcs model from the reference header. Used in writing world positions and field charts (can fail) wcs_file, wcs = reading.read_wcs_file(vastdir) ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir) _, _, _, first_frame = reading.extract_first_frame(vastdir) referene_frame_path = Path(reference_frame) reference_frame_filename = referene_frame_path.name logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}") logging.info(f"The first frame is '{first_frame}'") logging.info(f"Reference header is '{wcs_file}'") # Log filtering settings + check that reference frame is not inside of filter if args.jdfilter: logging.info(f"Filtering JD's: {args.jdfilter}") ref_inside_filter = (float(ref_jd) > args.jdfilter[0] and float(ref_jd) > args.jdfilter[1]) if ref_inside_filter: if not args.jdrefignore: assert not ref_inside_filter, "Reference frame JD is filtered" else: logging.info( "Reference frame JD is inside of the JD filter, but you indicated that's ok." ) if not os.path.isfile(wcs_file): full_ref_path = Path(args.fitsdir) / reference_frame_filename if not args.fitsdir and args.apikey: logging.error( "There is no plate-solved reference frame {wcs_file}, please specify both --apikey " "and --fitsdir.") sys.exit(0) rotation = reading.extract_reference_frame_rotation( vastdir, reference_frame_filename) assert (rotation == 0.0 ), f"Error: rotation is {rotation} and should always be 0.0" subprocess.Popen( f"python3 ./src/astrometry_api.py --apikey={args.apikey} " f"--upload={full_ref_path} --newfits={wcs_file} --private --no_commercial", shell=True, ) while not os.path.isfile(wcs_file): logging.info(f"Waiting for the astrometry.net plate solve...") time.sleep(10) wcs_file, wcs = reading.read_wcs_file(vastdir) star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs, args) stardict = get_localid_to_sd_dict(star_descriptions) logging.debug( f"First (max) 10 star descriptions: " f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}" ) write_augmented_autocandidates(vastdir, resultdir, stardict) write_augmented_all_stars(vastdir, resultdir, stardict) candidate_stars = utils.get_stars_with_metadata(star_descriptions, "CANDIDATE", exclude=["VSX"]) if args.selectcandidates: tag_candidates_as_selected(candidate_stars) logging.info(f"There are {len(candidate_stars)} candidate stars") vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX") logging.info(f"There are {len(vsx_stars)} vsx stars") selected_stars = utils.get_stars_with_metadata(star_descriptions, "SELECTEDTAG") # if args.selectvsx: # selected_stars = utils.concat_sd_lists(selected_stars, vsx_stars) logging.info(f"There are {len(selected_stars)} selected stars") compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars, candidate_stars) comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars, args.checkstarfile, vastdir, stardict, ref_jd) # Set comp stars for all interesting stars (stars which are interesting enough to measure) logging.info("Setting per star comparison stars...") if args.checkstarfile: utils.add_metadata(star_descriptions, CompStarData(compstar_ids=comp_stars.ids)) else: do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars, 10) logging.info( f"Using {thread_count} threads for phase plots, lightcurves, ...") if args.allstars: do_charts_vast.run( star_descriptions, comp_stars, vastdir, resultdir, "phase_all/", "light_all/", "aavso_all/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of ALL stars", ) else: if args.vsx: logging.info(f"Plotting {len(vsx_stars)} vsx stars...") do_charts_vast.run( vsx_stars, comp_stars, vastdir, resultdir, "phase_vsx/", "light_vsx/", "aavso_vsx/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of VSX stars", ) if args.radeccatalog or args.localidcatalog: do_charts_vast.run( selected_stars, comp_stars, vastdir, resultdir, "phase_selected/", "light_selected/", "aavso_selected", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of selected stars", ) if args.candidates: logging.info(f"Plotting {len(candidate_stars)} candidates...") do_charts_vast.run( candidate_stars, comp_stars, vastdir, resultdir, "phase_candidates/", "light_candidates/", "aavso_candidates/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of candidates", ) # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete ids = [x.local_id for x in selected_stars] logging.info( f"Writing selected files with {len(selected_stars)} selected stars: {ids}" ) write_selected_files(resultdir, vastdir, selected_stars) if args.field: do_charts_field.run_standard_field_charts(star_descriptions, wcs, fieldchartsdir, wcs_file, comp_stars) if args.stats: do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars, stardict, args.jdfilter) do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir, args.jdfilter) do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs, args.jdfilter) do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars, args.jdfilter) if args.site: ids = [x.local_id for x in selected_stars] logging.info( f"Creating HTML site with {len(selected_stars)} selected stars: {ids}" ) hugo_site.run(args.site, selected_stars, len(vsx_stars), referene_frame_path, resultdir, args.explore)