def construct_star_descriptions(vastdir: str, stars: List[int] = None):
vastdir = utils.add_trailing_slash(vastdir)
wcs_file, wcs = reading.read_wcs_file(vastdir)
all_files = utils.file_selector(the_dir=vastdir, match_pattern="*.dat")
if stars is not None:
all_files = [vastdir + reading.star_to_dat(int(x)) for x in stars]
sds = construct_raw_star_descriptions(vastdir, wcs, all_files)
return sds
def clean_and_create_resultdir(argsdir: str, vastdir: str):
resultdir = utils.add_trailing_slash(
argsdir) if argsdir is not None else vastdir
# if resultdir does not exist, create it
if not os.path.isdir(resultdir):
logging.info(
f"The resultdir '{resultdir}' does not exist, creating it...")
reading.create_dir(resultdir)
return resultdir
def meta_init(datadir: str):
# make sure datadir has trailing slash
datadir = utils.add_trailing_slash(datadir)
global settings
settings = Settings(datadir)
global init
# init = importlib.import_module(module_name, package=None)
# print("datadir is", datadir+'init.py')
init = imp.load_source('init', datadir+'init.py')
def home():
"""Displays the FotoJazz home page."""
filebrowse_path = ''
if len(sys.argv) > 1:
filebrowse_path = add_trailing_slash(sys.argv[1])
# The filebrowse area has to be rendered by a separate function,
# becase it's refreshed via AJAX.
filebrowse_files_rendered = photos(filebrowse_path, check_all=True)
return render_template('home.html', filebrowse_path=filebrowse_path,
filebrowse_files_rendered=filebrowse_files_rendered)
def home():
"""Displays the FotoJazz home page."""
filebrowse_path = ''
if len(sys.argv) > 1:
filebrowse_path = add_trailing_slash(sys.argv[1])
# The filebrowse area has to be rendered by a separate function,
# becase it's refreshed via AJAX.
filebrowse_files_rendered = photos(filebrowse_path, check_all=True)
return render_template('home.html',
filebrowse_path=filebrowse_path,
filebrowse_files_rendered=filebrowse_files_rendered)
def photos(photos_path='', check_all=False):
"""Renders the photo browsing block. This view is often called by
AJAX, as this block gets refreshed dynamically.
This view expects the following GET params:
- filenames_input: space-delimited string of absolute paths to
the photos that should be rendered with their
checkbox checked by default.
- check_all: boolean indicating if all photos should be rendered
with their checkbox checked by default. If true,
filenames_input is ignored.
- photos_path: absolute path to the directory from which to load
photos."""
filenames_str = request.args.get('filenames_input', '', type=str)
filenames_input = []
if filenames_str != '':
filenames_input = [x.strip() for x in filenames_str.split(' ')
if x != '']
if len(filenames_input):
filenames_input.sort()
check_all_arg = request.args.get('check_all', '', type=str)
if check_all_arg != '':
check_all = int(check_all_arg) and True or False
filebrowse_files = []
filebrowse_path = ''
filebrowse_error = ''
if not photos_path:
photos_path = request.args.get('photos_path', '', type=str)
if not photos_path:
filebrowse_error = 'No path specified.'
else:
filebrowse_path = add_trailing_slash(photos_path)
thumb_resize_width = app.config['THUMB_RESIZE_WIDTH']
thumb_resize_height = app.config['THUMB_RESIZE_HEIGHT']
if not path.isdir(filebrowse_path):
filebrowse_error = 'Specified path is not a valid directory.'
else:
glob_pattern = '%s%s' % (filebrowse_path, '*.[jJ][pP]*[gG]')
filenames = glob(glob_pattern)
filebrowse_files = [get_thumb_metadata(
filename,
thumb_resize_width,
thumb_resize_height,
checked=check_all or
filename in filenames_input)
for filename in filenames]
if not filebrowse_files:
filebrowse_error = 'No images in specified directory.'
else:
filebrowse_files.sort(
key=itemgetter('date_taken_timestamp'))
# Need to add timestamp to thumbnail img src's, as a unique url
# value to ensure fresh thumbs get shown on ajax refresh. Doesn't
# actually need to be passed as a get param to '/thumbs/' (although
# it does get passed), not used for anything else.
timestamp = int(time.time())
return render_template('fragments/photos.html',
filebrowse_files=filebrowse_files,
filenames_input=filenames_input,
filebrowse_error=filebrowse_error,
thumb_resize_width=thumb_resize_width,
thumb_resize_height=thumb_resize_height,
timestamp=timestamp)
def run_do_rest(args):
thread_count = args.threads
vastdir = utils.add_trailing_slash(args.datadir)
resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
do_light = args.light
do_phase = args.phase
do_aavso = args.aavso
logging.info(
f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
# get wcs model from the reference header. Used in writing world positions and field charts (can fail)
wcs_file, wcs = reading.read_wcs_file(vastdir)
ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
_, _, _, first_frame = reading.extract_first_frame(vastdir)
reference_frame_path = Path(reference_frame)
reference_frame_filename = reference_frame_path.name
wcs_file, wcs = perform_astrometry_net(args, vastdir, wcs_file, wcs,
reference_frame_filename)
logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
logging.info(f"The first frame is '{first_frame}'")
logging.info(f"Reference header is '{wcs_file}'")
check_that_reference_image_not_within_jdfilter(ref_jd, args.jdfilter,
args.jdrefignore)
star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
args)
stardict = get_localid_to_sd_dict(star_descriptions)
logging.debug(
f"First (max) 10 star descriptions: "
f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
)
write_augmented_autocandidates(vastdir, resultdir, stardict)
write_augmented_all_stars(vastdir, resultdir, stardict)
candidate_stars = utils.get_stars_with_metadata(star_descriptions,
"CANDIDATE",
exclude=["VSX"])
if args.selectcandidates:
tag_candidates_as_selected(candidate_stars)
logging.info(f"There are {len(candidate_stars)} candidate stars")
vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
logging.info(f"There are {len(vsx_stars)} vsx stars")
if args.allstars:
tag_all_stars_as_selected(star_descriptions)
selected_stars = utils.get_stars_with_metadata(star_descriptions,
"SELECTEDTAG")
logging.info(f"There are {len(selected_stars)} selected stars")
compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
candidate_stars)
comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
args.checkstarfile, vastdir,
stardict, ref_jd)
# Set comp stars for all interesting stars (stars which are interesting enough to measure)
logging.info("Setting per star comparison stars...")
if args.checkstarfile:
utils.add_metadata(star_descriptions,
CompStarData(compstar_ids=comp_stars.ids))
else:
do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
10)
logging.info(
f"Using {thread_count} threads for phase plots, lightcurves, ...")
if args.allstars:
do_charts_vast.run(
star_descriptions,
comp_stars,
vastdir,
resultdir,
"phase_all/",
"light_all/",
"aavso_all/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of ALL stars",
)
else:
if args.vsx:
logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
do_charts_vast.run(
vsx_stars,
comp_stars,
vastdir,
resultdir,
"phase_vsx/",
"light_vsx/",
"aavso_vsx/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of VSX stars",
)
if args.radeccatalog or args.localidcatalog:
do_charts_vast.run(
selected_stars,
comp_stars,
vastdir,
resultdir,
"phase_selected/",
"light_selected/",
"aavso_selected",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of selected stars",
)
if args.candidates:
logging.info(f"Plotting {len(candidate_stars)} candidates...")
do_charts_vast.run(
candidate_stars,
comp_stars,
vastdir,
resultdir,
"phase_candidates/",
"light_candidates/",
"aavso_candidates/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of candidates",
)
# starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
ids = [x.local_id for x in selected_stars]
logging.info(
f"Writing selected files with {len(selected_stars)} selected stars: {ids}"
)
write_selected_files(resultdir, vastdir, selected_stars)
# only create fieldcharts dir if we use it
fieldchartsdir = resultdir + "fieldcharts/"
if args.field or args.stats:
trash_and_recreate_dir(fieldchartsdir)
if args.field:
do_charts_field.run_standard_field_charts(star_descriptions, wcs,
fieldchartsdir, wcs_file,
comp_stars)
if args.stats:
do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
stardict, args.jdfilter)
do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
args.jdfilter)
do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
args.jdfilter)
do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
args.jdfilter)
if args.site:
ids = [x.local_id for x in selected_stars]
logging.info(
f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
)
hugo_site.run(args.site, selected_stars, len(vsx_stars),
reference_frame_path, resultdir, args.explore)
help="Fps of the movie",
nargs="?",
required=False,
default=30)
# parser.add_argument('-s', '--start',
# help="The start Julian Date",
# nargs='?', required=True)
# parser.add_argument('-e', '--end',
# help="The end Julian Date",
# nargs='?', required=True)
parser.add_argument("-x",
"--verbose",
help="Set logging to debug mode",
action="store_true")
args = parser.parse_args()
datadir = utils.add_trailing_slash(args.datadir)
datenow = datetime.now()
resultdir = Path(args.resultdir)
filehandler = f"{datadir}vastlog-{datenow:%Y%M%d-%H_%M_%S}.log"
fh = logging.FileHandler(filehandler)
fh.setLevel(logging.INFO)
# add the handlers to the logger
logger.addHandler(fh)
if args.verbose:
logger.setLevel(logging.DEBUG)
fh.setLevel(logging.DEBUG)
assert os.path.exists(args.datadir), "datadir does not exist"
# assert os.path.exists(args.resultdir), "resultdir does not exist" ==> this dir is created
assert os.path.exists(args.resultdir), "resultdir does not exist"
assert os.path.exists(args.fitsdir), "fitsdir does not exist"
def photos(photos_path='', check_all=False):
"""Renders the photo browsing block. This view is often called by
AJAX, as this block gets refreshed dynamically.
This view expects the following GET params:
- filenames_input: space-delimited string of absolute paths to
the photos that should be rendered with their
checkbox checked by default.
- check_all: boolean indicating if all photos should be rendered
with their checkbox checked by default. If true,
filenames_input is ignored.
- photos_path: absolute path to the directory from which to load
photos."""
filenames_str = request.args.get('filenames_input', '', type=str)
filenames_input = []
if filenames_str != '':
filenames_input = [
x.strip() for x in filenames_str.split(' ') if x != ''
]
if len(filenames_input):
filenames_input.sort()
check_all_arg = request.args.get('check_all', '', type=str)
if check_all_arg != '':
check_all = int(check_all_arg) and True or False
filebrowse_files = []
filebrowse_path = ''
filebrowse_error = ''
if not photos_path:
photos_path = request.args.get('photos_path', '', type=str)
if not photos_path:
filebrowse_error = 'No path specified.'
else:
filebrowse_path = add_trailing_slash(photos_path)
thumb_resize_width = app.config['THUMB_RESIZE_WIDTH']
thumb_resize_height = app.config['THUMB_RESIZE_HEIGHT']
if not path.isdir(filebrowse_path):
filebrowse_error = 'Specified path is not a valid directory.'
else:
glob_pattern = '%s%s' % (filebrowse_path, '*.[jJ][pP]*[gG]')
filenames = glob(glob_pattern)
filebrowse_files = [
get_thumb_metadata(filename,
thumb_resize_width,
thumb_resize_height,
checked=check_all
or filename in filenames_input)
for filename in filenames
]
if not filebrowse_files:
filebrowse_error = 'No images in specified directory.'
else:
filebrowse_files.sort(key=itemgetter('date_taken_timestamp'))
# Need to add timestamp to thumbnail img src's, as a unique url
# value to ensure fresh thumbs get shown on ajax refresh. Doesn't
# actually need to be passed as a get param to '/thumbs/' (although
# it does get passed), not used for anything else.
timestamp = int(time.time())
return render_template('fragments/photos.html',
filebrowse_files=filebrowse_files,
filenames_input=filenames_input,
filebrowse_error=filebrowse_error,
thumb_resize_width=thumb_resize_width,
thumb_resize_height=thumb_resize_height,
timestamp=timestamp)
def run_do_rest(args):
thread_count = args.threads
vastdir = utils.add_trailing_slash(args.datadir)
resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
fieldchartsdir = resultdir + "fieldcharts/"
do_light = args.light
do_phase = args.phase
do_aavso = args.aavso
logging.info(
f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
# get wcs model from the reference header. Used in writing world positions and field charts (can fail)
wcs_file, wcs = reading.read_wcs_file(vastdir)
ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
_, _, _, first_frame = reading.extract_first_frame(vastdir)
referene_frame_path = Path(reference_frame)
reference_frame_filename = referene_frame_path.name
logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
logging.info(f"The first frame is '{first_frame}'")
logging.info(f"Reference header is '{wcs_file}'")
# Log filtering settings + check that reference frame is not inside of filter
if args.jdfilter:
logging.info(f"Filtering JD's: {args.jdfilter}")
ref_inside_filter = (float(ref_jd) > args.jdfilter[0]
and float(ref_jd) > args.jdfilter[1])
if ref_inside_filter:
if not args.jdrefignore:
assert not ref_inside_filter, "Reference frame JD is filtered"
else:
logging.info(
"Reference frame JD is inside of the JD filter, but you indicated that's ok."
)
if not os.path.isfile(wcs_file):
full_ref_path = Path(args.fitsdir) / reference_frame_filename
if not args.fitsdir and args.apikey:
logging.error(
"There is no plate-solved reference frame {wcs_file}, please specify both --apikey "
"and --fitsdir.")
sys.exit(0)
rotation = reading.extract_reference_frame_rotation(
vastdir, reference_frame_filename)
assert (rotation == 0.0
), f"Error: rotation is {rotation} and should always be 0.0"
subprocess.Popen(
f"python3 ./src/astrometry_api.py --apikey={args.apikey} "
f"--upload={full_ref_path} --newfits={wcs_file} --private --no_commercial",
shell=True,
)
while not os.path.isfile(wcs_file):
logging.info(f"Waiting for the astrometry.net plate solve...")
time.sleep(10)
wcs_file, wcs = reading.read_wcs_file(vastdir)
star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
args)
stardict = get_localid_to_sd_dict(star_descriptions)
logging.debug(
f"First (max) 10 star descriptions: "
f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
)
write_augmented_autocandidates(vastdir, resultdir, stardict)
write_augmented_all_stars(vastdir, resultdir, stardict)
candidate_stars = utils.get_stars_with_metadata(star_descriptions,
"CANDIDATE",
exclude=["VSX"])
if args.selectcandidates:
tag_candidates_as_selected(candidate_stars)
logging.info(f"There are {len(candidate_stars)} candidate stars")
vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
logging.info(f"There are {len(vsx_stars)} vsx stars")
selected_stars = utils.get_stars_with_metadata(star_descriptions,
"SELECTEDTAG")
# if args.selectvsx:
# selected_stars = utils.concat_sd_lists(selected_stars, vsx_stars)
logging.info(f"There are {len(selected_stars)} selected stars")
compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
candidate_stars)
comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
args.checkstarfile, vastdir,
stardict, ref_jd)
# Set comp stars for all interesting stars (stars which are interesting enough to measure)
logging.info("Setting per star comparison stars...")
if args.checkstarfile:
utils.add_metadata(star_descriptions,
CompStarData(compstar_ids=comp_stars.ids))
else:
do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
10)
logging.info(
f"Using {thread_count} threads for phase plots, lightcurves, ...")
if args.allstars:
do_charts_vast.run(
star_descriptions,
comp_stars,
vastdir,
resultdir,
"phase_all/",
"light_all/",
"aavso_all/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of ALL stars",
)
else:
if args.vsx:
logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
do_charts_vast.run(
vsx_stars,
comp_stars,
vastdir,
resultdir,
"phase_vsx/",
"light_vsx/",
"aavso_vsx/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of VSX stars",
)
if args.radeccatalog or args.localidcatalog:
do_charts_vast.run(
selected_stars,
comp_stars,
vastdir,
resultdir,
"phase_selected/",
"light_selected/",
"aavso_selected",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of selected stars",
)
if args.candidates:
logging.info(f"Plotting {len(candidate_stars)} candidates...")
do_charts_vast.run(
candidate_stars,
comp_stars,
vastdir,
resultdir,
"phase_candidates/",
"light_candidates/",
"aavso_candidates/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of candidates",
)
# starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
ids = [x.local_id for x in selected_stars]
logging.info(
f"Writing selected files with {len(selected_stars)} selected stars: {ids}"
)
write_selected_files(resultdir, vastdir, selected_stars)
if args.field:
do_charts_field.run_standard_field_charts(star_descriptions, wcs,
fieldchartsdir, wcs_file,
comp_stars)
if args.stats:
do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
stardict, args.jdfilter)
do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
args.jdfilter)
do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
args.jdfilter)
do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
args.jdfilter)
if args.site:
ids = [x.local_id for x in selected_stars]
logging.info(
f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
)
hugo_site.run(args.site, selected_stars, len(vsx_stars),
referene_frame_path, resultdir, args.explore)