def abba_all(recipe_name, utdate, refdate="20140316", bands="HK",
starting_obsids=None, #interactive=False,
config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
if not bands in ["H", "K", "HK"]:
raise ValueError("bands must be one of 'H', 'K' or 'HK'")
fn = config.get_value('RECIPE_LOG_PATH', utdate)
from libs.recipes import Recipes #load_recipe_list, make_recipe_dict
recipe = Recipes(fn)
if starting_obsids is not None:
starting_obsids = map(int, starting_obsids.split(","))
selected = recipe.select(recipe_name, starting_obsids)
if not selected:
print "no recipe of with matching arguments is found"
process_abba_band = ProcessABBABand(utdate, refdate, config).process
for s in selected:
obsids = s[0]
frametypes = s[1]
for band in bands:
process_abba_band(recipe_name, band,
obsids, frametypes,
#do_interactive_figure=interactive
)
def get_pr(utdate, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
#from jj_recipe_base import ProcessBase
config = IGRINSConfig(config_file)
#refdate = config.get_value("REFDATE", None)
refdate = config.get("MASTER_CAL", "REFDATE")
pr = ProcessBase(utdate, refdate, config)
return pr
def abba_all(recipe_name, utdate, refdate="20140316", bands="HK",
starting_obsids=None, interactive=False,
config_file="recipe.config",
frac_slit=None,
cr_rejection_thresh=100.,
debug_output=False,
wavelength_increasing_order=False,
fill_nan=None,
lacosmics_thresh=0,
subtract_interorder_background=False,
):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
if not bands in ["H", "K", "HK"]:
raise ValueError("bands must be one of 'H', 'K' or 'HK'")
fn = config.get_value('RECIPE_LOG_PATH', utdate)
from libs.recipes import Recipes #load_recipe_list, make_recipe_dict
recipe = Recipes(fn)
if starting_obsids is not None:
starting_obsids = map(int, starting_obsids.split(","))
selected = recipe.select(recipe_name, starting_obsids)
if not selected:
print "no recipe of with matching arguments is found"
if frac_slit is not None:
frac_slit = map(float, frac_slit.split(","))
if len(frac_slit) !=2:
raise ValueError("frac_slit must be two floats separated by comma")
kwargs = dict(frac_slit=frac_slit,
cr_rejection_thresh=cr_rejection_thresh,
debug_output=debug_output,
wavelength_increasing_order=wavelength_increasing_order,
subtract_interorder_background=subtract_interorder_background,
fill_nan=fill_nan)
process_abba_band = ProcessABBABand(utdate, refdate,
config,
**kwargs).process
if len(selected) == 0:
print "No entry with given recipe is found : %s" % recipe_name
for s in selected:
obsids = s[0]
frametypes = s[1]
for band in bands:
process_abba_band(recipe_name, band,
obsids, frametypes,
#do_interactive_figure=interactive
)
def plot_spec(utdate,
refdate="20140316",
bands="HK",
starting_obsids=None,
interactive=False,
recipe_name="ALL_RECIPES",
config_file="recipe.config",
threshold_a0v=0.2,
multiply_model_a0v=False,
html_output=False):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
if not bands in ["H", "K", "HK"]:
raise ValueError("bands must be one of 'H', 'K' or 'HK'")
fn = config.get_value('RECIPE_LOG_PATH', utdate)
from libs.recipes import Recipes #load_recipe_list, make_recipe_dict
recipe = Recipes(fn)
if starting_obsids is not None:
starting_obsids = map(int, starting_obsids.split(","))
# recipe_name = "ALL_RECIPES"
selected = recipe.select(recipe_name, starting_obsids)
if not selected:
print "no recipe of with matching arguments is found"
for s in selected:
obsids = s[0]
frametypes = s[1]
recipe_name = s[2]["RECIPE"].strip()
objname = s[2]["OBJNAME"].strip()
if recipe_name not in [
"A0V_AB", "STELLAR_AB", "EXTENDED_AB", "EXTENDED_ONOFF"
]:
continue
for band in bands:
process_abba_band(recipe_name,
utdate,
refdate,
band,
obsids,
frametypes,
config,
do_interactive_figure=interactive,
threshold_a0v=threshold_a0v,
objname=objname,
multiply_model_a0v=multiply_model_a0v,
html_output=html_output)
def publish_html(utdate, config_file="recipe.config"):
#utdate = "20140713"
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
dirname = config.get_value("HTML_PATH", utdate)
from jinja2 import Environment, FileSystemLoader
env = Environment(loader=FileSystemLoader('jinja_templates'))
template = env.get_template('index.html')
sources = make_html(utdate, dirname)
s = template.render(utdate=utdate, sources=sources)
open(os.path.join(dirname, "index.html"), "w").write(s)
def publish_html(utdate, config_file="recipe.config"):
#utdate = "20140713"
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
dirname = config.get_value("HTML_PATH", utdate)
from jinja2 import Environment, FileSystemLoader
env = Environment(loader=FileSystemLoader('jinja_templates'))
template = env.get_template('index.html')
sources = make_html(utdate, dirname)
s = template.render(utdate=utdate, sources=sources)
open(os.path.join(dirname, "index.html"), "w").write(s)
def make_html(utdate, dirname, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
fn = config.get_value('RECIPE_LOG_PATH', utdate)
from libs.recipes import load_recipe_list
recipe_list = load_recipe_list(fn)
#recipe_dict = make_recipe_dict(recipe_list)
#spec_template = env.get_template('spec.html')
sources = []
for r, obsids, frametypes, desc in recipe_list:
if r in [
"A0V_AB",
"STELLAR_AB",
"EXTENDED_AB",
"EXTENDED_ONOFF",
]:
s = dict(
zip([
"name", "obj", "grp1", "grp2", "exptime", "recipe",
"obsids", "frametypes"
], desc))
#s["obsids"] = s["obsids"].strip()
s["nexp"] = len(obsids)
for band in "HK":
p = "igrins_spec_%04d_%s.html" % (obsids[0], band)
if os.path.exists(os.path.join(dirname, p)):
s["url_%s" % band] = p
for band in "HK":
p = "igrins_spec_%04dA0V_%s.html" % (obsids[0], band)
if os.path.exists(os.path.join(dirname, p)):
s["url_%s_A0V" % band] = p
sources.append(s)
# jsname = "igrins_spec_%04d_H.js" % obsids[0]
# ss = spec_template.render(utdate=utdate, jsname=jsname)
# open(os.path.join(dirname, s["url_H"]), "w").write(ss)
# jsname = "igrins_spec_%04d_K.js" % obsids[0]
# ss = spec_template.render(utdate=utdate, jsname=jsname)
# open(os.path.join(dirname, s["url_K"]), "w").write(ss)
return sources
def abba_all(recipe_name,
utdate,
refdate="20140316",
bands="HK",
starting_obsids=None,
interactive=False,
config_file="recipe.config",
frac_slit=None):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
if not bands in ["H", "K", "HK"]:
raise ValueError("bands must be one of 'H', 'K' or 'HK'")
fn = config.get_value('RECIPE_LOG_PATH', utdate)
from libs.recipes import Recipes #load_recipe_list, make_recipe_dict
recipe = Recipes(fn)
if starting_obsids is not None:
starting_obsids = map(int, starting_obsids.split(","))
selected = recipe.select(recipe_name, starting_obsids)
if not selected:
print "no recipe of with matching arguments is found"
if frac_slit is not None:
frac_slit = map(float, frac_slit.split(","))
if len(frac_slit) != 2:
raise ValueError("frac_slit must be two floats separated by comma")
process_abba_band = ProcessABBABand(utdate,
refdate,
config,
frac_slit=frac_slit).process
for s in selected:
obsids = s[0]
frametypes = s[1]
for band in bands:
process_abba_band(
recipe_name,
band,
obsids,
frametypes,
#do_interactive_figure=interactive
)
def plot_spec(utdate, refdate="20140316", bands="HK",
starting_obsids=None, interactive=False,
recipe_name = "ALL_RECIPES",
config_file="recipe.config",
threshold_a0v=0.2,
multiply_model_a0v=False,
html_output=False):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
if not bands in ["H", "K", "HK"]:
raise ValueError("bands must be one of 'H', 'K' or 'HK'")
fn = config.get_value('RECIPE_LOG_PATH', utdate)
from libs.recipes import Recipes #load_recipe_list, make_recipe_dict
recipe = Recipes(fn)
if starting_obsids is not None:
starting_obsids = map(int, starting_obsids.split(","))
# recipe_name = "ALL_RECIPES"
selected = recipe.select(recipe_name, starting_obsids)
if not selected:
print "no recipe of with matching arguments is found"
selected.sort()
for s in selected:
obsids = s[0]
frametypes = s[1]
recipe_name = s[2]["RECIPE"].strip()
objname = s[2]["OBJNAME"].strip()
target_type = recipe_name.split("_")[0]
if target_type not in ["A0V", "STELLAR", "EXTENDED"]:
print "Unsupported recipe : %s" % recipe_name
continue
for band in bands:
process_abba_band(recipe_name, utdate, refdate, band,
obsids, frametypes, config,
do_interactive_figure=interactive,
threshold_a0v=threshold_a0v,
objname=objname,
multiply_model_a0v=multiply_model_a0v,
html_output=html_output)
def __call__(self, utdate, bands="HK",
starting_obsids=None, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
self.config = IGRINSConfig(config_file)
self.refdate = self.config.get_value('REFDATE', utdate)
self._validate_bands(bands)
recipes = self.get_recipes(utdate)
starting_obsids_parsed = self.parse_starting_obsids(starting_obsids)
selected = recipes.select(self.RECIPE_NAME, starting_obsids_parsed)
self.run_selected_bands(utdate, selected, bands)
class RecipeBase(object):
""" The derived mus define RECIPE_NAME attribute and must implement
run_selected_bands method.
"""
def __init__(self, **kwargs):
self.kwargs = kwargs
def set_recipe_name(self, recipe_name):
self.RECIPE_NAME = recipe_name
def _validate_bands(self, bands):
if not bands in ["H", "K", "HK"]:
raise ValueError("bands must be one of 'H', 'K' or 'HK'")
def get_recipe_name(self, utdate):
fn = self.config.get_value('RECIPE_LOG_PATH', utdate)
return fn
def get_recipes(self, utdate):
fn = self.get_recipe_name(utdate)
from libs.recipes import Recipes #load_recipe_list, make_recipe_dict
return Recipes(fn)
def parse_starting_obsids(self, starting_obsids):
if starting_obsids is not None:
starting_obsids = map(int, starting_obsids.split(","))
return starting_obsids
else:
return None
def run_selected_bands_with_recipe(self, utdate, selected, bands):
# just ignore recipe
selected2 = [_[1:] for _ in selected]
self.run_selected_bands(utdate, selected2, bands)
def __call__(self, utdate, bands="HK",
starting_obsids=None, config_file="recipe.config"):
self.process(utdate, bands, starting_obsids, config_file)
def process(self, utdate, bands="HK",
starting_obsids=None, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
self.config = IGRINSConfig(config_file)
self.refdate = self.config.get("MASTER_CAL", "REFDATE")
self._validate_bands(bands)
recipes = self.get_recipes(utdate)
starting_obsids_parsed = self.parse_starting_obsids(starting_obsids)
selected = recipes.select_fnmatch(self.RECIPE_NAME,
starting_obsids_parsed)
self.run_selected_bands_with_recipe(utdate, selected, bands)
def make_html(utdate, dirname, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
fn = config.get_value('RECIPE_LOG_PATH', utdate)
from libs.recipes import load_recipe_list
recipe_list = load_recipe_list(fn)
#recipe_dict = make_recipe_dict(recipe_list)
#spec_template = env.get_template('spec.html')
sources = []
for r, obsids, frametypes, desc in recipe_list:
if r in ["A0V_AB", "STELLAR_AB",
"EXTENDED_AB",
"EXTENDED_ONOFF",
]:
s = dict(zip(["name", "obj", "grp1", "grp2", "exptime", "recipe", "obsids", "frametypes"],
desc))
#s["obsids"] = s["obsids"].strip()
s["nexp"] = len(obsids)
for band in "HK":
p = "igrins_spec_%04d_%s.html" % (obsids[0], band)
if os.path.exists(os.path.join(dirname, p)):
s["url_%s" % band] = p
for band in "HK":
p = "igrins_spec_%04dA0V_%s.html" % (obsids[0], band)
if os.path.exists(os.path.join(dirname, p)):
s["url_%s_A0V" % band] = p
sources.append(s)
# jsname = "igrins_spec_%04d_H.js" % obsids[0]
# ss = spec_template.render(utdate=utdate, jsname=jsname)
# open(os.path.join(dirname, s["url_H"]), "w").write(ss)
# jsname = "igrins_spec_%04d_K.js" % obsids[0]
# ss = spec_template.render(utdate=utdate, jsname=jsname)
# open(os.path.join(dirname, s["url_K"]), "w").write(ss)
return sources
def process(self, utdate, bands="HK",
starting_obsids=None, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
self.config = IGRINSConfig(config_file)
self.refdate = self.config.get("MASTER_CAL", "REFDATE")
self._validate_bands(bands)
recipes = self.get_recipes(utdate)
starting_obsids_parsed = self.parse_starting_obsids(starting_obsids)
selected = recipes.select_fnmatch(self.RECIPE_NAME,
starting_obsids_parsed)
self.run_selected_bands_with_recipe(utdate, selected, bands)
class RecipeBase(object):
""" The derived mus define RECIPE_NAME attribute and must implement
run_selected_bands method.
"""
def _validate_bands(self, bands):
if not bands in ["H", "K", "HK"]:
raise ValueError("bands must be one of 'H', 'K' or 'HK'")
def get_recipe_name(self, utdate):
fn = self.config.get_value('RECIPE_LOG_PATH', utdate)
return fn
def get_recipes(self, utdate):
fn = self.get_recipe_name(utdate)
from libs.recipes import Recipes #load_recipe_list, make_recipe_dict
return Recipes(fn)
def parse_starting_obsids(self, starting_obsids):
if starting_obsids is not None:
starting_obsids = map(int, starting_obsids.split(","))
return starting_obsids
else:
return None
def __call__(self, utdate, bands="HK",
starting_obsids=None, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
self.config = IGRINSConfig(config_file)
self.refdate = self.config.get_value('REFDATE', utdate)
self._validate_bands(bands)
recipes = self.get_recipes(utdate)
starting_obsids_parsed = self.parse_starting_obsids(starting_obsids)
selected = recipes.select(self.RECIPE_NAME, starting_obsids_parsed)
self.run_selected_bands(utdate, selected, bands)
def prepare_recipe_logs(utdate, config_file="recipe.config"):
from libs.igrins_config import IGRINSConfig
config = IGRINSConfig(config_file)
fn0 = config.get_value('INDATA_PATH', utdate)
# there could be two log files!
import glob
fn_list = glob.glob(os.path.join(fn0, "IGRINS_DT_Log_*-1_H.txt"))
print "loading DT log files:", fn_list
#fn = os.path.join(fn0, "IGRINS_DT_Log_%s-1_H.txt" % (utdate,))
# p_end_comma = re.compile(r",\s$")
# s = "".join(p_end_comma.sub(",\n", l) for l in lines)
#s = "".join(lines)
# dtype=[('FILENAME', 'S128'), ('OBSTIME', 'S128'), ('GROUP1', 'i'), ('GROUP2', 'i'), ('OBJNAME', 'S128'), ('OBJTYPE', 'S128'), ('FRAMETYPE', 'S128'), ('EXPTIME', 'd'), ('ROTPA', 'd'), ('RA', 'S128'), ('DEC', 'S128'), ('AM', 'd')]
# log file format for March and May, July is different.
dtype_=[('FILENAME', 'S128'),
('OBSTIME', 'S128'),
('GROUP1', 'i'),
('GROUP2', 'i'),
('OBJNAME', 'S128'),
('OBJTYPE', 'S128'),
('FRAMETYPE', 'S128'),
('EXPTIME', 'd'),
('ROTPA', 'd'),
('RA', 'S128'),
('DEC', 'S128'),
('AM', 'd'),
('OBSDATE', 'S128'),
('SEQID1', 'i'),
('SEQID2', 'i'),
('ALT', 'd'),
('AZI', 'd'),
('OBSERVER', 'S128'),
('EPOCH', 'S128'),
('AGPOS', 'S128'),
]
dtype_map = dict(dtype_)
dtype_replace = dict(SEQID1="GROUP1", SEQID2="GROUP2")
def load_data(fn):
lines = open(fn).readlines()
stripped_lines = [s1.strip() for s1 in lines[1].split(",")]
dtype = [(dtype_replace.get(s1, s1), dtype_map[s1]) for s1 in stripped_lines if s1]
l = np.genfromtxt(fn,
skip_header=2, delimiter=",", dtype=dtype)
return l
l_list = [load_data(fn) for fn in fn_list]
l = np.concatenate(l_list)
from itertools import groupby
groupby_keys = ["OBJNAME", "OBJTYPE", "GROUP1", "GROUP2", "EXPTIME"]
def keyfunc(l1):
return tuple(l1[k] for k in groupby_keys)
s_list = []
for lll in groupby(l, keyfunc):
grouper = list(lll[1])
obsids = [int(lll1[0].split(".")[0].split("_")[-1]) for lll1 in grouper]
frametypes = [lll1["FRAMETYPE"] for lll1 in grouper]
objtype = lll[0][1]
if objtype.lower() == "flat":
recipe = "FLAT"
elif objtype.lower() == "std":
recipe = "A0V_AB"
elif objtype.lower() == "tar":
recipe = "STELLAR_AB"
else:
recipe = "DEFAULT"
s1 = "%s, %s, %d, %d, %f," % lll[0]
s2 = "%s, %s, %s\n" % (recipe,
" ".join(map(str,obsids)),
" ".join(frametypes),
)
s_list.append(s1+" "+s2)
headers = groupby_keys + ["RECIPE", "OBSIDS", "FRAMETYPES"]
recipe_log_name = config.get_value('RECIPE_LOG_PATH', utdate)
fn_out = recipe_log_name + ".tmp"
fout = open(fn_out, "w")
fout.write(", ".join(headers) + "\n")
fout.write("# Avaiable recipes : FLAT, THAR, SKY_WVLSOL, A0V_AB, STELLAR_AB, EXTENDED_AB, EXTENDED_ONOFF\n")
fout.writelines(s_list)
fout.close()
print "A draft version of the recipe log is written to '%s'." % (fn_out,)
print "Make an adjusment and rename it to '%s'." % (recipe_log_name,)
