def start(args):
"""
nifsPipeline
This script is a full-featured NIFS data reduction pipeline. It can call up
to three "Steps".
This script does two things. It:
- gets data reduction parameters; either from an interactive input session or
an input file, and
- launches appropriate scripts to do the work. It can call up to 3 scripts directly:
1) nifsSort.py
2) nifsBaselineCalibration.py
3) nifsReduce.py
"""
# Save starting path for later use and change one directory up.
path = os.getcwd()
print "IT WORKED!"
# Get paths to built-in Nifty data. Special code in setup.py makes sure recipes/ and
# runtimeData/ will be installed when someone installs Nifty, and accessible in this way.
RECIPES_PATH = pkg_resources.resource_filename('nifty', 'recipes/')
RUNTIME_DATA_PATH = pkg_resources.resource_filename(
'nifty', 'runtimeData/')
# Format logging options.
FORMAT = '%(asctime)s %(message)s'
DATEFMT = datefmt()
# Set up the main logging file.
logging.basicConfig(filename='Nifty.log',
format=FORMAT,
datefmt=DATEFMT,
level=logging.DEBUG)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# This lets us logging.info(to stdout AND a logfile. Cool, huh?
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
logging.info("\n####################################")
logging.info("# #")
logging.info("# NIFTY #")
logging.info("# NIFS Data Reduction Pipeline #")
logging.info("# Version " + __version__ + " #")
logging.info("# July 25th, 2017 #")
logging.info("# Marie Lemoine-Busserolle #")
logging.info("# Gemini Observatory, Hilo, Hawaii #")
logging.info("# #")
logging.info("####################################\n")
# Make sure to change this if you change the default logfile.
logging.info('The log file is Nifty.log.')
# Read or write a configuration file, interactively, from defaults or from a provided file.
# Second argument is the name of the current script. This could be used to get script-dependent configuration.
GetConfig(args, "nifsPipeline")
# TODO(nat): fix this. It isn't recursively printing the dictionaries of values.
logging.info(
"\nParameters for this data reduction as read from ./config.cfg:\n")
with open('./config.cfg') as config_file:
config = ConfigObj(config_file, unrepr=True)
for i in config:
if isinstance(config[i], dict):
for k in config[i]:
logging.info(str(k) + " " + str(config[i][k]))
else:
logging.info(str(i) + " " + str(config[i]))
logging.info("")
# Load pipeline configuration from ./config.cfg that is used by this script.
with open('./config.cfg') as config_file:
# Load general config.
config = ConfigObj(config_file, unrepr=True)
manualMode = config['manualMode']
# Load pipeline specific config.
nifsPipelineConfig = config['nifsPipelineConfig']
sort = nifsPipelineConfig['sort']
calibrationReduction = nifsPipelineConfig['calibrationReduction']
telluricReduction = nifsPipelineConfig['telluricReduction']
scienceReduction = nifsPipelineConfig['scienceReduction']
telluricCorrection = nifsPipelineConfig['telluricCorrection']
fluxCalibration = nifsPipelineConfig['fluxCalibration']
merge = nifsPipelineConfig['merge']
###########################################################################
## SETUP COMPLETE ##
## BEGIN DATA REDUCTION ##
## ##
## Four Main Steps: ##
## 1) Sort the Raw Data - nifsSort.py ##
## 2) Reduce baseline calibrations - nifsBaselineCalibration.py ##
## 3) Reduce telluric observations - nifsReduce.py ##
## 4) Reduce science observations - nifsReduce.py ##
## ##
###########################################################################
###########################################################################
## STEP 1: Sort the raw data. ##
###########################################################################
if sort:
if manualMode:
a = raw_input('About to enter nifsSort.')
nifsSort.start()
# By now, we should have paths to the three types of raw data. Print them out.
printDirectoryLists()
###########################################################################
## STEP 2: Reduce baseline calibrations. ##
###########################################################################
if calibrationReduction:
if manualMode:
a = raw_input('About to enter nifsBaselineCalibration.')
nifsBaselineCalibration.start()
###########################################################################
## STEP 3: Reduce telluric observations. ##
###########################################################################
if telluricReduction:
if manualMode:
a = raw_input('About to enter nifsReduce to reduce Tellurics.')
nifsReduce.start('Telluric')
###########################################################################
## STEP 4: Reduce science observations. ##
###########################################################################
if scienceReduction:
if manualMode:
a = raw_input('About to enter nifsReduce to reduce science.')
nifsReduce.start('Science')
if telluricCorrection:
if manualMode:
a = raw_input(
'About to enter nifsTelluric to make and create telluric corrected cubes.'
)
nifsTelluric.run()
if fluxCalibration:
if manualMode:
a = raw_input(
'About to enter nifsFluxCalibrate to make and create flux calibrated and telluric corrected cubes.'
)
nifsFluxCalibrate.run()
if merge:
if manualMode:
a = raw_input(
'About to enter nifsMerge to merge final 3D data cubes to single cubes.'
)
nifsMerge.run()
###########################################################################
## Data Reduction Complete! ##
## Good luck with your science! ##
###########################################################################
logging.info(
'\n###########################################################')
logging.info('# #')
logging.info('# DATA REDUCTION COMPLETE #')
logging.info('# Good luck with your science! #')
logging.info('# Check out http://nifty4gemini.readthedocs.io/en/latest/ #')
logging.info('# For docs, tutorials and examples. #')
logging.info('# #')
logging.info(
'###########################################################\n')
return
def start(args):
"""
nifsPipeline
This script is a full-featured NIFS data reduction pipeline. It can call up
to three "Steps".
This script does two things. It:
- gets data reduction parameters; either from an interactive input session or
an input file, and
- launches appropriate scripts to do the work. It can call up to 3 scripts directly:
1) nifsSort.py
2) nifsBaselineCalibration.py
3) nifsReduce.py
"""
# Save path for later use and change one directory up.
path = os.getcwd()
# Get paths to Nifty data.
RECIPES_PATH = pkg_resources.resource_filename('nifty', 'recipes/')
RUNTIME_DATA_PATH = pkg_resources.resource_filename('nifty', 'runtimeData/')
# Format logging options.
FORMAT = '%(asctime)s %(message)s'
DATEFMT = datefmt()
# Set up the logging file.
logging.basicConfig(filename='Nifty.log',format=FORMAT,datefmt=DATEFMT,level=logging.DEBUG)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# This lets us logging.info(to stdout AND a logfile. Cool, huh?
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
logging.info("\nWARNING: This pipeline is untested and needs revision.")
# TODO(nat): revise this pipeline.
logging.info("\n####################################")
logging.info("# #")
logging.info("# NIFTY #")
logging.info("# NIFS Data Reduction Pipeline #")
logging.info("# Version "+ __version__+ " #")
logging.info("# July 25th, 2017 #")
logging.info("# Marie Lemoine-Busserolle #")
logging.info("# Gemini Observatory, Hilo, Hawaii #")
logging.info("# #")
logging.info("####################################\n")
# Make sure to change this if you change the default logfile.
logging.info('The log file is Nifty.log.')
# Read or write a configuration file, interactively or from some defaults.
# Second argument is the name of the current script. Used to get script-dependent configuration.
GetConfig(args, "nifsPipeline")
# TODO(nat): fix this. It isn't recursively printing the dictionaries of values.
logging.info("\nParameters for this data reduction as read from ./config.cfg:\n")
with open('./config.cfg') as config_file:
config = ConfigObj(config_file, unrepr=True)
for i in config:
logging.info(str(i) + " " + str(config[i]))
logging.info("")
# Load configuration from ./config.cfg that is used by this script.
with open('./config.cfg') as config_file:
# Load general config.
config = ConfigObj(config_file, unrepr=True)
manualMode = config['manualMode']
# Load pipeline specific config.
nifsPipelineConfig = config['nifsPipelineConfig']
sort = nifsPipelineConfig['sort']
calibrationReduction = nifsPipelineConfig['calibrationReduction']
telluricReduction = nifsPipelineConfig['telluricReduction']
scienceReduction = nifsPipelineConfig['scienceReduction']
###########################################################################
## SETUP COMPLETE ##
## BEGIN DATA REDUCTION ##
## ##
## Four Main Steps: ##
## 1) Sort the Raw Data - nifsSort.py ##
## 2) Reduce baseline calibrations - nifsBaselineCalibration.py ##
## 3) Reduce telluric observations - nifsReduce.py ##
## 4) Reduce science observations - nifsReduce.py ##
## ##
###########################################################################
###########################################################################
## STEP 1: Sort the raw data. ##
###########################################################################
if sort:
if manualMode:
a = raw_input('About to enter nifsSort.')
nifsSort.start()
# By now, we should have paths to the three types of raw data. Print them out.
printDirectoryLists()
with open('./config.cfg') as config_file:
# Load general config.
config = ConfigObj(config_file, unrepr=True)
scienceDirectoryList = config['scienceDirectoryList']
# This mode does a full reduction, science observation by science observation.
# For each science directory in science directory list:
# Reduce the associated calibrations
# Reduce the associated tellurics
# Reduce the science observation
for scienceObservation in scienceDirectoryList:
# Find associated calibrations by looping through calibration directory list.
# Split to this form: ('/Users/nat/tests/core/M85/20150508/J', 'obs36')
temp = os.path.split(scienceObservation)
# Split again to this form: ('/Users/nat/tests/core/M85/20150508', 'J')
temp2 = os.path.split(temp[0])
# Now temp[0] is calibation base path name, temp[1] is grating.
calibrationDirectory = temp2[0]+"/Calibrations_"+temp2[1]
# Have to convert it to a one-element list first.
calibrationDirectory = [calibrationDirectory]
# We now have our calibration directory for the given science!
# Now find associated telluric observations.
# temp[0] looks like: '/Users/nat/tests/core/M85/20150508/J'
telluricBaseName = temp[0]+"/Tellurics/"
os.chdir(telluricBaseName)
telluricDirectoryList = glob.glob("obs*")
# We must make these incomplete paths into full paths by adding the base path.
telluricDirectoryList = [telluricBaseName+x for x in telluricDirectoryList]
# Don't forget to change back to starting point.
os.chdir(path)
###########################################################################
## STEP 2: Reduce baseline calibrations. ##
###########################################################################
if calibrationReduction:
if manualMode:
a = raw_input('About to enter nifsBaselineCalibration.')
nifsBaselineCalibration.start(calibrationDirectoryList=calibrationDirectory)
###########################################################################
## STEP 3: Reduce telluric observations. ##
###########################################################################
if telluricReduction:
if manualMode:
a = raw_input('About to enter nifsReduce to reduce Tellurics.')
nifsReduce.start('Telluric', telluricDirectoryList=telluricDirectoryList)
###########################################################################
## STEP 4: Reduce science observations. ##
###########################################################################
if scienceReduction:
if manualMode:
a = raw_input('About to enter nifsReduce to reduce science.')
nifsReduce.start('Science')
###########################################################################
## Data Reduction Complete! ##
## Good luck with your science! ##
###########################################################################
logging.info('#########################################')
logging.info('# #')
logging.info('# DATA REDUCTION COMPLETE #')
logging.info('# Good luck with your science! #')
logging.info('# Check out ?? #')
logging.info('# For docs, tutorials and examples. #')
logging.info('# #')
logging.info('#########################################')
return
def start(args):
"""
NIFTY
This script launches a nifs data reduction.
It does two things; it:
- gets data reduction parameters; either from an interactive input session or
an input file
- launches appropriate scripts to do the work. It can call up to 3 scripts directly:
1) nifsSort.py
2) nifsBaselineCalibration.py
3) nifsReduce.py
"""
# Save path for later use and change one directory up.
path = os.getcwd()
# Get paths to Nifty data.
RECIPES_PATH = pkg_resources.resource_filename('nifty', 'recipes/')
RUNTIME_DATA_PATH = pkg_resources.resource_filename(
'nifty', 'runtimeData/')
# Format logging options.
FORMAT = '%(asctime)s %(message)s'
DATEFMT = datefmt()
# Set up the logging file.
logging.basicConfig(filename='Nifty.log',
format=FORMAT,
datefmt=DATEFMT,
level=logging.DEBUG)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# This lets us logging.info(to stdout AND a logfile. Cool, huh?
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
logging.info("\n####################################")
logging.info("# #")
logging.info("# NIFTY #")
logging.info("# NIFS Data Reduction Pipeline #")
logging.info("# Version " + __version__ + " #")
logging.info("# July 25th, 2017 #")
logging.info("# Marie Lemoine-Busserolle #")
logging.info("# Gemini Observatory, Hilo, Hawaii #")
logging.info("# #")
logging.info("####################################\n")
# Make sure to change this if you change the default logfile.
logging.info('The log file is Nifty.log.\n')
# Parse command line options.
parser = argparse.ArgumentParser(
description='Do a Gemini NIFS data reduction.')
# Ability to repeat the last data reduction
parser.add_argument(
'-r',
'--repeat',
dest='repeat',
default=False,
action='store_true',
help=
'Repeat the last data reduction, loading parameters from runtimeData/config.cfg.'
)
# Ability to load a built-in configuration file (recipe)
parser.add_argument(
'-l',
'--recipe',
dest='recipe',
action='store',
help=
'Load data reduction parameters from the a provided recipe. Default is default_input.cfg.'
)
# Ability to load your own configuration file
parser.add_argument(
dest='inputfile',
action='store',
help='Load data reduction parameters from <inputfile>.cfg.')
# Ability to do a quick and dirty fully automatic data reduction with no user input
# TODO(nat): make it so Nifty does this when you type "niftyRun" with no options
parser.add_argument(
'-f',
'--fullReduction',
dest='fullReduction',
default=False,
action='store_true',
help=
'Do a full reduction with data reduction parameters loaded from runtimeData/default_input.cfg'
)
args = parser.parse_args(args)
repeat = args.repeat
fullReduction = args.fullReduction
inputfile = args.inputfile
# Check if the user specified at command line to repeat the last Reduction, do a full default data reduction from a
# recipe file or do a full data reduction from a handmade file.
if not repeat and not fullReduction and not inputfile:
# If not get user input and check if user specified a full data reduction.
fullReduction = getUserInput()
# TODO(nat): Add proper documentation on supplying an input file name (the args option here).
if fullReduction:
# TODO(nat): move this code to a function.
# Read and use parameters of the last Reduction from runtimeData/config.cfg.
shutil.copy(RECIPES_PATH + 'default_input.cfg',
RUNTIME_DATA_PATH + 'config.cfg')
logging.info(
"\nData reduction parameters for this reduction were copied from recipes/default_input.cfg to runtimeData/config.cfg."
)
if inputfile:
# Load input from a .cfg file user specified at command line.
shutil.copy('./' + inputfile, RUNTIME_DATA_PATH + 'config.cfg')
logging.info("\nPipeline configuration for this data reduction was read from " + str(inputfile) + \
" and copied to ./config.cfg.")
else:
shutil.copy(RUNTIME_DATA_PATH + 'config.cfg', './config.cfg')
logging.info(
"\nPipeline configuration for this data reduction has been written to ./config.cfg"
)
# Print data reduction parameters for a user's peace-of-mind.
logging.info(
"\nParameters for this data reduction as read from that file:\n")
with open('./config.cfg') as config_file:
options = ConfigObj(config_file, unrepr=True)
for i in options:
logging.info(str(i) + " " + str(options[i]))
logging.info("")
# Define parameters used by this script:
with open('./config.cfg') as config_file:
options = ConfigObj(config_file, unrepr=True)
sort = options['sort']
calibrationReduction = options['calibrationReduction']
telluricReduction = options['telluricReduction']
scienceReduction = options['scienceReduction']
debug = options['debug']
###########################################################################
## SETUP COMPLETE ##
## BEGIN DATA REDUCTION ##
## ##
## Four Main Steps: ##
## 1) Sort the Raw Data - nifsSort.py ##
## 2) Reduce baseline calibrations - nifsBaselineCalibration.py ##
## 3) Reduce telluric observations - nifsReduce.py ##
## 4) Reduce science observations - nifsReduce.py ##
## ##
###########################################################################
###########################################################################
## STEP 1: Sort the raw data. ##
###########################################################################
if sort:
if debug:
a = raw_input('About to enter sort.')
nifsSort.start()
printDirectoryLists()
###########################################################################
## STEP 2: Reduce baseline calibrations. ##
###########################################################################
if calibrationReduction:
if debug:
a = raw_input('About to enter calibrate.')
nifsBaselineCalibration.start()
###########################################################################
## STEP 3: Reduce telluric observations. ##
###########################################################################
if telluricReduction:
if debug:
a = raw_input(
'About to enter reduce to reduce Telluric images, create telluric correction spectrum and blackbody spectrum.'
)
nifsReduce.start('Telluric')
###########################################################################
## STEP 4: Reduce science observations. ##
###########################################################################
if scienceReduction:
if debug:
a = raw_input('About to enter reduce to reduce science images.')
nifsReduce.start('Science')
###########################################################################
## Data Reduction Complete! ##
## Good luck with your science! ##
###########################################################################
logging.info('#########################################')
logging.info('# #')
logging.info('# DATA REDUCTION COMPLETE #')
logging.info('# Good luck with your science! #')
logging.info('# Check out ?? #')
logging.info('# For docs, recipes and examples. #')
logging.info('# #')
logging.info('#########################################')
return