def get_exposure_table_path(night=None, usespecprod=True):
"""
Defines the default path to save an exposure table. If night is given, it saves it under a monthly directory
to reduce the number of files in a large production directory.
Args:
night, int or str or None. The night corresponding to the exposure table. If None, no monthly subdirectory is used.
usespecprod, bool. Whether to use the master version in the exposure table repo or the version in a specprod.
Returns:
str. The full path to the directory where the exposure table should be written (or is already written). This
does not including the filename.
"""
# if night is None and 'PROD_NIGHT' in os.environ:
# night = os.environp['PROD_NIGHT']
if usespecprod:
basedir = define_variable_from_environment(env_name='DESI_SPECTRO_REDUX',
var_descr="The specprod path")
# subdir = define_variable_from_environment(env_name='USER', var_descr="Username for unique exposure table directories")
subdir = define_variable_from_environment(env_name='SPECPROD', var_descr="Use SPECPROD for unique exposure table directories")
basedir = pathjoin(basedir, subdir)
else:
basedir = define_variable_from_environment(env_name='DESI_SPECTRO_LOG',
var_descr="The exposure table repository path")
if night is None:
return pathjoin(basedir,'exposure_tables')
else:
month = night_to_month(night)
path = pathjoin(basedir,'exposure_tables',month)
return path
def get_processing_table_name(specprod=None, prodmod=None, extension='csv'):
"""
Defines the default processing name given the specprod of the production and the optional extension.
Args:
specprod, str or None. The name of the production. If None, it will be taken from the environment variable.
prodmod, str. Additional str that can be added to the production table name to further differentiate it.
Used in daily workflow to add the night to the name and make it unique from other nightly tables.
extension, str. The extension (and therefore data format) without a leading period of the saved table.
Default is 'csv'.
Returns:
str. The processing table name given the input night and extension.
"""
if specprod is None:
specprod = define_variable_from_environment(env_name='SPECPROD',
var_descr="Use SPECPROD for unique processing table directories")
if prodmod is not None:
prodname_modifier = '-' + str(prodmod)
elif 'SPECPROD_MOD' in os.environ:
prodname_modifier = '-' + os.environ['SPECPROD_MOD']
else:
prodname_modifier = ''
return f'processing_table_{specprod}{prodname_modifier}.{extension}'
def get_processing_table_path(specprod=None):
"""
Defines the default path to save a processing table. If specprod is not given, the environment variable
'SPECPROD' must exist.
Args:
specprod, str or None. The name of the production. If None, it will be taken from the environment variable.
Returns:
str. The full path to the directory where the processing table should be written (or is already written). This
does not including the filename.
"""
if specprod is None:
specprod = define_variable_from_environment(env_name='SPECPROD',
var_descr="Use SPECPROD for unique processing table directories")
basedir = define_variable_from_environment(env_name='DESI_SPECTRO_REDUX',
var_descr="The specprod path")
path = pathjoin(basedir, specprod, 'processing_tables')
return path
def get_processing_table_pathname(specprod=None, prodmod=None, extension='csv'): # base_path,specprod
"""
Defines the default pathname to save a processing table.
Args:
specprod, str or None. The name of the production. If None, it will be taken from the environment variable.
prodmod, str. Additional str that can be added to the production table name to further differentiate it.
Used in daily workflow to add the night to the name and make it unique from other nightly tables.
extension, str. The extension (and therefore data format) without a leading period of the saved table.
Default is 'csv'.
Returns:
str. The full pathname where the processing table should be written (or is already written). This
includes the filename.
"""
if specprod is None:
specprod = define_variable_from_environment(env_name='SPECPROD',
var_descr="Use SPECPROD for unique processing table directories")
path = get_processing_table_path(specprod)
table_name = get_processing_table_name(specprod, prodmod, extension)
return pathjoin(path, table_name)
def create_exposure_tables(nights=None, night_range=None, path_to_data=None, exp_table_path=None, obstypes=None, \
exp_filetype='csv', cameras='', bad_cameras='', badamps='',
verbose=False, no_specprod=False, overwrite_files=False):
"""
Generates processing tables for the nights requested. Requires exposure tables to exist on disk.
Args:
nights: str, int, or comma separated list. The night(s) to generate procesing tables for.
night_range: str, comma separated pair of nights in form YYYYMMDD,YYYYMMDD for first_night,last_night
specifying the beginning and end of a range of nights to be generated.
last_night should be inclusive.
path_to_data: str. The path to the raw data and request*.json and manifest* files.
exp_table_path: str. Full path to where to exposure tables should be saved, WITHOUT the monthly directory included.
obstypes: str or comma separated list of strings. The exposure OBSTYPE's that you want to include in the exposure table.
exp_filetype: str. The file extension (without the '.') of the exposure tables.
verbose: boolean. Whether to give verbose output information or not. True prints more information.
no_specprod: boolean. Create exposure table in repository location rather than the SPECPROD location
overwrite_files: boolean. Whether to overwrite processing tables if they exist. True overwrites.
cameras: str. Explicitly define the cameras for which you want to reduce the data. Should be a comma separated
list. Only numbers assumes you want to reduce r, b, and z for that camera. Otherwise specify
separately [brz][0-9].
bad_cameras: str. Explicitly define the cameras that you don't want to reduce the data. Should be a comma
separated list. Only numbers assumes you want to reduce r, b, and z for that camera.
Otherwise specify separately [brz][0-9].
badamps: str. Define amplifiers that you know to be bad and should not be processed. Should be a list separated
by comma or semicolon. Saved list will converted to semicolons. Each entry should be of the
form {camera}{spectrograph}{amp}, i.e. [brz][0-9][A-D].
Returns: Nothing
"""
if nights is None and night_range is None:
raise ValueError("Must specify either nights or night_range")
elif nights is not None and night_range is not None:
raise ValueError("Must only specify either nights or night_range, not both")
if nights is None or nights=='all':
nights = list()
for n in listpath(os.getenv('DESI_SPECTRO_DATA')):
#- nights are 20YYMMDD
if re.match('^20\d{6}$', n):
nights.append(n)
else:
nights = [ int(val.strip()) for val in nights.split(",") ]
nights = np.array(nights)
if night_range is not None:
if ',' not in night_range:
raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")
nightpair = night_range.split(',')
if len(nightpair) != 2 or not nightpair[0].isnumeric() or not nightpair[1].isnumeric():
raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")
first_night, last_night = nightpair
nights = nights[np.where(int(first_night)<=nights.astype(int))[0]]
nights = nights[np.where(int(last_night)>=nights.astype(int))[0]]
if obstypes is not None:
obstypes = [ val.strip('\t ') for val in obstypes.split(",") ]
else:
obstypes = default_exptypes_for_exptable()
print("Nights: ", nights)
print("Obs types: ", obstypes)
## Deal with cameras and amps, if given
camword = cameras
if camword != '':
camword = parse_cameras(camword)
badcamword = bad_cameras
if badcamword != '':
badcamword = parse_cameras(badcamword)
## Warn people if changing camword
finalcamword = 'a0123456789'
if camword is not None and badcamword is None:
badcamword = difference_camwords(finalcamword,camword)
finalcamword = camword
elif camword is not None and badcamword is not None:
finalcamword = difference_camwords(camword, badcamword)
badcamword = difference_camwords('a0123456789', finalcamword)
elif badcamword is not None:
finalcamword = difference_camwords(finalcamword,badcamword)
else:
badcamword = ''
if badcamword != '':
## Inform the user what will be done with it.
print(f"Modifying camword of data to be processed with badcamword: {badcamword}. " + \
f"Camword to be processed: {finalcamword}")
## Make sure badamps is formatted properly
if badamps is None:
badamps = ''
else:
badamps = validate_badamps(badamps)
## Define where to find the data
if path_to_data is None:
path_to_data = define_variable_from_environment(env_name='DESI_SPECTRO_DATA',
var_descr="The data path")
## Define where to save the data
usespecprod = (not no_specprod)
if exp_table_path is None:
exp_table_path = get_exposure_table_path(night=None,usespecprod=usespecprod)
## Make the save directory exists
os.makedirs(exp_table_path, exist_ok=True)
## Loop over nights
colnames, coltypes, coldefaults = get_exposure_table_column_defs(return_default_values=True)
for night in nights:
if str(night) not in listpath(path_to_data):
print(f'Night: {night} not in data directory {path_to_data}. Skipping')
continue
print(get_printable_banner(input_str=night))
## Create an astropy exposure table for the night
nightly_tab = instantiate_exposure_table()
## Loop through all exposures on disk
for exp in listpath(path_to_data,str(night)):
rowdict = summarize_exposure(path_to_data, night=night, exp=exp, obstypes=obstypes, \
colnames=colnames, coldefaults=coldefaults, verbosely=verbose)
if rowdict is not None and type(rowdict) is not str:
rowdict['BADCAMWORD'] = badcamword
rowdict['BADAMPS'] = badamps
## Add the dictionary of column values as a new row
nightly_tab.add_row(rowdict)
if verbose:
print("Rowdict:\n",rowdict,"\n\n")
if len(nightly_tab) > 0:
month = night_to_month(night)
exptab_path = pathjoin(exp_table_path,month)
os.makedirs(exptab_path,exist_ok=True)
exptab_name = get_exposure_table_name(night, extension=exp_filetype)
exptab_name = pathjoin(exptab_path, exptab_name)
write_table(nightly_tab, exptab_name, overwrite=overwrite_files)
else:
print('No rows to write to a file.')
print("Exposure table generations complete")
## Flush the outputs
sys.stdout.flush()
sys.stderr.flush()