def add_cut_routines(loop):
"""This function registers a series of common routines for cut
analysis. This is so that we don't have to keep repeating
ourselves to register these routines for each data set (train,
validate, test).
"""
# add a routine to load tod
loader_params = {
'output_key': 'tod',
'load_opts': {
'fix_sign': True,
'repair_pointing': True
}
}
loop.add_routine(TODLoader(**loader_params))
# add a routine to cut the sources
source_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'tod': 'tod'
},
'tag_source': '%s_source' % tag,
'no_noise': True,
'depot': DEPOT,
'write_depot': True,
}
loop.add_routine(CutSources(**source_params))
# add a routine to cut the planets
planets_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'tod': 'tod',
},
'tag_planet': '%s_planet' % tag,
'depot': DEPOT,
'pointing_par': {'source': 'fp_file', \
'filename': actpol_shared + "/RelativeOffsets/template_ar2_s16_170131.txt",
},
'mask_params': {
'radius': (8./60) #degrees
},
'mask_shift_generator': {
'source':'file',\
'filename':actpol_shared + '/TODOffsets/tod_offsets_2016_170131.txt',
'columns': [0,3,4],
'rescale_degrees': 1./60
},
'write_depot': True,
}
loop.add_routine(CutPlanets(**planets_params))
# add a routine to remove the sync pick up
sync_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'tod': 'tod',
},
'tag_sync': '%s' % tag,
'remove_sync': False,
'force_sync': False,
'depot': DEPOT,
'write_depot': True,
}
loop.add_routine(RemoveSyncPickup(**sync_params))
# add a routine to cut the glitches
partial_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'tod': 'tod',
},
'tag_partial': '%s_partial' % tag,
'include_mce': True,
'force_partial': False,
'glitchp': { 'nSig': 10., 'tGlitch' : 0.007, 'minSeparation': 30, \
'maxGlitch': 50000, 'highPassFc': 6.0, 'buffer': 200 },
'depot': DEPOT,
'write_depot': True,
}
loop.add_routine(CutPartial(**partial_params))
# add a routine to transform the TODs
transform_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'tod': 'tod',
},
'remove_mean': False,
'remove_median': True,
'detrend': False,
'remove_filter_gain': False,
'n_downsample': 1, # reduction with 2^n factor
}
loop.add_routine(TransformTOD(**transform_params))
# add a routine to analyze the scan
scan_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'scan': 'scan_params',
}
}
loop.add_routine(AnalyzeScan(**scan_params))
# add a routine to get the relevant detectors to look at
BASE_DIR = actpol_shared + '/ArrayData/2015/ar2/'
gd_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'dets': 'dets',
},
'source': 'individual',
'live': BASE_DIR + 'live_ext2.dict',
'dark': BASE_DIR + 'dark_ext.dict',
'exclude': BASE_DIR + 'exclude.dict'
}
loop.add_routine(GetDetectors(**gd_params))
# add a routine to calibrate DAQ units to pW using flatfield and
# responsivity
cal_params = {
'inputs': {
'tod': 'tod',
'dets': 'dets',
},
'outputs': {
'tod': 'tod',
'cal': 'calData'
},
'flatfield': "/mnt/act3/users/mhasse/shared/actpol_shared_depot/FlatFields/2015/" + \
"ff_mixed_ar2_v0_actpol2_2015_c9_v0_photon_it0.dict",
'config': [{
"type": "depot_cal",
"depot": DEPOT,
"tag": "actpol2_2016_biasstep",
"name": "biasstep"
}, {
"type": "constant",
"value": 0.821018,
"name": "DC bias factor"
}],
'forceNoResp': True,
'calibrateTOD': True,
}
loop.add_routine(CalibrateTOD(**cal_params))
# add a routine to find jumps in TOD
jump_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'jumps': 'jumps'
},
'dsStep': 4,
'window': 1,
}
loop.add_routine(FindJumps(**jump_params))
# add a routine to perform the fourior transform
fft_params = {
'inputs': {
'tod': 'tod'
},
'outputs': {
'tod': 'tod',
'fft': 'fft_data'
},
}
loop.add_routine(FouriorTransform(**fft_params))
# study the dark detectors using LF data
lf_dark_params = {
'inputs': {
'tod': 'tod',
'fft': 'fft_data',
'dets': 'dets',
'scan': 'scan_params',
},
'outputs': {
'lf_dark': 'lf_dark',
},
'cancelSync': False,
'doubleMode': False,
'freqRange': {
'fmin': 0.017,
'fshift': 0.009,
'band': 0.071,
'Nwin': 1,
},
}
loop.add_routine(AnalyzeDarkLF(**lf_dark_params))
# study the live detectors using LF data
lf_live_params = {
'inputs': {
'tod': 'tod',
'fft': 'fft_data',
'dets': 'dets',
'scan': 'scan_params',
'dark': 'lf_dark',
'cal': 'calData'
},
'outputs': {
'lf_live': 'lf_live',
},
'cancelSync': True,
'doubleMode': False,
'removeDark': True,
'freqRange': {
'fmin': 0.017,
'fshift': 0.009,
'band': 0.071,
'Nwin': 10,
},
'separateFreqs': False,
'darkModesParams': {
'useDarks': True,
'useSVD': True,
'Nmodes': 1,
'useTherm': False
},
}
loop.add_routine(AnalyzeLiveLF(**lf_live_params))
# get the drift errors
de_params = {
'inputs': {
'tod': 'tod',
'fft': 'fft_data',
'dets': 'dets',
'scan': 'scan_params',
},
'outputs': {
'drift': 'drift',
},
'driftFilter': 0.036,
'nmodes': 3,
}
loop.add_routine(GetDriftErrors(**de_params))
# study the live detectors in mid-freq
mf_params = {
'inputs': {
'tod': 'tod',
'fft': 'fft_data',
'dets': 'dets',
'scan': 'scan_params',
},
'outputs': {
'mf_live': 'mf_live'
},
'midFreqFilter': [0.3, 1.0],
'nmodes': 8,
}
loop.add_routine(AnalyzeLiveMF(**mf_params))
# study the live and dark detectors in HF
hf_params = {
'inputs': {
'tod': 'tod',
'fft': 'fft_data',
'dets': 'dets',
'scan': 'scan_params',
},
'outputs': {
'hf': 'hf'
},
'getPartial': False,
'highFreqFilter': [9.0, 19.0],
'nLiveModes': 10,
'nDarkModes': 3,
'highOrder': True,
}
loop.add_routine(AnalyzeHF(**hf_params))
# add the routine to compute jesse's features
params = {
'inputs': {
'tod': 'tod',
},
'outputs': {
'results': 'jesse_features',
}
}
loop.add_routine(JesseFeatures(**params))
# summarize the pickle parameters
summary_params = {
'inputs': {
# calculated features to include in the report
'features':
['lf_live', 'drift', 'mf_live', 'hf', 'jumps', 'jesse_features'],
},
'outputs': {
'report': 'report',
}
}
loop.add_routine(Summarize(**summary_params))
return loop
from todloop.routines import DataLoader
from todloop.tod import TODLoader, TODInfoLoader
from todloop.base import TODLoop
from routines import PlotGlitches
from calibration.routines import FixOpticalSign, CalibrateTOD
tod_id = 1300
loop = TODLoop()
loop.add_tod_list("../data/s16_pa3_list.txt")
#will load the data stored in the folder specified one by one
#and store the loaded data in the shared date store under a key called data
loop.add_routine(DataLoader(input_dir="../outputs/s16_pa3_list/cosig/", output_key="cuts"))
loop.add_routine(TODLoader(output_key="tod_data"))
loop.add_routine(FixOpticalSign(input_key="tod_data", output_key="tod_data"))
loop.add_routine(CalibrateTOD(input_key="tod_data",output_key="tod_data"))
loop.add_routine(PlotGlitches(tag=tod_id,tod_key="tod_data", cosig_key="cuts"))
loop.run(tod_id,tod_id + 1)
from todloop.tod import TODLoader, TODInfoLoader
from todloop.base import TODLoop
from todloop.routines import Logger
from routines import CompileCuts
loop = TODLoop()
loop.add_tod_list("data/s16_pa3_list.txt")
glitchp = {
'nSig': 10,
'tGlitch' : 0.007,
'minSeparation': 30,
'maxGlitch': 50000,
'highPassFc': 6.0,
'buffer': 0
}
loop.add_routine(TODLoader(output_key="tod_data", abspath=True))
# loop.add_routine(TODInfoLoader(output_key="tod_info"))
loop.add_routine(CompileCuts(input_key="tod_data", glitchp=glitchp, output_dir="outputs/s16_pa3_list/"))
loop.run(0, 10)