Python DataLoader Exemples

Exemple #1

Fichier : plot_glitches.py Projet : janvimadhani/ACT-Pipeline---OLD

from todloop.routines import DataLoader
from todloop.tod import TODLoader, TODInfoLoader
from todloop.base import TODLoop
from routines import PlotGlitches
from calibration.routines import FixOpticalSign, CalibrateTOD

tod_id = 1300
loop = TODLoop()
loop.add_tod_list("../data/s16_pa3_list.txt")

#will load the data stored in the folder specified one by one
#and store the loaded data in the shared date store under a key called data
loop.add_routine(DataLoader(input_dir="../outputs/s16_pa3_list/cosig/", output_key="cuts"))
loop.add_routine(TODLoader(output_key="tod_data"))
loop.add_routine(FixOpticalSign(input_key="tod_data", output_key="tod_data"))
loop.add_routine(CalibrateTOD(input_key="tod_data",output_key="tod_data"))
loop.add_routine(PlotGlitches(tag=tod_id,tod_key="tod_data", cosig_key="cuts"))
loop.run(tod_id,tod_id + 1)

Exemple #2

Fichier : get_tracks.py Projet : pitt-cosmos/act-glitch-studies

from todloop.base import TODLoop
from todloop.routines import DataLoader
from todloop.filters import DurationFilter, PixelFilter, SpreadFilter
from routines import GetTracks, PlotTracks

loop = TODLoop()
loop.add_tod_list("data/s17_pa4_sublist.txt")

# add sample handler
loop.add_routine(DataLoader(input_dir="outputs/s17_pa4_sublist_cosig/"))
#loop.add_routine(DataLoader(input_dir="outputs/coincident_signals_subset/"))

# add filters
#loop.add_routine(DurationFilter(max_duration=10))
#loop.add_routine(PixelFilter(max_pixels=5))

# add main routine
loop.add_routine(GetTracks(season='2017', array='AR4'))
#loop.add_routine(SpreadFilter(max_spread=1))

# add plot routine
#loop.add_routine(PlotTracks(output_dir="outputs/get_tracks_filtered/", season='2017', array='AR4'))
loop.add_routine(
    PlotTracks(output_dir="outputs/s17_pa4_sublist_tracks/",
               season='2017',
               array='AR4'))

# specify range of tods of interests
loop.run(0, 2)

Exemple #3

Fichier : covered_tods.py Projet : janvimadhani/ACT-Pipeline---OLD

from todloop.routines import DataLoader
from todloop.tod import TODLoader
from todloop.base import TODLoop
from routines import TimeSeries, PlotGlitches, Energy, SaveEvents, NPixelStudy, EnergyStudy, CRCorrelationFilter
from calibration.routines import FixOpticalSign, CalibrateTOD
"""
INITIALIZE TODLoop
"""
loop = TODLoop()
tod_id = 0
loop.add_tod_list("../data/covered_tods.txt")
loop.add_routine(
    DataLoader(input_dir="../outputs/covered_tods_cosig/", output_key="cuts"))
"""
LOAD TOD DATA
"""
loop.add_routine(TODLoader(output_key="tod_data"))
loop.add_routine(FixOpticalSign(input_key="tod_data", output_key="tod_data"))
loop.add_routine(CalibrateTOD(input_key="tod_data", output_key="tod_data"))
"""
ROUTINES
"""
loop.add_routine(TimeSeries(tod_key="tod_data", output_key="timeseries"))
loop.add_routine(
    Energy(timeseries_key="timeseries", output_key="energy_calculator"))
loop.add_routine(
    CRCorrelationFilter(timeseries_key="timeseries",
                        cosig_key="cuts",
                        tod_key="tod_data",
                        output_key="cr_cuts"))
#loop.add_routine(PlotGlitches(tag=tod_id,cosig_key="cr_cuts",tod_key="tod_data",timeseries_key = "timeseries"))

Exemple #4

from routines import TimeSeries, PlotGlitches, Energy, SaveEvents, NPixelStudy, EnergyStudy, CRCorrelationFilter, RaDecFilter, Timer
from calibration.routines import FixOpticalSign, CalibrateTOD
"""
INITIALIZE TODLoop
"""
loop = TODLoop()
tod_id = 15
loop.add_routine(Timer())
"""
ICECUBE
"""
#"""
loop.add_tod_list("../data/s17_icecube_list.txt")
loop.add_skip([128, 129, 130, 131])
loop.add_routine(
    DataLoader(input_dir="../outputs/s17_icecube_list/cosig/",
               output_key="cuts"))
#"""
"""
LOAD TOD DATA
"""
loop.add_routine(TODLoader(output_key="tod_data"))
loop.add_routine(FixOpticalSign(input_key="tod_data", output_key="tod_data"))
loop.add_routine(CalibrateTOD(input_key="tod_data", output_key="tod_data"))
"""
ROUTINES
"""
loop.add_routine(TimeSeries(tod_key="tod_data", output_key="timeseries"))
loop.add_routine(
    Energy(timeseries_key="timeseries", output_key="energy_calculator"))
loop.add_routine(
    SaveEvents(tag=tod_id,

Exemple #5

from todloop import TODLoop
from todloop.routines import DataLoader
from todloop.cosig import FindCosigs

# initialize TODLoop
loop = TODLoop()

# specify covered tod input list
loop.add_tod_list("inputs/covered_tods.txt")

# load cuts
loop.add_routine(DataLoader(input_dir="outputs/nSig_10/cuts",
                            output_key="cuts"))

# compile coincident signals and save data
loop.add_routine(FindCosigs(input_key="cuts", season='2017',
                            output_dir="outputs/nSig_10/cosigs"))

loop.run(0, 153)

Exemple #6

Fichier : cr_hist.py Projet : janvimadhani/ACT-Pipeline---OLD

from todloop.base import TODLoop
from todloop.tod import TODInfoLoader
from todloop.routines import DataLoader
from routines import NPixelStudy, CRHourStudy, CRPWVStudy, DurationStudy
from todloop.event import NPixelFilter, CoeffFilter

loop = TODLoop()
loop.add_tod_list("../data/s16_pa3_list.txt")
loop.add_routine(
    DataLoader(input_dir="../outputs/s16_pa3_list/events/",
               output_key="events"))

# loop.add_routine(NPixelFilter(min_pixels=0, max_pixels=20, input_key="events", output_key="events"))
# loop.add_routine(CoeffFilter(min_coeff=0.9, input_key="events", output_key="events"))
loop.add_routine(NPixelStudy())
loop.add_routine(DurationStudy())
# loop.add_routine(TODInfoLoader(output_key="tod_info"))
# loop.add_routine(CRHourStudy())
# loop.add_routine(CRPWVStudy())
loop.run(0, 10000)

Exemple #7

Fichier : glitches_hour.py Projet : pitt-cosmos/act-glitch-studies

from todloop.base import TODLoop
from todloop.tod import TODInfoLoader
from todloop.routines import DataLoader
from todloop.filters import PixelFilter, DurationFilter
from routines import GlitchHourStudy, GlitchPWVStudy, GlitchAzStudy

loop = TODLoop()
loop.add_tod_list("data/s16_pa3_list.txt")
loop.add_routine(TODInfoLoader(output_key="tod_info"))
loop.add_routine(
    DataLoader(input_dir="outputs/coincident_signals_subset/",
               output_key="cosig"))
#loop.add_routine(PixelFilter(input_key="cosig", output_key="cosig", max_pixels=7))
#loop.add_routine(DurationFilter(input_key="cosig", output_key="cosig", max_duration=10))
#loop.add_routine(GlitchHourStudy())
#loop.add_routine(GlitchPWVStudy())
loop.add_routine(GlitchAzStudy())
loop.run(0, 14000)

Exemple #8

Fichier : ra_dec.py Projet : pitt-cosmos/act-glitch-studies

from todloop.event import LoadRaDec, NPixelFilter, CoeffFilter, DurationFilter
from todloop.base import TODLoop
from calibration.routines import FixOpticalSign
from todloop.routines import DataLoader, Logger
from todloop.tod import TODLoader
from plot_events.routines import PlotEvents
from cr_hist.routines import RaDecStudy

loop = TODLoop()
loop.add_tod_list("../data/s17_icecube_list.txt")
loop.add_skip([128, 129, 130, 131])  # problematic ones
loop.add_routine(
    DataLoader(input_dir="../outputs/s17_icecube_list/events/",
               output_key="events"))
#loop.add_routine(NPixelFilter(min_pixels=0, max_pixels=3, input_key="events", output_key="events"))
#loop.add_routine(DurationFilter(min_duration=10, max_duration=60, input_key="events", output_key="events"))
loop.add_routine(
    CoeffFilter(min_coeff=0.0, input_key="events", output_key="events"))
# loop.add_routine(TODLoader(output_key="tod-data"))
# loop.add_routine(FixOpticalSign(input_key="tod-data", output_key="tod-data"))
# loop.add_routine(Logger(input_key="events"))
# loop.add_routine(PlotEvents(event_key="events", tod_key="tod-data"))
loop.add_routine(LoadRaDec(input_key="events", output_key="events"))
loop.add_routine(
    RaDecStudy(input_key="events", ra_range=[1.2, 1.4], dec_range=[0.08,
                                                                   0.11]))
loop.run(0, 127)

Exemple #9

from todloop.base import TODLoop
from todloop.routines import Logger, SaveData, DataLoader
from routines import RemoveMCE, TrimEdges, FindCosigs, FindEvents

tod_index = 152
loop = TODLoop()
loop.add_tod_list("../data/covered_tods.txt")
loop.add_routine(DataLoader(input_dir="../compile_cuts/outputs/covered_tods/", output_key="cut"))
loop.add_routine(RemoveMCE(input_key="cut", output_key="cut"))
loop.add_routine(TrimEdges(input_key="cut", output_key="cut"))
loop.add_routine(FindCosigs(input_key="cut", output_key="cosig", season='2017'))
loop.add_routine(FindEvents(input_key="cosig", output_key="data"))
loop.add_routine(Logger(input_key="data"))
loop.add_routine(SaveData(input_key="data", output_dir="../outputs/covered_tods_cosig/"))

loop.run(tod_index, tod_index + 1)

Exemple #10

import matplotlib
matplotlib.use("TKAgg")
from todloop import TODLoop
from todloop.routines import DataLoader, SaveData
from todloop.cosig import FindEvents
from todloop.tod import TODLoader, FixOpticalSign
from routines import PlotEvents

# initialize TODLoop
loop = TODLoop()

# specify covered tod input list
loop.add_tod_list("inputs/covered_tods.txt")

# load TOD
loop.add_routine(TODLoader(output_key="tod_data"))
loop.add_routine(FixOpticalSign(input_key="tod_data", output_key="tod_data"))
# load events
loop.add_routine(
    DataLoader(input_dir="outputs/nSig_10/events", output_key="events"))

# plot events
loop.add_routine(PlotEvents(event_key="events", tod_key="tod_data"))

loop.run(91, 153)

Exemple #11

Fichier : find_events.py Projet : guanyilun/cr-studies

from todloop import TODLoop
from todloop.routines import DataLoader, SaveData
from todloop.cosig import FindEvents

# initialize TODLoop
loop = TODLoop()

# specify covered tod input list
loop.add_tod_list("inputs/covered_tods.txt")

# load cosigs
loop.add_routine(
    DataLoader(input_dir="outputs/nSig_10/cosigs", output_key="cosig"))

# compile coincident signals and save data
loop.add_routine(FindEvents(input_key="cosig", output_key="events"))

# save data
loop.add_routine(
    SaveData(input_key="events", output_dir="outputs/nSig_10/events"))

loop.run(0, 153)

Exemple #12

from todloop.base import TODLoop
from todloop.tod import TODInfoLoader
from todloop.routines import DataLoader
from routines import NPixelStudy, CRHourStudy, CRPWVStudy, DurationStudy
from todloop.event import NPixelFilter, CoeffFilter

loop = TODLoop()
loop.add_tod_list("../data/s16_pa3_list.txt")
loop.add_routine(DataLoader(input_dir="../outputs/s16_pa3_list/events/", output_key="events"))

# loop.add_routine(NPixelFilter(min_pixels=0, max_pixels=20, input_key="events", output_key="events"))
loop.add_routine(CoeffFilter(min_coeff=0.9, input_key="events", output_key="events"))
loop.add_routine(NPixelStudy())
loop.add_routine(DurationStudy())
# loop.add_routine(TODInfoLoader(output_key="tod_info"))
# loop.add_routine(CRHourStudy())
# loop.add_routine(CRPWVStudy())
loop.run(0,10)

Exemple #13

from todloop.base import TODLoop
from todloop.routines import Logger, SaveData, DataLoader
from coincident_signals.routines import RemoveMCE, TrimEdges, FindCosigs, FindEvents

loop = TODLoop()
loop.add_tod_list("data/s17_icecube_list.txt")
loop.add_routine(DataLoader(input_dir="outputs/s17_icecube_list/cut/", output_key="cut"))
loop.add_routine(RemoveMCE(input_key="cut", output_key="cut"))
loop.add_routine(TrimEdges(input_key="cut", output_key="cut"))
loop.add_routine(FindCosigs(input_key="cut", output_key="cosig", season="2017"))
loop.add_routine(FindEvents(input_key="cosig", output_key="data"))
loop.add_routine(Logger(input_key="data"))
loop.add_routine(SaveData(input_key="data", output_dir="outputs/s17_icecube_list/cosig/"))

loop.run(0, 155)

Exemple #14

Fichier : transform.py Projet : pitt-cosmos/act-glitch-studies

from todloop.routines import Logger, DataLoader, SaveData
from todloop.tod import TODLoader, TODInfoLoader
from todloop.base import TODLoop
from routines import CosigToEvent

loop = TODLoop()
tod_id = 3731
loop.add_tod_list("../data/s16_pa3_list.txt")
loop.add_routine(
    DataLoader(input_dir="../outputs/s16_pa3_list/cosig/", output_key="cosig"))
loop.add_routine(CosigToEvent(input_key="cosig", output_key="events"))
loop.add_routine(
    SaveData(input_key="events",
             output_dir="../outputs/s16_pa3_list/events_raw/"))
loop.run(tod_id, tod_id + 1)