def get_baseline_rttc(det_id, hours=24):
"""Retrieve the median and std RTTC values for a given time interval [h]"""
print("Retrieving baseline RTTC")
now = time.time()
det_oid = km3db.tools.todetoid(det_id)
sds = km3db.StreamDS(container="pd")
det = kp.hardware.Detector(det_id=det_id)
clbmap = km3db.CLBMap(det_oid=det_oid)
runs = sds.runs(detid=det_id)
latest_run = int(runs.tail(1).RUN)
run_24h_ago = int(
runs[runs.UNIXSTARTTIME < (now - 60 * 60 * hours) * 1000].tail(1).RUN)
data = OrderedDict()
for param in ['wr_mu'] + ['wr_delta[%d]' % i for i in range(4)]:
data[param] = sds.datalognumbers(parameter_name=param,
detid=det_oid,
minrun=run_24h_ago,
maxrun=latest_run)
baselines = {}
for du in det.dus:
source_name = clbmap.base(du).upi
du_data = OrderedDict()
for param, df in data.items():
values = df[df.SOURCE_NAME == source_name].DATA_VALUE.values
if len(values) == 0:
du_data[param] = (np.nan, np.nan)
continue
du_data[param] = (np.median(values), np.std(values))
rttc_median = du_data['wr_mu'][0] - np.sum(
[du_data[p][0] for p in list(du_data.keys())[1:]])
rttc_std = du_data['wr_mu'][1] - np.sum(
[du_data[p][1] for p in list(du_data.keys())[1:]])
baselines[du] = (rttc_median, rttc_std)
return baselines
def runtable(det_id,
n=5,
run_range=None,
target=None,
compact=False,
sep="\t",
regex=None):
"""Print the run table of the last `n` runs for given detector"""
runs = km3db.StreamDS(container="nt").get("runs", detid=det_id)
if run_range is not None:
try:
from_run, to_run = [int(r) for r in run_range.split("-")]
except ValueError:
log.critical("Please specify a valid range (e.g. 3100-3200)!")
raise SystemExit
else:
runs = [r for r in runs if from_run <= r.run <= to_run]
if regex is not None:
try:
pattern = re.compile(regex)
except re.error:
log.error("Invalid regex!")
return
else:
runs = [
r for r in runs if re.match(pattern, r.runsetupname)
or re.match(pattern, r.runsetupid)
]
if target is not None:
runs = [r for r in runs if r.jobtarget == target.capitalize()]
if n is not None:
runs = runs[-n:]
if not runs:
log.warning("No runs found.")
return
if compact:
attributes = ["run", "runsetupname"]
header = sep.join(attributes)
def lineformatter(entry):
return sep.join([str(getattr(entry, attr)) for attr in attributes])
else:
header = sep.join(runs[0]._fields) # the dataset is homogenious
def lineformatter(entry):
return sep.join(map(str, entry))
print(header)
for entry in runs:
print(lineformatter(entry))
def __init__(self, det_id, should_upload_to_db, log_file="qrunqaqc.log"):
self.det_id = det_id
self.should_upload_to_db = should_upload_to_db
self.log = kp.logger.get_logger("qrunqaqc", filename=log_file)
self.log.info("QAQC analysis started for detector ID %s", det_id)
if should_upload_to_db:
self.log.info(
"The acquired data will be automatically updloaded to "
"the database after each successful run processing.")
self.jpp_version = kp.tools.get_jpp_version()
if self.jpp_version is None:
self.log.critical("Please load a Jpp environment")
raise SystemExit()
else:
print("Run quality determination using Jpp '{}'".format(
self.jpp_version))
self.sds = km3db.StreamDS(container="pd")
self.det_oid = km3db.tools.todetoid(det_id)
self.runtable = self.sds.get("runs", detid=self.det_id)
self.available_run_files = []
cwd = os.getcwd()
self.outdir = os.path.join(cwd, "qparams")
self.jobdir = os.path.join(cwd, "qjobs")
for path in [self.outdir, self.jobdir]:
if not os.path.exists(path):
os.makedirs(path)
self.blacklist = os.path.join(cwd,
"blacklist_{}.txt".format(self.det_id))
self.stats = defaultdict(int)
self._qparams = None
self._columns = None
self._blacklisted_run_ids = None
def configure(self):
self.plots_path = self.require('plots_path')
self.ytick_distance = self.get('ytick_distance', default=200)
self.min_dus = self.get('min_dus', default=1)
self.min_doms = self.get('min_doms', default=4)
self.det_id = self.require('det_id')
self.event_selection_table = self.get('event_selection_table',
default='event_selection')
self.logbook = self.get('logbook', default="Individual+Logbooks")
self.run_id = None
self.t0set = None
self.calib = None
self.max_z = None
self.last_plot_time = 0
self.lower_limits = {}
self.elog = self.get('elog', default=False)
self.sds = km3db.StreamDS()
self.index = 0
def runinfo(run_id, det_id):
runs = km3db.StreamDS(container="nt").get("runs", detid=det_id)
if runs is None:
log.error("No runs found for detector ID {}".format(det_id))
return
for idx, run in enumerate(runs):
if run.run == run_id:
break
else:
log.error("No run with ID {} found for detector ID".format(
run_id, det_id))
return
try:
next_run = runs[idx + 1]
except IndexError:
next_run = None
end_time = duration = float("NaN")
else:
duration = (next_run.unixstarttime - run.unixstarttime) / 1000 / 60
end_time = iso8601utc(next_run.unixstarttime / 1000)
print("Run {0} - detector ID: {1}".format(run_id, det_id))
print("-" * 42)
print(" Start time: {0}\n"
" End time: {1}\n"
" Duration [min]: {2:.2f}\n"
" Start time defined: {3}\n"
" Runsetup ID: {4}\n"
" Runsetup name: {5}\n"
" T0 Calibration ID: {6}".format(
iso8601utc(run.unixstarttime / 1000),
end_time,
duration,
bool(run.starttime_defined),
run.runsetupid,
run.runsetupname,
run.t0_calibsetid,
))
def get_data(stream, parameters, fmt):
"""Retrieve data for given stream and parameters, or None if not found"""
sds = km3db.StreamDS()
if stream not in sds.streams:
log.error("Stream '{}' not found in the database.".format(stream))
return
params = {}
if parameters:
for parameter in parameters:
if "=" not in parameter:
log.error("Invalid parameter syntax '{}'\n"
"The correct syntax is 'parameter=value'".format(
parameter))
continue
key, value = parameter.split("=")
params[key] = value
data = sds.get(stream, fmt, **params)
if data is not None:
try:
print(data)
except BrokenPipeError:
pass
else:
sds.help(stream)
import km3pipe as kp
from docopt import docopt
def diff(first, second):
second = set(second)
return [item for item in first if item not in second]
def duplicates(lst, item):
return [i for i, x in enumerate(lst) if x == item]
args = docopt(__doc__)
sds = km3db.StreamDS(container="pd")
try:
detid = int(args['-d'])
except ValueError:
detid = (args['-d'])
if type(detid)==int:
detid = km3db.tools.todetoid(detid)
directory = args['-o']
ACOUSTIC_BEACONS = [0, 0, 0]
N_DOMS = 18
N_ABS = 3
DOMS = range(N_DOMS + 1)
def main():
args = docopt(__doc__)
DET_ID = int(args["-d"])
TMAX = int(args["-t"])
ET_PER_RUN = int(args["-e"]) * 60 # [s]
RUNS_PER_JOB = int(args["-n"])
VMEM = args["-m"]
CWD = os.getcwd()
LOG_PATH = args["-l"]
JOB_NAME = args["-j"]
CALIB_PATH = os.path.join(CWD, args["OUTPUT_PATH"])
RUN_SUBSTR = args["-s"]
DRYRUN = args["-q"]
if not os.path.exists(CALIB_PATH):
os.makedirs(CALIB_PATH)
run_table = km3db.StreamDS(container="pd").get("run_table", det_id=DET_ID)
phys_run_table = run_table[run_table.RUNSETUPNAME.str.contains(RUN_SUBSTR)]
phys_runs = set(phys_run_table.RUN)
processed_runs = set(
int(re.search("_\\d{8}_(\\d{8})", s).group(1))
for s in glob(os.path.join(CALIB_PATH, "*.k40_cal.p"))
)
remaining_runs = list(phys_runs - processed_runs)
print("Remaining runs: {}".format(remaining_runs))
s = Script()
for job_id, runs_chunk in enumerate(kp.tools.chunks(remaining_runs, RUNS_PER_JOB)):
n_runs = len(runs_chunk)
print("Preparing batch script for a chunk of {} runs.".format(len(runs_chunk)))
s.add("cd $TMPDIR; mkdir -p $USER; cd $USER")
for run in runs_chunk:
s.add("echo Processing {}:".format(run))
irods_path = kp.tools.irods_path(DET_ID, run)
root_filename = os.path.basename(irods_path)
calib_filename = root_filename + ".k40_cal.p"
s.add("iget -v {}".format(irods_path))
s.add(
"CTMIN=$(JPrint -f {}|grep '^ctMin'|awk '{{print $2}}')".format(
root_filename
)
)
s.add(
"k40calib {} {} -t {} -c $CTMIN -o {}".format(
root_filename, DET_ID, TMAX, calib_filename
)
)
s.add("cp {} {}".format(calib_filename, CALIB_PATH))
s.add("rm -f {}".format(root_filename))
s.add("rm -f {}".format(calib_filename))
s.add("echo Run {} processed.".format(run))
s.add("echo " + 42 * "=")
walltime = time.strftime("%H:%M:%S", time.gmtime(ET_PER_RUN * n_runs))
qsub(
s,
"{}_{}".format(JOB_NAME, job_id),
walltime=walltime,
vmem=VMEM,
log_path=LOG_PATH,
irods=True,
dryrun=DRYRUN,
)
if DRYRUN:
break
s.clear()
def available_streams():
"""Show a short list of available streams."""
sds = km3db.StreamDS()
print("Available streams: ")
print(", ".join(sorted(sds.streams)))
def print_info(stream):
"""Print the information about a stream"""
sds = km3db.StreamDS()
sds.help(stream)
def print_streams():
"""Print all available streams with their full description"""
sds = km3db.StreamDS()
sds.print_streams()
