max_num_gch)
glitches = glitches[-max_num_gch:]
if len(cleans) > max_num_cln:
logger.info(' downselecting to the %d most recent cleans' %
max_num_cln)
cleans = cleans[-max_num_cln:]
output = idq.combine_separated_output(these_columns,
[glitches, cleans])
### define weights over time
output['weight'] = calibration.weights(output['GPS'],
weight_type="uniform")
if not opts.dont_cluster:
cluster_dat = idq.dat(output_dir, classifier, ifo, "clustered",
usertag, gpsstart - lookback, lookback +
stride) ### write clustered dat file
logger.info(' writing %s' % cluster_dat)
idq.output_to_datfile(output, cluster_dat)
else:
cluster_dat = idq.dat(output_dir, classifier, ifo, "unclustered",
usertag, gpsstart - lookback,
lookback + stride)
logger.info(' writing %s' % cluster_dat)
idq.output_to_datfile(output, cluster_dat)
### compute rcg from output
r, c, g = idq.dat_to_rcg(output)
logger.info(' N_gch = %d , N_cln = %d' % (g[-1], c[-1]))
lines.append(line)
trained_ranges = dict(
zip(idq.extract_trained_ranges(lines, flavor), lines))
trained_range = idq.best_range(gps, trained_ranges.keys())
lines = trained_ranges[trained_range]
### find best calibration data
cache = calibration_cache[classifier]
lines = cache.readlines(
) ### calibration caches are forced to have a simpler format that train caches
calib_ranges = dict(zip(idq.extract_calibration_ranges(lines), lines))
calib_range = idq.best_range(gps, calib_ranges.keys())
effmap, fapmap = idq.rank_to_EffFAPmap(calib_ranges[calib_range])
### compute filenames
dat = idq.dat(this_output_dir, classifier, ifo, trained_range, usertag,
gps_start, twopadding)
gchxml = idq.xml(directory, classifier, ifo, trained_range,
calib_range, gchtag, gps_start, twopadding)
dats[classifier] = dat ### store for combiners
### perform evalutation
miniconfig = classifiersD[classifier]['config']
if classifiersD[classifier]['mla']:
returncode = idq.execute(flavor,
lines,
dat,
miniconfig,
gps_start_time=gps_start,
gps_end_time=gps_padd,
dir=this_output_dir,
these_columns, glitches, cleans = idq.separate_output( output )
glitches.sort(key=lambda l: l[these_columns['GPS']])
cleans.sort(key=lambda l: l[these_columns['GPS']])
if len(glitches) > max_num_gch:
logger.info(' downselecting to the %d most recent glitches'%max_num_gch)
glitches = glitches[-max_num_gch:]
if len(cleans) > max_num_cln:
logger.info(' downselecting to the %d most recent cleans'%max_num_cln)
cleans = cleans[-max_num_cln:]
output = idq.combine_separated_output( these_columns, [glitches, cleans] )
### define weights over time
output['weight'] = calibration.weights( output['GPS'], weight_type="uniform" )
if not opts.dont_cluster:
cluster_dat = idq.dat(output_dir, classifier, ifo, "clustered", usertag, gpsstart-lookback, lookback+stride) ### write clustered dat file
logger.info(' writing %s'%cluster_dat)
idq.output_to_datfile( output, cluster_dat )
else:
cluster_dat = idq.dat(output_dir, classifier, ifo, "unclustered", usertag, gpsstart-lookback, lookback+stride)
logger.info(' writing %s'%cluster_dat)
idq.output_to_datfile( output, cluster_dat )
### compute rcg from output
r, c, g = idq.dat_to_rcg( output )
logger.info(' N_gch = %d , N_cln = %d'%(g[-1], c[-1]))
### dump into roc file
roc = idq.roc(output_dir, classifier, ifo, usertag, gpsstart-lookback, lookback+stride)
lines.append( line )
line = []
lines.append( line )
trained_ranges = dict( zip( idq.extract_trained_ranges( lines, flavor ), lines ) )
trained_range = idq.best_range( gps, trained_ranges.keys() )
lines = trained_ranges[trained_range]
### find best calibration data
cache = calibration_cache[classifier]
lines = cache.readlines() ### calibration caches are forced to have a simpler format that train caches
calib_ranges = dict( zip(idq.extract_calibration_ranges( lines ), lines ) )
calib_range = idq.best_range( gps, calib_ranges.keys() )
effmap, fapmap = idq.rank_to_EffFAPmap( calib_ranges[calib_range] )
### compute filenames
dat = idq.dat(this_output_dir, classifier, ifo, trained_range, usertag, gps_start, twopadding)
gchxml = idq.xml(directory, classifier, ifo, trained_range, calib_range, gchtag, gps_start, twopadding)
dats[classifier] = dat ### store for combiners
### perform evalutation
miniconfig = classifiersD[classifier]['config']
if classifiersD[classifier]['mla']:
returncode = idq.execute(flavor, lines, dat, miniconfig, gps_start_time=gps_start, gps_end_time=gps_padd, dir=this_output_dir, trgdict=pat, auxmvc_vectors=auxmvc_vectors)
else:
returncode = idq.execute(flavor, lines, dat, miniconfig, gps_start_time=gps_start, gps_end_time=gps_padd, dir=this_output_dir, trgdict=trgdict, samples=samples, samples_header=samples_header)
### check if process has been execited correctly
if returncode:
raise Warning('%s predict failed'%classifier)
raise Warning(' skipping %s timeseries'%classifier)
