max_num_gch) glitches = glitches[-max_num_gch:] if len(cleans) > max_num_cln: logger.info(' downselecting to the %d most recent cleans' % max_num_cln) cleans = cleans[-max_num_cln:] output = idq.combine_separated_output(these_columns, [glitches, cleans]) ### define weights over time output['weight'] = calibration.weights(output['GPS'], weight_type="uniform") if not opts.dont_cluster: cluster_dat = idq.dat(output_dir, classifier, ifo, "clustered", usertag, gpsstart - lookback, lookback + stride) ### write clustered dat file logger.info(' writing %s' % cluster_dat) idq.output_to_datfile(output, cluster_dat) else: cluster_dat = idq.dat(output_dir, classifier, ifo, "unclustered", usertag, gpsstart - lookback, lookback + stride) logger.info(' writing %s' % cluster_dat) idq.output_to_datfile(output, cluster_dat) ### compute rcg from output r, c, g = idq.dat_to_rcg(output) logger.info(' N_gch = %d , N_cln = %d' % (g[-1], c[-1]))
lines.append(line) trained_ranges = dict( zip(idq.extract_trained_ranges(lines, flavor), lines)) trained_range = idq.best_range(gps, trained_ranges.keys()) lines = trained_ranges[trained_range] ### find best calibration data cache = calibration_cache[classifier] lines = cache.readlines( ) ### calibration caches are forced to have a simpler format that train caches calib_ranges = dict(zip(idq.extract_calibration_ranges(lines), lines)) calib_range = idq.best_range(gps, calib_ranges.keys()) effmap, fapmap = idq.rank_to_EffFAPmap(calib_ranges[calib_range]) ### compute filenames dat = idq.dat(this_output_dir, classifier, ifo, trained_range, usertag, gps_start, twopadding) gchxml = idq.xml(directory, classifier, ifo, trained_range, calib_range, gchtag, gps_start, twopadding) dats[classifier] = dat ### store for combiners ### perform evalutation miniconfig = classifiersD[classifier]['config'] if classifiersD[classifier]['mla']: returncode = idq.execute(flavor, lines, dat, miniconfig, gps_start_time=gps_start, gps_end_time=gps_padd, dir=this_output_dir,
these_columns, glitches, cleans = idq.separate_output( output ) glitches.sort(key=lambda l: l[these_columns['GPS']]) cleans.sort(key=lambda l: l[these_columns['GPS']]) if len(glitches) > max_num_gch: logger.info(' downselecting to the %d most recent glitches'%max_num_gch) glitches = glitches[-max_num_gch:] if len(cleans) > max_num_cln: logger.info(' downselecting to the %d most recent cleans'%max_num_cln) cleans = cleans[-max_num_cln:] output = idq.combine_separated_output( these_columns, [glitches, cleans] ) ### define weights over time output['weight'] = calibration.weights( output['GPS'], weight_type="uniform" ) if not opts.dont_cluster: cluster_dat = idq.dat(output_dir, classifier, ifo, "clustered", usertag, gpsstart-lookback, lookback+stride) ### write clustered dat file logger.info(' writing %s'%cluster_dat) idq.output_to_datfile( output, cluster_dat ) else: cluster_dat = idq.dat(output_dir, classifier, ifo, "unclustered", usertag, gpsstart-lookback, lookback+stride) logger.info(' writing %s'%cluster_dat) idq.output_to_datfile( output, cluster_dat ) ### compute rcg from output r, c, g = idq.dat_to_rcg( output ) logger.info(' N_gch = %d , N_cln = %d'%(g[-1], c[-1])) ### dump into roc file roc = idq.roc(output_dir, classifier, ifo, usertag, gpsstart-lookback, lookback+stride)
lines.append( line ) line = [] lines.append( line ) trained_ranges = dict( zip( idq.extract_trained_ranges( lines, flavor ), lines ) ) trained_range = idq.best_range( gps, trained_ranges.keys() ) lines = trained_ranges[trained_range] ### find best calibration data cache = calibration_cache[classifier] lines = cache.readlines() ### calibration caches are forced to have a simpler format that train caches calib_ranges = dict( zip(idq.extract_calibration_ranges( lines ), lines ) ) calib_range = idq.best_range( gps, calib_ranges.keys() ) effmap, fapmap = idq.rank_to_EffFAPmap( calib_ranges[calib_range] ) ### compute filenames dat = idq.dat(this_output_dir, classifier, ifo, trained_range, usertag, gps_start, twopadding) gchxml = idq.xml(directory, classifier, ifo, trained_range, calib_range, gchtag, gps_start, twopadding) dats[classifier] = dat ### store for combiners ### perform evalutation miniconfig = classifiersD[classifier]['config'] if classifiersD[classifier]['mla']: returncode = idq.execute(flavor, lines, dat, miniconfig, gps_start_time=gps_start, gps_end_time=gps_padd, dir=this_output_dir, trgdict=pat, auxmvc_vectors=auxmvc_vectors) else: returncode = idq.execute(flavor, lines, dat, miniconfig, gps_start_time=gps_start, gps_end_time=gps_padd, dir=this_output_dir, trgdict=trgdict, samples=samples, samples_header=samples_header) ### check if process has been execited correctly if returncode: raise Warning('%s predict failed'%classifier) raise Warning(' skipping %s timeseries'%classifier)