def init_job_types(config_parser, job_types = ("datafind", "rm", "binj", "power", "lladd", "binjfind", "bucluster", "bucut", "burca", "burca2", "sqlite", "burcatailor")): """ Construct definitions of the submit files. """ global datafindjob, rmjob, binjjob, powerjob, lladdjob, binjfindjob, buclusterjob, llb2mjob, bucutjob, burcajob, burca2job, sqlitejob, burcatailorjob # ligo_data_find if "datafind" in job_types: datafindjob = pipeline.LSCDataFindJob(os.path.join(os.getcwd(), get_cache_dir(config_parser)), os.path.join(os.getcwd(), get_out_dir(config_parser)), config_parser) # rm if "rm" in job_types: rmjob = RMJob(config_parser) # lalapps_binj if "binj" in job_types: binjjob = BurstInjJob(config_parser) # lalapps_power if "power" in job_types: powerjob = PowerJob(config_parser) # ligolw_add if "lladd" in job_types: lladdjob = pipeline.LigolwAddJob(os.path.join(get_out_dir(config_parser)), config_parser) lladdjob.cache_dir = get_cache_dir(config_parser) # lalapps_binjfind if "binjfind" in job_types: binjfindjob = BinjfindJob(config_parser) # lalapps_bucut if "bucut" in job_types: bucutjob = BucutJob(config_parser) # lalapps_bucluster if "bucluster" in job_types: buclusterjob = BuclusterJob(config_parser) # lalapps_burca if "burca" in job_types: burcajob = BurcaJob(config_parser) # lalapps_burca2 if "burca2" in job_types: burca2job = Burca2Job(config_parser) # ligolw_sqlite if "sqlite" in job_types: sqlitejob = SQLiteJob(config_parser) # lalapps_burca_tailor if "burcatailor" in job_types: burcatailorjob = BurcaTailorJob(config_parser)
for ifo in ifo_list: if not cp.has_option('segments', ifo.lower() + '-analyze'): continue # decide if we need to segment the data available_segments = get_valid_segments( cp.get('segfind', 'segment-url'), cp.get('framefind', 'base-dir'), ifo, cp.get('segments', ifo.lower() + '-analyze'), gps_start_time, gps_end_time) if not available_segments: print("No available segments for %s, skipping" % ifo) continue # create the Condor jobs that will be used in the DAG df_job = pipeline.LSCDataFindJob('cache', 'logs', cp) tmplt_job = inspiral.TmpltBankJob(cp) # Based on S6A results ttrigscan clustering has # been replaced with 30-ms window clustering # ts_job = TrigscanJob(cp) si_job_coarse = SiClusterJobCoarse(cp) si_job_fine = SiClusterJobFine(cp) cp_job = FilesystemJob('cp') # Add ifo-specific template config if cp.has_section(ifo.lower() + '-tmpltbank'): tmplt_job.add_ini_opts(cp, ifo.lower() + '-tmpltbank') # Create a job to split the template into parallelization pieces
segdir=os.path.join(opts.run_path,'segments') checkDir(segdir) os.chdir(segdir) science_segs={} seg_files={} segs={} basename="nest_%.3f-%.3f"%(starttime,endtime) # Create DAG ################# daglogfile=os.path.join(opts.dag_log_path,basename+'.log') dagfile=os.path.join(opts.run_path,basename) dag = pipeline.CondorDAG(daglogfile) dag.set_dag_file(dagfile) datafind_job = pipeline.LSCDataFindJob(cache_dir,opts.jobs_log_path,cp) datafind_job.add_opt('url-type','file') datafind_job.set_sub_file(os.path.join(opts.run_path,'datafind.sub')) # Build list of science segments # Covers entire time range. Not necessarily all used for ifo in ifos: if not opts.ignore_science_mode and types[ifo] not in fakeTypes: seg_files[ifo]=inspiralutils.science_segments(ifo,cp) segfile=open(seg_files[ifo]) #segs[ifo]=segmentsUtils.fromfilenames([seg_files[ifo]]) segs[ifo]=segmentsUtils.fromsegwizard(segfile) segs[ifo].coalesce() segfile.close() else: # If we skip the segdb step, just construct a large segment print 'Faking segment from %i to %i\n'%(datastart,dataend)
subsuffix = '.sub' # create the Condor jobs that will be used in the DAG mkdir_job = strain.MkdirJob('logs',cp) mkdir_node = strain.MkdirNode(mkdir_job,'cache') if opts.write_dax: dag.add_node(mkdir_node) # try and make a directory to store the cache files and job logs try: os.mkdir('logs') except: pass # #try: os.mkdir('cache') #except: pass df_job = pipeline.LSCDataFindJob('cache','logs',cp,opts.write_dax) noise_job = strain.NoiseJob(cp,opts.write_dax) qscan_job = strain.qscanJob(cp) # submit files df_job.set_sub_file( opts.basename + '.datafind'+ subsuffix ) noise_job.set_sub_file( opts.basename + '.noisecomp' + subsuffix ) # get the pad and chunk lengths from the values in the ini file length = int(cp.get('pipeline', 'segment-length')) # get the ifo to filter ifo = cp.get('pipeline','ifo') datatype_hoft = cp.get('input','type-hoft') datatype_derr = cp.get('input','type-derr')
# create the Condor jobs that will be used in the DAG # datafind: frame_types = [] try: lsync_file = cp.get('pipeline','lsync-cache-file') try: frame_types.append(cp.get('input','ligo-type')) except: pass try: frame_types.append(cp.get('input','virgo-type')) except: pass try: frame_types.append(cp.get('input','geo-type')) except: pass frame_types = [t for t in frame_types if t] except: lsync_file = None df_job = pipeline.LSCDataFindJob( 'cache','logs',cp,opts.dax,lsync_file,'|'.join(frame_types)) df_job.set_sub_file( basename + '.datafind'+ subsuffix ) # tmpltbank: tmplt_jobs = {} for ifo in ifo_list: tmplt_jobs[ifo] = inspiral.TmpltBankJob(cp,opts.dax) tmplt_jobs[ifo].set_sub_file( basename + '.tmpltbank_' + ifo + subsuffix ) # inspinj: inspinj_job = inspiral.InspInjJob(cp) inspinj_job.set_sub_file( basename + '.inspinj' + subsuffix ) if opts.noop_inspinj: inspinj_job.add_condor_cmd("noop_job", "true")