def dload_dbase(cores=8, path="/group/mwaops/k_smith/"): epn_url = "http://www.epta.eu.org/epndb/json/" wget_cmd = "" for _ in range(cores - 1): wget_cmd = wget_cmd + "wget -r -np -N {0} & ".format(epn_url) wget_cmd = wget_cmd + "wget -r -np -N {0}".format(epn_url) #create slurm job: commands = [] commands.append("cd {0}".format(path)) commands.append('echo "Downloading epn database to {0}"'.format(path)) commands.append("srun --export=all -N 1 -n 1 {0}".format(wget_cmd)) slurm_kwargs = { "partition": "gpuq", "time": "12:00:00", "nodes": "1", "ntasks-per-node": "{0}".format(cores), "gres": "gpu:1" } name = "epn_dload" batch_dir = path + "/wget_batch" submit_slurm(name, commands, batch_dir=batch_dir, slurm_kwargs=slurm_kwargs, submit=True, vcstools_version="master")
def tar_job_wrapper(hsm_work_dir, file_list, remove=True): print(file_list) for fn, fits_dir in enumerate(file_list): if fits_dir.endswith("\n"): fits_dir = fits_dir[:-1] pointing = fits_dir.split("/")[-1] #only lets 5 jobs at a time to not flood the transfer (could probably handle more if os.path.exists(fits_dir): print(pointing) your_slurm_queue_check(max_queue=5, grep='tar', queue='workq') commands = [] commands.append('cd {}'.format(fits_dir)) commands.append("srun -n 1 -c 10 tar zcvvf - *.fits | ssh hpc-hsm.pawsey.org.au 'cat - > {0}/temp_{1}.tar.gz'".format(hsm_work_dir, fn)) commands.append('errorcode=$?') commands.append('if [ $errorcode == "0" ]; then') commands.append(' echo mv temp_{0}.tar.gz {1}_pointing.tar.gz'.format(fn, pointing)) commands.append(' ssh hpc-hsm.pawsey.org.au "mv {0}/temp_{1}.tar.gz {0}/{2}_pointing.tar.gz"'.format(hsm_work_dir, fn, pointing)) if remove: commands.append(' cd ..') commands.append(' rm -r {}'.format(pointing)) commands.append('fi') #TODO and a move command submit_slurm('tar_{0}_{1}'.format(pointing,fn), commands, batch_dir="./", slurm_kwargs={"time": "5:00:00", "partition": "workq"}, submit=True, export='ALL') else: print('{} does not exist'.format(pointing))
def stokes_fold(run_params): """ Launches the stokes_fold part of the data processing pipeling Parameters: ----------- run_params: object The run_params object defined by data_processing_pipeline """ launch_line = stokes_launch_line(run_params) commands = [launch_line] name = "Stokes_Fold_init_{0}_{1}".format(run_params.pulsar, run_params.obsid) comp_config = load_config_file() batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) job_id = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "00:10:00"},\ module_list=["mwa_search/{0}".format(run_params.mwa_search),\ "dspsr/master", "psrchive/master"],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) logger.info("Job successfully submitted: {0}".format(name)) return job_id
def write_cont_scripts(self): """Function to write RTS submission script in the case of a "standard" contiguous bandwidth observation. This is relatively simple and just requires us to use the standard format RTS configuration file. Returns ------- jobids : list of ints A list of all the job IDs submitted to the system compute queue. """ logger.info( "Writing RTS configuration script for contiguous bandwidth observation" ) fname = "{0}/rts_{1}.in".format( self.rts_out_dir, self.cal_obsid) # output file name to write with open(fname, 'w') as f: f.write(self.base_str) jobids = [] nnodes = 25 # number of required GPU nodes - 1 per coarse channels + 1 master node rts_batch = "RTS_{0}".format(self.cal_obsid) slurm_kwargs = { "partition": "gpuq", "chdir": "{0}".format(self.rts_out_dir), "time": "2:00:00", "nodes": "{0}".format(nnodes), "gres": "gpu:1", "ntasks-per-node": "1" } module_list = ["RTS/master"] commands = list(self.script_body) # make a copy of body to then extend commands.append("export UCX_MEMTYPE_CACHE=n") commands.append("srun --export=all -N {0} -n {0} rts_gpu {1}".format( nnodes, fname)) hostname = socket.gethostname() if hostname.startswith("galaxy"): mem = 1024 else: mem = 10240 jobid = submit_slurm( rts_batch, commands, slurm_kwargs=slurm_kwargs, module_list=module_list, batch_dir=self.batch_dir, submit=self.submit, queue='gpuq', export="NONE", mem=mem, load_vcstools=self.offline, #load if offline vcstools_version=self.vcstools_version) jobids.append(jobid) return jobids
def submit_rm_cor_rvm(run_params): """ Runs the RM correction on the dspsr archive and writes the result to a text file. Relaunches the stokes_fold script afterwards for plotting Parameters: ----------- run_params: object The run_params object from data_processing_pipeline.py """ os.chdir(run_params.pointing_dir) filenames_dict = create_filenames(run_params) #Look for files in pointing dir is_rmsynth = isfile(filenames_dict["rmsynth"]) is_rmfit = isfile(filenames_dict["rmfit"]) #Correct for RM if is_rmsynth: logger.info("Using RM synthesis result for correction") rm_dict = rm_synthesis.read_rmsynth_out(filenames_dict["rmsynth"]) RM = rm_dict["0"]["rm"] elif is_rmfit: logger.info("Using rmfit result for correction") RM = find_RM_from_file(filenames_dict["rmfit"])[0] if not RM: RM = find_RM_from_cat(run_params.pulsar)[0] run_params.RM = RM commands = add_rm_cor_to_commands(run_params.pointing_dir, run_params.RM, filenames_dict["archive1"], filenames_dict["archive2"], filenames_dict["ascii"]) #RVM fitting commands = add_rvm_to_commands(run_params.pointing_dir, filenames_dict["archive2"], rvmfile=filenames_dict["rvmfit"], chimap=filenames_dict["chimap"],\ commands=commands, res=run_params.rvmres) #relaunch if not run_params.stop: launch_line = dpp.stokes_launch_line(run_params) commands.append(launch_line) job_name = "ipfb_RMcor_RVM_{}".format(run_params.file_prefix) if not run_params.ipfb: job_name = job_name[5:] batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) job_id = submit_slurm(job_name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "06:00:00"},\ module_list=["mwa_search/{0}".format(run_params.mwa_search), "psrchive/master"],\ submit=True, vcstools_version=run_params.vcs_tools, mem="") return job_id
def submit_dspsr(run_params): #run dspsr launch_line = "stokes_fold.py -m f -d {0} -p {1} -b {2} -s {3} -L {4} --mwa_search {5}\ --vcs_tools {6}"\ .format(run_params.pointing_dir, run_params.pulsar, run_params.nbins,\ run_params.subint, run_params.loglvl, run_params.mwa_search,\ run_params.vcs_tools) if run_params.stop == True: launch_line += " -S" commands = [] commands.append("psrcat -e {0} > {1}/{0}.eph".format( run_params.pulsar, run_params.pointing_dir)) commands.append("echo 'Running DSPSR folding...\n'") commands.append("dspsr -cont -U 4000 -A -L {0} -E {3}/{1}.eph -K -b {2} -O {3}/{1}_subint_{0} {3}/*.fits"\ .format(run_params.subint, run_params.pulsar, run_params.nbins, run_params.pointing_dir)) commands.append("echo 'Attempting to find rotation measure.\nOutputting result to {0}/{1}_rmfit.txt\n'"\ .format(run_params.pointing_dir, run_params.pulsar)) commands.append("rmfit {0}/{1}_subint_{2}.ar -t > {0}/{1}_rmfit.txt"\ .format(run_params.pointing_dir, run_params.pulsar, run_params.subint)) #rerun the script commands.append(launch_line) name = "dspsr_RM_{0}_{1}".format(run_params.pulsar, run_params.obsid) comp_config = config.load_config_file() batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "08:00:00"},\ module_list=["mwa_search/{0}".format(run_params.mwa_search),\ "dspsr/master", "psrchive/master"],\ submit=True, vcstools_version=run_params.vcs_tools) logger.info("Job submitted for dspsr using\n\ pointing directory: {0}\n\ pulsar: {1}"\ .format(run_params.pointing_dir, run_params.pulsar))
def submit_RM_correct(run_params): #correct for RM and submit plot launch_line = "stokes_fold.py -m p -d {0} -p {1} -b {2} -s {3} -L {4} --mwa_search {5}\ --vcs_tools {6}"\ .format(run_params.pointing_dir, run_params.pulsar, run_params.nbins,\ run_params.subint, run_params.loglvl, run_params.mwa_search,\ run_params.vcs_tools)#, run_params.stop) if run_params.stop == True: launch_line += " -S" commands = [] #correct for RM commands.append("echo 'Correcting for input rotation measure\n'") commands.append("pam -e ar2 -R {0} {1}/{2}_subint_{3}.ar"\ .format(run_params.RM, run_params.pointing_dir, run_params.pulsar, run_params.subint)) #Turn the archive into a readable ascii file commands.append("echo 'Wiritng result to text file\n'") commands.append("pdv -FTt {0}/{1}_subint_{2}.ar2 > {0}/{1}_archive.txt".\ format(run_params.pointing_dir, run_params.pulsar, run_params.subint)) #launch plotting commands.append(launch_line) name = "RMcor_plt_{0}_{1}".format(run_params.pulsar, run_params.obsid) comp_config = config.load_config_file() batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "02:00:00"},\ module_list=["mwa_search/{0}".format(run_params.mwa_search), "psrchive/master"],\ submit=True, vcstools_version=run_params.vcs_tools)
def binfind(run_params): """ Launches the binfinding part of the data processing pipeline Parameters: ----------- run_params: object The run_params object defined by data_proces_pipeline """ launch_line = binfinder_launch_line(run_params) commands = [launch_line] #decide how much time to allocate based on number of pointings n_pointings = len(run_params.pointing_dir) if n_pointings < 100: time = "00:30:00" elif n_pointings < 400: time = "02:00:00" elif n_pointings < 1000: time = "05:00:00" else: time = "10:00:00" name = "bf_initiate_{0}_{1}".format(run_params.pulsar, run_params.obsid) logger.info("Submitting binfinder script:") logger.info("") logger.info("Job Name: {}".format(name)) comp_config = load_config_file() batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) job_id = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": time},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ 'presto/no-python'],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) logger.info("Job successfully submitted: {0}".format(name)) return job_id
def submit_multiple_pointings(run_params): """ Submits many pointings on either 50 or 100 bins depending on the pulsar's period. Launches the next step of binfinder Parameters: ----------- run_params: object The run_params object defined in data_processing_pipeline """ job_ids = [] #Check number of bins to use bin_limit = bin_sampling_limit(run_params.pulsar) if bin_limit < 100: nbins = 50 else: nbins = 64 logger.info("Submitting multiple prepfold jobs:") logger.info("Pulsar name: {}".format(run_params.pulsar)) logger.info("Number of bins to fold on: {}".format(nbins)) #submit a job for each pointing for i, pointing in enumerate(run_params.pointing_dir): if not glob.glob(os.path.join(pointing, "*.pfd.bestprof")): prepfold_dict = make_my_fold_dict(run_params, nbins, True) #create slurm job: commands = add_prepfold_to_commands(pointing, files="*.fits", pulsar=run_params.pulsar, **prepfold_dict) name = "bf_multi_{0}_{1}_{2}_bins_{3}".format( run_params.pulsar, run_params.obsid, nbins, i) batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid, "batch") time = dpp.prepfold_time_alloc(prepfold_dict, run_params.beg, run_params.end) if time > 86399.: logger.warn( "Estimation for prepfold time greater than one day") time = 86399 time = str(datetime.timedelta(seconds=int(time))) logger.info("Submitting pointing: {0}".format(pointing)) logger.info("Job name: {}".format(name)) myid = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": time},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ 'presto/master'],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) logger.info("Pulsar name: {}".format( run_params.pulsar)) logger.info("Number of bins to fold on: {}".format(nbins)) logger.info("Job name: {}".format(name)) logger.info("Time Allocation: {}".format(time)) logger.info("Job ID: {}".format(myid)) job_ids.append(myid) else: logger.info( "Pointing {} already has a folded profile. Not Folding".format( pointing)) launch_line = dpp.binfinder_launch_line(run_params, dpp=False) commands = [launch_line] name = "bf_post_multi_{0}".format(run_params.pulsar) batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid, "batch") logger.info("Submitting post-prepfold binfinder:") logger.info("Job name: {}".format(name)) myid = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "00:30:00"},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ "presto/master"],\ submit=True, depend=job_ids, depend_type="afterany",\ vcstools_version=run_params.vcs_tools)
def submit_to_db_and_continue(run_params, best_bins): """ Submits the best fold profile to the pulsar database. Will also submit .ppps, .png and .pfd Parameters: ----------- run_params: object The run_params object defined in data_processing_pipeline best_bins: int """ ppps = "*_{}*_bins*.pfd.ps".format(best_bins) ppps = glob.glob(os.path.join(run_params.pointing_dir, ppps))[0] bestprof = "*_{}*_bins*.pfd.bestprof".format(best_bins) bestprof = glob.glob(os.path.join(run_params.pointing_dir, bestprof))[0] png = "*_{}*_bins*.png".format(best_bins) png = glob.glob(os.path.join(run_params.pointing_dir, png))[0] pfd = "*_{}*_bins*.pfd".format(best_bins) pfd = glob.glob(os.path.join(run_params.pointing_dir, pfd))[0] commands = [] commands.append("cd {}".format(run_params.pointing_dir)) commands.append( "echo 'Submitting profile to database with {} bins'".format(best_bins)) commands.append('submit_to_database.py -o {0} --cal_id {1} -p {2} --bestprof {3} --ppps {4}'\ .format(run_params.obsid, run_params.cal_id, run_params.pulsar, bestprof, ppps)) #Make a nice plot plotting_toolkit.plot_bestprof(os.path.join(run_params.pointing_dir, bestprof),\ out_dir=run_params.pointing_dir) bin_lim = bin_sampling_limit(run_params.pulsar) if bin_lim > 100: b_standard = 100 else: b_standard = 50 if best_bins != b_standard: #do the same for 100/50 bin profiles depending on whether this is an msp or not ppps = "*_{}*_bins*.pfd.ps".format(b_standard) ppps = glob.glob(os.path.join(run_params.pointing_dir, ppps))[0] bestprof = "*_{}*_bins*.pfd.bestprof".format(b_standard) bestprof = glob.glob(os.path.join(run_params.pointing_dir, bestprof))[0] png = "*_{}*_bins*.png".format(b_standard) png = glob.glob(os.path.join(run_params.pointing_dir, png))[0] pfd = "*_{}*_bins*.pfd".format(b_standard) pfd = glob.glob(os.path.join(run_params.pointing_dir, pfd))[0] commands.append( "echo 'Submitting profile to database with {} bins'".format( b_standard)) commands.append('submit_to_database.py -o {0} --cal_id {1} -p {2} --bestprof {3} --ppps {4}'\ .format(run_params.obsid, run_params.cal_id, run_params.pulsar, bestprof, ppps)) if run_params.stokes_dep: #submit inverse pfb profile if it exists ipfb_archive = os.path.join( run_params.pointing_dir, "{0}_{1}_ipfb_archive.txt".format(run_params.obsid, run_params.pulsar)) commands.append("echo 'Submitting inverse PFB profile to database'") commands.append("submit_to_database.py -o {0} --cal_id {1} -p {2} --ascii {3} --ppps {4} --start {5} --stop {6}"\ .format(run_params.obsid, run_params.cal_id, run_params.pulsar, ipfb_archive, ppps,\ run_params.beg, run_params.end)) #Move the pointing directory move_loc = os.path.join(comp_config["base_product_dir"], run_params.obsid, "data_products") pointing = run_params.pointing_dir.split("/") pointing = [i for i in pointing if i != ""] new_pointing_dir = os.path.join(move_loc, pointing[-1]) logger.info("New pointing directory: {}".format(new_pointing_dir)) #in case the previous command fails. Don't move stuff around commands.append("errorcode=$?") commands.append("echo 'errorcode' $errorcode") commands.append('if [ "$errorcode" != "0" ]; then') commands.append(" echo 'Submission Failure!'") commands.append(" exit $errorcode") commands.append("fi") commands.append( "echo 'submitted profile to database: {0}'".format(bestprof)) commands.append("echo 'Moving directory {0} to location {1}'".format( run_params.pointing_dir, move_loc)) commands.append("mkdir -p {}".format(move_loc)) if run_params.pulsar[-1].isalpha(): commands.append("cp -ru {0} {1}".format(run_params.pointing_dir, new_pointing_dir)) else: commands.append("mv {0} {1}".format(run_params.pointing_dir, new_pointing_dir)) #submit job name = "Submit_db_{0}_{1}".format(run_params.pulsar, run_params.obsid) batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid, "batch") logger.info( "Submitting submission script for profile: {0}".format(bestprof)) logger.info("Job name: {}".format(name)) dep_id = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "04:00:00"},\ depend=run_params.stokes_dep, module_list=['mwa_search/{0}'.format(run_params.mwa_search)],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) if not run_params.stop: #Run stokes fold run_params.stokes_bins = best_bins launch_line = dpp.stokes_launch_line(run_params, dpp=True, custom_pointing=new_pointing_dir) commands = [launch_line] name = "dpp_stokes_{0}_{1}".format(run_params.pulsar, run_params.obsid) batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid, "batch") logger.info("Submitting Stokes Fold script") logger.info("Job Name: {}".format(name)) #wait for pfb inversion if it exists dep_ids = [dep_id] if run_params.stokes_dep: dep_ids.append(run_params.stokes_dep) submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "00:20:00"},\ depend=dep_ids, depend_type="afterany",\ module_list=['mwa_search/{0}'.format(run_params.mwa_search)],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
def submit_prepfold(run_params, nbins, initial=None): """ Submits a prepfold job for the given parameters Parameters: ----------- run_params: object The run_params object defined by data_processing_pipeline nbins: int The number of bins to fold on initial: boolean OPTIONAL - Whether this is the first fold on this pulsar. This affects the search range.\ If none, will try to figure it out. Default: None """ if initial is None: profs_in_dir = glob.glob( os.path.join(run_params.pointing_dir, "*.pfd.bestprof")) if profs_in_dir: initial = False else: initial = True if initial or is_binary(run_params.pulsar): psr = run_params.pulsar else: psr = None prepfold_dict = make_my_fold_dict(run_params, nbins, initial) #make the commands commands = [] commands.append( "echo '############### Prepfolding on {} bins ###############'".format( nbins)) commands = add_prepfold_to_commands(run_params.pointing_dir, files="*.fits", pulsar=psr, commands=commands, **prepfold_dict) #Check if prepfold worked: commands.append("errorcode=$?") commands.append("echo 'errorcode' $errorcode") commands.append('if [ "$errorcode" != "0" ]; then') commands.append(" echo 'Prepfold operation failure!'") commands.append(" exit $errorcode") commands.append("fi") #binfinder relaunch: commands.append( "echo '############### Relaunching binfinder script ###############'") bf_relaunch = dpp.binfinder_launch_line(run_params) commands.append(bf_relaunch) batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid, "batch") name = "bf_{0}_{1}_{2}_bins".format(run_params.pulsar, run_params.obsid, nbins) time = dpp.prepfold_time_alloc(prepfold_dict, run_params.beg, run_params.end) if time > 86399.: logger.warn("Estimation for prepfold time greater than one day") time = 86399 time = str(datetime.timedelta(seconds=int(time))) logger.info("Submitting prepfold and resubmission job:") job_id = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": time},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ 'presto/master'],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) logger.info("Pointing directory: {}".format( run_params.pointing_dir)) logger.info("Pulsar name: {}".format(run_params.pulsar)) logger.info("Number of bins to fold on: {}".format(nbins)) logger.info("Job name: {}".format(name)) logger.info("Time Allocation: {}".format(time)) logger.info("Job ID: {}".format(job_id)) return job_id
def submit_multifold(run_params, nbins=100): job_ids = [] comp_config = config.load_config_file() #Check beam coverage for the pulsar start, end = pulsar_beam_coverage(run_params.obsid, run_params.pulsar) logger.info( "start and end of pulsar beam coverage for on-disk files:{0}, {1}". format(start, end)) if start >= 1. or end < 0.: logger.error( "pulsar is not in beam for any of the on-disk files. Ending...") sys.exit(1) for i, pointing in enumerate(run_params.pointing_dir): logger.info("submitting pointing:{0}".format(pointing)) #os.chdir(pointing) #create slurm job: commands = [] commands = add_prepfold_to_commands(commands, pointing, run_params.pulsar, run_params.obsid,\ start=start, end=end, nbins=nbins) name = "multifold_binfind_{0}_{1}".format(run_params.pulsar, i) batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) myid = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "2:00:00"},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ 'presto/no-python'],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) job_ids.append(myid) #Now submit the check script if run_params.stop == True: stop = "-S" else: stop = "" p = "" for pointing in run_params.pointing_dir: p += " " + pointing commands = [] commands.append("binfinder.py -m b -d {0} -O {1} -p {2} -o {3} -L {4} {5} --vcs_tools {6}\ --mwa_search {7} --force_initial -p {8}"\ .format(p, run_params.cal_id, run_params.pulsar, run_params.obsid, run_params.loglvl,\ stop, run_params.vcs_tools, run_params.mwa_search, run_params.pulsar)) name = "best_pointing_{0}".format(run_params.pulsar) batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) myid = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "00:30:00"},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ "presto/no-python"],\ submit=True, depend=job_ids, depend_type="afterany",\ vcstools_version="master")
def submit_dspsr_rmfit(run_params): """ Runs dspsr on fits files and relaunches the stokes fold script Parameters: ----------- run_params: object The run_params object from data_processing_pipeline.py """ if not run_params.cand: enter, leave, _ = binfinder.find_fold_times\ (run_params.pulsar, run_params.obsid, run_params.beg, run_params.end, min_z_power=[0.3, 0.1]) obs_int = run_params.end - run_params.beg if enter is None or leave is None: logger.warn( "{} not in beam for given times. Will use entire integration time to fold." .format(run_params.pulsar)) logger.warn("Used the following parameters:") logger.warn("pulsar: {}".format(run_params.pulsar)) logger.warn("obsid: {}".format(run_params.obsid)) logger.warn("beg: {}".format(run_params.beg)) logger.warn("end: {}".format(run_params.end)) enter_sec = 0 duration = obs_int else: duration = (leave - enter) * obs_int enter_sec = enter * obs_int logger.info("{0} enters beam at {1} and leaves at {2}".format( run_params.pulsar, enter, leave)) logger.info("Integration time: {}".format(duration)) else: enter_sec = None duration = None filenames_dict = create_filenames(run_params) #dspsr command commands = add_dspsr_fold_to_commands(run_params.pulsar, run_params.pointing_dir, run_params.stokes_bins, out_pref=run_params.file_prefix,\ seek=enter_sec, total=duration, subint=run_params.subint, dspsr_ops=run_params.dspsr_ops,\ dm=run_params.dm, period=run_params.period) #rmfit command commands = add_rm_fit_to_commands(run_params.pulsar, run_params.pointing_dir, filenames_dict["archive1"], out_name=filenames_dict["rmfit"], commands=commands) #rmsynth command commands = add_rmsynth_to_commands(run_params.pointing_dir, filenames_dict["archive1"], label=run_params.file_prefix, write=True, plot=True, keep_QUV=False, commands=commands) #rerun the script if not run_params.stop: launch_line = dpp.stokes_launch_line(run_params) commands.append(launch_line) name = "DSPSR_RMfit_{0}".format(run_params.file_prefix) batch_dir = "{}".format( ospj(comp_config['base_product_dir'], run_params.obsid, "batch")) job_id = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "08:00:00"},\ module_list=["mwa_search/{0}".format(run_params.mwa_search),\ "dspsr/master", "psrchive/master"],\ submit=True, vcstools_version=run_params.vcs_tools, mem="") logger.info("Job submitted using\n\ pointing directory: {0}\n\ pulsar: {1}"\ .format(run_params.pointing_dir, run_params.pulsar)) return job_id
def submit_inverse_pfb_fold(run_params, stop=False): """ Submits the inverse pfb folding script and fits RM Parameters: ----------- run_params: object The run_params object defined in data_processing_pipeline Returns: -------- job_id: int The id of the submitted job """ #Find beam coverage for known pulsars if not run_params.cand: enter, leave, _ = binfinder.find_fold_times(run_params.pulsar, run_params.obsid, run_params.beg, run_params.end, min_z_power=[0.3, 0.1]) obs_int = run_params.end - run_params.beg if enter is None or leave is None: logger.warn( "{} not in beam for given times. Will use entire integration time to fold." .format(run_params.pulsar)) logger.warn("Used the following parameters:") logger.warn("pulsar: {}".format(run_params.pulsar)) logger.warn("obsid: {}".format(run_params.obsid)) logger.warn("beg: {}".format(run_params.beg)) logger.warn("end: {}".format(run_params.end)) enter_sec = 0 duration = obs_int else: duration = (leave - enter) * obs_int enter_sec = enter * obs_int logger.info("{0} enters beam at {1} and leaves at {2}".format( run_params.pulsar, enter, leave)) logger.info("Integration time: {}".format(duration)) else: enter_sec = None duration = None #pfb inversion filenames_dict = create_filenames(run_params) commands = add_pfb_inversion_to_commands(run_params.pointing_dir, run_params.pulsar, run_params.obsid, filenames_dict["archive1"], filenames_dict["ascii"],\ seek=enter_sec, total=duration, tscrunch=duration, dm=run_params.dm, period=run_params.period) #launch RM fitting commands = add_rm_fit_to_commands(run_params.pulsar, run_params.pointing_dir, filenames_dict["archive1"], out_name=filenames_dict["rmfit"], commands=commands) #launch RM synthesis commands = add_rmsynth_to_commands(run_params.pointing_dir, filenames_dict["archive1"], write=True, plot=True, keep_QUV=False, label=run_params.file_prefix, commands=commands) if not stop: #Relaunch stokes_fold.py launch_line = dpp.stokes_launch_line(run_params) commands.append(launch_line) elif not run_params.stop: launch_line = dpp.stokes_launch_line(run_params) commands.append(launch_line) batch_dir = ospj(comp_config['base_product_dir'], run_params.obsid, "batch") name = "inverse_pfb_{0}".format(run_params.file_prefix) logger.info("Submitting inverse pfb job:") logger.info("Pointing directory: {}".format(run_params.pointing_dir)) logger.info("Pulsar name: {}".format(run_params.pulsar)) logger.info("Job name: {}".format(name)) job_id = submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "10:00:00"},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ "dspsr", "psrchive"],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) return job_id
commands.append("ar_loc=" + str(args.archive)[:-3]) elif args.single_pulse_series: commands.append("ar_loc=" + str(args.single_pulse_series)[:-3]) elif args.u_ippd or args.u_waterfall or args.u_archive: commands.append("srun -n 1 -c $ncpus dspsr -U 600 -E {0}.par -b {1} -A -cont -O {2}_{0} {3}".format(pulsar, num_bins, obsid, fits_files_loc)) commands.append("ar_loc={0}_{1}".format(obsid,pulsar)) if args.u_ippd: commands.append("pav -CDFTp -N1,1 -g {0}_{1}.prof.ps/cps ".format(obsid,pulsar) +\ "${ar_loc}.ar") if args.u_waterfall: commands.append('psrplot -pG -jCDTp -j "B {0}" -D {1}_{2}.freq.vs.phase.ps/cps '.\ format(num_bins,obsid,pulsar)+ "${ar_loc}.ar") if args.u_ppps: commands.append("psrcat -e {0} > {0}.eph".format(pulsar)) commands.append("srun -n 1 -c $ncpus prepfold -ncpus $ncpus -o {0} -topo -runavg -noclip -par {1}.eph -nsub 256 ".format(obsid, pulsar) + "${fits}") commands.append("rm {0}.eph".format(pulsar)) if args.u_single_pulse_series: commands.append("srun -n 1 dspsr -U 600 -E {0}.par -b {1} -cont -s -K ".\ format(pulsar,num_bins) + "${fits}") commands.append('psraddstring="psradd -o {0}_{1}.ts.ar "'.format(obsid,pulsar)) commands.append("ts=(pulse*.ar)") commands.append('for ((i=0;i<${#ts[@]};i++)); do psraddstring=${psraddstring}" "${ts[i]} ; done') commands.append("srun -n 1 -c $ncpus $psraddstring") commands.append("rm pulse*.ar") commands.append("rm {0}.par".format(pulsar)) job_id = submit_slurm(dspsr_batch, commands, batch_dir="./", slurm_kwargs={"time": "6:50:00", "partition": "workq"}, submit=True)
def submit_prepfold(run_params, nbins=100, finish=False): if nbins is not int: nbins = int(float(nbins)) comp_config = config.load_config_file() #Check to see if there is a 100 bin fold already bin_lim = bin_sampling_limit(run_params.pulsar) if len(glob.glob("*_100_bins**{0}*bestprof".format( run_params.pulsar))) > 0 and bin_lim > 100 and nbins is not 100: #add a prepfold command for 100 bins logger.info("Folding on 100 bins for pointing {0}".format( run_params.pointing)) commands = [] commands = add_to_prepfold(commands, run_params.pointing_dir, run_params.pulsar, run_params.obsid, nbins=100) name = "binfinder_prepfold_only_{0}_100".format(run_params.pulsar) batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "2:00:00"},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ 'presto/no-python'],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) logger.info("Prepfold job successfully submitted: {0}".format(name)) launch_line = "binfinder.py -d {0} -t {1} -O {2} -o {3} -L {4} --prevbins {5} --vcs_tools {6}\ --mwa_search {7} -p {8}"\ .format(run_params.pointing_dir, run_params.threshold, run_params.cal_id,\ run_params.obsid, run_params.loglvl, nbins, run_params.vcs_tools,\ run_params.mwa_search, run_params.pulsar) if run_params.stop == True: launch_line += " -S" logger.info("Submitting job for {0} bins".format(nbins)) #create slurm job: commands = [] if firstrun == True: commands = add_to_prepfold(commands, run_params.pointing_dir, run_params.pulsar, run_params.obsid, nbins=100) commands = add_prepfold_to_commands(commands, run_params.pointing_dir, run_params.pulsar, run_params.obsid, nbins=nbins) if finish == False: #Rerun this script commands.append( 'echo "Running script again. Passing prevbins = {0}"'.format( nbins)) launch_line += " -m f" else: #Run again only once and without prepfold commands.append( 'echo "Running script again without folding. Passing prevbins = {0}"' .format(nbins)) launch_line += " -m e" commands.append(launch_line) comp_config = config.load_config_file() name = "binfinder_{0}_{1}".format(run_params.pulsar, nbins) batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "2:00:00"},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search),\ 'presto/no-python'],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools)) logger.info("Job successfully submitted: {0}".format(name))
def write_picket_fence_scripts(self): """Function to write RTS submission scripts in the case of a picket-fence observation. A significant amount of extra information needs to be determined and placed in the RTS configuration files. There will be: 1 RTS configuration file per set of adjacent single channels (subband) The only exception to that rule is where the 129 boundary is crossed, in which case that subband will be split in two. Returns ------- jobids : list of ints A list of all the job IDs submitted to the system compute queue. """ logger.info( "Sorting picket-fence channels and determining subband info...") self.sort_obs_channels() hichan_groups, lochan_groups = self.construct_subbands() # write out the RTS config files and keep track of the number of nodes required for each count = 0 lodict, count = self.get_subband_config(lochan_groups, self.rts_out_dir, "low", count) hidict, count = self.get_subband_config(hichan_groups, self.rts_out_dir, "high", count) chan_file_dict = lodict.copy() chan_file_dict.update(hidict) # Now submit the RTS jobs logger.info("Writing individual subband RTS configuration scripts") hostname = socket.gethostname() if hostname.startswith("galaxy"): mem = 1024 else: mem = 10240 jobids = [] for k, v in chan_file_dict.items(): nnodes = v + 1 chans = k.split('_')[-1].split(".")[0] rts_batch = "RTS_{0}_{1}".format(self.cal_obsid, chans) slurm_kwargs = { "partition": "gpuq", "chdir": "{0}".format(self.rts_out_dir), "time": "2:00:00", "nodes": "{0}".format(nnodes), "gres": "gpu:1", "ntasks-per-node": "1" } module_list = ["RTS/master"] commands = list( self.script_body) # make a copy of body to then extend commands.append("export UCX_MEMTYPE_CACHE=n") commands.append( "srun --export=all -N {0} -n {0} rts_gpu {1}".format( nnodes, k)) jobid = submit_slurm(rts_batch, commands, slurm_kwargs=slurm_kwargs, module_list=module_list, batch_dir=self.batch_dir, submit=self.submit, queue='gpuq', export="NONE", mem=mem, load_vcstools=False) jobids.append(jobid) return jobids
def submit_to_db(run_params, ideal_bins): logger.info("submitting profile to database: {0}".format( run_params.bestprof)) #Add path to filenames for submit script cwd = os.getcwd() ppps = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd.ps".format( ideal_bins, run_params.pulsar[1:]))[0] bestprof_name = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd.bestprof".format( ideal_bins, run_params.pulsar[1:]))[0] png_output = cwd + "/" + glob.glob("*{0}_bins*{1}*.png".format( ideal_bins, run_params.pulsar[1:]))[0] pfd = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd".format( ideal_bins, run_params.pulsar[1:]))[0] #do the same for 100 bin profiles ppps_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.pfd.ps".format( run_params.pulsar[1:]))[0] bestprof_name_100 = cwd + "/" + glob.glob( "*_100_bins*{0}*.pfd.bestprof".format(run_params.pulsar[1:]))[0] png_output_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.png".format( run_params.pulsar[1:]))[0] pfd_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.pfd".format( run_params.pulsar[1:]))[0] products = [ppps, bestprof_name, png_output, pfd,\ ppps_100, bestprof_name_100, png_output_100, pfd_100] #move all of these data products to a suitable directory data_dir = "/group/mwaops/vcs/{0}/data_products/{1}".format( run_params.obsid, run_params.pulsar) for product in products: data_process_pipeline.copy_data(product, data_dir) commands = [] commands.append('submit_to_database.py -o {0} --cal_id {1} -p {2} --bestprof {3} --ppps {4}'\ .format(run_params.obsid, run_params.cal_id, run_params.pulsar, bestprof_name, ppps)) commands.append( 'echo "submitted profile to database: {0}"'.format(bestprof_name)) if run_params.stop == False: #Run stokes fold commands.append("data_process_pipeline.py -d {0} -O {1} -p {2} -o {3} -b {4} -L {5}\ --mwa_search {6} --vcs_tools {7} -m s"\ .format(run_params.pointing_dir, run_params.cal_id, run_params.pulsar,\ run_params.obsid, run_params.best_bins, run_params.loglvl, run_params.mwa_search,\ run_params.vcs_tools)) #commands.append('echo "Searching for pulsar using the pipeline to test the pipelines effectivness"') #commands.append('mwa_search_pipeline.py -o {0} -a --search --pulsar {1} -O {2}\ # --code_comment "Known pulsar auto test"'.format(run_params.obsid, run_params.pulsar,\ # run_params.cal_id)) name = "Submit_{0}_{1}".format(run_params.pulsar, run_params.obsid) comp_config = config.load_config_file() batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'], run_params.obsid) submit_slurm(name, commands,\ batch_dir=batch_dir,\ slurm_kwargs={"time": "00:05:00"},\ module_list=['mwa_search/{0}'.format(run_params.mwa_search)],\ submit=True, vcstools_version="{0}".format(run_params.vcs_tools))