def dload_dbase(cores=8, path="/group/mwaops/k_smith/"):
epn_url = "http://www.epta.eu.org/epndb/json/"
wget_cmd = ""
for _ in range(cores - 1):
wget_cmd = wget_cmd + "wget -r -np -N {0} & ".format(epn_url)
wget_cmd = wget_cmd + "wget -r -np -N {0}".format(epn_url)
#create slurm job:
commands = []
commands.append("cd {0}".format(path))
commands.append('echo "Downloading epn database to {0}"'.format(path))
commands.append("srun --export=all -N 1 -n 1 {0}".format(wget_cmd))
slurm_kwargs = {
"partition": "gpuq",
"time": "12:00:00",
"nodes": "1",
"ntasks-per-node": "{0}".format(cores),
"gres": "gpu:1"
}
name = "epn_dload"
batch_dir = path + "/wget_batch"
submit_slurm(name,
commands,
batch_dir=batch_dir,
slurm_kwargs=slurm_kwargs,
submit=True,
vcstools_version="master")
def tar_job_wrapper(hsm_work_dir, file_list, remove=True):
print(file_list)
for fn, fits_dir in enumerate(file_list):
if fits_dir.endswith("\n"):
fits_dir = fits_dir[:-1]
pointing = fits_dir.split("/")[-1]
#only lets 5 jobs at a time to not flood the transfer (could probably handle more
if os.path.exists(fits_dir):
print(pointing)
your_slurm_queue_check(max_queue=5, grep='tar', queue='workq')
commands = []
commands.append('cd {}'.format(fits_dir))
commands.append("srun -n 1 -c 10 tar zcvvf - *.fits | ssh hpc-hsm.pawsey.org.au 'cat - > {0}/temp_{1}.tar.gz'".format(hsm_work_dir, fn))
commands.append('errorcode=$?')
commands.append('if [ $errorcode == "0" ]; then')
commands.append(' echo mv temp_{0}.tar.gz {1}_pointing.tar.gz'.format(fn, pointing))
commands.append(' ssh hpc-hsm.pawsey.org.au "mv {0}/temp_{1}.tar.gz {0}/{2}_pointing.tar.gz"'.format(hsm_work_dir, fn, pointing))
if remove:
commands.append(' cd ..')
commands.append(' rm -r {}'.format(pointing))
commands.append('fi')
#TODO and a move command
submit_slurm('tar_{0}_{1}'.format(pointing,fn), commands,
batch_dir="./",
slurm_kwargs={"time": "5:00:00", "partition": "workq"},
submit=True, export='ALL')
else:
print('{} does not exist'.format(pointing))
def stokes_fold(run_params):
"""
Launches the stokes_fold part of the data processing pipeling
Parameters:
-----------
run_params: object
The run_params object defined by data_processing_pipeline
"""
launch_line = stokes_launch_line(run_params)
commands = [launch_line]
name = "Stokes_Fold_init_{0}_{1}".format(run_params.pulsar,
run_params.obsid)
comp_config = load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
job_id = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "00:10:00"},\
module_list=["mwa_search/{0}".format(run_params.mwa_search),\
"dspsr/master", "psrchive/master"],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Job successfully submitted: {0}".format(name))
return job_id
def write_cont_scripts(self):
"""Function to write RTS submission script in the case of a "standard"
contiguous bandwidth observation. This is relatively simple and just
requires us to use the standard format RTS configuration file.
Returns
-------
jobids : list of ints
A list of all the job IDs submitted to the system compute queue.
"""
logger.info(
"Writing RTS configuration script for contiguous bandwidth observation"
)
fname = "{0}/rts_{1}.in".format(
self.rts_out_dir, self.cal_obsid) # output file name to write
with open(fname, 'w') as f:
f.write(self.base_str)
jobids = []
nnodes = 25 # number of required GPU nodes - 1 per coarse channels + 1 master node
rts_batch = "RTS_{0}".format(self.cal_obsid)
slurm_kwargs = {
"partition": "gpuq",
"chdir": "{0}".format(self.rts_out_dir),
"time": "2:00:00",
"nodes": "{0}".format(nnodes),
"gres": "gpu:1",
"ntasks-per-node": "1"
}
module_list = ["RTS/master"]
commands = list(self.script_body) # make a copy of body to then extend
commands.append("export UCX_MEMTYPE_CACHE=n")
commands.append("srun --export=all -N {0} -n {0} rts_gpu {1}".format(
nnodes, fname))
hostname = socket.gethostname()
if hostname.startswith("galaxy"):
mem = 1024
else:
mem = 10240
jobid = submit_slurm(
rts_batch,
commands,
slurm_kwargs=slurm_kwargs,
module_list=module_list,
batch_dir=self.batch_dir,
submit=self.submit,
queue='gpuq',
export="NONE",
mem=mem,
load_vcstools=self.offline, #load if offline
vcstools_version=self.vcstools_version)
jobids.append(jobid)
return jobids
def submit_rm_cor_rvm(run_params):
"""
Runs the RM correction on the dspsr archive and writes the result to a text file.
Relaunches the stokes_fold script afterwards for plotting
Parameters:
-----------
run_params: object
The run_params object from data_processing_pipeline.py
"""
os.chdir(run_params.pointing_dir)
filenames_dict = create_filenames(run_params)
#Look for files in pointing dir
is_rmsynth = isfile(filenames_dict["rmsynth"])
is_rmfit = isfile(filenames_dict["rmfit"])
#Correct for RM
if is_rmsynth:
logger.info("Using RM synthesis result for correction")
rm_dict = rm_synthesis.read_rmsynth_out(filenames_dict["rmsynth"])
RM = rm_dict["0"]["rm"]
elif is_rmfit:
logger.info("Using rmfit result for correction")
RM = find_RM_from_file(filenames_dict["rmfit"])[0]
if not RM:
RM = find_RM_from_cat(run_params.pulsar)[0]
run_params.RM = RM
commands = add_rm_cor_to_commands(run_params.pointing_dir, run_params.RM,
filenames_dict["archive1"],
filenames_dict["archive2"],
filenames_dict["ascii"])
#RVM fitting
commands = add_rvm_to_commands(run_params.pointing_dir, filenames_dict["archive2"], rvmfile=filenames_dict["rvmfit"], chimap=filenames_dict["chimap"],\
commands=commands, res=run_params.rvmres)
#relaunch
if not run_params.stop:
launch_line = dpp.stokes_launch_line(run_params)
commands.append(launch_line)
job_name = "ipfb_RMcor_RVM_{}".format(run_params.file_prefix)
if not run_params.ipfb:
job_name = job_name[5:]
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
job_id = submit_slurm(job_name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "06:00:00"},\
module_list=["mwa_search/{0}".format(run_params.mwa_search),
"psrchive/master"],\
submit=True, vcstools_version=run_params.vcs_tools, mem="")
return job_id
def submit_dspsr(run_params):
#run dspsr
launch_line = "stokes_fold.py -m f -d {0} -p {1} -b {2} -s {3} -L {4} --mwa_search {5}\
--vcs_tools {6}"\
.format(run_params.pointing_dir, run_params.pulsar, run_params.nbins,\
run_params.subint, run_params.loglvl, run_params.mwa_search,\
run_params.vcs_tools)
if run_params.stop == True:
launch_line += " -S"
commands = []
commands.append("psrcat -e {0} > {1}/{0}.eph".format(
run_params.pulsar, run_params.pointing_dir))
commands.append("echo 'Running DSPSR folding...\n'")
commands.append("dspsr -cont -U 4000 -A -L {0} -E {3}/{1}.eph -K -b {2} -O {3}/{1}_subint_{0} {3}/*.fits"\
.format(run_params.subint, run_params.pulsar, run_params.nbins, run_params.pointing_dir))
commands.append("echo 'Attempting to find rotation measure.\nOutputting result to {0}/{1}_rmfit.txt\n'"\
.format(run_params.pointing_dir, run_params.pulsar))
commands.append("rmfit {0}/{1}_subint_{2}.ar -t > {0}/{1}_rmfit.txt"\
.format(run_params.pointing_dir, run_params.pulsar, run_params.subint))
#rerun the script
commands.append(launch_line)
name = "dspsr_RM_{0}_{1}".format(run_params.pulsar, run_params.obsid)
comp_config = config.load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "08:00:00"},\
module_list=["mwa_search/{0}".format(run_params.mwa_search),\
"dspsr/master", "psrchive/master"],\
submit=True, vcstools_version=run_params.vcs_tools)
logger.info("Job submitted for dspsr using\n\
pointing directory: {0}\n\
pulsar: {1}"\
.format(run_params.pointing_dir, run_params.pulsar))
def submit_RM_correct(run_params):
#correct for RM and submit plot
launch_line = "stokes_fold.py -m p -d {0} -p {1} -b {2} -s {3} -L {4} --mwa_search {5}\
--vcs_tools {6}"\
.format(run_params.pointing_dir, run_params.pulsar, run_params.nbins,\
run_params.subint, run_params.loglvl, run_params.mwa_search,\
run_params.vcs_tools)#, run_params.stop)
if run_params.stop == True:
launch_line += " -S"
commands = []
#correct for RM
commands.append("echo 'Correcting for input rotation measure\n'")
commands.append("pam -e ar2 -R {0} {1}/{2}_subint_{3}.ar"\
.format(run_params.RM, run_params.pointing_dir, run_params.pulsar, run_params.subint))
#Turn the archive into a readable ascii file
commands.append("echo 'Wiritng result to text file\n'")
commands.append("pdv -FTt {0}/{1}_subint_{2}.ar2 > {0}/{1}_archive.txt".\
format(run_params.pointing_dir, run_params.pulsar, run_params.subint))
#launch plotting
commands.append(launch_line)
name = "RMcor_plt_{0}_{1}".format(run_params.pulsar, run_params.obsid)
comp_config = config.load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "02:00:00"},\
module_list=["mwa_search/{0}".format(run_params.mwa_search),
"psrchive/master"],\
submit=True, vcstools_version=run_params.vcs_tools)
def binfind(run_params):
"""
Launches the binfinding part of the data processing pipeline
Parameters:
-----------
run_params: object
The run_params object defined by data_proces_pipeline
"""
launch_line = binfinder_launch_line(run_params)
commands = [launch_line]
#decide how much time to allocate based on number of pointings
n_pointings = len(run_params.pointing_dir)
if n_pointings < 100:
time = "00:30:00"
elif n_pointings < 400:
time = "02:00:00"
elif n_pointings < 1000:
time = "05:00:00"
else:
time = "10:00:00"
name = "bf_initiate_{0}_{1}".format(run_params.pulsar, run_params.obsid)
logger.info("Submitting binfinder script:")
logger.info("")
logger.info("Job Name: {}".format(name))
comp_config = load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
job_id = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": time},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/no-python'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Job successfully submitted: {0}".format(name))
return job_id
def submit_multiple_pointings(run_params):
"""
Submits many pointings on either 50 or 100 bins depending on the pulsar's period. Launches the next step of binfinder
Parameters:
-----------
run_params: object
The run_params object defined in data_processing_pipeline
"""
job_ids = []
#Check number of bins to use
bin_limit = bin_sampling_limit(run_params.pulsar)
if bin_limit < 100:
nbins = 50
else:
nbins = 64
logger.info("Submitting multiple prepfold jobs:")
logger.info("Pulsar name: {}".format(run_params.pulsar))
logger.info("Number of bins to fold on: {}".format(nbins))
#submit a job for each pointing
for i, pointing in enumerate(run_params.pointing_dir):
if not glob.glob(os.path.join(pointing, "*.pfd.bestprof")):
prepfold_dict = make_my_fold_dict(run_params, nbins, True)
#create slurm job:
commands = add_prepfold_to_commands(pointing,
files="*.fits",
pulsar=run_params.pulsar,
**prepfold_dict)
name = "bf_multi_{0}_{1}_{2}_bins_{3}".format(
run_params.pulsar, run_params.obsid, nbins, i)
batch_dir = os.path.join(comp_config['base_product_dir'],
run_params.obsid, "batch")
time = dpp.prepfold_time_alloc(prepfold_dict, run_params.beg,
run_params.end)
if time > 86399.:
logger.warn(
"Estimation for prepfold time greater than one day")
time = 86399
time = str(datetime.timedelta(seconds=int(time)))
logger.info("Submitting pointing: {0}".format(pointing))
logger.info("Job name: {}".format(name))
myid = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": time},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/master'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Pulsar name: {}".format(
run_params.pulsar))
logger.info("Number of bins to fold on: {}".format(nbins))
logger.info("Job name: {}".format(name))
logger.info("Time Allocation: {}".format(time))
logger.info("Job ID: {}".format(myid))
job_ids.append(myid)
else:
logger.info(
"Pointing {} already has a folded profile. Not Folding".format(
pointing))
launch_line = dpp.binfinder_launch_line(run_params, dpp=False)
commands = [launch_line]
name = "bf_post_multi_{0}".format(run_params.pulsar)
batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid,
"batch")
logger.info("Submitting post-prepfold binfinder:")
logger.info("Job name: {}".format(name))
myid = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "00:30:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
"presto/master"],\
submit=True, depend=job_ids, depend_type="afterany",\
vcstools_version=run_params.vcs_tools)
def submit_to_db_and_continue(run_params, best_bins):
"""
Submits the best fold profile to the pulsar database. Will also submit .ppps, .png and .pfd
Parameters:
-----------
run_params: object
The run_params object defined in data_processing_pipeline
best_bins: int
"""
ppps = "*_{}*_bins*.pfd.ps".format(best_bins)
ppps = glob.glob(os.path.join(run_params.pointing_dir, ppps))[0]
bestprof = "*_{}*_bins*.pfd.bestprof".format(best_bins)
bestprof = glob.glob(os.path.join(run_params.pointing_dir, bestprof))[0]
png = "*_{}*_bins*.png".format(best_bins)
png = glob.glob(os.path.join(run_params.pointing_dir, png))[0]
pfd = "*_{}*_bins*.pfd".format(best_bins)
pfd = glob.glob(os.path.join(run_params.pointing_dir, pfd))[0]
commands = []
commands.append("cd {}".format(run_params.pointing_dir))
commands.append(
"echo 'Submitting profile to database with {} bins'".format(best_bins))
commands.append('submit_to_database.py -o {0} --cal_id {1} -p {2} --bestprof {3} --ppps {4}'\
.format(run_params.obsid, run_params.cal_id, run_params.pulsar, bestprof, ppps))
#Make a nice plot
plotting_toolkit.plot_bestprof(os.path.join(run_params.pointing_dir, bestprof),\
out_dir=run_params.pointing_dir)
bin_lim = bin_sampling_limit(run_params.pulsar)
if bin_lim > 100:
b_standard = 100
else:
b_standard = 50
if best_bins != b_standard:
#do the same for 100/50 bin profiles depending on whether this is an msp or not
ppps = "*_{}*_bins*.pfd.ps".format(b_standard)
ppps = glob.glob(os.path.join(run_params.pointing_dir, ppps))[0]
bestprof = "*_{}*_bins*.pfd.bestprof".format(b_standard)
bestprof = glob.glob(os.path.join(run_params.pointing_dir,
bestprof))[0]
png = "*_{}*_bins*.png".format(b_standard)
png = glob.glob(os.path.join(run_params.pointing_dir, png))[0]
pfd = "*_{}*_bins*.pfd".format(b_standard)
pfd = glob.glob(os.path.join(run_params.pointing_dir, pfd))[0]
commands.append(
"echo 'Submitting profile to database with {} bins'".format(
b_standard))
commands.append('submit_to_database.py -o {0} --cal_id {1} -p {2} --bestprof {3} --ppps {4}'\
.format(run_params.obsid, run_params.cal_id, run_params.pulsar, bestprof, ppps))
if run_params.stokes_dep:
#submit inverse pfb profile if it exists
ipfb_archive = os.path.join(
run_params.pointing_dir,
"{0}_{1}_ipfb_archive.txt".format(run_params.obsid,
run_params.pulsar))
commands.append("echo 'Submitting inverse PFB profile to database'")
commands.append("submit_to_database.py -o {0} --cal_id {1} -p {2} --ascii {3} --ppps {4} --start {5} --stop {6}"\
.format(run_params.obsid, run_params.cal_id, run_params.pulsar, ipfb_archive, ppps,\
run_params.beg, run_params.end))
#Move the pointing directory
move_loc = os.path.join(comp_config["base_product_dir"], run_params.obsid,
"data_products")
pointing = run_params.pointing_dir.split("/")
pointing = [i for i in pointing if i != ""]
new_pointing_dir = os.path.join(move_loc, pointing[-1])
logger.info("New pointing directory: {}".format(new_pointing_dir))
#in case the previous command fails. Don't move stuff around
commands.append("errorcode=$?")
commands.append("echo 'errorcode' $errorcode")
commands.append('if [ "$errorcode" != "0" ]; then')
commands.append(" echo 'Submission Failure!'")
commands.append(" exit $errorcode")
commands.append("fi")
commands.append(
"echo 'submitted profile to database: {0}'".format(bestprof))
commands.append("echo 'Moving directory {0} to location {1}'".format(
run_params.pointing_dir, move_loc))
commands.append("mkdir -p {}".format(move_loc))
if run_params.pulsar[-1].isalpha():
commands.append("cp -ru {0} {1}".format(run_params.pointing_dir,
new_pointing_dir))
else:
commands.append("mv {0} {1}".format(run_params.pointing_dir,
new_pointing_dir))
#submit job
name = "Submit_db_{0}_{1}".format(run_params.pulsar, run_params.obsid)
batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid,
"batch")
logger.info(
"Submitting submission script for profile: {0}".format(bestprof))
logger.info("Job name: {}".format(name))
dep_id = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "04:00:00"},\
depend=run_params.stokes_dep,
module_list=['mwa_search/{0}'.format(run_params.mwa_search)],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
if not run_params.stop:
#Run stokes fold
run_params.stokes_bins = best_bins
launch_line = dpp.stokes_launch_line(run_params,
dpp=True,
custom_pointing=new_pointing_dir)
commands = [launch_line]
name = "dpp_stokes_{0}_{1}".format(run_params.pulsar, run_params.obsid)
batch_dir = os.path.join(comp_config['base_product_dir'],
run_params.obsid, "batch")
logger.info("Submitting Stokes Fold script")
logger.info("Job Name: {}".format(name))
#wait for pfb inversion if it exists
dep_ids = [dep_id]
if run_params.stokes_dep:
dep_ids.append(run_params.stokes_dep)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "00:20:00"},\
depend=dep_ids, depend_type="afterany",\
module_list=['mwa_search/{0}'.format(run_params.mwa_search)],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
def submit_prepfold(run_params, nbins, initial=None):
"""
Submits a prepfold job for the given parameters
Parameters:
-----------
run_params: object
The run_params object defined by data_processing_pipeline
nbins: int
The number of bins to fold on
initial: boolean
OPTIONAL - Whether this is the first fold on this pulsar. This affects the search range.\
If none, will try to figure it out. Default: None
"""
if initial is None:
profs_in_dir = glob.glob(
os.path.join(run_params.pointing_dir, "*.pfd.bestprof"))
if profs_in_dir:
initial = False
else:
initial = True
if initial or is_binary(run_params.pulsar):
psr = run_params.pulsar
else:
psr = None
prepfold_dict = make_my_fold_dict(run_params, nbins, initial)
#make the commands
commands = []
commands.append(
"echo '############### Prepfolding on {} bins ###############'".format(
nbins))
commands = add_prepfold_to_commands(run_params.pointing_dir,
files="*.fits",
pulsar=psr,
commands=commands,
**prepfold_dict)
#Check if prepfold worked:
commands.append("errorcode=$?")
commands.append("echo 'errorcode' $errorcode")
commands.append('if [ "$errorcode" != "0" ]; then')
commands.append(" echo 'Prepfold operation failure!'")
commands.append(" exit $errorcode")
commands.append("fi")
#binfinder relaunch:
commands.append(
"echo '############### Relaunching binfinder script ###############'")
bf_relaunch = dpp.binfinder_launch_line(run_params)
commands.append(bf_relaunch)
batch_dir = os.path.join(comp_config['base_product_dir'], run_params.obsid,
"batch")
name = "bf_{0}_{1}_{2}_bins".format(run_params.pulsar, run_params.obsid,
nbins)
time = dpp.prepfold_time_alloc(prepfold_dict, run_params.beg,
run_params.end)
if time > 86399.:
logger.warn("Estimation for prepfold time greater than one day")
time = 86399
time = str(datetime.timedelta(seconds=int(time)))
logger.info("Submitting prepfold and resubmission job:")
job_id = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": time},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/master'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Pointing directory: {}".format(
run_params.pointing_dir))
logger.info("Pulsar name: {}".format(run_params.pulsar))
logger.info("Number of bins to fold on: {}".format(nbins))
logger.info("Job name: {}".format(name))
logger.info("Time Allocation: {}".format(time))
logger.info("Job ID: {}".format(job_id))
return job_id
def submit_multifold(run_params, nbins=100):
job_ids = []
comp_config = config.load_config_file()
#Check beam coverage for the pulsar
start, end = pulsar_beam_coverage(run_params.obsid, run_params.pulsar)
logger.info(
"start and end of pulsar beam coverage for on-disk files:{0}, {1}".
format(start, end))
if start >= 1. or end < 0.:
logger.error(
"pulsar is not in beam for any of the on-disk files. Ending...")
sys.exit(1)
for i, pointing in enumerate(run_params.pointing_dir):
logger.info("submitting pointing:{0}".format(pointing))
#os.chdir(pointing)
#create slurm job:
commands = []
commands = add_prepfold_to_commands(commands, pointing, run_params.pulsar, run_params.obsid,\
start=start, end=end, nbins=nbins)
name = "multifold_binfind_{0}_{1}".format(run_params.pulsar, i)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
myid = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "2:00:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/no-python'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
job_ids.append(myid)
#Now submit the check script
if run_params.stop == True:
stop = "-S"
else:
stop = ""
p = ""
for pointing in run_params.pointing_dir:
p += " " + pointing
commands = []
commands.append("binfinder.py -m b -d {0} -O {1} -p {2} -o {3} -L {4} {5} --vcs_tools {6}\
--mwa_search {7} --force_initial -p {8}"\
.format(p, run_params.cal_id, run_params.pulsar, run_params.obsid, run_params.loglvl,\
stop, run_params.vcs_tools, run_params.mwa_search, run_params.pulsar))
name = "best_pointing_{0}".format(run_params.pulsar)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
myid = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "00:30:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
"presto/no-python"],\
submit=True, depend=job_ids, depend_type="afterany",\
vcstools_version="master")
def submit_dspsr_rmfit(run_params):
"""
Runs dspsr on fits files and relaunches the stokes fold script
Parameters:
-----------
run_params: object
The run_params object from data_processing_pipeline.py
"""
if not run_params.cand:
enter, leave, _ = binfinder.find_fold_times\
(run_params.pulsar, run_params.obsid, run_params.beg, run_params.end, min_z_power=[0.3, 0.1])
obs_int = run_params.end - run_params.beg
if enter is None or leave is None:
logger.warn(
"{} not in beam for given times. Will use entire integration time to fold."
.format(run_params.pulsar))
logger.warn("Used the following parameters:")
logger.warn("pulsar: {}".format(run_params.pulsar))
logger.warn("obsid: {}".format(run_params.obsid))
logger.warn("beg: {}".format(run_params.beg))
logger.warn("end: {}".format(run_params.end))
enter_sec = 0
duration = obs_int
else:
duration = (leave - enter) * obs_int
enter_sec = enter * obs_int
logger.info("{0} enters beam at {1} and leaves at {2}".format(
run_params.pulsar, enter, leave))
logger.info("Integration time: {}".format(duration))
else:
enter_sec = None
duration = None
filenames_dict = create_filenames(run_params)
#dspsr command
commands = add_dspsr_fold_to_commands(run_params.pulsar, run_params.pointing_dir, run_params.stokes_bins, out_pref=run_params.file_prefix,\
seek=enter_sec, total=duration, subint=run_params.subint, dspsr_ops=run_params.dspsr_ops,\
dm=run_params.dm, period=run_params.period)
#rmfit command
commands = add_rm_fit_to_commands(run_params.pulsar,
run_params.pointing_dir,
filenames_dict["archive1"],
out_name=filenames_dict["rmfit"],
commands=commands)
#rmsynth command
commands = add_rmsynth_to_commands(run_params.pointing_dir,
filenames_dict["archive1"],
label=run_params.file_prefix,
write=True,
plot=True,
keep_QUV=False,
commands=commands)
#rerun the script
if not run_params.stop:
launch_line = dpp.stokes_launch_line(run_params)
commands.append(launch_line)
name = "DSPSR_RMfit_{0}".format(run_params.file_prefix)
batch_dir = "{}".format(
ospj(comp_config['base_product_dir'], run_params.obsid, "batch"))
job_id = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "08:00:00"},\
module_list=["mwa_search/{0}".format(run_params.mwa_search),\
"dspsr/master", "psrchive/master"],\
submit=True, vcstools_version=run_params.vcs_tools, mem="")
logger.info("Job submitted using\n\
pointing directory: {0}\n\
pulsar: {1}"\
.format(run_params.pointing_dir, run_params.pulsar))
return job_id
def submit_inverse_pfb_fold(run_params, stop=False):
"""
Submits the inverse pfb folding script and fits RM
Parameters:
-----------
run_params: object
The run_params object defined in data_processing_pipeline
Returns:
--------
job_id: int
The id of the submitted job
"""
#Find beam coverage for known pulsars
if not run_params.cand:
enter, leave, _ = binfinder.find_fold_times(run_params.pulsar,
run_params.obsid,
run_params.beg,
run_params.end,
min_z_power=[0.3, 0.1])
obs_int = run_params.end - run_params.beg
if enter is None or leave is None:
logger.warn(
"{} not in beam for given times. Will use entire integration time to fold."
.format(run_params.pulsar))
logger.warn("Used the following parameters:")
logger.warn("pulsar: {}".format(run_params.pulsar))
logger.warn("obsid: {}".format(run_params.obsid))
logger.warn("beg: {}".format(run_params.beg))
logger.warn("end: {}".format(run_params.end))
enter_sec = 0
duration = obs_int
else:
duration = (leave - enter) * obs_int
enter_sec = enter * obs_int
logger.info("{0} enters beam at {1} and leaves at {2}".format(
run_params.pulsar, enter, leave))
logger.info("Integration time: {}".format(duration))
else:
enter_sec = None
duration = None
#pfb inversion
filenames_dict = create_filenames(run_params)
commands = add_pfb_inversion_to_commands(run_params.pointing_dir, run_params.pulsar, run_params.obsid, filenames_dict["archive1"], filenames_dict["ascii"],\
seek=enter_sec, total=duration, tscrunch=duration, dm=run_params.dm, period=run_params.period)
#launch RM fitting
commands = add_rm_fit_to_commands(run_params.pulsar,
run_params.pointing_dir,
filenames_dict["archive1"],
out_name=filenames_dict["rmfit"],
commands=commands)
#launch RM synthesis
commands = add_rmsynth_to_commands(run_params.pointing_dir,
filenames_dict["archive1"],
write=True,
plot=True,
keep_QUV=False,
label=run_params.file_prefix,
commands=commands)
if not stop:
#Relaunch stokes_fold.py
launch_line = dpp.stokes_launch_line(run_params)
commands.append(launch_line)
elif not run_params.stop:
launch_line = dpp.stokes_launch_line(run_params)
commands.append(launch_line)
batch_dir = ospj(comp_config['base_product_dir'], run_params.obsid,
"batch")
name = "inverse_pfb_{0}".format(run_params.file_prefix)
logger.info("Submitting inverse pfb job:")
logger.info("Pointing directory: {}".format(run_params.pointing_dir))
logger.info("Pulsar name: {}".format(run_params.pulsar))
logger.info("Job name: {}".format(name))
job_id = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "10:00:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
"dspsr", "psrchive"],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
return job_id
commands.append("ar_loc=" + str(args.archive)[:-3])
elif args.single_pulse_series:
commands.append("ar_loc=" + str(args.single_pulse_series)[:-3])
elif args.u_ippd or args.u_waterfall or args.u_archive:
commands.append("srun -n 1 -c $ncpus dspsr -U 600 -E {0}.par -b {1} -A -cont -O {2}_{0} {3}".format(pulsar, num_bins, obsid, fits_files_loc))
commands.append("ar_loc={0}_{1}".format(obsid,pulsar))
if args.u_ippd:
commands.append("pav -CDFTp -N1,1 -g {0}_{1}.prof.ps/cps ".format(obsid,pulsar) +\
"${ar_loc}.ar")
if args.u_waterfall:
commands.append('psrplot -pG -jCDTp -j "B {0}" -D {1}_{2}.freq.vs.phase.ps/cps '.\
format(num_bins,obsid,pulsar)+ "${ar_loc}.ar")
if args.u_ppps:
commands.append("psrcat -e {0} > {0}.eph".format(pulsar))
commands.append("srun -n 1 -c $ncpus prepfold -ncpus $ncpus -o {0} -topo -runavg -noclip -par {1}.eph -nsub 256 ".format(obsid, pulsar) + "${fits}")
commands.append("rm {0}.eph".format(pulsar))
if args.u_single_pulse_series:
commands.append("srun -n 1 dspsr -U 600 -E {0}.par -b {1} -cont -s -K ".\
format(pulsar,num_bins) + "${fits}")
commands.append('psraddstring="psradd -o {0}_{1}.ts.ar "'.format(obsid,pulsar))
commands.append("ts=(pulse*.ar)")
commands.append('for ((i=0;i<${#ts[@]};i++)); do psraddstring=${psraddstring}" "${ts[i]} ; done')
commands.append("srun -n 1 -c $ncpus $psraddstring")
commands.append("rm pulse*.ar")
commands.append("rm {0}.par".format(pulsar))
job_id = submit_slurm(dspsr_batch, commands,
batch_dir="./",
slurm_kwargs={"time": "6:50:00", "partition": "workq"},
submit=True)
def submit_prepfold(run_params, nbins=100, finish=False):
if nbins is not int:
nbins = int(float(nbins))
comp_config = config.load_config_file()
#Check to see if there is a 100 bin fold already
bin_lim = bin_sampling_limit(run_params.pulsar)
if len(glob.glob("*_100_bins**{0}*bestprof".format(
run_params.pulsar))) > 0 and bin_lim > 100 and nbins is not 100:
#add a prepfold command for 100 bins
logger.info("Folding on 100 bins for pointing {0}".format(
run_params.pointing))
commands = []
commands = add_to_prepfold(commands,
run_params.pointing_dir,
run_params.pulsar,
run_params.obsid,
nbins=100)
name = "binfinder_prepfold_only_{0}_100".format(run_params.pulsar)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "2:00:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/no-python'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Prepfold job successfully submitted: {0}".format(name))
launch_line = "binfinder.py -d {0} -t {1} -O {2} -o {3} -L {4} --prevbins {5} --vcs_tools {6}\
--mwa_search {7} -p {8}"\
.format(run_params.pointing_dir, run_params.threshold, run_params.cal_id,\
run_params.obsid, run_params.loglvl, nbins, run_params.vcs_tools,\
run_params.mwa_search, run_params.pulsar)
if run_params.stop == True:
launch_line += " -S"
logger.info("Submitting job for {0} bins".format(nbins))
#create slurm job:
commands = []
if firstrun == True:
commands = add_to_prepfold(commands,
run_params.pointing_dir,
run_params.pulsar,
run_params.obsid,
nbins=100)
commands = add_prepfold_to_commands(commands,
run_params.pointing_dir,
run_params.pulsar,
run_params.obsid,
nbins=nbins)
if finish == False:
#Rerun this script
commands.append(
'echo "Running script again. Passing prevbins = {0}"'.format(
nbins))
launch_line += " -m f"
else:
#Run again only once and without prepfold
commands.append(
'echo "Running script again without folding. Passing prevbins = {0}"'
.format(nbins))
launch_line += " -m e"
commands.append(launch_line)
comp_config = config.load_config_file()
name = "binfinder_{0}_{1}".format(run_params.pulsar, nbins)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "2:00:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/no-python'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Job successfully submitted: {0}".format(name))
def write_picket_fence_scripts(self):
"""Function to write RTS submission scripts in the case of a picket-fence
observation. A significant amount of extra information needs to be
determined and placed in the RTS configuration files. There will be:
1 RTS configuration file per set of adjacent single channels (subband)
The only exception to that rule is where the 129 boundary is crossed,
in which case that subband will be split in two.
Returns
-------
jobids : list of ints
A list of all the job IDs submitted to the system compute queue.
"""
logger.info(
"Sorting picket-fence channels and determining subband info...")
self.sort_obs_channels()
hichan_groups, lochan_groups = self.construct_subbands()
# write out the RTS config files and keep track of the number of nodes required for each
count = 0
lodict, count = self.get_subband_config(lochan_groups,
self.rts_out_dir, "low", count)
hidict, count = self.get_subband_config(hichan_groups,
self.rts_out_dir, "high",
count)
chan_file_dict = lodict.copy()
chan_file_dict.update(hidict)
# Now submit the RTS jobs
logger.info("Writing individual subband RTS configuration scripts")
hostname = socket.gethostname()
if hostname.startswith("galaxy"):
mem = 1024
else:
mem = 10240
jobids = []
for k, v in chan_file_dict.items():
nnodes = v + 1
chans = k.split('_')[-1].split(".")[0]
rts_batch = "RTS_{0}_{1}".format(self.cal_obsid, chans)
slurm_kwargs = {
"partition": "gpuq",
"chdir": "{0}".format(self.rts_out_dir),
"time": "2:00:00",
"nodes": "{0}".format(nnodes),
"gres": "gpu:1",
"ntasks-per-node": "1"
}
module_list = ["RTS/master"]
commands = list(
self.script_body) # make a copy of body to then extend
commands.append("export UCX_MEMTYPE_CACHE=n")
commands.append(
"srun --export=all -N {0} -n {0} rts_gpu {1}".format(
nnodes, k))
jobid = submit_slurm(rts_batch,
commands,
slurm_kwargs=slurm_kwargs,
module_list=module_list,
batch_dir=self.batch_dir,
submit=self.submit,
queue='gpuq',
export="NONE",
mem=mem,
load_vcstools=False)
jobids.append(jobid)
return jobids
def submit_to_db(run_params, ideal_bins):
logger.info("submitting profile to database: {0}".format(
run_params.bestprof))
#Add path to filenames for submit script
cwd = os.getcwd()
ppps = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd.ps".format(
ideal_bins, run_params.pulsar[1:]))[0]
bestprof_name = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd.bestprof".format(
ideal_bins, run_params.pulsar[1:]))[0]
png_output = cwd + "/" + glob.glob("*{0}_bins*{1}*.png".format(
ideal_bins, run_params.pulsar[1:]))[0]
pfd = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd".format(
ideal_bins, run_params.pulsar[1:]))[0]
#do the same for 100 bin profiles
ppps_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.pfd.ps".format(
run_params.pulsar[1:]))[0]
bestprof_name_100 = cwd + "/" + glob.glob(
"*_100_bins*{0}*.pfd.bestprof".format(run_params.pulsar[1:]))[0]
png_output_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.png".format(
run_params.pulsar[1:]))[0]
pfd_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.pfd".format(
run_params.pulsar[1:]))[0]
products = [ppps, bestprof_name, png_output, pfd,\
ppps_100, bestprof_name_100, png_output_100, pfd_100]
#move all of these data products to a suitable directory
data_dir = "/group/mwaops/vcs/{0}/data_products/{1}".format(
run_params.obsid, run_params.pulsar)
for product in products:
data_process_pipeline.copy_data(product, data_dir)
commands = []
commands.append('submit_to_database.py -o {0} --cal_id {1} -p {2} --bestprof {3} --ppps {4}'\
.format(run_params.obsid, run_params.cal_id, run_params.pulsar, bestprof_name, ppps))
commands.append(
'echo "submitted profile to database: {0}"'.format(bestprof_name))
if run_params.stop == False:
#Run stokes fold
commands.append("data_process_pipeline.py -d {0} -O {1} -p {2} -o {3} -b {4} -L {5}\
--mwa_search {6} --vcs_tools {7} -m s"\
.format(run_params.pointing_dir, run_params.cal_id, run_params.pulsar,\
run_params.obsid, run_params.best_bins, run_params.loglvl, run_params.mwa_search,\
run_params.vcs_tools))
#commands.append('echo "Searching for pulsar using the pipeline to test the pipelines effectivness"')
#commands.append('mwa_search_pipeline.py -o {0} -a --search --pulsar {1} -O {2}\
# --code_comment "Known pulsar auto test"'.format(run_params.obsid, run_params.pulsar,\
# run_params.cal_id))
name = "Submit_{0}_{1}".format(run_params.pulsar, run_params.obsid)
comp_config = config.load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "00:05:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search)],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
