def remove_raw(obs):
comp_config = load_config_file()
raw_folder = os.path.join(comp_config['base_data_dir'], str(obs), "raw")
combined_folder = os.path.join(comp_config['base_data_dir'], str(obs),
"combined")
raw_files = False
tar_files = False
combined_files = False
ics_files = False
for file in glob.iglob("{0}/*".format(raw_folder)):
if re.search('dat', file):
raw_files = True
for file in glob.iglob("{0}/*".format(combined_folder)):
if re.search('tar', file):
tar_files = True
if re.search('ch\d{3}', file):
combined_files = True
if re.search('ics', file):
ics_files = True
if raw_files:
munlink_files(raw_folder, "dat")
if tar_files:
munlink_files(combined_folder, "tar")
if combined_files:
munlink_files(combined_folder, "ch\d{3}")
if ics_files:
munlink_files(combined_folder, "ics")
def opt_parser(loglevels):
try:
comp_config = load_config_file()
except Exception:
# No computer found so making a default for the argparse help to work
comp_config = {'base_data_dir': "/astro/mwavcs/vcs/"}
parser = argparse.ArgumentParser(
description="scripts to check sanity of downloads and recombine.")
parser.add_argument("-m", "--mode", type=str, choices=['download','recombine'],\
help="Mode you want to run: download, recombine", dest='mode', default=None)
parser.add_argument("-d", "--data_type", type=str, choices=['11','15','16', 'raw','ics','tar_ics'],\
help="Only necessary when checking downloads. Types refer to those as definded " + \
"in voltdownload.py: 11 = Raw, 15 = ICS only, 16 = ICS and tarballs of recombined data.", \
dest='data_type', default=None)
parser.add_argument("-o", "--obs", metavar="OBS ID", type=int, dest='obsID',\
help="Observation ID you want to process [no default]", default=None)
parser.add_argument("-b", "--begin", metavar="start", type=int, dest='begin',\
help="gps time of first file to check on [default=%(default)s]",\
default=None)
parser.add_argument("-e", "--end", metavar="stop", type=int, dest='end',\
help="gps time of last file to check on [default=%(default)s]",\
default=None)
parser.add_argument(
"-a",
"--all",
action="store_true",
default=False,
help=
"Perform on entire observation span. Use instead of -b & -e. [default=%(default)s]"
)
parser.add_argument("-i", "--increment", metavar="time increment", type=int, \
dest='increment',\
help="Effectively the number of seconds to ckeck for " +\
"starting at start time [default=%(default)s]",\
default=None)
parser.add_argument("-s", "--size", type=int, dest='size',\
help="The files size in bytes that you expect all files" +\
" to have. Per default will figure this out from files on he archive" +\
"We expect 253440000 (download raw), 327680000" +\
" (recombined, not ics), 7865368576 (tarballs)", default=None)
parser.add_argument("-S", "--size_ics", type=int, help='Size in bytes that' +\
"you expect the ics files to have. Default = %(default)s", \
dest='size_ics', default=30720000)
parser.add_argument('-w', '--work_dir', type=str, dest='work_dir',\
help="Directory to check the files in. " +\
"Default is {0}[obsID]/[raw,combined]".format(comp_config['base_data_dir']))
parser.add_argument("-V",
"--version",
action="store_true",
help="Print version and quit")
parser.add_argument("-L",
"--loglvl",
type=str,
help="Logger verbosity level. Default: INFO",
choices=loglevels.keys(),
default="INFO")
return parser.parse_args()
def check_data(obsid, beg=None, dur=None, base_dir=None):
"""
Checks to see if all of the recombined files exist on disk
Parameters:
-----------
obsid: int
The observation ID to check
beg: int
OPTIONAL - The beginning time of the files to check. If none, will use entire obs. Default: None
dur: int
OPTIONAL - The duration in seconds to check since the beginning time. If none, will use entire obs. Default: None
base_dir: string
OPTIONAL - The base directory to use. If none, will load from config. Default: None
Returns:
---------
check: boolean
True - all files are on disk. False - not all files are on disk
"""
if base_dir is None:
comp_config = load_config_file()
base_dir = comp_config['base_data_dir']
comb_dir = "{0}{1}/combined".format(base_dir, obsid)
if not isinstance(beg, int):
beg = int(beg)
if not isinstance(dur, int):
dur = int(dur)
#Check to see if the files are combined properly
if beg is not None and dur is not None:
logger.info("Checking recombined files beginning at {0} and ending at {1}. Duration: {2} seconds"\
.format(beg, (beg+dur), dur))
error = check_recombine(obsid,
startsec=beg,
n_secs=dur,
directory=comb_dir)
else:
logger.warn(
"No start time information supplied. Comparing files with full obs"
)
error = check_recombine(obsid, directory=comb_dir)
if error == True:
check = False
else:
check = True
return check
def download_cal(obs_id, cal_obs_id, data_dir, product_dir,
vcstools_version="master", nice=0):
#Load computer dependant config file
comp_config = load_config_file()
batch_dir = os.path.join(product_dir, 'batch')
product_dir = os.path.join(product_dir, 'cal', str(cal_obs_id))
vis_dir = os.path.join(data_dir, 'vis')
mdir(vis_dir, 'Calibrator vis', gid=comp_config['gid'])
mdir(product_dir, 'Calibrator product', gid=comp_config['gid'])
mdir(batch_dir, 'Batch', gid=comp_config['gid'])
# Downloads the visablities to /astro/mwavcs/vcs/[cal_obs_id]/vis
# but creates a link to it here /astro/mwavcs/vcs/[obs_id]/cal/[cal_obs_id]
csvfile = os.path.join(batch_dir, "{0}_dl.csv".format(cal_obs_id))
create_link(data_dir, 'vis', product_dir, 'vis')
obsdownload_batch = "caldownload_{0}".format(cal_obs_id)
secs_to_run = "03:00:00" # sometimes the staging can take a while...
module_list = ["manta-ray-client/python3"]
commands = []
commands.append("csvfile={0}".format(csvfile))
commands.append('cd {0}'.format(vis_dir))
commands.append('if [[ -z ${MWA_ASVO_API_KEY} ]]')
commands.append('then')
commands.append(' echo "Error, MWA_ASVO_API_KEY not set"')
commands.append(' echo "Cannot use client"')
commands.append(' echo "Please read the MWA ASVO documentation '
'about setting this (https://wiki.mwatelescope.org/'
'display/MP/MWA+ASVO%3A+Release+Notes)"')
commands.append(' exit 1')
commands.append('fi')
commands.append('echo "obs_id={0}, job_type=d, download_type=vis" > {1}'.\
format(cal_obs_id,csvfile))
commands.append('mwa_client --csv={0} --dir={1}'.format(csvfile,vis_dir))
#commands.append("ln -sfn {0} {1}/{2}".format(data_dir, product_dir, 'vis'))
commands.append('unzip *.zip')
submit_slurm(obsdownload_batch, commands, batch_dir=batch_dir,
module_list=module_list,
slurm_kwargs={"time": secs_to_run, "nice": nice},
vcstools_version=vcstools_version, queue="copyq",
export="NONE", mem=4096,
# Manually handing it the module dir as it should only run
module_dir='/group/mwa/software/modulefiles')
def rmfit_quad(archive, phase_min, phase_max):
"""Runs the PRSCHIVE rmfit command as a python subprocess using the -w option for a quadratic fit.
Parameters
----------
archive : `str`
The name of the archive file to take as an input,
phase_min : `float`
The minimum phase to begin the fit, should be a float between 0 and 1,
phase_max : `float`
The maximum phase to use to fit, should be a float between 0 and 1,
"""
comp_config = load_config_file()
commands = [comp_config["prschive_container"]]
commands.append("rmfit")
commands.append("-m")
commands.append("-10,10,20")
commands.append("-w")
commands.append(f"{phase_min},{phase_max}")
commands.append("-Y")
commands.append(f"{archive}")
subprocess.run(commands)
def remove_beamformed(obs, pointing=None):
comp_config = load_config_file()
pointing_folder = os.path.join(comp_config['base_data_dir'], str(obs),
"pointings")
if not pointing:
authority = (
'No pointing specified, would you like to remove all pointings for this observation?'
)
if (authority == "Y") or (authority == "y"):
pointings = glob.glob("{0}/*:*:*:*:*")
if not pointings:
logger.error("No valid pointings in {0}. Exiting...")
sys.exit(0)
else:
for pointing in pointings:
logger.info(
"Checking if pointing {0} has been uploaded to MWA Pulsar Database..."
.format(pointing))
# Upload to MWA Pulsar Database if not there already
# Remove each pointing
return
def launch_pabeam_sim(obsid,
pointing,
begin,
duration,
source_name="noname",
metafits_file=None,
flagged_tiles=None,
delays=None,
efficiency=1,
vcstools_version='master',
args=None,
common_metadata=None,
output_dir=None):
"""Submit a job to run the pabeam code to estimate the system equivelent
flux density and a dependent job to resume the submit_to_databse.py code if `args` is given.
Parameters
----------
obsid : `int`
The MWA observation ID.
pointing : `str`
The pointing of the simulation in the format HH:MM:SS.SS_DD:MM:SS.SS.
begin : `int`
The begining of the simulation in GPS time.
duration : `int`
The duration of the simulation in seconds (used to calculate the end of the simulation).
source_name : `str`, optional
The name of the source to be used to label output files. |br| Default: "noname".
metafits_file : `str`, optional
The location of the metafits file. If none given will assume the default location.
flagged_tiles : `list`, optional
A list of the flagged tiles. If none given will assume no tiles were flagged.
efficiency : `float`, optional
Frequency and pointing dependent array efficiency. |br| Default: 1.
vcstools_version : `str`, optional
VCSTools version to load in the job.
args : `dict`, optional
The argument parse dictionary from submit_to_database.py.
If supplied will launch a dependedn job with submit_to_databse.py to complete the script.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`.
output_dir : `str`
The output directory of the simulation results. By default will put it in the VCS directory under <obsid>/sefd_simulations.
Examples
--------
A simple example:
>>>launch_pabeam_sim(1206977296, "12:49:12_+27:12:00", 1206977300, 600, source_name="SEFD_test", output_dir=".")
"""
# Load computer dependant config file
comp_config = load_config_file()
# Ensure metafits file is there
data_dir = "{}{}".format(comp_config['base_data_dir'], obsid)
ensure_metafits(data_dir, obsid, "{0}_metafits_ppds.fits".format(obsid))
# Perform metadata calls
if common_metadata is None:
common_metadata = get_common_obs_metadata(obsid)
# Get frequencies
centre_freq = common_metadata[5] * 10e5
low_freq = common_metadata[6][0] * 1.28 * 10e5
high_freq = common_metadata[6][-1] * 1.28 * 10e5
sim_freqs = [str(low_freq), str(centre_freq), str(high_freq)]
# Calculate required pixel res and cores/mem
array_phase = get_obs_array_phase(obsid)
fwhm = calc_ta_fwhm(high_freq / 10e5, array_phase=array_phase) #degrees
phi_res = theta_res = fwhm / 3
if phi_res < 0.015:
# Going any smaller causes memory errors
phi_res = theta_res = 0.015
npixels = 360. // phi_res + 90. // theta_res
cores_required = npixels * len(sim_freqs) // 600
nodes_required = cores_required // 24 + 1
# Make directories
batch_dir = "{}/batch".format(data_dir)
if output_dir is None:
sefd_dir = "{}/sefd_simulations".format(data_dir)
else:
sefd_dir = output_dir
if not os.path.exists(batch_dir):
mdir(batch_dir, "Batch", gid=comp_config['gid'])
if not os.path.exists(sefd_dir):
mdir(sefd_dir, "SEFD", gid=comp_config['gid'])
# Parse defaults
if metafits_file is None:
metafits_file = "{0}{1}/{1}_metafits_ppds.fits".format(
comp_config['base_data_dir'], obsid)
# Get delays if none given
if delays is None:
delays = get_common_obs_metadata(obsid)[4][0]
print(delays)
print(' '.join(np.array(delays, dtype=str)))
# Set up pabeam command
command = 'srun --export=all -u -n {} pabeam.py'.format(
int(nodes_required * 24))
command += ' -o {}'.format(obsid)
command += ' -b {}'.format(begin)
command += ' -d {}'.format(int(duration))
command += ' -s {}'.format(
int(duration // 4 - 1)) # force 4 time steps to get reasonable std
command += ' -e {}'.format(efficiency)
command += ' --metafits {}'.format(metafits_file)
command += ' -p {}'.format(pointing)
command += ' --grid_res {:.3f} {:.3f}'.format(theta_res, phi_res)
command += ' --delays {}'.format(' '.join(np.array(delays, dtype=str)))
command += ' --out_dir {}'.format(sefd_dir)
command += ' --out_name {}'.format(source_name)
command += ' --freq {}'.format(" ".join(sim_freqs))
if flagged_tiles is not None:
logger.debug("flagged_tiles: {}".format(flagged_tiles))
command += ' --flagged_tiles {}'.format(' '.join(flagged_tiles))
# Set up and launch job
batch_file_name = 'pabeam_{}_{}_{}'.format(obsid, source_name, pointing)
job_id = submit_slurm(batch_file_name, [command],
batch_dir=batch_dir,
slurm_kwargs={
"time": datetime.timedelta(seconds=10 * 60 * 60),
"nodes": int(nodes_required)
},
module_list=['hyperbeam-python'],
queue='cpuq',
cpu_threads=24,
mem=12288,
vcstools_version=vcstools_version)
if args:
# Set up dependant submit_to_database.py job
submit_args = vars(args)
# Add sefd_file argument
submit_args['sefd_file'] = "{}/{}*stats".format(sefd_dir, source_name)
command_str = "submit_to_database.py"
for key, val in submit_args.items():
if val:
if val == True:
command_str += " --{}".format(key)
else:
command_str += " --{} {}".format(key, val)
batch_file_name = 'submit_to_database_{}_{}_{}'.format(
obsid, source_name, pointing)
job_id_dependant = submit_slurm(
batch_file_name, [command_str],
batch_dir=batch_dir,
slurm_kwargs={"time": datetime.timedelta(seconds=1 * 60 * 60)},
queue='cpuq',
vcstools_version=vcstools_version,
depend=[job_id])
return job_id, job_id_dependant
def __init__(self, obsid, cal_obsid, metafits, srclist, n_int_bins=6, datadir=None, outdir=None, offline=False, beam_model="FEE2016", vcstools_version="master"):
self.obsid = obsid # target observation ID
self.cal_obsid = cal_obsid # calibrator observation ID
self.offline = offline # switch to decide if offline correlated data or not
self.utctime = None # start UTC time
self.nfine_chan = None # number of fine channels
self.channels = None # actual channel numbers
self.fine_cbw = None # fine channel bandwidth
self.max_frequency = None # the maximum frequency used by the RTS to calculate decorrelation
self.corr_dump_time = None # correlator dump times (i.e. integration time)
self.n_dumps_to_average = None # number of integration times to use for calibration
self.PB_HA = None # primary beam HA
self.PB_DEC = None # primary beam Dec
self.freq_base = None # frequency base for RTS
self.JD = None # time base for RTS
self.metafits_RTSform = None # modified metafits file name for RTS
self.ArrayPositionLat = -26.70331940 # MWA latitude
self.ArrayPositionLong = 116.6708152 # MWA longitude
self.n_integration_bins = n_int_bins # number of visibility integration groups for RTS
self.base_str = None # base string to be written to file, will be editted by RTScal
self.beam_model = beam_model # The beam model to use for the calibration solutions. Either 'ANALYTIC' or 'FEE2016'
self.beam_model_bool = None
self.vcstools_version = vcstools_version
comp_config = load_config_file()
# Check to make sure paths and files exist:
# First, check that the actual data directory exists
if datadir is None:
# use the default data path
self.data_dir = os.path.join(comp_config['base_data_dir'], str(obsid), "cal", str(cal_obsid), "vis")
logger.info("Using default calibrator data path: {0}".format(self.data_dir))
if os.path.exists(os.path.realpath(self.data_dir)) is False:
errmsg = "Default data directory ({0}) does not exist. Aborting.".format(self.data_dir)
logger.error(errmsg)
raise CalibrationError(errmsg)
elif os.path.isdir(datadir):
self.data_dir = os.path.realpath(datadir)
logger.info("Using the user specified data directory: {0}".format(datadir))
else:
errmsg = "Data directory ({0}) does not exist. Aborting.".format(datadir)
logger.error(errmsg)
raise CalibrationError(errmsg)
# Then check if the specified output and batch directories exists
if outdir is None:
# this is the default
logger.info("Assuming default directory structure...")
self.output_dir = os.path.join(comp_config['base_data_dir'], str(self.obsid), "cal", str(self.cal_obsid), "rts")
self.batch_dir =os.path.join(comp_config['base_data_dir'], str(self.obsid), "batch")
logger.debug("RTS output directory is {0}".format(self.output_dir))
logger.debug("Batch directory is {0}".format(self.batch_dir))
mdir(self.output_dir, "RTS", gid=comp_config['gid'])
mdir(self.batch_dir, "Batch", gid=comp_config['gid'])
else:
# mdir handles if the directory already exists
self.output_dir = os.path.realpath(outdir + "/rts")
self.batch_dir = os.path.realpath(outdir + "/batch")
logger.warning("Non-standard RTS output path: {0}".format(self.output_dir))
logger.warning("Non-standard batch directory path: {0}".format(self.batch_dir))
mdir(self.output_dir, "RTS", gid=comp_config['gid'])
mdir(self.batch_dir, "Batch", gid=comp_config['gid'])
# Then check that the metafits file exists
if os.path.isfile(metafits) is False:
# file doesn't exist
errmsg = "Given metafits file ({0}) does not exist.".format(metafits)
logger.error(errmsg)
raise CalibrationError(errmsg)
elif "_ppds" not in metafits:
# file doesn't have the correct naming convention
errmsg = "Looks like you have an old-style metafits. " \
"You'll need to download the new version, which is named like: " \
"{0}_metafits_ppds.fits.".format(obsid)
logger.error(errmsg)
raise CalibrationError(errmsg)
else:
logger.info("Metafits file exists and is named correctly.")
self.metafits = os.path.realpath(metafits)
logger.debug(" {0}".format(self.metafits))
# the check that the source list exists
if os.path.isfile(srclist) is False:
# file doesn't exist
errmsg = "Given source list file ({0}) does not exist.".format(srclist)
logger.error(errmsg)
raise CalibrationError(errmsg)
else:
logger.info("Checking source list file exists... Ok")
self.source_list = os.path.realpath(srclist)
# Check the 'beam_model' is one of the correct choices
choices = ("FEE2016", "ANALYTIC")
if self.beam_model not in choices:
errmsg = "Given beam model: {0} not an available choice: {1}".format(self.beam_model, choices)
logger.error(errmsg)
raise CalibrationError(errmsg)
else:
logger.info("Using {0} beam model for calibration solution".format(self.beam_model))
self.beam_model_bool=int(bool(self.beam_model == "ANALYTIC")) #produces 1 for ANALYTIC, 0 for FEE2016
# set some RTS flags based on if we have offline correlated data or not
logger.info("Setting RTS data input flags...")
if self.offline:
self.useCorrInput = 1
self.readDirect = 0
logger.debug("Offline correlation")
else:
self.useCorrInput = 0
self.readDirect = 1
logger.debug("Online correlation")
def vcs_download(obsid, start_time, stop_time, increment, data_dir,
product_dir, parallel,
ics=False, n_untar=2, keep="", vcstools_version="master",
nice=0):
#Load computer dependant config file
comp_config = load_config_file()
logger.info("Downloading files from archive")
voltdownload = "voltdownload.py"
obsinfo = meta.getmeta(service='obs', params={'obs_id':str(obsid)})
comb_del_check = meta.combined_deleted_check(obsid, begin=start_time, end=stop_time)
data_format = obsinfo['dataquality']
if data_format == 1 or (comb_del_check and data_format == 6):
# either only the raw data is available (data_format == 1)
# or there was combined files but they were deleted (comb_del_check and data_format == 6)
target_dir = link = '/raw'
if ics:
logger.error("Data have not been recombined in the "
"archive yet. Exiting")
sys.exit(0)
data_type = 11
dl_dir = "{0}/{1}".format(data_dir, target_dir)
dir_description = "Raw"
elif data_format == 6:
target_dir = link = '/combined'
if ics:
data_type = 15
else:
data_type = 16
dl_dir = "{0}/{1}".format(data_dir, target_dir)
dir_description = "Combined"
else:
logger.error("Unable to determine data format from archive. Exiting")
sys.exit(0)
mdir(dl_dir, dir_description, gid=comp_config['gid'])
create_link(data_dir, target_dir, product_dir, link)
batch_dir = product_dir+"/batch/"
for time_to_get in range(start_time,stop_time,increment):
if time_to_get + increment > stop_time:
increment = stop_time - time_to_get + 1
#need to subtract 1 from increment since voltdownload wants how many
#seconds PAST the first one
voltdownload_batch = "volt_{0}".format(time_to_get)
check_batch = "check_volt_{0}".format(time_to_get)
volt_secs_to_run = datetime.timedelta(seconds=500*increment)
check_secs_to_run = "15:00"
if data_type == 16:
check_secs_to_run = "10:15:00"
checks = "checks.py"
# Write out the checks batch file but don't submit it
commands = []
commands.append("newcount=0")
commands.append("let oldcount=$newcount-1")
commands.append("sed -i -e \"s/oldcount=${{oldcount}}/oldcount=${{newcount}}/\" {0}".\
format(batch_dir+voltdownload_batch+".batch"))
commands.append("oldcount=$newcount; let newcount=$newcount+1")
commands.append("sed -i -e \"s/_${{oldcount}}.out/_${{newcount}}.out/\" {0}".\
format(batch_dir+voltdownload_batch+".batch"))
checks_command = "-m download -o {0} -w {1} -b {2} -i {3} --data_type {4}".format(obsid,
dl_dir, time_to_get, increment, data_type)
commands.append('{0} {1}'.format(checks, checks_command))
commands.append("if [ $? -eq 1 ];then")
commands.append("sbatch {0}".format(batch_dir+voltdownload_batch+".batch"))
# if we have tarballs we send the untar jobs to the workq
if data_type == 16:
commands.append("else")
untar = 'untar.sh'
untar_command = "-w {0} -o {1} -b {2} -e {3} -j {4} {5}".format(dl_dir,
obsid, time_to_get, time_to_get+increment-1, n_untar,
keep)
commands.append('{0} {1}'.format(untar, untar_command))
#commands.append("sbatch {0}.batch".format(batch_dir+tar_batch))
commands.append("fi")
# Download and checks should be done on Zeus's cpuq. This will only work
# on Galaxy as the Ozstar workflow is different
submit_slurm(check_batch, commands, batch_dir=batch_dir,
slurm_kwargs={"time": check_secs_to_run,
"nice": nice},
vcstools_version=vcstools_version, submit=False,
outfile=batch_dir+check_batch+"_0.out",
queue="zcpuq", export="NONE", mem=10240,
# Manually handing it the module dir as it should only run
module_dir='/group/mwa/software/modulefiles')
# Write out the tar batch file if in mode 15
#if format == 16:
# body = []
# for t in range(time_to_get, time_to_get+increment):
# body.append("aprun tar -xf {0}/1149620392_{1}_combined.tar".format(dl_dir,t))
# submit_slurm(tar_batch,body,batch_dir=working_dir+"/batch/", slurm_kwargs={"time":"1:00:00", "partition":"gpuq" })
#module_list=["mwa-voltage/master"]
#removed the master version load because by default we load the python 3 version
module_list=[]
body = []
body.append("oldcount=0")
body.append("let newcount=$oldcount+1")
body.append("if [ ${newcount} -gt 10 ]; then")
body.append("echo \"Tried ten times, this is silly. Aborting here.\";exit")
body.append("fi")
body.append("sed -i -e \"s/newcount=${{oldcount}}/newcount=${{newcount}}/\" {0}\n".\
format(batch_dir+check_batch+".batch"))
body.append("sed -i -e \"s/_${{oldcount}}.out/_${{newcount}}.out/\" {0}".\
format(batch_dir+check_batch+".batch"))
body.append("sbatch -d afterany:${{SLURM_JOB_ID}} {0}".\
format(batch_dir+check_batch+".batch"))
voltdownload_command = "--obs={0} --type={1} --from={2} --duration={3} --parallel={4}"\
" --dir={5}".format(obsid, data_type, time_to_get, increment-1,
parallel, dl_dir)
body.append("{0} {1}".format(voltdownload, voltdownload_command))
submit_slurm(voltdownload_batch, body, batch_dir=batch_dir,
module_list=module_list,
slurm_kwargs={"time" : str(volt_secs_to_run),
"nice" : nice},
vcstools_version=vcstools_version,
outfile=batch_dir+voltdownload_batch+"_1.out",
queue="copyq", export="NONE", mem=5120,
# Manually handing it the module dir as it should only run
module_dir='/group/mwa/software/modulefiles')
def coherent_beam(obs_id, start, stop, data_dir, product_dir, batch_dir,
metafits_file, nfine_chan, pointing_list,
rts_flag_file=None, bf_formats=None, DI_dir=None,
execpath=None, calibration_type='rts', ipfb_filter="LSQ12",
vcstools_version="master", nice=0, channels_to_beamform=None,
beam_version="FEE2016"):
"""
This function runs the new version of the beamformer. It is modelled after
the old function above and will likely be able to be streamlined after
working implementation (SET)
Streamlining underway, as well as full replacement of the old function (SET March 28, 2018)
"""
#Load computer dependant config file
comp_config = load_config_file()
# If execpath is given, change the make_beam executable command
# otherwise, it should be on the PATH if vcstools has been installed
if execpath:
make_beam_cmd = "{0}/make_beam".format(execpath)
make_beam_version_cmd = "{0}/make_beam -V".format(execpath)
else:
make_beam_cmd = "make_beam"
make_beam_version_cmd = "make_beam -V"
make_beam_version = subprocess.Popen(make_beam_version_cmd,
stdout=subprocess.PIPE, shell=True).communicate()[0]
logger.info("Current version of make_beam = {0}".format(make_beam_version.strip()))
metafile = "{0}/{1}.meta".format(product_dir, obs_id)
channels = None
# No channels given so first check for a metafile
if os.path.isfile(metafile):
logger.info("Found observation metafile: {0}".format(metafile))
with open(metafile, 'r') as m:
for line in m.readlines():
if line.startswith("channels"):
channels = line.split(",")[1:]
channels = np.array(channels, dtype=np.int)
else:
logger.debug("No metafile in {0}".format(metafile))
logger.debug("Channels before meta.get_channels: {0}".format(channels))
# If channels is still None get_channels will get it from the metadata
channels = meta.get_channels(obs_id, channels=channels)
# Make a metafile containing the channels so no future metadata calls are required
if not os.path.isfile(metafile):
with open(metafile, "w") as m:
m.write("#Metadata for obs ID {0} required to determine if: normal or "
"picket-fence\n".format(obs_id))
m.write("channels,{0}".format(",".join([str(c) for c in channels])))
channels = np.array(channels, dtype=np.int)
hichans = [c for c in channels if c>128]
lochans = [c for c in channels if c<=128]
lochans.extend(list(reversed(hichans)))
ordered_channels = lochans
if channels_to_beamform is None:
# If no channels_to_beamform given fold on everything
channels_to_beamform = ordered_channels
# Run for each coarse channel. Calculates delays and makes beam
if not DI_dir:
logger.error("You need to specify the path to the calibrator files, "
"either where the DIJs are or where the Offringa "
"calibration_solutions.bin file is. Aborting here")
sys.exit(0)
DI_dir = os.path.abspath(DI_dir)
# make_beam_small requires the start time in UTC, get it from the start
utctime = gps_to_utc(start)
P_dir = os.path.join(product_dir, "pointings")
mdir(P_dir, "Pointings", gid=comp_config['gid'])
mdir(os.path.join(product_dir, "incoh"), "Incoh", gid=comp_config['gid'])
# startjobs = True
# Set up supercomputer dependant parameters
import socket
hostname = socket.gethostname()
if hostname.startswith('john') or hostname.startswith('farnarkle'):
max_pointing = 120
else:
max_pointing = 15
if comp_config['ssd_dir'] is None:
temp_mem = None
else:
#Work out required SSD size
obs_length = stop - start + 1.
temp_mem = int(0.0012 * obs_length * max_pointing + 1)
temp_mem_single = int(0.0024 * obs_length + 2)
if "-s" not in bf_format:
temp_mem = temp_mem * 4
temp_mem_single = temp_mem_single *4
# set up SLURM requirements
if len(pointing_list) > max_pointing:
seconds_to_run = 8 * (stop - start + 1) * max_pointing
else:
seconds_to_run = 8 * (stop - start + 1) * len(pointing_list)
if seconds_to_run > 86399.:
secs_to_run = datetime.timedelta(seconds=86399)
else:
secs_to_run = datetime.timedelta(seconds=seconds_to_run)
# Get the project id (eg G0057) from the metafits file
with pyfits.open(metafits_file) as hdul:
project_id = hdul[0].header['project']
# splits the pointing list into lists of length max_pointing
pointing_list_list = list(chunks(pointing_list, max_pointing))
time_now = str(datetime.datetime.now()).replace(" ", "_")
logging.info("Running make_beam")
job_id_list_list = []
for pl, pointing_list in enumerate(pointing_list_list):
pointing_str = ",".join(pointing_list)
# Run one coarse channel per node
job_id_list = []
for gpubox, coarse_chan in enumerate(ordered_channels, 1):
if coarse_chan not in channels_to_beamform:
continue
if calibration_type == 'rts':
#chan_list = get_frequencies(metafits_file, resort=True)
DI_file = "{0}/DI_JonesMatrices_node{1:0>3}.dat".format(DI_dir, gpubox)
jones_option = "-J {0}".format(DI_file)
elif calibration_type == 'offringa':
#chan_list = get_frequencies(metafits_file, resort=False)
DI_file = "{0}/calibration_solution.bin".format(DI_dir)
jones_option = "-O {0} -C {1}".format(DI_file, int(gpubox) - 1)
else:
logger.info("Please an accepted calibratin type. Aborting here.")
sys.exit(0)
# Making pointing directories
for pointing in pointing_list:
mdir("{0}/{1}".format(P_dir, pointing), "Pointing {0}".format(pointing), gid=comp_config['gid'])
n_omp_threads = 1
if "v" in bf_formats:
for pointing in pointing_list:
make_beam_small_batch = "mb_{0}_ch{1}".format(pointing, coarse_chan)
module_list = [comp_config['container_module']]
commands = []
commands.append("cd {0}/{1}".format(P_dir,pointing))
runline = "srun --export=all -n 1"
runline += " -c {}".format(n_omp_threads)
if comp_config['container_command'] !='':
runline += " {} '".format(comp_config['container_command'])
runline += " {}".format(make_beam_cmd)
runline += " -o {}".format(obs_id)
runline += " -b {}".format(start)
runline += " -e {}".format(stop)
runline += " -a 128"
runline += " -n 128"
runline += " -f {}".format(coarse_chan)
runline += " {}".format(jones_option)
runline += " -d {}/combined".format(data_dir)
runline += " -P {}".format(pointing)
runline += " -r 10000"
runline += " -m {}".format(metafits_file)
runline += " -z {}".format(utctime)
runline += " {}".format(bf_formats)
runline += " -F {}".format(rts_flag_file)
runline += " -S {}".format(ipfb_filter)
if beam_version == "ANALYTIC":
runline += " -H"
if comp_config['container_command'] !='':
runline += "'"
commands.append(runline)
job_id = submit_slurm(make_beam_small_batch, commands,
batch_dir=batch_dir, module_list=module_list,
slurm_kwargs={"time":secs_to_run, "nice":nice},
queue='gpuq', vcstools_version=vcstools_version,#forces olf version with vdif
submit=True, export="NONE", gpu_res=1,
cpu_threads=n_omp_threads,
mem=comp_config['gpu_beamform_mem'], temp_mem=temp_mem_single)
job_id_list.append(job_id)
else:
make_beam_small_batch = "mb_{0}_{1}_ch{2}".format(pl, time_now, coarse_chan)
module_list = [comp_config['container_module']]
commands = []
if comp_config['ssd_dir'] is None:
# Write outputs to SSDs if on Ozstar
commands.append("cd {0}".format(P_dir))
else:
commands.append("cd {0}".format(comp_config['ssd_dir']))
runline = "srun --export=all -n 1"
runline += " -c {}".format(n_omp_threads)
if comp_config['container_command'] !='':
runline += " {} '".format(comp_config['container_command'])
runline += " {}".format(make_beam_cmd)
runline += " -o {}".format(obs_id)
runline += " -b {}".format(start)
runline += " -e {}".format(stop)
runline += " -a 128"
runline += " -n 128"
runline += " -f {}".format(coarse_chan)
runline += " {}".format(jones_option)
runline += " -d {}/combined".format(data_dir)
runline += " -P {}".format(pointing_str)
runline += " -r 10000"
runline += " -m {}".format(metafits_file)
runline += " -z {}".format(utctime)
runline += " {}".format(bf_formats)
runline += " -F {}".format(rts_flag_file)
if beam_version == "ANALYTIC":
runline += " -H"
if comp_config['container_command'] !='':
runline += "'"
commands.append(runline)
commands.append("")
if comp_config['ssd_dir'] is not None:
for pointing in pointing_list:
commands.append("cp {0}/{1}/{2}_{3}_{1}_ch{4}_00*.fits "
"{5}/{1}/".format(comp_config['ssd_dir'], pointing,
project_id, obs_id, coarse_chan, P_dir))
if 'i' in bf_formats:
commands.append("cp {0}/{1}/{2}_{3}_{1}_ch{4}_00*.fits "
"{5}/{1}/".format(comp_config['ssd_dir'], "incoh",
project_id, obs_id, coarse_chan, product_dir))
commands.append("")
job_id = submit_slurm(make_beam_small_batch, commands,
batch_dir=batch_dir, module_list=module_list,
slurm_kwargs={"time":secs_to_run, "nice":nice},
queue='gpuq', vcstools_version=vcstools_version,
submit=True, export="NONE", gpu_res=1,
cpu_threads=n_omp_threads,
mem=comp_config['gpu_beamform_mem'], temp_mem=temp_mem)
job_id_list.append(job_id)
job_id_list_list.append(job_id_list)
return job_id_list_list, make_beam_small_batch.split('ch')[0]
def analyise_and_flux_cal(pulsar, bestprof_data,
flagged_tiles=None,
calid=None,
common_metadata=None,
trcvr=data_load.TRCVR_FILE,
simple_sefd=False, sefd_file=None,
vcstools_version='master',
args=None,
flux_method="radiometer"):
"""Analyise a pulse profile and calculates its flux density
Parameters
----------
pulsar : `str`
The pulsar's Jname
bestprof_data: list
The output list from the function :py:meth:`vcstools.prof_utils.get_from_bestprof`
Optional parameters:
-------------------
flagged_tiles : `str`
The location of the flagged_tiles.txt file. If it's in the default location you can just supply the calid.
calid : `int`
The calibration ID of the detection. This is used to find the flagged_tiles.txt file.
common_metadata: list
The output of mwa_metadb_utils.get_common_obs_metadata(). If not supplied it will be downloaded.
trcvr : `str`
The file location of antena temperatures.
simple_sefd : `boolean`
If True perfom just a simple SEFD calculation instead of simulating the phased array beam response over the sky. Default: False.
sefd_file : `str`
The location of the pabeam.py's simulation of the phased array beam response over the sky output file. If not supplied will launch a pabeam.py simulation.
vcstools_version : `str`
The version of vcstools to use for the pabeam.py simulation. Default: master.
args: Namespace
The args from argparse to be used for job resubmission. Default: None.
Returns
-------
det_kwargs: dict
det_kwargs["flux"]: The mean flux density of the pulsar in mJy
det_kwargs["flux_error"]: The flux desnity error in mJy
det_kwargs["width"]: The equivalent width of the pulsar in ms
det_kwargs["width_error"]: The error of the equivalent width in ms
det_kwargs["scattering"]: The scattering width in s
det_kwargs["scattering_error"]: The error of the scattering in s
"""
# Load computer dependant config file
comp_config = load_config_file()
#unpack the bestprof_data
obsid, prof_psr, _, period, _, sigma, beg, t_int, profile, num_bins, pointing = bestprof_data
period=float(period)
num_bins=int(num_bins)
# Perform metadata calls
if common_metadata is None:
common_metadata = get_common_obs_metadata(obsid)
obsid, ra, dec, dura, [xdelays, ydelays], centrefreq, channels = common_metadata
# assume full bandwidth of 30.72 MHz
bandwidth = 30.72e6
# Find pulsar ra and dec
_, pul_ra, pul_dec = get_psrcat_ra_dec(pulsar_list=[pulsar])[0]
# Work out flagged tiles from calbration directory
if not flagged_tiles:
if calid:
flagged_file = os.path.join(comp_config['base_data_dir'], obsid, "cal", calid, "rts", "flagged_tiles.txt")
if os.path.exists(flagged_file):
with open(flagged_file, "r") as ftf:
flagged_tiles = []
reader = csv.reader(ftf)
for row in reader:
flagged_tiles.append(row)
flagged_tiles = np.array(flagged_tiles).flatten()
else:
logger.warn("No flagged_tiles.txt file found so assuming no tiles have been flagged")
flagged_tiles = []
else:
logger.warn("No flagged_tiles or calid provided so assuming no tiles have been flagged")
flagged_tiles = []
# Calc SEFD from the T_sys and gain
if simple_sefd:
t_sys, _, gain, u_gain = find_t_sys_gain(pulsar, obsid,
common_metadata=common_metadata,
beg=beg, end=(t_int + beg - 1))
sefd = tsys / gain
else:
if sefd_file is None:
launch_pabeam_sim(obsid, pointing, beg, t_int,
source_name=pulsar,
vcstools_version=vcstools_version,
flagged_tiles=flagged_tiles,
delays=xdelays,
args=args,
common_metadata=common_metadata)
sys.exit(0)
else:
sefd_freq_time, sefd, u_sefd = read_sefd_file(sefd_file)
#estimate S/N
try:
g_fitter = gfit(profile)
g_fitter.plot_name = f"{obsid}_{pulsar}_{num_bins}_bins_gaussian_fit.png"
g_fitter.component_plot_name = f"{obsid}_{pulsar}_{num_bins}_bins_gaussian_components.png"
g_fitter.auto_fit()
g_fitter.plot_fit()
prof_dict = g_fitter.fit_dict
except (prof_utils.ProfileLengthError, prof_utils.NoFitError):
logger.info("Profile couldn't be fit. Using old style of profile analysis")
prof_dict = prof_utils.auto_analyse_pulse_prof(profile, period)
if not prof_dict:
logger.warn("Profile could not be analysed using any methods")
det_kwargs = {}
det_kwargs["flux"] = None
det_kwargs["flux_error"] = None
det_kwargs["width"] = None
det_kwargs["width_error"] = None
det_kwargs["scattering"] = None
det_kwargs["scattering_error"] = None
return det_kwargs, None, None
# Unpack dictionary
sn = prof_dict["sn"]
u_sn = prof_dict["sn_e"]
profile = prof_dict["profile"]
on_pulse_bool = prof_dict["on_pulse_bool"]
noise_std = prof_dict["noise_std"]
noise_mean = prof_dict["noise_mean"]
w_equiv_phase = prof_dict["Weq"]
u_w_equiv_phase = prof_dict["Weq_e"]
w_equiv_bins = w_equiv_phase * num_bins
u_w_equiv_bins = w_equiv_phase * num_bins
w_equiv_ms = period * w_equiv_phase
u_w_equiv_ms = period * u_w_equiv_phase
scattering = prof_dict["Wscat"]*period/1000 #convert to seconds
u_scattering = prof_dict["Wscat_e"]*period/1000
scattered = prof_dict["scattered"]
logger.info("Profile scattered? {0}".format(scattered))
logger.info("S/N: {0} +/- {1}".format(sn, u_sn))
#logger.debug("Gain {0} K/Jy".format(gain))
logger.debug("Equivalent width in ms: {0}".format(w_equiv_ms))
#logger.debug("T_sys: {0} K".format(t_sys))
logger.debug("Detection time: {0}".format(t_int))
logger.debug("Number of bins: {0}".format(num_bins))
# Renormalise around the noise mean
noise = []
noise_i = []
on_pulse = []
on_pulse_i = []
for p, b, i in zip(profile, on_pulse_bool, range(len(profile))):
if not b:
noise.append(p)
noise_i.append(i)
else:
on_pulse.append(p)
on_pulse_i.append(i)
plt.scatter(on_pulse_i, on_pulse, color="blue")
plt.scatter(noise_i, noise, color="red")
plt.savefig("test.png")
noise_mean = np.mean(noise)
print(f"Noise mean: {noise_mean}")
profile = (profile - noise_mean) / max(profile - noise_mean)
logger.debug(list(profile))
logger.debug(on_pulse_bool)
logger.debug(noise_std, w_equiv_bins, sefd, u_sefd, t_int)
# Final calc of the mean flux density in mJy
if flux_method == "radiometer":
S_mean, u_S_mean = flux_calc_radiometer_equation(profile, on_pulse_bool,
noise_std, w_equiv_bins,
sefd, u_sefd, t_int, bandwidth=bandwidth)
elif flux_method == "flux_profile":
S_mean, u_S_mean = flux_calc_flux_profile(profile,
noise_std,
sefd, u_sefd, t_int, bandwidth=bandwidth)
logger.info('Smean {0:.3f} +/- {1:.3f} mJy'.format(S_mean, u_S_mean))
#prevent TypeError caused by trying to format Nones given to fluxes for highly scattered pulsars
S_mean = float("{0:.3f}".format(S_mean))
u_S_mean = float("{0:.3f}".format(u_S_mean))
# Plot flux comparisons for ANTF
freq_all, flux_all, flux_err_all, _ = flux_from_atnf(pulsar)
logger.debug("Freqs: {0}".format(freq_all))
logger.debug("Fluxes: {0}".format(flux_all))
logger.debug("Flux Errors: {0}".format(flux_err_all))
logger.debug("{0} there are {1} flux values available on the ATNF database"\
.format(pulsar, len(flux_all)))
# Check if there is enough data to estimate the flux
#if len(flux_all) == 0:
# logger.debug("{} no flux values on archive. Cannot estimate flux.".format(pulsar))
#elif ( len(flux_all) == 1 ) and ( ( not spind ) or ( not spind_err ) ):
# logger.debug("{} has only a single flux value and no spectral index. Cannot estimate flux. Will return Nones".format(pulsar))
#else:
# spind, spind_err, K, covar_mat = find_spind(pulsar, freq_all, flux_all, flux_err_all)
# plot_flux_estimation(pulsar, freq_all, flux_all, flux_err_all, spind,
# my_nu=centrefreq, my_S=S_mean, my_S_e=u_S_mean, obsid=obsid,
# a_err=spind_err, K=K, covar_mat=covar_mat)
#format data for uploading
w_equiv_ms = float("{0:.2f}".format(w_equiv_ms))
u_w_equiv_ms = float("{0:.2f}".format(u_w_equiv_ms))
scattering = float("{0:.5f}".format(scattering))
u_scattering = float("{0:.5f}".format(u_scattering))
det_kwargs = {}
det_kwargs["flux"] = S_mean
det_kwargs["flux_error"] = u_S_mean
det_kwargs["width"] = w_equiv_ms
det_kwargs["width_error"] = u_w_equiv_ms
det_kwargs["scattering"] = scattering
det_kwargs["scattering_error"] = u_scattering
return det_kwargs, sn, u_sn
def vcs_recombine(obsid, start_time, stop_time, increment, data_dir, product_dir,
vcstools_version="master", nice=0):
#Load computer dependant config file
comp_config = load_config_file()
logger.info("Running recombine on files")
jobs_per_node = 8
target_dir = link = 'combined'
mdir(data_dir + '/' + target_dir, 'Combined', gid=comp_config['gid'])
create_link(data_dir, target_dir, product_dir, link)
batch_dir = product_dir+"/batch/"
recombine = "recombine.py"
checks = "checks.py"
recombine_binary = "recombine"
for time_to_get in range(start_time,stop_time,increment):
process_nsecs = increment if (time_to_get + increment <= stop_time) \
else (stop_time - time_to_get + 1)
if (jobs_per_node > process_nsecs):
jobs_per_node = process_nsecs
nodes = (increment+(-increment%jobs_per_node))//jobs_per_node + 1 # Integer division with ceiling result plus 1 for master node
recombine_batch = "recombine_{0}".format(time_to_get)
check_batch = "check_recombine_{0}".format(time_to_get)
#module_list = ["module switch PrgEnv-cray PrgEnv-gnu", "python/3.6.3", "numpy/1.13.3", "mwa-voltage/master"]
module_list = ["mwa-voltage/master"]
commands = []
commands.append("newcount=0")
commands.append("let oldcount=$newcount-1")
commands.append("sed -i -e \"s/oldcount=${{oldcount}}/oldcount=${{newcount}}/\" {0}".\
format(batch_dir+recombine_batch+".batch"))
commands.append("oldcount=$newcount; let newcount=$newcount+1")
commands.append("sed -i -e \"s/_${{oldcount}}.out/_${{newcount}}.out/\" {0}".\
format(batch_dir+recombine_batch+".batch"))
checks_command = "-m recombine -o {0} -w {1}/combined/ -b {2} -i {3}".format(obsid,
data_dir, time_to_get, process_nsecs)
commands.append("{0} {1}".format(checks, checks_command))
commands.append("if [ $? -eq 1 ];then")
commands.append("sbatch {0}".format(batch_dir+recombine_batch+".batch"))
commands.append("fi")
submit_slurm(check_batch, commands, batch_dir=batch_dir,
module_list=module_list,
slurm_kwargs={"time": "15:00", "nice": nice},
vcstools_version=vcstools_version, submit=False,
outfile=batch_dir+check_batch+"_0.out",
queue='gpuq', export="NONE")
#module_list = ["module switch PrgEnv-cray PrgEnv-gnu", "python/3.6.3",
# "numpy/1.13.3", "mwa-voltage/master", "mpi4py", "cfitsio"]
module_list = ["mwa-voltage/master", "mpi4py"]
commands = []
commands.append("oldcount=0")
commands.append("let newcount=$oldcount+1")
commands.append("if [ ${newcount} -gt 10 ]; then")
commands.append("echo \"Tried ten times, this is silly. Aborting here.\";exit")
commands.append("fi")
commands.append("sed -i -e \"s/newcount=${{oldcount}}/newcount=${{newcount}}/\" {0}".\
format(batch_dir+check_batch+".batch"))
commands.append("sed -i -e \"s/_${{oldcount}}.out/_${{newcount}}.out/\" {0}".\
format(batch_dir+check_batch+".batch"))
commands.append("sbatch -d afterany:${{SLURM_JOB_ID}} {0}".\
format(batch_dir+check_batch+".batch"))
recombine_command = "-o {0} -s {1} -w {2} -e {3}".format(obsid, time_to_get,
data_dir, recombine_binary)
commands.append("srun --export=all {0} {1}".format(recombine, recombine_command))
submit_slurm(recombine_batch, commands, batch_dir=batch_dir,
module_list=module_list,
slurm_kwargs={"time": "06:00:00", "nodes": str(nodes),
"ntasks-per-node": jobs_per_node,
"nice": nice},
vcstools_version=vcstools_version,
outfile=batch_dir+recombine_batch+"_1.out",
queue='gpuq', export="NONE")
def find_fwhm_and_plot(obsid, pointing):
pointing_list = []
sn = []
comp_config = load_config_file()
for d in glob.glob("{0}/{1}/pointings/*".format(
comp_config['base_product_dir'], obsid)):
bestprof_file = glob.glob("{0}/{1}*_PSR_2330-2005.pfd.bestprof".format(
d, obsid))
if len(bestprof_file) == 1:
with open(bestprof_file[0]) as bestfile:
for i in bestfile.readlines():
if i.startswith("# Prob(Noise)"):
pointing_list.append(d.split("/")[-1])
sn.append(float(i.split("(")[-1][1:-7]))
#find max for a FWHM test
#max_index = sn.index(max(sn))
ra_hex = pointing.split("_")[0]
dec_hex = pointing.split("_")[1]
print(ra_hex, dec_hex)
coord = SkyCoord(ra_hex, dec_hex, unit=(u.hourangle, u.deg))
dec_centre = coord.dec.degree
ra_centre = coord.ra.degree
ras = []
decs = []
ra_line = []
ra_sn_line = []
dec_line = []
dec_sn_line = []
print(sn)
for i in range(len(sn)):
rah, dech = pointing_list[i].split("_")
coord = SkyCoord(rah, dech, unit=(u.hourangle, u.deg))
ras.append(coord.ra.degree)
decs.append(coord.dec.degree)
if decs[i] == dec_centre:
ra_line.append(ras[i])
ra_sn_line.append(sn[i])
if ras[i] == ra_centre:
dec_line.append(decs[i])
dec_sn_line.append(sn[i])
max_ra_i = np.argmax(ra_sn_line)
max_dec_i = np.argmax(dec_sn_line)
max_coord = SkyCoord(ra_line[max_ra_i],
dec_line[max_dec_i],
unit=(u.deg, u.deg))
ra_max_hex = max_coord.ra.to_string(unit=u.hour, sep=':')
dec_max_hex = max_coord.dec.to_string(unit=u.degree, sep=':')
print("sn max coord: {0}_{1}".format(ra_max_hex, dec_max_hex))
#sort and calc FWHM
ra_sn_line = [x for _, x in sorted(zip(ra_line, ra_sn_line))]
ra_line = sorted(ra_line)
print(ra_sn_line, ra_line)
dec_sn_line = [x for _, x in sorted(zip(dec_line, dec_sn_line))]
dec_line = sorted(dec_line)
print(dec_sn_line, dec_line)
ra_sn_line = np.array(ra_sn_line)
dec_sn_line = np.array(dec_sn_line)
spline = UnivariateSpline(ra_line,
ra_sn_line - np.max(ra_sn_line) / 2.,
s=0)
print(spline.roots())
if len(spline.roots()) == 2:
r1, r2 = spline.roots()
ra_FWHM = abs(r1 - r2)
print("raw ra FHWM: " + str(ra_FWHM))
cor_ra_FWHM = float(ra_FWHM) * math.cos(np.radians(dec_centre))
print("corrected ra FWHM: {0}".format(cor_ra_FWHM))
else:
print("No detectable ra FWHM (too many roots)")
spline = UnivariateSpline(dec_line,
dec_sn_line - np.max(dec_sn_line) / 2.,
s=0)
if len(spline.roots()) == 2:
r1, r2 = spline.roots()
dec_FWHM = abs(r1 - r2)
print("raw dec FHWM: " + str(dec_FWHM))
cor_dec_FWHM = float(dec_FWHM) * math.cos(
np.radians(dec_centre) + np.radians(26.7))**2
print("corrected dec FWHM: {0}".format(cor_dec_FWHM))
else:
print("No detectable dec FWHM (too many roots)")
diff = 10**20
# Find the min diff by comparing difference
# of all possible pairs in given array
n = len(dec_line)
for i in range(n - 1):
for j in range(i + 1, n):
if abs(dec_line[i] - dec_line[j]) < diff:
diff = abs(dec_line[i] - dec_line[j])
n = len(ra_line)
for i in range(n - 1):
for j in range(i + 1, n):
if abs(ra_line[i] - ra_line[j]) < diff:
diff = abs(ra_line[i] - ra_line[j])
diff = 0.01
print("Diff: {}".format(diff))
ras = np.array(ras)
decs = np.array(decs)
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111)
plt.grid(True)
#sort by sn
ras = [x for _, x in sorted(zip(sn, ras))]
decs = [x for _, x in sorted(zip(sn, decs))]
sn = sorted(sn)
cm = plt.cm.get_cmap('plasma', 20)
ax.grid(color='r', linestyle='-', linewidth=2)
sp = plt.scatter(ras, decs, c=sn, s=(1000 * diff)**2, cmap=cm)
#plt.gray()
plt.colorbar(sp)
plt.savefig("{0}_position_heatmap.png".format(obsid))
plt.show()
def write_batch_files(obsid,
begin,
end,
ra,
dec,
freq,
flaggedtiles,
step=500,
thetares=0.05,
phires=0.05,
nnodes=1,
eff=1,
beam_model='hyperbeam',
maploc="$PWD",
odir=None,
delays=[0] * 16,
write=True,
write_showspec=False,
vcstools_version='master',
metafits_loc=None):
comp_config = load_config_file()
times = np.arange(begin, end, step=step)
times = np.append(times, end)
#nprocesses = 32 * nnodes
nprocesses = 1 * nnodes
flags = " ".join(flaggedtiles)
if odir is None:
# Make default directories
product_dir = os.path.join(comp_config['base_data_dir'], obsid,
'pabeam', '{}_{}'.format(ra, dec))
batch_dir = os.path.join(comp_config['base_data_dir'], obsid, 'batch')
else:
product_dir = batch_dir = odir
mdir(product_dir, 'Product Dir', gid=comp_config['gid'])
mdir(batch_dir, 'Batch Dir', gid=comp_config['gid'])
# Loop over all times
for i in range(len(times)):
fname = "make_pabeam_{0}_{1}_{2}_{3:.2f}MHz".format(
ra, dec, times[i], freq / 1e6)
onamebase = "{0}_{1}_{2:.2f}MHz_tres{3}_pres{4}_{5}_{6}".format(
obsid, float(times[i]), freq / 1e6, thetares, phires, ra, dec)
commands = []
# Write out params
commands.append("nprocesses={}".format(nprocesses))
commands.append("obsid={}".format(obsid))
commands.append("""ra='"{}"'""".format(ra))
commands.append("""dec='"{}"'""".format(dec))
commands.append("freq={}".format(freq))
commands.append("eff={}".format(eff))
commands.append('flags="{}"'.format(flags))
commands.append('delays="{}"'.format(delays))
commands.append("tres={}".format(thetares))
commands.append("pres={}".format(phires))
commands.append("obstime={}".format(times[i]))
commands.append("odir={}".format(product_dir))
commands.append("metafits_loc={}".format(metafits_loc))
commands.append('beam="{}"'.format(beam_model))
# TODO remove this once hyperbeam is installed with python
commands.append(
"export PYTHONPATH=$PYTHONPATH:/pawsey/mwa/software/python3/hyperbeam/v0.3.0/lib/python3.8/site-packages"
)
# Main command
pabeam_command = "srun --export=all -u -n ${nprocesses} pabeam.py " +\
"-o ${obsid} -f ${freq} -t ${obstime} -e ${eff} -p ${ra} ${dec} --metafits ${metafits_loc} " +\
"--flagged_tiles ${flags} --grid_res ${tres} ${pres} --out_dir ${odir} --beam_model ${beam} --delays ${delays}"
if write:
pabeam_command = pabeam_command + " --write"
commands.append('cd {}'.format(product_dir))
commands.append('echo "{}"'.format(pabeam_command))
commands.append(pabeam_command)
# Combine the output files into one
commands.append(pabeam_concat_cmd.format(onamebase,
onamebase + ".dat"))
# Remove the partial beam pattern files written by processes
commands.append("rm {0}\n".format(onamebase + ".*.dat"))
module_list = ['mpi4py', 'hyperbeam/v0.3.0']
submit_slurm(fname,
commands,
batch_dir=batch_dir,
module_list=module_list,
slurm_kwargs={
"time": "12:00:00",
"nodes": nnodes,
"ntasks-per-node": nprocesses
},
vcstools_version=vcstools_version,
queue='cpuq',
mem=10240)
if write_showspec:
# Now write the showspec batch for this time
write_showspec_batch(times[i], obsid, ra, dec, freq,
(90 / thetares) + 1, 360 / phires,
onamebase + ".dat", maploc)
def vcs_correlate(obsid,start,stop,increment, data_dir, product_dir, ft_res,
metafits, vcstools_version="master", nice=0):
#Load computer dependant config file
comp_config = load_config_file()
logger.info("Correlating files at {0} kHz and {1} milliseconds".\
format(ft_res[0], ft_res[1]))
batch_dir = product_dir+"/batch/"
target_dir = link = 'vis'
if data_dir == product_dir:
corr_dir = "{0}/cal/{1}/{2}".format(product_dir, obsid, target_dir)
else:
corr_dir = "{0}/{1}".format(data_dir, target_dir)
product_dir = "{0}/cal/{1}/".format(product_dir, obsid)
mdir(product_dir, "Correlator", gid=comp_config['gid'])
mdir(corr_dir, "Correlator Product", gid=comp_config['gid'])
create_link(data_dir, target_dir, product_dir, link)
chan_list = get_frequencies(metafits_file, resort=True)
#gpu_int = 0.01 # Code was compiled with a hard-coded 100 sample minimum intigration. For 'normal' data this means 0.01 seconds
gpu_int = 10 # Code was compiled with a hard-coded 100 sample minimum integration. For 'normal' data this means 10 milliseconds.
integrations=int(ft_res[1]/gpu_int)
#num_frames=int(1.0/ft_res[1])
num_frames=int(1000/ft_res[1])
logger.info("Input chan list is {0}".format(chan_list))
for time_to_get in range(start,stop,increment):
inc_start = time_to_get
inc_stop = time_to_get+increment
for index,channel in enumerate(chan_list):
gpubox_label = (index+1)
f=[]
for time_to_corr in range(inc_start,inc_stop,1):
file_to_process = "{0}/combined/{1}_{2}_ch{3:0>2}.dat".\
format(data_dir,obsid,time_to_corr,channel)
#check the file exists
if (os.path.isfile(file_to_process) == True):
f.append(file_to_process)
#now have a full list of files
#for this increment
#and this channel
if (len(f) > 0):
corr_batch = "correlator_{0}_gpubox{1:0>2}".format(inc_start,gpubox_label)
body = []
to_corr = 0
for file in f:
(current_time,_) = os.path.splitext(os.path.basename(file))
(obsid,gpstime,_) = current_time.split('_')
t = Time(int(gpstime), format='gps', scale='utc')
unix_time = int(t.unix)
offline_correlator_command = "-o {0}/{1} -s {2} -r {3} -i {4} -n {5} "\
"-c {6:0>2} -d {7}".format(corr_dir, obsid,
unix_time, num_frames, integrations,
int(ft_res[0]/10), gpubox_label, file)
body.append("{0} {1}".format("offline_correlator", offline_correlator_command))
to_corr += 1
#module_list = ["module switch PrgEnv-cray PrgEnv-gnu"]
module_list = ["offline_correlator/v1.0.0"]
secs_to_run = str(datetime.timedelta(seconds=2*12*num_frames*to_corr))
# added factor two on 10 April 2017 as galaxy seemed really slow...
submit_slurm(corr_batch, body, module_list=module_list,
slurm_kwargs={"time": secs_to_run, "nice": nice},
queue='gpuq', vcstools_version=vcstools_version,
batch_dir=batch_dir, export="NONE")
else:
logger.error("Couldn't find any recombine files. Aborting here.")
from os.path import join
import logging
from glob import glob
from vcstools.config import load_config_file
from vcstools.job_submit import submit_slurm
from dpp.helper_files import glob_pfds
comp_config = load_config_file()
logger = logging.getLogger(__name__)
def submit_prepfold_products_db(cfg, dep_id=None, dep_type="afterany"):
"""Submits the best fold profile to the pulsar database. Will also submit .ppps"""
my_pointing = cfg["source"]["my_pointing"]
# We will upload the init fold and the best post fold
bin_list = list(cfg["folds"][my_pointing]["init"].keys())
bin_list.append(cfg["source"]["my_bins"])
jids = []
for bin_count in bin_list:
commands = []
commands.append(f"cd {cfg['files']['psr_dir']}")
# Get the files to upload
try:
ppps = glob_pfds(cfg, my_pointing, bin_count, pfd_type=".ps")[0]
except IndexError as e:
raise IndexError(f"No ppps files found in dir: {cfg['files']['psr_dir']} for pointing {my_pointing} and bin count {bin_count}")
try:
bestprof = glob_pfds(cfg, my_pointing, bin_count, pfd_type=".bestprof")[0]
except IndexError as e:
raise IndexError(f"No bestprof files found in dir: {cfg['files']['psr_dir']} for pointing {my_pointing} and bin count {bin_count}")
def submit_slurm(name,
commands,
tmpl=SLURM_TMPL,
slurm_kwargs=None,
module_list=[],
vcstools_version="master",
batch_dir="batch/",
depend=None,
depend_type='afterok',
submit=True,
outfile=None,
queue="cpuq",
export="NONE",
gpu_res=None,
mem=1024,
cpu_threads=1,
temp_mem=None,
nice=0,
shebag='#!/bin/bash -l',
module_dir=None,
load_vcstools=True):
"""Making this function to cleanly submit SLURM jobs using a simple template.
Parameters
----------
name : `str`
The base name that is used to create the "`name`.batch" and "`name`.out" files.
commands : `list`
Each item in the list is a line of the bash script commands you want to run.
tmpl : `str`, optional
A template header string with format place holders: export, outfile,
cluster, header and script.
This is used to create the final string to be written to the job script.
For this function, it is required to be SLURM compliant.
|br| Default: `SLURM_TMPL`
slurm_kwargs : `dict`, optional
A dictionary of SLURM keyword, value pairs to fill in whatever is not
in the template supplied to `tmpl`.
|br| Default: `{}` (empty dictionary, i.e. no additional header parameters).
module_list : `list`, optional
A list of module names (including versions if applicable) that will
be included in the header for the batch
scripts. e.g. ["vcstools/master", "mwa-voltage/master", "presto/master"] would append
|br| module load vcstools/master
|br| module load mwa-voltage/master
|br| module load presto/master
|br| to the header of the batch script. This can also invoke "module use ..." commands.
NOTE: /group/mwa/software/modulefiles is used and vcstools/master is loaded by default.
vcstools_version : `str`, optional
The version of vcstools to load. |br| Default: master.
batch_dir : `str`, optional
The directory where you want to write the batch scripts
|br| Default: "batch/". (i.e. it will write to `$PWD/batch_dir`).
depend : `list`, optional
A list of the SLURM job IDs that your would like this job to depend on.
If `None` then it is assumed there is no dependency on any other job.
|br| Default: `None`.
depend_type : `str`, optional
The type of slurm dependancy required. For example if you wanted the
job to run after the jobs have been terminated use 'afterany'.
|br| Default: "afterok".
submit : `boolean`, optional
Whether to write and submit the job scripts (`True`) or only write the scripts (`False`).
|br| Default: `True`.
outfile : `str`, optional
The output file name if "`name`.out" is not desirable.
|br| Default: `None` (i.e. "`batch_dir`/`name`.out")
queue : `str`, optional
The type of queue you require (cpuq, gpuq or copyq) then the script will
choose the correct partitions and clusters for the job to run on
Default: "cpuq"
export : `str`, optional
Switch that lets SLURM use your login environment on the compute
nodes ("ALL") or not ("NONE").
|br| Default: "None".
gpu_res : `int`, optional
Number of GPUs that the SLURM job will reserve.
|br| Default: "None".
mem : `int`, optional
The MB of ram required for your slurm job.
|br| Default: 8192.
cpu_threads : `int`, optional
The number of CPU threads required for your slurm job.
|br| Default: 1.
Returns
-------
jobid : `int`
The unique SLURM job ID associated with the submitted job.
"""
if slurm_kwargs is None:
slurm_kwargs = {}
#Work out which partition and cluster to use based on the supercomputer
#(in config file) and queue required
comp_config = load_config_file()
if queue == 'cpuq':
cluster = comp_config['cpuq_cluster']
partition = comp_config['cpuq_partition']
elif queue == 'gpuq':
cluster = comp_config['gpuq_cluster']
partition = comp_config['gpuq_partition']
if gpu_res is None:
# No gpus reserved so change it to a default of 1
gpu_res = 1
elif queue == 'copyq':
cluster = comp_config['copyq_cluster']
partition = comp_config['copyq_partition']
elif queue == 'zcpuq':
# Download and checks should be done on Zeus's cpuq. This will only work
# on Galaxy as the Ozstar workflow is different
cluster = comp_config['zcpuq_cluster']
partition = comp_config['zcpuq_partition']
else:
logger.error("No queue found, please use cpuq, gpuq or copyq")
header = []
if batch_dir.endswith("/") is False:
batch_dir += "/"
# define file names (both the batch job file and the output file)
jobfile = batch_dir + name + ".batch"
if not outfile:
outfile = batch_dir + name + ".out"
# create the header from supplied arguments
for k, v in slurm_kwargs.items():
if len(k) > 1:
k = "--" + k + "="
else:
k = "-" + k + " "
header.append("#SBATCH {0}{1}".format(k, v))
# check if there are dependencies, and if so include that in the header
if depend is not None:
#assumes append is a list but if not will make an educated guess of how to reformat it
if isinstance(depend, int):
#assume it's ben given a single job id
header.append("#SBATCH --dependency={0}:{1}".format(
depend_type, depend))
if isinstance(depend, str):
if ":" in depend:
#assume it has been given an already formated string
if depend.startswith(":"):
depend = depend[1:]
#or a single jobid
header.append("#SBATCH --dependency={0}:{1}".format(
depend_type, depend))
if isinstance(depend, list):
depend_str = ""
for job_id in depend:
depend_str += ":" + str(job_id)
header.append("#SBATCH --dependency={0}{1}".format(
depend_type, depend_str))
# add a gpu res to header
if gpu_res is not None:
header.append('#SBATCH --gres=gpu:{0}'.format(gpu_res))
# add temp SSD memory to combat I/O issues. Only availble on Ozstar
hostname = socket.gethostname()
if temp_mem is not None:
header.append("#SBATCH --tmp={0}GB".format(temp_mem))
if module_dir is None:
module_dir = comp_config['module_dir']
# now join the header into one string
header = "\n".join(header)
# construct the module loads
if load_vcstools:
modules = ["module load vcstools/{0}\n".format(vcstools_version)]
else:
modules = []
switches = []
for m in module_list:
if m == "vcstools":
# don't do anything as vcstools is loaded automatically
continue
if "module switch" in m:
# if a module switch command is included rather than just a module name, then add it to a separate list
switches.append(m)
elif "module" in m:
modules.append("{0}\n".format(m))
else:
modules.append("module load {0}\n".format(m))
# join the module loads and switches into a single string
switches = "\n".join(switches)
modules = "\n".join(modules)
# join the commands into a single string
commands = "\n".join(commands)
# some little hacks to make jobs work on the shanghai server
if hostname.startswith('x86') or hostname.startswith('arm'):
if vcstools_version == 'master':
vcstools_version = 'cpu-master'
if export == "NONE":
export = "ALL"
if shebag == "#!/bin/bash -l":
shebag = "#!/bin/bash"
# format the template script
tmpl = tmpl.format(shebag=shebag,
script=commands,
outfile=outfile,
header=header,
switches=switches,
modules=modules,
cluster=cluster,
partition=partition,
export=export,
account=comp_config['group_account'][queue],
module_dir=module_dir,
threads=cpu_threads,
mem=mem,
nice=nice)
# write the formatted template to the job file for submission
with open(jobfile, "w") as fh:
fh.write(tmpl)
# submit the jobs
batch_submit_line = "sbatch {0}".format(jobfile)
jobid = None
if submit:
submit_cmd = subprocess.Popen(batch_submit_line,
shell=True,
stdout=subprocess.PIPE)
for line in submit_cmd.stdout:
if b"Submitted" in line:
jobid = str(line.split(b" ")[3].decode())
if jobid is None:
logger.debug(batch_submit_line)
logger.debug(submit_cmd.stdout)
return
else:
return jobid
else:
return
