def create_map_fits(num,configfile,cal_scan,output_dir,data_dir,crop_file,waveband):
'''Creates a calibrated, cropped map, with and without matched filtering, of a SCUBA-2 scan. The following data reduction
commands are performed in starlink:
(1) Checks if in the data_products directory there is an existing folder
for each calibrator scan number or 'default' in the case of auto calibration.
If not, it creates the folder
(2) Creates a list with the names of raw data by calling create_list function
(3) Runs smurf.makemap on the raw data
(4) Gets the FCF value for the given cal_scan number or uses default values in cause of auto calibration.
(5) Runs kappa.cmult to apply the FCF correction to the map, converting pW to
Jy/beam. The output is a new file with '_cal' at the end, indicating that
the FCF was applied.
(6) Runs picard.crop_scuba2_images on the output of step (5) to crop the map
(7) Converts the output of step 6 into fits files
INPUT: num - target scan number (str)
configfile - name of configuration file WITH full path (str)
cal_scan - calibration scan number or 'default' if auto calibration selected (str)
output_dir - path to directory where the data products are stored (str)
data_dir - path to directory where the raw data is stored (str)
crop_file - cropping parameter file (str)
waveband - '8' or '4' for 850um or 450um (str)
OUTPUT: Nothing returned, but function creates,
- Calibrated and cropped map files (sdf and fits)
- Calibrated and cropped map files with matched filter applied (sdf and fits)
'''
final_outputdir=output_dir+'/data_products/'
# Check if directory of cal_scan exists already, if not, create directory
if not os.path.isdir(output_dir+'calibrate_'+cal_scan):
os.mkdir(output_dir+'calibrate_'+cal_scan)
output_dir2 = output_dir+'calibrate_'+cal_scan+'/'
# Create list
create_list(num,data_dir,output_dir,cal_scan,waveband)
# create map
smurf.makemap(in_='^'+output_dir2+'mylist'+num+'_'+waveband+'.lst',out=output_dir2+'scan_'+num+'_'+waveband,config = '^'+configfile)
# get FCF value
if cal_scan=='default' and waveband=='8':
fcf_val= 537
elif cal_scan=='default' and waveband=='4':
fcf_val= 491
else:
fcf_val = FCF(output_dir2+'scan_'+cal_scan+'_'+waveband+'.sdf')
# run cmult
kappa.cmult(in_=output_dir2+'scan_'+num+'_'+waveband+'.sdf',scalar = fcf_val, out = output_dir2+'scan_'+num+'_'+waveband+'_cal')
# cropping map
cropped_map = picard.crop_scuba2_images([output_dir2+'scan_'+num+'_'+waveband+'_cal.sdf'],recpars = crop_file)
# move to Data products directory
os.system('mv '+cropped_map.datafiles[0]+' '+output_dir2)
#make a matched filter version
mf_map=picard.scuba2_matched_filter([output_dir2+'scan_'+num+'_'+waveband+'_cal_crop.sdf'])
os.system('mv '+[item for item in mf_map.datafiles if '_mf' in item][0]+' '+output_dir2+'scan_'+num+'_'+waveband+'_cal_crop_mf.sdf')
# create fits file
convert.ndf2fits(in_= output_dir2+'scan_'+num+'_'+waveband+'_cal_crop.sdf',out = output_dir2+'scan_'+num+'_'+waveband+'_cal_crop.fits')
convert.ndf2fits(in_= output_dir2+'scan_'+num+'_'+waveband+'_cal_crop_mf.sdf',out = output_dir2+'scan_'+num+'_'+waveband+'_cal_crop_mf.fits')
#copy all final processed maps to the data_products directory
if cal_scan=='default':
os.system('cp -r '+output_dir2+'scan_'+num+'_'+waveband+'_cal_crop.fits'+' '+final_outputdir)
os.system('cp -r '+output_dir2+'scan_'+num+'_'+waveband+'_cal_crop_mf.fits'+' '+final_outputdir)
def create_pol_timing(nums,data_dir,output_dir,date,configfile,waveband):
'''Creates calibrated, cropped I,Q,U timing cubes of a POL-2 scan. The following data reduction
commands are performed in starlink:
(1) Checks if in the data_products directory there is an existing folder
for each calibrator scan number or 'default' in the case of auto calibration.
If not, it creates the folder
(2) Creates a list with the names of raw data by calling create_list function
(3) Runs smurf.pol2map command on the raw polarization data to create a
total intensity cube in pW. It is run again, with the first I cube as input, to produce the
final I, Q and U cubes.
(3) Runs kappa.cmult to apply defualt FCFs to IQU cubes (individual scans only)
(4) IQU cubes are converted to fits files (individual scans only)
INPUT: nums - list of strings of target scan numbers
data_dir - path to directory where raw data is stored (str)
output_dir - path to directory where the data products are stored (str)
data - date of observation (YYYMMDD, str)
configfile - name of configuration file WITH full path (str)
crop_file - name of cropping parameter file WITH full path (str)
waveband - '8' or '4' for 850um or 450um (str)
OUTPUT: Nothing returned, but function creates,
- Calibrated I,Q,U timing cubes (individual scans only)
'''
final_outputdir=output_dir+'/data_products/'
if waveband=='8':
fcf_val=725
elif waveband=='4':
fcf_val=962
output_dir2 = output_dir+'calibrate_default/'
smurf.pol2map(in_ = '^'+output_dir2+'mylistall_'+waveband+'.lst', iout = output_dir2+'stokes_icube_'+waveband+'/'+'mosaic_Imap',qout='!',\
uout='!', mapdir = output_dir2+'stokes_icube_'+waveband, qudir = output_dir2+'stokes_qucube_'+waveband,jy=True, fcf='!',skyloop=False,\
config = '^'+configfile)
smurf.pol2map(in_=output_dir2+'stokes_qucube_'+waveband+'/*',iout = output_dir2+'stokes_icube_'+waveband+'/'+'mosaic_Imap_2',
qout=output_dir2+'stokes_qucube_'+waveband+'/'+'mosaic_Qmap', uout=output_dir2+'stokes_qucube_'+waveband+'/'+'mosaic_Umap',
mapdir=output_dir2+'stokes_icube_'+waveband+'/',mask=output_dir2+'stokes_icube_'+waveband+'/'+'mosaic_Imap',\
maskout1=output_dir2+'stokes_icube_'+waveband+'/'+'astmask',
maskout2=output_dir2+'stokes_icube_'+waveband+'/'+'pcamask',ipref=output_dir2+'stokes_icube_'+waveband+'/'+'mosaic_Imap_2',
cat=output_dir2+'stokes_qucube_'+waveband+'/'+'mycat',debias=True,jy=True, fcf='!',skyloop=False,\
config = '^'+configfile)
#calibrate and convert all output I,Q,U maps (only indv scans have cubes) to fits
for n in nums:
#need to cut name length down as starlink gives truncation errors for stackframes task!
for i in ['I','Q','U']:
os.system('mv '+output_dir2+'stokes_icube_'+waveband+'/'+date+'_000'+n+'*_'+i+'map.sdf'+' '+output_dir2+'stokes_icube_'+waveband+'/'+'scan_'+n+'_'+i+'map.sdf')
list_pol=[glob.glob(output_dir2+'stokes_icube_'+waveband+'/'+'scan_'+n+'_'+i+'map.sdf')[0] for i in ['I','Q','U']]
for item in list_pol:
smurf.stackframes(in_ = item.strip('.sdf')+'.more.smurf.shortmaps',out = item.strip('.sdf')+'_cube', sort = False, sortby = '')
kappa.cmult(in_=item.strip('.sdf')+'_cube.sdf',scalar = fcf_val, out = item.strip('.sdf')+'_cube_cal.sdf')
convert.ndf2fits(in_=item.strip('.sdf')+'_cube_cal.sdf',out = item.strip('.sdf')+'_cube_cal.fits')
#copy all final processed maps to the data_products directory
os.system('cp -r '+item.strip('.sdf')+'_cube_cal.fits '+final_outputdir)
def timing_cube(num,configfile,cal_scan,output_dir,waveband):
'''Creates a calibrated timing cube (RA,DEC,time) of a SCUBA-2 scan. The following data reduction
commands are performed in starlink:
(1) Checks if in the data_products directory there is an existing folder
for each calibrator scan number or 'default' in the case of auto calibration.
If not, it creates the folder
(2) Runs smurf.makemap on the raw data for the specified scan (the raw data list file was already
created in the full scan map function)
(3) Runs smurf.stackframes to create a cube
(4) Gets the FCF value for the given cal_scan number or uses default values in cause of auto calibration.
(5) Runs kappa.cmult to apply the FCF correction to the map, converting pW
to Jy/beam. The output is a new file with '_shortmp_cube_cal' at the
end,indicating that the FCF was applied.
(6) Converts the output of step 5 into fits files
INPUT: num - target scan number (str)
configfile - name of configuration file, WITH the full path (str)
cal_scan - calibration scan number or 'default' if auto calibration selected (str)
output_dir - path to directory where the data products are stored (str)
OUTPUT: Nothing returned, but function creates,
- Timing cube (RA,DEC,time; sdf and fits files)
'''
final_outputdir=output_dir+'/data_products/'
# Check if directory of cal_scan exists already, if not, create directory
if not os.path.isdir(output_dir+'calibrate_'+cal_scan):
os.mkdir(output_dir+'calibrate_'+cal_scan)
output_dir2 = output_dir+'calibrate_'+cal_scan+'/'
# making timing cube
smurf.makemap(in_='^'+output_dir2+'mylist'+num+'_'+waveband+'.lst',out=output_dir2+'scan_'+num+'_'+waveband+'_shortmp',config = '^'+configfile)
# stacking frames
smurf.stackframes(in_ = output_dir2+'scan_'+num+'_'+waveband+'_shortmp.more.smurf.shortmaps', out = output_dir2+'scan_'+num+'_'+waveband+'_shortmp_cube', sort = False, sortby = '')
# get FCF val
if cal_scan=='default' and waveband=='8':
fcf_val= 537
elif cal_scan=='default' and waveband=='4':
fcf_val= 491
else:
fcf_val = FCF(output_dir2+'scan_'+cal_scan+'_'+waveband+'.sdf')
# run cmult
kappa.cmult(in_=output_dir2+'scan_'+num+'_'+waveband+'_shortmp_cube.sdf',scalar = fcf_val, out = output_dir2+'scan_'+num+'_'+waveband+'_shortmp_cube_cal')
# create .fits file
convert.ndf2fits(in_=output_dir2+'scan_'+num+'_'+waveband+'_shortmp_cube_cal.sdf',out = output_dir2+'scan_'+num+'_'+waveband+'_shortmp_cube_cal.fits')
#copy all final processed maps to the data_products directory
if cal_scan=='default':
os.system('cp -r '+ output_dir2+'scan_'+num+'_'+waveband+'_shortmp_cube_cal.fits '+final_outputdir)
file1.close()
# Step 2
smurf.makemap(in_='^' + output_dir + 'mylist37.lst',
out=output_dir + 'calibrator_scan_37',
config='^' + data_dir + 'dimmconfig_bright_compact.lis')
# Step 4
result_FCF = picard.scuba2_check_cal([output_dir + 'calibrator_scan_8.sdf'])
fcffile = result_FCF.logfiles[2]
FCF8 = float(ascii.read(fcffile)['col12']) # gets the FCF value
# Step 5
kappa.cmult(in_=output_dir + 'calibrator_scan_37.sdf',
scalar=FCF8,
out=output_dir + 'calibrator_scan_37_cal')
'''
Cropping target maps
'''
cropped_map = picard.crop_scuba2_images(
[output_dir + 'calibrator_scan_37_cal.sdf'],
recpars=output_dir + 'crop_parameters.lis')
os.system('rm -rf ' + output_dir + cropped_map.datafiles[0])
os.system('mv ' + cropped_map.datafiles[0] + ' ' + output_dir)
# Step 6
convert.ndf2fits(in_=output_dir + 'calibrator_scan_37_cal.sdf',
out=output_dir + 'calibrator_scan_37_cal.fits')
def create_pol_map(nums,data_dir,output_dir,date,configfile,crop_file,waveband):
'''Creates calibrated, cropped stokes cubes of a POL-2 scan. The following data reduction
commands are performed in starlink:
(1) Checks if in the data_products directory there is an existing folder
for each calibrator scan number or 'default' in the case of auto calibration.
If not, it creates the folder
(2) Creates a list with the names of raw data by calling create_list function
(3) Runs smurf.pol2map command on the raw polarization data to create a
total intensity map in pW. It is run again, with the first I map as input, to produce the
final I, Q and U maps, and the vector catalogue.
(3) Runs kappa.cmult to apply defualt FCFs to IQU maps (individual scans and mosaics)
(4) Runs picard.crop_scuba2_images to crop IQU maps (individual scans and mosaics)
(5) IQU maps are converted to fits files, and a stokes cube is built (individual scans and mosaics)
INPUT: nums - list of strings of target scan numbers
data_dir - path to directory where raw data is stored (str)
output_dir - path to directory where the data products are stored (str)
date - date of observation (YYYMMDD, str)
configfile - name of configuration file WITH full path (str)
crop_file - name of cropping parameter file WITH full path (str)
waveband - '8' or '4' for 850um or 450um (str)
OUTPUT: Nothing returned, but function creates,
- Calibrated, cropped stokes cubes (individual scans and mosaics with and without matched filters; fits files)
'''
final_outputdir=output_dir+'/data_products/'
if waveband=='8':
fcf_val=725
elif waveband=='4':
fcf_val=962
if not os.path.isdir(output_dir+'calibrate_default'):
os.mkdir(output_dir+'calibrate_default')
output_dir2 = output_dir+'calibrate_default/'
create_list(nums,data_dir,output_dir,'default',waveband)
smurf.pol2map(in_ = '^'+output_dir2+'mylistall_'+waveband+'.lst', iout = output_dir2+'stokes_i_'+waveband+'/'+'mosaic_Imap',qout='!',\
uout='!', mapdir = output_dir2+'stokes_i_'+waveband, qudir = output_dir2+'stokes_qu_'+waveband,jy=True, fcf='!',skyloop=False,\
config = '^'+configfile)
smurf.pol2map(in_=output_dir2+'stokes_qu_'+waveband+'/*',iout = output_dir2+'stokes_i_'+waveband+'/'+'mosaic_Imap_2',
qout=output_dir2+'stokes_qu_'+waveband+'/'+'mosaic_Qmap', uout=output_dir2+'stokes_qu_'+waveband+'/'+'mosaic_Umap',
mapdir=output_dir2+'stokes_i_'+waveband+'/',mask=output_dir2+'stokes_i_'+waveband+'/'+'mosaic_Imap',\
maskout1=output_dir2+'stokes_i_'+waveband+'/'+'astmask',
maskout2=output_dir2+'stokes_i_'+waveband+'/'+'pcamask',ipref=output_dir2+'stokes_i_'+waveband+'/'+'mosaic_Imap_2',
cat=output_dir2+'stokes_qu_'+waveband+'/'+'mycat',debias=True,jy=True, fcf='!',skyloop=False,\
config = '^'+configfile)
#calibrate,crop and convert all output I,Q,U maps (indv scans and mosaics, both with matched filter versions) to fits
for n in nums:
list_pol=[glob.glob(output_dir2+'stokes_i_'+waveband+'/'+date+'_000'+n+'*_'+i+'map.sdf')[0] for i in ['I','Q','U']]
for item in list_pol:
kappa.cmult(in_=item,scalar = fcf_val, out = item.strip('.sdf')+'_cal.sdf')
cropped_map = picard.crop_scuba2_images([item.strip('.sdf')+'_cal.sdf'],recpars = crop_file)
os.system('mv '+cropped_map.datafiles[0]+' '+output_dir2+'stokes_i_'+waveband)
mf_map=picard.scuba2_matched_filter([item.strip('.sdf')+'_cal_crop.sdf'])
os.system('mv '+[i for i in mf_map.datafiles if '_mf' in i][0]+' '+item.strip('.sdf')+'_cal_crop_mf.sdf')
convert.ndf2fits(in_=item.strip('.sdf')+'_cal_crop.sdf',out = item.strip('.sdf')+'_cal_crop.fits')
convert.ndf2fits(in_=item.strip('.sdf')+'_cal_crop_mf.sdf',out = item.strip('.sdf')+'_cal_crop_mf.fits')
lpf=[glob.glob(output_dir2+'stokes_i_'+waveband+'/'+date+'_000'+n+'*_'+i+'map_cal_crop.fits')[0] for i in ['I','Q','U']]
stokes_cube=create_stokes_cubes(lpf,output_dir2,waveband,date,n,'n','n')
lpf_mf=[glob.glob(output_dir2+'stokes_i_'+waveband+'/'+date+'_000'+n+'*_'+i+'map_cal_crop_mf.fits')[0] for i in ['I','Q','U']]
stokes_cube_mf=create_stokes_cubes(lpf_mf,output_dir2,waveband,date,n,'n','y')
#copy all final processed maps to the data_products directory
os.system('cp -r '+stokes_cube+' '+final_outputdir)
os.system('cp -r '+stokes_cube_mf+' '+final_outputdir)
list_pol_mosaic=glob.glob(output_dir2+'stokes_*'+waveband+'/mosaic_*map.sdf')
for item in list_pol_mosaic:
kappa.cmult(in_=item,scalar = fcf_val, out = item.strip('.sdf')+'_cal.sdf')
cropped_map = picard.crop_scuba2_images([item.strip('.sdf')+'_cal.sdf'],recpars = crop_file)
os.system('mv '+cropped_map.datafiles[0]+' '+output_dir2+item.split('/')[-2])
mf_map=picard.scuba2_matched_filter([item.strip('.sdf')+'_cal_crop.sdf'])
os.system('mv '+[i for i in mf_map.datafiles if '_mf' in i][0]+' '+item.strip('.sdf')+'_cal_crop_mf.sdf')
convert.ndf2fits(in_=item.strip('.sdf')+'_cal_crop.sdf',out = item.strip('.sdf')+'_cal_crop.fits')
convert.ndf2fits(in_=item.strip('.sdf')+'_cal_crop_mf.sdf',out = item.strip('.sdf')+'_cal_crop_mf.fits')
lpfmos=[glob.glob(output_dir2+'stokes_*'+waveband+'/mosaic_'+str(i)+'map_cal_crop.fits')[0] for i in ['I','Q','U']]
stokes_cube_mosaic=create_stokes_cubes(lpfmos,output_dir2,waveband,date,'','y','n')
lpfmos_mf=[glob.glob(output_dir2+'stokes_*'+waveband+'/mosaic_'+str(i)+'map_cal_crop_mf.fits')[0] for i in ['I','Q','U']]
stokes_cube_mosaic_mf=create_stokes_cubes(lpfmos_mf,output_dir2,waveband,date,'','y','y')
#copy all final processed maps to the data_products directory
os.system('cp -r '+stokes_cube_mosaic+' '+final_outputdir)
os.system('cp -r '+stokes_cube_mosaic_mf+' '+final_outputdir)
print('\nMaking the ACcal files - 850 microns....')
# Apply the relative FCF correction using kappa
apply_relFCF_AC("tables/Transient_"+eachregion+"_run_"+str(alignment_iteration)+"_"+wave+".table",wave)
print('\nConverting the units to mJy/beam....')
newCR3files = sorted(list(glob.glob('*'+wave+'*CR3.sdf')))
#Change to mJy/beam!
for eachCR3file in newCR3files:
firstconv = mJy_arcsec2_850_FCF
convfactor = mJy_beam_850_FCF
pwfilename = eachCR3file.split('.sdf')[0]+'_pw.sdf'
kappa.cdiv(eachCR3file,firstconv,pwfilename)
mJybmfilename = eachCR3file.split('.sdf')[0]+'_mJybm.sdf'
kappa.cmult(pwfilename,convfactor,mJybmfilename)
kappa.setunits(mJybmfilename,'mJy/beam')
os.system('mv -f '+mJybmfilename+' '+eachCR3file)
print('\nCleaning Up....')
# Move the newly aligned and flux calibrated files to their own directory and cleanup
os.system('mv '+eachregion+'*_CR3.sdf '+eachregion+'*CR3_ACcal.sdf '+new_direc_for_corrected_data)
os.system('rm -f *pcor.txt makemaps.sh *_'+wave+'.txt')
######################################
######################################
# 450 Micron Cross/Auto-Correlation Methods
######################################
######################################
######################################