def generate_script(project_dict, OT_dict, comments=False, split=True, template=False):
"""Generates a custom imaging script. Only used for imaging."""
os.chdir(project_dict['project_path'])
parameters = sg.get_parameters(project_dict = project_dict, OT_dict = OT_dict)
script = sg.script_data_prep(parameters, project_dict, comments, template)
if split == True:
prep_script = script
script = ''
spws = sg.sort_spws(parameters)
continfo = spws[0]
lineinfo = spws[1]
linespws = spws[2]
script = sg.make_continuum(script,parameters, project_dict, continfo, comments, template,flagchannels=False)
script = sg.image_setup(script,parameters, comments, template)
script = sg.cont_image(script,parameters, comments, template)
if lineinfo != []:
script = sg.contsub(script,parameters, continfo, linespws, comments)
script = sg.line_image(script,parameters, lineinfo, comments)
script = sg.pbcor_fits(script)
if split == True:
sg.write_script(prep_script,project_dict, template, filename = 'scriptForImaging_Prep.py')
sg.write_script(script,project_dict, template, filename = 'scriptForImaging.py')
#sg.write_script(script,project_dict, filename = 'scriptForImaging.py')
# cleanup temp files
OT_info.cleanup(parameters['AOT'])
def main(project_dict=False, AOT=False):
"""Asks the user for project information or reads the information from the provided dictionaries to write to the README"""
if project_dict == False:
project_dict = project_info.main()
if AOT == False:
AOT_dict = OT_info.getOTinfo(project_dict['SB_name'], AOTpath=False)
else:
AOT_dict = OT_info.getOTinfo(project_dict['SB_name'],AOTpath=AOT)
AOTdir = AOT_dict[0]
readme_info = getInfo(project_dict,AOT_dict)
write2Readme(project_dict['project_type'], project_dict['project_path'], readme_info, project_dict['casa_version']) # why won't this write??
#fill_README.testwrite()
cleanup(AOTdir['AOT'])
def generate(SB_name, project_path = False, comments = True, template=False, split=True): #template to get template, split for generating 2 scripts instead of 1
"""Main method. Options are: project_path ='directory_where_project_lives', comments=True/False ie verbose or non-verbose, template = True/False, split = True/False ie two scripts or one"""
if (SB_name == 'Template' or SB_name == 'template'):
template = True
if project_path == False:
project_path = os.getcwd()
if template == True:
parameters = template_parameters()
project_dict = {'project_path': project_path, 'project_type': 'Imaging', 'casa_version': '4.4.0'}
flagchannels = '2:1201~2199,3:1201~2199' #whatever is in the example
print 'this option not completely ready yet!'
else:
dictionaries = pi.most_info(SB_name, project_path=project_path)
project_dict = dictionaries[0]
OT_dict = dictionaries[1]
os.chdir(project_path)
flagchannels=False
parameters = get_parameters(project_dict = project_dict, OT_dict = OT_dict) #skip up to here
#print parameters['vwidth']
script = script_data_prep(parameters, project_dict, comments, template)
if split == True:
prep_script = script
script = ''
#print prep_script
spws = sort_spws(parameters) #skip this?
continfo = spws[0] #and this
lineinfo = spws[1] #and this
linespws = spws[2] # and this ... but check first
script = make_continuum(script,parameters, project_dict, continfo, comments, template, flagchannels)
script = image_setup(script,parameters, comments, template)
script = cont_image(script,parameters, comments, template)
if lineinfo != []:
script = contsub(script,parameters, continfo, linespws, comments)
script = line_image(script,parameters, lineinfo, comments)
print parameters['vwidth']
script = pbcor_fits(script)
if split == True:
write_script(prep_script,project_dict, template, filename = 'scriptForImaging_Prep.py')
write_script(script,project_dict, template, filename = 'scriptForImaging.py')
else:
write_script(script,project_dict, template) #, filename = 'scriptForImaging_test.py') #diff filename for template
# cleanup temp files
if template == False:
OT_info.cleanup(parameters['AOT']) # don't need this for template
def fillReadme(project_dict, AOT=False):
"""Fills out the header information in the README"""
#print AOT, project_dict['SB_name']
AOT_dict = OT_info.getOTinfo(project_dict['SB_name'],AOTpath=AOT)
AOTdir = AOT_dict[0]
readme_info = fill_README.getInfo(project_dict,AOT_dict)
fill_README.write2Readme(project_dict['project_type'], project_dict['project_path'], readme_info, project_dict['casa_version']) # why won't this write??
#fill_README.testwrite()
#fill_README.cleanup(AOTdir['AOT'])
return AOT_dict
def generate(SB_name, project_path = False, comments = True):
if project_path == False:
project_path = os.getcwd()
dictionaries = pi.most_info(SB_name, project_path=project_path)
project_dict = dictionaries[0]
OT_dict = dictionaries[1]
os.chdir(project_path)
parameters = get_parameters(project_dict = project_dict, OT_dict = OT_dict)
script = script_data_prep(parameters, project_dict, comments)
spws = sort_spws(parameters)
continfo = spws[0]
lineinfo = spws[1]
linespws = spws[2]
script = make_continuum(script,parameters, project_dict, continfo, comments, flagchannels=False)
script = image_setup(script,parameters, comments)
script = cont_image(script,parameters, comments)
if lineinfo != []:
script = contsub(script,parameters, continfo, linespws, comments)
script = line_image(script,parameters, lineinfo, comments)
script = pbcor_fits(script)
write_script(script,project_dict, filename = 'scriptForImaging_test.py')
# cleanup temp files
OT_info.cleanup(parameters['AOT'])
def most_info(SBname=False, project_path = False): # project_path is the /lustre/naasc/username/Reduce_XXX directory
import OT_info as OT
import glob
import os
import sys
#proj_type = project_type()
if project_path == False:
project_path = directory()
if SBname == False:
SBname = SB_name()
# figure out the project type:
os.chdir(project_path)
#print project_path
folders = glob.glob('*')
if 'sg_ouss_id' in folders:
proj_type = "Imaging"
version = casa_version()
elif 'Imaging' in folders:
proj_type = "Manual"
version = '4.4.0'
elif 'S.MOUS.uid' in folders[0]: # in case people still do that
proj_type = "Imaging"
version = casa_version()
project_path = project_path + folders[0]
else:
sys.exit('ERROR: Wrong directory or wrong structure in directory! Please make sure folder names comply with NA arc best practices for imaging or manual calibration.')
os.chdir(project_path)
AOT = glob.glob('*.aot')
if not AOT:
AOTpath = False
else:
AOTpath = os.getcwd()+'/' + AOT[0]
print AOTpath
OT_dict = OT.getOTinfo(SBname, AOTpath=AOTpath)
AOTfile = OT_dict[0]['AOT']
project_number = AOTfile[AOTfile.rfind('/')+1:len(AOTfile)]
project_number = project_number[0:14]
codes = ['N/A', 'N/A']
asdm = ['N/A']
project_dict = project_info_dict(proj_type, project_path, project_number, codes, SBname, asdm, version)
#print project_dict, OT_dict
return project_dict, OT_dict
def get_parameters(project_dict=False, OT_dict=False):
"""gets the parameter dictionary from list_imparameters_CASA.py """
if project_dict == False:
project_dict = pi.main() #will query the user for project details. Should never happen when running the generate() method, this is mainly for testing purposes
if OT_dict == False:
OT_dict = OT_info.getOTinfo(project_dict['SB_name'])
XMLroots=OT_dict[1]
science_root = XMLroots['science_root']
namespaces = XMLroots['namespaces']
nms = li.getnum_ms(project_dict['project_type'], project_dict['project_path'])
listobs = li.getListobs()
line = listobs[0]
index = listobs[1]
nspws = li.getNspw(line, index)
cell = li.getCell(line)
scifld = li.getScienceFields(line, index)
#specinfo = li.getLines(index, int(nspws[0]))
spwdict = li.getSpwInfo(science_root, namespaces, nspws[1])
rfreqHz = li.getRestFreq(science_root, namespaces)
vwidth = li.getVelWidth(science_root, namespaces, rfreqHz=rfreqHz)
rframe = li.getRefFrame(science_root, namespaces)
sg = OT_dict[0]
AOT = OT_dict[0]['AOT']
sName = li.sourceName(science_root, namespaces, project_dict['SB_name'])
mosaic = li.mosaicBool(namespaces, XMLroots['science_root'], sName)
if mosaic == 'true':
pc = li.getPhasecenter(science_root, namespaces)
else:
pc = 'N/A'
# fill dictionary
pc = li.getPhasecenter(science_root, namespaces)
lastfield = scifld[scifld.rfind(',')+1:]
parameters = {'project_number': project_dict['project_number'],'SB_name': project_dict['SB_name'],'nms':nms,'mosaic': mosaic, 'scifields': scifld,'scifield0': str(scifld[0]), 'scifield1': str(lastfield), 'cellsize': cell[0], 'imsize':cell[1], 'rframe':rframe, 'vwidth':vwidth[0], 'rwidth': vwidth[1], 'rwidthunit': vwidth[2], 'spw_dict': spwdict, 'rfreq':str(float(rfreqHz)*1e-9), 'plotcmd': '', 'sourceName': sName, 'phasecenter': pc, 'AOT': AOT}
print parameters
return parameters
def get_parameters(project_dict=False, OT_dict=False):
if project_dict == False:
project_dict = pi.main()
if OT_dict == False:
OT_dict = OT_info.getOTinfo(project_dict['SB_name'])
XMLroots=OT_dict[1]
science_root = XMLroots['science_root']
namespaces = XMLroots['namespaces']
nms = li.getnum_ms(project_dict['project_type'], project_dict['project_path'])
listobs = li.getListobs()
line = listobs[0]
index = listobs[1]
nspws = li.getNspw(line, index)
cell = li.getCell(line)
scifld = li.getScienceFields(line, index)
specinfo = li.getLines(index, int(nspws[0]))
spwdict = li.getSpwInfo(science_root, namespaces, nspws[1])
rfreqHz = li.getRestFreq(science_root, namespaces)
vwidth = li.getVelWidth(science_root, namespaces, rfreqHz=rfreqHz)
rframe = li.getRefFrame(science_root, namespaces)
sg = OT_dict[0]
AOT = OT_dict[0]['AOT']
sName = li.sourceName(science_root, namespaces, project_dict['SB_name'])
#mosaic = li.mosaicBool(namespaces, XMLroots['proj_root'], sg['science_goal'])
mosaic = li.mosaicBool(namespaces, XMLroots['science_root'], sName)
if mosaic == 'true':
pc = li.getPhasecenter(science_root, namespaces)
else:
pc = 'N/A'
# fill dictionary
pc = li.getPhasecenter(science_root, namespaces)
lastfield = scifld[scifld.rfind(' ')-1:]
parameters = {'project_number': project_dict['project_number'],'SB_name': project_dict['SB_name'],'nms':nms, 'specinfo':specinfo,'mosaic': mosaic, 'scifields': scifld,'scifield0': scifld[0], 'scifield1': lastfield, 'cellsize': cell[0], 'imsize':cell[1], 'rframe':rframe, 'vwidth':vwidth[0], 'rwidth': vwidth[1], 'rwidthunit': vwidth[2], 'spw_dict': spwdict, 'rfreq':str(float(rfreqHz)*1e-9), 'plotcmd': '', 'sourceName': sName, 'phasecenter': pc, 'AOT': AOT}
return parameters
def main(lbc=False): # will ask for the OT file twice ... fix that
"""For imaging:
Asks the user to enter information about the project, downloads and untars data from pipetemp, runs ASDMExportLight and the pipeline, applies fixes to the piperestore script, copies the flagtemplate files over, gets the imaging script, untars the weblog, fills the readme, and generates the file imaging_parameters.txt
For manual reduction:
Creates the directory tree, runs ASDMExportLight and generates calibration script, fills the readme, asks the user whether or not to run the script through once and generate weblog plots in a pdf
"""
project_dict = pi.main()
OTfile = OT_info.getAOT(project_number=project_dict['project_number'])
makeDirectories(project_dict)
if project_dict['project_type'] == 'Imaging':
untarStuff(project_dict)
os.chdir('%s/sg_ouss_id/group_ouss_id/member_ouss_id/' % project_dict['project_path'])
project_dict['casa_version'] = pi.casa_version()
if project_dict['casa_version'] != '4.3.1':
pipelineChanges()
downloadASDM(project_dict,lbc=lbc)
'''
#input array is: ['project_type','asdm','lbc']
input_array = []
for asdm in project_dict['asdm']:
input_array.append([project_dict['project_type'], asdm, lbc])
pool_size = multiprocessing.cpu_count() * 4
pool = multiprocessing.Pool(processes=pool_size, initializer=start_process,)
pool_outputs= pool.map(downloadASDM_parallel, input_array)
'''
copyFiles(project_dict) #don't actually do this yet
#project_dict['casa_version'] = pi.casa_version()
runPipeline(project_dict)
#getScript(project_dict) #don't need this if we're generating the script
untarWeblog()
OT_dict = fillReadme(project_dict, AOT=OTfile)
#OT_dict = OT_info.getOTinfo(project_dict['SB_name'], AOTpath = OTfile) # just fill the dictionary instead....
#li.main(project_dict=project_dict, OT_dict = OT_dict)
generate_script(project_dict, OT_dict)
else :
os.chdir('%s' % project_dict['project_path'])
script_name = downloadASDM(project_dict)
'''
pool_size = multiprocessing.cpu_count() * 4
pool = multiprocessing.Pool(processes=pool_size, initializer=start_process,)
pool_outputs= pool.map(downloadASDM_parallel, project_dict[asdm], lbc=lbc)
script_name=pool_outputs
'''
#getScript(project_dict)
OT_dict = fillReadme(project_dict, AOT=OTfile)
'''
for asdm in project_dict['asdm']:
os.chdir('%s/Calibration_%s' % (project_dict['project_path'],str(asdm[asdm.rfind('X'):]).lstrip()))
ms_name = glob.glob('*.ms')
snr = subprocess.Popen(["casa", "-c", "au.gaincalSNR(\'%s\')" % ms_name[0]], shell=False,stdout=subprocess.PIPE, stderr=subprocess.STDOUT) #this doesn't work yet for some reason...
snr.wait()
print 'The results of running au.gaincalSNR are in %s/gaincalSNR.txt' % os.getcwd()
gaincal = snr.communicate()
if os.path.isfile('gaincalSNR.txt') == False:
os.mknod('gaincalSNR.txt')
text = open('gaincalSNR.txt','w')
text.writelines(gaincal[0])
text.close()
'''
#runscript = raw_input('Run calibration script through and check the visibilities by generating a pdf of the weblog plots? Warning - this will take a while and do it for all the asdms!')
#os.chdir('../')
#if runscript == 'Y':
# for asdm in project_dict['asdm']:
# os.chdir('%s/Calibration_%s' % (project_dict['project_path'],str(asdm[-4:])))
# script_name = glob.glob('*.py')
# script = open(script_name, 'rw+') # insert line mysteps = [] at beginning of script...
# steps = script.readlines()
# steps[1] = 'mysteps = []\n'
# script.writelines(steps)
# script.close()
# subprocess.call(["casa", "-c", "'%s'" % script_name[0]])
# script = open(script_name, 'rw+') # remove line mysteps = []
# steps = script.readlines()
# steps[1] = '\n'
# script.writelines(steps)
# script.close()
# subprocess.call(["casa", "-c", "/lustre/naasc/elastufk/Python/checkvispdf.py"]) # does this need the path?
#os.chdir('../')
print 'Make sure to run the script generator once your calibrated or combined ms is in the Imaging/ directory!'
