def optimize_uvmin(rootname, mslist, colname, uvmin_limit=None):
uvminfile = rootname + '_uvmin.txt'
report('Optimizing uvmin for self-cal')
if os.path.isfile(uvminfile):
result = float(open(uvminfile).readlines()[0].rstrip())
else:
level = sumdico(rootname)
result = find_uvmin(mslist, level, colname=colname) * 1.1
print 'Will use shortest baseline of', result, 'km'
with open(uvminfile, 'w') as f:
f.write('%f\n' % result)
if uvmin_limit is not None and result < uvmin_limit:
result = uvmin_limit
return result
def do_run_job(name, basedir, qsubfile=None, do_field=True, prefix='ddfp'):
config = ''
workdir = basedir + '/' + name
g = glob.glob(workdir + '/tier1*.cfg')
if len(g) > 0:
print 'Local config file exists, using that'
config = ',CONFIG=' + g[0]
if qsubfile is None:
qsubfile = '/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
report('Submit job')
os.system('qsub -N ' + prefix + '-' + name + ' -v WD=' + workdir + config +
' ' + qsubfile)
if do_field:
update_status(name, 'Queued', workdir=workdir)
def make_external_mask(fname,
templatename,
use_tgss=True,
options=None,
extended_use=None,
clobber=False):
if options is None:
options = o # attempt to get global
if options['restart'] and os.path.isfile(fname) and not clobber:
warn('External mask already exists, not creating it')
else:
report('Make blank external mask')
hdus = fits.open(templatename)
hdus[0].data = np.zeros_like(hdus[0].data, dtype=np.int32)
hdus.writeto(fname, clobber=True)
hdus.close()
if use_tgss and options['tgss'] is not None:
report('Merging the mask with TGSS catalogue')
# TGSS path is provided, this means we want to add the positions of bright TGSS sources to the mask
modify_mask(fname,
fname,
options['tgss'],
options['tgss_radius'],
options['tgss_flux'],
do_extended=options['tgss_extended'],
cellsize=options['cellsize'],
pointsize=options['tgss_pointlike'])
if options['region'] is not None:
report('Merging with mask with user-specified region')
add_manual_mask(fname, options['region'], fname)
if options['extended_size'] is not None and extended_use is not None:
report('Merging with automatic extended mask')
merge_mask(fname, extended_use, fname)
def do_run_job(name,basedir,qsubfile=None,do_field=True,prefix='ddfp',dysco=False):
config=''
workdir=basedir+'/'+name
g=glob.glob(workdir+'/tier1*.cfg')
if len(g)>0:
print 'Local config file exists, using that'
config=',CONFIG='+g[0]
if qsubfile is None:
qsubfile='/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
if dysco:
qsubfile=qsubfile.replace('.qsub','-fdr14.qsub')
report('Submit job')
os.system('qsub -N '+prefix+'-'+name+' -v WD='+workdir+config+' '+qsubfile)
if do_field:
update_status(name,'Queued',workdir=workdir)
def clearcache(mslist, options):
cachedir = find_cache_dir(options)
report('Clearing cache for ' + mslist)
if os.path.isfile(mslist):
filenames = [l.strip() for l in open(mslist, 'r').readlines()]
else:
filenames = []
try:
rmtglob(cachedir + '/' + mslist + '*.ddfcache')
rmtglob(mslist + '*.ddfcache')
except OSError:
pass
for f in filenames:
try:
rmtglob(cachedir + '/' + f + '*.ddfcache')
except OSError:
pass
def check_imaging_weight(mslist_name):
# returns a boolean that says whether it did something
result = False
report('Checking for IMAGING_WEIGHT in input MSS')
mslist = [s.strip() for s in open(mslist_name).readlines()]
for ms in mslist:
t = pt.table(ms)
try:
dummy = t.getcoldesc('IMAGING_WEIGHT')
except RuntimeError:
dummy = None
t.close()
if dummy is not None:
warn('Table ' + ms + ' already has imaging weights')
else:
pt.addImagingColumns(ms)
result = True
return result
def run_reprocess(wd=None):
# by default assume we're in the working directory at this point
if wd is not None:
os.chdir(wd)
update_status(None,'Running')
solsfile = glob.glob('DDS3_full*smoothed.npz')
if len(solsfile) < 1:
die('Cannot find the correct solution file -- exiting')
solsfile = str(solsfile[0])
o = options('reprocess-vlow.cfg',option_list)
cubefiles=['image_full_vlow_QU.cube.dirty.fits','image_full_vlow_QU.cube.dirty.corr.fits']
cthreads=[]
flist=[]
ddf_kw = {}
do_polcubes('DATA','[DDS3_full_smoothed,DDS3_full_slow]',[o['image_uvmin'],1.600000],'image_full_vlow',ddf_kw,beamsize=o['vlow_psf_arcsec'],imsize=o['vlow_imsize'],cellsize=o['vlow_cell'],robust=o['vlow_robust'],options=o,catcher=None)
if o['compress_polcubes']:
for cubefile in cubefiles:
if o['restart'] and os.path.isfile(cubefile+'.fz'):
warn('Compressed cube file '+cubefile+'.fz already exists, not starting compression thread')
else:
report('Starting compression thread for '+cubefile)
thread = threading.Thread(target=compress_fits, args=(cubefile,o['fpack_q']))
thread.start()
cthreads.append(thread)
flist.append(cubefile)
if o['compress_polcubes']:
# cthreads and flist exist
for thread in cthreads:
if thread.isAlive():
warn('Waiting for a compression thread to finish')
thread.join()
if o['delete_compressed']:
for f in flist:
warn('Deleting compressed file %s' % f)
os.remove(f)
update_status(None,'Complete')
# matching with catalogs
removelist=[]
for cat in o['list']:
print 'Doing catalogue',cat
if crossmatch_image(o['catprefix'] + '.cat.fits',cat,catdir=o['catdir'])>10:
filter_catalog(o['catprefix'] + '.cat.fits',o['catprefix']+'.cat.fits_'+cat+'_match.fits',o['pbimage'],o['catprefix']+'.cat.fits_'+cat+'_match_filtered.fits',cat,options=o)
else:
print 'Insufficient matches, abandoning catalogue'
removelist.append(cat)
for cat in removelist:
o['list'].remove(cat)
# Astrometric plots
if 'FIRST' in o['list']:
report('Plotting position offsets')
plot_position_offset('%s.cat.fits_FIRST_match_filtered.fits'%o['catprefix'],o['pbimage'],'%s.cat.fits_FIRST_match_filtered_positions.png'%o['catprefix'],'FIRST',options=o)
t=Table.read(o['catprefix']+'.cat.fits_FIRST_match_filtered.fits')
bsra=np.percentile(bootstrap(t['FIRST_dRA'],np.mean,10000),(16,84))
bsdec=np.percentile(bootstrap(t['FIRST_dDEC'],np.mean,10000),(16,84))
mdra=np.mean(t['FIRST_dRA'])
mddec=np.mean(t['FIRST_dDEC'])
print 'Mean delta RA is %.3f arcsec (1-sigma %.3f -- %.3f arcsec)' % (mdra,bsra[0],bsra[1])
print 'Mean delta DEC is %.3f arcsec (1-sigma %.3f -- %.3f arcsec)' % (mddec,bsdec[0],bsdec[1])
first_ra=mdra
first_dec=mddec
report('Plotting per-facet position offsets')
do_plot_facet_offsets(t,tesselfile,o['catprefix']+'.cat.fits_FIRST_match_filtered_offsets.png')
t['FIRST_dRA']-=mdra
def main(msin, config_path):
o = options(config_path, option_list)
if o['pbimage'] is None:
die('pbimage must be specified')
# fix up the new list-type options
for i, cat in enumerate(o['list']):
try:
o[cat] = o['filenames'][i]
except:
pass
try:
o[cat + '_matchrad'] = o['radii'][i]
except:
pass
try:
o[cat + '_fluxfactor'] = o['fluxfactor'][i]
except:
pass
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# pybdsm source finding
sfind_image(o['catprefix'], o['pbimage'], o['sfind_pixel_fraction'])
# matching with catalogs
for cat in o['list']:
print 'Doing catalogue', cat
crossmatch_image(o['catprefix'] + '.cat.fits', cat)
filter_catalog(o['catprefix'] + '.cat.fits',
o['catprefix'] + '.cat.fits_' + cat + '_match.fits',
o['pbimage'],
o['catprefix'] + '.cat.fits_' + cat +
'_match_filtered.fits',
cat,
options=o)
# Filter catalogs (only keep isolated compact sources within 3deg of pointing centre)
# Astrometric plots
if 'FIRST' in o['list']:
report('Plotting position offsets')
plot_position_offset(
'%s.cat.fits_FIRST_match_filtered.fits' % o['catprefix'],
o['pbimage'],
'%s.cat.fits_FIRST_match_filtered_positions.png' % o['catprefix'],
'FIRST',
options=o)
t = Table.read(o['catprefix'] + '.cat.fits_FIRST_match_filtered.fits')
bsra = np.percentile(bootstrap(t['FIRST_dRA'], np.mean, 10000),
(16, 84))
bsdec = np.percentile(bootstrap(t['FIRST_dDEC'], np.mean, 10000),
(16, 84))
mdra = np.mean(t['FIRST_dRA'])
mddec = np.mean(t['FIRST_dDEC'])
print 'Mean delta RA is %.3f arcsec (1-sigma %.3f -- %.3f arcsec)' % (
mdra, bsra[0], bsra[1])
print 'Mean delta DEC is %.3f arcsec (1-sigma %.3f -- %.3f arcsec)' % (
mddec, bsdec[0], bsdec[1])
report('Plotting flux ratios')
# Flux ratio plots (only compact sources)
plot_flux_ratios(
'%s.cat.fits_FIRST_match_filtered.fits' % o['catprefix'],
o['pbimage'],
'%s.cat.fits_FIRST_match_filtered_fluxerrors.png' % o['catprefix'],
options=o)
report('Plotting flux scale comparison')
# Flux scale comparison plots
if 'TGSS' in o['list']:
plot_flux_errors(
'%s.cat.fits_TGSS_match_filtered.fits' % o['catprefix'],
o['pbimage'],
'%s.cat.fits_TGSS_match_filtered_fluxratio.png' % o['catprefix'],
'TGSS',
options=o)
t = Table.read(o['catprefix'] + '.cat.fits_TGSS_match_filtered.fits')
ratios = t['Total_flux'] / (t['TGSS_Total_flux'] /
o['TGSS_fluxfactor'])
bsratio = np.percentile(bootstrap(ratios, np.median, 10000), (16, 84))
print 'Median LOFAR/TGSS ratio is %.3f (1-sigma %.3f -- %.3f)' % (
np.median(ratios), bsratio[0], bsratio[1])
if 'NVSS' in o['list']:
t = Table.read(o['catprefix'] + '.cat.fits_NVSS_match_filtered.fits')
t = t[t['Total_flux'] > 10e-3]
ratios = t['Total_flux'] / t['NVSS_Total_flux']
bsratio = np.percentile(bootstrap(ratios, np.median, 10000), (16, 84))
print 'Median LOFAR/NVSS ratio is %.3f (1-sigma %.3f -- %.3f)' % (
np.median(ratios), bsratio[0], bsratio[1])
# Noise estimate
hdu = fits.open(o['pbimage'])
imagenoise = get_rms(hdu)
print 'An estimate of the image noise is %.3f muJy/beam' % (imagenoise *
1E6)
return 0
filter_catalog(o['catprefix'] + '.cat.fits',
o['catprefix'] + '.cat.fits_' + cat + '_match.fits',
o['pbimage'],
o['catprefix'] + '.cat.fits_' + cat +
'_match_filtered.fits',
cat,
options=o)
else:
print 'No matches, abandoning catalogue'
removelist.append(cat)
for cat in removelist:
o['list'].remove(cat)
# Astrometric plots
if 'FIRST' in o['list']:
report('Plotting position offsets')
plot_position_offset(
'%s.cat.fits_FIRST_match_filtered.fits' % o['catprefix'],
o['pbimage'],
'%s.cat.fits_FIRST_match_filtered_positions.png' % o['catprefix'],
'FIRST',
options=o)
t = Table.read(o['catprefix'] + '.cat.fits_FIRST_match_filtered.fits')
bsra = np.percentile(bootstrap(t['FIRST_dRA'], np.mean, 10000),
(16, 84))
bsdec = np.percentile(bootstrap(t['FIRST_dDEC'], np.mean, 10000),
(16, 84))
mdra = np.mean(t['FIRST_dRA'])
mddec = np.mean(t['FIRST_dDEC'])
print 'Mean delta RA is %.3f arcsec (1-sigma %.3f -- %.3f arcsec)' % (
rootdir = '/data/lofar/mjh'
os.chdir(rootdir)
name = sys.argv[1]
try:
qsubfile = sys.argv[2]
except:
qsubfile = '/home/mjh/git/ddf-pipeline/pipeline.qsub'
try:
os.mkdir(name)
except OSError:
warn('Working directory already exists')
pass
os.chdir(name)
report('Downloading data')
if not download_dataset('https://lofar-webdav.grid.sara.nl',
'/SKSP/' + name + '/'):
die('Download failed to get the right number of files')
report('Unpacking data')
unpack()
report('Deleting tar files')
os.system('rm *.tar.gz')
report('Making ms lists')
if make_list():
report('Submit job')
os.system('qsub -N ddfp-' + name + ' -v WD=' + rootdir + '/' + name + ' ' +
qsubfile)
os.chdir(rootdir)
name=sys.argv[1]
if name[0]!='P' and name[0]!='L':
die('This code should be used only with field or observation names',database=False)
do_field=(name[0]=='P')
try:
os.mkdir(name)
except OSError:
warn('Working directory already exists')
pass
os.chdir(name)
report('Downloading data')
if do_field:
success=download_field(name)
else:
success=download_dataset('https://lofar-webdav.grid.sara.nl','/SKSP/'+name+'/')
if not success:
die('Download failed, see earlier errors',database=False)
report('Unpacking data')
unpack()
if do_field:
unpack_db_update()
report('Deleting tar files')
def do_run_pipeline(name, basedir):
if name[0] != 'P' and name[0] != 'L':
die('This code should be used only with field or observation names',
database=False)
do_field = (name[0] == 'P')
try:
qsubfile = sys.argv[2]
except:
qsubfile = '/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
workdir = basedir + '/' + name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
report('Downloading data')
if do_field:
success = download_field(name, basedir=basedir)
else:
success = download_dataset('https://lofar-webdav.grid.sara.nl',
'/SKSP/' + name + '/',
basedir=basedir)
if not success:
die('Download failed, see earlier errors', database=False)
report('Unpacking data')
try:
unpack(workdir=workdir)
except RuntimeError:
if do_field:
update_status(name, 'List failed', workdir=workdir)
raise
if do_field:
update_status(name, 'Unpacked', workdir=workdir)
report('Deleting tar files')
os.system('rm ' + workdir + '/*.tar.gz')
os.system('rm ' + workdir + '/*.tar')
averaged = False
report('Checking structure')
g = glob.glob(workdir + '/*.ms')
msl = MSList(None, mss=g)
uobsids = set(msl.obsids)
for thisobs in uobsids:
# check one MS with each ID
for m, ch, o in zip(msl.mss, msl.channels, msl.obsids):
if o == thisobs:
channels = len(ch)
print 'MS', m, 'has', channels, 'channels'
if channels > 20:
update_status(name, 'Averaging', workdir=workdir)
print 'Averaging needed for', thisobs, '!'
averaged = True
average(wildcard=workdir + '/*' + thisobs + '*')
os.system('rm -r ' + workdir + '/*' + thisobs +
'*pre-cal.ms')
break
report('Making ms lists')
success = make_list(workdir=workdir)
if do_field:
list_db_update(success, workdir=workdir)
if not success:
die('make_list could not construct the MS list', database=False)
report('Creating custom config file from template')
make_custom_config(name, workdir, do_field, averaged)
# now run the job
do_run_job(name, basedir=basedir, qsubfile=None, do_field=do_field)
def do_offsets(o):
# o is the options file
if o['mode'] != 'normal' and o['mode'] != 'test':
raise NotImplementedError('Offsets called with mode ' + o['mode'])
image_root = 'image_full_ampphase_di_m.NS'
method = o['method']
report('Determining astrometric offsets with method ' + method +
' in mode ' + o['mode'])
report('Merging downloaded catalogues')
if os.path.isfile(method + '.fits'):
warn('Merged file exists, reading from disk instead')
data = Table.read(method + '.fits')
else:
if method == 'pslocal':
data = Table.read(method + '/' + method + '.txt', format='ascii')
data['RA'].name = 'ra'
data['DEC'].name = 'dec'
data.write(method + '.fits')
else:
kwargs = {}
if 'panstarrs' in method:
kwargs['rastr'] = 'ramean'
kwargs['decstr'] = 'decmean'
data = merge_cat(method, **kwargs)
if o['mode'] == 'test':
image_root += '_shift'
method += '-test'
report('Running PyBDSM on LOFAR image, please wait...')
catfile = image_root + '.offset_cat.fits'
gaulfile = catfile.replace('cat', 'gaul')
if os.path.isfile(catfile):
warn('Catalogue already exists, skipping pybdsf run')
else:
if o['mode'] == 'test':
suffix = 'facetRestored'
else:
suffix = 'restored'
pbimage = image_root + '.int.' + suffix + '.fits'
nonpbimage = image_root + '.app.' + suffix + '.fits'
img = bdsm.process_image(pbimage,
detection_image=nonpbimage,
thresh_isl=4.0,
thresh_pix=5.0,
rms_box=(150, 15),
rms_map=True,
mean_map='zero',
ini_method='intensity',
adaptive_rms_box=True,
adaptive_thresh=150,
rms_box_bright=(60, 15),
group_by_isl=False,
group_tol=10.0,
output_opts=True,
output_all=True,
atrous_do=False,
flagging_opts=True,
flag_maxsize_fwhm=0.5,
advanced_opts=True,
blank_limit=None)
img.write_catalog(outfile=catfile,
catalog_type='srl',
format='fits',
correct_proj='True')
img.write_catalog(outfile=gaulfile,
catalog_type='gaul',
format='fits',
correct_proj='True')
lofar = Table.read(catfile)
print len(lofar), 'LOFAR sources before filtering'
filter = (lofar['E_RA'] * 3600.0) < 2.0
filter &= (lofar['E_DEC'] * 3600.0) < 2.0
filter &= (lofar['Maj'] * 3600.0) < 10
lofar = lofar[filter]
print len(lofar), 'LOFAR sources after filtering'
regfile = image_root + '.tessel.reg'
cra, cdec = get_centpos()
report('Set up structure')
NDir = np.load("image_dirin_SSD_m.npy.ClusterCat.npy").shape[0]
oo = Offsets(method,
n=NDir,
imroot=image_root,
cellsize=o['cellsize'],
fitmethod=o['fit'])
report('Label table')
lofar_l = oo.r.add_facet_labels(lofar)
report('Finding offsets')
oo.find_offsets(lofar_l, data)
report('Fitting offsets')
oo.fit_offsets()
report('Making plots and saving output')
#oo.plot_fits(method+'-fits.pdf')
oo.save_fits()
oo.plot_offsets()
if 'test' not in o['mode']:
oo.save(method + '-fit_state.pickle')
report('Making astrometry error map, please wait')
oo.make_astrometry_map('astromap.fits', 20)
oo.offsets_to_facetshift('facet-offset.txt')
def do_offsets(o):
# o is the options file
if o['mode']!='normal' and o['mode']!='test':
raise NotImplementedError('Offsets called with mode '+o['mode'])
image_root='image_full_ampphase_di_m.NS'
method=o['method']
report('Determining astrometric offsets with method '+method+' in mode '+o['mode'])
report('Merging downloaded catalogues')
if os.path.isfile(method+'.fits'):
warn('Merged file exists, reading from disk instead')
data=Table.read(method+'.fits')
else:
if method=='pslocal':
data=Table.read(method+'/'+method+'.txt',format='ascii')
data['RA'].name='ra'
data['DEC'].name='dec'
data.write(method+'.fits')
else:
kwargs={}
if 'panstarrs' in method:
kwargs['rastr']='ramean'
kwargs['decstr']='decmean'
data=merge_cat(method,**kwargs)
if o['mode']=='test':
image_root+='_shift'
method+='-test'
report('Running PyBDSM on LOFAR image, please wait...')
catfile=image_root+'.offset_cat.fits'
gaulfile=catfile.replace('cat','gaul')
if os.path.isfile(catfile):
warn('Catalogue already exists, skipping pybdsf run')
else:
if o['mode']=='test':
suffix='facetRestored'
else:
suffix='restored'
pbimage=image_root+'.int.'+suffix+'.fits'
nonpbimage=image_root+'.app.'+suffix+'.fits'
img = bdsm.process_image(pbimage, detection_image=nonpbimage, thresh_isl=4.0, thresh_pix=5.0, rms_box=(150,15), rms_map=True, mean_map='zero', ini_method='intensity', adaptive_rms_box=True, adaptive_thresh=150, rms_box_bright=(60,15), group_by_isl=False, group_tol=10.0,output_opts=True, output_all=True, atrous_do=False, flagging_opts=True, flag_maxsize_fwhm=0.5,advanced_opts=True, blank_limit=None)
img.write_catalog(outfile=catfile,catalog_type='srl',format='fits',correct_proj='True')
img.write_catalog(outfile=gaulfile,catalog_type='gaul',format='fits',correct_proj='True')
lofar=Table.read(catfile)
print len(lofar),'LOFAR sources before filtering'
filter=(lofar['E_RA']*3600.0)<2.0
filter&=(lofar['E_DEC']*3600.0)<2.0
filter&=(lofar['Maj']*3600.0)<10
lofar=lofar[filter]
print len(lofar),'LOFAR sources after filtering'
regfile=image_root+'.tessel.reg'
cra,cdec=get_centpos()
report('Set up structure')
NDir=np.load("image_dirin_SSD_m.npy.ClusterCat.npy").shape[0]
oo=Offsets(method,n=NDir,imroot=image_root,cellsize=o['cellsize'],fitmethod=o['fit'])
report('Label table')
lofar_l=oo.r.add_facet_labels(lofar)
report('Finding offsets')
oo.find_offsets(lofar_l,data)
report('Fitting offsets')
oo.fit_offsets()
report('Making plots and saving output')
#oo.plot_fits(method+'-fits.pdf')
oo.save_fits()
oo.plot_offsets()
if 'test' not in o['mode']:
oo.save(method+'-fit_state.pickle')
report('Making astrometry error map, please wait')
oo.make_astrometry_map('astromap.fits',20)
oo.offsets_to_facetshift('facet-offset.txt')
def do_run_selfcal(name,basedir,inarchivedir,outarchivedir):
startdir = os.getcwd()
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
sdb.close()
fields = extractdict['fields'].split(',')
selfcal_status = extractdict['selfcal_status']
extract_status = extractdict['extract_status'].split(',')
print 'Working on ',name, 'in fields', fields,'current selfcal status',selfcal_status
workdir=basedir+'/'+name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
print 'In directory', os.getcwd()
os.chdir(workdir)
# Update status to running here
selfcal_status = 'STARTED'
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['selfcal_status'] = selfcal_status
sdb.db_set('reprocessing',extractdict)
sdb.close()
print 'Updated status to STARTED for',name
time.sleep(2.0)
print 'Starting rsync'
fieldstring = ''
for fieldid, field in enumerate(fields):
print field, fields
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
sdb.close()
extract_status = extractdict['extract_status'].split(',')
if extract_status[fieldid] == 'EDONE':
cmd = '%s/%s/%s/%s_%s*archive*'%(inarchivedir,name,field,field,name)
observations = check_output('ssh [email protected] ls -d ' + cmd, shell=True)
print 'ssh [email protected] ls -d ' + cmd
observations = observations.split('\n')[:-1] # remove last empty item in this list
else:
observations = []
print 'DATA LOCATIONS', observations
print 'FIELDS', fields
print 'EXTRACT STATUS', extract_status
for observation in observations:
print observation
report('Copying data from %s'%observation)
#'+ inarchivedir +'/' + name + '/' + field +'/' + field +'_'+name +'.dysco.sub.shift.avg.weights.ms.archive')
do_rsync_download(observation.split('/')[-1],inarchivedir +'/'+name + '/'+field +'/',workdir)
fieldstring += observation.split('/')[-1] + ' '
#'%s_%s.dysco.sub.shift.avg.weights.ms.archive '%(field,name)
fieldstring = fieldstring[:-1]
# Update status to copied here
report('Updating %s status to copied'%name)
selfcal_status = 'COPIED'
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['selfcal_status'] = selfcal_status
uvminstr = str(extractdict['uvmin'])
uvmin = extractdict['uvmin']
sdb.db_set('reprocessing',extractdict)
sdb.close()
# Create boxfile
report('Create ds9 region file for extraction')
create_ds9_region('%s.ds9.reg'%name,extractdict['ra'],extractdict['decl'],extractdict['size'])
# Run subtract code
print os.getcwd(), 'working here'
if uvmin > 0.0:
print ('runwsclean.py --uvmin=%s -b %s.ds9.reg -i %s %s'%(uvminstr,name,name+"_image",fieldstring))
os.system('runwsclean.py --uvmin=%s -b %s.ds9.reg -i %s %s'%(uvminstr,name,name+"_image",fieldstring))
else:
print ('runwsclean.py -b %s.ds9.reg -i %s %s'%(name,name+"_image",fieldstring))
os.system('runwsclean.py -b %s.ds9.reg -i %s %s'%(name,name+"_image",fieldstring))
report('Archiving the results to %s'%outarchivedir)
os.chdir(workdir)
f = glob.glob('%s.ds9.tar.gz'%(name))
do_rsync_upload(name,outarchivedir,f)
# update the database to give success
selfcal_status = 'SDONE'
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['selfcal_status'] = selfcal_status
sdb.db_set('reprocessing',extractdict)
sdb.close()
print 'Updated status to SDONE for',name
def do_run_pipeline(name,basedir,qsubfile=None,do_field=True):
'''
set do_field False for the now obsolete behaviour of downloading
and imaging a particular observation
'''
if qsubfile is None:
qsubfile='/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
workdir=basedir+'/'+name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
report('Downloading data')
if do_field:
success=download_field(name,basedir=basedir)
else:
success=download_dataset('https://lofar-webdav.grid.sara.nl','/SKSP/'+name+'/',basedir=basedir)
if not success:
die('Download failed, see earlier errors',database=False)
report('Unpacking data')
try:
unpack(workdir=workdir)
except RuntimeError:
if do_field:
update_status(name,'Unpack failed',workdir=workdir)
raise
if do_field:
update_status(name,'Unpacked',workdir=workdir)
report('Deleting tar files')
os.system('rm '+workdir+'/*.tar.gz')
os.system('rm '+workdir+'/*.tar')
averaged=False
report('Checking structure')
g=glob.glob(workdir+'/*.ms')
msl=MSList(None,mss=g)
dysco=np.any(msl.dysco)
uobsids=set(msl.obsids)
for thisobs in uobsids:
# check one MS with each ID
for m,ch,o,hc in zip(msl.mss,msl.channels,msl.obsids,msl.hascorrected):
if o==thisobs:
if not(hc):
print('MS',m,'has no corrected_data column, force use of DATA')
averaged=True
channels=len(ch)
print('MS',m,'has',channels,'channels')
if channels>20:
update_status(name,'Averaging',workdir=workdir)
print('Averaging needed for',thisobs,'!')
averaged=True
average(wildcard=workdir+'/*'+thisobs+'*')
os.system('rm -r '+workdir+'/*'+thisobs+'*pre-cal.ms')
break
report('Making ms lists')
success=make_list(workdir=workdir)
if do_field:
list_db_update(success,workdir=workdir)
if not success:
die('make_list could not construct the MS list',database=False)
report('Creating custom config file from template')
make_custom_config(name,workdir,do_field,averaged)
# now run the job
do_run_job(name,basedir=basedir,qsubfile=None,do_field=do_field,dysco=dysco)
def do_run_subtract(name,basedir,inarchivedir,outarchivedir,force=False):
startdir = os.getcwd()
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
sdb.close()
fields = extractdict['fields'].split(',')
extract_status = extractdict['extract_status'].split(',')
print 'Working on ',name, 'in fields', fields,'which have status',extract_status
for i in range(0,len(fields)):
os.chdir(startdir)
if not(extract_status[i] == 'EREADY' or (force and extract_status[i] == 'STARTED')):
continue
field = fields[i]
workdir=basedir+'/'+name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
print 'In directory', os.getcwd()
os.chdir(workdir)
# Update status to running here
extract_status[i] = 'STARTED'
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['extract_status'] = ','.join(extract_status)
sdb.db_set('reprocessing',extractdict)
sdb.close()
print 'Updated status to STARTED for',field,name
time.sleep(2.0)
report('Copying data from %s'%inarchivedir)
# WANT TO MAKE THIS INTO A RSYNC SO THAT IT CAN BE DONE OUTSIDE LEIDEN
#os.system('cp -r %s/%s %s'%(inarchivedir,field,workdir))
do_rsync_download(field,inarchivedir,workdir)
# Update status to copied here
extract_status[i] = 'COPIED'
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['extract_status'] = ','.join(extract_status)
sdb.db_set('reprocessing',extractdict)
sdb.close()
print 'Updated status to COPIED for',field,name
# Create boxfile
create_ds9_region('%s.ds9.reg'%name,extractdict['ra'],extractdict['decl'],extractdict['size'])
# Run subtract code
print os.getcwd(), 'working here'
os.chdir(field)
print ('sub-sources-outside-region.py -b %s/%s.ds9.reg -p %s'%(workdir,name,name))
result=os.system('sub-sources-outside-region.py -b %s/%s.ds9.reg -p %s'%(workdir,name,name))
if result!=0:
raise RuntimeError('sub-sources-outside-region.py failed with error code %i' % result)
# Archive the results need an rsync code this is just the *archive file that needs to be archived.
#os.system('mkdir %s/%s'%(outarchivedir,name))
#os.system('mkdir %s/%s/%s'%(outarchivedir,name,field))
os.chdir(workdir)
f = glob.glob('%s/*.archive*'%(field))
do_rsync_upload(name,field,f)
#print ('cp -r %s_%s.dysco.sub.shift.avg.weights.ms.archive %s/%s/%s'%(field,name,outarchivedir,name,field))
#os.system('cp -r %s_%s.dysco.sub.shift.avg.weights.ms.archive %s/%s/%s'%(field,name,outarchivedir,name,field))
# update the database to give success
extract_status[i] = 'EDONE'
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['extract_status'] = ','.join(extract_status)
sdb.db_set('reprocessing',extractdict)
sdb.close()
print 'Updated status to EDONE for',field,name
# update the database to give selfcal status as SREADY
selfcal_status = 'SREADY'
sdb=SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['selfcal_status'] = selfcal_status
sdb.db_set('reprocessing',extractdict)
sdb.close()
print 'Updated status to SREADY for',name
def do_run_subtract(name, basedir, inarchivedir, outarchivedir, force=False):
startdir = os.getcwd()
sdb = SurveysDB()
extractdict = sdb.get_reprocessing(name)
sdb.close()
fields = extractdict['fields'].split(',')
extract_status = extractdict['extract_status'].split(',')
try:
bad_pointings = extractdict['bad_pointings'].split(',')
except AttributeError:
bad_pointings = ['']
print('Working on ', name, 'in fields', fields, 'which have status',
extract_status)
for i in range(0, len(fields)):
os.chdir(startdir)
if not (extract_status[i] == 'EREADY' or
(force and extract_status[i] == 'STARTED')):
continue
field = fields[i]
if field in bad_pointings:
print('Field', field,
'in bad pointings -- skipping and setting to BADP')
sdb = SurveysDB()
extractdict = sdb.get_reprocessing(name)
extract_status[i] = 'BADP'
extractdict['extract_status'] = ','.join(extract_status)
sdb.db_set('reprocessing', extractdict)
sdb.close()
continue
workdir = basedir + '/' + name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
print('In directory', os.getcwd())
os.chdir(workdir)
# Update status to running here
extract_status[i] = 'STARTED'
sdb = SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['extract_status'] = ','.join(extract_status)
sdb.db_set('reprocessing', extractdict)
sdb.close()
print('Updated status to STARTED for', field, name)
time.sleep(2.0)
report('Copying data from %s' % inarchivedir)
# WANT TO MAKE THIS INTO A RSYNC SO THAT IT CAN BE DONE OUTSIDE LEIDEN
#os.system('cp -r %s/%s %s'%(inarchivedir,field,workdir))
do_rsync_download(field, inarchivedir, workdir)
# Update status to copied here
extract_status[i] = 'COPIED'
sdb = SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['extract_status'] = ','.join(extract_status)
sdb.db_set('reprocessing', extractdict)
sdb.close()
print('Updated status to COPIED for', field, name)
# Create boxfile
create_ds9_region('%s.ds9.reg' % name, extractdict['ra'],
extractdict['decl'], extractdict['size'])
# Run subtract code
print(os.getcwd(), 'working here')
os.chdir(field)
print('sub-sources-outside-region.py -b %s/%s.ds9.reg -p %s' %
(workdir, name, name))
result = os.system(
'sub-sources-outside-region.py -b %s/%s.ds9.reg -p %s' %
(workdir, name, name))
if result != 0:
raise RuntimeError(
'sub-sources-outside-region.py failed with error code %i' %
result)
# Archive the results need an rsync code this is just the *archive file that needs to be archived.
#os.system('mkdir %s/%s'%(outarchivedir,name))
#os.system('mkdir %s/%s/%s'%(outarchivedir,name,field))
os.chdir(workdir)
f = glob.glob('%s/*.archive*' % (field))
do_rsync_upload(name, field, f)
#print ('cp -r %s_%s.dysco.sub.shift.avg.weights.ms.archive %s/%s/%s'%(field,name,outarchivedir,name,field))
#os.system('cp -r %s_%s.dysco.sub.shift.avg.weights.ms.archive %s/%s/%s'%(field,name,outarchivedir,name,field))
# update the database to give success
extract_status[i] = 'EDONE'
sdb = SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['extract_status'] = ','.join(extract_status)
sdb.db_set('reprocessing', extractdict)
sdb.close()
print('Updated status to EDONE for', field, name)
# update the database to give selfcal status as SREADY
selfcal_status = 'SREADY'
sdb = SurveysDB()
extractdict = sdb.get_reprocessing(name)
extractdict['selfcal_status'] = selfcal_status
sdb.db_set('reprocessing', extractdict)
sdb.close()
print('Updated status to SREADY for', name)
'SolsFile': ddsols,
'WSize': interval
}))
return outsols
def full_clearcache(o):
clearcache(o['mslist'], o)
clearcache('temp_mslist.txt', o)
if o['full_mslist'] is not None:
clearcache(o['full_mslist'], o)
if __name__ == '__main__':
# Main loop
report('Welcome to ddf-pipeline, version ' + __version__)
if len(sys.argv) < 2:
warn(
'pipeline.py must be called with at least one parameter file or a command-line\noption list.\nE.g "pipeline.py example.cfg second_example.cfg --solutions-robust=0.1"\nSee below for a complete list of possible options with their default values.'
)
print_options(option_list)
sys.exit(1)
o = options(sys.argv[1:], option_list)
if o['catch_signal']:
catcher = Catcher()
else:
catcher = None
uvrange = [o['image_uvmin'], o['uvmax']]
