def do_rsync(name, basedir, f):
other = False
if use_database():
with SurveysDB(readonly=True) as sdb:
idd = sdb.get_field(name)
if not (idd['lotss_field'] > 0) or idd['proprietary_date'] is not None:
other = True
workdir = basedir + '/' + name
if os.environ['DDF_PIPELINE_CLUSTER'] != 'paracluster':
target = os.environ[
'DDF_PIPELINE_LEIDENUSER'] + '@ssh.strw.leidenuniv.nl:'
else:
target = ''
f = [fle for fle in f if os.path.exists(workdir + '/' + fle)]
while True:
s = 'cd ' + workdir + '; rsync -avz --progress --safe-links --perms --chmod=ugo+rX --partial --timeout=20 ' + ' '.join(
f
) + ' ' + target + '/disks/paradata/shimwell/LoTSS-DR2/archive/' + name
if other:
s = s.replace('DR2/archive', 'DR2/archive_other')
print('Running command:', s)
retval = call(s, shell=True)
if retval == 0:
break
print('Non-zero return value', retval)
if retval != 30:
raise RuntimeError('rsync failed unexpectedly')
sleep(10)
def check(self):
if self.stop:
if use_database():
update_status(None, 'Stopped')
os.system('CleanSHM.py')
raise RuntimeError(
'Caught user-defined exception, terminating gracefully')
def update_status(name,status):
if not use_database():
return
if name is None:
# work it out
id=get_id(workdir=os.getcwd())
else:
id=name
with SurveysDB() as sdb:
idd=sdb.get_field(id)
if idd is None:
raise RuntimeError('Unable to find database entry for field "%s".' % id)
idd['vlow_reprocess']=status
sdb.set_field(idd)
rms=imagenoise*1e6
print 'An estimate of the image noise is %.3f muJy/beam' % rms
drs=do_dr_checker(o['catprefix']+'.cat.fits',o['pbimage'],verbose=False,peak=0.4)
dr=np.median(drs)
print 'Median dynamic range is',dr
# fit source counts
if o['fit_sourcecounts']:
from fit_sourcecounts import do_fit_sourcecounts
sc_norm,sc_index,scale=do_fit_sourcecounts(rms=imagenoise)
else:
sc_norm=sc_index=scale=None
print rms,dr,catsources,first_ra,first_dec,tgss_scale,nvss_scale,sc_norm,sc_index,scale
if use_database():
id=get_id()
with SurveysDB() as sdb:
result=sdb.create_quality(id)
result['rms']=rms
result['dr']=dr
result['catsources']=catsources
result['first_ra']=first_ra
result['first_dec']=first_dec
result['tgss_scale']=tgss_scale
result['nvss_scale']=nvss_scale
result['sc_norm']=sc_norm
result['sc_index']=sc_index
result['scale']=scale
sdb.set_quality(result)
def die(s,database=True):
print(bcolors.FAIL+s+bcolors.ENDC)
if database and use_database():
update_status(None,'Failed')
raise Exception(s)
o['pbimage'],
verbose=False,
peak=0.4)
dr = np.median(drs)
print 'Median dynamic range is', dr
# fit source counts
if o['fit_sourcecounts']:
from fit_sourcecounts import do_fit_sourcecounts
sc_norm, sc_index, scale = do_fit_sourcecounts(rms=imagenoise)
else:
sc_norm = sc_index = scale = None
print rms, dr, catsources, first_ra, first_dec, tgss_scale, nvss_scale, sc_norm, sc_index, scale
if use_database():
id = get_id()
with SurveysDB() as sdb:
result = sdb.create_quality(id)
result['rms'] = rms
result['dr'] = dr
result['catsources'] = catsources
result['first_ra'] = first_ra
result['first_dec'] = first_dec
result['tgss_scale'] = tgss_scale
result['nvss_scale'] = nvss_scale
result['sc_norm'] = sc_norm
result['sc_index'] = sc_index
result['scale'] = scale
sdb.set_quality(result)
def die(s,database=True):
print bcolors.FAIL+s+bcolors.ENDC
if database and use_database():
update_status(None,'Failed')
raise Exception(s)
def check(self):
if self.stop:
if use_database():
update_status(None,'Stopped')
os.system('CleanSHM.py')
raise RuntimeError('Caught user-defined exception, terminating gracefully')
