def do_rsync(name, basedir, f): other = False if use_database(): with SurveysDB(readonly=True) as sdb: idd = sdb.get_field(name) if not (idd['lotss_field'] > 0) or idd['proprietary_date'] is not None: other = True workdir = basedir + '/' + name if os.environ['DDF_PIPELINE_CLUSTER'] != 'paracluster': target = os.environ[ 'DDF_PIPELINE_LEIDENUSER'] + '@ssh.strw.leidenuniv.nl:' else: target = '' f = [fle for fle in f if os.path.exists(workdir + '/' + fle)] while True: s = 'cd ' + workdir + '; rsync -avz --progress --safe-links --perms --chmod=ugo+rX --partial --timeout=20 ' + ' '.join( f ) + ' ' + target + '/disks/paradata/shimwell/LoTSS-DR2/archive/' + name if other: s = s.replace('DR2/archive', 'DR2/archive_other') print('Running command:', s) retval = call(s, shell=True) if retval == 0: break print('Non-zero return value', retval) if retval != 30: raise RuntimeError('rsync failed unexpectedly') sleep(10)
def check(self): if self.stop: if use_database(): update_status(None, 'Stopped') os.system('CleanSHM.py') raise RuntimeError( 'Caught user-defined exception, terminating gracefully')
def update_status(name,status): if not use_database(): return if name is None: # work it out id=get_id(workdir=os.getcwd()) else: id=name with SurveysDB() as sdb: idd=sdb.get_field(id) if idd is None: raise RuntimeError('Unable to find database entry for field "%s".' % id) idd['vlow_reprocess']=status sdb.set_field(idd)
rms=imagenoise*1e6 print 'An estimate of the image noise is %.3f muJy/beam' % rms drs=do_dr_checker(o['catprefix']+'.cat.fits',o['pbimage'],verbose=False,peak=0.4) dr=np.median(drs) print 'Median dynamic range is',dr # fit source counts if o['fit_sourcecounts']: from fit_sourcecounts import do_fit_sourcecounts sc_norm,sc_index,scale=do_fit_sourcecounts(rms=imagenoise) else: sc_norm=sc_index=scale=None print rms,dr,catsources,first_ra,first_dec,tgss_scale,nvss_scale,sc_norm,sc_index,scale if use_database(): id=get_id() with SurveysDB() as sdb: result=sdb.create_quality(id) result['rms']=rms result['dr']=dr result['catsources']=catsources result['first_ra']=first_ra result['first_dec']=first_dec result['tgss_scale']=tgss_scale result['nvss_scale']=nvss_scale result['sc_norm']=sc_norm result['sc_index']=sc_index result['scale']=scale sdb.set_quality(result)
def die(s,database=True): print(bcolors.FAIL+s+bcolors.ENDC) if database and use_database(): update_status(None,'Failed') raise Exception(s)
o['pbimage'], verbose=False, peak=0.4) dr = np.median(drs) print 'Median dynamic range is', dr # fit source counts if o['fit_sourcecounts']: from fit_sourcecounts import do_fit_sourcecounts sc_norm, sc_index, scale = do_fit_sourcecounts(rms=imagenoise) else: sc_norm = sc_index = scale = None print rms, dr, catsources, first_ra, first_dec, tgss_scale, nvss_scale, sc_norm, sc_index, scale if use_database(): id = get_id() with SurveysDB() as sdb: result = sdb.create_quality(id) result['rms'] = rms result['dr'] = dr result['catsources'] = catsources result['first_ra'] = first_ra result['first_dec'] = first_dec result['tgss_scale'] = tgss_scale result['nvss_scale'] = nvss_scale result['sc_norm'] = sc_norm result['sc_index'] = sc_index result['scale'] = scale sdb.set_quality(result)
def die(s,database=True): print bcolors.FAIL+s+bcolors.ENDC if database and use_database(): update_status(None,'Failed') raise Exception(s)
def check(self): if self.stop: if use_database(): update_status(None,'Stopped') os.system('CleanSHM.py') raise RuntimeError('Caught user-defined exception, terminating gracefully')