def cleanup(objects,fail = False) :
""" Method to close up the logs and clean up temporary files
objetcs - objects listing
fail - whether the data reduction failed or not
"""
if(globals.isSci2) :
log.run("rm -rf *.log junk antpos.list bw.sel rms.* flux.log.* fit.log* *.map",[],logit=True)
else :
log.run("rm -rf *.log junk antpos.list bw.sel rms.* flux.log.* fit.log* *.beam *.clean *.map *.bm",[],logit=True)
# move the data files to directories based on purpose
log.run("mkdir -p fits",[],logit=False)
log.run("\mv *.fits fits/.",[],logit=False)
for obj in objects._sources :
transferFile("sources",obj._file)
for obj in objects._fluxcals :
transferFile("fluxcals",obj._file)
for obj in objects._passcals :
transferFile("passcals",obj._file)
for obj in objects._gaincals :
transferFile("gaincals",obj._file)
# close everything up
del objects
log.writeLog("Data reduction by CARMA pipeline version %s completed at %s" % (version.VERSION,time.ctime()))
log.closeLog()
prepend(log.closeScript())
if(not fail) :
print "\nData reduction complete"
else :
print "\nData reduction failed"
def calSort(objects) :
""" Function to sort calibrators and flag any overlapping data
input :
objetcs - a list of the objects
returns :
none
"""
# check for full overlap first
done = []
for gcal in objects._gaincals :
for gcal2 in objects._gaincals :
if(gcal._name != gcal2._name and not(gcal._name in done)) :
if(gcal._start[0] >= gcal2._start[0] and gcal._end[-1] <= gcal2._end[-1]) :
done.append(gcal._name)
objects.convertGainToSource(gcal)
# check for partial overlap
done = []
for gcal in objects._gaincals :
for gcal2 in objects._gaincals :
if(gcal._name != gcal2._name and not(gcal2._name in done)) :
if(gcal._end[-1] > gcal2._start[0] and gcal._start[0] < gcal2._start[0]) : # we have an overlap that needs to be taken care of
# flag the overlaping data in the later data set
log.writeComment("Flag the overlaping data in the later data set")
args = []
args.append(globals.Variable("vis",gcal2._file))
args.append(globals.Variable("select","time'('%s,%s')'" % (calculations.unconvertTime(gcal2._start[0] - 0.0083),calculations.unconvertTime(gcal._end[-1] + 0.00833))))
args.append(globals.Variable("flagval","flag"))
sys = log.run("uvflag",args)
if(sys == 0) :
log.writeLog("Flagged %s from %s to %s to avoid primary calibrator overlap" % (gcal2._file,calculations.unconvertTime(gcal2._start[0] - 0.0083),calculations.unconvertTime(gcal._end[-1] + 0.00833)))
done.append(gcal)
async def udpclientsend(udpclient,mqttclient):
while True:
if not msgque.empty():
msg = msgque.get()
# msg = input(">> ").strip()
ip_port = (sensorgateway.ip, udpobj.port)
try:
udpclient.sendto(msg.encode('utf-8'),ip_port)
except Exception as e:
logger.writeLog(gendate() + ' 发送UDP请求指令失败->'+ msg + '\nReson->'+ str(e),'udpmain.log')
print(gendate() + ' 等待接收信息--->')
for i in range(0,3):
await asyncio.sleep(1)
try:
data,server_addr = udpclient.recvfrom(1024)
except Exception:
pass
else:
print(data, server_addr)
res = json.loads(data)
if res['model'] == 'weather':
if res['cmd'] == 'read_rsp':
mqttinfo = sensorweather.genMQTTinfo(sensorgateway.location,
res['params'][1]['temperature'],
res['params'][2]['humidity'],
res['params'][3]['pressure'],
res['params'][0]['battery_voltage'])
mqttclient.publish(sensorgateway.weatherpubtopic,json.dumps(mqttinfo))
break
i = i + 1
await asyncio.sleep(0.01)
def on_message(client, userdata, msg):
'''
命令报文格式 {
"gatewaysid":"aaaaaaa",
"sensorsid":"dfdfdfd",
"sensortype":"weather", # weather plug leak
"cmd":"read", # read on off
}
'''
msgstr = str(msg.payload, 'utf-8')
# print(gendate() + " " + msg.topic+"--->"+ msgstr)
if len(str.strip(msgstr)) > 1: #判断报文是否为空
try:
msgobj = json.loads(msgstr)
findobj = False #存放找到的设备信息
#1.判断报文中携带的 网关SID是否存在
for siditem in sensorgateway.sidlist:
if siditem['sid'] == msgobj['gatewaysid']:
findobj = siditem
break
if findobj != False:
if findobj['token'] != 0:
if msgobj['sensortype'] == "plug" and (
msgobj['cmd'] == 'on' or msgobj['cmd'] == 'off'):
insstr = sensorplug.writeStatus(
msgobj['sensorsid'], findobj['key'], msgobj['cmd'],
findobj["token"])
tempobj = {
"gatewaysid": findobj['sid'],
"gatewayip": findobj['ip'],
"insstr": insstr
}
msgque.put(tempobj)
print(gendate() + ' 生成控制开关指令--->' + str(tempobj))
elif findobj['sensortype'] == "weather" and findobj[
'cmd'] == 'weather':
# insstr = sensorweather.readDevStatus()
# msgque.put(insstr)
# print(gendate() + ' 生成查询温湿度大气压指令--->' + insstr)
pass
else:
# print(gendate() + ' 此指令无法识别--->' + msgstr)
pass
else:
print(gendate() + ' 无法获取正确的网关心跳Token，指令生成失败, 稍后请重试: ' +
msgstr)
else:
# logger.writeLog(gendate() + ' 未找到要执行操作的设备->'+ msg + '\nReson->'+ msgstr,'mqtt.log')
print(gendate() + ' 未找到要执行操作的设备->' + msg + '\nReson->' +
msgstr)
except Exception as e:
logger.writeLog(
gendate() + ' MQTT指令格式错误->' + msg + '\nReson->' + str(e),
'mqtt.log')
else:
logger.writeLog(gendate() + ' MQTT指令格式错误->' + '\nReson->' + msgstr,
'mqtt.log')
def __init__(self):
'''
初始化，从配置文件读取相关加密参数
'''
try:
# 读取配置文件
f = open("config.yaml", "r+", encoding="utf-8")
fstream = f.read()
configobj = yaml.safe_load(fstream)
# self.secretKey = configobj['gateway']['key'].encode('utf-8')
self.iv = a2b_hex(configobj['gateway']['iv'])
except Exception as e:
logger.writeLog("AES加密组件初始化失败" + str(e), 'encrypt.log')
def __init__(self):
'''
初始化，从配置文件读取MQTT参数
'''
try:
# 读取配置文件
f = open("config.yaml", "r+", encoding="utf-8")
fstream = f.read()
configobj = yaml.safe_load(fstream)
self.host = configobj['mqtt']['host']
self.port = configobj['mqtt']['port']
self.keepalive = configobj['mqtt']['keepalive']
except Exception as e:
logger.writeLog("MQTT组件初始化失败" + str(e), 'mqtt.log')
def getPrimaryFluxCal(objects) :
""" Method to determine the primary flux calibrator
searches the objects looking for the best one in descending order of preference
input :
objects - the objects object
returns:
the name of the primary flux calibrator
"""
primaryFluxcal = Fluxcal()
if(fluxcal["NEPTUNE"]) :
for fcal in objects._fluxcals :
if(fcal._name == "NEPTUNE") :
primaryFluxcal = fcal
elif(fluxcal["URANUS"]) :
for fcal in objects._fluxcals :
if(fcal._name == "URANUS") :
primaryFluxcal = fcal
elif(fluxcal["MARS"]) :
for fcal in objects._fluxcals :
if(fcal._name == "MARS") :
primaryFluxcal = fcal
elif(fluxcal["MWC349"]) :
for fcal in objects._fluxcals :
if(fcal._name == "MWC349") :
primaryFluxcal = fcal
elif(fluxcal["MERCURY"]) :
for fcal in objects._fluxcals :
if(fcal._name == "MERCURY") :
primaryFluxcal = fcal
elif(fluxcal["JUPITER"]) :
for fcal in objects._fluxcals :
if(fcal._name == "JUPITER") :
primaryFluxcal = fcal
elif(fluxcal["SATURN"]) :
for fcal in objects._fluxcals :
if(fcal._name == "SATURN") :
primaryFluxcal = fcal
elif(fluxcal["VENUS"]):
for fcal in objects._fluxcals :
if(fcal._name == "VENUS") :
primaryFluxcal = fcal
elif(fluxcal["PLUTO"]) :
for fcal in objects._fluxcals :
if(fcal._name == "PLUTO") :
primaryFluxcal = fcal
else:
primaryFluxcal = objects._fluxcals[0]
log.writeLog("Using %s as a flux calibrator for bootflux" % (primaryFluxcal._name))
return primaryFluxcal
def calculateMosaicCleanRegion(pointing,cellsize,freq,offset) :
""" Method to calculate the clean region for a multipoint mosaic
input :
pointing - a list of the pointing positions in offset arcseconds from the phase center
cellsize - the cellsize in arcseconds
freq - observing frequency in GHz
offset - offset from 0,0 of the phase center in the map
returns :
the clean region
"""
polys = []
temp = ""
for i in pointing :
temp = temp + "%i %i\n" % (i[0],i[1])
pointing = sortPointing(pointing,offset)
# create the hexagonal regions to clean for each pointing
for center in range(0,len(pointing)) :
polys.append(createPoly(pointing[center],cellsize,freq,offset))
# see which ones overlap and merge them one at a time
# since all regions are the same size then they can only overlap if one has a point inside the other
while(len(polys) > 1) :
inside = [False,False,False,False,False,False,False,False,False]
overlap = False
for poly in range(1,len(polys)) :
for coord in range(0,8) :
if(pointInPoly(polys[0],polys[poly][coord][0],polys[poly][coord][1])) :
inside[coord] = True
overlap = True
if(overlap) :
inside[8] = inside[0]
mergePoly(polys,0,poly,inside)
break
if(not overlap) :
log.writeLog("regions do not overlap")
# something is not right
return
doubles = True
while(doubles) :
doubles = False
for i in range(0,len(polys[0]) - 1) :
if(polys[0][i] == polys[0][i + 1]) :
del polys[0][i]
doubles = True
break
region = "polygon'('"
for i in polys[0] :
region = region + "%i,%i," % (max(1,min(i[0],(offset[0]*2.0))),max(1,min(i[1],(offset[1]*2.0))))
region = region[:-1] + "')'"
return region
def __init__(self):
'''
初始化，从配置文件读取相关UDP参数
'''
try:
# 读取配置文件
f = open("config.yaml", "r+", encoding="utf-8")
fstream = f.read()
configobj = yaml.safe_load(fstream)
self.bindaddr = configobj['udp']['bindaddr']
self.desip = configobj['udp']['desip']
self.port = configobj['udp']['port']
# print(configobj)
except Exception as e:
logger.writeLog("UDP组件初始化失败" + str(e), 'udp.log')
def parse_http_request(pkt):
if not pkt.haslayer(Raw):
return
# if latest ack number same as last_ack means: it's segmented
nonlocal payload, last_ack
if pkt[IP].ack == last_ack:
payload = payload + pkt[Raw].load.decode('utf-8', 'ignore')
else:
payload = pkt[Raw].load.decode('utf-8', 'ignore')
last_ack = pkt[IP].ack
# find request url and log it if found
url = find_url_from_headers(extract_headers_from_request_payload(payload))
if url:
log(url)
writeLog(url)
def __init__(self):
'''
初始化，从配置文件读取网关参数
'''
try:
# 读取配置文件
f = open("config.yaml", "r+", encoding="utf-8")
fstream = f.read()
configobj = yaml.safe_load(fstream)
self.sidlist = configobj['gateway']['sidlist']
self.location = configobj['gateway']['location']
self.desport = configobj['gateway']['port']
self.interval = configobj['gateway']['interval']
self.weatherpubtopic = configobj['gateway']['weatherpubtopic']
self.leakpubtopic = configobj['gateway']['leakpubtopic']
# print(configobj)
except Exception as e:
logger.writeLog("网关组件初始化失败" + str(e), 'dev.log')
def __init__(self, devtype="noise"):
'''
初始化，从配置文件读取相关串口Modbus参数
'''
try:
self.devtype = devtype
# 读取配置文件
f = open("config.yaml", "r+", encoding="utf-8")
fstream = f.read()
configobj = yaml.safe_load(fstream)
#通用部分
self.sid = configobj['uartmodbus'][devtype]['sid']
self.devname = configobj['uartmodbus'][devtype]['devname']
self.unit = configobj['uartmodbus'][devtype]['unit']
self.method = configobj['uartmodbus'][devtype]['method']
self.port = configobj['uartmodbus'][devtype]['port']
self.timeout = configobj['uartmodbus'][devtype]['timeout']
self.baudrate = configobj['uartmodbus'][devtype]['baudrate']
self.pubtopic = configobj['uartmodbus'][devtype]['pubtopic']
self.location = configobj['uartmodbus'][devtype]['location']
self.interval = configobj['uartmodbus'][devtype]['interval']
self.addrcode = configobj['uartmodbus'][devtype]['holdingreg'][
'addrcode']
self.startaddr = configobj['uartmodbus'][devtype]['holdingreg'][
'startaddr']
self.datalen = configobj['uartmodbus'][devtype]['holdingreg'][
'datalen']
if devtype == 'pt100':
self.damrange = configobj['uartmodbus'][devtype]['damrange']
self.damexpand = configobj['uartmodbus'][devtype]['damexpand']
self.minma = configobj['uartmodbus'][devtype]['minma'] #最低电流值
self.mapercelsius = configobj['uartmodbus'][devtype][
'mapercelsius'] #对应电流
if devtype == 'pressure':
self.waterpremax = configobj['uartmodbus'][devtype][
'waterpremax']
self.waterpremin = configobj['uartmodbus'][devtype][
'waterpremin']
except Exception as e:
logger.writeLog("串口uartmodbus组件初始化失败-->" + str(e),
'uartmodbus.log')
def startup(visFile,args) :
""" Method that starts the common processes for all data reduction
input:
visFile - the name of the main visibility file
"""
# start logging system
log.openScriptLog("reductionScript.csh")
log.openMessageLog("reduction.notes")
version.versionCheck()
log.run("rm -rf sources passcals gaincals fluxcals fits",[])
if(len(args) > 0) :
log.run("tar",args,logit=True,fatal=True)
log.writeAll("\n")
# add notes to the history file about the data reduction
args = []
args.append(globals.Variable("in",visFile))
args.append(globals.Variable("comment","'Data reduction by CARMA pipeline version %s started at %s with preferences:'" % (version.VERSION,time.ctime())))
args.append(globals.Variable("file",filePath))
args.append(globals.Variable("lines","6,100"))
log.run("addhis",args,logit=False)
log.writeAll("\n")
log.writeLog("Data reduction by CARMA pipeline version %s started at %s\n" %(version.VERSION,time.ctime()))
def splitHybrid(passbandcal, numberOfWins) :
""" Method to split hybrid data into seperate files for each bandwidth and window
inputs :
passbandcal - a passband calibrator object
numberOfWins - the total number of windows
returns :
none
"""
log.writeComment("Splitting %s into separate hybrid components" % (passbandcal._name))
array = []
halfWin = numberOfWins/2
if(globals.isSci2) :
halfWin = globals.ENDWINDOW
for i in range(globals.STARTWINDOW, halfWin + 1) :
array.append(0)
for window in range(1, halfWin + 1) :
# select out the individual windows
fileEnd = str(random.randint(1,100000))
sys = log.run("rm -rf temp.w%i.%s; sleep 3",[],fatal=True)
args = []
args.append(globals.Variable("vis",passbandcal._file))
args.append(globals.Variable("select","win'('%i')'" % (window)))
args.append(globals.Variable("out","temp.w%i.%s" % (window,fileEnd)))
sys = log.run("uvcat",args,fatal=True)
if(sys != 0) :
log.writeLog("FAILED: uvcat vis=%s select=win'('%i')' out=temp.w%i%s" % (passbandcal._file, window, window,fileEnd))
exit(0)
# list all spectral data for the window
args = []
args.append(globals.Variable("vis","temp.w%i.%s" % (window,fileEnd)))
args.append(globals.Variable("options","spec"))
args.append(globals.Variable("recnum","0"))
args.append(globals.Variable("log","pcal.log"))
log.run("uvlist",args,fatal=True)
input = open("pcal.log",'r')
uvList = input.readlines()
input.close()
uvList.reverse()
bandwidths = []
nchan = 0
# gather the different bandwidth data
while(len(uvList) > 0) :
line = uvList.pop()
if("number of channels" in line) :
splitLine = line.split()
nchan = int(splitLine[4])
elif("frequency interval" in line) :
splitLine = line.split()
bw = sources.findBandwidth(float(splitLine[3])*1000.0*nchan)
bandwidths.append(bw)
passbandcal.setHybridChans(bw, nchan)
first = True
# for each bandwidth detected separate out the data for both USB and LSB windows
for bw in bandwidths :
if(bw != bandwidths[0]) :
if(first) :
passbandcal.setHybridConf(bandwidths[0])
first = False
passbandcal.setHybridConf(bw)
for config in passbandcal._hybridConf :
if(passbandcal._hybridConf.get(config)) :
fileEnd = str(random.randint(1,100000))
array[window - 1] = config
arrayString = "%i" % (array[0])
for m in range(1, len(array)) :
arrayString = arrayString + ",%i" % (array[m])
args = []
args.append(globals.Variable("vis",passbandcal._file))
args.append(globals.Variable("bw",arrayString))
args.append(globals.Variable("out","temp.pbcal." + fileEnd))
log.run("bwsel",args)
args = []
args.append(globals.Variable("vis","temp.pbcal." + fileEnd))
args.append(globals.Variable("select","win'('%i')'" % (window)))
args.append(globals.Variable("out",passbandcal._file,".%i.w%i" % (config, window)))
log.run("uvcat",args)
args = []
args.append(globals.Variable("vis","temp.pbcal." + fileEnd))
args.append(globals.Variable("select","win'('%i')'" % (window + halfWin)))
args.append(globals.Variable("out",passbandcal._file,".%i.w%i" % (config, window + halfWin)))
log.run("uvcat",args)
array[window - 1] = 0
log.run("rm -rf temp.*",[])
def cleanWindow(source,window,csub) :
""" Method to clean a spectral window
input :
source - the source data
window - which window to clean
csub - has contuinuum subtraction been done
returns :
none
"""
global rmsList
lastIter = 10000.0
addon = ""
fileEnd = str(random.randint(1,100000))
if(csub) :
addon = ".cs"
# do the initial clean
log.run("mossdi map=%s.w%i.map beam=%s.w%i.beam out=%s.w%i.clean niters=100 region=%s" % (source._file+addon, window, source._file+addon, window, source._file+addon, window,source.getCleanRegion()),[],fatal=True,logit=False)
# get the beam size
log.run("mospsf beam=%s.w%i.beam out=%s.w%i.bm""" % (source._file+addon, window, source._file+addon, window),[],fatal=True,logit=False)
log.run("imfit in=%s.w%i.bm object=beam region=arcsec,box'('-5,-5,5,5')' > fit.log.%s" % (source._file+addon, window,fileEnd),[],fatal=True,logit=False)
input = open("fit.log.%s" % (fileEnd),'r')
fileList = input.readlines()
input.close()
majorAxis = 0.0
minorAxis = 0.0
positionAngle = 0.0
while(len(fileList) > 0) :
line = fileList.pop()
if("Position angle" in line) :
splitLine = line.split()
positionAngle = float(splitLine[3])
elif("Minor axis" in line) :
splitLine = line.split()
minorAxis = float(splitLine[3])
elif("Major axis" in line) :
splitLine = line.split()
majorAxis = float(splitLine[3])
log.run("restor model=%s.w%i.clean map=%s.w%i.map beam=%s.w%i.beam fwhm=%f,%f pa=%f out=%s.w%i.finalmap" % (source._file+addon, window, source._file+addon, window, source._file+addon, window, majorAxis,minorAxis,positionAngle, source._file+addon, window),[],logit=False)
region = dict()
region[0] = "%i,%i,%i,%i" % (int(math.floor(imsize/4)),int(math.floor((imsize/4) + (imsize/6))),int(math.floor((imsize/4) + (imsize/6))),int(math.floor((imsize/4) + (imsize/3))))
region[1] = "%i,%i,%i,%i" % (int(math.floor((imsize/4) + (imsize/6))),int(math.floor((imsize/4) + (imsize/3))),int(math.floor((imsize/4) + (imsize/3))),int(math.floor(3*imsize/4)))
region[2] = "%i,%i,%i,%i" % (int(math.floor((imsize/4) + (imsize/3))),int(math.floor((imsize/4) + (imsize/6))),int(math.floor(3*imsize/4)),int(math.floor((imsize/4) + (imsize/3))))
region[3] = "%i,%i,%i,%i" % (int(math.floor((imsize/4) + (imsize/6))),int(math.floor(imsize/4)),int(math.floor((imsize/4) + (imsize/3))),int(math.floor((imsize/4) + (imsize/6))))
rms = 1000000.0
best = 0
# determine the rms noise
for j in range(4) :
log.run("imstat in=%s.w%i.finalmap region=box'('%s')' device=/NULL > rms.log.%s""" % (source._file+addon, window, region.get(j),fileEnd),[],logit=False)
input = open("rms.log.%s" % (fileEnd),'r')
fileList = input.readlines()
input.close()
fileList.reverse()
while(len(fileList) > 0) :
line = fileList.pop()
if("Frequency" in line) :
line = fileList.pop()
temprms = float(line[42:51])
if(temprms < rms) :
rms = temprms
best = j
converged = False
run = 1
cutoff = 1000.0
# loop over clean and restor until the noise level settles down or until 10 iterations are complete
while((not converged) and (run <= 5)) :
cutoff = rms * p.preferences.get("cleanThreshold")
log.run("rm -rf %s.w%i.clean %s.w%i.finalmap; sleep 3" % (source._file+addon, window, source._file+addon, window),[],fatal=True,logit=False)
log.run("mossdi map=%s.w%i.map beam=%s.w%i.beam out=%s.w%i.clean niters=50000 cutoff=%f region=%s" % (source._file+addon, window, source._file+addon, window, source._file+addon, window, cutoff,source.getCleanRegion()),[],fatal=True,logit=False)
log.run("restor model=%s.w%i.clean map=%s.w%i.map beam=%s.w%i.beam fwhm=%f,%f pa=%f out=%s.w%i.finalmap" % (source._file+addon, window, source._file+addon, window, source._file+addon, window, majorAxis,minorAxis,positionAngle, source._file+addon, window),[],fatal=True,logit=False)
log.run("imstat in=%s.w%i.finalmap region=box'('%s')' device=/NULL > rms.log.%s" % (source._file+addon, window, region.get(best),fileEnd),[],logit=False)
input = open("rms.log.%s" % (fileEnd),'r')
fileList = input.readlines()
input.close()
fileList.reverse()
while(len(fileList) > 0) :
line = fileList.pop()
if("cube" in line) :
line = fileList.pop()
temprms = float(line[42:51])
if(abs(temprms-rms) < 0.05 * rms) : # there must be less than a 5 % change
converged = True
else :
rms = temprms
run += 1
log.writeComment("Cleaning window %i" % (window))
if(run > 5) :
log.writeComment("Did not reach noise level cutoff, performed 5 iterations.")
args = []
args.append(globals.Variable("map",source._file,addon + ".w%i.map" % (window)))
args.append(globals.Variable("beam",source._file,addon + ".w%i.beam" % (window)))
args.append(globals.Variable("out",source._file,addon + ".w%i.clean" % (window)))
args.append(globals.Variable("niters","50000"))
args.append(globals.Variable("cutoff",str(cutoff)))
args.append(globals.Variable("region",source.getCleanRegion()))
log.run("mossdi",args,execute=False)
args = []
args.append(globals.Variable("model",source._file,addon + ".w%i.clean" % (window)))
args.append(globals.Variable("map",source._file,addon + ".w%i.map" % (window)))
args.append(globals.Variable("beam",source._file,addon + ".w%i.beam" % (window)))
args.append(globals.Variable("fwhm","%f,%f" % (majorAxis,minorAxis)))
args.append(globals.Variable("pa",str(positionAngle)))
args.append(globals.Variable("out",source._file,addon + ".w%i.finalmap" % (window)))
log.run("restor",args,execute=False)
args = []
args.append(globals.Variable("in",source._file,addon + ".w%i.finalmap" % (window)))
args.append(globals.Variable("op","xyout"))
args.append(globals.Variable("out",source._file,addon + ".w%i.fits" % (window)))
log.run("fits",args,logit=False)
log.writeScript("\n")
log.writeLog("Data reduction of %s window %i complete. Final map(%s.w%i.finalmap) has a noise level of %f Jy/beam\n" % (source._name, window, source._file+addon, window, rms))
rmsList[window] = rms
startupTeardown.endFile("%s.w%i.finalmap" % (source._file+addon,window))
def getWindowInfo(visFile,object,isSource=False,isGaincal=False,isPasscal=False,isFluxcal=False) :
""" Method to separate the main miriad file into smaller ones for each object
input :
visFile - the main MIRIAD visibility file
object - the name of the object to separate
source - whether the object is a source
gaincal - whether the object is a gain calibrator
passcal - whether the object is a passband calibrator
fluxcal - whether the object is a flux calibrator
"""
global doDecor
numberOfWins = len(object._bandwidths)
# if the output file exists then we must create a copy for the current purpose
log.writeComment("Separating data into individual, window based, files")
if(os.path.exists(object._file) == 1) :
id = ""
if(isGaincal) :
id = globals.GAINCALEND
elif(isPasscal) :
id = globals.PASSCALEND
elif(isFluxcal) :
id = globals.FLUXCALEND
else :
log.writeLog("Error: File %s exists exiting" % (object._file))
raise Exception, "Error: File %s exists exiting" % (object._file)
print "SOURCE copy",object._file
args = []
args.append(globals.Variable(None,object._file))
args.append(globals.Variable(None,object._file,"." + id))
log.run("cp -r",args,fatal=True)
object._file = object._file +"." + id
else :
print "SOURCE new",object._file
args = []
args.append(globals.Variable("vis",visFile))
args.append(globals.Variable("select",object._name,"')'","-auto,source'('"))
args.append(globals.Variable("out",object._file))
log.run("uvcat",args,fatal=True)
if(object._name == "NOISE") :
args = []
args.append(globals.Variable("vis","NOISE"))
args.append(globals.Variable("options","noisecal"))
args.append(globals.Variable("out","NOISE",".conj; sleep 3; rm -r"))
args.append(globals.Variable(None,"NOISE","; sleep 3; mv"))
args.append(globals.Variable(None,"NOISE",".conj"))
args.append(globals.Variable(None,"NOISE"))
log.run("uvcal",args)
# correct for decorrelation if needed
if(doDecor and object._name != "NOISE") :
log.writeComment("Correcting for decorrelation on long baselines")
fileEnd = str(random.randint(1,100000))
args = []
args.append(globals.Variable("vis",object._file))
args.append(globals.Variable("options","nocal,nopass,nopol"))
args.append(globals.Variable("delaymax","8500"))
args.append(globals.Variable("out","temp." + fileEnd,"; sleep 3; rm -rf"))
args.append(globals.Variable(None,object._file,"/*; rm -rf"))
args.append(globals.Variable(None,object._file,"; sleep 3; mv temp." + fileEnd))
args.append(globals.Variable(None,object._file))
log.run("uvdecor",args,fatal=True)
if(isPasscal and object.isHybrid()) :
hybrid.splitHybrid(object, numberOfWins)
else :
# separate each window
log.writeAll("\n")
log.writeComment("Separating individual windows")
superwin = {"LSB" : [], "USB" : []}
for window in range(globals.STARTWINDOW, globals.ENDWINDOW + 1) :
if(not object.isSuper(window)) :
args = []
args.append(globals.Variable("vis",object._file))
args.append(globals.Variable("select","window'('%i')'" % (window)))
args.append(globals.Variable("out",object._file,".w%i" % (window)))
log.run("uvcat",args)
else :
if(window <= numberOfWins/2) :
superwin["LSB"].append(window)
else :
superwin["USB"].append(window)
# separate out the super windows
lsbString = ""
usbString = ""
for window in superwin["LSB"] :
lsbString += "%i," % (window)
for window in superwin["USB"] :
usbString += "%i," % (window)
if(len(superwin["LSB"]) > 0) :
object._lsbGood = True
args = []
args.append(globals.Variable("vis",object._file))
args.append(globals.Variable("select","window'('%s')'" % (lsbString[:-1])))
args.append(globals.Variable("out",object._file,".LSB"))
log.run("uvcat",args)
if(len(superwin["USB"]) > 0) :
object._usbGood = True
args = []
args.append(globals.Variable("vis",object._file))
args.append(globals.Variable("select","window'('%s')'" % (usbString[:-1])))
args.append(globals.Variable("out",object._file,".USB"))
log.run("uvcat",args)
object.calcSuperChans()
for window in range(globals.STARTWINDOW, globals.ENDWINDOW + 1) :
# flag edge channels
if(object._bandwidths[window - 1] == 62) :
log.writeAll("\n")
log.writeComment("Flagging edge channels in 62 MHz windows")
args = []
args.append(globals.Variable("vis",object._file,".w%i" % (window)))
args.append(globals.Variable("edge","3,3"))
args.append(globals.Variable("flagval","flag"))
log.run("uvflag",args)
def invertContinuum(objects, obsFreq, avgBaseline, windows, individual = False) :
""" Method to create continuum maps
input :
objects - the objects
obsFreq - observing frequency in GHz
avgBaseline - average baseline in lambda
windows - which windows to invert
individual - whther we are only inverting individual windows
returns :
none
"""
global cellsize
global imsize
global continDone
fileEnd = str(random.randint(1,100000))
# calculate the optimal cell size, based on the median baseline in klambda, want at least 5 pixels across the beam
# but only if the user did not specify it in the preferences file
if(p.preferences.get("cellSize") < 0 and cellsize == None) :
cellsize = calculations.calcCellSize(avgBaseline,0.0)
log.writeComment("Based on average baseline of %f, using a cell size of %f arcseconds." % (avgBaseline, cellsize))
elif(cellsize == None) :
cellsize = p.preferences.get("cellSize")
log.writeComment("Using cellsize %f" % (cellsize))
iterate = True
# calculate the optimal image size, based on the cell size and observing frequency
# but only if the user did not specify an image size in the preferences file
if(p.preferences.get("imageSize") < 0 and imsize == None) :
imsize = calculations.calcImsize(obsFreq, cellsize)
elif(imsize == None) :
iterate = False
imsize = p.preferences.get("imageSize")
# invert each source in turn
sourceList = deepcopy(objects._sources)
if(not individual) :
sourceList.append(deepcopy(objects._gaincals[0]))
for source in sourceList :
log.writeComment("Inverting continuum of %s" % (source._name))
if(isinstance(source,sources.Source)) :
if(source._mosaic) :
if(imsize > 1000) :
imsize = int(imsize * 1.5)
else :
imsize = int(imsize * 2.0)
continDone = True
invertString = ""
invertList = []
startChan = 1
numChan = source._numChans[0]
# make sure we start at the correct channel number
if(source.haveSuper() and not individual) :
for ending in ["LSB","USB"] :
if(not source._lsbGood and ending == "LSB") :
continue
if(not source._usbGood and ending == "USB") :
continue
startChan = 1
numChan = source.getSuperNumChans()
invertString = invertString + ",%s.%s" % (source._file, ending)
invertList.append([source._file,".%s" % (ending)])
args = []
args.append(globals.Variable("vis",source._file,".%s" % (ending)))
args.append(globals.Variable("options","unflagged"))
args.append(globals.Variable("out","temp." + fileEnd + "; sleep 3; rm -rf "))
args.append(globals.Variable(None,source._file,".%s/*; rm -rf " % (ending)))
args.append(globals.Variable(None,source._file,".%s; sleep 3; mv " % (ending)))
args.append(globals.Variable(None,"temp." + fileEnd))
args.append(globals.Variable(None,source._file,".%s" % (ending)))
log.run("uvcat",args,fatal=True)
elif(len(windows) != 0) :
for window in windows :
if(source._bandwidths[window - 1] == 62) :
startChan = 4
numChan = source._numChans[window - 1] - 6
invertString = invertString + ",%s.w%i" % (source._file, window)
invertList.append([source._file,".w%i" % (window)])
args = []
args.append(globals.Variable("vis",source._file,".w%i" % (window)))
args.append(globals.Variable("options","unflagged"))
args.append(globals.Variable("out","temp." + fileEnd + "; sleep 3; rm -rf "))
args.append(globals.Variable(None,source._file,".w%i/*; rm -rf " % (window)))
args.append(globals.Variable(None,source._file,".w%i; sleep 3; mv " % (window)))
args.append(globals.Variable(None,"temp." + fileEnd))
args.append(globals.Variable(None,source._file,".w%i" % (window)))
log.run("uvcat",args,fatal=True)
else :
for window in range(globals.STARTWINDOW, globals.ENDWINDOW + 1) :
if(source._bandwidths[window - 1] == 62) :
startChan = 4
numChan = source._numChans[window - 1] - 6
invertString = invertString + ",%s.w%i" % (source._file, window)
invertList.append([source._file,".w%i" % (window)])
args = []
args.append(globals.Variable("vis",source._file,".w%i" % (window)))
args.append(globals.Variable("options","unflagged"))
args.append(globals.Variable("out","temp." + fileEnd + "; sleep 3; rm -rf "))
args.append(globals.Variable(None,source._file,".w%i/*; rm -rf " % (window)))
args.append(globals.Variable(None,source._file,".w%i; sleep 3; mv " % (window)))
args.append(globals.Variable(None,"temp." + fileEnd))
args.append(globals.Variable(None,source._file,".w%i" % (window)))
log.run("uvcat",args,fatal=True)
invertString = invertString[1:]
lastbad = 0
noise = 0.5
endings = [""]
selects=[""]
tempCell = cellsize
tempImg = imsize
if(globals.isSci2 and not individual) :
endings.append(".short")
if(globals.obsFreq() < 50.0) :
selects.append(" select=uvrange'('0,2')'")
selects.append(" select=uvrange'('2,2000000')'")
else :
selects.append(" select=uvrange'('0,6')'")
selects.append(" select=uvrange'('6,6000000')'")
endings.append(".long")
# keep inverting until we minimize the number of rejected visibilities
fileName = source._file
for i in range(0,len(endings)) :
if(endings[i].find(".short") >= 0) :
cellsize = calculations.calcCellSize(globals.avgShortBaseline(),0.0)
imsize = calculations.calcImsize(obsFreq, cellsize)
elif(endings[i].find(".long") >= 0) :
cellsize = calculations.calcCellSize(globals.avgLongBaseline(),0.0)
imsize = calculations.calcImsize(obsFreq, cellsize)
else :
cellsize = tempCell
imsize = tempImg
fName = ""
fEnd = ""
ok = False
end = endings[i]
select = selects[i]
if(individual) :
fName = fileName
fEnd = ".contin.w%i" % (windows[0])
fileName += ".contin.w%i" % (windows[0])
while(not ok) :
log.run("invert vis=%s map=%s.map beam=%s.beam imsize=%i cell=%f sup=0%s options=systemp,double,mfs,mosaic line=channel,1,%i,%i >& junk" % (invertString, fileName + end, fileName + end,imsize,cellsize,select, startChan, numChan),[],fatal=True,logit=False)
input = open("junk",'r')
fileList = input.readlines()
input.close()
found = False
while(len(fileList) > 0 and iterate) :
line = fileList.pop()
if("Visibilities rejected" in line) :
splitLine = line.split()
if(lastbad != int(splitLine[5])) :
cellsize = calculations.calcCellSize(avgBaseline,cellsize)
log.run("rm -rf %s %s; sleep 3" % (fileName + end + ".map", fileName + end + ".beam"),[],logit=False)
found = True
else :
cellsize *= 1.1
log.run("invert vis=%s map=%s.map beam=%s.beam imsize=%i cell=%f sup=0%s options=systemp,double,mfs,mosaic line=channel,1,%i,%i >& junk" % (invertString, fileName + end, fileName + end, imsize,cellsize, select, startChan, numChan),[],fatal=True,logit=False)
if("Theoretical rms" in line) :
noise = float(line.split(":")[1])
if(not found) :
ok = True
log.writeLog("Using an image size of %f for %s" % (imsize, source._name))
args = []
args.append(globals.Variable("vis",invertList[0][0],invertList[0][1]))
temp = invertList.pop(0)
for vis in invertList :
args.append(globals.Variable("ADD",vis[0],vis[1]))
invertList.append(temp)
if(individual) :
args.append(globals.Variable("map",fName,fEnd + ".map"))
args.append(globals.Variable("beam",fName,fEnd + ".beam"))
else :
args.append(globals.Variable("map",source._file,end + ".map"))
args.append(globals.Variable("beam",source._file,end + ".beam"))
args.append(globals.Variable("imsize",str(imsize)))
args.append(globals.Variable("cell",str(cellsize)))
args.append(globals.Variable("sup","0"))
if(select != "") :
args.append(globals.Variable(None,select))
args.append(globals.Variable("options","systemp,double,mfs,mosaic"))
args.append(globals.Variable("line","channel,1,%i,%i" % (startChan, numChan)))
log.run("invert",args,execute=False)
lastIter = 10000.0
# treat multipoint mosics differently
log.run("mossdi map=%s.map beam=%s.beam out=%s.clean niters=50000 cutoff=%f region=quarter" % (fileName + end, fileName + end, fileName + end,noise*2.5),[],fatal=True,logit=False)
# calculate the correst synthesized beam
log.run("mospsf beam=%s.beam out=%s.bm" % (fileName + end, fileName + end),[],fatal=True,logit=False)
log.run("imfit in=%s.bm object=beam region=arcsec,box'('-5,-5,5,5')' > fit.log.%s" % (fileName + end,fileEnd),[],fatal=True,logit=False)
input = open("fit.log.%s" % (fileEnd),'r')
fileList = input.readlines()
input.close()
majorAxis = 0.0
minorAxis = 0.0
positionAngle = 0.0
# get the beam parameters
while(len(fileList) > 0) :
line = fileList.pop()
if("Position angle" in line) :
splitLine = line.split()
positionAngle = float(splitLine[3])
elif("Minor axis" in line) :
splitLine = line.split()
minorAxis = float(splitLine[3])
elif("Major axis" in line) :
splitLine = line.split()
majorAxis = float(splitLine[3])
log.run("restor model=%s.clean map=%s.map beam=%s.beam fwhm=%f,%f pa=%f out=%s.finalmap" % (fileName + end, fileName + end, fileName + end, majorAxis,minorAxis,positionAngle, fileName + end),[],logit=False)
# get the initial noise level
region = dict()
region[0] = "%i,%i,%i,%i" % (int(math.floor(imsize/4)),int(math.floor((imsize/4) + (imsize/6))),int(math.floor((imsize/4) + (imsize/6))),int(math.floor((imsize/4) + (imsize/3))))
region[1] = "%i,%i,%i,%i" % (int(math.floor((imsize/4) + (imsize/6))),int(math.floor((imsize/4) + (imsize/3))),int(math.floor((imsize/4) + (imsize/3))),int(math.floor(3*imsize/4)))
region[2] = "%i,%i,%i,%i" % (int(math.floor((imsize/4) + (imsize/3))),int(math.floor((imsize/4) + (imsize/6))),int(math.floor(3*imsize/4)),int(math.floor((imsize/4) + (imsize/3))))
region[3] = "%i,%i,%i,%i" % (int(math.floor((imsize/4) + (imsize/6))),int(math.floor(imsize/4)),int(math.floor((imsize/4) + (imsize/3))),int(math.floor((imsize/4) + (imsize/6))))
rms = 1000000.0
best = 0
for j in range(4) :
#log.writeComment("Locating best region for noise calculation.")
log.run("imstat in=%s.finalmap region=box'('%s')' device=/NULL > rms.log.%s" % (fileName + end, region.get(j),fileEnd),[],logit=False)
input = open("rms.log.%s" % (fileEnd),'r')
fileList = input.readlines()
input.close()
fileList.reverse()
while(len(fileList) > 0) :
line = fileList.pop()
if("Frequency" in line) :
line = fileList.pop()
temprms = float(line[42:51])
if(temprms < rms) :
rms = temprms
best = j
#log.writeComment("Found region %i is best: %s" % (best, region.get(best)))
# determine the clean region either from user input, number of pointings, or automatically from the data
cleanReg = ""
if(isinstance(source,sources.Source)) :
if(source.getContinuumCleanRegion() == None) :
if(not source._mosaic) :
if(p.preferences.get("doAutoCleanRegion")) :
source.setContinuumCleanRegion(peakLocator.findCleanRegion("%s.finalmap" % (fileName + end),rms,"%s.bm" % (fileName),source._pointingCenters))
if(source.getContinuumCleanRegion() == "quarter") :
source.setCleanRegion("quarter")
elif(p.preferences.get("cleanRegion") == "quarter") :
source.setContinuumCleanRegion("quarter")
source.setCleanRegion("quarter")
else :
boxsize = int(p.preferences.get("cleanRegion"))
source.setContinuumCleanRegion("arcsec,box'(-%f,-%f,%f,%f)'" % (boxsize,boxsize,boxsize,boxsize))
source.setCleanRegion("arcsec,box'(-%f,-%f,%f,%f)'" % (boxsize,boxsize,boxsize,boxsize))
else :
if(p.preferences.get("doAutoCleanRegion")) :
print "AUTOCLEAN+++++++++++++++++++++++++++++"#################POINTING CENTERS
print source._pointingCenters
source.setContinuumCleanRegion(peakLocator.findCleanRegion("%s.finalmap" % (fileName + end),rms,"%s.bm" % (fileName + end),source._pointingCenters))
source.setCleanRegion(source.getContinuumCleanRegion())
else :
offset = [int(miriadClasses.imhead("%s.map" % fileName+end+addon,"crpix1")),int(miriadClasses.imhead("%s.map" % fileName+end+addon,"crpix2"))]
source.setContinuumCleanRegion(calculations.calculateMosaicCleanRegion(source._pointingCenters,cellsize,obsFreq,offset))
source.setCleanRegion(source.getContinuumCleanRegion())
log.writeLog("mosaic " + source.getCleanRegion())
cleanReg = source.getCleanRegion()
else :
# we are mapping a calibrator
cleanReg = "quarter"
converged = False
log.writeComment("Cleaning image to %f times the noise level, or 100000 iterations, whichever comes first" % (p.preferences.get("cleanThreshold")))
run = 1
cutoff = 1000.0
# loop over clean and restor until the noise level settles down or until 5 iterations are done
while((not converged) and (run <= 5)) :
cutoff = rms * p.preferences.get("cleanThreshold")
log.run("rm -rf %s.clean %s.finalmap; sleep 3" % (fileName + end, fileName + end),[],fatal=True,logit=False)
log.run("mossdi map=%s.map beam=%s.beam out=%s.clean niters=50000 cutoff=%f region=%s" % (fileName + end, fileName + end, fileName + end, cutoff,cleanReg),[],fatal=True,logit=False)
log.run("restor model=%s.clean map=%s.map beam=%s.beam fwhm=%f,%f pa=%f out=%s.finalmap" % (fileName + end,fileName + end, fileName + end, majorAxis,minorAxis,positionAngle,fileName + end),[],fatal=True,logit=False)
log.run("imstat in=%s.finalmap region=box'('%s')' device=/NULL > rms.log.%s" % (fileName + end, region.get(best),fileEnd),[],logit=False)
input = open("rms.log.%s" % (fileEnd),'r')
fileList = input.readlines()
input.close()
fileList.reverse()
while(len(fileList) > 0) :
line = fileList.pop()
if("Frequency" in line) :
line = fileList.pop()
temprms = float(line[42:51])
if(abs(temprms-rms) < 0.05 * rms) : # there must be less than a 5 % change
converged = True
else :
rms = temprms
run += 1
if(run > 5) :
log.writeComment("Did not reach noise level cutoff, performed 5 iterations.")
args = []
if(individual) :
args.append(globals.Variable("map",fName,fEnd + ".map"))
args.append(globals.Variable("beam",fName,fEnd + ".beam"))
args.append(globals.Variable("out",fName,fEnd + ".clean"))
else :
args.append(globals.Variable("map",source._file,end + ".map"))
args.append(globals.Variable("beam",source._file,end + ".beam"))
args.append(globals.Variable("out",source._file,end + ".clean"))
args.append(globals.Variable("niters","50000"))
args.append(globals.Variable("cutoff",str(cutoff)))
args.append(globals.Variable("region",cleanReg))
log.run("mossdi",args,execute=False)
args = []
if(individual) :
args.append(globals.Variable("model",fName,fEnd + ".clean"))
args.append(globals.Variable("map",fName,fEnd + ".map"))
args.append(globals.Variable("beam",fName,fEnd + ".beam"))
args.append(globals.Variable("out",fName,fEnd + ".finalmap"))
else :
args.append(globals.Variable("model",source._file,end + ".clean"))
args.append(globals.Variable("map",source._file,end + ".map"))
args.append(globals.Variable("beam",source._file,end + ".beam"))
args.append(globals.Variable("out",source._file,end + ".finalmap"))
args.append(globals.Variable("fwhm","%f,%f" % (majorAxis,minorAxis)))
args.append(globals.Variable("pa",str(positionAngle)))
log.run("restor",args,execute=False)
args = []
if(individual) :
args.append(globals.Variable("in",fName,fEnd + ".finalmap"))
args.append(globals.Variable("out",fName,fEnd + ".fits"))
else :
args.append(globals.Variable("in",source._file,end + ".finalmap"))
args.append(globals.Variable("out",source._file,end + ".fits"))
args.append(globals.Variable("op","xyout"))
log.run("fits",args,logit=False)
log.writeComment("Data reduction of %s complete. Final map(%s.finalmap) has a noise level of %f Jy/beam" % (source._name, fileName + end, rms))
startupTeardown.endFile("%s.finalmap" % (fileName + end))
fluxTrack = False
prjData = calculations.splitObsblockId(visFile)
project = prjData["project"]
if(project == "flux") :
fluxTrack = True
SPECTRAL = False
CONTINUUM = True
if("S" in prjData["configuration"]) :
globals.isSci2 = True
# initialize what needs to be initialized
proj = ""
startupTeardown.startup(visFile,args)
log.writeLog("Pipeline is doing %s data reduction" % (type))
# run listobs
listobs.runListobs(visFile, objects)
# get global variables
obsFreq = globals.obsFreq()
obsDate = globals.obsDate()
avgBaseline = globals.avgBaseline()
refant = globals.refant()
# determine which windows to process
if(len(objects._sources) > 0) :
numberOfWins = len(objects._sources[0]._bandwidths)
elif(len(objects._gaincals) > 0) :
numberOfWins = len(objects._gaincals[0]._bandwidths)
def cust_print(fmt, fg=None, bg=None, style=None, level="", saveLog=True):
"""
prints table of formatted text format options
"""
COLCODE = {
'k': 0, # black
'r': 1, # red
'g': 2, # green
'y': 3, # yellow
'b': 4, # blue
'm': 5, # magenta
'c': 6, # cyan
'w': 7 # white
}
FMTCODE = {
'b': 1, # bold
'f': 2, # faint
'i': 3, # italic
'u': 4, # underline
'x': 5, # blinking
'y': 6, # fast blinking
'r': 7, # reverse
'h': 8, # hide
's': 9, # strikethrough
}
# properties
props = []
if isinstance(style, str):
props = [FMTCODE[s] for s in style]
if isinstance(fg, str):
props.append(30 + COLCODE[fg])
if isinstance(bg, str):
props.append(40 + COLCODE[bg])
# display
props = ';'.join([str(x) for x in props])
if props:
print('\x1b[%sm%s\x1b[0m' % (props, fmt))
else:
print(fmt)
if saveLog:
# logging.info(fmt)
LOG_LEVEL = {
None: 'info',
'k': 'info', # black
'g': 'info', # green
'b': 'debug', # blue
'y': 'warning', # yellow
'r': 'error', # red
'm': 'critical', # magenta
'c': 'info', # cyan
'w': 'info' # white
}
# 没有指定日志级别时，以颜色进行界定日志级别
if level == "":
level = LOG_LEVEL[fg]
writeLog(fmt, level)