VERBOSE = 0
arg = getParam(args,'mode',msgLib,'verbose')
print "MODE",arg
if arg == 'verbose':
msgLib.message("VERBOSE RUN")
VERBOSE = 1
print "\n\n[INFO] script: %s" % script
print "[INFO] ARG#",len(sys.argv)
print "[INFO] ARGS",sys.argv
#
# RUN ARGUMENTS:
#
network = getParam(args,'net',msgLib,None)
station = getParam(args,'sta',msgLib,None)
location = staLib.getLocation(getParam(args,'loc',msgLib,None))
channel = getParam(args,'chan',msgLib,None)
if len(channel) <3:
channel += "*"
#
# PSD files are all HOURLY files with 50% overlap computed as part of the polarization product
# date parameter of the hourly PSDs to start, it starts at hour 00:00:00
# - HOURLY files YYYY-MM-DD
#
dataDirectory = param.dataDirectory
startDateTime = getParam(args,'start',msgLib,None).split("T")[0] # we always want to start from the begining of the day, so we discard user hours, if any
startDateTime += "T00:00:00"
tStart = UTCDateTime(startDateTime)
startYear = tStart.strftime("%Y")
startMonth = tStart.strftime("%m")
def qc3Stream(stream,segmentLength,window,sortedChannelList,channelGroups,VERBOSE):
streamList = str(stream)
if VERBOSE:
print("[QC3-INFO] there are ",streamList,"\n\n")
streamArray= streamList.split("\n")
streamContent = []
for i in range(1,len(streamArray)): # first line is title
streamContent.append(streamArray[i]+ "|" + str(i))
#
# sort to make sure related records are one after another
#
streamContent = sorted(streamContent)
#
# extract the list, one recrd (line) at a time and group them
#
qcRecordList = []
previousGroupName = ""
groupCount = -1
groupChannels = []
groupRecords = []
groupNames = []
staInfoList = []
timeInfoList = []
chanInfoList = []
recordInfoList = []
for i in range(len(streamContent)):
#
# reset the list for each record (line)
#
thisStaInfoList = []
thisTimeInfoList = []
thisChanInfoList = []
thisRecordInfoList = []
#
# RECORD: NM.SIUC..BHE | 2009-11-01T11:00:00.019537Z - 2009-11-01T11:59:59.994537Z | 40.0 Hz, 144000 samples|1
# | | | |
# staInfo timeInfo chanInfo recInfo
#
# from each record extract parts
#
staInfo,timeInfo,chanInfo,recInfo = streamContent[i].split("|")
#
# from each part extract list
#
# 0 1 2 3
# thisStaInfoList = [NET,STA,LOC,CHAN]
#
thisStaInfoList = staInfo.strip().split(".")
thisStaInfoList[2] = staLib.getLocation(thisStaInfoList[2]) # replace blank locations with "--"
#
# 0 1
# thisTimeInfoList = [START,END]
#
thisTimeInfoList.append(timeInfo.strip().split(" - "))
#
# 0 1 2 3
# thisChanInfoList = [SAMPLING,UNIT,SAMPLES,TEXT]
#
thisChanInfoList.append(chanInfo.strip().split(" "))
#
# thisRecordInfoList = RECORD
#
thisRecordInfoList.append(int(recInfo.strip()))
#
# name each record as a channel group (do not include channel)
#
thisGroupName = ".".join(thisStaInfoList[ii] for ii in range(len(thisStaInfoList)-1))
#
# starting the first group, start saving info
#
if thisGroupName != previousGroupName:
groupCount +=1
if VERBOSE:
print("[QC-INFO] started group",groupCount,":",thisGroupName)
groupNames.append(thisGroupName)
groupChannels.append([])
groupRecords.append([])
previousGroupName = thisGroupName
groupChannels[groupCount].append(thisStaInfoList[-1]) # save channel names
groupRecords[groupCount].append(i)
#
# note: the following arrays are not grouped, hence extend and not append
#
timeInfoList.extend(thisTimeInfoList)
chanInfoList.extend(thisChanInfoList)
staInfoList.extend([thisStaInfoList])
recordInfoList.extend(thisRecordInfoList)
if VERBOSE:
print("\n[QC3-INFO]found ",len(groupRecords)," record groups")
#
# QC each group
#
for i in range(len(groupRecords)):
#
# all group elements are in, start the QC
#
qcPassed = True
if VERBOSE:
print("\n[QC3-INFO] QC for record group ",groupNames[i])
#
# create a sorted list of unique channels
#
channelList = sorted(set(groupChannels[i]))
if VERBOSE:
print("[QC3-INFO] channel list:",channelList)
#
# Missing Channels?
#
# - based on missing records
#
if len(groupRecords[i]) < 3:
print("[QC3-rejected] missing channels records, received ",len(groupRecords[i]))
qcPassed = False
else:
#
# - based on channels missing from channel list
#
if channelList not in sortedChannelList:
print("[QC3-rejected] missing channels records from",groupNames[i]," got (",channelList,"while expecting",sortedChannelList,")")
qcPassed = False
#
# - channel list is valid
#
else:
print("[QC3-passed] channel list complete",channelList)
#
# Gaps?
#
# this is a simple rejection based on gaps. A better choice will be to take segments and process those with sufficient length
# but with 3 channels involved, this will be too complicated -- manoch 2014-04-18
#
if len(groupRecords[i]) > 3:
print("[QC3-check] gaps in ",groupNames[i])
qcPassed = False
else:
print("[QC3-passed] no gaps in",groupNames[i])
#
# check for sampling rates
#
rec1,rec2,rec3 = list(map(int,groupRecords[i]))
samplingFrequency01 = float(chanInfoList[rec1][0])
samplingFrequency02 = float(chanInfoList[rec2][0])
samplingFrequency03 = float(chanInfoList[rec3][0])
if samplingFrequency01 != samplingFrequency02 or samplingFrequency01 != samplingFrequency03:
print("[QC3-rejected] sampling frequencies do not match!(",samplingFrequency01, samplingFrequency02, samplingFrequency03)
qcPassed = False
else:
print("[QC3-passed] sampling frequencies",[samplingFrequency01,samplingFrequency02,samplingFrequency03])
#
# check for mismatched start time - Note: there are exactly 3 records
#
delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec2][0]))
delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec3][0]))
samplerate = 1.0 / float(chanInfoList[rec1][0])
#
# calculate number of points needed for FFT (as a power of 2) based on the run parameters
#
nSampNeeded = 2**int(math.log(int((float(segmentLength) / samplerate +1)/window), 2)) # make sure it is power of 2
if delay01 == 0.0 and delay02 == 0.0:
print("[QC3-Passed] start time")
else:
if delay01 > 0.0 and delay01 < samplerate:
print("[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample")
elif delay01 > 0.0 and delay01 >= samplerate:
print("[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more")
qcPassed = False
if delay02 > 0.0 and delay02 < samplerate:
print("[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample")
elif delay02 > 0.0 and delay02 >= samplerate:
print("[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more")
qcPassed = False
#
# check for insufficient number of samples
#
if qcPassed:
samplesList = list(map(float,list(zip(chanInfoList[rec1],chanInfoList[rec2],chanInfoList[rec3]))[2]))
if VERBOSE:
print("[QC3-Info] samples:",samplesList)
minimumSamples = np.min(samplesList)
if minimumSamples < nSampNeeded:
print("[QC3-Failed] wanted minimum of", nSampNeeded, " but got only", minimumSamples)
qcPassed = False
else:
print("[QC3-Passed] wanted minimum of", nSampNeeded, " got ", minimumSamples)
#
# mismatched end time
#
delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec2][1]))
delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec3][1]))
samplerate = 1.0 / float(chanInfoList[rec1][0])
qcPassed = True
if delay01 == 0.0 and delay02 == 0.0:
print("[QC3-Passed] end time")
#
# for information only, we know we have enough samples!
#
else:
if delay01 > 0.0 and delay01 < samplerate:
print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample")
elif delay01 > 0.0 and delay01 >= samplerate:
print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more")
if delay02 > 0.0 and delay02 < samplerate:
print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample")
elif delay02 > 0.0 and delay02 >= samplerate:
print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more")
#
# end of the QC save qcPassed flag
#
if qcPassed:
#
# put records in proper channel order
# first go through each requested channel group
#
for chanList in channelGroups:
#
# found the matching channel group?
#
if groupChannels[i][0] in chanList and groupChannels[i][1] in chanList and groupChannels[i][2] in chanList:
print("[QC-INFO] output channel order should be",chanList)
orderedGroupRecords = [-1,-1,-1]
orderedGroupRecords[chanList.index(groupChannels[i][0])] = groupRecords[i][0]
orderedGroupRecords[chanList.index(groupChannels[i][1])] = groupRecords[i][1]
orderedGroupRecords[chanList.index(groupChannels[i][2])] = groupRecords[i][2]
qcRecordList.append(orderedGroupRecords)
if VERBOSE:
print("[QC-INFO] passed records:",qcRecordList)
return qcRecordList
VERBOSE = 0
arg = getParam(args, 'mode', msgLib, 'verbose')
print "MODE", arg
if arg == 'verbose':
msgLib.message("VERBOSE RUN")
VERBOSE = 1
print "\n\n[INFO] script: %s" % script
print "[INFO] ARG#", len(sys.argv)
print "[INFO] ARGS", sys.argv
#
# RUN ARGUMENTS:
#
network = getParam(args, 'net', msgLib, None)
station = getParam(args, 'sta', msgLib, None)
location = staLib.getLocation(getParam(args, 'loc', msgLib, None))
channel = getParam(args, 'chan', msgLib, None)
if len(channel) < 3:
channel += "*"
#
# PSD files are all HOURLY files with 50% overlap computed as part of the polarization product
# date parameter of the hourly PSDs to start, it starts at hour 00:00:00
# - HOURLY files YYYY-MM-DD
#
dataDirectory = param.dataDirectory
startDateTime = getParam(args, 'start', msgLib, None)
tStart = UTCDateTime(startDateTime)
startYear = tStart.strftime("%Y")
startMonth = tStart.strftime("%m")
startDay = tStart.strftime("%d")
irisClient = IrisClient(user_agent=msgLib.param(param, 'userAgent').userAgent)
client = Client(user_agent=msgLib.param(param, 'userAgent').userAgent)
action = "" # keep track of what you are doing
#
# RUN ARGUMENTS:
#
t0 = time()
t1 = fileLib.timeIt("START", t0)
print "\n"
msgLib.message("START")
inNetwork = getParam(args, 'net', msgLib, None)
inStation = getParam(args, 'sta', msgLib, None)
inLocation = staLib.getLocation(getParam(args, 'loc', msgLib, None))
inChannel = getParam(args, 'chan', msgLib,
msgLib.param(param, 'channel').channel)
maxPeriod = msgLib.param(param, 'maxT').maxT * pow(
2,
msgLib.param(param, 'octaveWindowWidth').octaveWindowWidth /
2.0) # maximum period needed to compute value at maxT period point
minFrequency = 1.0 / float(
maxPeriod
) # minimum frequency needed to compute value at 1.0/maxT frequency point
if TIMING > 0:
t0 = fileLib.timeIt("ARTGS", t0)
if VERBOSE > 0:
print "[INFO] MAX PERIOD: " + str(msgLib.param(param, 'maxT').maxT)
client = Client(user_agent=msgLib.param(param,'userAgent').userAgent)
action = "" # keep track of what you are doing
#
# RUN ARGUMENTS:
#
t0 = time()
t1 = fileLib.timeIt("START",t0)
print "\n"
msgLib.message("START")
inNetwork = getParam(args,'net',msgLib,None)
inStation = getParam(args,'sta',msgLib,None)
inLocation = staLib.getLocation(getParam(args,'loc',msgLib,None))
inChannel = getParam(args,'chan',msgLib,msgLib.param(param,'channel').channel)
maxPeriod = msgLib.param(param,'maxT').maxT * pow(2,msgLib.param(param,'octaveWindowWidth').octaveWindowWidth/2.0) # maximum period needed to compute value at maxT period point
minFrequency = 1.0/float(maxPeriod) # minimum frequency needed to compute value at 1.0/maxT frequency point
if TIMING >0 :
t0 = fileLib.timeIt("ARTGS",t0)
if VERBOSE >0 :
print "[INFO] MAX PERIOD: "+str(msgLib.param(param,'maxT').maxT)
print "[INFO] CALL: "+str(sys.argv)
#
# less than 3 characters station name triggers wildcard
#
def qc3Stream(stream,segmentLength,window,sortedChannelList,channelGroups,VERBOSE):
streamList = str(stream)
if VERBOSE:
print "[QC3-INFO] there are ",streamList,"\n\n"
streamArray= streamList.split("\n")
streamContent = []
for i in xrange(1,len(streamArray)): # first line is title
streamContent.append(streamArray[i]+ "|" + str(i))
#
# sort to make sure related records are one after another
#
streamContent = sorted(streamContent)
#
# extract the list, one recrd (line) at a time and group them
#
qcRecordList = []
previousGroupName = ""
groupCount = -1
groupChannels = []
groupRecords = []
groupNames = []
staInfoList = []
timeInfoList = []
chanInfoList = []
recordInfoList = []
for i in xrange(len(streamContent)):
#
# reset the list for each record (line)
#
thisStaInfoList = []
thisTimeInfoList = []
thisChanInfoList = []
thisRecordInfoList = []
#
# RECORD: NM.SIUC..BHE | 2009-11-01T11:00:00.019537Z - 2009-11-01T11:59:59.994537Z | 40.0 Hz, 144000 samples|1
# | | | |
# staInfo timeInfo chanInfo recInfo
#
# from each record extract parts
#
staInfo,timeInfo,chanInfo,recInfo = streamContent[i].split("|")
#
# from each part extract list
#
# 0 1 2 3
# thisStaInfoList = [NET,STA,LOC,CHAN]
#
thisStaInfoList = staInfo.strip().split(".")
thisStaInfoList[2] = staLib.getLocation(thisStaInfoList[2]) # replace blank locations with "--"
#
# 0 1
# thisTimeInfoList = [START,END]
#
thisTimeInfoList.append(timeInfo.strip().split(" - "))
#
# 0 1 2 3
# thisChanInfoList = [SAMPLING,UNIT,SAMPLES,TEXT]
#
thisChanInfoList.append(chanInfo.strip().split(" "))
#
# thisRecordInfoList = RECORD
#
thisRecordInfoList.append(int(recInfo.strip()))
#
# name each record as a channel group (do not include channel)
#
thisGroupName = ".".join(thisStaInfoList[ii] for ii in xrange(len(thisStaInfoList)-1))
#
# starting the first group, start saving info
#
if thisGroupName != previousGroupName:
groupCount +=1
if VERBOSE:
print "[QC-INFO] started group",groupCount,":",thisGroupName
groupNames.append(thisGroupName)
groupChannels.append([])
groupRecords.append([])
previousGroupName = thisGroupName
groupChannels[groupCount].append(thisStaInfoList[-1]) # save channel names
groupRecords[groupCount].append(i)
#
# note: the following arrays are not grouped, hence extend and not append
#
timeInfoList.extend(thisTimeInfoList)
chanInfoList.extend(thisChanInfoList)
staInfoList.extend([thisStaInfoList])
recordInfoList.extend(thisRecordInfoList)
if VERBOSE:
print "\n[QC3-INFO]found ",len(groupRecords)," record groups"
#
# QC each group
#
for i in xrange(len(groupRecords)):
#
# all group elements are in, start the QC
#
qcPassed = True
if VERBOSE:
print "\n[QC3-INFO] QC for record group ",groupNames[i]
#
# create a sorted list of unique channels
#
channelList = sorted(set(groupChannels[i]))
if VERBOSE:
print "[QC3-INFO] channel list:",channelList
#
# Missing Channels?
#
# - based on missing records
#
if len(groupRecords[i]) < 3:
print "[QC3-rejected] missing channels records, received ",len(groupRecords[i])
qcPassed = False
else:
#
# - based on channels missing from channel list
#
if channelList not in sortedChannelList:
print "[QC3-rejected] missing channels records from",groupNames[i]," got (",channelList,"while expecting",sortedChannelList,")"
qcPassed = False
#
# - channel list is valid
#
else:
print "[QC3-passed] channel list complete",channelList
#
# Gaps?
#
# this is a simple rejection based on gaps. A better choice will be to take segments and process those with sufficient length
# but with 3 channels involved, this will be too complicated -- manoch 2014-04-18
#
if len(groupRecords[i]) > 3:
print "[QC3-check] gaps in ",groupNames[i]
qcPassed = False
else:
print "[QC3-passed] no gaps in",groupNames[i]
#
# check for sampling rates
#
rec1,rec2,rec3 = map(int,groupRecords[i])
samplingFrequency01 = float(chanInfoList[rec1][0])
samplingFrequency02 = float(chanInfoList[rec2][0])
samplingFrequency03 = float(chanInfoList[rec3][0])
if samplingFrequency01 != samplingFrequency02 or samplingFrequency01 != samplingFrequency03:
print "[QC3-rejected] sampling frequencies do not match!(",samplingFrequency01, samplingFrequency02, samplingFrequency03
qcPassed = False
else:
print "[QC3-passed] sampling frequencies",[samplingFrequency01,samplingFrequency02,samplingFrequency03]
#
# check for mismatched start time - Note: there are exactly 3 records
#
delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec2][0]))
delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec3][0]))
samplerate = 1.0 / float(chanInfoList[rec1][0])
#
# calculate number of points needed for FFT (as a power of 2) based on the run parameters
#
nSampNeeded = 2**int(math.log(int((float(segmentLength) / samplerate +1)/window), 2)) # make sure it is power of 2
if delay01 == 0.0 and delay02 == 0.0:
print "[QC3-Passed] start time"
else:
if delay01 > 0.0 and delay01 < samplerate:
print "[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample"
elif delay01 > 0.0 and delay01 >= samplerate:
print "[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more"
qcPassed = False
if delay02 > 0.0 and delay02 < samplerate:
print "[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample"
elif delay02 > 0.0 and delay02 >= samplerate:
print "[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more"
qcPassed = False
#
# check for insufficient number of samples
#
if qcPassed:
samplesList = map(float,zip(chanInfoList[rec1],chanInfoList[rec2],chanInfoList[rec3])[2])
if VERBOSE:
print "[QC3-Info] samples:",samplesList
minimumSamples = np.min(samplesList)
if minimumSamples < nSampNeeded:
print "[QC3-Failed] wanted minimum of", nSampNeeded, " but got only", minimumSamples
qcPassed = False
else:
print "[QC3-Passed] wanted minimum of", nSampNeeded, " got ", minimumSamples
#
# mismatched end time
#
delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec2][1]))
delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec3][1]))
samplerate = 1.0 / float(chanInfoList[rec1][0])
qcPassed = True
if delay01 == 0.0 and delay02 == 0.0:
print "[QC3-Passed] end time"
#
# for information only, we know we have enough samples!
#
else:
if delay01 > 0.0 and delay01 < samplerate:
print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample"
elif delay01 > 0.0 and delay01 >= samplerate:
print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more"
if delay02 > 0.0 and delay02 < samplerate:
print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample"
elif delay02 > 0.0 and delay02 >= samplerate:
print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more"
#
# end of the QC save qcPassed flag
#
if qcPassed:
#
# put records in proper channel order
# first go through each requested channel group
#
for chanList in channelGroups:
#
# found the matching channel group?
#
if groupChannels[i][0] in chanList and groupChannels[i][1] in chanList and groupChannels[i][2] in chanList:
print "[QC-INFO] output channel order should be",chanList
orderedGroupRecords = [-1,-1,-1]
orderedGroupRecords[chanList.index(groupChannels[i][0])] = groupRecords[i][0]
orderedGroupRecords[chanList.index(groupChannels[i][1])] = groupRecords[i][1]
orderedGroupRecords[chanList.index(groupChannels[i][2])] = groupRecords[i][2]
qcRecordList.append(orderedGroupRecords)
if VERBOSE:
print "[QC-INFO] passed records:",qcRecordList
return qcRecordList
