VERBOSE = 0 arg = getParam(args,'mode',msgLib,'verbose') print "MODE",arg if arg == 'verbose': msgLib.message("VERBOSE RUN") VERBOSE = 1 print "\n\n[INFO] script: %s" % script print "[INFO] ARG#",len(sys.argv) print "[INFO] ARGS",sys.argv # # RUN ARGUMENTS: # network = getParam(args,'net',msgLib,None) station = getParam(args,'sta',msgLib,None) location = staLib.getLocation(getParam(args,'loc',msgLib,None)) channel = getParam(args,'chan',msgLib,None) if len(channel) <3: channel += "*" # # PSD files are all HOURLY files with 50% overlap computed as part of the polarization product # date parameter of the hourly PSDs to start, it starts at hour 00:00:00 # - HOURLY files YYYY-MM-DD # dataDirectory = param.dataDirectory startDateTime = getParam(args,'start',msgLib,None).split("T")[0] # we always want to start from the begining of the day, so we discard user hours, if any startDateTime += "T00:00:00" tStart = UTCDateTime(startDateTime) startYear = tStart.strftime("%Y") startMonth = tStart.strftime("%m")
def qc3Stream(stream,segmentLength,window,sortedChannelList,channelGroups,VERBOSE): streamList = str(stream) if VERBOSE: print("[QC3-INFO] there are ",streamList,"\n\n") streamArray= streamList.split("\n") streamContent = [] for i in range(1,len(streamArray)): # first line is title streamContent.append(streamArray[i]+ "|" + str(i)) # # sort to make sure related records are one after another # streamContent = sorted(streamContent) # # extract the list, one recrd (line) at a time and group them # qcRecordList = [] previousGroupName = "" groupCount = -1 groupChannels = [] groupRecords = [] groupNames = [] staInfoList = [] timeInfoList = [] chanInfoList = [] recordInfoList = [] for i in range(len(streamContent)): # # reset the list for each record (line) # thisStaInfoList = [] thisTimeInfoList = [] thisChanInfoList = [] thisRecordInfoList = [] # # RECORD: NM.SIUC..BHE | 2009-11-01T11:00:00.019537Z - 2009-11-01T11:59:59.994537Z | 40.0 Hz, 144000 samples|1 # | | | | # staInfo timeInfo chanInfo recInfo # # from each record extract parts # staInfo,timeInfo,chanInfo,recInfo = streamContent[i].split("|") # # from each part extract list # # 0 1 2 3 # thisStaInfoList = [NET,STA,LOC,CHAN] # thisStaInfoList = staInfo.strip().split(".") thisStaInfoList[2] = staLib.getLocation(thisStaInfoList[2]) # replace blank locations with "--" # # 0 1 # thisTimeInfoList = [START,END] # thisTimeInfoList.append(timeInfo.strip().split(" - ")) # # 0 1 2 3 # thisChanInfoList = [SAMPLING,UNIT,SAMPLES,TEXT] # thisChanInfoList.append(chanInfo.strip().split(" ")) # # thisRecordInfoList = RECORD # thisRecordInfoList.append(int(recInfo.strip())) # # name each record as a channel group (do not include channel) # thisGroupName = ".".join(thisStaInfoList[ii] for ii in range(len(thisStaInfoList)-1)) # # starting the first group, start saving info # if thisGroupName != previousGroupName: groupCount +=1 if VERBOSE: print("[QC-INFO] started group",groupCount,":",thisGroupName) groupNames.append(thisGroupName) groupChannels.append([]) groupRecords.append([]) previousGroupName = thisGroupName groupChannels[groupCount].append(thisStaInfoList[-1]) # save channel names groupRecords[groupCount].append(i) # # note: the following arrays are not grouped, hence extend and not append # timeInfoList.extend(thisTimeInfoList) chanInfoList.extend(thisChanInfoList) staInfoList.extend([thisStaInfoList]) recordInfoList.extend(thisRecordInfoList) if VERBOSE: print("\n[QC3-INFO]found ",len(groupRecords)," record groups") # # QC each group # for i in range(len(groupRecords)): # # all group elements are in, start the QC # qcPassed = True if VERBOSE: print("\n[QC3-INFO] QC for record group ",groupNames[i]) # # create a sorted list of unique channels # channelList = sorted(set(groupChannels[i])) if VERBOSE: print("[QC3-INFO] channel list:",channelList) # # Missing Channels? # # - based on missing records # if len(groupRecords[i]) < 3: print("[QC3-rejected] missing channels records, received ",len(groupRecords[i])) qcPassed = False else: # # - based on channels missing from channel list # if channelList not in sortedChannelList: print("[QC3-rejected] missing channels records from",groupNames[i]," got (",channelList,"while expecting",sortedChannelList,")") qcPassed = False # # - channel list is valid # else: print("[QC3-passed] channel list complete",channelList) # # Gaps? # # this is a simple rejection based on gaps. A better choice will be to take segments and process those with sufficient length # but with 3 channels involved, this will be too complicated -- manoch 2014-04-18 # if len(groupRecords[i]) > 3: print("[QC3-check] gaps in ",groupNames[i]) qcPassed = False else: print("[QC3-passed] no gaps in",groupNames[i]) # # check for sampling rates # rec1,rec2,rec3 = list(map(int,groupRecords[i])) samplingFrequency01 = float(chanInfoList[rec1][0]) samplingFrequency02 = float(chanInfoList[rec2][0]) samplingFrequency03 = float(chanInfoList[rec3][0]) if samplingFrequency01 != samplingFrequency02 or samplingFrequency01 != samplingFrequency03: print("[QC3-rejected] sampling frequencies do not match!(",samplingFrequency01, samplingFrequency02, samplingFrequency03) qcPassed = False else: print("[QC3-passed] sampling frequencies",[samplingFrequency01,samplingFrequency02,samplingFrequency03]) # # check for mismatched start time - Note: there are exactly 3 records # delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec2][0])) delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec3][0])) samplerate = 1.0 / float(chanInfoList[rec1][0]) # # calculate number of points needed for FFT (as a power of 2) based on the run parameters # nSampNeeded = 2**int(math.log(int((float(segmentLength) / samplerate +1)/window), 2)) # make sure it is power of 2 if delay01 == 0.0 and delay02 == 0.0: print("[QC3-Passed] start time") else: if delay01 > 0.0 and delay01 < samplerate: print("[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample") elif delay01 > 0.0 and delay01 >= samplerate: print("[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more") qcPassed = False if delay02 > 0.0 and delay02 < samplerate: print("[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample") elif delay02 > 0.0 and delay02 >= samplerate: print("[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more") qcPassed = False # # check for insufficient number of samples # if qcPassed: samplesList = list(map(float,list(zip(chanInfoList[rec1],chanInfoList[rec2],chanInfoList[rec3]))[2])) if VERBOSE: print("[QC3-Info] samples:",samplesList) minimumSamples = np.min(samplesList) if minimumSamples < nSampNeeded: print("[QC3-Failed] wanted minimum of", nSampNeeded, " but got only", minimumSamples) qcPassed = False else: print("[QC3-Passed] wanted minimum of", nSampNeeded, " got ", minimumSamples) # # mismatched end time # delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec2][1])) delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec3][1])) samplerate = 1.0 / float(chanInfoList[rec1][0]) qcPassed = True if delay01 == 0.0 and delay02 == 0.0: print("[QC3-Passed] end time") # # for information only, we know we have enough samples! # else: if delay01 > 0.0 and delay01 < samplerate: print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample") elif delay01 > 0.0 and delay01 >= samplerate: print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more") if delay02 > 0.0 and delay02 < samplerate: print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample") elif delay02 > 0.0 and delay02 >= samplerate: print("[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more") # # end of the QC save qcPassed flag # if qcPassed: # # put records in proper channel order # first go through each requested channel group # for chanList in channelGroups: # # found the matching channel group? # if groupChannels[i][0] in chanList and groupChannels[i][1] in chanList and groupChannels[i][2] in chanList: print("[QC-INFO] output channel order should be",chanList) orderedGroupRecords = [-1,-1,-1] orderedGroupRecords[chanList.index(groupChannels[i][0])] = groupRecords[i][0] orderedGroupRecords[chanList.index(groupChannels[i][1])] = groupRecords[i][1] orderedGroupRecords[chanList.index(groupChannels[i][2])] = groupRecords[i][2] qcRecordList.append(orderedGroupRecords) if VERBOSE: print("[QC-INFO] passed records:",qcRecordList) return qcRecordList
VERBOSE = 0 arg = getParam(args, 'mode', msgLib, 'verbose') print "MODE", arg if arg == 'verbose': msgLib.message("VERBOSE RUN") VERBOSE = 1 print "\n\n[INFO] script: %s" % script print "[INFO] ARG#", len(sys.argv) print "[INFO] ARGS", sys.argv # # RUN ARGUMENTS: # network = getParam(args, 'net', msgLib, None) station = getParam(args, 'sta', msgLib, None) location = staLib.getLocation(getParam(args, 'loc', msgLib, None)) channel = getParam(args, 'chan', msgLib, None) if len(channel) < 3: channel += "*" # # PSD files are all HOURLY files with 50% overlap computed as part of the polarization product # date parameter of the hourly PSDs to start, it starts at hour 00:00:00 # - HOURLY files YYYY-MM-DD # dataDirectory = param.dataDirectory startDateTime = getParam(args, 'start', msgLib, None) tStart = UTCDateTime(startDateTime) startYear = tStart.strftime("%Y") startMonth = tStart.strftime("%m") startDay = tStart.strftime("%d")
irisClient = IrisClient(user_agent=msgLib.param(param, 'userAgent').userAgent) client = Client(user_agent=msgLib.param(param, 'userAgent').userAgent) action = "" # keep track of what you are doing # # RUN ARGUMENTS: # t0 = time() t1 = fileLib.timeIt("START", t0) print "\n" msgLib.message("START") inNetwork = getParam(args, 'net', msgLib, None) inStation = getParam(args, 'sta', msgLib, None) inLocation = staLib.getLocation(getParam(args, 'loc', msgLib, None)) inChannel = getParam(args, 'chan', msgLib, msgLib.param(param, 'channel').channel) maxPeriod = msgLib.param(param, 'maxT').maxT * pow( 2, msgLib.param(param, 'octaveWindowWidth').octaveWindowWidth / 2.0) # maximum period needed to compute value at maxT period point minFrequency = 1.0 / float( maxPeriod ) # minimum frequency needed to compute value at 1.0/maxT frequency point if TIMING > 0: t0 = fileLib.timeIt("ARTGS", t0) if VERBOSE > 0: print "[INFO] MAX PERIOD: " + str(msgLib.param(param, 'maxT').maxT)
client = Client(user_agent=msgLib.param(param,'userAgent').userAgent) action = "" # keep track of what you are doing # # RUN ARGUMENTS: # t0 = time() t1 = fileLib.timeIt("START",t0) print "\n" msgLib.message("START") inNetwork = getParam(args,'net',msgLib,None) inStation = getParam(args,'sta',msgLib,None) inLocation = staLib.getLocation(getParam(args,'loc',msgLib,None)) inChannel = getParam(args,'chan',msgLib,msgLib.param(param,'channel').channel) maxPeriod = msgLib.param(param,'maxT').maxT * pow(2,msgLib.param(param,'octaveWindowWidth').octaveWindowWidth/2.0) # maximum period needed to compute value at maxT period point minFrequency = 1.0/float(maxPeriod) # minimum frequency needed to compute value at 1.0/maxT frequency point if TIMING >0 : t0 = fileLib.timeIt("ARTGS",t0) if VERBOSE >0 : print "[INFO] MAX PERIOD: "+str(msgLib.param(param,'maxT').maxT) print "[INFO] CALL: "+str(sys.argv) # # less than 3 characters station name triggers wildcard #
def qc3Stream(stream,segmentLength,window,sortedChannelList,channelGroups,VERBOSE): streamList = str(stream) if VERBOSE: print "[QC3-INFO] there are ",streamList,"\n\n" streamArray= streamList.split("\n") streamContent = [] for i in xrange(1,len(streamArray)): # first line is title streamContent.append(streamArray[i]+ "|" + str(i)) # # sort to make sure related records are one after another # streamContent = sorted(streamContent) # # extract the list, one recrd (line) at a time and group them # qcRecordList = [] previousGroupName = "" groupCount = -1 groupChannels = [] groupRecords = [] groupNames = [] staInfoList = [] timeInfoList = [] chanInfoList = [] recordInfoList = [] for i in xrange(len(streamContent)): # # reset the list for each record (line) # thisStaInfoList = [] thisTimeInfoList = [] thisChanInfoList = [] thisRecordInfoList = [] # # RECORD: NM.SIUC..BHE | 2009-11-01T11:00:00.019537Z - 2009-11-01T11:59:59.994537Z | 40.0 Hz, 144000 samples|1 # | | | | # staInfo timeInfo chanInfo recInfo # # from each record extract parts # staInfo,timeInfo,chanInfo,recInfo = streamContent[i].split("|") # # from each part extract list # # 0 1 2 3 # thisStaInfoList = [NET,STA,LOC,CHAN] # thisStaInfoList = staInfo.strip().split(".") thisStaInfoList[2] = staLib.getLocation(thisStaInfoList[2]) # replace blank locations with "--" # # 0 1 # thisTimeInfoList = [START,END] # thisTimeInfoList.append(timeInfo.strip().split(" - ")) # # 0 1 2 3 # thisChanInfoList = [SAMPLING,UNIT,SAMPLES,TEXT] # thisChanInfoList.append(chanInfo.strip().split(" ")) # # thisRecordInfoList = RECORD # thisRecordInfoList.append(int(recInfo.strip())) # # name each record as a channel group (do not include channel) # thisGroupName = ".".join(thisStaInfoList[ii] for ii in xrange(len(thisStaInfoList)-1)) # # starting the first group, start saving info # if thisGroupName != previousGroupName: groupCount +=1 if VERBOSE: print "[QC-INFO] started group",groupCount,":",thisGroupName groupNames.append(thisGroupName) groupChannels.append([]) groupRecords.append([]) previousGroupName = thisGroupName groupChannels[groupCount].append(thisStaInfoList[-1]) # save channel names groupRecords[groupCount].append(i) # # note: the following arrays are not grouped, hence extend and not append # timeInfoList.extend(thisTimeInfoList) chanInfoList.extend(thisChanInfoList) staInfoList.extend([thisStaInfoList]) recordInfoList.extend(thisRecordInfoList) if VERBOSE: print "\n[QC3-INFO]found ",len(groupRecords)," record groups" # # QC each group # for i in xrange(len(groupRecords)): # # all group elements are in, start the QC # qcPassed = True if VERBOSE: print "\n[QC3-INFO] QC for record group ",groupNames[i] # # create a sorted list of unique channels # channelList = sorted(set(groupChannels[i])) if VERBOSE: print "[QC3-INFO] channel list:",channelList # # Missing Channels? # # - based on missing records # if len(groupRecords[i]) < 3: print "[QC3-rejected] missing channels records, received ",len(groupRecords[i]) qcPassed = False else: # # - based on channels missing from channel list # if channelList not in sortedChannelList: print "[QC3-rejected] missing channels records from",groupNames[i]," got (",channelList,"while expecting",sortedChannelList,")" qcPassed = False # # - channel list is valid # else: print "[QC3-passed] channel list complete",channelList # # Gaps? # # this is a simple rejection based on gaps. A better choice will be to take segments and process those with sufficient length # but with 3 channels involved, this will be too complicated -- manoch 2014-04-18 # if len(groupRecords[i]) > 3: print "[QC3-check] gaps in ",groupNames[i] qcPassed = False else: print "[QC3-passed] no gaps in",groupNames[i] # # check for sampling rates # rec1,rec2,rec3 = map(int,groupRecords[i]) samplingFrequency01 = float(chanInfoList[rec1][0]) samplingFrequency02 = float(chanInfoList[rec2][0]) samplingFrequency03 = float(chanInfoList[rec3][0]) if samplingFrequency01 != samplingFrequency02 or samplingFrequency01 != samplingFrequency03: print "[QC3-rejected] sampling frequencies do not match!(",samplingFrequency01, samplingFrequency02, samplingFrequency03 qcPassed = False else: print "[QC3-passed] sampling frequencies",[samplingFrequency01,samplingFrequency02,samplingFrequency03] # # check for mismatched start time - Note: there are exactly 3 records # delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec2][0])) delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][0]) - utcdatetime.UTCDateTime(timeInfoList[rec3][0])) samplerate = 1.0 / float(chanInfoList[rec1][0]) # # calculate number of points needed for FFT (as a power of 2) based on the run parameters # nSampNeeded = 2**int(math.log(int((float(segmentLength) / samplerate +1)/window), 2)) # make sure it is power of 2 if delay01 == 0.0 and delay02 == 0.0: print "[QC3-Passed] start time" else: if delay01 > 0.0 and delay01 < samplerate: print "[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample" elif delay01 > 0.0 and delay01 >= samplerate: print "[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more" qcPassed = False if delay02 > 0.0 and delay02 < samplerate: print "[QC3-Passed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample" elif delay02 > 0.0 and delay02 >= samplerate: print "[QC3-Failed] start time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more" qcPassed = False # # check for insufficient number of samples # if qcPassed: samplesList = map(float,zip(chanInfoList[rec1],chanInfoList[rec2],chanInfoList[rec3])[2]) if VERBOSE: print "[QC3-Info] samples:",samplesList minimumSamples = np.min(samplesList) if minimumSamples < nSampNeeded: print "[QC3-Failed] wanted minimum of", nSampNeeded, " but got only", minimumSamples qcPassed = False else: print "[QC3-Passed] wanted minimum of", nSampNeeded, " got ", minimumSamples # # mismatched end time # delay01 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec2][1])) delay02 = np.abs(utcdatetime.UTCDateTime(timeInfoList[rec1][1]) - utcdatetime.UTCDateTime(timeInfoList[rec3][1])) samplerate = 1.0 / float(chanInfoList[rec1][0]) qcPassed = True if delay01 == 0.0 and delay02 == 0.0: print "[QC3-Passed] end time" # # for information only, we know we have enough samples! # else: if delay01 > 0.0 and delay01 < samplerate: print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, less than 1 sample" elif delay01 > 0.0 and delay01 >= samplerate: print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec2]), "is",delay01,"s, is 1 sample or more" if delay02 > 0.0 and delay02 < samplerate: print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, less than 1 sample" elif delay02 > 0.0 and delay02 >= samplerate: print "[QC3-Info] end time difference between", '.'.join(staInfoList[rec1]), "and", '.'.join(staInfoList[rec3]), "is",delay02,"s, is 1 sample or more" # # end of the QC save qcPassed flag # if qcPassed: # # put records in proper channel order # first go through each requested channel group # for chanList in channelGroups: # # found the matching channel group? # if groupChannels[i][0] in chanList and groupChannels[i][1] in chanList and groupChannels[i][2] in chanList: print "[QC-INFO] output channel order should be",chanList orderedGroupRecords = [-1,-1,-1] orderedGroupRecords[chanList.index(groupChannels[i][0])] = groupRecords[i][0] orderedGroupRecords[chanList.index(groupChannels[i][1])] = groupRecords[i][1] orderedGroupRecords[chanList.index(groupChannels[i][2])] = groupRecords[i][2] qcRecordList.append(orderedGroupRecords) if VERBOSE: print "[QC-INFO] passed records:",qcRecordList return qcRecordList