class ParallelDeconvFilter(object): def __init__(self): # Initialize kill object for class self.killproc = Kill() def deconvFilter(self, stream, response, filters): # ---------------------------------------- # Deconvolve/filter each station, filters # are based on channel IDs # ---------------------------------------- streamID = stream[0].getId() tmpstr = re.split("\\.", streamID) networkID = tmpstr[0].strip() stationID = tmpstr[1].strip() respID = response['filename'].strip() netstatID = networkID + stationID # Get filter types from filters[{},{},...] if streamID in filters['streamID']: filtertype = filters['filtertype'] freqX = filters['freqX'] freqY = filters['freqY'] # Try/catch block for sensitivity subprocess try: print "Stream/Filter: " + netstatID + " / " + str(filtertype) # Deconvolution (removes sensitivity) sensitivity = "Sensitivity:" # pull sensitivity from RESP file grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " + respID + " | tail -1") self.subprocess = True # flag for exceptions (if !subprocess return) subproc = subprocess.Popen([grepSensitivity], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (out, err) = subproc.communicate(timeout=10) # waits for child proc # Store/print pids for exception kills self.parentpid = os.getppid() self.childpid = os.getpid() self.gchildpid = subproc.pid # Pull sensitivity from subproc tmps = out.strip() tmps = re.split(':', tmps) s = float(tmps[1].strip()) sys.stdout.flush() sys.stderr.flush() self.subprocess = False # subprocess finished # deconvolution (this will be a flag for the user) # stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4), # seedresp=response, taper='True') # Remove transient response and decimate signals to SR=1Hz decfactor = int(stream[0].stats.sampling_rate) stream.detrend('demean') # removes mean in data set #stream.taper(max_percentage=0.01/2.0, type='cosine') # cos tapers beginning/end to remove transient resp stream.decimate(decfactor, no_filter=True, strict_length=False) # Filter stream based on channel (remove sensitivity) if filtertype == "bandpass": print "Bandpass filter: %.3f-%.3fHz" % (freqX, freqY) maxval = np.amax(stream[0].data) stream.filter(filtertype, freqmin=freqX, freqmax=freqY, corners=4) # bp filter stream[0].data = stream[0].data / s elif filtertype == "bandstop": print "Bandstop filter: %.3f-%.3fHz" % (freqX, freqY) maxval = np.amax(stream[0].data) stream.filter(filtertype, freqmin=freqX, freqmax=freqY, corners=4) # notch filter stream[0].data = stream[0].data / s elif filtertype == "lowpass": print "Lowpass filter: %.2f" % freqX stream.filter(filtertype, freq=freqX, corners=4) # lp filter stream[0].data = stream[0].data / s elif filtertype == "highpass": print "Highpass filter: %.2f" % freqX stream.filter(filtertype, freq=freqX, corners=4) # hp filter stream[0].data = stream[0].data / s print "Filtered stream: " + str(stream) + "\n" return stream except subprocess.TimeoutExpired: print "TimeoutExpired deconvFilter(): terminate workers..." if self.subprocess: signum = signal.SIGKILL killargs = {'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum} self.killproc.killSubprocess(**killargs) sys.stdout.flush() sys.stdout.flush() raise TimeoutExpiredError() return # return to deconvFilter pool except KeyboardInterrupt: print "KeyboardInterrupt deconvFilter(): terminate workers..." if self.subprocess: signum = signal.SIGKILL killargs = {'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum} self.killproc.killSubprocess(**killargs) raise KeyboardInterruptError() return except Exception as e: print "UnknownException deconvFilter(): " + str(e) if self.subprocess: signum = signal.SIGKILL killargs = {'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum} self.killproc.killSubprocess(**killargs) return def launchWorkers(self, stream, streamlen, response, filters): # --------------------------------- # Simulate/filter queried stations # --------------------------------- print "-------deconvFilter() Pool-------\n" # Merge traces to eliminate small data lengths, # method 0 => no overlap of traces (i.e. overwriting # of previous trace data, gaps fill overlaps) # method 1 => fill overlaps using interpolation for # values between both vectors for x num of samples for i in range(streamlen): try: stream[i].merge(method=1, fill_value='interpolate', interpolation_samples=100) except Exception, e: print 'Error merging traces:', e # Deconvolution/Prefilter # Initialize multiprocessing pools PROCESSES = multiprocessing.cpu_count() print "PROCESSES: " + str(PROCESSES) print "streamlen: " + str(streamlen) + "\n" pool = multiprocessing.Pool(PROCESSES) try: self.poolpid = os.getpid() self.poolname = "deconvFilter()" flt_streams = pool.map(unwrap_self_deconvFilter, zip([self]*streamlen, stream, response, filters)) pool.close() pool.join() self.flt_streams = flt_streams print "-------deconvFilter() Pool Complete-------\n\n" except TimeoutExpiredError: print "\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..." # find/kill all child processes killargs = {'pid': self.poolpid, 'name': self.poolname} self.killproc.killPool(**killargs) except KeyboardInterrupt: print "\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..." killargs = {'pid': self.poolpid, 'name': self.poolname} self.killproc.killPool(**killargs) else: # cleanup (close pool of workers) pool.close() pool.join()
class ParallelDeconvFilter(object): def __init__(self): # Initialize kill object for class self.killproc = Kill() def deconvFilter(self, stream, response, filters): # ---------------------------------------- # Deconvolve/filter each station, filters # are based on channel IDs # ---------------------------------------- streamID = stream[0].get_id() tmpstr = re.split("\\.", streamID) networkID = tmpstr[0].strip() stationID = tmpstr[1].strip() respID = response['filename'].strip() netstatID = networkID + stationID # Get filter types from filters[{},{},...] if streamID in filters['streamID']: filtertype = filters['filtertype'] freqX = filters['freqX'] freqY = filters['freqY'] # Try/catch block for sensitivity subprocess try: #print("Stream/Filter: " + netstatID + " / " + str(filtertype)) # Deconvolution (removes sensitivity) sensitivity = "Sensitivity:" # pull sensitivity from RESP file grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " + respID + " | tail -1") self.subprocess = True # flag for exceptions (if !subprocess return) subproc = subprocess.Popen([grepSensitivity], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (out, err) = subproc.communicate(timeout=10) # waits for child proc out = out.decode("utf-8") out = str(out) # Store/print pids for exception kills self.parentpid = os.getppid() self.childpid = os.getpid() self.gchildpid = subproc.pid # Pull sensitivity from subproc tmps = out.strip() tmps = re.split(':', tmps) tmps = tmps[1] tmps = tmps.replace(' ', '') tmps = tmps.replace("'", "") s = float(tmps) print("Stream/Filter: " + netstatID + " / " + str(filtertype) + " Sensitivity: " + str(s)) sys.stdout.flush() sys.stderr.flush() self.subprocess = False # subprocess finished # deconvolution (this will be a flag for the user) # stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4), # seedresp=response, taper='True') # Remove transient response and decimate signals to SR=1Hz decfactor = int(stream[0].stats.sampling_rate) stream.detrend('demean') # removes mean in data set #stream.taper(max_percentage=0.01/2.0, type='cosine') # cos tapers beginning/end to remove transient resp stream.decimate(decfactor, no_filter=True, strict_length=False) # Filter stream based on channel (remove sensitivity) if filtertype == "bandpass": print("Bandpass filter: %.3f-%.3fHz" % (freqX, freqY)) maxval = np.amax(stream[0].data) stream.filter(filtertype, freqmin=freqX, freqmax=freqY, corners=4) # bp filter stream[0].data = stream[0].data / s elif filtertype == "bandstop": print("Bandstop filter: %.3f-%.3fHz" % (freqX, freqY)) maxval = np.amax(stream[0].data) stream.filter(filtertype, freqmin=freqX, freqmax=freqY, corners=4) # notch filter stream[0].data = stream[0].data / s elif filtertype == "lowpass": print("Lowpass filter: %.2f" % freqX) stream.filter(filtertype, freq=freqX, corners=4) # lp filter stream[0].data = stream[0].data / s elif filtertype == "highpass": print("Highpass filter: %.2f" % freqX) stream.filter(filtertype, freq=freqX, corners=4) # hp filter stream[0].data = stream[0].data / s print("Filtered stream: " + str(stream) + "\n") return stream except subprocess.TimeoutExpired: print("TimeoutExpired deconvFilter(): terminate workers...") if self.subprocess: signum = signal.SIGKILL killargs = { 'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum } self.killproc.killSubprocess(**killargs) sys.stdout.flush() sys.stdout.flush() raise TimeoutExpiredError() return # return to deconvFilter pool except KeyboardInterrupt: print("KeyboardInterrupt deconvFilter(): terminate workers...") if self.subprocess: signum = signal.SIGKILL killargs = { 'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum } self.killproc.killSubprocess(**killargs) raise KeyboardInterruptError() return except Exception as e: PrintException() print("UnknownException deconvFilter(): " + str(e)) if self.subprocess: signum = signal.SIGKILL killargs = { 'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum } self.killproc.killSubprocess(**killargs) return def launchWorkers(self, stream, streamlen, response, filters): # --------------------------------- # Simulate/filter queried stations # --------------------------------- print("-------deconvFilter() Pool-------\n") # Merge traces to eliminate small data lengths, # method 0 => no overlap of traces (i.e. overwriting # of previous trace data, gaps fill overlaps) # method 1 => fill overlaps using interpolation for # values between both vectors for x num of samples for i in range(streamlen): try: stream[i].merge(method=1, fill_value='interpolate', interpolation_samples=100) except Exception as e: print('Error merging traces:', e) # Deconvolution/Prefilter # Initialize multiprocessing pools PROCESSES = multiprocessing.cpu_count() print("PROCESSES: " + str(PROCESSES)) print("streamlen: " + str(streamlen) + "\n") pool = multiprocessing.Pool(PROCESSES) try: self.poolpid = os.getpid() self.poolname = "deconvFilter()" flt_streams = pool.map( unwrap_self_deconvFilter, list(zip([self] * streamlen, stream, response, filters))) pool.close() pool.join() self.flt_streams = flt_streams print("-------deconvFilter() Pool Complete-------\n\n") except TimeoutExpiredError: print( "\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..." ) # find/kill all child processes killargs = {'pid': self.poolpid, 'name': self.poolname} self.killproc.killPool(**killargs) except KeyboardInterrupt: print( "\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..." ) killargs = {'pid': self.poolpid, 'name': self.poolname} self.killproc.killPool(**killargs) else: # cleanup (close pool of workers) pool.close() pool.join()
class ParallelCwbQuery(object): def __init__(self): # Initialize kill object for class self.killproc = Kill() def cwbQuery(self, station): # ------------------------------------------------ # Pull specific station seed files using CWBQuery # ------------------------------------------------ for attempt in range(self.cwbattempts): try: cmd = ("java -jar " + self.cwbquery + " -s " + '"' + station + '"' + " -b " + '"' + self.datetimeQuery + '"' + " -d " + '"' + str(self.duration) + '"' + " -t dcc512 -o " + self.seedpath + "%N_%y_%j -h " + '"' + self.ipaddress + '"') #print(cmd) # may want to implement a logger to track system # pid hangs and program exceptions #print ("java -jar " + self.cwbquery + " -s " + '"'+station+'"' + # " -b " + '"'+self.datetimeQuery+'"' + " -d " + # '"'+str(self.duration)+'"' + " -t dcc512 -o " + self.seedpath+"%N_%y_%j -h " + # '"'+self.ipaddress+'"') subproc = subprocess.Popen([cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid, shell=True) (out, err) = subproc.communicate(timeout=self.cwbtimeout) # waits #time.sleep(2) out = out.decode("utf-8") err = err.decode("utf-8") # Set pids and kill args for killSubprocess() method self.parentpid = os.getppid() self.childpid = os.getpid() self.gchildpid = subproc.pid #print("parent pid: " + str(self.parentpid)) #print("child pid: " + str(self.childpid)) #print("gchild pid: " + str(self.gchildpid)) if (len(out) == 0): print("Query on " + station) print("Returned 0 blocks\n") print(str(out)) print(str(err)) sys.stdout.flush() sys.stderr.flush() except subprocess.TimeoutExpired: print("TimeoutExpired cwbQuery(): retrying (attempt %d)..." % attempt) time.sleep(self.cwbsleep) if attempt == (self.cwbattempts - 1): print("TimeoutExpired cwbQuery(): terminate workers...") signum = signal.SIGKILL killargs = { 'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum } self.killproc.killSubprocess(**killargs) sys.stdout.flush() sys.stderr.flush() raise TimeoutExpiredError() return # returns to cwbQuery pool except KeyboardInterrupt: print("KeyboardInterrupt cwbQuery(): terminate workers...") signum = signal.SIGKILL killargs = { 'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum } self.killproc.killSubprocess(**killargs) raise KeyboardInterruptError() return except Exception as e: print("UnknownException cwbQuery(): " + str(e)) signum = signal.SIGKILL killargs = { 'childpid': self.childpid, 'gchildpid': self.gchildpid, 'signum': signum } self.killproc.killSubprocess(**killargs) return else: break def launchWorkers(self, stationinfo, cwbquery, cwbattempts, cwbsleep, cwbtimeout, datetimeQuery, duration, seedpath, ipaddress): # --------------------------------------------- # Initialize all vars needed to run cwbQuery() # --------------------------------------------- print("------cwbQuery() Pool------\n") files = glob.glob(seedpath + "*") self.cwbquery = cwbquery self.cwbattempts = cwbattempts self.cwbsleep = cwbsleep self.cwbtimeout = cwbtimeout self.datetimeQuery = datetimeQuery self.duration = duration self.seedpath = seedpath self.ipaddress = ipaddress for f in files: os.remove(f) # remove tmp seed files from SeedFiles dir stationlen = len(stationinfo) # --------------------------------------------- # Create multiprocessing pools to run multiple # instances of cwbQuery() # --------------------------------------------- PROCESSES = multiprocessing.cpu_count() print("PROCESSES: " + str(PROCESSES)) print("stationlen: " + str(stationlen) + "\n") pool = multiprocessing.Pool(PROCESSES) try: self.poolpid = os.getpid() self.poolname = "cwbQuery()" #print "pool PID: " + str(self.poolpid) + "\n" pool.map(unwrap_self_cwbQuery, list(zip([self] * stationlen, stationinfo))) # pool.close()/pool.terminate() must be called before pool.join() # pool.close(): prevents more tasks from being submitted to pool, # once tasks have been completed the worker processes # will exit # pool.terminate(): tops worker processes immediately without # completing outstanding work, when the pool # object is garbage collected terminate() will # be called immediately # pool.join(): wait for worker processes to exit pool.close() pool.join() print("------cwbQuery() Pool Complete------\n\n") except TimeoutExpiredError: print("\nTimeoutExpired parallelcwbQuery(): terminating pool...") # find/kill all child processes killargs = {'pid': self.poolpid, 'name': self.poolname} self.killproc.killPool(**killargs) except KeyboardInterrupt: print( "\nKeyboardInterrupt parallelcwbQuery(): terminating pool...") killargs = {'pid': self.poolpid, 'name': self.poolname} self.killproc.killPool(**killargs) else: # cleanup (close pool of workers) pool.close() pool.join()