class ParallelDeconvFilter(object):
def __init__(self):
# Initialize kill object for class
self.killproc = Kill()
def deconvFilter(self, stream, response, filters):
# ----------------------------------------
# Deconvolve/filter each station, filters
# are based on channel IDs
# ----------------------------------------
streamID = stream[0].getId()
tmpstr = re.split("\\.", streamID)
networkID = tmpstr[0].strip()
stationID = tmpstr[1].strip()
respID = response['filename'].strip()
netstatID = networkID + stationID
# Get filter types from filters[{},{},...]
if streamID in filters['streamID']:
filtertype = filters['filtertype']
freqX = filters['freqX']
freqY = filters['freqY']
# Try/catch block for sensitivity subprocess
try:
print "Stream/Filter: " + netstatID + " / " + str(filtertype)
# Deconvolution (removes sensitivity)
sensitivity = "Sensitivity:" # pull sensitivity from RESP file
grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " +
respID + " | tail -1")
self.subprocess = True # flag for exceptions (if !subprocess return)
subproc = subprocess.Popen([grepSensitivity], stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
(out, err) = subproc.communicate(timeout=10) # waits for child proc
# Store/print pids for exception kills
self.parentpid = os.getppid()
self.childpid = os.getpid()
self.gchildpid = subproc.pid
# Pull sensitivity from subproc
tmps = out.strip()
tmps = re.split(':', tmps)
s = float(tmps[1].strip())
sys.stdout.flush()
sys.stderr.flush()
self.subprocess = False # subprocess finished
# deconvolution (this will be a flag for the user)
# stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4),
# seedresp=response, taper='True')
# Remove transient response and decimate signals to SR=1Hz
decfactor = int(stream[0].stats.sampling_rate)
stream.detrend('demean') # removes mean in data set
#stream.taper(max_percentage=0.01/2.0, type='cosine') # cos tapers beginning/end to remove transient resp
stream.decimate(decfactor, no_filter=True, strict_length=False)
# Filter stream based on channel (remove sensitivity)
if filtertype == "bandpass":
print "Bandpass filter: %.3f-%.3fHz" % (freqX, freqY)
maxval = np.amax(stream[0].data)
stream.filter(filtertype, freqmin=freqX,
freqmax=freqY, corners=4) # bp filter
stream[0].data = stream[0].data / s
elif filtertype == "bandstop":
print "Bandstop filter: %.3f-%.3fHz" % (freqX, freqY)
maxval = np.amax(stream[0].data)
stream.filter(filtertype, freqmin=freqX,
freqmax=freqY, corners=4) # notch filter
stream[0].data = stream[0].data / s
elif filtertype == "lowpass":
print "Lowpass filter: %.2f" % freqX
stream.filter(filtertype, freq=freqX, corners=4) # lp filter
stream[0].data = stream[0].data / s
elif filtertype == "highpass":
print "Highpass filter: %.2f" % freqX
stream.filter(filtertype, freq=freqX, corners=4) # hp filter
stream[0].data = stream[0].data / s
print "Filtered stream: " + str(stream) + "\n"
return stream
except subprocess.TimeoutExpired:
print "TimeoutExpired deconvFilter(): terminate workers..."
if self.subprocess:
signum = signal.SIGKILL
killargs = {'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum}
self.killproc.killSubprocess(**killargs)
sys.stdout.flush()
sys.stdout.flush()
raise TimeoutExpiredError()
return # return to deconvFilter pool
except KeyboardInterrupt:
print "KeyboardInterrupt deconvFilter(): terminate workers..."
if self.subprocess:
signum = signal.SIGKILL
killargs = {'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum}
self.killproc.killSubprocess(**killargs)
raise KeyboardInterruptError()
return
except Exception as e:
print "UnknownException deconvFilter(): " + str(e)
if self.subprocess:
signum = signal.SIGKILL
killargs = {'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum}
self.killproc.killSubprocess(**killargs)
return
def launchWorkers(self, stream, streamlen, response, filters):
# ---------------------------------
# Simulate/filter queried stations
# ---------------------------------
print "-------deconvFilter() Pool-------\n"
# Merge traces to eliminate small data lengths,
# method 0 => no overlap of traces (i.e. overwriting
# of previous trace data, gaps fill overlaps)
# method 1 => fill overlaps using interpolation for
# values between both vectors for x num of samples
for i in range(streamlen):
try:
stream[i].merge(method=1, fill_value='interpolate',
interpolation_samples=100)
except Exception, e:
print 'Error merging traces:', e
# Deconvolution/Prefilter
# Initialize multiprocessing pools
PROCESSES = multiprocessing.cpu_count()
print "PROCESSES: " + str(PROCESSES)
print "streamlen: " + str(streamlen) + "\n"
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "deconvFilter()"
flt_streams = pool.map(unwrap_self_deconvFilter,
zip([self]*streamlen, stream, response, filters))
pool.close()
pool.join()
self.flt_streams = flt_streams
print "-------deconvFilter() Pool Complete-------\n\n"
except TimeoutExpiredError:
print "\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..."
# find/kill all child processes
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
except KeyboardInterrupt:
print "\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..."
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()
class ParallelDeconvFilter(object):
def __init__(self):
# Initialize kill object for class
self.killproc = Kill()
def deconvFilter(self, stream, response, filters):
# ----------------------------------------
# Deconvolve/filter each station, filters
# are based on channel IDs
# ----------------------------------------
streamID = stream[0].get_id()
tmpstr = re.split("\\.", streamID)
networkID = tmpstr[0].strip()
stationID = tmpstr[1].strip()
respID = response['filename'].strip()
netstatID = networkID + stationID
# Get filter types from filters[{},{},...]
if streamID in filters['streamID']:
filtertype = filters['filtertype']
freqX = filters['freqX']
freqY = filters['freqY']
# Try/catch block for sensitivity subprocess
try:
#print("Stream/Filter: " + netstatID + " / " + str(filtertype))
# Deconvolution (removes sensitivity)
sensitivity = "Sensitivity:" # pull sensitivity from RESP file
grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " +
respID + " | tail -1")
self.subprocess = True # flag for exceptions (if !subprocess return)
subproc = subprocess.Popen([grepSensitivity],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(out,
err) = subproc.communicate(timeout=10) # waits for child proc
out = out.decode("utf-8")
out = str(out)
# Store/print pids for exception kills
self.parentpid = os.getppid()
self.childpid = os.getpid()
self.gchildpid = subproc.pid
# Pull sensitivity from subproc
tmps = out.strip()
tmps = re.split(':', tmps)
tmps = tmps[1]
tmps = tmps.replace(' ', '')
tmps = tmps.replace("'", "")
s = float(tmps)
print("Stream/Filter: " + netstatID + " / " + str(filtertype) +
" Sensitivity: " + str(s))
sys.stdout.flush()
sys.stderr.flush()
self.subprocess = False # subprocess finished
# deconvolution (this will be a flag for the user)
# stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4),
# seedresp=response, taper='True')
# Remove transient response and decimate signals to SR=1Hz
decfactor = int(stream[0].stats.sampling_rate)
stream.detrend('demean') # removes mean in data set
#stream.taper(max_percentage=0.01/2.0, type='cosine') # cos tapers beginning/end to remove transient resp
stream.decimate(decfactor, no_filter=True, strict_length=False)
# Filter stream based on channel (remove sensitivity)
if filtertype == "bandpass":
print("Bandpass filter: %.3f-%.3fHz" % (freqX, freqY))
maxval = np.amax(stream[0].data)
stream.filter(filtertype,
freqmin=freqX,
freqmax=freqY,
corners=4) # bp filter
stream[0].data = stream[0].data / s
elif filtertype == "bandstop":
print("Bandstop filter: %.3f-%.3fHz" % (freqX, freqY))
maxval = np.amax(stream[0].data)
stream.filter(filtertype,
freqmin=freqX,
freqmax=freqY,
corners=4) # notch filter
stream[0].data = stream[0].data / s
elif filtertype == "lowpass":
print("Lowpass filter: %.2f" % freqX)
stream.filter(filtertype, freq=freqX, corners=4) # lp filter
stream[0].data = stream[0].data / s
elif filtertype == "highpass":
print("Highpass filter: %.2f" % freqX)
stream.filter(filtertype, freq=freqX, corners=4) # hp filter
stream[0].data = stream[0].data / s
print("Filtered stream: " + str(stream) + "\n")
return stream
except subprocess.TimeoutExpired:
print("TimeoutExpired deconvFilter(): terminate workers...")
if self.subprocess:
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
sys.stdout.flush()
sys.stdout.flush()
raise TimeoutExpiredError()
return # return to deconvFilter pool
except KeyboardInterrupt:
print("KeyboardInterrupt deconvFilter(): terminate workers...")
if self.subprocess:
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
raise KeyboardInterruptError()
return
except Exception as e:
PrintException()
print("UnknownException deconvFilter(): " + str(e))
if self.subprocess:
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
return
def launchWorkers(self, stream, streamlen, response, filters):
# ---------------------------------
# Simulate/filter queried stations
# ---------------------------------
print("-------deconvFilter() Pool-------\n")
# Merge traces to eliminate small data lengths,
# method 0 => no overlap of traces (i.e. overwriting
# of previous trace data, gaps fill overlaps)
# method 1 => fill overlaps using interpolation for
# values between both vectors for x num of samples
for i in range(streamlen):
try:
stream[i].merge(method=1,
fill_value='interpolate',
interpolation_samples=100)
except Exception as e:
print('Error merging traces:', e)
# Deconvolution/Prefilter
# Initialize multiprocessing pools
PROCESSES = multiprocessing.cpu_count()
print("PROCESSES: " + str(PROCESSES))
print("streamlen: " + str(streamlen) + "\n")
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "deconvFilter()"
flt_streams = pool.map(
unwrap_self_deconvFilter,
list(zip([self] * streamlen, stream, response, filters)))
pool.close()
pool.join()
self.flt_streams = flt_streams
print("-------deconvFilter() Pool Complete-------\n\n")
except TimeoutExpiredError:
print(
"\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..."
)
# find/kill all child processes
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
except KeyboardInterrupt:
print(
"\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..."
)
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()
class ParallelCwbQuery(object):
def __init__(self):
# Initialize kill object for class
self.killproc = Kill()
def cwbQuery(self, station):
# ------------------------------------------------
# Pull specific station seed files using CWBQuery
# ------------------------------------------------
for attempt in range(self.cwbattempts):
try:
cmd = ("java -jar " + self.cwbquery + " -s " + '"' + station +
'"' + " -b " + '"' + self.datetimeQuery + '"' + " -d " +
'"' + str(self.duration) + '"' + " -t dcc512 -o " +
self.seedpath + "%N_%y_%j -h " + '"' + self.ipaddress +
'"')
#print(cmd)
# may want to implement a logger to track system
# pid hangs and program exceptions
#print ("java -jar " + self.cwbquery + " -s " + '"'+station+'"' +
# " -b " + '"'+self.datetimeQuery+'"' + " -d " +
# '"'+str(self.duration)+'"' + " -t dcc512 -o " + self.seedpath+"%N_%y_%j -h " +
# '"'+self.ipaddress+'"')
subproc = subprocess.Popen([cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid,
shell=True)
(out,
err) = subproc.communicate(timeout=self.cwbtimeout) # waits
#time.sleep(2)
out = out.decode("utf-8")
err = err.decode("utf-8")
# Set pids and kill args for killSubprocess() method
self.parentpid = os.getppid()
self.childpid = os.getpid()
self.gchildpid = subproc.pid
#print("parent pid: " + str(self.parentpid))
#print("child pid: " + str(self.childpid))
#print("gchild pid: " + str(self.gchildpid))
if (len(out) == 0):
print("Query on " + station)
print("Returned 0 blocks\n")
print(str(out))
print(str(err))
sys.stdout.flush()
sys.stderr.flush()
except subprocess.TimeoutExpired:
print("TimeoutExpired cwbQuery(): retrying (attempt %d)..." %
attempt)
time.sleep(self.cwbsleep)
if attempt == (self.cwbattempts - 1):
print("TimeoutExpired cwbQuery(): terminate workers...")
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
sys.stdout.flush()
sys.stderr.flush()
raise TimeoutExpiredError()
return # returns to cwbQuery pool
except KeyboardInterrupt:
print("KeyboardInterrupt cwbQuery(): terminate workers...")
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
raise KeyboardInterruptError()
return
except Exception as e:
print("UnknownException cwbQuery(): " + str(e))
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
return
else:
break
def launchWorkers(self, stationinfo, cwbquery, cwbattempts, cwbsleep,
cwbtimeout, datetimeQuery, duration, seedpath,
ipaddress):
# ---------------------------------------------
# Initialize all vars needed to run cwbQuery()
# ---------------------------------------------
print("------cwbQuery() Pool------\n")
files = glob.glob(seedpath + "*")
self.cwbquery = cwbquery
self.cwbattempts = cwbattempts
self.cwbsleep = cwbsleep
self.cwbtimeout = cwbtimeout
self.datetimeQuery = datetimeQuery
self.duration = duration
self.seedpath = seedpath
self.ipaddress = ipaddress
for f in files:
os.remove(f) # remove tmp seed files from SeedFiles dir
stationlen = len(stationinfo)
# ---------------------------------------------
# Create multiprocessing pools to run multiple
# instances of cwbQuery()
# ---------------------------------------------
PROCESSES = multiprocessing.cpu_count()
print("PROCESSES: " + str(PROCESSES))
print("stationlen: " + str(stationlen) + "\n")
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "cwbQuery()"
#print "pool PID: " + str(self.poolpid) + "\n"
pool.map(unwrap_self_cwbQuery,
list(zip([self] * stationlen, stationinfo)))
# pool.close()/pool.terminate() must be called before pool.join()
# pool.close(): prevents more tasks from being submitted to pool,
# once tasks have been completed the worker processes
# will exit
# pool.terminate(): tops worker processes immediately without
# completing outstanding work, when the pool
# object is garbage collected terminate() will
# be called immediately
# pool.join(): wait for worker processes to exit
pool.close()
pool.join()
print("------cwbQuery() Pool Complete------\n\n")
except TimeoutExpiredError:
print("\nTimeoutExpired parallelcwbQuery(): terminating pool...")
# find/kill all child processes
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
except KeyboardInterrupt:
print(
"\nKeyboardInterrupt parallelcwbQuery(): terminating pool...")
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()
