def __init__(self):
# Initializes kill object for pool
self.killproc = Kill()
def __init__(self):
# Initialize kill object for class
self.killproc = Kill()
class ParallelPlotVelocity(object):
def __init__(self):
# Initializes kill object for pool
self.killproc = Kill()
def plotVelocity(self, stream, stationName, filters):
# --------------------------------
# Plots filtered/magnified streams
# --------------------------------
try:
streamID = stream[0].get_id()
magnification = self.magnification[
streamID] # magnification for station[i]
trspacing = self.vertrange / magnification * 1000.0 # trace spacing
# Get filter coefficients for every station
if streamID in filters['streamID']:
filtertype = filters['filtertype']
freqX = filters['freqX']
freqY = filters['freqY']
# set bounds x-label
if filtertype == "highpass":
bounds = str(freqX)
elif filtertype == "bandpass":
bounds = str(freqX) + "-" + str(freqY)
elif filtertype == "bandstop":
bounds = str(freqX) + "-" + str(freqY)
elif filtertype == "lowpass":
bounds = str(freqX)
# pass explicit figure instance to set correct title and attributes
# pix, resx, and resy are from station.cfg
# - pix is dpi (arbitrary, but using 80 is easiest)
# - resx and resy are in pixels - should be 800 x 600 to match expectations
# of earthquake.usgs.gov Monitoring web pages
dpl = plt.figure(dpi=self.pix,
figsize=(self.resx / self.pix,
self.resy / self.pix))
titlestartTime = self.datetimePlotstart.strftime("%Y/%m/%d %H:%M")
titlestartTime = titlestartTime + " UTC"
plotstart = self.datetimePlotstart
plotend = self.datetimePlotend
plotday = plotstart.day
plothour = plotstart.hour
# Need to check for streams that have a start time greater
# than the query time, then trim based on the nearest hour
streamstart = stream[0].stats.starttime.datetime
streamstart = streamstart.strftime("%Y%m%d_%H:%M:00")
streamstart = UTCDateTime(streamstart)
if (streamstart.datetime <= plotstart.datetime):
#print streamID + ": " + str(streamstart.datetime) + " < " + str(plotstart.datetime) + "\n"
# Trim stream to starttime of plot
# Round up to the nearest sample, this will take care
# of sample drift for non-Q330 signals
stream.trim(
starttime=plotstart, endtime=plotend,
nearest_sample=True) # selects sample nearest trim time
# Check trimmed hour and round if != plotstart hour
trimmedhour = stream[0].stats.starttime.hour
if (trimmedhour != plothour):
stream[0].stats.starttime.day = plotday
stream[0].stats.starttime.hour = plothour
stream[0].stats.starttime.minute = 0
stream[0].stats.starttime.second = 0
stream[0].stats.starttime.microsecond = 0
elif (streamstart.datetime > plotstart.datetime):
#print streamID + ": " + str(streamstart.datetime) + " > " + str(plotstart.datetime) + "\n"
# Trim stream to nearest hour when
# the plot start time is less than
# the stream start time
year = streamstart.year # stream stats (date/time)
month = streamstart.month
day = streamstart.day
hour = streamstart.hour
minute = 0 # 00 to account for shift
second = 0
currtime = datetime(year, month, day, hour, minute, second, 0)
if int(hour) != 23:
# increase trim time to next hour if h != 23
trimtime = currtime + timedelta(hours=1)
else:
# trim to next day if h = 23
hour = 0 # set time to 00:00:00
minute = 0
second = 0
trimtime = datetime(year, month, day, hour, minute, second,
0) + timedelta(days=1)
trimtime = UTCDateTime(trimtime)
startday = trimtime.day
starthour = trimtime.hour
stream.trim(
starttime=trimtime, endtime=plotend,
nearest_sample=True) # selects sample nearest trim time
# Check trimmed hour and round if != trimhour
trimmedhour = stream[0].stats.starttime.hour
if (trimmedhour != starthour):
stream[0].stats.starttime.day = startday
stream[0].stats.starttime.hour = starthour
stream[0].stats.starttime.minute = 0
stream[0].stats.starttime.second = 0
stream[0].stats.starttime.microsecond = 0
print("\nPlotting: " + str(stream))
stream.plot(startime=plotstart,
endtime=plotend,
type='dayplot',
interval=60,
vertical_scaling_range=self.vertrange,
right_vertical_labels=False,
number_of_ticks=7,
one_tick_per_line=True,
color=['k'],
fig=dpl,
show_y_UTC_label=True,
title_size=-1)
# set title, x/y labels and tick marks
plt.title(streamID.replace('.', ' ') + " " + "Starts: " +
str(titlestartTime),
fontsize=12)
plt.xlabel('Time [m]\n(%s: %sHz Trace Spacing: %.2e mm/s)' %
(str(filtertype), str(bounds), trspacing),
fontsize=10)
plt.ylabel('Time [h]', fontsize=10)
locs, labels = plt.yticks() # pull current locs/labels
hours = [0 for i in range(24)] # 24 hours
# Create list of hours (if missing data, fill in beginning hours)
if len(labels) < len(hours):
tmptime = re.split(':', labels[0].get_text())
starthour = int(tmptime[0])
hour = 0 # fill in hour
lastindex = len(hours) - len(labels)
i = lastindex
# Stream start hour can be < or > than the plot
# start hour (if > then subtract, else start from
# plot hour and add)
# **NOTE: This fixes negative indexing
if (plothour < starthour):
while (i > 0): # fill beginning hours
hour = starthour - i
hours[lastindex - i] = str(hour) + ":00"
i = i - 1
i = 0
for i in range(len(labels)): # fill remaining hours
tmptime = re.split(':', labels[i].get_text())
hour = int(tmptime[0])
hours[i + lastindex] = str(hour) + ":00"
else: # plothour > starthour
while (i > 0):
if (i > starthour):
hour = plothour + (lastindex - i)
hours[lastindex - i] = str(hour) + ":00"
elif (i <= starthour): # start at 0
hour = starthour - i
hours[lastindex - i] = str(hour) + ":00"
i = i - 1
i = 0
for i in range(len(labels)): # fill remaining hours
tmptime = re.split(':', labels[i].get_text())
hour = int(tmptime[0])
hours[i + lastindex] = str(hour) + ":00"
elif len(labels) == len(hours):
for i in range(len(labels)): # extract hours from labels
tmptime = re.split(':', labels[i].get_text())
hour = int(tmptime[0])
hours[i] = str(hour) + ":00"
# Create tick position list
position = [i + 0.5 for i in range(24)]
position = position[::-1] # reverse list
plt.yticks(position, hours, fontsize=9) # times in position
#dpi=self.pix, size=(self.resx,self.resy))
# port to 3.6 cut off bottom legend, this fixes the problem
plt.gcf().subplots_adjust(bottom=0.15)
# GHSC version - use station as plot name
name_components = stationName.split(".")
station = name_components[1]
plt.savefig(station + "." + self.imgformat)
plt.close(dpl)
except KeyboardInterrupt:
print("KeyboardInterrupt plotVelocity(): terminate workers...")
raise KeyboardInterruptError()
return # return to plotVelocity() pool
except Exception as e:
print("UnknownException plotVelocity(): " + str(e))
return
def launchWorkers(self, streams, plotspath, stationName, magnification,
vertrange, datetimePlotstart, datetimePlotend, resx,
resy, pix, imgformat, filters):
# ------------------------
# Pool of plotting workers
# ------------------------
print("------plotVelocity() Pool------\n")
self.magnification = magnification
self.vertrange = vertrange
self.datetimePlotstart = datetimePlotstart
self.datetimePlotend = datetimePlotend
self.resx = resx
self.resy = resy
self.pix = pix
self.imgformat = imgformat
streamlen = len(streams)
# clear output plots dir
os.chdir(plotspath)
imgfiles = glob.glob(plotspath + "*")
for f in imgfiles:
os.remove(f) # remove tmp png files from OutputPlots dir
# Initialize multiprocessing pools for plotting
PROCESSES = multiprocessing.cpu_count()
print("PROCESSES: " + str(PROCESSES))
print("streamlen: " + str(streamlen) + "\n")
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "plotVelocity()"
#print "pool PID: " + str(self.poolpid) + "\n"
pool.map(unwrap_self_plotVelocity,
list(
zip([self] * streamlen, streams, stationName,
filters))) # thread plots
pool.close()
pool.join()
print("\n------plotVelocity() Pool Complete------\n\n")
except KeyboardInterrupt:
print(
"KeyboardInterrupt parallelplotVelocity(): terminating pool..."
)
# find/kill all child processes
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
except Exception as e:
print("Exception parallelplotVelocity(): terminating pool: " +
str(e))
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()
def test_proc_info(self):
client = Kill()
client.handle()
class ParallelDeconvFilter(object):
def __init__(self):
# Initialize kill object for class
self.killproc = Kill()
def deconvFilter(self, stream, response, filters):
# ----------------------------------------
# Deconvolve/filter each station, filters
# are based on channel IDs
# ----------------------------------------
streamID = stream[0].get_id()
tmpstr = re.split("\\.", streamID)
networkID = tmpstr[0].strip()
stationID = tmpstr[1].strip()
respID = response['filename'].strip()
netstatID = networkID + stationID
# Get filter types from filters[{},{},...]
if streamID in filters['streamID']:
filtertype = filters['filtertype']
freqX = filters['freqX']
freqY = filters['freqY']
# Try/catch block for sensitivity subprocess
try:
#print("Stream/Filter: " + netstatID + " / " + str(filtertype))
# Deconvolution (removes sensitivity)
sensitivity = "Sensitivity:" # pull sensitivity from RESP file
grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " +
respID + " | tail -1")
self.subprocess = True # flag for exceptions (if !subprocess return)
subproc = subprocess.Popen([grepSensitivity],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(out,
err) = subproc.communicate(timeout=10) # waits for child proc
out = out.decode("utf-8")
out = str(out)
# Store/print pids for exception kills
self.parentpid = os.getppid()
self.childpid = os.getpid()
self.gchildpid = subproc.pid
# Pull sensitivity from subproc
tmps = out.strip()
tmps = re.split(':', tmps)
tmps = tmps[1]
tmps = tmps.replace(' ', '')
tmps = tmps.replace("'", "")
s = float(tmps)
print("Stream/Filter: " + netstatID + " / " + str(filtertype) +
" Sensitivity: " + str(s))
sys.stdout.flush()
sys.stderr.flush()
self.subprocess = False # subprocess finished
# deconvolution (this will be a flag for the user)
# stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4),
# seedresp=response, taper='True')
# Remove transient response and decimate signals to SR=1Hz
decfactor = int(stream[0].stats.sampling_rate)
stream.detrend('demean') # removes mean in data set
#stream.taper(max_percentage=0.01/2.0, type='cosine') # cos tapers beginning/end to remove transient resp
stream.decimate(decfactor, no_filter=True, strict_length=False)
# Filter stream based on channel (remove sensitivity)
if filtertype == "bandpass":
print("Bandpass filter: %.3f-%.3fHz" % (freqX, freqY))
maxval = np.amax(stream[0].data)
stream.filter(filtertype,
freqmin=freqX,
freqmax=freqY,
corners=4) # bp filter
stream[0].data = stream[0].data / s
elif filtertype == "bandstop":
print("Bandstop filter: %.3f-%.3fHz" % (freqX, freqY))
maxval = np.amax(stream[0].data)
stream.filter(filtertype,
freqmin=freqX,
freqmax=freqY,
corners=4) # notch filter
stream[0].data = stream[0].data / s
elif filtertype == "lowpass":
print("Lowpass filter: %.2f" % freqX)
stream.filter(filtertype, freq=freqX, corners=4) # lp filter
stream[0].data = stream[0].data / s
elif filtertype == "highpass":
print("Highpass filter: %.2f" % freqX)
stream.filter(filtertype, freq=freqX, corners=4) # hp filter
stream[0].data = stream[0].data / s
print("Filtered stream: " + str(stream) + "\n")
return stream
except subprocess.TimeoutExpired:
print("TimeoutExpired deconvFilter(): terminate workers...")
if self.subprocess:
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
sys.stdout.flush()
sys.stdout.flush()
raise TimeoutExpiredError()
return # return to deconvFilter pool
except KeyboardInterrupt:
print("KeyboardInterrupt deconvFilter(): terminate workers...")
if self.subprocess:
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
raise KeyboardInterruptError()
return
except Exception as e:
PrintException()
print("UnknownException deconvFilter(): " + str(e))
if self.subprocess:
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
return
def launchWorkers(self, stream, streamlen, response, filters):
# ---------------------------------
# Simulate/filter queried stations
# ---------------------------------
print("-------deconvFilter() Pool-------\n")
# Merge traces to eliminate small data lengths,
# method 0 => no overlap of traces (i.e. overwriting
# of previous trace data, gaps fill overlaps)
# method 1 => fill overlaps using interpolation for
# values between both vectors for x num of samples
for i in range(streamlen):
try:
stream[i].merge(method=1,
fill_value='interpolate',
interpolation_samples=100)
except Exception as e:
print('Error merging traces:', e)
# Deconvolution/Prefilter
# Initialize multiprocessing pools
PROCESSES = multiprocessing.cpu_count()
print("PROCESSES: " + str(PROCESSES))
print("streamlen: " + str(streamlen) + "\n")
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "deconvFilter()"
flt_streams = pool.map(
unwrap_self_deconvFilter,
list(zip([self] * streamlen, stream, response, filters)))
pool.close()
pool.join()
self.flt_streams = flt_streams
print("-------deconvFilter() Pool Complete-------\n\n")
except TimeoutExpiredError:
print(
"\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..."
)
# find/kill all child processes
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
except KeyboardInterrupt:
print(
"\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..."
)
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()
def __init__(self): # Initialize kill object for class self.killproc = Kill()
class ParallelDeconvFilter(object):
def __init__(self):
# Initialize kill object for class
self.killproc = Kill()
def deconvFilter(self, stream, response, filters):
# ----------------------------------------
# Deconvolve/filter each station, filters
# are based on channel IDs
# ----------------------------------------
streamID = stream[0].getId()
tmpstr = re.split("\\.", streamID)
networkID = tmpstr[0].strip()
stationID = tmpstr[1].strip()
respID = response['filename'].strip()
netstatID = networkID + stationID
# Get filter types from filters[{},{},...]
if streamID in filters['streamID']:
filtertype = filters['filtertype']
freqX = filters['freqX']
freqY = filters['freqY']
# Try/catch block for sensitivity subprocess
try:
print "Stream/Filter: " + netstatID + " / " + str(filtertype)
# Deconvolution (removes sensitivity)
sensitivity = "Sensitivity:" # pull sensitivity from RESP file
grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " +
respID + " | tail -1")
self.subprocess = True # flag for exceptions (if !subprocess return)
subproc = subprocess.Popen([grepSensitivity], stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
(out, err) = subproc.communicate(timeout=10) # waits for child proc
# Store/print pids for exception kills
self.parentpid = os.getppid()
self.childpid = os.getpid()
self.gchildpid = subproc.pid
# Pull sensitivity from subproc
tmps = out.strip()
tmps = re.split(':', tmps)
s = float(tmps[1].strip())
sys.stdout.flush()
sys.stderr.flush()
self.subprocess = False # subprocess finished
# deconvolution (this will be a flag for the user)
# stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4),
# seedresp=response, taper='True')
# Remove transient response and decimate signals to SR=1Hz
decfactor = int(stream[0].stats.sampling_rate)
stream.detrend('demean') # removes mean in data set
#stream.taper(max_percentage=0.01/2.0, type='cosine') # cos tapers beginning/end to remove transient resp
stream.decimate(decfactor, no_filter=True, strict_length=False)
# Filter stream based on channel (remove sensitivity)
if filtertype == "bandpass":
print "Bandpass filter: %.3f-%.3fHz" % (freqX, freqY)
maxval = np.amax(stream[0].data)
stream.filter(filtertype, freqmin=freqX,
freqmax=freqY, corners=4) # bp filter
stream[0].data = stream[0].data / s
elif filtertype == "bandstop":
print "Bandstop filter: %.3f-%.3fHz" % (freqX, freqY)
maxval = np.amax(stream[0].data)
stream.filter(filtertype, freqmin=freqX,
freqmax=freqY, corners=4) # notch filter
stream[0].data = stream[0].data / s
elif filtertype == "lowpass":
print "Lowpass filter: %.2f" % freqX
stream.filter(filtertype, freq=freqX, corners=4) # lp filter
stream[0].data = stream[0].data / s
elif filtertype == "highpass":
print "Highpass filter: %.2f" % freqX
stream.filter(filtertype, freq=freqX, corners=4) # hp filter
stream[0].data = stream[0].data / s
print "Filtered stream: " + str(stream) + "\n"
return stream
except subprocess.TimeoutExpired:
print "TimeoutExpired deconvFilter(): terminate workers..."
if self.subprocess:
signum = signal.SIGKILL
killargs = {'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum}
self.killproc.killSubprocess(**killargs)
sys.stdout.flush()
sys.stdout.flush()
raise TimeoutExpiredError()
return # return to deconvFilter pool
except KeyboardInterrupt:
print "KeyboardInterrupt deconvFilter(): terminate workers..."
if self.subprocess:
signum = signal.SIGKILL
killargs = {'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum}
self.killproc.killSubprocess(**killargs)
raise KeyboardInterruptError()
return
except Exception as e:
print "UnknownException deconvFilter(): " + str(e)
if self.subprocess:
signum = signal.SIGKILL
killargs = {'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum}
self.killproc.killSubprocess(**killargs)
return
def launchWorkers(self, stream, streamlen, response, filters):
# ---------------------------------
# Simulate/filter queried stations
# ---------------------------------
print "-------deconvFilter() Pool-------\n"
# Merge traces to eliminate small data lengths,
# method 0 => no overlap of traces (i.e. overwriting
# of previous trace data, gaps fill overlaps)
# method 1 => fill overlaps using interpolation for
# values between both vectors for x num of samples
for i in range(streamlen):
try:
stream[i].merge(method=1, fill_value='interpolate',
interpolation_samples=100)
except Exception, e:
print 'Error merging traces:', e
# Deconvolution/Prefilter
# Initialize multiprocessing pools
PROCESSES = multiprocessing.cpu_count()
print "PROCESSES: " + str(PROCESSES)
print "streamlen: " + str(streamlen) + "\n"
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "deconvFilter()"
flt_streams = pool.map(unwrap_self_deconvFilter,
zip([self]*streamlen, stream, response, filters))
pool.close()
pool.join()
self.flt_streams = flt_streams
print "-------deconvFilter() Pool Complete-------\n\n"
except TimeoutExpiredError:
print "\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..."
# find/kill all child processes
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
except KeyboardInterrupt:
print "\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..."
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()
class ParallelCwbQuery(object):
def __init__(self):
# Initialize kill object for class
self.killproc = Kill()
def cwbQuery(self, station):
# ------------------------------------------------
# Pull specific station seed files using CWBQuery
# ------------------------------------------------
for attempt in range(self.cwbattempts):
try:
cmd = ("java -jar " + self.cwbquery + " -s " + '"' + station +
'"' + " -b " + '"' + self.datetimeQuery + '"' + " -d " +
'"' + str(self.duration) + '"' + " -t dcc512 -o " +
self.seedpath + "%N_%y_%j -h " + '"' + self.ipaddress +
'"')
#print(cmd)
# may want to implement a logger to track system
# pid hangs and program exceptions
#print ("java -jar " + self.cwbquery + " -s " + '"'+station+'"' +
# " -b " + '"'+self.datetimeQuery+'"' + " -d " +
# '"'+str(self.duration)+'"' + " -t dcc512 -o " + self.seedpath+"%N_%y_%j -h " +
# '"'+self.ipaddress+'"')
subproc = subprocess.Popen([cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid,
shell=True)
(out,
err) = subproc.communicate(timeout=self.cwbtimeout) # waits
#time.sleep(2)
out = out.decode("utf-8")
err = err.decode("utf-8")
# Set pids and kill args for killSubprocess() method
self.parentpid = os.getppid()
self.childpid = os.getpid()
self.gchildpid = subproc.pid
#print("parent pid: " + str(self.parentpid))
#print("child pid: " + str(self.childpid))
#print("gchild pid: " + str(self.gchildpid))
if (len(out) == 0):
print("Query on " + station)
print("Returned 0 blocks\n")
print(str(out))
print(str(err))
sys.stdout.flush()
sys.stderr.flush()
except subprocess.TimeoutExpired:
print("TimeoutExpired cwbQuery(): retrying (attempt %d)..." %
attempt)
time.sleep(self.cwbsleep)
if attempt == (self.cwbattempts - 1):
print("TimeoutExpired cwbQuery(): terminate workers...")
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
sys.stdout.flush()
sys.stderr.flush()
raise TimeoutExpiredError()
return # returns to cwbQuery pool
except KeyboardInterrupt:
print("KeyboardInterrupt cwbQuery(): terminate workers...")
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
raise KeyboardInterruptError()
return
except Exception as e:
print("UnknownException cwbQuery(): " + str(e))
signum = signal.SIGKILL
killargs = {
'childpid': self.childpid,
'gchildpid': self.gchildpid,
'signum': signum
}
self.killproc.killSubprocess(**killargs)
return
else:
break
def launchWorkers(self, stationinfo, cwbquery, cwbattempts, cwbsleep,
cwbtimeout, datetimeQuery, duration, seedpath,
ipaddress):
# ---------------------------------------------
# Initialize all vars needed to run cwbQuery()
# ---------------------------------------------
print("------cwbQuery() Pool------\n")
files = glob.glob(seedpath + "*")
self.cwbquery = cwbquery
self.cwbattempts = cwbattempts
self.cwbsleep = cwbsleep
self.cwbtimeout = cwbtimeout
self.datetimeQuery = datetimeQuery
self.duration = duration
self.seedpath = seedpath
self.ipaddress = ipaddress
for f in files:
os.remove(f) # remove tmp seed files from SeedFiles dir
stationlen = len(stationinfo)
# ---------------------------------------------
# Create multiprocessing pools to run multiple
# instances of cwbQuery()
# ---------------------------------------------
PROCESSES = multiprocessing.cpu_count()
print("PROCESSES: " + str(PROCESSES))
print("stationlen: " + str(stationlen) + "\n")
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "cwbQuery()"
#print "pool PID: " + str(self.poolpid) + "\n"
pool.map(unwrap_self_cwbQuery,
list(zip([self] * stationlen, stationinfo)))
# pool.close()/pool.terminate() must be called before pool.join()
# pool.close(): prevents more tasks from being submitted to pool,
# once tasks have been completed the worker processes
# will exit
# pool.terminate(): tops worker processes immediately without
# completing outstanding work, when the pool
# object is garbage collected terminate() will
# be called immediately
# pool.join(): wait for worker processes to exit
pool.close()
pool.join()
print("------cwbQuery() Pool Complete------\n\n")
except TimeoutExpiredError:
print("\nTimeoutExpired parallelcwbQuery(): terminating pool...")
# find/kill all child processes
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
except KeyboardInterrupt:
print(
"\nKeyboardInterrupt parallelcwbQuery(): terminating pool...")
killargs = {'pid': self.poolpid, 'name': self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()
def __init__(self):
# Initializes kill object for pool
self.killproc = Kill()
class ParallelPlotVelocity(object):
def __init__(self):
# Initializes kill object for pool
self.killproc = Kill()
def plotVelocity(self, stream, stationName, filters):
# --------------------------------
# Plots filtered/magnified streams
# --------------------------------
try:
streamID = stream[0].getId()
magnification = self.magnification[streamID] # magnification for station[i]
trspacing = self.vertrange / magnification * 1000.0 # trace spacing
# Get filter coefficients for every station
if streamID in filters["streamID"]:
filtertype = filters["filtertype"]
freqX = filters["freqX"]
freqY = filters["freqY"]
# set bounds x-label
if filtertype == "highpass":
bounds = str(freqX)
elif filtertype == "bandpass":
bounds = str(freqX) + "-" + str(freqY)
elif filtertype == "bandstop":
bounds = str(freqX) + "-" + str(freqY)
elif filtertype == "lowpass":
bounds = str(freqX)
# pass explicit figure instance to set correct title and attributes
dpl = plt.figure()
titlestartTime = self.datetimePlotstart.strftime("%Y/%m/%d %H:%M")
titlestartTime = titlestartTime + " UTC"
plotstart = self.datetimePlotstart
plotend = self.datetimePlotend
plotday = plotstart.day
plothour = plotstart.hour
# Need to check for streams that have a start time greater
# than the query time, then trim based on the nearest hour
streamstart = stream[0].stats.starttime.datetime
streamstart = streamstart.strftime("%Y%m%d_%H:%M:00")
streamstart = UTCDateTime(streamstart)
if streamstart.datetime <= plotstart.datetime:
# print streamID + ": " + str(streamstart.datetime) + " < " + str(plotstart.datetime) + "\n"
# Trim stream to starttime of plot
# Round up to the nearest sample, this will take care
# of sample drift for non-Q330 signals
stream.trim(
starttime=plotstart, endtime=plotend, nearest_sample=True
) # selects sample nearest trim time
# Check trimmed hour and round if != plotstart hour
trimmedhour = stream[0].stats.starttime.hour
if trimmedhour != plothour:
stream[0].stats.starttime.day = plotday
stream[0].stats.starttime.hour = plothour
stream[0].stats.starttime.minute = 0
stream[0].stats.starttime.second = 0
stream[0].stats.starttime.microsecond = 0
elif streamstart.datetime > plotstart.datetime:
# print streamID + ": " + str(streamstart.datetime) + " > " + str(plotstart.datetime) + "\n"
# Trim stream to nearest hour when
# the plot start time is less than
# the stream start time
year = streamstart.year # stream stats (date/time)
month = streamstart.month
day = streamstart.day
hour = streamstart.hour
minute = 0 # 00 to account for shift
second = 0
currtime = datetime(year, month, day, hour, minute, second, 0)
if int(hour) != 23:
# increase trim time to next hour if h != 23
trimtime = currtime + timedelta(hours=1)
else:
# trim to next day if h = 23
hour = 0 # set time to 00:00:00
minute = 0
second = 0
trimtime = datetime(year, month, day, hour, minute, second, 0) + timedelta(days=1)
trimtime = UTCDateTime(trimtime)
startday = trimtime.day
starthour = trimtime.hour
stream.trim(
starttime=trimtime, endtime=plotend, nearest_sample=True
) # selects sample nearest trim time
# Check trimmed hour and round if != trimhour
trimmedhour = stream[0].stats.starttime.hour
if trimmedhour != starthour:
stream[0].stats.starttime.day = startday
stream[0].stats.starttime.hour = starthour
stream[0].stats.starttime.minute = 0
stream[0].stats.starttime.second = 0
stream[0].stats.starttime.microsecond = 0
print "Plotting: " + str(stream)
stream.plot(
startime=plotstart,
endtime=plotend,
type="dayplot",
interval=60,
vertical_scaling_range=self.vertrange,
right_vertical_labels=False,
number_of_ticks=7,
one_tick_per_line=True,
color=["k"],
fig=dpl,
show_y_UTC_label=True,
size=(self.resx, self.resy),
dpi=self.pix,
title_size=-1,
)
# set title, x/y labels and tick marks
plt.title(streamID + " " + "Start: " + str(titlestartTime), fontsize=12)
plt.xlabel(
"Time [m]\n(%s: %sHz Trace Spacing: %.2e mm/s)" % (str(filtertype), str(bounds), trspacing),
fontsize=10,
)
plt.ylabel("Time [h]", fontsize=10)
locs, labels = plt.yticks() # pull current locs/labels
hours = [0 for i in range(24)] # 24 hours
# Create list of hours (if missing data, fill in beginning hours)
if len(labels) < len(hours):
tmptime = re.split(":", labels[0].get_text())
starthour = int(tmptime[0])
hour = 0 # fill in hour
lastindex = len(hours) - len(labels)
i = lastindex
# Stream start hour can be < or > than the plot
# start hour (if > then subtract, else start from
# plot hour and add)
# **NOTE: This fixes negative indexing
if plothour < starthour:
while i > 0: # fill beginning hours
hour = starthour - i
hours[lastindex - i] = str(hour) + ":00"
i = i - 1
i = 0
for i in range(len(labels)): # fill remaining hours
tmptime = re.split(":", labels[i].get_text())
hour = int(tmptime[0])
hours[i + lastindex] = str(hour) + ":00"
else: # plothour > starthour
while i > 0:
if i > starthour:
hour = plothour + (lastindex - i)
hours[lastindex - i] = str(hour) + ":00"
elif i <= starthour: # start at 0
hour = starthour - i
hours[lastindex - i] = str(hour) + ":00"
i = i - 1
i = 0
for i in range(len(labels)): # fill remaining hours
tmptime = re.split(":", labels[i].get_text())
hour = int(tmptime[0])
hours[i + lastindex] = str(hour) + ":00"
elif len(labels) == len(hours):
for i in range(len(labels)): # extract hours from labels
tmptime = re.split(":", labels[i].get_text())
hour = int(tmptime[0])
hours[i] = str(hour) + ":00"
# Create tick position list
position = [i + 0.5 for i in range(24)]
position = position[::-1] # reverse list
plt.yticks(position, hours, fontsize=9) # times in position
# dpi=self.pix, size=(self.resx,self.resy))
plt.savefig(stationName + "." + self.imgformat)
plt.close(dpl)
except KeyboardInterrupt:
print "KeyboardInterrupt plotVelocity(): terminate workers..."
raise KeyboardInterruptError()
return # return to plotVelocity() pool
except Exception as e:
print "UnknownException plotVelocity(): " + str(e)
return
def launchWorkers(
self,
streams,
plotspath,
stationName,
magnification,
vertrange,
datetimePlotstart,
datetimePlotend,
resx,
resy,
pix,
imgformat,
filters,
):
# ------------------------
# Pool of plotting workers
# ------------------------
print "------plotVelocity() Pool------\n"
self.magnification = magnification
self.vertrange = vertrange
self.datetimePlotstart = datetimePlotstart
self.datetimePlotend = datetimePlotend
self.resx = resx
self.resy = resy
self.pix = pix
self.imgformat = imgformat
streamlen = len(streams)
# clear output plots dir
os.chdir(plotspath)
imgfiles = glob.glob(plotspath + "*")
for f in imgfiles:
os.remove(f) # remove tmp png files from OutputPlots dir
# Initialize multiprocessing pools for plotting
PROCESSES = multiprocessing.cpu_count()
print "PROCESSES: " + str(PROCESSES)
print "streamlen: " + str(streamlen) + "\n"
pool = multiprocessing.Pool(PROCESSES)
try:
self.poolpid = os.getpid()
self.poolname = "plotVelocity()"
# print "pool PID: " + str(self.poolpid) + "\n"
pool.map(unwrap_self_plotVelocity, zip([self] * streamlen, streams, stationName, filters)) # thread plots
pool.close()
pool.join()
print "\n------plotVelocity() Pool Complete------\n\n"
except KeyboardInterrupt:
print "KeyboardInterrupt parallelplotVelocity(): terminating pool..."
# find/kill all child processes
killargs = {"pid": self.poolpid, "name": self.poolname}
self.killproc.killPool(**killargs)
except Exception as e:
print "Exception parallelplotVelocity(): terminating pool: " + str(e)
killargs = {"pid": self.poolpid, "name": self.poolname}
self.killproc.killPool(**killargs)
else:
# cleanup (close pool of workers)
pool.close()
pool.join()