Exemplo n.º 1
0
 def __init__(self):
     # Initializes kill object for pool
     self.killproc = Kill()
Exemplo n.º 2
0
 def __init__(self):
     # Initialize kill object for class
     self.killproc = Kill()
Exemplo n.º 3
0
class ParallelPlotVelocity(object):
    def __init__(self):
        # Initializes kill object for pool
        self.killproc = Kill()

    def plotVelocity(self, stream, stationName, filters):
        # --------------------------------
        # Plots filtered/magnified streams
        # --------------------------------
        try:
            streamID = stream[0].get_id()
            magnification = self.magnification[
                streamID]  # magnification for station[i]
            trspacing = self.vertrange / magnification * 1000.0  # trace spacing
            # Get filter coefficients for every station
            if streamID in filters['streamID']:
                filtertype = filters['filtertype']
                freqX = filters['freqX']
                freqY = filters['freqY']

                # set bounds x-label
                if filtertype == "highpass":
                    bounds = str(freqX)
                elif filtertype == "bandpass":
                    bounds = str(freqX) + "-" + str(freqY)
                elif filtertype == "bandstop":
                    bounds = str(freqX) + "-" + str(freqY)
                elif filtertype == "lowpass":
                    bounds = str(freqX)

            # pass explicit figure instance to set correct title and attributes
            # pix, resx, and resy are from station.cfg
            # - pix is dpi (arbitrary, but using 80 is easiest)
            # - resx and resy are in pixels - should be 800 x 600 to match expectations
            #   of earthquake.usgs.gov Monitoring web pages
            dpl = plt.figure(dpi=self.pix,
                             figsize=(self.resx / self.pix,
                                      self.resy / self.pix))

            titlestartTime = self.datetimePlotstart.strftime("%Y/%m/%d %H:%M")
            titlestartTime = titlestartTime + " UTC"
            plotstart = self.datetimePlotstart
            plotend = self.datetimePlotend
            plotday = plotstart.day
            plothour = plotstart.hour

            # Need to check for streams that have a start time greater
            # than the query time, then trim based on the nearest hour
            streamstart = stream[0].stats.starttime.datetime
            streamstart = streamstart.strftime("%Y%m%d_%H:%M:00")
            streamstart = UTCDateTime(streamstart)
            if (streamstart.datetime <= plotstart.datetime):
                #print streamID + ": " + str(streamstart.datetime) + " < " + str(plotstart.datetime) + "\n"

                # Trim stream to starttime of plot
                # Round up to the nearest sample, this will take care
                # of sample drift for non-Q330 signals
                stream.trim(
                    starttime=plotstart, endtime=plotend,
                    nearest_sample=True)  # selects sample nearest trim time

                # Check trimmed hour and round if != plotstart hour
                trimmedhour = stream[0].stats.starttime.hour
                if (trimmedhour != plothour):
                    stream[0].stats.starttime.day = plotday
                    stream[0].stats.starttime.hour = plothour
                    stream[0].stats.starttime.minute = 0
                    stream[0].stats.starttime.second = 0
                    stream[0].stats.starttime.microsecond = 0

            elif (streamstart.datetime > plotstart.datetime):
                #print streamID + ": " + str(streamstart.datetime) + " > " + str(plotstart.datetime) + "\n"

                # Trim stream to nearest hour when
                # the plot start time is less than
                # the stream start time
                year = streamstart.year  # stream stats (date/time)
                month = streamstart.month
                day = streamstart.day
                hour = streamstart.hour
                minute = 0  # 00 to account for shift
                second = 0
                currtime = datetime(year, month, day, hour, minute, second, 0)
                if int(hour) != 23:
                    # increase trim time to next hour if h != 23
                    trimtime = currtime + timedelta(hours=1)
                else:
                    # trim to next day if h = 23
                    hour = 0  # set time to 00:00:00
                    minute = 0
                    second = 0
                    trimtime = datetime(year, month, day, hour, minute, second,
                                        0) + timedelta(days=1)
                trimtime = UTCDateTime(trimtime)
                startday = trimtime.day
                starthour = trimtime.hour
                stream.trim(
                    starttime=trimtime, endtime=plotend,
                    nearest_sample=True)  # selects sample nearest trim time

                # Check trimmed hour and round if != trimhour
                trimmedhour = stream[0].stats.starttime.hour
                if (trimmedhour != starthour):
                    stream[0].stats.starttime.day = startday
                    stream[0].stats.starttime.hour = starthour
                    stream[0].stats.starttime.minute = 0
                    stream[0].stats.starttime.second = 0
                    stream[0].stats.starttime.microsecond = 0

            print("\nPlotting: " + str(stream))
            stream.plot(startime=plotstart,
                        endtime=plotend,
                        type='dayplot',
                        interval=60,
                        vertical_scaling_range=self.vertrange,
                        right_vertical_labels=False,
                        number_of_ticks=7,
                        one_tick_per_line=True,
                        color=['k'],
                        fig=dpl,
                        show_y_UTC_label=True,
                        title_size=-1)

            # set title, x/y labels and tick marks
            plt.title(streamID.replace('.', ' ') + "  " + "Starts: " +
                      str(titlestartTime),
                      fontsize=12)
            plt.xlabel('Time [m]\n(%s: %sHz  Trace Spacing: %.2e mm/s)' %
                       (str(filtertype), str(bounds), trspacing),
                       fontsize=10)
            plt.ylabel('Time [h]', fontsize=10)
            locs, labels = plt.yticks()  # pull current locs/labels

            hours = [0 for i in range(24)]  # 24 hours
            # Create list of hours (if missing data, fill in beginning hours)
            if len(labels) < len(hours):
                tmptime = re.split(':', labels[0].get_text())
                starthour = int(tmptime[0])
                hour = 0  # fill in hour
                lastindex = len(hours) - len(labels)
                i = lastindex

                # Stream start hour can be < or > than the plot
                # start hour (if > then subtract, else start from
                # plot hour and add)
                # **NOTE: This fixes negative indexing
                if (plothour < starthour):
                    while (i > 0):  # fill beginning hours
                        hour = starthour - i
                        hours[lastindex - i] = str(hour) + ":00"
                        i = i - 1
                    i = 0
                    for i in range(len(labels)):  # fill remaining hours
                        tmptime = re.split(':', labels[i].get_text())
                        hour = int(tmptime[0])
                        hours[i + lastindex] = str(hour) + ":00"
                else:  # plothour > starthour
                    while (i > 0):
                        if (i > starthour):
                            hour = plothour + (lastindex - i)
                            hours[lastindex - i] = str(hour) + ":00"
                        elif (i <= starthour):  # start at 0
                            hour = starthour - i
                            hours[lastindex - i] = str(hour) + ":00"
                        i = i - 1
                    i = 0
                    for i in range(len(labels)):  # fill remaining hours
                        tmptime = re.split(':', labels[i].get_text())
                        hour = int(tmptime[0])
                        hours[i + lastindex] = str(hour) + ":00"
            elif len(labels) == len(hours):
                for i in range(len(labels)):  # extract hours from labels
                    tmptime = re.split(':', labels[i].get_text())
                    hour = int(tmptime[0])
                    hours[i] = str(hour) + ":00"

            # Create tick position list
            position = [i + 0.5 for i in range(24)]
            position = position[::-1]  # reverse list
            plt.yticks(position, hours, fontsize=9)  # times in position
            #dpi=self.pix, size=(self.resx,self.resy))

            # port to 3.6 cut off bottom legend, this fixes the problem
            plt.gcf().subplots_adjust(bottom=0.15)

            # GHSC version - use station as plot name
            name_components = stationName.split(".")
            station = name_components[1]

            plt.savefig(station + "." + self.imgformat)
            plt.close(dpl)
        except KeyboardInterrupt:
            print("KeyboardInterrupt plotVelocity(): terminate workers...")
            raise KeyboardInterruptError()
            return  # return to plotVelocity() pool
        except Exception as e:
            print("UnknownException plotVelocity(): " + str(e))
            return

    def launchWorkers(self, streams, plotspath, stationName, magnification,
                      vertrange, datetimePlotstart, datetimePlotend, resx,
                      resy, pix, imgformat, filters):
        # ------------------------
        # Pool of plotting workers
        # ------------------------
        print("------plotVelocity() Pool------\n")
        self.magnification = magnification
        self.vertrange = vertrange
        self.datetimePlotstart = datetimePlotstart
        self.datetimePlotend = datetimePlotend
        self.resx = resx
        self.resy = resy
        self.pix = pix
        self.imgformat = imgformat

        streamlen = len(streams)
        # clear output plots dir
        os.chdir(plotspath)
        imgfiles = glob.glob(plotspath + "*")
        for f in imgfiles:
            os.remove(f)  # remove tmp png files from OutputPlots dir

        # Initialize multiprocessing pools for plotting
        PROCESSES = multiprocessing.cpu_count()
        print("PROCESSES:	" + str(PROCESSES))
        print("streamlen:	" + str(streamlen) + "\n")
        pool = multiprocessing.Pool(PROCESSES)
        try:
            self.poolpid = os.getpid()
            self.poolname = "plotVelocity()"
            #print "pool PID:	" + str(self.poolpid) + "\n"
            pool.map(unwrap_self_plotVelocity,
                     list(
                         zip([self] * streamlen, streams, stationName,
                             filters)))  # thread plots
            pool.close()
            pool.join()
            print("\n------plotVelocity() Pool Complete------\n\n")
        except KeyboardInterrupt:
            print(
                "KeyboardInterrupt parallelplotVelocity(): terminating pool..."
            )
            # find/kill all child processes
            killargs = {'pid': self.poolpid, 'name': self.poolname}
            self.killproc.killPool(**killargs)
        except Exception as e:
            print("Exception parallelplotVelocity(): terminating pool: " +
                  str(e))
            killargs = {'pid': self.poolpid, 'name': self.poolname}
            self.killproc.killPool(**killargs)
        else:
            # cleanup (close pool of workers)
            pool.close()
            pool.join()
Exemplo n.º 4
0
 def test_proc_info(self):
     client = Kill()
     client.handle()
Exemplo n.º 5
0
class ParallelDeconvFilter(object):
    def __init__(self):
        # Initialize kill object for class
        self.killproc = Kill()

    def deconvFilter(self, stream, response, filters):
        # ----------------------------------------
        # Deconvolve/filter each station, filters
        # are based on channel IDs
        # ----------------------------------------
        streamID = stream[0].get_id()
        tmpstr = re.split("\\.", streamID)
        networkID = tmpstr[0].strip()
        stationID = tmpstr[1].strip()
        respID = response['filename'].strip()
        netstatID = networkID + stationID

        # Get filter types from filters[{},{},...]
        if streamID in filters['streamID']:
            filtertype = filters['filtertype']
            freqX = filters['freqX']
            freqY = filters['freqY']

        # Try/catch block for sensitivity subprocess
        try:
            #print("Stream/Filter: " + netstatID + " / " + str(filtertype))

            # Deconvolution (removes sensitivity)
            sensitivity = "Sensitivity:"  # pull sensitivity from RESP file
            grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " +
                               respID + " | tail -1")
            self.subprocess = True  # flag for exceptions (if !subprocess return)
            subproc = subprocess.Popen([grepSensitivity],
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE,
                                       shell=True)
            (out,
             err) = subproc.communicate(timeout=10)  # waits for child proc
            out = out.decode("utf-8")
            out = str(out)

            # Store/print pids for exception kills
            self.parentpid = os.getppid()
            self.childpid = os.getpid()
            self.gchildpid = subproc.pid

            # Pull sensitivity from subproc
            tmps = out.strip()
            tmps = re.split(':', tmps)
            tmps = tmps[1]
            tmps = tmps.replace(' ', '')
            tmps = tmps.replace("'", "")

            s = float(tmps)
            print("Stream/Filter: " + netstatID + " / " + str(filtertype) +
                  " Sensitivity: " + str(s))
            sys.stdout.flush()
            sys.stderr.flush()
            self.subprocess = False  # subprocess finished

            # deconvolution (this will be a flag for the user)
            # stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4),
            # 	seedresp=response, taper='True')

            # Remove transient response and decimate signals to SR=1Hz
            decfactor = int(stream[0].stats.sampling_rate)
            stream.detrend('demean')  # removes mean in data set
            #stream.taper(max_percentage=0.01/2.0, type='cosine')	# cos tapers beginning/end to remove transient resp
            stream.decimate(decfactor, no_filter=True, strict_length=False)

            # Filter stream based on channel (remove sensitivity)
            if filtertype == "bandpass":
                print("Bandpass filter: %.3f-%.3fHz" % (freqX, freqY))
                maxval = np.amax(stream[0].data)
                stream.filter(filtertype,
                              freqmin=freqX,
                              freqmax=freqY,
                              corners=4)  # bp filter
                stream[0].data = stream[0].data / s
            elif filtertype == "bandstop":
                print("Bandstop filter: %.3f-%.3fHz" % (freqX, freqY))
                maxval = np.amax(stream[0].data)
                stream.filter(filtertype,
                              freqmin=freqX,
                              freqmax=freqY,
                              corners=4)  # notch filter
                stream[0].data = stream[0].data / s
            elif filtertype == "lowpass":
                print("Lowpass filter: %.2f" % freqX)
                stream.filter(filtertype, freq=freqX, corners=4)  # lp filter
                stream[0].data = stream[0].data / s
            elif filtertype == "highpass":
                print("Highpass filter: %.2f" % freqX)
                stream.filter(filtertype, freq=freqX, corners=4)  # hp filter
                stream[0].data = stream[0].data / s
            print("Filtered stream: " + str(stream) + "\n")
            return stream
        except subprocess.TimeoutExpired:
            print("TimeoutExpired deconvFilter(): terminate workers...")
            if self.subprocess:
                signum = signal.SIGKILL
                killargs = {
                    'childpid': self.childpid,
                    'gchildpid': self.gchildpid,
                    'signum': signum
                }
                self.killproc.killSubprocess(**killargs)
            sys.stdout.flush()
            sys.stdout.flush()
            raise TimeoutExpiredError()
            return  # return to deconvFilter pool
        except KeyboardInterrupt:
            print("KeyboardInterrupt deconvFilter(): terminate workers...")
            if self.subprocess:
                signum = signal.SIGKILL
                killargs = {
                    'childpid': self.childpid,
                    'gchildpid': self.gchildpid,
                    'signum': signum
                }
                self.killproc.killSubprocess(**killargs)
            raise KeyboardInterruptError()
            return
        except Exception as e:
            PrintException()
            print("UnknownException deconvFilter(): " + str(e))
            if self.subprocess:
                signum = signal.SIGKILL
                killargs = {
                    'childpid': self.childpid,
                    'gchildpid': self.gchildpid,
                    'signum': signum
                }
                self.killproc.killSubprocess(**killargs)
            return

    def launchWorkers(self, stream, streamlen, response, filters):
        # ---------------------------------
        # Simulate/filter queried stations
        # ---------------------------------
        print("-------deconvFilter() Pool-------\n")
        # Merge traces to eliminate small data lengths,
        # method 0 => no overlap of traces (i.e. overwriting
        # of previous trace data, gaps fill overlaps)
        # method 1 => fill overlaps using interpolation for
        # values between both vectors for x num of samples
        for i in range(streamlen):
            try:
                stream[i].merge(method=1,
                                fill_value='interpolate',
                                interpolation_samples=100)
            except Exception as e:
                print('Error merging traces:', e)

        # Deconvolution/Prefilter
        # Initialize multiprocessing pools
        PROCESSES = multiprocessing.cpu_count()
        print("PROCESSES:	" + str(PROCESSES))
        print("streamlen:	" + str(streamlen) + "\n")

        pool = multiprocessing.Pool(PROCESSES)
        try:
            self.poolpid = os.getpid()
            self.poolname = "deconvFilter()"
            flt_streams = pool.map(
                unwrap_self_deconvFilter,
                list(zip([self] * streamlen, stream, response, filters)))
            pool.close()
            pool.join()
            self.flt_streams = flt_streams
            print("-------deconvFilter() Pool Complete-------\n\n")
        except TimeoutExpiredError:
            print(
                "\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..."
            )
            # find/kill all child processes
            killargs = {'pid': self.poolpid, 'name': self.poolname}
            self.killproc.killPool(**killargs)
        except KeyboardInterrupt:
            print(
                "\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..."
            )
            killargs = {'pid': self.poolpid, 'name': self.poolname}
            self.killproc.killPool(**killargs)
        else:
            # cleanup (close pool of workers)
            pool.close()
            pool.join()
	def __init__(self):
		# Initialize kill object for class
		self.killproc = Kill()
class ParallelDeconvFilter(object):
	def __init__(self):
		# Initialize kill object for class
		self.killproc = Kill()

	def deconvFilter(self, stream, response, filters):
		# ----------------------------------------
		# Deconvolve/filter each station, filters
		# are based on channel IDs	
		# ----------------------------------------
		streamID = stream[0].getId()	
		tmpstr = re.split("\\.", streamID) 
		networkID = tmpstr[0].strip()	
		stationID = tmpstr[1].strip()
		respID = response['filename'].strip()
		netstatID = networkID + stationID 
	
		# Get filter types from filters[{},{},...]
		if streamID in filters['streamID']:
			filtertype = filters['filtertype']
			freqX = filters['freqX']
			freqY = filters['freqY']

		# Try/catch block for sensitivity subprocess
		try:
			print "Stream/Filter: " + netstatID + " / " + str(filtertype) 

			# Deconvolution (removes sensitivity)
			sensitivity = "Sensitivity:"	# pull sensitivity from RESP file
			grepSensitivity = ("grep " + '"' + sensitivity + '"' + " " +
				respID + " | tail -1")
			self.subprocess = True	# flag for exceptions (if !subprocess return)
			subproc = subprocess.Popen([grepSensitivity], stdout=subprocess.PIPE,
				stderr=subprocess.PIPE, shell=True)
			(out, err) = subproc.communicate(timeout=10)	# waits for child proc

			# Store/print pids for exception kills
			self.parentpid = os.getppid()
			self.childpid = os.getpid()
			self.gchildpid = subproc.pid

			# Pull sensitivity from subproc
			tmps = out.strip()
			tmps = re.split(':', tmps)
			s = float(tmps[1].strip())
			sys.stdout.flush()
			sys.stderr.flush()
			self.subprocess = False	# subprocess finished

			# deconvolution (this will be a flag for the user)
			# stream.simulate(paz_remove=None, pre_filt=(c1, c2, c3, c4), 
			# 	seedresp=response, taper='True') 

			# Remove transient response and decimate signals to SR=1Hz 
			decfactor = int(stream[0].stats.sampling_rate)
			stream.detrend('demean')	# removes mean in data set
			#stream.taper(max_percentage=0.01/2.0, type='cosine')	# cos tapers beginning/end to remove transient resp
			stream.decimate(decfactor, no_filter=True, strict_length=False)	

			# Filter stream based on channel (remove sensitivity) 
			if filtertype == "bandpass":
				print "Bandpass filter: %.3f-%.3fHz" % (freqX, freqY)
				maxval = np.amax(stream[0].data) 
				stream.filter(filtertype, freqmin=freqX,
					freqmax=freqY, corners=4)	# bp filter 
				stream[0].data = stream[0].data / s
			elif filtertype == "bandstop":
				print "Bandstop filter: %.3f-%.3fHz" % (freqX, freqY)
				maxval = np.amax(stream[0].data)
				stream.filter(filtertype, freqmin=freqX,
					freqmax=freqY, corners=4)	# notch filter
				stream[0].data = stream[0].data / s
			elif filtertype == "lowpass":
				print "Lowpass filter: %.2f" % freqX 
				stream.filter(filtertype, freq=freqX, corners=4) # lp filter 
				stream[0].data = stream[0].data / s
			elif filtertype == "highpass":
				print "Highpass filter: %.2f" % freqX 
				stream.filter(filtertype, freq=freqX, corners=4) # hp filter
				stream[0].data = stream[0].data / s
			print "Filtered stream: " + str(stream) + "\n"
			return stream
		except subprocess.TimeoutExpired:
			print "TimeoutExpired deconvFilter(): terminate workers..."
			if self.subprocess:
				signum = signal.SIGKILL
				killargs = {'childpid': self.childpid,
					    'gchildpid': self.gchildpid,
					    'signum': signum}
				self.killproc.killSubprocess(**killargs)
			sys.stdout.flush()
			sys.stdout.flush()
			raise TimeoutExpiredError()
			return	# return to deconvFilter pool
		except KeyboardInterrupt:
			print "KeyboardInterrupt deconvFilter(): terminate workers..."
			if self.subprocess:
				signum = signal.SIGKILL
				killargs = {'childpid': self.childpid,
					    'gchildpid': self.gchildpid,
					    'signum': signum}
				self.killproc.killSubprocess(**killargs)
			raise KeyboardInterruptError()
			return
		except Exception as e:
			print "UnknownException deconvFilter(): " + str(e)
			if self.subprocess:
				signum = signal.SIGKILL
				killargs = {'childpid': self.childpid,
					    'gchildpid': self.gchildpid,
					    'signum': signum}
				self.killproc.killSubprocess(**killargs)
			return

	def launchWorkers(self, stream, streamlen, response, filters):
		# ---------------------------------
		# Simulate/filter queried stations
		# ---------------------------------
		print "-------deconvFilter() Pool-------\n"
		# Merge traces to eliminate small data lengths, 
		# method 0 => no overlap of traces (i.e. overwriting
		# of previous trace data, gaps fill overlaps)
		# method 1 => fill overlaps using interpolation for
		# values between both vectors for x num of samples
		for i in range(streamlen):
			try:
				stream[i].merge(method=1, fill_value='interpolate',
					interpolation_samples=100)
			except Exception, e:
				print 'Error merging traces:', e

		# Deconvolution/Prefilter
		# Initialize multiprocessing pools
		PROCESSES = multiprocessing.cpu_count()
		print "PROCESSES:	" + str(PROCESSES)
		print "streamlen:	" + str(streamlen) + "\n"	

		pool = multiprocessing.Pool(PROCESSES)
		try:
			self.poolpid = os.getpid()
			self.poolname = "deconvFilter()"
			flt_streams = pool.map(unwrap_self_deconvFilter,
				zip([self]*streamlen, stream, response, filters))
			pool.close()
			pool.join()
			self.flt_streams = flt_streams
			print "-------deconvFilter() Pool Complete-------\n\n"
		except TimeoutExpiredError:
			print "\nTimeoutExpiredError parallelDeconvFilter(): terminating pool..."
			# find/kill all child processes
			killargs = {'pid': self.poolpid, 'name': self.poolname}
			self.killproc.killPool(**killargs)
		except KeyboardInterrupt:
			print "\nKeyboardInterrupt parallelDeconvFilter(): terminating pool..."
			killargs = {'pid': self.poolpid, 'name': self.poolname}
			self.killproc.killPool(**killargs)
		else:
			# cleanup (close pool of workers)
			pool.close()
			pool.join()
Exemplo n.º 8
0
class ParallelCwbQuery(object):
    def __init__(self):
        # Initialize kill object for class
        self.killproc = Kill()

    def cwbQuery(self, station):
        # ------------------------------------------------
        # Pull specific station seed files using CWBQuery
        # ------------------------------------------------
        for attempt in range(self.cwbattempts):
            try:
                cmd = ("java -jar " + self.cwbquery + " -s " + '"' + station +
                       '"' + " -b " + '"' + self.datetimeQuery + '"' + " -d " +
                       '"' + str(self.duration) + '"' + " -t dcc512 -o " +
                       self.seedpath + "%N_%y_%j -h " + '"' + self.ipaddress +
                       '"')
                #print(cmd)
                # may want to implement a logger to track system
                # pid hangs and program exceptions
                #print ("java -jar " + self.cwbquery + " -s " + '"'+station+'"' +
                #	" -b " + '"'+self.datetimeQuery+'"' + " -d " +
                #	'"'+str(self.duration)+'"' + " -t dcc512 -o " + self.seedpath+"%N_%y_%j -h " +
                #	'"'+self.ipaddress+'"')
                subproc = subprocess.Popen([cmd],
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE,
                                           preexec_fn=os.setsid,
                                           shell=True)
                (out,
                 err) = subproc.communicate(timeout=self.cwbtimeout)  # waits
                #time.sleep(2)
                out = out.decode("utf-8")
                err = err.decode("utf-8")

                # Set pids and kill args for killSubprocess() method
                self.parentpid = os.getppid()
                self.childpid = os.getpid()
                self.gchildpid = subproc.pid
                #print("parent pid: " + str(self.parentpid))
                #print("child  pid: " + str(self.childpid))
                #print("gchild pid: " + str(self.gchildpid))
                if (len(out) == 0):
                    print("Query on " + station)
                    print("Returned 0 blocks\n")
                print(str(out))
                print(str(err))
                sys.stdout.flush()
                sys.stderr.flush()
            except subprocess.TimeoutExpired:
                print("TimeoutExpired cwbQuery(): retrying (attempt %d)..." %
                      attempt)
                time.sleep(self.cwbsleep)

                if attempt == (self.cwbattempts - 1):
                    print("TimeoutExpired cwbQuery(): terminate workers...")
                    signum = signal.SIGKILL
                    killargs = {
                        'childpid': self.childpid,
                        'gchildpid': self.gchildpid,
                        'signum': signum
                    }
                    self.killproc.killSubprocess(**killargs)
                    sys.stdout.flush()
                    sys.stderr.flush()
                    raise TimeoutExpiredError()
                    return  # returns to cwbQuery pool
            except KeyboardInterrupt:
                print("KeyboardInterrupt cwbQuery(): terminate workers...")
                signum = signal.SIGKILL
                killargs = {
                    'childpid': self.childpid,
                    'gchildpid': self.gchildpid,
                    'signum': signum
                }
                self.killproc.killSubprocess(**killargs)
                raise KeyboardInterruptError()
                return
            except Exception as e:
                print("UnknownException cwbQuery(): " + str(e))
                signum = signal.SIGKILL
                killargs = {
                    'childpid': self.childpid,
                    'gchildpid': self.gchildpid,
                    'signum': signum
                }
                self.killproc.killSubprocess(**killargs)
                return
            else:
                break

    def launchWorkers(self, stationinfo, cwbquery, cwbattempts, cwbsleep,
                      cwbtimeout, datetimeQuery, duration, seedpath,
                      ipaddress):
        # ---------------------------------------------
        # Initialize all vars needed to run cwbQuery()
        # ---------------------------------------------
        print("------cwbQuery() Pool------\n")
        files = glob.glob(seedpath + "*")
        self.cwbquery = cwbquery
        self.cwbattempts = cwbattempts
        self.cwbsleep = cwbsleep
        self.cwbtimeout = cwbtimeout
        self.datetimeQuery = datetimeQuery
        self.duration = duration
        self.seedpath = seedpath
        self.ipaddress = ipaddress

        for f in files:
            os.remove(f)  # remove tmp seed files from SeedFiles dir
        stationlen = len(stationinfo)

        # ---------------------------------------------
        # Create multiprocessing pools to run multiple
        # instances of cwbQuery()
        # ---------------------------------------------
        PROCESSES = multiprocessing.cpu_count()
        print("PROCESSES:	" + str(PROCESSES))
        print("stationlen:	" + str(stationlen) + "\n")
        pool = multiprocessing.Pool(PROCESSES)
        try:
            self.poolpid = os.getpid()
            self.poolname = "cwbQuery()"
            #print "pool PID:	" + str(self.poolpid) + "\n"
            pool.map(unwrap_self_cwbQuery,
                     list(zip([self] * stationlen, stationinfo)))

            # pool.close()/pool.terminate() must be called before pool.join()
            # pool.close(): prevents more tasks from being submitted to pool,
            #               once tasks have been completed the worker processes
            #		will exit
            # pool.terminate(): tops worker processes immediately without
            #		    completing outstanding work, when the pool
            #		    object is garbage collected terminate() will
            #		    be called immediately
            # pool.join(): wait for worker processes to exit
            pool.close()
            pool.join()
            print("------cwbQuery() Pool Complete------\n\n")
        except TimeoutExpiredError:
            print("\nTimeoutExpired parallelcwbQuery(): terminating pool...")
            # find/kill all child processes
            killargs = {'pid': self.poolpid, 'name': self.poolname}
            self.killproc.killPool(**killargs)
        except KeyboardInterrupt:
            print(
                "\nKeyboardInterrupt parallelcwbQuery(): terminating pool...")
            killargs = {'pid': self.poolpid, 'name': self.poolname}
            self.killproc.killPool(**killargs)
        else:
            # cleanup (close pool of workers)
            pool.close()
            pool.join()
 def __init__(self):
     # Initializes kill object for pool
     self.killproc = Kill()
class ParallelPlotVelocity(object):
    def __init__(self):
        # Initializes kill object for pool
        self.killproc = Kill()

    def plotVelocity(self, stream, stationName, filters):
        # --------------------------------
        # Plots filtered/magnified streams
        # --------------------------------
        try:
            streamID = stream[0].getId()
            magnification = self.magnification[streamID]  # magnification for station[i]
            trspacing = self.vertrange / magnification * 1000.0  # trace spacing
            # Get filter coefficients for every station
            if streamID in filters["streamID"]:
                filtertype = filters["filtertype"]
                freqX = filters["freqX"]
                freqY = filters["freqY"]

                # set bounds x-label
                if filtertype == "highpass":
                    bounds = str(freqX)
                elif filtertype == "bandpass":
                    bounds = str(freqX) + "-" + str(freqY)
                elif filtertype == "bandstop":
                    bounds = str(freqX) + "-" + str(freqY)
                elif filtertype == "lowpass":
                    bounds = str(freqX)

                    # pass explicit figure instance to set correct title and attributes
            dpl = plt.figure()
            titlestartTime = self.datetimePlotstart.strftime("%Y/%m/%d %H:%M")
            titlestartTime = titlestartTime + " UTC"
            plotstart = self.datetimePlotstart
            plotend = self.datetimePlotend
            plotday = plotstart.day
            plothour = plotstart.hour

            # Need to check for streams that have a start time greater
            # than the query time, then trim based on the nearest hour
            streamstart = stream[0].stats.starttime.datetime
            streamstart = streamstart.strftime("%Y%m%d_%H:%M:00")
            streamstart = UTCDateTime(streamstart)
            if streamstart.datetime <= plotstart.datetime:
                # print streamID + ": " + str(streamstart.datetime) + " < " + str(plotstart.datetime) + "\n"

                # Trim stream to starttime of plot
                # Round up to the nearest sample, this will take care
                # of sample drift for non-Q330 signals
                stream.trim(
                    starttime=plotstart, endtime=plotend, nearest_sample=True
                )  # selects sample nearest trim time

                # Check trimmed hour and round if != plotstart hour
                trimmedhour = stream[0].stats.starttime.hour
                if trimmedhour != plothour:
                    stream[0].stats.starttime.day = plotday
                    stream[0].stats.starttime.hour = plothour
                    stream[0].stats.starttime.minute = 0
                    stream[0].stats.starttime.second = 0
                    stream[0].stats.starttime.microsecond = 0

            elif streamstart.datetime > plotstart.datetime:
                # print streamID + ": " + str(streamstart.datetime) + " > " + str(plotstart.datetime) + "\n"

                # Trim stream to nearest hour when
                # the plot start time is less than
                # the stream start time
                year = streamstart.year  # stream stats (date/time)
                month = streamstart.month
                day = streamstart.day
                hour = streamstart.hour
                minute = 0  # 00 to account for shift
                second = 0
                currtime = datetime(year, month, day, hour, minute, second, 0)
                if int(hour) != 23:
                    # increase trim time to next hour if h != 23
                    trimtime = currtime + timedelta(hours=1)
                else:
                    # trim to next day if h = 23
                    hour = 0  # set time to 00:00:00
                    minute = 0
                    second = 0
                    trimtime = datetime(year, month, day, hour, minute, second, 0) + timedelta(days=1)
                trimtime = UTCDateTime(trimtime)
                startday = trimtime.day
                starthour = trimtime.hour
                stream.trim(
                    starttime=trimtime, endtime=plotend, nearest_sample=True
                )  # selects sample nearest trim time

                # Check trimmed hour and round if != trimhour
                trimmedhour = stream[0].stats.starttime.hour
                if trimmedhour != starthour:
                    stream[0].stats.starttime.day = startday
                    stream[0].stats.starttime.hour = starthour
                    stream[0].stats.starttime.minute = 0
                    stream[0].stats.starttime.second = 0
                    stream[0].stats.starttime.microsecond = 0

            print "Plotting: " + str(stream)
            stream.plot(
                startime=plotstart,
                endtime=plotend,
                type="dayplot",
                interval=60,
                vertical_scaling_range=self.vertrange,
                right_vertical_labels=False,
                number_of_ticks=7,
                one_tick_per_line=True,
                color=["k"],
                fig=dpl,
                show_y_UTC_label=True,
                size=(self.resx, self.resy),
                dpi=self.pix,
                title_size=-1,
            )

            # set title, x/y labels and tick marks
            plt.title(streamID + "  " + "Start: " + str(titlestartTime), fontsize=12)
            plt.xlabel(
                "Time [m]\n(%s: %sHz  Trace Spacing: %.2e mm/s)" % (str(filtertype), str(bounds), trspacing),
                fontsize=10,
            )
            plt.ylabel("Time [h]", fontsize=10)
            locs, labels = plt.yticks()  # pull current locs/labels

            hours = [0 for i in range(24)]  # 24 hours
            # Create list of hours (if missing data, fill in beginning hours)
            if len(labels) < len(hours):
                tmptime = re.split(":", labels[0].get_text())
                starthour = int(tmptime[0])
                hour = 0  # fill in hour
                lastindex = len(hours) - len(labels)
                i = lastindex

                # Stream start hour can be < or > than the plot
                # start hour (if > then subtract, else start from
                # plot hour and add)
                # **NOTE: This fixes negative indexing
                if plothour < starthour:
                    while i > 0:  # fill beginning hours
                        hour = starthour - i
                        hours[lastindex - i] = str(hour) + ":00"
                        i = i - 1
                    i = 0
                    for i in range(len(labels)):  # fill remaining hours
                        tmptime = re.split(":", labels[i].get_text())
                        hour = int(tmptime[0])
                        hours[i + lastindex] = str(hour) + ":00"
                else:  # plothour > starthour
                    while i > 0:
                        if i > starthour:
                            hour = plothour + (lastindex - i)
                            hours[lastindex - i] = str(hour) + ":00"
                        elif i <= starthour:  # start at 0
                            hour = starthour - i
                            hours[lastindex - i] = str(hour) + ":00"
                        i = i - 1
                    i = 0
                    for i in range(len(labels)):  # fill remaining hours
                        tmptime = re.split(":", labels[i].get_text())
                        hour = int(tmptime[0])
                        hours[i + lastindex] = str(hour) + ":00"
            elif len(labels) == len(hours):
                for i in range(len(labels)):  # extract hours from labels
                    tmptime = re.split(":", labels[i].get_text())
                    hour = int(tmptime[0])
                    hours[i] = str(hour) + ":00"

                    # Create tick position list
            position = [i + 0.5 for i in range(24)]
            position = position[::-1]  # reverse list
            plt.yticks(position, hours, fontsize=9)  # times in position
            # dpi=self.pix, size=(self.resx,self.resy))
            plt.savefig(stationName + "." + self.imgformat)
            plt.close(dpl)
        except KeyboardInterrupt:
            print "KeyboardInterrupt plotVelocity(): terminate workers..."
            raise KeyboardInterruptError()
            return  # return to plotVelocity() pool
        except Exception as e:
            print "UnknownException plotVelocity(): " + str(e)
            return

    def launchWorkers(
        self,
        streams,
        plotspath,
        stationName,
        magnification,
        vertrange,
        datetimePlotstart,
        datetimePlotend,
        resx,
        resy,
        pix,
        imgformat,
        filters,
    ):
        # ------------------------
        # Pool of plotting workers
        # ------------------------
        print "------plotVelocity() Pool------\n"
        self.magnification = magnification
        self.vertrange = vertrange
        self.datetimePlotstart = datetimePlotstart
        self.datetimePlotend = datetimePlotend
        self.resx = resx
        self.resy = resy
        self.pix = pix
        self.imgformat = imgformat

        streamlen = len(streams)
        # clear output plots dir
        os.chdir(plotspath)
        imgfiles = glob.glob(plotspath + "*")
        for f in imgfiles:
            os.remove(f)  # remove tmp png files from OutputPlots dir

            # Initialize multiprocessing pools for plotting
        PROCESSES = multiprocessing.cpu_count()
        print "PROCESSES:	" + str(PROCESSES)
        print "streamlen:	" + str(streamlen) + "\n"
        pool = multiprocessing.Pool(PROCESSES)
        try:
            self.poolpid = os.getpid()
            self.poolname = "plotVelocity()"
            # print "pool PID:	" + str(self.poolpid) + "\n"
            pool.map(unwrap_self_plotVelocity, zip([self] * streamlen, streams, stationName, filters))  # thread plots
            pool.close()
            pool.join()
            print "\n------plotVelocity() Pool Complete------\n\n"
        except KeyboardInterrupt:
            print "KeyboardInterrupt parallelplotVelocity(): terminating pool..."
            # find/kill all child processes
            killargs = {"pid": self.poolpid, "name": self.poolname}
            self.killproc.killPool(**killargs)
        except Exception as e:
            print "Exception parallelplotVelocity(): terminating pool: " + str(e)
            killargs = {"pid": self.poolpid, "name": self.poolname}
            self.killproc.killPool(**killargs)
        else:
            # cleanup (close pool of workers)
            pool.close()
            pool.join()