def getq(self): ''' Reads data from the queue and updates the stream. :rtype: bool :return: Returns ``True`` if stream is updated, otherwise ``False``. ''' d = self.queue.get(True, timeout=None) self.queue.task_done() if 'TERM' in str(d): self.alive = False printM('Exiting.', self.sender) sys.exit() elif str(d.decode('UTF-8')).split(' ')[0] in [ 'ALARM', 'RESET', 'IMGPATH' ]: pass else: if rs.getCHN(d) in self.chans: self.stream = rs.update_stream(stream=self.stream, d=d, fill_value=None) return True else: return False
def write(self, stream=False): if not stream: self.last = self.stream[0].stats.endtime - timedelta(seconds=5) stream = self.stream.copy().slice(endtime=self.last, nearest_sample=False) for t in stream: enc = 'STEIM2' # encoding if isinstance(t.data, RS.np.ma.masked_array): t.data = t.data.filled( fill_value=0) # fill array (to avoid obspy write error) outfile = self.outdir + '/%s.%s.00.%s.D.%s.%s' % ( t.stats.network, t.stats.station, t.stats.channel, self.y, self.j) if os.path.exists(os.path.abspath(outfile)): with open(outfile, 'ab') as fh: if self.debug: printM( 'Writing %s records to %s' % (len(t.data), outfile), self.sender) t.write(fh, format='MSEED', encoding=enc) else: if self.debug: printM('Writing %s new file %s' % (len(t.data), outfile), self.sender) t.write(outfile, format='MSEED', encoding=enc)
def __init__(self, q, data_dir, testing=False, debug=False, cha='all'): """ Initialize the process """ super().__init__() self.sender = 'Write' self.alive = True self.testing = testing self.debug = debug if self.testing: self.debug = True self.queue = q self.stream = rs.Stream() self.outdir = os.path.join(data_dir, 'data') self.outfiles = [] self.chans = [] helpers.set_channels(self, cha) printM('Writing channels: %s' % self.chans, self.sender) self.numchns = rs.numchns self.stime = 1 / rs.sps self.inv = rs.inv printM('Starting.', self.sender)
def __init__(self, consumer_key, consumer_secret, access_token, access_token_secret, q=False, tweet_images=False, ): """ Initialize the process """ super().__init__() self.sender = 'Tweeter' self.alarm = False self.alive = True self.tweet_images = tweet_images self.fmt = '%Y-%m-%d %H:%M:%S UTC' self.region = ' - region: %s' % RS.region.title() if RS.region else '' if q: self.queue = q else: printM('ERROR: no queue passed to consumer! Thread will exit now!', self.sender) sys.stdout.flush() self.alive = False sys.exit() self.twitter = Twython( consumer_key, consumer_secret, access_token, access_token_secret ) self.message0 = '(#RaspberryShake station %s.%s%s) Event detected at' % (RS.net, RS.stn, self.region) self.message1 = '(#RaspberryShake station %s.%s%s) Image of event detected at' % (RS.net, RS.stn, self.region) printM('Starting.', self.sender)
def get_inventory(sender='get_inventory'): ''' .. role:: pycode(code) :language: python Downloads the station inventory from the Raspberry Shake FDSN and stores it as an :py:class:`obspy.core.inventory.inventory.Inventory` object which is available globally. In this example, we get the R940D station inventory from the Raspberry Shake FDSN: .. code-block:: python >>> import rsudp.raspberryshake as rs >>> rs.initRSlib(dport=8888, rsstn='R940D') >>> inv = rs.get_inventory() >>> print(inv) Inventory created at 2020-02-21T20:37:34.246777Z Sending institution: SeisComP3 (gempa testbed) Contains: Networks (1): AM Stations (1): AM.R940D (Raspberry Shake Citizen Science Station) Channels (2): AM.R940D.00.EHZ, AM.R940D.00.HDF :param sender: `(optional)` The name of the function calling the :py:func:`rsudp.printM` logging function :type str: str or None :rtype: obspy.core.inventory.inventory.Inventory or bool :return: The inventory of the Raspberry Shake station in the :pycode:`rsudp.raspberryshake.stn` variable. ''' global inv, stn, region sender = 'get_inventory' if 'Z0000' in stn: printW('No station name given, continuing without inventory.', sender) inv = False else: try: printM('Fetching inventory for station %s.%s from Raspberry Shake FDSN.' % (net, stn), sender) url = 'https://fdsnws.raspberryshakedata.com/fdsnws/station/1/query?network=%s&station=%s&level=resp&nodata=404&format=xml' % ( net, stn)#, str(UTCDateTime.now()-timedelta(seconds=14400))) inv = read_inventory(url) region = FlinnEngdahl().get_region(inv[0][0].longitude, inv[0][0].latitude) printM('Inventory fetch successful. Station region is %s' % (region), sender) except (IndexError, HTTPError): printW('No inventory found for %s. Are you forwarding your Shake data?' % stn, sender) printW('Deconvolution will only be available if data forwarding is on.', sender, spaces=True) printW('Access the config page of the web front end for details.', sender, spaces=True) printW('More info at https://manual.raspberryshake.org/quickstart.html', sender, spaces=True) inv = False region = False except Exception as e: printE('Inventory fetch failed!', sender) printE('Error detail: %s' % e, sender, spaces=True) inv = False region = False return inv
def __init__(self, testing=False, soundloc=False, q=False): """ .. _pydub.AudioSegment: https://github.com/jiaaro/pydub/blob/master/API.markdown#audiosegment Initializes the alert sound listener thread. Needs a pydub.AudioSegment_ to play and a :class:`queue.Queue` to listen on. """ super().__init__() self.sender = 'AlertSound' self.alive = True self.testing = testing self.sound = soundloc self.devnull = open(os.devnull, 'w') self._init_sound() if q: self.queue = q else: printE('no queue passed to the consumer thread! We will exit now!', self.sender) sys.stdout.flush() self.alive = False sys.exit() printM('Starting.', self.sender)
def __init__( self, consumer_key, consumer_secret, access_token, access_token_secret, q=False, tweet_images=False, ): """ Initialize the process """ super().__init__() self.sender = 'Tweeter' self.alarm = False self.alive = True self.tweet_images = tweet_images self.fmt = '%Y-%m-%d %H:%M:%S UTC' self.region = '%s' % RS.region.title() if RS.region else '' if q: self.queue = q else: printM('ERROR: no queue passed to consumer! Thread will exit now!', self.sender) sys.stdout.flush() self.alive = False sys.exit() self.twitter = Twython(consumer_key, consumer_secret, access_token, access_token_secret) printM('Starting.', self.sender)
def send(self): ''' Send the latest line in the open file to the specified port at localhost. If the next line's timestamp is the same, that line will also be sent immediately. If the next line does not contain the same timestamp, the program will seek back to the last line read and then break for a new loop. If the line contains ``TERM``, the program will set ``self.alive = False`` and prepare to exit. ''' l = self.f.readline() if ('TERM' in l.decode('utf-8')) or (l.decode('utf-8') == ''): printM('End of file.', self.sender) self.alive = False else: ts = rs.getTIME(l) self.sock.sendto(l, (self.addr, self.port)) while True: self.pos = self.f.tell() l = self.f.readline() if 'TERM' in l.decode('utf-8'): break if rs.getTIME(l) == ts: self.sock.sendto(l, (self.addr, self.port)) else: self.f.seek(self.pos) break
def openSOCK(host=''): ''' .. role:: pycode(code) :language: python Initialize a socket at the port specified by :pycode:`rsudp.raspberryshake.port`. Called by :py:func:`rsudp.raspberryshake.initRSlib`, must be done before :py:func:`rsudp.raspberryshake.set_params`. :param str host: self-referential location at which to open a listening port (defaults to :pycode:`''` which resolves to :pycode:`'localhost'`) :raise IOError: if the library is not initialized (:py:func:`rsudp.raspberryshake.initRSlib`) prior to running this function :raise OSError: if the program cannot bind to the specified port number ''' global sockopen sockopen = False if initd: HP = '%s:%s' % ('localhost', port) printM("Opening socket on %s (HOST:PORT)" % HP, 'openSOCK') try: sock.bind((host, port)) sockopen = True except Exception as e: printE('Could not bind to port %s. Is another program using it?' % port) printE('Detail: %s' % e, announce=False) raise OSError(e) else: raise IOError( "Before opening a socket, you must initialize this raspberryshake library by calling initRSlib(dport=XXXXX, rssta='R0E05') first." )
def _messagetests(self, d): ''' Run tests on a message to see if a specific one has been passed. If so, mark the test passed. :param bytes d: a data packet from the queue ''' if 'TERM' in str(d): printM('Got TERM message...', sender=self.sender) t.TEST['c_TERM'][1] = True self.alive = False elif 'ALARM' in str(d): printM('Got ALARM message with time %s' % (helpers.fsec(helpers.get_msg_time(d))), sender=self.sender) t.TEST['c_ALARM'][1] = True elif 'RESET' in str(d): printM('Got RESET message with time %s' % (helpers.fsec(helpers.get_msg_time(d))), sender=self.sender) t.TEST['c_RESET'][1] = True elif 'IMGPATH' in str(d): printM('Got IMGPATH message with time %s' % (helpers.fsec(helpers.get_msg_time(d))), sender=self.sender) printM('and path %s' % (helpers.get_msg_path(d)), sender=self.sender) t.TEST['c_IMGPATH'][1] = True
def _tasks(self): ''' Execute tasks based on the states of sub-consumers. ''' for thread in self.threads: # for each thread here if thread.alarm: # if there is an alarm in a sub thread, send the ALARM message to the queues self.queue.put(helpers.msg_alarm(thread.alarm)) printM( '%s thread has indicated alarm state, sending ALARM message to queues' % thread.sender, sender=self.sender) # now re-arm the trigger thread.alarm = False if thread.alarm_reset: # if there's an alarm_reset flag in a sub thread, send a RESET message self.queue.put(helpers.msg_reset(thread.alarm_reset)) printM( '%s thread has indicated alarm reset, sending RESET message to queues' % thread.sender, sender=self.sender) # re-arm the trigger thread.alarm_reset = False if not thread.alive: # if a thread stops, set the stop flag self.stop = True
def _tracewrite(self, t): ''' Processing for the :py:func:`rsudp.c_write.Write.write` function. Writes an input trace to disk. :type t: obspy.core.trace.Trace :param t: The trace segment to write to disk. ''' enc = 'STEIM2' # encoding if isinstance(t.data, rs.np.ma.masked_array): t.data = t.data.filled( fill_value=0) # fill array (to avoid obspy write error) outfile = self.outdir + '/%s.%s.00.%s.D.%s.%s' % ( t.stats.network, t.stats.station, t.stats.channel, self.y, self.j) if not outfile in self.outfiles: self.outfiles.append(outfile) if os.path.exists(os.path.abspath(outfile)): with open(outfile, 'ab') as fh: t.write(fh, format='MSEED', encoding=enc) if self.debug: printM('%s records to %s' % (len(t.data), outfile), self.sender) else: t.write(outfile, format='MSEED', encoding=enc) if self.debug: printM('%s records to new file %s' % (len(t.data), outfile), self.sender)
def __init__(self, q=False, codefile=False, win_ovr=False): """ Initializes the custom code execution thread. """ super().__init__() self.sender = 'Custom' self.alive = True self.codefile = False self.win_ovr = win_ovr if codefile: if (os.path.exists(os.path.expanduser(codefile))) and ( os.path.splitext(codefile)[1]): self.codefile = os.path.expanduser(codefile) printM('Custom code file to run: %s' % self.codefile, sender=self.sender) else: printW( 'No python file exists at %s. No custom code will be run during alarms.' % codefile, sender=self.sender) else: printW( 'No custom code file set. No custom code will be run during alarms.', sender=self.sender) if (os.name in 'nt') and (not self.win_ovr): printE( 'Using Windows with custom alert code! Your code MUST have UNIX/Mac newline characters!' ) printE( 'Please use a conversion tool like dos2unix to convert line endings', spaces=True) printE( '(https://en.wikipedia.org/wiki/Unix2dos) to make your code file', spaces=True) printE('readable to the Python interpreter.', spaces=True) printE( 'Once you have done that, please set "win_override" to true', spaces=True) printE('in the settings file.', spaces=True) printE( '(see also footnote [1] on this page: https://docs.python.org/3/library/functions.html#id2)', spaces=True) printE('THREAD EXITING, please correct and restart!', self.sender, spaces=True) sys.exit(2) else: pass if q: self.queue = q else: printE('no queue passed to the consumer thread! We will exit now!', self.sender) sys.stdout.flush() self.alive = False sys.exit() printM('Starting.', self.sender)
def _set_deconv(self, deconv): """ This function sets the deconvolution units. Allowed values are as follows: .. |ms2| replace:: m/s\ :sup:`2`\ - ``'VEL'`` - velocity (m/s) - ``'ACC'`` - acceleration (|ms2|) - ``'GRAV'`` - fraction of acceleration due to gravity (g, or 9.81 |ms2|) - ``'DISP'`` - displacement (m) - ``'CHAN'`` - channel-specific unit calculation, i.e. ``'VEL'`` for geophone channels and ``'ACC'`` for accelerometer channels :param str deconv: ``'VEL'``, ``'ACC'``, ``'GRAV'``, ``'DISP'``, or ``'CHAN'`` """ deconv = deconv.upper() if deconv else False self.deconv = deconv if (deconv in rs.UNITS) else False if self.deconv and rs.inv: self.units = '%s (%s)' % (rs.UNITS[self.deconv][0], rs.UNITS[self.deconv][1]) if ( self.deconv in rs.UNITS) else self.units printM('Signal deconvolution set to %s' % (self.deconv), self.sender) else: self.units = rs.UNITS['CHAN'][1] self.deconv = False printM('RSAM stream units are %s' % (self.units.strip(' ').lower()), self.sender)
def cancel_tests(settings, MPL, plot, quiet): ''' Cancel some tests if they don't need to be run. :param dict settings: the dictionary of settings for program execution :param bool plot: whether or not to plot (``False`` == no plot) :param bool quiet: whether or not to play sounds (``True`` == no sound) :rtype: dict :return: settings dictionary to test with ''' global TEST if plot: if MPL: TEST['d_matplotlib'][1] = True else: printW('matplotlib backend failed to load') else: settings['plot']['enabled'] = False del TEST['d_matplotlib'] del TEST['c_IMGPATH'] printM('Plot is disabled') if quiet: settings['alertsound']['enabled'] = False del TEST['d_pydub'] printM('Alert sound is disabled') return settings
def __init__(self, addr, port, cha, q): """ Initialize the process """ super().__init__() self.sender = 'Forward' printM('Starting.', self.sender) self.queue = q self.addr = addr self.port = port self.chans = [] cha = RS.chns if (cha == 'all') else cha cha = list(cha) if isinstance(cha, str) else cha l = RS.chns for c in l: n = 0 for uch in cha: if (uch.upper() in c) and (c not in str(self.chans)): self.chans.append(c) n += 1 if len(self.chans) < 1: self.chans = RS.chns self.alarm = False self.running = True self.alive = True
def run(self): """ Distributes queue objects to execute various other tasks: for example, it may be used to populate ObsPy streams for various things like plotting, alert triggers, and ground motion calculation. """ try: while self.running: p = self.queue.get() self.queue.task_done() for q in self.destinations: q.put(p) if 'TERM' in str(p): printM('Exiting.', self.sender) break if self.testing: TEST['x_masterqueue'][1] = True except Exception as e: return e sys.exit()
def _exit(self): """ Exits the thread. """ self.alive = False printM('Exiting.', self.sender) sys.exit()
def write(self, stream=False): ''' Writes a segment of the stream to disk as miniSEED, and appends it to the file in question. If there is no file (i.e. if the program is just starting or a new UTC day has just started, then this function writes to a new file). :type stream: obspy.core.stream.Stream or bool :param stream: The stream segment to write. If ``False``, the program has just started. ''' if not stream: self.last = self.stream[0].stats.endtime - timedelta(seconds=5) stream = self.stream.copy().slice(endtime=self.last, nearest_sample=False) for t in stream: enc = 'STEIM2' # encoding if isinstance(t.data, rs.np.ma.masked_array): t.data = t.data.filled( fill_value=0) # fill array (to avoid obspy write error) outfile = self.outdir + '/%s.%s.00.%s.D.%s.%s' % ( t.stats.network, t.stats.station, t.stats.channel, self.y, self.j) if os.path.exists(os.path.abspath(outfile)): with open(outfile, 'ab') as fh: if self.debug: printM( 'Writing %s records to %s' % (len(t.data), outfile), self.sender) t.write(fh, format='MSEED', encoding=enc) else: if self.debug: printM('Writing %s new file %s' % (len(t.data), outfile), self.sender) t.write(outfile, format='MSEED', encoding=enc)
def _load_sound(self): ''' Loads MP3 sound if possible, then writes to wav. Catches a ``FileNotFoundError`` when no player can be loaded. ''' try: soundloc = self.sound self.sound = AudioSegment.from_file(self.sound, format="mp3") printM('Loaded %.2f sec alert sound from %s' % (len(self.sound) / 1000., soundloc), sender=self.sender) self.wavloc = '%s.wav' % os.path.splitext(soundloc)[0] if 'ffplay' in PLAYER: self._write_wav() except FileNotFoundError as e: printE('Error loading player - %s' % (e), sender=self.sender) printW( "You have chosen to play a sound, but don't have ffmpeg or libav installed.", sender=self.sender) printW('Sound playback requires one of these dependencies.', sender=self.sender, spaces=True) printW("To install either dependency, follow the instructions at:", sender=self.sender, spaces=True) printW('https://github.com/jiaaro/pydub#playback', sender=self.sender, spaces=True) printW('The program will now continue without sound playback.', sender=self.sender, spaces=True) self.sound = False
def set_params(): ''' .. role:: pycode(code) :language: python Read a data packet off the port. Called by :py:func:`rsudp.raspberryshake.initRSlib`, must be done after :py:func:`rsudp.raspberryshake.openSOCK` but before :py:func:`rsudp.raspberryshake.getDATA`. Will wait :pycode:`rsudp.raspberryshake.to` seconds for data before raising a no data exception (only available with UNIX socket types). ''' global to, firstaddr if os.name not in 'nt': # signal alarm not available on windows signal.signal(signal.SIGALRM, handler) signal.alarm(to) # alarm time set with timeout value data, (firstaddr, connport) = sock.recvfrom(2048) if os.name not in 'nt': signal.alarm( 0) # once data has been received, turn alarm completely off to = 0 # otherwise it erroneously triggers after keyboardinterrupt getTR(getCHNS()[0]) getSR(tf, data) getTTLCHN() printM('Available channels: %s' % chns, 'Init') get_inventory()
def run(self): """ The heart of the plotting routine. Begins by updating the queue to populate a :py:`obspy.core.stream.Stream` object, then setting up the main plot. The first time through the main loop, the plot is not drawn. After that, the plot is drawn every time all channels are updated. Any plots containing a spectrogram and more than 1 channel are drawn at most every second (1000 ms). All other plots are drawn at most every quarter second (250 ms). """ self.getq() # block until data is flowing from the consumer for i in range((self.totchns) * 2): # fill up a stream object self.getq() self.set_sps() self.deconvolve() self.setup_plot() n = 0 # number of iterations without plotting i = 0 # number of plot events without clearing the linecache u = -1 # number of blocked queue calls (must be -1 at startup) while True: # main loop while True: if self.alive == False: # break if the user has closed the plot break n += 1 self.save_timer += 1 if self.queue.qsize() > 0: self.getq() time.sleep( 0.009 ) # wait a ms to see if another packet will arrive else: u += 1 if self.getq() else 0 if n > (self.delay * RS.numchns): n = 0 break if self.alive == False: # break if the user has closed the plot printM('Exiting.', self.sender) break if i > 10: linecache.clearcache() i = 0 else: i += 1 self.stream = RS.copy(self.stream) self.raw = RS.copy(self.raw) self.deconvolve() self.update_plot() if u >= 0: # avoiding a matplotlib broadcast error self.loop() if self.save: if (self.save_timer > self.save[0][0]): self._eventsave() u = 0 time.sleep(0.005) # wait a ms to see if another packet will arrive sys.stdout.flush() return
def _play(self): printM('Playing alert sound...', sender=self.sender) if 'ffplay' in PLAYER: self._play_quiet() else: play(self.sound) if self.testing: TEST['c_play'][1] = True
def main(): ''' When run from the command line, pass in a value of seconds as an argument to set the packet loss for reporting period. for example, to report packet loss statistics every hour, run the following command (if rsudp is installed in your environment, i.e. activate using ``conda activate rsudp``, then): .. code-block:: bash rs-packetloss -s 3600 -p 18001 ''' hlp_txt = ''' ######################################################## ## R A S P B E R R Y S H A K E ## ## UDP Packet Loss Reporter ## ## by Richard Boaz ## ## Copyleft 2019 ## ## ## ## Reports data packet loss over a specified period ## ## of seconds. ## ## ## ## Supply -p (port) and -f (frequency) to change ## ## the port and frequency to report packet loss ## ## statistics. ## ## ## ## Requires: ## ## - rsudp ## ## ## ## The following example sets the port to 18001 ## ## and report frequency to 1 hour ## ## ## ######################################################## ## ## ## $ rs-packetloss -p 18001 -f 3600 ## ## ## ######################################################## ''' f, p = 60, 8888 opts, args = getopt.getopt(sys.argv[1:], 'hp:f:', ['help', 'port=', 'frequency=']) for o, a in opts: if o in ('-h, --help'): print(hlp_txt) exit(0) if o in ('-p', 'port='): p = int(a) if o in ('-f', 'frequency='): f = int(a) try: run(printFREQ=f, port=p) except KeyboardInterrupt: print('') printM('Quitting...')
def getq(self): d = self.queue.get() self.queue.task_done() if 'TERM' in str(d): self.alive = False printM('Exiting.', self.sender) sys.exit() else: return d
def _print_filt(self): ''' Prints stream filtering information. ''' if self.filt == 'bandpass': printM('Alert stream will be %s filtered from %s to %s Hz' % (self.filt, self.freqmin, self.freqmax), self.sender) elif self.filt in ('lowpass', 'highpass'): modifier = 'below' if self.filt in 'lowpass' else 'above' printM('Alert stream will be %s filtered %s %s Hz' % (self.filt, modifier, self.freq), self.sender)
def _print_rsam(self): """ Print the current RSAM analysis """ if not self.quiet: msg = '%s Current RSAM: mean %s median %s min %s max %s' % ( (self.stream[0].stats.starttime + timedelta( seconds=len(self.stream[0].data) * self.stream[0].stats.delta)).strftime('%Y-%m-%d %H:%M:%S'), self.rsam[0], self.rsam[1], self.rsam[2], self.rsam[3]) printM(msg, self.sender)
def __init__(self, queue, destinations): """ Initialize the process """ super().__init__() self.sender = 'Consumer' printM('Starting.', self.sender) self.queue = queue self.destinations = destinations self.running = True
def signal_handler(signal, frame): ''' The signal handler for the CTRL+C keystroke. :param int signum: signal number :param int frame: frame number ''' print() printM("Quitting...") sys.exit(0)
def _write_wav(self): ''' FFPlay can only play raw wav sounds without verbosity, so to support non-verbose mode we must export to .wav prior to playing a sound. This function checks for an existing wav file and if it does not exist, writes a new one. ''' if not os.path.isfile(self.wavloc): self.sound.export(self.wavloc, format="wav") printM('Wrote wav version of sound file %s' % (self.wavloc), self.sender)