def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n'): """ ``` port - UDP port on which to write records. prefix - If non-empty, prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud ``` """ self.port = port self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform(fields='1:') # strip off timestamp self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False
def __init__(self, port, filebase, instrument): """ ``` port - UDP port on which to write records. filebase - Prefix string to be matched (with a following "*") to fine files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud instrument - Instrument name prefix to add before sendind out on wire ``` """ self.filebase = filebase self.reader = LogfileReader(filebase=filebase, use_timestamps=True) self.slice_n = SliceTransform( fields='1:') # grab 2nd and subsequent fields self.timestamp = TimestampTransform() self.prefix = PrefixTransform(instrument) self.writer = UDPWriter(port=port) self.instrument = instrument self.first_time = True self.quit_flag = False
def __init__(self, port, filebase=None, record_format=None, time_format=TIME_FORMAT, eol='\n'): """ ``` port - UDP port on which to write records. time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud record_format If specified, a custom record format to use for extracting timestamp and record. The default is '{timestamp:ti} {record}' ``` """ self.port = port self.time_format = time_format self.filebase = filebase self.record_format = record_format or '{timestamp:ti} {record}' self.compiled_record_format = parse.compile(self.record_format) # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, record_format=self.record_format, time_format=self.time_format) self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False
class SimNetwork: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, filebase, instrument): """ ``` port - UDP port on which to write records. filebase - Prefix string to be matched (with a following "*") to fine files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud instrument - Instrument name prefix to add before sendind out on wire ``` """ self.filebase = filebase self.reader = LogfileReader(filebase=filebase, use_timestamps=True) self.slice_n = SliceTransform( fields='1:') # grab 2nd and subsequent fields self.timestamp = TimestampTransform() self.prefix = PrefixTransform(instrument) self.writer = UDPWriter(port=port) self.instrument = instrument self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s', self.instrument) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.instrument) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True) continue # Strip off timestamp and tack on a new one record = self.slice_n.transform(record) record = self.timestamp.transform(record) # Add instrument name back on, and write to specified network record = self.prefix.transform(record) self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.instrument)
class SimUDP: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n'): """ ``` port - UDP port on which to write records. prefix - If non-empty, prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud ``` """ self.port = port self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform(fields='1:') # strip off timestamp self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s: %s', self.port, self.filebase) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.prefix) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True) continue # Strip off timestamp record = self.slice_n.transform(record) if not record: continue record = record.strip() if not record: continue if self.timestamp: # do we want to add a timestamp? record = self.timestamp.transform(record) if self.prefix: # do we want to add prefix? record = self.prefix.transform(record) self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.prefix)
if new_args.write_udp: eol = all_args.network_eol for addr_str in new_args.write_udp.split(','): addr = addr_str.split(':') dest = '' interface = '' port = int(addr[-1]) # port is last arg if len(addr) > 1: dest = addr[-2] # destination (multi/broadcast) is prev arg if len(addr) > 2: interface = addr[-3] # interface is first arg if len(addr) > 3: parser.error('Format error for --write_udp argument. Format ' 'should be [[interface:]destination:]port') writers.append(UDPWriter(port=port, destination=dest, interface=interface, eol=eol)) if new_args.write_redis: for channel in new_args.write_redis.split(','): writers.append(RedisWriter(channel=channel)) if new_args.write_record_screen: writers.append(RecordScreenWriter()) if new_args.write_database: password = all_args.database_password # Parse out values for user@host:database. We count on # --database_password having been specified somewhere. (user, host_db) = new_args.write_database.split('@') (host, database) = host_db.split(':') writers.append(DatabaseWriter(database=database, host=host,
class SimUDP: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, filebase=None, record_format=None, time_format=TIME_FORMAT, eol='\n'): """ ``` port - UDP port on which to write records. time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud record_format If specified, a custom record format to use for extracting timestamp and record. The default is '{timestamp:ti} {record}' ``` """ self.port = port self.time_format = time_format self.filebase = filebase self.record_format = record_format or '{timestamp:ti} {record}' self.compiled_record_format = parse.compile(self.record_format) # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, record_format=self.record_format, time_format=self.time_format) self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s: %s', self.port, self.filebase) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.prefix) self.reader = LogfileReader( filebase=self.filebase, record_format=self.record_format, use_timestamps=True) continue # We've now got a record. Try parsing timestamp off it try: parsed_record = self.compiled_record_format.parse( record).named record = parsed_record['record'] # We had a problem parsing. Discard record and try reading next one. except (KeyError, ValueError, AttributeError): logging.warning('Unable to parse record into "%s"', self.record_format) logging.warning('Record: %s', record) continue if not record: continue self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.prefix)
def write_udp(self, port, dest, data, eol=None, interval=0, delay=0): writer = UDPWriter(port=port, destination=dest, eol=eol) time.sleep(delay) for line in data: writer.write(line) time.sleep(interval)