def _process_metadata_dict(self):
"""
This method process a full metadata dictionary which results in
the generation of a metadata particle
:return: None
"""
file_time = self._metadata_dict[Pco2wAbcDataParticleKey.FILE_TIME]
if file_time is None:
message = "Unable to create metadata particle due to missing file time"
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
else:
utc_timestamp = formatted_timestamp_utc_time(
file_time, "%Y%m%d %H%M%S")
ntp_timestamp = float(ntplib.system_to_ntp_time(utc_timestamp))
particle = self._extract_sample(self._metadata_class,
None,
self._metadata_dict,
internal_timestamp=ntp_timestamp)
self._record_buffer.append(particle)
def _generate_metadata_particle(self):
"""
This function generates a metadata particle.
"""
if self._metadata_matches_dict[MetadataMatchKey.FILE_TIME_MATCH] is None:
message = "Unable to create metadata particle due to missing file time"
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
else:
particle_data = dict()
for key in self._metadata_matches_dict.keys():
log.trace('key: %s, particle_data: %s', key, particle_data)
if self._metadata_matches_dict[key]:
self._process_metadata_match_dict(key, particle_data)
utc_time = formatted_timestamp_utc_time(
particle_data[PhsenAbcdefImodemDataParticleKey.FILE_TIME],
"%Y%m%d %H%M%S")
ntp_timestamp = ntplib.system_to_ntp_time(utc_time)
# Generate the metadata particle class and add the
# result to the list of particles to be returned.
particle = self._extract_sample(self._metadata_particle_class,
None,
particle_data,
ntp_timestamp)
if particle is not None:
log.trace("Appending metadata particle to record buffer: %s", particle.generate())
self._record_buffer.append(particle)
def _process_metadata_dict(self):
"""
This method process a full metadata dictionary which results in
the generation of a metadata particle
:return: None
"""
file_time = self._metadata_dict[Pco2wAbcDataParticleKey.FILE_TIME]
if file_time is None:
message = "Unable to create metadata particle due to missing file time"
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
else:
utc_timestamp = formatted_timestamp_utc_time(file_time,
"%Y%m%d %H%M%S")
ntp_timestamp = float(ntplib.system_to_ntp_time(utc_timestamp))
particle = self._extract_sample(
self._metadata_class, None,
self._metadata_dict, ntp_timestamp)
self._record_buffer.append(particle)
def _generate_metadata_particle(self):
"""
This function generates a metadata particle.
"""
particle_data = dict()
for key in self._metadata_matches_dict.keys():
self._process_metadata_match_dict(key, particle_data)
utc_time = formatted_timestamp_utc_time(
particle_data[CtdmoGhqrImodemDataParticleKey.DATE_TIME_STRING],
"%Y%m%d %H%M%S")
ntp_time = ntplib.system_to_ntp_time(utc_time)
# Generate the metadata particle class and add the
# result to the list of particles to be returned.
particle = self._extract_sample(self.metadata_particle_class,
None,
particle_data,
internal_timestamp=ntp_time)
if particle is not None:
log.debug("Appending metadata particle to record buffer")
self._record_buffer.append(particle)
def _generate_metadata_particle(self):
"""
This function generates a metadata particle.
"""
particle_data = dict()
for key in self._metadata_matches_dict.keys():
self._process_metadata_match_dict(key, particle_data)
utc_time = formatted_timestamp_utc_time(
particle_data[CtdmoGhqrImodemDataParticleKey.DATE_TIME_STRING],
"%Y%m%d %H%M%S")
ntp_time = ntplib.system_to_ntp_time(utc_time)
# Generate the metadata particle class and add the
# result to the list of particles to be returned.
particle = self._extract_sample(self.metadata_particle_class,
None,
particle_data,
ntp_time)
if particle is not None:
log.debug("Appending metadata particle to record buffer")
self._record_buffer.append(particle)
def process_wave(self, fields):
date_time_str = fields[1] + SPACE + fields[0] # concatenate date and time
date_time_utc = formatted_timestamp_utc_time(date_time_str, DATE_TIME_FORMAT)
date_time_ntp = ntplib.system_to_ntp_time(date_time_utc)
self._eng_data[7] = date_time_ntp
self._eng_data[8:] = fields[2:]
return
def __init__(self,
config,
stream_handle,
filename,
exception_callback):
self.timer_diff = None
self._timer_start = None
self._timer_rollover = 0
self._start_time_utc = utilities.formatted_timestamp_utc_time(filename[:15],
"%Y%m%d_%H%M%S")
try:
# Get the particle classes to publish from the configuration
particle_classes_dict = config.get(DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT)
self._accel_particle_class = particle_classes_dict.get(MopakParticleClassType.ACCEL_PARTICLE_CLASS)
self._rate_particle_class = particle_classes_dict.get(MopakParticleClassType.RATE_PARTICLE_CLASS)
except Exception:
log.error('Parser configuration missing or incorrect')
raise ConfigurationException
super(MopakODclParser, self).__init__(config,
stream_handle,
exception_callback)
def parse_file(self):
"""
Parse the *.mp4 file.
"""
match = FILE_PATH_MATCHER.match(self._stream_handle.name)
if match:
file_datetime = match.group('Date') + match.group('Time')
time_stamp = ntplib.system_to_ntp_time(
utilities.formatted_timestamp_utc_time(file_datetime,
TIMESTAMP_FORMAT))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(CamhdAInstrumentDataParticle,
None,
match.group('Path'),
internal_timestamp=time_stamp)
log.debug('Parsed particle: %s', particle.generate_dict())
self._record_buffer.append(particle)
else:
# Files retrieved from the instrument should always match the timestamp naming convention
self.recov_exception_callback(
"Unable to extract file time from input file name: %s."
"Expected format REFDES-YYYYmmddTHHMMSSZ.mp4" %
self._stream_handle.name)
def _generate_metadata_particle(self):
"""
This function generates a metadata particle.
"""
if self._metadata_matches_dict[
MetadataMatchKey.FILE_TIME_MATCH] is None:
message = "Unable to create metadata particle due to missing file time"
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
else:
particle_data = dict()
for key in self._metadata_matches_dict.keys():
log.trace('key: %s, particle_data: %s', key, particle_data)
if self._metadata_matches_dict[key]:
self._process_metadata_match_dict(key, particle_data)
utc_time = formatted_timestamp_utc_time(
particle_data[PhsenAbcdefImodemDataParticleKey.FILE_TIME],
"%Y%m%d %H%M%S")
ntp_timestamp = ntplib.system_to_ntp_time(utc_time)
# Generate the metadata particle class and add the
# result to the list of particles to be returned.
particle = self._extract_sample(self._metadata_particle_class,
None,
particle_data,
internal_timestamp=ntp_timestamp)
if particle is not None:
log.trace("Appending metadata particle to record buffer: %s",
particle.generate())
self._record_buffer.append(particle)
def __init__(self, config, state, stream_handle, filename, state_callback,
publish_callback, exception_callback, *args, **kwargs):
self.timer_diff = None
self._read_state = {
StateKey.POSITION: 0,
StateKey.TIMER_ROLLOVER: 0,
StateKey.TIMER_START: None
}
# convert the date / time string from the file name to a starting time in seconds UTC
self._start_time_utc = utilities.formatted_timestamp_utc_time(
filename[:15], "%Y%m%d_%H%M%S")
try:
# Get the particle classes to publish from the configuration
particle_classes_dict = config.get(
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT)
self._accel_particle_class = particle_classes_dict.get(
MopakParticleClassType.ACCEL_PARTCICLE_CLASS)
self._rate_particle_class = particle_classes_dict.get(
MopakParticleClassType.RATE_PARTICLE_CLASS)
except Exception as e:
log.error('Parser configuration missing or incorrect')
raise ConfigurationException
super(MopakODclParser,
self).__init__(config, stream_handle, state, self.sieve_function,
state_callback, publish_callback,
exception_callback)
if state:
self.set_state(state)
def process_date(self, fields):
date_time_str = fields[0] + SPACE + fields[1] # concatenate date and time
date_time_utc = formatted_timestamp_utc_time(date_time_str, DATE_TIME_FORMAT)
date_time_utc += float(fields[2])*3600 # adjust for timezone (%z format is not supported in Python 2.7)
date_time_ntp = ntplib.system_to_ntp_time(date_time_utc)
self._eng_data[0] = date_time_ntp
return
def parse_file(self):
"""
Parse the zplsc_c log file (averaged condensed data).
Read file line by line. Values are extracted from lines containing condensed ASCII data
@return: dictionary of data values with the particle names as keys or None
"""
# Loop over all lines in the data file and parse the data to generate particles
for number, line in enumerate(self._stream_handle, start=1):
# Check if this is the dcl status log
match = DCL_LOG_MATCHER.match(line)
if match is not None:
log.trace("MATCHED DCL_LOG_MATCHER: %s: %s", number,
match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument phase status log
match = PHASE_STATUS_MATCHER.match(line)
if match is not None:
log.trace("MATCHED PHASE_STATUS_MATCHER: %s: %s", number,
match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument condensed ASCII data
match = SENSOR_DATA_MATCHER.match(line)
if match is not None:
log.trace("MATCHED SENSOR_DATA_MATCHER: %s: %s", number,
match.groups())
# Extract the condensed ASCII data from this line
data_dict = self.parse_line(match)
if data_dict is None:
log.error('Erroneous data found in line %s: %s', number,
line)
continue
# Convert the DCL timestamp into the particle timestamp
time_stamp = ntplib.system_to_ntp_time(
utilities.formatted_timestamp_utc_time(
match.group('dcl_timestamp'),
utilities.DCL_CONTROLLER_TIMESTAMP_FORMAT))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(ZplscCInstrumentDataParticle,
None, data_dict, time_stamp)
if particle is not None:
log.trace('Parsed particle: %s' % particle.generate_dict())
self._record_buffer.append(particle)
continue
# Error, line did not match any expected regex
self._exception_callback(
RecoverableSampleException('Unknown data found in line %s:%s' %
(number, line)))
def parse_file(self):
"""
Parse the zplsc_c log file (averaged condensed data).
Read file line by line. Values are extracted from lines containing condensed ASCII data
@return: dictionary of data values with the particle names as keys or None
"""
# Loop over all lines in the data file and parse the data to generate particles
for number, line in enumerate(self._stream_handle, start=1):
# Check if this is the dcl status log
match = DCL_LOG_MATCHER.match(line)
if match is not None:
log.trace("MATCHED DCL_LOG_MATCHER: %s: %s", number, match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument phase status log
match = PHASE_STATUS_MATCHER.match(line)
if match is not None:
log.trace("MATCHED PHASE_STATUS_MATCHER: %s: %s", number, match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument condensed ASCII data
match = SENSOR_DATA_MATCHER.match(line)
if match is not None:
log.trace("MATCHED SENSOR_DATA_MATCHER: %s: %s", number, match.groups())
# Extract the condensed ASCII data from this line
data_dict = self.parse_line(match)
if data_dict is None:
log.error('Erroneous data found in line %s: %s', number, line)
continue
# Convert the DCL timestamp into the particle timestamp
time_stamp = ntplib.system_to_ntp_time(
utilities.formatted_timestamp_utc_time(
match.group('dcl_timestamp'), utilities.DCL_CONTROLLER_TIMESTAMP_FORMAT))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(
ZplscCInstrumentDataParticle, None, data_dict, time_stamp)
if particle is not None:
log.trace('Parsed particle: %s' % particle.generate_dict())
self._record_buffer.append(particle)
continue
# Error, line did not match any expected regex
self._exception_callback(
RecoverableSampleException('Unknown data found in line %s:%s' % (number, line)))
def parse_file(self):
"""
Parse the *.mp4 file.
"""
match = FILE_PATH_MATCHER.match(self._stream_handle.name)
if match:
file_datetime = match.group('Date') + match.group('Time')
time_stamp = ntplib.system_to_ntp_time(
utilities.formatted_timestamp_utc_time(file_datetime, TIMESTAMP_FORMAT))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(CamhdAInstrumentDataParticle, None, match.group('Path'),
time_stamp)
log.debug('Parsed particle: %s', particle.generate_dict())
self._record_buffer.append(particle)
else:
# Files retrieved from the instrument should always match the timestamp naming convention
self.recov_exception_callback("Unable to extract file time from input file name: %s."
"Expected format REFDES-YYYYmmddTHHMMSSZ.mp4" % self._stream_handle.name)
def __init__(self,
config,
state,
stream_handle,
filename,
state_callback,
publish_callback,
exception_callback,
*args, **kwargs):
self.timer_diff = None
self._read_state = {StateKey.POSITION: 0, StateKey.TIMER_ROLLOVER: 0, StateKey.TIMER_START: None}
# convert the date / time string from the file name to a starting time in seconds UTC
self._start_time_utc = utilities.formatted_timestamp_utc_time(filename[:15],
"%Y%m%d_%H%M%S")
try:
# Get the particle classes to publish from the configuration
particle_classes_dict = config.get(DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT)
self._accel_particle_class = particle_classes_dict.get(MopakParticleClassType.ACCEL_PARTCICLE_CLASS)
self._rate_particle_class = particle_classes_dict.get(MopakParticleClassType.RATE_PARTICLE_CLASS)
except Exception as e:
log.error('Parser configuration missing or incorrect')
raise ConfigurationException
super(MopakODclParser, self).__init__(config,
stream_handle,
state,
self.sieve_function,
state_callback,
publish_callback,
exception_callback)
if state:
self.set_state(state)
def parse_file(self):
"""
Parse through the file, pulling single lines and comparing to
the established patterns, generating particles for data lines
"""
for line in self._stream_handle:
message = 'data line \n%s' % line
log.debug(message)
# First check for valid FLORT DJ DCL data
# If this is a valid sensor data record,
# use the extracted fields to generate a particle.
sensor_match = SENSOR_DATA_MATCHER.match(line)
if sensor_match is not None:
self._particle_class._data_particle_map = INSTRUMENT_PARTICLE_MAP
log.debug('FLORT DJ match found')
else:
log.debug('FLORT DJ match NOT found')
# check for a match against the FLORT D data in a combined
# CTDBP FLORT instrument record
sensor_match = CTDBP_FLORT_MATCHER.match(line)
if sensor_match is not None:
self._particle_class._data_particle_map = CTDBP_FLORT_PARTICLE_MAP
log.debug('check for CTDBP/FLORT match')
if sensor_match is not None:
# FLORT data matched against one of the patterns
log.debug('record found')
# DCL Controller timestamp is the port_timestamp
dcl_controller_timestamp = sensor_match.groups()[SENSOR_GROUP_TIMESTAMP]
port_timestamp = dcl_time_to_ntp(dcl_controller_timestamp)
if self._particle_class._data_particle_map == INSTRUMENT_PARTICLE_MAP:
# For valid FLORT DJ data, Instrument timestamp is the internal_timestamp
instrument_timestamp = sensor_match.groups()[SENSOR_GROUP_SENSOR_DATE] \
+ ' ' + sensor_match.groups()[SENSOR_GROUP_SENSOR_TIME]
internal_timestamp = timestamp_mmddyyhhmmss_to_ntp(instrument_timestamp)
else:
# _data_particle_map is CTDBP_FLORT_PARTICLE_MAP
utc_time = formatted_timestamp_utc_time(sensor_match.groups()[CTDBP_FLORT_GROUP_DATE_TIME],
"%d %b %Y %H:%M:%S")
instrument_timestamp = ntplib.system_to_ntp_time(utc_time)
internal_timestamp = instrument_timestamp
# using port_timestamp as preferred_ts because internal_timestamp is not accurate
particle = self._extract_sample(self._particle_class,
None,
sensor_match.groups(),
port_timestamp=port_timestamp,
internal_timestamp=internal_timestamp,
preferred_ts=DataParticleKey.PORT_TIMESTAMP)
# increment state for this chunk even if we don't
# get a particle
self._record_buffer.append(particle)
# It's not a sensor data record, see if it's a metadata record.
else:
log.debug('No data recs found, check for meta record')
# If it's a valid metadata record, ignore it.
# Otherwise generate warning for unknown data.
meta_match = METADATA_MATCHER.match(line)
if meta_match is None:
error_message = 'Unknown data found in chunk %s' % line
log.warn(error_message)
self._exception_callback(UnexpectedDataException(error_message))
def parse_file(self):
"""
Parse the FCoeff*.txt file. Create a chunk from valid data in the file.
Build a data particle from the chunk.
"""
file_time_dict = {}
dir_freq_dict = {}
freq_band_dict = {}
sensor_data_dict = {
AdcptMFCoeffParticleKey.FREQ_BAND: [],
AdcptMFCoeffParticleKey.BANDWIDTH_BAND: [],
AdcptMFCoeffParticleKey.ENERGY_BAND: [],
AdcptMFCoeffParticleKey.DIR_BAND: [],
AdcptMFCoeffParticleKey.A1_BAND: [],
AdcptMFCoeffParticleKey.B1_BAND: [],
AdcptMFCoeffParticleKey.A2_BAND: [],
AdcptMFCoeffParticleKey.B2_BAND: [],
AdcptMFCoeffParticleKey.CHECK_BAND: []
}
# Extract the file time from the file name
input_file_name = self._stream_handle.name
match = FILE_NAME_MATCHER.match(input_file_name)
if match:
file_time_dict = match.groupdict()
else:
self.recov_exception_callback(
'Unable to extract file time from FCoeff input file name: %s ' % input_file_name)
# read the first line in the file
line = self._stream_handle.readline()
while line:
if EMPTY_LINE_MATCHER.match(line):
# ignore blank lines, do nothing
pass
elif HEADER_MATCHER.match(line):
# we need header records to extract useful information
for matcher in HEADER_MATCHER_LIST:
header_match = matcher.match(line)
if header_match is not None:
if matcher is DIR_FREQ_MATCHER:
dir_freq_dict = header_match.groupdict()
elif matcher is FREQ_BAND_MATCHER:
freq_band_dict = header_match.groupdict()
else:
#ignore
pass
elif FCOEFF_DATA_MATCHER.match(line):
# Extract a row of data
sensor_match = FCOEFF_DATA_MATCHER.match(line)
sensor_data_dict[AdcptMFCoeffParticleKey.FREQ_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.FREQ_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.BANDWIDTH_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.BANDWIDTH_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.ENERGY_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.ENERGY_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.DIR_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.DIR_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.A1_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.A1_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.B1_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.B1_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.A2_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.A2_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.B2_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.B2_BAND))
sensor_data_dict[AdcptMFCoeffParticleKey.CHECK_BAND].append(
sensor_match.group(AdcptMFCoeffParticleKey.CHECK_BAND))
else:
# Generate a warning for unknown data
self.recov_exception_callback('Unexpected data found in line %s' % line)
# read the next line in the file
line = self._stream_handle.readline()
# Construct parsed data list to hand over to the Data Particle class for particle creation
# Make all the collected data effectively into one long dictionary
parsed_dict = dict(chain(file_time_dict.iteritems(),
dir_freq_dict.iteritems(),
freq_band_dict.iteritems(),
sensor_data_dict.iteritems()))
error_flag = False
# Check if all parameter data is accounted for
for name in FCOEFF_ENCODING_RULES:
try:
if parsed_dict[name[0]]:
log.trace("parsed_dict[%s]: %s", name[0], parsed_dict[name[0]])
except KeyError:
self.recov_exception_callback('Missing particle data: %s' % name[0])
error_flag = True
# Don't create a particle if data is missing
if error_flag:
return
# Check if the specified number of frequencies were retrieved from the data
fcoeff_data_length = len(sensor_data_dict[AdcptMFCoeffParticleKey.FREQ_BAND])
if fcoeff_data_length != int(dir_freq_dict[AdcptMFCoeffParticleKey.NUM_FREQ]):
self.recov_exception_callback(
'Unexpected number of frequencies in FCoeff Matrix: expected %s, got %s'
% (dir_freq_dict[AdcptMFCoeffParticleKey.NUM_FREQ], fcoeff_data_length))
# Don't create a particle if data is missing
return
# Convert the filename timestamp into the particle timestamp
time_stamp = ntplib.system_to_ntp_time(
utilities.formatted_timestamp_utc_time(file_time_dict[AdcptMFCoeffParticleKey.FILE_TIME]
, TIMESTAMP_FORMAT))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(AdcptMFCoeffInstrumentDataParticle,
None, parsed_dict, internal_timestamp=time_stamp)
log.trace('Parsed particle: %s' % particle.generate_dict())
self._record_buffer.append(particle)
def parse_file(self):
"""
Parse through the file, pulling single lines and comparing to
the established patterns, generating particles for data lines
"""
for line in self._stream_handle:
message = 'data line \n%s' % line
log.debug(message)
# First check for valid FLORT DJ DCL data
# If this is a valid sensor data record,
# use the extracted fields to generate a particle.
sensor_match = SENSOR_DATA_MATCHER.match(line)
if sensor_match is not None:
self._particle_class._data_particle_map = INSTRUMENT_PARTICLE_MAP
log.debug('FLORT DJ match found')
else:
log.debug('FLORT DJ match NOT found')
# check for a match against the FLORT D data in a combined
# CTDBP FLORT instrument record
sensor_match = CTDBP_FLORT_MATCHER.match(line)
if sensor_match is not None:
self._particle_class._data_particle_map = CTDBP_FLORT_PARTICLE_MAP
log.debug('check for CTDBP/FLORT match')
if sensor_match is not None:
# FLORT data matched against one of the patterns
log.debug('record found')
# DCL Controller timestamp is the port_timestamp
dcl_controller_timestamp = sensor_match.groups()[SENSOR_GROUP_TIMESTAMP]
port_timestamp = dcl_time_to_ntp(dcl_controller_timestamp)
if self._particle_class._data_particle_map == INSTRUMENT_PARTICLE_MAP:
# For valid FLORT DJ data, Instrument timestamp is the internal_timestamp
instrument_timestamp = sensor_match.groups()[SENSOR_GROUP_SENSOR_DATE] \
+ ' ' + sensor_match.groups()[SENSOR_GROUP_SENSOR_TIME]
internal_timestamp = timestamp_mmddyyhhmmss_to_ntp(instrument_timestamp)
else:
# _data_particle_map is CTDBP_FLORT_PARTICLE_MAP
utc_time = formatted_timestamp_utc_time(sensor_match.groups()[CTDBP_FLORT_GROUP_DATE_TIME],
"%d %b %Y %H:%M:%S")
instrument_timestamp = ntplib.system_to_ntp_time(utc_time)
internal_timestamp = instrument_timestamp
# using port_timestamp as preferred_ts because internal_timestamp is not accurate
particle = self._extract_sample(self._particle_class,
None,
sensor_match.groups(),
port_timestamp=port_timestamp,
internal_timestamp=internal_timestamp,
preferred_ts=DataParticleKey.PORT_TIMESTAMP)
# increment state for this chunk even if we don't
# get a particle
self._record_buffer.append(particle)
# It's not a sensor data record, see if it's a metadata record.
else:
log.debug('No data recs found, check for meta record')
# If it's a valid metadata record, ignore it.
# Otherwise generate warning for unknown data.
meta_match = METADATA_MATCHER.match(line)
if meta_match is None:
error_message = 'Unknown data found in chunk %s' % line
log.warn(error_message)
self._exception_callback(UnexpectedDataException(error_message))