def parse_chunks(self):
"""
Parse out any pending data chunks in the chunker. If
it is a valid data piece, build a particle, update the position and
timestamp. Go until the chunker has no more valid data.
@retval a list of tuples with sample particles encountered in this
parsing, plus the state. An empty list of nothing was parsed.
"""
result_particles = []
# all non-data packets will be read along with all the data, so we can't just use the fact that
# there is or is not non-data to determine when a new sequence should occur. The non-data will
# keep getting shifted lower as things get cleaned out, and when it reaches the 0 index the non-data
# is actually next
(nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()
non_data_flag = False
if non_data is not None and non_end <= start:
log.debug('start setting non_data_flag')
non_data_flag = True
sample_count = 0
new_seq = 0
while (chunk != None):
header_match = SIO_HEADER_MATCHER.match(chunk)
sample_count = 0
new_seq = 0
log.debug('parsing header %s', header_match.group(0)[1:32])
if header_match.group(1) == self._instrument_id:
# Check for missing data between records
if non_data_flag or self._new_seq_flag:
log.debug("Non matching data packet detected")
# reset non data flag and new sequence flags now
# that we have made a new sequence
non_data = None
non_data_flag = False
self._new_seq_flag = False
self.start_new_sequence()
# need to figure out if there is a new sequence the first time through,
# since if we are using in process data we don't read unprocessed again
new_seq = 1
# need to do special processing on data to handle escape sequences
# replace 0x182b with 0x2b and 0x1858 into 0x18
processed_match = chunk.replace(b'\x182b', b'\x2b')
processed_match = processed_match.replace(b'\x1858', b'\x18')
log.debug("matched chunk header %s", processed_match[1:32])
data_match = DATA_MATCHER.search(processed_match)
if data_match:
log.debug('Found data match in chunk %s', processed_match[1:32])
# get the time from the header
posix_time = int(header_match.group(3), 16)
# convert from posix to local time
log.debug('utc timestamp %s', datetime.utcfromtimestamp(posix_time))
self._timestamp = ntplib.system_to_ntp_time(float(posix_time))
log.debug("Converted time \"%s\" (unix: %s) into %s", header_match.group(3),
posix_time, self._timestamp)
# particle-ize the data block received, return the record
sample = self._extract_sample(DostadParserDataParticle,
DATA_MATCHER,
data_match.group(0),
self._timestamp)
if sample:
# create particle
result_particles.append(sample)
sample_count += 1
self._chunk_sample_count.append(sample_count)
self._chunk_new_seq.append(new_seq)
(nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()
# need to set a flag in case we read a chunk not matching the instrument ID and overwrite the non_data
if non_data is not None and non_end <= start:
log.debug('setting non_data_flag')
non_data_flag = True
return result_particles
def parse_chunks(self):
"""
Parse out any pending data chunks in the chunker. If
it is a valid data piece, build a particle, update the position and
timestamp. Go until the chunker has no more valid data.
@retval a list of tuples with sample particles encountered in this
parsing, plus the state. An empty list of nothing was parsed.
"""
result_particles = []
(nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()
# all non-data packets will be read along with all the data, so we can't just use the fact that
# there is or is not non-data to determine when a new sequence should occur. The non-data will
# keep getting shifted lower as actual data get cleaned out, so as long as the end of non-data
# is before the start of the data it counts
non_data_flag = False
if non_data is not None and non_end <= start:
non_data_flag = True
sample_count = 0
prev_sample = None
new_seq = 0
while (chunk != None):
header_match = SIO_HEADER_MATCHER.match(chunk)
sample_count = 0
prev_sample = None
new_seq = 0
if header_match.group(1) == self._instrument_id:
# Check for missing data between records
if non_data_flag or self._new_seq_flag:
log.trace("Non matching data packet detected")
# reset non data flag and new sequence flags now
# that we have made a new sequence
non_data_flag = False
self._new_seq_flag = False
# No longer starting new sequences on gaps. for now.
#self.start_new_sequence()
new_seq = 1
# need to do special processing on data to handle escape sequences
# replace 0x182b with 0x2b and 0x1858 into 0x18
chunk = chunk.replace(b'\x182b', b'\x2b')
chunk = chunk.replace(b'\x1858', b'\x18')
log.debug("matched chunk header %s", chunk[1:32])
for data_match in DATA_MATCHER.finditer(chunk):
# check if the end of the last sample connects to the start of the next sample
if prev_sample is not None:
if data_match.start(0) != prev_sample:
log.error('extra data found between samples, leaving out the rest of this chunk')
break
prev_sample = data_match.end(0)
# the timestamp is part of the data, pull out the time stamp
# convert from binary to hex string
asciihex_id = binascii.b2a_hex(data_match.group(0)[0:1])
induct_id = int(asciihex_id, 16)
if induct_id == self._config.get('inductive_id'):
asciihextime = binascii.b2a_hex(data_match.group(1))
# reverse byte order in time hex string
timehex_reverse = asciihextime[6:8] + asciihextime[4:6] + \
asciihextime[2:4] + asciihextime[0:2]
# time is in seconds since Jan 1 2000, convert to timestamp
log.trace("time in hex:%s, in seconds:%d", timehex_reverse,
int(timehex_reverse, 16))
self._timestamp = self._convert_time_to_timestamp(int(timehex_reverse, 16))
# particle-ize the data block received, return the record
sample = self._extract_sample(CtdmoParserDataParticle,
DATA_MATCHER,
data_match.group(0),
self._timestamp)
if sample:
# create particle
result_particles.append(sample)
sample_count += 1
# keep track of how many samples were found in this chunk
self._chunk_sample_count.append(sample_count)
self._chunk_new_seq.append(new_seq)
(nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()
# need to set a flag in case we read a chunk not matching the instrument ID and overwrite the non_data
if non_data is not None and non_end <= start:
non_data_flag = True
return result_particles
def parse_chunks(self):
"""
Parse out any pending data chunks in the chunker. If
it is a valid data piece, build a particle, update the position and
timestamp. Go until the chunker has no more valid data.
@retval a list of tuples with sample particles encountered in this
parsing, plus the state. An empty list of nothing was parsed.
"""
result_particles = []
# all non-data packets will be read along with all the data, so we can't just use the fact that
# there is or is not non-data to determine when a new sequence should occur. The non-data will
# keep getting shifted lower as things get cleaned out, and when it reaches the 0 index the non-data
# is actually next
(nd_timestamp, non_data, non_start,
non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start,
end) = self._chunker.get_next_data_with_index()
non_data_flag = False
if non_data is not None and non_end <= start:
log.debug('start setting non_data_flag')
non_data_flag = True
sample_count = 0
new_seq = 0
while (chunk != None):
header_match = SIO_HEADER_MATCHER.match(chunk)
sample_count = 0
new_seq = 0
log.debug('parsing header %s', header_match.group(0)[1:32])
if header_match.group(1) == self._instrument_id:
# Check for missing data between records
if non_data_flag or self._new_seq_flag:
log.debug("Non matching data packet detected")
# reset non data flag and new sequence flags now
# that we have made a new sequence
non_data = None
non_data_flag = False
self._new_seq_flag = False
self.start_new_sequence()
# need to figure out if there is a new sequence the first time through,
# since if we are using in process data we don't read unprocessed again
new_seq = 1
# need to do special processing on data to handle escape sequences
# replace 0x182b with 0x2b and 0x1858 into 0x18
processed_match = chunk.replace(b'\x182b', b'\x2b')
processed_match = processed_match.replace(b'\x1858', b'\x18')
log.debug("matched chunk header %s", processed_match[1:32])
data_match = DATA_MATCHER.search(processed_match)
if data_match:
log.debug('Found data match in chunk %s',
processed_match[1:32])
# get the time from the header
posix_time = int(header_match.group(3), 16)
# convert from posix to local time
log.debug('utc timestamp %s',
datetime.utcfromtimestamp(posix_time))
self._timestamp = ntplib.system_to_ntp_time(
float(posix_time))
log.debug("Converted time \"%s\" (unix: %s) into %s",
header_match.group(3), posix_time,
self._timestamp)
# particle-ize the data block received, return the record
sample = self._extract_sample(DostadParserDataParticle,
DATA_MATCHER,
data_match.group(0),
self._timestamp)
if sample:
# create particle
result_particles.append(sample)
sample_count += 1
self._chunk_sample_count.append(sample_count)
self._chunk_new_seq.append(new_seq)
(nd_timestamp, non_data, non_start,
non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start,
end) = self._chunker.get_next_data_with_index()
# need to set a flag in case we read a chunk not matching the instrument ID and overwrite the non_data
if non_data is not None and non_end <= start:
log.debug('setting non_data_flag')
non_data_flag = True
return result_particles
def parse_chunks(self):
"""
Parse out any pending data chunks in the chunker. If
it is a valid data piece, build a particle, update the position and
timestamp. Go until the chunker has no more valid data.
@retval a list of tuples with sample particles encountered in this
parsing, plus the state. An empty list of nothing was parsed.
"""
result_particles = []
(nd_timestamp, non_data, non_start,
non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start,
end) = self._chunker.get_next_data_with_index()
# all non-data packets will be read along with all the data, so we can't just use the fact that
# there is or is not non-data to determine when a new sequence should occur. The non-data will
# keep getting shifted lower as actual data get cleaned out, so as long as the end of non-data
# is before the start of the data it counts
non_data_flag = False
if non_data is not None and non_end <= start:
non_data_flag = True
sample_count = 0
prev_sample = None
new_seq = 0
while (chunk != None):
header_match = SIO_HEADER_MATCHER.match(chunk)
sample_count = 0
prev_sample = None
new_seq = 0
if header_match.group(1) == self._instrument_id:
# Check for missing data between records
if non_data_flag or self._new_seq_flag:
log.trace("Non matching data packet detected")
# reset non data flag and new sequence flags now
# that we have made a new sequence
non_data_flag = False
self._new_seq_flag = False
self.start_new_sequence()
new_seq = 1
# need to do special processing on data to handle escape sequences
# replace 0x182b with 0x2b and 0x1858 into 0x18
chunk = chunk.replace(b'\x182b', b'\x2b')
chunk = chunk.replace(b'\x1858', b'\x18')
log.debug("matched chunk header %s", chunk[1:32])
for data_match in DATA_MATCHER.finditer(chunk):
# check if the end of the last sample connects to the start of the next sample
if prev_sample is not None:
if data_match.start(0) != prev_sample:
log.error(
'extra data found between samples, leaving out the rest of this chunk'
)
break
prev_sample = data_match.end(0)
# the timestamp is part of the data, pull out the time stamp
# convert from binary to hex string
asciihextime = binascii.b2a_hex(data_match.group(1))
# reverse byte order in time hex string
timehex_reverse = asciihextime[6:8] + asciihextime[4:6] + \
asciihextime[2:4] + asciihextime[0:2]
# time is in seconds since Jan 1 2000, convert to timestamp
log.trace("time in hex:%s, in seconds:%d", timehex_reverse,
int(timehex_reverse, 16))
self._timestamp = self._convert_time_to_timestamp(
int(timehex_reverse, 16))
# particle-ize the data block received, return the record
sample = self._extract_sample(CtdmoParserDataParticle,
DATA_MATCHER,
data_match.group(0),
self._timestamp)
if sample:
# create particle
result_particles.append(sample)
sample_count += 1
# keep track of how many samples were found in this chunk
self._chunk_sample_count.append(sample_count)
self._chunk_new_seq.append(new_seq)
(nd_timestamp, non_data, non_start,
non_end) = self._chunker.get_next_non_data_with_index(clean=False)
(timestamp, chunk, start,
end) = self._chunker.get_next_data_with_index()
# need to set a flag in case we read a chunk not matching the instrument ID and overwrite the non_data
if non_data is not None and non_end <= start:
non_data_flag = True
return result_particles
