def dataReceived(self, data):
self.buffer += data
self.totalData += len(data)
parseBuffer = DataBuffer(self.buffer)
while parseBuffer.lenLeft() > 0:
packetType = ord(parseBuffer.read(1))
#logging.debug("packet 0x%x" % (packetType,))
try:
format = PACKET_FORMATS[packetType]
except KeyError:
logging.error("invalid packet type - 0x%x %r %r" %
(packetType, len(self.buffer), self.buffer))
logging.error("last 0x%x" % (self.lastPacket, ))
#self.transport.loseConnection()
return
try:
#format = PACKET_FORMATS[PACKET_NEW]
parts = list(format.decode(parseBuffer) or [])
except IncompleteDataError:
break
#pass
self.buffer = parseBuffer.peek()
#interesting benchmark stuffs
#self.packetCounts[packetType] += 1
#self.counter += 1
#if self.counter % 1000 == 0:
# d = time.time()-self.lastCountTime
# logging.debug("1000 in %r - %r packets/s - %r kB/s" % (d, 1000/d, self.totalData/1000/d))
# logging.debug(repr(self.packetCounts.most_common(5)))
# self.lastCountTime = time.time()
# self.totalData = 0
# self.packetCounts.clear()
self.lastPacket = packetType
for handler in self.packetHandlers[packetType]:
try:
ret = handler(parts)
#logging.info(parts)
if ret == False:
return
except Exception as ex:
logging.error("Exception in handling packet 0x%02x:" %
(packetType, ))
logging.exception(ex)
def dataReceived(self, data):
self.buffer += data
self.totalData += len(data)
parseBuffer = DataBuffer(self.buffer)
while parseBuffer.lenLeft() > 0:
packetType = ord(parseBuffer.read(1))
#logging.debug("packet 0x%x" % (packetType,))
try:
format = PACKET_FORMATS[packetType]
except KeyError:
logging.error("invalid packet type - 0x%x %r %r" % (packetType,
len(self.buffer), self.buffer))
logging.error("last 0x%x" % (self.lastPacket,))
#self.transport.loseConnection()
return
try:
#format = PACKET_FORMATS[PACKET_NEW]
parts = list(format.decode(parseBuffer) or [])
except IncompleteDataError:
break
#pass
self.buffer = parseBuffer.peek()
#interesting benchmark stuffs
#self.packetCounts[packetType] += 1
#self.counter += 1
#if self.counter % 1000 == 0:
# d = time.time()-self.lastCountTime
# logging.debug("1000 in %r - %r packets/s - %r kB/s" % (d, 1000/d, self.totalData/1000/d))
# logging.debug(repr(self.packetCounts.most_common(5)))
# self.lastCountTime = time.time()
# self.totalData = 0
# self.packetCounts.clear()
self.lastPacket = packetType
for handler in self.packetHandlers[packetType]:
try:
ret = handler(parts)
#logging.info(parts)
if ret == False:
return
except Exception as ex:
logging.error("Exception in handling packet 0x%02x:" % (packetType,))
logging.exception(ex)
class BaseConnection(object):
def __init__(self):
self.__buffer = DataBuffer()
def dump_buffer(self):
self.__buffer.dump()
def command(self, text, timeout = 1.0, wait_response = True):
data = (text + "\n").encode("utf-8")
self._write(data)
if wait_response and text.endswith("?"):
return self.readline(timeout = timeout)
else:
time.sleep(0.2)
def readline(self, codec = "utf-8", timeout = 1.0):
return self.__buffer.getline(codec = codec, timeout = timeout)
def get(self, length, timeout = 1.0):
return self.__buffer.get(length, timeout = timeout)
def get_tmc_data(self, timeout = 5.0):
header = self.__buffer.get(2)
assert(header[0] == ord("#"))
digit_count = int(chr(header[1]))
data_length = self.__buffer.get(digit_count)
data_length = int(data_length.decode("ascii"))
data = self.__buffer.get(data_length, timeout = timeout)
# Yes, that's pretty stupid. But it sends a newline after binary,
# length-delimited data.
newline = self.__buffer.get(1)
assert(newline[0] == 10)
return data
def _put(self, data):
# print("<-", data)
self.__buffer.put(data)
def _write(self, data):
print("->", data)
self._raw_write(data)
def _raw_write(self, data):
raise Exception(NotImplemented)
def test_stream(self):
buf = DataBuffer()
buf2 = DataBuffer()
buf3 = DataBuffer()
stream = DataStream()
stream.buffer_stream.put(buf)
stream.buffer_stream.put(buf2)
stream.buffer_stream.put(buf3)
stream.stream()
assert(stream.result_stream.get() == buf)
assert(stream.result_stream.empty() == False)
stream.result_stream.task_done()
assert(stream.result_stream.get() == buf2)
assert(stream.result_stream.empty() == False)
stream.result_stream.task_done()
assert(stream.result_stream.get() == buf3)
assert(stream.result_stream.empty() == True)
stream.result_stream.task_done()
assert(stream.result_stream.empty() == True)
def process(self):
counter=0
while counter < self.stream_size and self.translation_finished == False:
Translator.process(self)
reader = self.wb.get_sheet_by_name(self.sheet_name)
skipfirst = 0
for row in reader.rows:
if skipfirst != 0:
buf = DataBuffer()
start_element = row[0].value
#If the row is not empty, populate and push the data buffer
if start_element != '' and start_element is not None:
for col in row:
if col.value != '' and col.value is not None:
buf.append(col.value)
if self.type == 0:
buf.type = self.current_sheet+1
else:
but.type = self.current_sheet+6
buf.next_status()
self.output_queue.put(buf)
#If the row is empty, flip the section finished flag
else:
self.section_finished = True
counter+=1
skipfirst+=1
self.last_read+=self.stream_size
self.section_finished = True
def dataReceived(self, data):
self.buffer += data
parseBuffer = DataBuffer(self.buffer)
while parseBuffer.lenLeft() > 0:
packetType = ord(parseBuffer.read(1))
#logging.debug("packet 0x%x" % (packetType,))
try:
format = PACKET_FORMATS[packetType]
except KeyError:
logging.error("invalid packet type - 0x%x %r %r" %
(packetType, len(self.buffer), self.buffer))
self.transport.loseConnection()
return
try:
parts = list(format.decode(parseBuffer) or [])
except IncompleteDataError:
break
self.buffer = parseBuffer.peek()
self.counter += 1
#TODO: send the keepalive at some set interval
if self.counter % 300 == 0:
self.sendPacked(PACKET_KEEPALIVE)
for handler in self.packetHandlers[packetType]:
try:
ret = handler(parts)
if ret == False:
return
except Exception as ex:
log.err(ex)
def dataReceived(self, data):
self.buffer += data
parseBuffer = DataBuffer(self.buffer)
while parseBuffer.lenLeft() > 0:
packetType = ord(parseBuffer.read(1))
#logging.debug("packet 0x%x" % (packetType,))
try:
format = PACKET_FORMATS[packetType]
except KeyError:
logging.error("invalid packet type - 0x%x %r %r" % (packetType,
len(self.buffer), self.buffer))
self.transport.loseConnection()
return
try:
parts = list(format.decode(parseBuffer) or [])
except IncompleteDataError:
break
self.buffer = parseBuffer.peek()
self.counter += 1
#TODO: send the keepalive at some set interval
if self.counter % 300 == 0:
self.sendPacked(PACKET_KEEPALIVE)
for handler in self.packetHandlers[packetType]:
try:
ret = handler(parts)
if ret == False:
return
except Exception as ex:
log.err(ex)
def process(self):
self.con = lite.connect(self.input_file)
with self.con:
cur = self.con.cursor()
cur.execute("SELECT * FROM {} LIMIT ? OFFSET ?".format(self.table_name), (self.stream_size, self.last_read))
rows = cur.fetchall()
self.last_read+=len(rows)
if len(rows) < self.stream_size:
self.section_finished = True
if rows is not None and len(rows) != 0:
for row in rows:
buf = DataBuffer()
for col in row:
buf.append(col)
if self.type == 0:
buf.type = self.current_table+1
else:
buf.type = self.current_table+1
buf.next_status()
self.output_queue.put(buf)
Translator.process(self)
def process(self):
print('CSVTranslator Process Called')
with open(self.input_file, 'rb') as csvfile:
self.reader = csv.reader(csvfile, delimiter=',', quotechar='|')
#Counter to reset on next section
k=0
#Process Counter
j=0
#Last Read
i=self.last_read
for row in self.reader:
if j - i < self.stream_size and j >= i:
#Check for the type of the data buffer
if row[0] != self.current_type:
self.next_section()
self.current_type = row[0]
print('Current Type set to %s' % (self.current_type))
k=0
else:
k+=1
#Create the data buffer
buf = DataBuffer()
print('Data buffer initialized')
skipfirst = 0
for col in row:
if skipfirst != 0:
buf.append(col)
print('%s appended to data buffer' % (col))
skipfirst+=1
buf.type = self.current_type
#The buffer has been translated
buf.next_status()
print('Buffer type set to %s' % (self.current_type))
self.output_queue.put(buf)
print('Buffer written to queue')
j+=1
self.last_read+=self.stream_size
print('Last read set to %s' % (self.last_read))
def __init__(self): self.__buffer = DataBuffer()
def test_buffer(self):
buf = DataBuffer()
#Append
buf.append(1)
assert(buf.data[0] == 1)
buf.append(2)
assert(buf.data[0] == 1)
assert(buf.data[1] == 2)
buf.append(3)
#Remove
buf.remove(1)
assert(buf.data[0] == 2)
#Clear
buf.clear()
assert(buf.length() == 0)
#Next Status
assert(buf.status == 0)
buf.next_status()
assert(buf.status == 1)
buf.next_status()
assert(buf.status == 2)
buf.next_status()
assert(buf.status == 3)
buf.next_status()
assert(buf.status == 3)
#Add Error
buf.add_error('Test')
assert(buf.error[0] == 'Test')
buf.add_error('Test2')
assert(buf.error[0] == 'Test')
assert(buf.error[1] == 'Test2')
#Clear Erros
buf.clear_error()
assert(len(buf.error) == 0)