class xferXmodem(xferBase):
maxpayload = -1 # Don't care
increment = 128
last_ok = 0
mode = "xmodem"
def __init__(self, terminal, params={}):
super().__init__(terminal, params)
def connect(self, chip):
def getc(size, timeout=10):
ret = self.term.read(size)
return ret or None
def putc(data, timeout=10):
return self.term.write(data) # note that this ignores the timeout
self.modem = XMODEM(getc, putc, mode=self.mode)
return super().connect(chip)
def _write(self, destaddr, buffer, callback=None):
def wrap_callback(total_packets, success_count, error_count):
if self.last_ok != success_count and success_count != 0:
if callback:
callback(total_packets * self.increment,
success_count * self.increment, self.increment)
self.last_ok = success_count
if not isinstance(buffer, io.IOBase):
buffer = io.BytesIO(buffer)
ln = self.stream_size(buffer)
return self.modem.send(buffer, retry=128, callback=wrap_callback)
def _read(self, srcaddr, length, callback=None):
def wrap_callback(total_packets, success_count, error_count,
packet_size):
if self.last_ok != success_count and success_count != 0:
if callback:
callback(length, success_count * packet_size, packet_size)
self.last_ok = success_count
stream = io.BytesIO()
#HACK: Check if this stuff is merged: https://github.com/tehmaze/xmodem/pull/53
spec = inspect.getargspec(self.modem.recv)
if "callback" in spec.args:
self.modem.recv(stream,
crc_mode=0,
retry=128,
callback=wrap_callback)
else:
print(
"WARN: No progressbar will be shown, because xmodem library is too old"
)
print(
"WARN: Please update (pip install --upgrade xmodem) to see progressbar during readdout"
)
self.modem.recv(stream, crc_mode=0, retry=128)
return stream.getvalue()
def download(filename):
modem = XMODEM(getc, putc)
cmd = 'upload ' + filename
send_cmd(cmd) # b'upload trend.log\n'
receive_line() # b'Start receiving now.\r\n'
stream = open(FILENAME, 'wb')
print('*Start receiving (XMODEM-CRC)')
ser.flushInput()
modem.recv(stream, crc_mode=1)
line = receive_line() # b'28822 Bytes Transmitted.\r\n'
size = int(line.split()[0])
stream.truncate(size)
stream.close()
print('*downloaded ' + filename + ' (' + str(size) + ' bytes)')
return size
def test_xmodem_recv_bad_checksum():
"""
Using external program for send, verify checksum fail in XMODEM.recv().
"""
# Given,
_, send_filename = tempfile.mkstemp()
try:
with open(send_filename, 'wb') as stream:
fill_binary_data(stream)
proc = subprocess.Popen(
(send_prog, '--xmodem', '--verbose', send_filename),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0)
getc = functools.partial(_proc_getc_fail_16bit_checksum, proc=proc)
putc = functools.partial(_proc_putc, proc=proc)
xmodem = XMODEM(getc, putc)
recv_stream = BytesIO()
# Exercise,
status = xmodem.recv(recv_stream, timeout=5, crc_mode=1)
# Verify,
assert status == recv_stream.tell()
verify_binary_data(recv_stream, padding=b'\x1a')
proc.wait()
assert proc.returncode == 0
finally:
os.unlink(send_filename)
def test_xmodem_recv_bad_checksum():
"""
Using external program for send, verify checksum fail in XMODEM.recv().
"""
# Given,
_, send_filename = tempfile.mkstemp()
try:
with open(send_filename, 'wb') as stream:
fill_binary_data(stream)
proc = subprocess.Popen(
(send_prog, '--xmodem', '--verbose', send_filename),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
bufsize=0)
getc = functools.partial(_proc_getc_fail_16bit_checksum, proc=proc)
putc = functools.partial(_proc_putc, proc=proc)
xmodem = XMODEM(getc, putc)
recv_stream = BytesIO()
# Exercise,
status = xmodem.recv(recv_stream, timeout=5, crc_mode=1)
# Verify,
assert status == recv_stream.tell()
verify_binary_data(recv_stream, padding=b'\x1a')
proc.wait()
assert proc.returncode == 0
finally:
os.unlink(send_filename)
def download(outfile):
print("Starting download")
# Initiate xmodem download
exchangeCommand("dx", "CTRL+C to cancel.\r\n")
xm = XMODEM(getc, putc)
print("Downloading", end="", flush=True)
n = xm.recv(outfile, crc_mode=False, retry=102, quiet=False)
print("") # newline
print("Bytes received: " + str(n))
return n
class xferXmodem(xferBase):
increment = 128
mode = "xmodem"
def __init__(self, terminal):
super().__init__(terminal)
def connect(self, chip):
ser = self.term.ser
def getc(size, timeout=10):
ret = ser.read(size)
return ret or None
def putc(data, timeout=10):
return ser.write(data) # note that this ignores the timeout
self.modem = XMODEM(getc, putc, mode=self.mode)
return super().connect(chip)
def _send(self, stream, destaddr, desc="Sending stream"):
total = self.stream_size(stream)
terminal = self.term
increment = self.increment
terminal.progress_start(desc, total)
self.last_ok = 0
def callback(total_packets, success_count, error_count):
if self.last_ok != success_count and success_count != 0:
terminal.progress_update(total_packets * increment,
success_count * increment, increment)
self.last_ok = success_count
ret = self.modem.send(stream, retry=128, callback=callback)
terminal.progress_end()
return ret
def _recv(self, stream, srcaddr, total, desc='Receiving stream'):
terminal = self.term
increment = self.increment
#terminal.progress_start(desc, total)
self.last_ok = 0
def callback(total_packets, success_count, error_count):
if self.last_ok != success_count and success_count != 0:
terminal.progress_update(total_packets * increment,
success_count * increment, increment)
self.last_ok = success_count
#FixMe: modem.recv doesn't support callback mechanism
ret = self.modem.recv(stream, crc_mode=0, retry=128)
#terminal.progress_end()
return ret
class Server(FakeIO, threading.Thread):
def __init__(self, io):
threading.Thread.__init__(self)
self.io = io
self.stream = StringIO()
def getc(self, data, timeout=0):
return self.io.getc(data, 1)
def putc(self, data, timeout=0):
return self.io.putc(data, 1)
def run(self):
self.xmodem = XMODEM(self.getc, self.putc)
print('%s %s' % ('s.recv', file=self.xmodem.recv(self.stream)))
class Server(FakeIO, threading.Thread):
def __init__(self, io):
threading.Thread.__init__(self)
self.io = io
self.stream = StringIO.StringIO()
def getc(self, data, timeout=0):
return self.io.getc(data, 1)
def putc(self, data, timeout=0):
return self.io.putc(data, 1)
def run(self):
self.xmodem = XMODEM(self.getc, self.putc)
print 's.recv', self.xmodem.recv(self.stream)
print 'got'
print self.stream.getvalue()
class Server(FakeIO, threading.Thread):
def __init__(self, io):
threading.Thread.__init__(self)
self.io = io
self.stream = StringIO.StringIO()
def getc(self, data, timeout=0):
return self.io.getc(data, 1)
def putc(self, data, timeout=0):
return self.io.putc(data, 1)
def run(self):
self.xmodem = XMODEM(self.getc, self.putc)
print 's.recv', self.xmodem.recv(self.stream)
print 'got'
print self.stream.getvalue()
def DownloadFileuZ(self, session_id):
modem = XMODEM(self.Serial.getc, self.Serial.putc)
print(
self.Commands.QueryRespond("DOWNLOAD," + str(session_id) +
"XMODEM"))
download_bytes = 0
t1_start = time.perf_counter()
download_bytes = modem.recv(filebuf, crc_mode=1)
t1_stop = time.perf_counter()
elapsed = t1_stop - t1_start
bytes_per_sec = downloadbytes / elapsed
print("Received %s in %f.0 seconds: (%f.0 bytes/sec)" %
(self.Human_Bytes(download_bytes), elapsed, bytes_per_sec))
filebuf.close
self.serial.timeout = self.TIMEOUT
return download_bytes
modem = XMODEM(getc, putc)
from io import BytesIO
text = b'testText'
stream = BytesIO(text)
#stream = open('text.txt', 'rb') # send file over xmodem
ret = modem.send(
stream) # xmodem use ^Z (0x1a) as empty symbol for 128 byte buffer
print('send:', ret)
if ret is True:
#recvBuffer = b''
stream = BytesIO()
#stream = open('get.txt', 'wb') # receive file over xmodem
ret = modem.recv(stream)
print('recv:', ret)
stream.seek(0)
print(stream.read())
"""
stream = open('file.bin', 'rb')
modem.send(stream)
stream = open('output', 'wb')
modem.recv(stream)
"""
def initiateRetrieveBinary(self, device):
device_type = 'Transponder' if 'Slots' in device else 'Meter'
# Functions for the XMODEM utility
def getc(size, timeout=3):
# return self.serial.read(size) or None
self.serial.timeout = timeout
self.serial.inter_byte_timeout = None
r = self.serial.read(size)
if len(r) != size:
return (None)
else:
return (r)
def putc(data, timeout=3):
return self.serial.write(
data) # Note that this ignores the timeout
# Download the binary file with meter readings
self.updateQuery(
"Requesting Events",
"Sending event command with range {} to {} {}".format(
self.query['Range'], device_type,
device['SerialNumber']), True)
self.sendln("event -XIgD -> {}".format(self.query['Range']),
20) # "05/16/2018"
# Wait for the device to be ready again
time.sleep(60)
self.updateQuery("Receiving Events",
"Attempting to receive events in binary", True)
transferred = False
try:
modem = XMODEM(getc, putc)
bin_path = "{}\\{}\\{}-{}.bin".format(self.pathToReadingsDirectory,
self.today, self.query['Id'],
device['Id'])
mdt_path = "{}\\{}\\{}-{}.bin.mdt".format(
self.pathToReadingsDirectory, self.today, self.query['Id'],
device['Id'])
# Create the directory paths if they don't exist
try:
os.makedirs(os.path.dirname(bin_path))
os.makedirs(os.path.dirname(mdt_path))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
stream = open(bin_path, 'wb')
if modem.recv(stream, retry=10, timeout=4, crc_mode=0) != None:
stream.flush()
stream.close()
self.updateQuery(
'Receiving Events',
'Received events for {} {}'.format(device_type,
device['SerialNumber']),
True)
transferred = True
# self.sendln('attn')
# time.sleep(0.5)
# self.sendln('attn')
# time.sleep(0.5)
else:
raise ConnectionError()
except serial.SerialException as e:
msg = 'Could not receive events for {} {}: {}'.format(
device_type, device['SerialNumber'], e)
self.updateQuery('Receiving Events', msg, True, 'warn', error=msg)
except ConnectionError as e:
msg = 'Could not receive events for {} {}: {}'.format(
device_type, device['SerialNumber'], e)
self.updateQuery('Receiving Events', msg, True, 'warn', error=msg)
if transferred:
# Convert the BIN file to MDT format
if not self.convertBinToMdt(bin_path):
msg = 'Could not convert BIN file to MDT format'
self.updateQuery('Converting', msg, True, 'warn', error=msg)
return
# Convert MDT file (which is CSV) to JSON
if not self.convertMdtToJson(mdt_path):
msg = 'Could not convert MDT file to JSON'
self.updateQuery('Converting', msg, True, 'warn', error=msg)
return
class UART:
def __init__(self, port_number):
self.port = config.PORT_NAME[port_number]
self.serial = serial.Serial(
port=self.port,
baudrate=115200,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
bytesize=serial.EIGHTBITS,
timeout=1)
# setup xmodem for image transfers
self.modem = XMODEM(self.getc, self.putc)
self.logger = app_api.logging_setup("UART")
def getc(self, size, timeout=1):
return self.serial.read(size) or None
def putc(self, data, timeout=1):
return self.serial.write(data) # note that this ignores the timeout
def transfer_image(self, filename):
try:
# open uart port
self.serial.close()
self.serial.open()
if self.serial.isOpen():
self.logger.debug("UART port {} is open. Waiting for file...".format(self.port))
stream = open(filename, 'wb+')
result = self.modem.recv(stream)
self.serial.close()
return result
else:
self.serial.close()
self.logger.warn("Could not open serial port for file transfer: {}".format(self.port))
return False
except Exception as e:
self.logger.warn("Exception trying to read file: {} from xmodem stream: {}|{}".format(filename, type(e).__name__,str(e)))
return False
# send message to hardware over uart
def write(self, message):
try:
# open uart port
self.serial.close()
self.serial.open()
if self.serial.isOpen():
# if uart port is open, try to send encoded string message
self.serial.write(str(message + '\r\n').encode('utf-8'))
self.serial.close()
self.logger.debug("UART port {} is open. Sent message: {}".format(self.port, str(message)))
return True
else:
# if could not open uart port, return failure
self.serial.close()
self.logger.warn("Could not open serial port: {}".format(self.port))
return False
# return failure if exception during write/encoding
except Exception as e:
self.logger.warn("Error sending message {} over uart port {}: {}".format(str(message), self.port, str(e)))
self.serial.close()
return False
# get message from hardware over uart
def read(self):
try:
message = None
self.serial.close()
self.serial.open()
if self.serial.isOpen():
# if uart port is open, try to read something
message = self.serial.readline()
message = message.decode('utf-8')
self.logger.debug("Uart port {} is open. Read line: {}".format(self.port,message))
self.serial.close()
return True, message
else:
# if could not open uart port, return failure
self.serial.close()
self.logger.warn("Could not open serial port: {}".format(self.port))
return False, None
# return failure if exception during read/decoding
except Exception as e:
self.logger.warn("Error reading message: {} over uart port {}: {}".format(message, self.port, str(e)))
self.serial.close
return False, None
class appredirector(redirector):
welcome = b'''
RC Module's __ __
_______ ______ ___ / /_ ____ ____ / /_
/ ___/ / / / __ `__ \/ __ \/ __ \/ __ \/ __/
/ / / /_/ / / / / / / /_/ / /_/ / /_/ / /_
/_/ \__,_/_/ /_/ /_/_.___/\____/\____/\__/
boot: Native Debug Environment
boot: host: Hit 'X' for X-Modem upload
'''
def cleanup(self, fatal):
#TODO: Kill the app
self.pipe.kill()
os.unlink(self.tempfile)
return super().cleanup(fatal)
def run(self):
self.nodelay(self.socket)
self._buf = b""
def getc(size, timeout=10):
ready = select.select([self.socket], [], [], timeout)
ret = None
if ready[0]:
ret = self.socket.recv(size, socket.MSG_DONTWAIT)
return ret
def putc(data, timeout=10):
ret = self.socket.sendall(data, socket.MSG_DONTWAIT)
return len(data)
self.modem = XMODEM(getc, putc, mode="xmodem1k")
self.socket.sendall(self.welcome, socket.MSG_DONTWAIT)
while self.socket.recv(1) != b"X":
pass
#Dirty Workaround: Some ethernet controllers (xilinx) on prototype board
#Dirty Workaround: cause a huge wait when instructed to send just ONE byte
ret = self.socket.sendall(b"CC", socket.MSG_DONTWAIT)
tmp = tempfile.NamedTemporaryFile(mode="wb+",
prefix="rumboot_daemon_temp_",
delete=False)
try:
self.modem.recv(tmp)
except Exception as e:
self.socket.sendall(str(e), socket.MSG_DONTWAIT)
return 0
tmp.close()
os.chmod(tmp.name, 755)
self.pipe = subprocess.Popen([tmp.name],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self.tempfile = tmp.name
return super().run()
def readSerial(self):
return self.pipe.stdout.read()
def writeSerial(self, data):
return self.pipe.stdin.write(data)
def loop_once(self):
fromserial = bytearray(b"")
fromsocket = bytearray(b"")
ready_to_read, ready_to_write, in_error = select.select(
[self.socket, self.pipe.stdout], [self.socket, self.pipe.stdin],
[self.socket], 1)
for sock in ready_to_read:
if sock == self.pipe.stdout:
fromserial = fromserial + self.readSerial()
if self.socket in ready_to_write:
sent = self.socket.send(fromserial)
del fromserial[0:sent]
if sock == self.socket:
tmp = self.socket.recv(1024)
if len(tmp) == 0:
self.cleanup(False)
return False
fromsocket = fromsocket + bytearray(tmp)
if self.pipe.stdin in ready_to_write:
ret = self.writeSerial(fromsocket)
del fromsocket[0:ret]
for sock in in_error:
if sock == self.serial.fileno():
print("Something bad with serial port")
self.cleanup(True)
return False
if sock == self.socket:
print("Disconnect?")
self.cleanup(False)
return False
if self.pipe.poll() is not None:
self.socket.sendall(
f"boot: host: Back in rom, code {self.pipe.poll()}\n".encode())
self.cleanup(False)
return False
return True
print "Transfer files between remote machine and host - uses XMODEM protocol" print "at 115200 baud rate." print "" print "Usage:" print " python transfile filename (from/to) serialport" print "" print "Example, put MBASIC.COM to emulated machine:" print " python transfile.py MBASIC.COM to /dev/cu.SLAB_USBtoUART" print "Example, get MBASIC.COM from emulated machine:" print " python transfile.py MBASIC.COM from /dev/cu.SLAB_USBtoUART" sys.exit() filename = sys.argv[1] cmd = sys.argv[2].lower() # from/to serialPort = sys.argv[3] ser = serial.Serial(serialPort, baudrate=115200, timeout=None) modem = XMODEM(getc, putc) if cmd == 'from': # copy from remote machine stream = open(filename, 'wb') modem.recv(stream) if cmd == 'to': # copy to remote machine stream = open(filename, 'rb') modem.send(stream)
def getc(size, timeout=3):
w, t, f = select.select([so], [], [], timeout)
if w:
data = so.read(size)
else:
data = None
print 'getc(', repr(data), ')'
return data
def putc(data, timeout=3):
w, t, f = select.select([], [si], [], timeout)
if t:
si.write(data)
si.flush()
size = len(data)
else:
size = None
print 'putc(', repr(data), repr(size), ')'
return size
stream = StringIO.StringIO()
xmodem = XMODEM(getc, putc)
nbytes = xmodem.recv(stream, retry=8)
print >> sys.stderr, 'received', nbytes, 'bytes'
print >> sys.stderr, stream.getvalue()
sys.exit(int(nbytes == 0))
w,t,f = select.select([so], [], [], timeout)
if w:
data = so.read(size)
else:
data = None
print 'getc(', repr(data), ')'
return data
def putc(data, timeout=3):
w,t,f = select.select([], [si], [], timeout)
if t:
si.write(data)
si.flush()
size = len(data)
else:
size = None
print 'putc(', repr(data), repr(size), ')'
return size
stream = StringIO.StringIO()
xmodem = XMODEM(getc, putc)
nbytes = xmodem.recv(stream, retry=8)
print >> sys.stderr, 'received', nbytes, 'bytes'
print >> sys.stderr, stream.getvalue()
sys.exit(int(nbytes == 0))
