class FilesystemOutboundMessageQueue(wolk.OutboundMessageQueue):
def __init__(self, path="."):
if path == ".":
self.queue = PersistentQueue("FileOutboundMessageQueue")
else:
self.queue = PersistentQueue("FileOutboundMessageQueue", path)
def put(self, message):
self.queue.push(message)
def get(self):
message = self.queue.pop()
self.queue.flush()
return message
def peek(self):
if not self.queue.peek():
self.queue.clear()
return None
else:
return self.queue.peek()
class TestPersistentQueue:
def setup_method(self):
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}.queue'.format(self.__class__.__name__, random)
self.queue = PersistentQueue(filename)
def teardown_method(self):
if os.path.isfile(self.queue.filename):
os.remove(self.queue.filename)
def test_simple(self):
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}.queue'.format(self.__class__.__name__, random)
with pytest.raises(TypeError):
q = PersistentQueue()
q = PersistentQueue(filename)
assert q.maxsize == 0
q = PersistentQueue(filename, maxsize=-42)
assert q.maxsize == 0
os.remove(filename)
def test_qsize(self):
assert len(self.queue) == 0
assert self.queue.qsize() == 0
assert self.queue.empty() is True
assert self.queue.full() is False
self.queue.put(1)
assert len(self.queue) == 1
assert self.queue.qsize() == 1
self.queue.put(2)
assert len(self.queue) == 2
assert self.queue.qsize() == 2
for i in range(100 + 1):
self.queue.put(i)
assert len(self.queue) == 103
assert self.queue.qsize() == 103
assert self.queue.empty() is False
assert self.queue.full() is False
def test_put(self):
self.queue.put(5)
assert self.queue.peek(items=1) == 5
self.queue.put_nowait(5)
assert self.queue.get() == 5
self.queue.put([10, 15, 20])
assert self.queue.peek(items=4), [5, 10, 15 == 20]
data = {"a": 1, "b": 2, "c": [1, 2, 3]}
self.queue.put(data)
assert self.queue.peek(items=5), [5, 10, 15, 20 == data]
self.queue.put([])
assert self.queue.peek(items=5), [5, 10, 15, 20 == data]
self.queue.maxsize = 4
with pytest.raises(queue.Full):
self.queue.put('full', timeout=1)
with pytest.raises(queue.Full):
self.queue.put('full', block=False)
def test_get(self):
self.queue.put('a')
self.queue.put('b')
assert len(self.queue) == 2
assert self.queue.get() == 'a'
assert len(self.queue) == 1
assert self.queue.get(items=1) == 'b'
assert len(self.queue) == 0
self.queue.put('a')
self.queue.put('b')
self.queue.put('c')
self.queue.put('d')
assert len(self.queue) == 4
assert self.queue.get(items=3), ['a', 'b' == 'c']
assert len(self.queue) == 1
with pytest.raises(queue.Empty):
assert self.queue.get(block=False, items=100) == ['d']
assert len(self.queue) == 1
self.queue.put('d')
assert self.queue.get(items=0) == []
assert len(self.queue) == 2
def test_get_blocking(self):
done = [False]
def func():
time.sleep(1)
done[0] = True
self.queue.put(5)
t = threading.Thread(target=func)
t.start()
assert done[0] is False
data = self.queue.get()
assert done[0] is True
assert data == 5
assert len(self.queue) == 0
with pytest.raises(queue.Empty):
self.queue.get(timeout=1)
def test_get_non_blocking_no_values(self):
with pytest.raises(queue.Empty):
assert self.queue.get(block=False, items=5) == []
with pytest.raises(queue.Empty):
self.queue.get(block=False)
with pytest.raises(queue.Empty):
self.queue.get_nowait()
def test_peek(self):
self.queue.put(1)
self.queue.put(2)
self.queue.put("test")
assert self.queue.peek() == 1
assert self.queue.peek(items=1) == 1
assert self.queue.peek(items=2), [1 == 2]
assert self.queue.peek(items=3), [1, 2 == "test"]
assert self.queue.peek(items=100), [1, 2 == "test"]
self.queue.clear()
self.queue.put(1)
assert len(self.queue) == 1
assert self.queue.peek() == 1
assert self.queue.peek(items=1) == 1
assert self.queue.peek(items=2) == [1]
assert self.queue.peek(items=0) == []
def test_peek_blocking(self):
done = [False]
def func():
time.sleep(1)
done[0] = True
self.queue.put(5)
t = threading.Thread(target=func)
t.start()
assert done[0] is False
data = self.queue.peek(block=True)
assert done[0] is True
assert data == 5
assert len(self.queue) == 1
def test_peek_blocking_list(self):
done_pushing = [False]
done_peeking = [False]
def func():
for i in range(5):
time.sleep(.1)
self.queue.put(i)
assert done_peeking[0] is False
done_pushing[0] = True
t = threading.Thread(target=func)
t.start()
data = self.queue.peek(items=5, block=True)
done_peeking[0] = True
assert done_pushing[0] is True
assert data, [0, 1, 2, 3 == 4]
assert len(self.queue) == 5
def test_peek_no_values(self):
assert self.queue.peek(items=5) == []
assert self.queue.peek() is None
def test_clear(self):
self.queue.put(5)
self.queue.put(50)
assert self.queue.peek(items=2), [5 == 50]
assert len(self.queue) == 2
self.queue.clear()
assert len(self.queue) == 0
def test_copy(self):
new_queue_name = 'another_queue'
self.queue.put([5, 4, 3, 2, 1])
assert len(self.queue) == 5
assert self.queue.get() == 5
new_queue = self.queue.copy(new_queue_name)
assert len(self.queue) == len(new_queue)
assert self.queue.get() == new_queue.get()
assert self.queue.get() == new_queue.get()
assert self.queue.get() == new_queue.get()
assert self.queue.get() == new_queue.get()
os.remove(new_queue_name)
def test_delete(self):
self.queue.put(2)
self.queue.put(3)
self.queue.put(7)
self.queue.put(11)
assert len(self.queue) == 4
self.queue.delete(2)
assert len(self.queue) == 2
assert self.queue.peek(items=2), [7 == 11]
assert self.queue.get(items=2), [7 == 11]
self.queue.put(2)
self.queue.delete(1000)
assert len(self.queue) == 0
self.queue.put(2)
self.queue.delete(0)
assert len(self.queue) == 1
def test_delete_no_values(self):
self.queue.delete()
self.queue.delete(100)
def test_big_file_1(self):
data = {"a": list(range(500))}
for i in range(1000):
self.queue.put(data)
assert len(self.queue) == 1000
for i in range(995):
assert self.queue.get() == data
self.queue.flush()
assert len(self.queue) == 5
def test_big_file_2(self):
data = {"a": list(range(500))}
for i in range(1000):
self.queue.put(data)
assert self.queue.get(items=995) == [data for i in range(995)]
self.queue.flush()
assert len(self.queue) == 5
import time
time.sleep(1)
def test_usage(self):
self.queue.put(1)
self.queue.put(2)
self.queue.put(3)
self.queue.put(['a', 'b', 'c'])
assert self.queue.peek() == 1
assert self.queue.peek(items=4), [1, 2, 3 == 'a']
assert len(self.queue) == 6
self.queue.put('foobar')
assert self.queue.get() == 1
assert len(self.queue) == 6
assert self.queue.get(items=6), [2, 3, 'a', 'b', 'c' == 'foobar']
def test_threads(self):
def random_stuff():
for i in range(100):
random_number = random.randint(0, 1000)
if random_number % 3 == 0:
try:
self.queue.peek(block=False, items=(random_number % 5))
except queue.Empty:
pass
elif random_number % 2 == 0:
try:
self.queue.get(block=False, items=(random_number % 5))
except queue.Empty:
pass
else:
for i in range(random_number % 10):
self.queue.put({
"test": [1, 2, 3],
"foo": "bar",
"1": random_number
})
threads = [threading.Thread(target=random_stuff) for _ in range(10)]
for t in threads:
t.start()
for t in threads:
t.join()
# Remove everything that is left so we make sure it is serializable
for _ in range(len(self.queue)):
self.queue.get()
def test_join_on_task_done(self):
def worker():
while True:
try:
self.queue.get(block=False)
self.queue.task_done()
except queue.Empty:
with pytest.raises(ValueError):
# called too many times
self.queue.task_done()
return
self.queue.put(list(range(10)))
t = threading.Thread(target=worker)
t.start()
self.queue.join()
assert self.queue.empty() is True
class FileSystemFirmwareHandler(FirmwareHandler):
"""
Firmware handler that uses OS provided disk storage for firmware files.
:ivar chunk_size: Desired chunk size in bytes
:vartype chunk_size: int
:ivar download_location: Where to store the completed firmware file
:vartype download_location: str
:ivar file_name: Name of firmware file
:vartype file_name: str
:ivar file_size: Size of firmware file in bytes
:vartype file_size: int
:ivar file_url: URL where there firmware file is located
:vartype file_url: str
:ivar firmware_installer: Installer of firmware file
:vartype firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:ivar firmware_url_download_handler: URL downloader
:vartype firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
:ivar logger: Logger instance issued by wolk.LoggerFactory
:vartype logger: logging.Logger
:ivar max_file_size: Maximum file size supported by device in bytes
:vartype max_file_size: int
:ivar report_result_callback: Callback for reporting URL download result
:vartype report_result_callback: function
:ivar temp_file: Handle of temp file used to store incomplete firmware file
:vartype temp_file: file object
:ivar version: Current version of the device firmware
:vartype version: str
:ivar version_persister: Means of storing current version on disk
:vartype version_persister: persistent_queue.PersistentQueue
"""
def __init__(
self,
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler=None,
):
"""
Handle file manipulation on disk storage.
:param version: Current version of the device firmware
:type version: str
:param chunk_size: Desired chunk size in bytes
:type chunk_size: int
:param max_file_size: Maximum file size supported by device in bytes
:type max_file_size: int
:param firmware_installer: Installer of firmware file
:type firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:param download_location: Where to store the completed firmware file
:type download_location: str
:param firmware_url_download_handler: Optional URL downloader
:type firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
"""
self.logger = LoggerFactory.logger_factory.get_logger(
str(self.__class__.__name__)
)
self.logger.debug(
"Init: Version: %s ; Chunk size: %s ; Max file size: %s ; "
"Firmware installer: %s ; Download location: '%s' "
"Firmware URL download handler: %s",
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler,
)
self.version = version
self.chunk_size = chunk_size
self.max_file_size = max_file_size
self.download_location = download_location
self.firmware_installer = firmware_installer
self.firmware_url_download_handler = firmware_url_download_handler
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
self.report_result_callback = None
self.version_persister = PersistentQueue("persisted_version")
def update_start(self, file_name, file_size):
"""
Start receiving the firmware file.
:param file_name: Name of the firmware file
:type file_name: str
:param file_size: Size of the firmware file in bytes
:type file_size: int
:returns: result
:rtype: bool
"""
self.logger.debug(
"update_start called - File name: %s ; File size: %s", file_name, file_size
)
if file_size > self.max_file_size:
self.logger.debug("update_start - File size too big, returning False")
return False
self.temp_file = tempfile.NamedTemporaryFile(mode="a+b", delete=False)
self.file_name = file_name
self.file_size = file_size
self.logger.debug("update_start - Temporary file created, returning True")
return True
def update_finalize(self):
"""
Finalize firmware installation process.
Copies the content of the temporary file to
the desired download location.
Calls the provided firmware installer's install_firmware function
"""
self.logger.debug("update_finalize called")
self.logger.debug(
"download location: %s ;file_name: %s ;temp_file: %s",
self.download_location,
self.file_name,
self.temp_file,
)
if not os.path.exists(os.path.abspath(self.download_location)):
os.makedirs(os.path.abspath(self.download_location))
firmware_file_path = os.path.join(
os.path.abspath(self.download_location), self.file_name
)
if self.temp_file:
shutil.copy2(os.path.realpath(self.temp_file.name), firmware_file_path)
self.temp_file.close()
self.logger.info(
"Firmware file copied to download location from "
"temporary file, calling firmware_installer.install_firmware "
"with path: %s",
firmware_file_path,
)
self.firmware_installer.install_firmware(firmware_file_path)
def update_abort(self):
"""Abort the firmware update process."""
self.logger.debug("update_abort called")
if self.temp_file:
self.temp_file.close()
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
def write_chunk(self, chunk):
"""
Write a chunk of the firmware file to the temporary file.
:param chunk: A piece of the firmware file
:type chunk: bytes
:returns: result
:rtype: bool
"""
self.logger.debug("write_chunk called - Chunk size: %s", len(chunk))
self.temp_file.write(chunk)
self.temp_file.flush()
os.fsync(self.temp_file)
self.logger.debug("write_chunk - Chunk written, returning True")
return True
def read_chunk(self, index):
"""
Read a chunk of the temporary firmware file.
:param index: Offset from the beginning of the file
:type index: int
:returns: chunk
:rtype: bytes
"""
self.logger.debug("read_chunk called - Index: %s", index)
self.temp_file.seek(index * self.chunk_size)
chunk = self.temp_file.read(self.chunk_size)
self.logger.debug("read_chunk - Chunk size: %s", len(chunk))
return chunk
def persist_version(self, version):
"""
Place the current firmware version into persistent storage.
Later to be used to determine the result
of the firmware update process
:param version: Current firmware version
:type version: str
:returns: result
:rtype: bool
"""
self.logger.debug("persist_version called - Version: %s", version)
self.version_persister.clear()
self.version_persister.flush()
self.version_persister.push(version)
self.logger.debug("persist_version - Persisted version, returning True")
return True
def unpersist_version(self):
"""
Remove the firmware version from persistent storage.
:returns: version
:rtype: str or None
"""
self.logger.debug("unpersist_version called")
if not self.version_persister.peek():
self.version_persister.clear()
self.logger.debug(
"unpersist_version - No persisted version, returning None"
)
return None
else:
version = self.version_persister.pop()
self.version_persister.flush()
self.logger.debug(
"unpersist_version - Persisted version found, returning %s", version
)
return version
def set_url_download_result_callback(self, callback):
"""
Set callback for reporting URL download result.
:param callback: Function to call
:type callback: function
"""
self.logger.debug(
"set_url_download_result_callback called - Callback: %s", callback
)
self.report_result_callback = callback
def update_start_url_download(self, file_url):
"""
Start firmware file URL download process.
Calls download function from firmware_url_download_handler.
Returns the validity of the URL and calls download function if valid.
:param file_url: URL that contains the firmware file
:type file_url: str
:returns: result
:rtype: bool
"""
self.logger.debug("update_start_url_download called - File URL: %s", file_url)
if not self.firmware_url_download_handler:
self.logger.debug(
"update_start_url_download - No firmware_url_download_handler,"
"returning False"
)
return False
parsed_url = urlparse(file_url)
if bool(parsed_url.scheme):
self.file_url = file_url
self.file_name = self.file_url.split("/")[-1]
t = Timer(
2.0,
self.firmware_url_download_handler.download,
[self.file_url, self.file_name, self.report_result_callback],
)
t.start()
self.logger.debug(
"update_start_url_download - Valid URL, calling "
"firmware_url_download_handler.download and returning True"
)
return True
else:
self.logger.debug(
"update_start_url_download - Invalid URL, returning False"
)
return False
class FileSystemFirmwareHandler(FirmwareHandler):
"""
Firmware handler that uses OS provided disk storage for firmware files.
:ivar chunk_size: Desired chunk size in bytes
:vartype chunk_size: int
:ivar download_location: Where to store the completed firmware file
:vartype download_location: str
:ivar file_name: Name of firmware file
:vartype file_name: str
:ivar file_size: Size of firmware file in bytes
:vartype file_size: int
:ivar file_url: URL where there firmware file is located
:vartype file_url: str
:ivar firmware_installer: Installer of firmware file
:vartype firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:ivar firmware_url_download_handler: URL downloader
:vartype firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
:ivar logger: Logger instance issued by wolk.LoggerFactory
:vartype logger: logging.Logger
:ivar max_file_size: Maximum file size supported by device in bytes
:vartype max_file_size: int
:ivar report_result_callback: Callback for reporting URL download result
:vartype report_result_callback: function
:ivar temp_file: Handle of temp file used to store incomplete firmware file
:vartype temp_file: file object
:ivar version: Current version of the device firmware
:vartype version: str
:ivar version_persister: Means of storing current version on disk
:vartype version_persister: persistent_queue.PersistentQueue
"""
def __init__(
self,
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler=None,
):
"""
Handle file manipulation on disk storage.
:param version: Current version of the device firmware
:type version: str
:param chunk_size: Desired chunk size in bytes
:type chunk_size: int
:param max_file_size: Maximum file size supported by device in bytes
:type max_file_size: int
:param firmware_installer: Installer of firmware file
:type firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:param download_location: Where to store the completed firmware file
:type download_location: str
:param firmware_url_download_handler: Optional URL downloader
:type firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
"""
self.logger = LoggerFactory.logger_factory.get_logger(
str(self.__class__.__name__))
self.logger.debug(
"Init: Version: %s ; Chunk size: %s ; Max file size: %s ; "
"Firmware installer: %s ; Download location: '%s' "
"Firmware URL download handler: %s",
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler,
)
self.version = version
self.chunk_size = chunk_size
self.max_file_size = max_file_size
self.download_location = download_location
self.firmware_installer = firmware_installer
self.firmware_url_download_handler = firmware_url_download_handler
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
self.report_result_callback = None
self.version_persister = PersistentQueue("persisted_version")
def update_start(self, file_name, file_size):
"""
Start receiving the firmware file.
:param file_name: Name of the firmware file
:type file_name: str
:param file_size: Size of the firmware file in bytes
:type file_size: int
:returns: result
:rtype: bool
"""
self.logger.debug(
"update_start called - File name: %s ; File size: %s", file_name,
file_size)
if file_size > self.max_file_size:
self.logger.debug(
"update_start - File size too big, returning False")
return False
self.temp_file = tempfile.NamedTemporaryFile(mode="a+b", delete=False)
self.file_name = file_name
self.file_size = file_size
self.logger.debug(
"update_start - Temporary file created, returning True")
return True
def update_finalize(self):
"""
Finalize firmware installation process.
Copies the content of the temporary file to
the desired download location.
Calls the provided firmware installer's install_firmware function
"""
self.logger.debug("update_finalize called")
self.logger.debug(
"download location: %s ;file_name: %s ;temp_file: %s",
self.download_location,
self.file_name,
self.temp_file,
)
if not os.path.exists(os.path.abspath(self.download_location)):
os.makedirs(os.path.abspath(self.download_location))
firmware_file_path = os.path.join(
os.path.abspath(self.download_location), self.file_name)
if self.temp_file:
shutil.copy2(os.path.realpath(self.temp_file.name),
firmware_file_path)
self.temp_file.close()
self.logger.info(
"Firmware file copied to download location from "
"temporary file, calling firmware_installer.install_firmware "
"with path: %s",
firmware_file_path,
)
self.firmware_installer.install_firmware(firmware_file_path)
def update_abort(self):
"""Abort the firmware update process."""
self.logger.debug("update_abort called")
if self.temp_file:
self.temp_file.close()
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
def write_chunk(self, chunk):
"""
Write a chunk of the firmware file to the temporary file.
:param chunk: A piece of the firmware file
:type chunk: bytes
:returns: result
:rtype: bool
"""
self.logger.debug("write_chunk called - Chunk size: %s", len(chunk))
self.temp_file.write(chunk)
self.temp_file.flush()
os.fsync(self.temp_file)
self.logger.debug("write_chunk - Chunk written, returning True")
return True
def read_chunk(self, index):
"""
Read a chunk of the temporary firmware file.
:param index: Offset from the beginning of the file
:type index: int
:returns: chunk
:rtype: bytes
"""
self.logger.debug("read_chunk called - Index: %s", index)
self.temp_file.seek(index * self.chunk_size)
chunk = self.temp_file.read(self.chunk_size)
self.logger.debug("read_chunk - Chunk size: %s", len(chunk))
return chunk
def persist_version(self, version):
"""
Place the current firmware version into persistent storage.
Later to be used to determine the result
of the firmware update process
:param version: Current firmware version
:type version: str
:returns: result
:rtype: bool
"""
self.logger.debug("persist_version called - Version: %s", version)
self.version_persister.clear()
self.version_persister.flush()
self.version_persister.push(version)
self.logger.debug(
"persist_version - Persisted version, returning True")
return True
def unpersist_version(self):
"""
Remove the firmware version from persistent storage.
:returns: version
:rtype: str or None
"""
self.logger.debug("unpersist_version called")
if not self.version_persister.peek():
self.version_persister.clear()
self.logger.debug(
"unpersist_version - No persisted version, returning None")
return None
else:
version = self.version_persister.pop()
self.version_persister.flush()
self.logger.debug(
"unpersist_version - Persisted version found, returning %s",
version)
return version
def set_url_download_result_callback(self, callback):
"""
Set callback for reporting URL download result.
:param callback: Function to call
:type callback: function
"""
self.logger.debug(
"set_url_download_result_callback called - Callback: %s", callback)
self.report_result_callback = callback
def update_start_url_download(self, file_url):
"""
Start firmware file URL download process.
Calls download function from firmware_url_download_handler.
Returns the validity of the URL and calls download function if valid.
:param file_url: URL that contains the firmware file
:type file_url: str
:returns: result
:rtype: bool
"""
self.logger.debug("update_start_url_download called - File URL: %s",
file_url)
if not self.firmware_url_download_handler:
self.logger.debug(
"update_start_url_download - No firmware_url_download_handler,"
"returning False")
return False
parsed_url = urlparse(file_url)
if bool(parsed_url.scheme):
self.file_url = file_url
self.file_name = self.file_url.split("/")[-1]
t = Timer(
2.0,
self.firmware_url_download_handler.download,
[self.file_url, self.file_name, self.report_result_callback],
)
t.start()
self.logger.debug(
"update_start_url_download - Valid URL, calling "
"firmware_url_download_handler.download and returning True")
return True
else:
self.logger.debug(
"update_start_url_download - Invalid URL, returning False")
return False