def __init__(self):
"""
Robonomics liability persistence node initialisation.
"""
rospy.init_node('robonomics_liability_persistence')
self.persistence_update_interval = rospy.get_param(
'~persistence_update_interval', 0.1)
self.__liability_executions_lock = Lock()
self.__liability_timestamps_lock = Lock()
self.__liability_executions = shelve.open(
"robonomics_liability_executions.persistence")
self.__liability_executions_timestamps = shelve.open(
"robonomics_liability_executions_timestamps.persistence")
self.__liability_executions_timestamps_queue = PersistentQueue(
'robonomics_liability_executions_timestamps.queue')
rospy.Subscriber('persistence/add', Liability, self.__add_liability)
rospy.Subscriber('persistence/del', Liability, self.__del_liability)
rospy.Subscriber("persistence/update_timestamp",
LiabilityExecutionTimestamp,
self.__update_liability_execution_timestamp_handler)
rospy.Service("persistence/exists", PersistenceContainsLiability,
self.__liability_exists)
rospy.Service("persistence/get_liability_timestamp",
PersistenceLiabilityTimestamp,
self.__get_liability_timestamp)
self.__incoming_liability_topic = rospy.Publisher('incoming',
Liability,
queue_size=10)
self.__restore_executions()
def setup_method(self):
import bson
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}'.format(self.__class__.__name__, random)
self.queue = PersistentQueue(filename,
loads=bson.loads,
dumps=bson.dumps)
def test_simple(self):
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}.queue'.format(self.__class__.__name__, random)
with pytest.raises(TypeError):
q = PersistentQueue()
q = PersistentQueue(filename)
assert q.maxsize == 0
q = PersistentQueue(filename, maxsize=-42)
assert q.maxsize == 0
os.remove(filename)
def __init__(
self,
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler=None,
):
"""
Handle file manipulation on disk storage.
:param version: Current version of the device firmware
:type version: str
:param chunk_size: Desired chunk size in bytes
:type chunk_size: int
:param max_file_size: Maximum file size supported by device in bytes
:type max_file_size: int
:param firmware_installer: Installer of firmware file
:type firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:param download_location: Where to store the completed firmware file
:type download_location: str
:param firmware_url_download_handler: Optional URL downloader
:type firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
"""
self.logger = LoggerFactory.logger_factory.get_logger(
str(self.__class__.__name__))
self.logger.debug(
"Init: Version: %s ; Chunk size: %s ; Max file size: %s ; "
"Firmware installer: %s ; Download location: '%s' "
"Firmware URL download handler: %s",
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler,
)
self.version = version
self.chunk_size = chunk_size
self.max_file_size = max_file_size
self.download_location = download_location
self.firmware_installer = firmware_installer
self.firmware_url_download_handler = firmware_url_download_handler
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
self.report_result_callback = None
self.version_persister = PersistentQueue("persisted_version")
def setup_method(self):
import msgpack
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}'.format(self.__class__.__name__, random)
self.queue = PersistentQueue(filename,
loads=msgpack.unpackb,
dumps=msgpack.packb)
def getQueue(self, key, create=False):
queue = self.queues.get(key)
if queue is None and create:
queue = PersistentQueue(os.path.join(self.qspath, key))
serverStats['current_bytes'] += queue.initialBytes
serverStats['total_items'] += queue.total_items
self.queues[key] = queue
return queue
def __init__(self):
'''
Robonomics liability tracking node initialisation.
'''
rospy.init_node('robonomics_liability_listener')
web3_http_provider = rospy.get_param('~web3_http_provider')
http_provider = HTTPProvider(web3_http_provider)
web3_ws_provider = rospy.get_param('~web3_ws_provider')
ws_provider = WebsocketProvider(web3_ws_provider)
ens_contract = rospy.get_param('~ens_contract', None)
self.ens = ENS(http_provider, addr=ens_contract)
self.web3 = Web3(http_provider, ens=self.ens)
self.web3ws = Web3(ws_provider, ens=self.ens)
from web3.middleware import geth_poa_middleware
# inject the poa compatibility middleware to the innermost layer
self.web3.middleware_stack.inject(geth_poa_middleware, layer=0)
self.ens.web3.middleware_stack.inject(geth_poa_middleware, layer=0)
self.poll_interval = rospy.get_param('~poll_interval', 5)
self.liability = rospy.Publisher('incoming', Liability, queue_size=10)
self.create_liability_filter()
self.liability_abi = json.loads(
rospy.get_param('~liability_contract_abi'))
self.liability_finalization_checker = finalization_checker.FinalizationChecker(
self.liability_abi,
web3_http_provider=web3_http_provider,
ens_contract=ens_contract)
self.finalized = rospy.Publisher('finalized', String, queue_size=10)
self.liabilities_queue = PersistentQueue(
'robonomics_liability_listener.queue')
self.result_handler()
class TestPersistentQueueWithBson:
def setup_method(self):
import bson
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}'.format(self.__class__.__name__, random)
self.queue = PersistentQueue(filename,
loads=bson.loads,
dumps=bson.dumps)
def teardown_method(self):
if os.path.isfile(self.queue.filename):
os.remove(self.queue.filename)
def test_big_file_1(self):
data = {"a": list(range(500))}
for i in range(1000):
self.queue.put(data)
assert len(self.queue) == 1000
for i in range(995):
assert self.queue.get() == data
self.queue.flush()
assert len(self.queue) == 5
def test_big_file_2(self):
data = {"a": list(range(500))}
for i in range(1000):
self.queue.put(data)
assert self.queue.get(items=995) == [data for i in range(995)]
self.queue.flush()
assert len(self.queue) == 5
import time
time.sleep(1)
def main(broker, data_file, interval):
print("Loading queue...")
queue = PersistentQueue('mqtt.queue')
print("Starting producer")
producer = Thread(target=start_sensors,
args=(queue, data_file, interval))
producer.start()
print("Starting MQTT publisher...")
client = mqtt.Client()
client.connect(broker)
client.loop_start()
while True:
data = queue.peek(blocking=True)
# Convert all byte strings to strings
print("Publishing data")
info = client.publish('devices/sensor001/data',
payload=json.dumps(data),
qos=1)
info.wait_for_publish()
queue.delete()
queue.flush()
def __init__(
self,
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler=None,
):
"""
Handle file manipulation on disk storage.
:param version: Current version of the device firmware
:type version: str
:param chunk_size: Desired chunk size in bytes
:type chunk_size: int
:param max_file_size: Maximum file size supported by device in bytes
:type max_file_size: int
:param firmware_installer: Installer of firmware file
:type firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:param download_location: Where to store the completed firmware file
:type download_location: str
:param firmware_url_download_handler: Optional URL downloader
:type firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
"""
self.logger = LoggerFactory.logger_factory.get_logger(
str(self.__class__.__name__)
)
self.logger.debug(
"Init: Version: %s ; Chunk size: %s ; Max file size: %s ; "
"Firmware installer: %s ; Download location: '%s' "
"Firmware URL download handler: %s",
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler,
)
self.version = version
self.chunk_size = chunk_size
self.max_file_size = max_file_size
self.download_location = download_location
self.firmware_installer = firmware_installer
self.firmware_url_download_handler = firmware_url_download_handler
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
self.report_result_callback = None
self.version_persister = PersistentQueue("persisted_version")
class FilesystemOutboundMessageQueue(wolk.OutboundMessageQueue):
def __init__(self, path="."):
if path == ".":
self.queue = PersistentQueue("FileOutboundMessageQueue")
else:
self.queue = PersistentQueue("FileOutboundMessageQueue", path)
def put(self, message):
self.queue.push(message)
def get(self):
message = self.queue.pop()
self.queue.flush()
return message
def peek(self):
if not self.queue.peek():
self.queue.clear()
return None
else:
return self.queue.peek()
def __main__():
threads = []
statistics_queue = PersistentQueue('tmp/statistics.queue')
statistics_semaphore = threading.Semaphore(statistics_queue.count())
verification_queue = PersistentQueue('tmp/verification.queue')
verification_semaphore = threading.Semaphore(verification_queue.count())
storage_queue = PersistentQueue('tmp/storage.queue')
storage_semaphore = threading.Semaphore(storage_queue.count())
statistics_thread = StatisticsThread(statistics_semaphore,
statistics_queue, verification_queue,
storage_queue)
statistics_thread.start()
threads.append(statistics_thread)
verification_thread = VerificationThread(statistics_semaphore,
statistics_queue,
verification_semaphore,
verification_queue,
storage_semaphore, storage_queue)
verification_thread.start()
threads.append(verification_thread)
storage_thread = StorageThread(statistics_semaphore, statistics_queue,
storage_semaphore, storage_queue)
storage_thread.start()
threads.append(storage_thread)
for hostname in UNITS.keys():
thread = CollectorThread(verification_semaphore, verification_queue,
statistics_semaphore, statistics_queue,
hostname)
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
class Listener:
def __init__(self):
'''
Robonomics liability tracking node initialisation.
'''
rospy.init_node('robonomics_liability_listener')
web3_http_provider = rospy.get_param('~web3_http_provider')
http_provider = HTTPProvider(web3_http_provider)
web3_ws_provider = rospy.get_param('~web3_ws_provider')
ws_provider = WebsocketProvider(web3_ws_provider)
ens_contract = rospy.get_param('~ens_contract', None)
self.ens = ENS(http_provider, addr=ens_contract)
self.web3 = Web3(http_provider, ens=self.ens)
self.web3ws = Web3(ws_provider, ens=self.ens)
from web3.middleware import geth_poa_middleware
# inject the poa compatibility middleware to the innermost layer
self.web3.middleware_stack.inject(geth_poa_middleware, layer=0)
self.ens.web3.middleware_stack.inject(geth_poa_middleware, layer=0)
self.poll_interval = rospy.get_param('~poll_interval', 5)
self.liability = rospy.Publisher('incoming', Liability, queue_size=10)
self.create_liability_filter()
self.liability_abi = json.loads(
rospy.get_param('~liability_contract_abi'))
self.liability_finalization_checker = finalization_checker.FinalizationChecker(
self.liability_abi,
web3_http_provider=web3_http_provider,
ens_contract=ens_contract)
self.finalized = rospy.Publisher('finalized', String, queue_size=10)
self.liabilities_queue = PersistentQueue(
'robonomics_liability_listener.queue')
self.result = rospy.Publisher('infochan/eth/signing/result',
Result,
queue_size=10)
self.persistence_contains_liability = rospy.ServiceProxy(
'persistence/exists', PersistenceContainsLiability)
def liability_finalize(msg):
rospy.logdebug("liability_finalize: msg is: %s", msg)
is_finalized = False
while is_finalized is not True:
self.result.publish(msg)
time.sleep(30)
# TODO: move sleep time to rop parameter with 30 seconds by default
is_finalized = self.liability_finalization_checker.finalized(
msg.liability.address)
self.finalized.publish(msg.liability.address)
rospy.Subscriber('result', Result, liability_finalize)
def create_liability_filter(self):
try:
factory_abi = json.loads(rospy.get_param('~factory_contract_abi'))
factory_address = self.ens.address(
rospy.get_param('~factory_contract'))
factory = self.web3ws.eth.contract(factory_address,
abi=factory_abi)
self.liability_filter = factory.eventFilter('NewLiability')
except Exception as e:
rospy.logwarn(
"Failed to create liability filter with exception: \"%s\"", e)
def liability_read(self, address):
'''
Read liability from blockchain to message.
'''
c = self.web3.eth.contract(address, abi=self.liability_abi)
msg = Liability()
msg.address.address = address
msg.model.multihash = multihash.decode(
c.call().model()).encode('base58').decode()
msg.objective.multihash = multihash.decode(
c.call().objective()).encode('base58').decode()
msg.promisee.address = c.call().promisee()
msg.promisor.address = c.call().promisor()
msg.lighthouse.address = c.call().lighthouse()
msg.token.address = c.call().token()
msg.cost.uint256 = str(c.call().cost())
msg.validator.address = c.call().validator()
msg.validatorFee.uint256 = str(c.call().validatorFee())
rospy.logdebug('New liability readed: %s', msg)
return msg
def spin(self):
'''
Waiting for the new liabilities.
'''
def liability_filter_thread():
try:
for entry in self.liability_filter.get_new_entries():
liability_address = entry['args']['liability']
self.liabilities_queue.push(liability_address)
rospy.loginfo(
"New liability added to persistence queue: %s",
liability_address)
except Exception as e:
rospy.logerr('listener liability filter exception: %s', e)
self.create_liability_filter()
Timer(self.poll_interval, liability_filter_thread).start()
def liabilities_queue_handler():
entry = self.liabilities_queue.peek()
if entry is not None:
try:
rospy.wait_for_service(
self.persistence_contains_liability.resolved_name)
liability_already_in_persistence = self.persistence_contains_liability(
entry)
# TODO: verify that liability for my saved in persistence before pop them from liabilities_queue
if not liability_already_in_persistence.exists:
self.liability.publish(self.liability_read(entry))
self.liabilities_queue.pop()
rospy.loginfo("Liability read successfully: %s", entry)
except Exception as e:
rospy.logerr('Liability %s read exception: %s', entry, e)
Timer(self.poll_interval, liabilities_queue_handler).start()
liability_filter_thread()
liabilities_queue_handler()
rospy.spin()
FPLAYER_ADDR = "192.168.0.19"
FPLAYER_PORT = 54243
index_name = datetime.datetime.now().strftime("pi-events-%Y.%m.%d")
esConnection = None
def getEsConnection():
global esConnection
if not esConnection:
esConnection = httplib.HTTPConnection("server1", 9200)
return esConnection
sessions_queue = PersistentQueue("Session.queue")
def index_doc(doc, doctype, docid=None):
doc['ts'] = datetime.datetime.now().isoformat()
if docid:
url = "%s/%s/%s" % (index_name, doctype, docid)
else:
url = "%s/%s" % (index_name, doctype)
indexing_req = getEsConnection().request("POST", url, json.dumps(doc))
resp = esConnection.getresponse()
rc = resp.status
if (rc / 100) != 2:
raise Exception("unable to index document. URL=%s rc=%s error=%s" %
(url, rc, resp.read()))
def __init__(self, path="."):
if path == ".":
self.queue = PersistentQueue("FileOutboundMessageQueue")
else:
self.queue = PersistentQueue("FileOutboundMessageQueue", path)
class FileSystemFirmwareHandler(FirmwareHandler):
"""
Firmware handler that uses OS provided disk storage for firmware files.
:ivar chunk_size: Desired chunk size in bytes
:vartype chunk_size: int
:ivar download_location: Where to store the completed firmware file
:vartype download_location: str
:ivar file_name: Name of firmware file
:vartype file_name: str
:ivar file_size: Size of firmware file in bytes
:vartype file_size: int
:ivar file_url: URL where there firmware file is located
:vartype file_url: str
:ivar firmware_installer: Installer of firmware file
:vartype firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:ivar firmware_url_download_handler: URL downloader
:vartype firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
:ivar logger: Logger instance issued by wolk.LoggerFactory
:vartype logger: logging.Logger
:ivar max_file_size: Maximum file size supported by device in bytes
:vartype max_file_size: int
:ivar report_result_callback: Callback for reporting URL download result
:vartype report_result_callback: function
:ivar temp_file: Handle of temp file used to store incomplete firmware file
:vartype temp_file: file object
:ivar version: Current version of the device firmware
:vartype version: str
:ivar version_persister: Means of storing current version on disk
:vartype version_persister: persistent_queue.PersistentQueue
"""
def __init__(
self,
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler=None,
):
"""
Handle file manipulation on disk storage.
:param version: Current version of the device firmware
:type version: str
:param chunk_size: Desired chunk size in bytes
:type chunk_size: int
:param max_file_size: Maximum file size supported by device in bytes
:type max_file_size: int
:param firmware_installer: Installer of firmware file
:type firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:param download_location: Where to store the completed firmware file
:type download_location: str
:param firmware_url_download_handler: Optional URL downloader
:type firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
"""
self.logger = LoggerFactory.logger_factory.get_logger(
str(self.__class__.__name__)
)
self.logger.debug(
"Init: Version: %s ; Chunk size: %s ; Max file size: %s ; "
"Firmware installer: %s ; Download location: '%s' "
"Firmware URL download handler: %s",
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler,
)
self.version = version
self.chunk_size = chunk_size
self.max_file_size = max_file_size
self.download_location = download_location
self.firmware_installer = firmware_installer
self.firmware_url_download_handler = firmware_url_download_handler
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
self.report_result_callback = None
self.version_persister = PersistentQueue("persisted_version")
def update_start(self, file_name, file_size):
"""
Start receiving the firmware file.
:param file_name: Name of the firmware file
:type file_name: str
:param file_size: Size of the firmware file in bytes
:type file_size: int
:returns: result
:rtype: bool
"""
self.logger.debug(
"update_start called - File name: %s ; File size: %s", file_name, file_size
)
if file_size > self.max_file_size:
self.logger.debug("update_start - File size too big, returning False")
return False
self.temp_file = tempfile.NamedTemporaryFile(mode="a+b", delete=False)
self.file_name = file_name
self.file_size = file_size
self.logger.debug("update_start - Temporary file created, returning True")
return True
def update_finalize(self):
"""
Finalize firmware installation process.
Copies the content of the temporary file to
the desired download location.
Calls the provided firmware installer's install_firmware function
"""
self.logger.debug("update_finalize called")
self.logger.debug(
"download location: %s ;file_name: %s ;temp_file: %s",
self.download_location,
self.file_name,
self.temp_file,
)
if not os.path.exists(os.path.abspath(self.download_location)):
os.makedirs(os.path.abspath(self.download_location))
firmware_file_path = os.path.join(
os.path.abspath(self.download_location), self.file_name
)
if self.temp_file:
shutil.copy2(os.path.realpath(self.temp_file.name), firmware_file_path)
self.temp_file.close()
self.logger.info(
"Firmware file copied to download location from "
"temporary file, calling firmware_installer.install_firmware "
"with path: %s",
firmware_file_path,
)
self.firmware_installer.install_firmware(firmware_file_path)
def update_abort(self):
"""Abort the firmware update process."""
self.logger.debug("update_abort called")
if self.temp_file:
self.temp_file.close()
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
def write_chunk(self, chunk):
"""
Write a chunk of the firmware file to the temporary file.
:param chunk: A piece of the firmware file
:type chunk: bytes
:returns: result
:rtype: bool
"""
self.logger.debug("write_chunk called - Chunk size: %s", len(chunk))
self.temp_file.write(chunk)
self.temp_file.flush()
os.fsync(self.temp_file)
self.logger.debug("write_chunk - Chunk written, returning True")
return True
def read_chunk(self, index):
"""
Read a chunk of the temporary firmware file.
:param index: Offset from the beginning of the file
:type index: int
:returns: chunk
:rtype: bytes
"""
self.logger.debug("read_chunk called - Index: %s", index)
self.temp_file.seek(index * self.chunk_size)
chunk = self.temp_file.read(self.chunk_size)
self.logger.debug("read_chunk - Chunk size: %s", len(chunk))
return chunk
def persist_version(self, version):
"""
Place the current firmware version into persistent storage.
Later to be used to determine the result
of the firmware update process
:param version: Current firmware version
:type version: str
:returns: result
:rtype: bool
"""
self.logger.debug("persist_version called - Version: %s", version)
self.version_persister.clear()
self.version_persister.flush()
self.version_persister.push(version)
self.logger.debug("persist_version - Persisted version, returning True")
return True
def unpersist_version(self):
"""
Remove the firmware version from persistent storage.
:returns: version
:rtype: str or None
"""
self.logger.debug("unpersist_version called")
if not self.version_persister.peek():
self.version_persister.clear()
self.logger.debug(
"unpersist_version - No persisted version, returning None"
)
return None
else:
version = self.version_persister.pop()
self.version_persister.flush()
self.logger.debug(
"unpersist_version - Persisted version found, returning %s", version
)
return version
def set_url_download_result_callback(self, callback):
"""
Set callback for reporting URL download result.
:param callback: Function to call
:type callback: function
"""
self.logger.debug(
"set_url_download_result_callback called - Callback: %s", callback
)
self.report_result_callback = callback
def update_start_url_download(self, file_url):
"""
Start firmware file URL download process.
Calls download function from firmware_url_download_handler.
Returns the validity of the URL and calls download function if valid.
:param file_url: URL that contains the firmware file
:type file_url: str
:returns: result
:rtype: bool
"""
self.logger.debug("update_start_url_download called - File URL: %s", file_url)
if not self.firmware_url_download_handler:
self.logger.debug(
"update_start_url_download - No firmware_url_download_handler,"
"returning False"
)
return False
parsed_url = urlparse(file_url)
if bool(parsed_url.scheme):
self.file_url = file_url
self.file_name = self.file_url.split("/")[-1]
t = Timer(
2.0,
self.firmware_url_download_handler.download,
[self.file_url, self.file_name, self.report_result_callback],
)
t.start()
self.logger.debug(
"update_start_url_download - Valid URL, calling "
"firmware_url_download_handler.download and returning True"
)
return True
else:
self.logger.debug(
"update_start_url_download - Invalid URL, returning False"
)
return False
class LiabilityExecutionsPersistence:
def __init__(self):
"""
Robonomics liability persistence node initialisation.
"""
rospy.init_node('robonomics_liability_persistence')
self.persistence_update_interval = rospy.get_param(
'~persistence_update_interval', 0.1)
self.__liability_executions_lock = Lock()
self.__liability_timestamps_lock = Lock()
self.__liability_executions = shelve.open(
"robonomics_liability_executions.persistence")
self.__liability_executions_timestamps = shelve.open(
"robonomics_liability_executions_timestamps.persistence")
self.__liability_executions_timestamps_queue = PersistentQueue(
'robonomics_liability_executions_timestamps.queue')
rospy.Subscriber('persistence/add', Liability, self.__add_liability)
rospy.Subscriber('persistence/del', Liability, self.__del_liability)
rospy.Subscriber("persistence/update_timestamp",
LiabilityExecutionTimestamp,
self.__update_liability_execution_timestamp_handler)
rospy.Service("persistence/exists", PersistenceContainsLiability,
self.__liability_exists)
rospy.Service("persistence/get_liability_timestamp",
PersistenceLiabilityTimestamp,
self.__get_liability_timestamp)
self.__incoming_liability_topic = rospy.Publisher('incoming',
Liability,
queue_size=10)
self.__restore_executions()
def __update_liability_execution_timestamp_handler(self, msg):
self.__liability_executions_timestamps_queue.push(msg)
def __update_liability_execution_timestamp(self, msg):
rospy.logdebug("update liability %s execution timestamp",
msg.address.address)
if msg.address.address not in self.__liability_executions_timestamps:
rospy.logwarn(
"liability %s already unregistered from timestamps persistence",
msg.address.address)
return
try:
self.__liability_timestamps_lock.acquire()
self.__liability_executions_timestamps[
msg.address.address] = msg.timestamp
self.__liability_executions_timestamps.sync()
finally:
self.__liability_timestamps_lock.release()
rospy.logdebug(
"Persistence liability %s timestamp %s", msg.address.address,
self.__liability_executions_timestamps[msg.address.address])
def __liability_exists(self, msg):
return PersistenceContainsLiabilityResponse(
msg.address.address in self.__liability_executions)
def __register_liability_in_timestamp_persistence(self, msg):
try:
self.__liability_timestamps_lock.acquire()
if msg.address.address not in self.__liability_executions_timestamps:
self.__liability_executions_timestamps[
msg.address.address] = rospy.Time.from_sec(0)
rospy.loginfo(
"Timestamps persistence contains %s value for liability %s",
self.__liability_executions_timestamps[msg.address.address],
msg.address.address)
self.__liability_executions_timestamps.sync()
finally:
self.__liability_timestamps_lock.release()
def __add_liability(self, msg):
try:
self.__liability_executions_lock.acquire()
self.__liability_executions[msg.address.address] = msg
self.__register_liability_in_timestamp_persistence(msg)
self.__liability_executions.sync()
rospy.loginfo(
"Liability %s added to liabilities executions persistence store",
msg.address.address)
finally:
self.__liability_executions_lock.release()
def __del_liability(self, msg):
try:
self.__liability_executions_lock.acquire()
del self.__liability_executions[msg.address.address]
self.__liability_executions.sync()
rospy.loginfo(
"Liability %s deleted from liabilities executions persistence store",
msg.address.address)
except KeyError:
rospy.logwarn(
"Liability %s not found in liabilities executions persistence store",
msg.address.address)
finally:
self.__liability_executions_lock.release()
try:
self.__liability_timestamps_lock.acquire()
del self.__liability_executions_timestamps[msg.address.address]
self.__liability_executions_timestamps.sync()
rospy.loginfo(
"Liability %s deleted from liabilities timestamps persistence store",
msg.address.address)
except KeyError:
rospy.logwarn(
"Liability %s not found in liabilities timestamps persistence store",
msg.address.address)
finally:
self.__liability_timestamps_lock.release()
def __restore_executions(self):
try:
self.__liability_executions_lock.acquire()
executions = list(self.__liability_executions.values())
finally:
self.__liability_executions_lock.release()
time.sleep(5)
for liability in executions:
self.__incoming_liability_topic.publish(liability)
rospy.logwarn(
"Liability %s received from liabilities executions persistence store",
liability.address.address)
def __get_liability_timestamp(self, msg):
timestamp = rospy.Time.from_sec(0)
liability_address = msg.address.address
queue_entry = self.__liability_executions_timestamps_queue.peek()
while queue_entry is not None:
time.sleep(0.1)
queue_entry = self.__liability_executions_timestamps_queue.peek()
try:
self.__liability_timestamps_lock.acquire()
timestamp = self.__liability_executions_timestamps[
liability_address]
except KeyError as e:
rospy.logwarn(
"Unable to get known execution timestamp for liability %s",
liability_address)
finally:
self.__liability_timestamps_lock.release()
return PersistenceLiabilityTimestampResponse(timestamp)
def spin(self):
def update_liability_timestamp_queue_handler():
entry = self.__liability_executions_timestamps_queue.peek()
if entry is not None:
self.__update_liability_execution_timestamp(entry)
self.__liability_executions_timestamps_queue.pop()
Timer(self.persistence_update_interval,
update_liability_timestamp_queue_handler).start()
update_liability_timestamp_queue_handler()
rospy.spin()
def setup_method(self):
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}.queue'.format(self.__class__.__name__, random)
self.queue = PersistentQueue(filename)
class TestPersistentQueue:
def setup_method(self):
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}.queue'.format(self.__class__.__name__, random)
self.queue = PersistentQueue(filename)
def teardown_method(self):
if os.path.isfile(self.queue.filename):
os.remove(self.queue.filename)
def test_simple(self):
random = str(uuid.uuid4()).replace('-', '')
filename = '{}_{}.queue'.format(self.__class__.__name__, random)
with pytest.raises(TypeError):
q = PersistentQueue()
q = PersistentQueue(filename)
assert q.maxsize == 0
q = PersistentQueue(filename, maxsize=-42)
assert q.maxsize == 0
os.remove(filename)
def test_qsize(self):
assert len(self.queue) == 0
assert self.queue.qsize() == 0
assert self.queue.empty() is True
assert self.queue.full() is False
self.queue.put(1)
assert len(self.queue) == 1
assert self.queue.qsize() == 1
self.queue.put(2)
assert len(self.queue) == 2
assert self.queue.qsize() == 2
for i in range(100 + 1):
self.queue.put(i)
assert len(self.queue) == 103
assert self.queue.qsize() == 103
assert self.queue.empty() is False
assert self.queue.full() is False
def test_put(self):
self.queue.put(5)
assert self.queue.peek(items=1) == 5
self.queue.put_nowait(5)
assert self.queue.get() == 5
self.queue.put([10, 15, 20])
assert self.queue.peek(items=4), [5, 10, 15 == 20]
data = {"a": 1, "b": 2, "c": [1, 2, 3]}
self.queue.put(data)
assert self.queue.peek(items=5), [5, 10, 15, 20 == data]
self.queue.put([])
assert self.queue.peek(items=5), [5, 10, 15, 20 == data]
self.queue.maxsize = 4
with pytest.raises(queue.Full):
self.queue.put('full', timeout=1)
with pytest.raises(queue.Full):
self.queue.put('full', block=False)
def test_get(self):
self.queue.put('a')
self.queue.put('b')
assert len(self.queue) == 2
assert self.queue.get() == 'a'
assert len(self.queue) == 1
assert self.queue.get(items=1) == 'b'
assert len(self.queue) == 0
self.queue.put('a')
self.queue.put('b')
self.queue.put('c')
self.queue.put('d')
assert len(self.queue) == 4
assert self.queue.get(items=3), ['a', 'b' == 'c']
assert len(self.queue) == 1
with pytest.raises(queue.Empty):
assert self.queue.get(block=False, items=100) == ['d']
assert len(self.queue) == 1
self.queue.put('d')
assert self.queue.get(items=0) == []
assert len(self.queue) == 2
def test_get_blocking(self):
done = [False]
def func():
time.sleep(1)
done[0] = True
self.queue.put(5)
t = threading.Thread(target=func)
t.start()
assert done[0] is False
data = self.queue.get()
assert done[0] is True
assert data == 5
assert len(self.queue) == 0
with pytest.raises(queue.Empty):
self.queue.get(timeout=1)
def test_get_non_blocking_no_values(self):
with pytest.raises(queue.Empty):
assert self.queue.get(block=False, items=5) == []
with pytest.raises(queue.Empty):
self.queue.get(block=False)
with pytest.raises(queue.Empty):
self.queue.get_nowait()
def test_peek(self):
self.queue.put(1)
self.queue.put(2)
self.queue.put("test")
assert self.queue.peek() == 1
assert self.queue.peek(items=1) == 1
assert self.queue.peek(items=2), [1 == 2]
assert self.queue.peek(items=3), [1, 2 == "test"]
assert self.queue.peek(items=100), [1, 2 == "test"]
self.queue.clear()
self.queue.put(1)
assert len(self.queue) == 1
assert self.queue.peek() == 1
assert self.queue.peek(items=1) == 1
assert self.queue.peek(items=2) == [1]
assert self.queue.peek(items=0) == []
def test_peek_blocking(self):
done = [False]
def func():
time.sleep(1)
done[0] = True
self.queue.put(5)
t = threading.Thread(target=func)
t.start()
assert done[0] is False
data = self.queue.peek(block=True)
assert done[0] is True
assert data == 5
assert len(self.queue) == 1
def test_peek_blocking_list(self):
done_pushing = [False]
done_peeking = [False]
def func():
for i in range(5):
time.sleep(.1)
self.queue.put(i)
assert done_peeking[0] is False
done_pushing[0] = True
t = threading.Thread(target=func)
t.start()
data = self.queue.peek(items=5, block=True)
done_peeking[0] = True
assert done_pushing[0] is True
assert data, [0, 1, 2, 3 == 4]
assert len(self.queue) == 5
def test_peek_no_values(self):
assert self.queue.peek(items=5) == []
assert self.queue.peek() is None
def test_clear(self):
self.queue.put(5)
self.queue.put(50)
assert self.queue.peek(items=2), [5 == 50]
assert len(self.queue) == 2
self.queue.clear()
assert len(self.queue) == 0
def test_copy(self):
new_queue_name = 'another_queue'
self.queue.put([5, 4, 3, 2, 1])
assert len(self.queue) == 5
assert self.queue.get() == 5
new_queue = self.queue.copy(new_queue_name)
assert len(self.queue) == len(new_queue)
assert self.queue.get() == new_queue.get()
assert self.queue.get() == new_queue.get()
assert self.queue.get() == new_queue.get()
assert self.queue.get() == new_queue.get()
os.remove(new_queue_name)
def test_delete(self):
self.queue.put(2)
self.queue.put(3)
self.queue.put(7)
self.queue.put(11)
assert len(self.queue) == 4
self.queue.delete(2)
assert len(self.queue) == 2
assert self.queue.peek(items=2), [7 == 11]
assert self.queue.get(items=2), [7 == 11]
self.queue.put(2)
self.queue.delete(1000)
assert len(self.queue) == 0
self.queue.put(2)
self.queue.delete(0)
assert len(self.queue) == 1
def test_delete_no_values(self):
self.queue.delete()
self.queue.delete(100)
def test_big_file_1(self):
data = {"a": list(range(500))}
for i in range(1000):
self.queue.put(data)
assert len(self.queue) == 1000
for i in range(995):
assert self.queue.get() == data
self.queue.flush()
assert len(self.queue) == 5
def test_big_file_2(self):
data = {"a": list(range(500))}
for i in range(1000):
self.queue.put(data)
assert self.queue.get(items=995) == [data for i in range(995)]
self.queue.flush()
assert len(self.queue) == 5
import time
time.sleep(1)
def test_usage(self):
self.queue.put(1)
self.queue.put(2)
self.queue.put(3)
self.queue.put(['a', 'b', 'c'])
assert self.queue.peek() == 1
assert self.queue.peek(items=4), [1, 2, 3 == 'a']
assert len(self.queue) == 6
self.queue.put('foobar')
assert self.queue.get() == 1
assert len(self.queue) == 6
assert self.queue.get(items=6), [2, 3, 'a', 'b', 'c' == 'foobar']
def test_threads(self):
def random_stuff():
for i in range(100):
random_number = random.randint(0, 1000)
if random_number % 3 == 0:
try:
self.queue.peek(block=False, items=(random_number % 5))
except queue.Empty:
pass
elif random_number % 2 == 0:
try:
self.queue.get(block=False, items=(random_number % 5))
except queue.Empty:
pass
else:
for i in range(random_number % 10):
self.queue.put({
"test": [1, 2, 3],
"foo": "bar",
"1": random_number
})
threads = [threading.Thread(target=random_stuff) for _ in range(10)]
for t in threads:
t.start()
for t in threads:
t.join()
# Remove everything that is left so we make sure it is serializable
for _ in range(len(self.queue)):
self.queue.get()
def test_join_on_task_done(self):
def worker():
while True:
try:
self.queue.get(block=False)
self.queue.task_done()
except queue.Empty:
with pytest.raises(ValueError):
# called too many times
self.queue.task_done()
return
self.queue.put(list(range(10)))
t = threading.Thread(target=worker)
t.start()
self.queue.join()
assert self.queue.empty() is True
def __init__(self):
'''
Robonomics liability tracking node initialisation.
'''
rospy.init_node('robonomics_liability_listener')
web3_http_provider = rospy.get_param('~web3_http_provider')
http_provider = HTTPProvider(web3_http_provider)
web3_ws_provider = rospy.get_param('~web3_ws_provider')
ws_provider = WebsocketProvider(web3_ws_provider)
ens_contract = rospy.get_param('~ens_contract', None)
self.ens = ENS(http_provider, addr=ens_contract)
self.web3 = Web3(http_provider, ens=self.ens)
self.web3ws = Web3(ws_provider, ens=self.ens)
from web3.middleware import geth_poa_middleware
# inject the poa compatibility middleware to the innermost layer
self.web3.middleware_stack.inject(geth_poa_middleware, layer=0)
self.ens.web3.middleware_stack.inject(geth_poa_middleware, layer=0)
self.poll_interval = rospy.get_param('~poll_interval', 5)
self.liability = rospy.Publisher('incoming', Liability, queue_size=10)
self.create_liability_filter()
self.liability_abi = json.loads(
rospy.get_param('~liability_contract_abi'))
self.liability_finalization_checker = finalization_checker.FinalizationChecker(
self.liability_abi,
web3_http_provider=web3_http_provider,
ens_contract=ens_contract)
self.finalized = rospy.Publisher('finalized', String, queue_size=10)
self.liabilities_queue = PersistentQueue(
'robonomics_liability_listener.queue')
self.result = rospy.Publisher('infochan/eth/signing/result',
Result,
queue_size=10)
self.persistence_contains_liability = rospy.ServiceProxy(
'persistence/exists', PersistenceContainsLiability)
def liability_finalize(msg):
rospy.logdebug("liability_finalize: msg is: %s", msg)
is_finalized = False
while is_finalized is not True:
self.result.publish(msg)
time.sleep(30)
# TODO: move sleep time to rop parameter with 30 seconds by default
is_finalized = self.liability_finalization_checker.finalized(
msg.liability.address)
self.finalized.publish(msg.liability.address)
rospy.Subscriber('result', Result, liability_finalize)
def test_queue(self):
q = PersistentQueue()
obj = {'test':'1'}
q.put(obj)
self.assertEqual(obj, q.get())
class FileSystemFirmwareHandler(FirmwareHandler):
"""
Firmware handler that uses OS provided disk storage for firmware files.
:ivar chunk_size: Desired chunk size in bytes
:vartype chunk_size: int
:ivar download_location: Where to store the completed firmware file
:vartype download_location: str
:ivar file_name: Name of firmware file
:vartype file_name: str
:ivar file_size: Size of firmware file in bytes
:vartype file_size: int
:ivar file_url: URL where there firmware file is located
:vartype file_url: str
:ivar firmware_installer: Installer of firmware file
:vartype firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:ivar firmware_url_download_handler: URL downloader
:vartype firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
:ivar logger: Logger instance issued by wolk.LoggerFactory
:vartype logger: logging.Logger
:ivar max_file_size: Maximum file size supported by device in bytes
:vartype max_file_size: int
:ivar report_result_callback: Callback for reporting URL download result
:vartype report_result_callback: function
:ivar temp_file: Handle of temp file used to store incomplete firmware file
:vartype temp_file: file object
:ivar version: Current version of the device firmware
:vartype version: str
:ivar version_persister: Means of storing current version on disk
:vartype version_persister: persistent_queue.PersistentQueue
"""
def __init__(
self,
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler=None,
):
"""
Handle file manipulation on disk storage.
:param version: Current version of the device firmware
:type version: str
:param chunk_size: Desired chunk size in bytes
:type chunk_size: int
:param max_file_size: Maximum file size supported by device in bytes
:type max_file_size: int
:param firmware_installer: Installer of firmware file
:type firmware_installer: wolk.interfaces.FirmwareInstaller.FirmwareInstaller
:param download_location: Where to store the completed firmware file
:type download_location: str
:param firmware_url_download_handler: Optional URL downloader
:type firmware_url_download_handler: wolk.interfaces.FirmwareURLDownloadHandler.FirmwareURLDownloadHandler or None
"""
self.logger = LoggerFactory.logger_factory.get_logger(
str(self.__class__.__name__))
self.logger.debug(
"Init: Version: %s ; Chunk size: %s ; Max file size: %s ; "
"Firmware installer: %s ; Download location: '%s' "
"Firmware URL download handler: %s",
version,
chunk_size,
max_file_size,
firmware_installer,
download_location,
firmware_url_download_handler,
)
self.version = version
self.chunk_size = chunk_size
self.max_file_size = max_file_size
self.download_location = download_location
self.firmware_installer = firmware_installer
self.firmware_url_download_handler = firmware_url_download_handler
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
self.report_result_callback = None
self.version_persister = PersistentQueue("persisted_version")
def update_start(self, file_name, file_size):
"""
Start receiving the firmware file.
:param file_name: Name of the firmware file
:type file_name: str
:param file_size: Size of the firmware file in bytes
:type file_size: int
:returns: result
:rtype: bool
"""
self.logger.debug(
"update_start called - File name: %s ; File size: %s", file_name,
file_size)
if file_size > self.max_file_size:
self.logger.debug(
"update_start - File size too big, returning False")
return False
self.temp_file = tempfile.NamedTemporaryFile(mode="a+b", delete=False)
self.file_name = file_name
self.file_size = file_size
self.logger.debug(
"update_start - Temporary file created, returning True")
return True
def update_finalize(self):
"""
Finalize firmware installation process.
Copies the content of the temporary file to
the desired download location.
Calls the provided firmware installer's install_firmware function
"""
self.logger.debug("update_finalize called")
self.logger.debug(
"download location: %s ;file_name: %s ;temp_file: %s",
self.download_location,
self.file_name,
self.temp_file,
)
if not os.path.exists(os.path.abspath(self.download_location)):
os.makedirs(os.path.abspath(self.download_location))
firmware_file_path = os.path.join(
os.path.abspath(self.download_location), self.file_name)
if self.temp_file:
shutil.copy2(os.path.realpath(self.temp_file.name),
firmware_file_path)
self.temp_file.close()
self.logger.info(
"Firmware file copied to download location from "
"temporary file, calling firmware_installer.install_firmware "
"with path: %s",
firmware_file_path,
)
self.firmware_installer.install_firmware(firmware_file_path)
def update_abort(self):
"""Abort the firmware update process."""
self.logger.debug("update_abort called")
if self.temp_file:
self.temp_file.close()
self.temp_file = None
self.file_name = None
self.file_size = None
self.file_url = None
def write_chunk(self, chunk):
"""
Write a chunk of the firmware file to the temporary file.
:param chunk: A piece of the firmware file
:type chunk: bytes
:returns: result
:rtype: bool
"""
self.logger.debug("write_chunk called - Chunk size: %s", len(chunk))
self.temp_file.write(chunk)
self.temp_file.flush()
os.fsync(self.temp_file)
self.logger.debug("write_chunk - Chunk written, returning True")
return True
def read_chunk(self, index):
"""
Read a chunk of the temporary firmware file.
:param index: Offset from the beginning of the file
:type index: int
:returns: chunk
:rtype: bytes
"""
self.logger.debug("read_chunk called - Index: %s", index)
self.temp_file.seek(index * self.chunk_size)
chunk = self.temp_file.read(self.chunk_size)
self.logger.debug("read_chunk - Chunk size: %s", len(chunk))
return chunk
def persist_version(self, version):
"""
Place the current firmware version into persistent storage.
Later to be used to determine the result
of the firmware update process
:param version: Current firmware version
:type version: str
:returns: result
:rtype: bool
"""
self.logger.debug("persist_version called - Version: %s", version)
self.version_persister.clear()
self.version_persister.flush()
self.version_persister.push(version)
self.logger.debug(
"persist_version - Persisted version, returning True")
return True
def unpersist_version(self):
"""
Remove the firmware version from persistent storage.
:returns: version
:rtype: str or None
"""
self.logger.debug("unpersist_version called")
if not self.version_persister.peek():
self.version_persister.clear()
self.logger.debug(
"unpersist_version - No persisted version, returning None")
return None
else:
version = self.version_persister.pop()
self.version_persister.flush()
self.logger.debug(
"unpersist_version - Persisted version found, returning %s",
version)
return version
def set_url_download_result_callback(self, callback):
"""
Set callback for reporting URL download result.
:param callback: Function to call
:type callback: function
"""
self.logger.debug(
"set_url_download_result_callback called - Callback: %s", callback)
self.report_result_callback = callback
def update_start_url_download(self, file_url):
"""
Start firmware file URL download process.
Calls download function from firmware_url_download_handler.
Returns the validity of the URL and calls download function if valid.
:param file_url: URL that contains the firmware file
:type file_url: str
:returns: result
:rtype: bool
"""
self.logger.debug("update_start_url_download called - File URL: %s",
file_url)
if not self.firmware_url_download_handler:
self.logger.debug(
"update_start_url_download - No firmware_url_download_handler,"
"returning False")
return False
parsed_url = urlparse(file_url)
if bool(parsed_url.scheme):
self.file_url = file_url
self.file_name = self.file_url.split("/")[-1]
t = Timer(
2.0,
self.firmware_url_download_handler.download,
[self.file_url, self.file_name, self.report_result_callback],
)
t.start()
self.logger.debug(
"update_start_url_download - Valid URL, calling "
"firmware_url_download_handler.download and returning True")
return True
else:
self.logger.debug(
"update_start_url_download - Invalid URL, returning False")
return False