def get_next_element_when_ready(self): self.first_element_changed.acquire() try: isNotReady = True while isNotReady: if self._qsize() > 0: first_element = heapq.nsmallest(1, self.queue)[0] if isinstance(first_element, SystemExit): first_element = self._get() break if not first_element.flag_alive: log.debug("Early termination of dead metric") first_element = self._get() break timeout = ( first_element.get_next_run_time() - getUTCmillis() ) / 1000.0 log.debug("Waiting on acquired first_element_changed LOCK " + "for: %.2f" % timeout) self.first_element_changed.wait(timeout) else: self.first_element_changed.wait() first_element = heapq.nsmallest(1, self.queue)[0] if isinstance(first_element, SystemExit): first_element = self._get() break if (first_element.get_next_run_time() - getUTCmillis()) <= 0 \ or not first_element.flag_alive: isNotReady = False first_element = self._get() return first_element finally: self.first_element_changed.release()
def get_next_element_when_ready(self): self.first_element_changed.acquire() try: isNotReady = True while isNotReady: if self._qsize() > 0: first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if not first_element.flag_alive: log.debug("Early termination of dead metric") first_element = self._get() break timeout = (first_element.get_next_run_time() - getUTCmillis()) / 1000.0 log.debug( "Waiting on acquired first_element_changed LOCK " + "for: %.2f" % timeout) self.first_element_changed.wait(timeout) else: self.first_element_changed.wait() first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if (first_element.get_next_run_time() - getUTCmillis()) <= 0 \ or not first_element.flag_alive: isNotReady = False first_element = self._get() return first_element finally: self.first_element_changed.release()
def process(self, message): if message is not None: self.metric_list = [] print "Message in process: ", message metrics_message = { } #stores all the metric data and action taken corres. to them if self.timeout_occured: self.timeout_occured = False log.warning("No action taken, all metrics not available.") #can't return data as we don't know which metric has not got its value, it may be rpm, may be any metric, we are not checking for it print "All metrics are not available, please check the metric collecting sensors." return None #raise MyException("All metrics are not available, please check the metric collecting sensors.") else: result = self.model_rule(*message) counter = 0 counter = 0 if (result == 0) else counter + 1 if (counter >= self.exceed_limit): self.actuator_udm(1) metrics_message['result'] = 1 self.counter = 0 else: self.actuator_udm(0) metrics_message['result'] = 0 for i in range(self.no_args): metric_name = str( inspect.getargspec(self.model_rule)[0][i]) metrics_message[metric_name] = message[i] metrics_message['timestamp'] = getUTCmillis() json_data = json.dumps(metrics_message) return json_data
def simulated_list_of_timestamps_values_sampling_function(): random.seed(time.clock()) current_time = getUTCmillis() list_of_timestamp_value_tuples = [] for step in range(0, 25, 5): list_of_timestamp_value_tuples.append((current_time - step*1000, random.randint(0, 20))) return list_of_timestamp_value_tuples
def _window(self, collected_value, filtered_value): """ Windowing scheme. :param collected_value: Collected value by sampling function. :param filtered_value: Filtered value by sampling function. :return: Filtered value (or) collected value at the end of every time window. """ # Next window time has elapsed. if getUTCmillis() >= self.next_window_time: # At-least one sample has not passed so far during this window. if not self.sample_passed and filtered_value is None: self._set_next_window_time() log.info("Sending collected value for this window.") return collected_value # At-least one sample has (or will be) passed by now. else: self._set_next_window_time() return filtered_value # Could be filtered-value or None # Next window time has not elapsed. else: if filtered_value is not None: self.sample_passed = True # At-least one sample has passed during this window return filtered_value
def set_properties(self, reg_entity_obj, properties): # RegisteredMetric get parent's resid; RegisteredEntity gets own resid reg_entity_id = reg_entity_obj.reg_entity_id if isinstance(reg_entity_obj, RegisteredMetric): entity = reg_entity_obj.parent.ref_entity else: entity = reg_entity_obj.ref_entity log.info("Properties defined for resource {0}".format(entity.name)) self.comms.send( json.dumps( self._properties(self.next_id(), reg_entity_id, entity.entity_type, getUTCmillis(), properties))) if entity.entity_type == "HelixGateway": with self.file_ops_lock: self.store_reg_entity_attributes("EdgeSystem", entity.name, reg_entity_obj.reg_entity_id, None, properties) else: # get dev_type, and prop_dict if possible with self.file_ops_lock: self.store_reg_entity_attributes("Devices", entity.name, reg_entity_obj.reg_entity_id, entity.entity_type, properties)
def set_system_properties(self, reg_entity_obj, system_properties): """ Used to set the system property for RegisteredEntity(EdgeSystem) Registered Devices will inherit the system properties from the parent in create_relationship call :param reg_entity_obj: RegisteredEntity Object :param system_properties: System Property List :return: """ if isinstance(reg_entity_obj, RegisteredMetric): reg_entity_obj.parent.sys_properties = system_properties entity = reg_entity_obj.parent.ref_entity else: reg_entity_obj.sys_properties = system_properties entity = reg_entity_obj.ref_entity # set_properties internal response queue set_sys_prop_resp_q = Queue.Queue() # create and add register request into req_list (waiting list) transaction_id = self._next_id() req = Request(transaction_id, set_sys_prop_resp_q) log.debug("Updating set system properties response queue for transaction_id:{0}".format(transaction_id)) with self._req_ops_lock: self._req_dict.update({transaction_id: req}) self.comms.send(json.dumps( self._properties(transaction_id, entity.entity_type, entity.entity_id, entity.name, getUTCmillis(), system_properties))) response = self._handle_response(set_sys_prop_resp_q.get(True, timeout)) if response: log.info("System Properties defined for resource {0}".format(entity.name)) else: raise Exception("Setting System Properties for resource {0} failed".format(entity.name))
def start_collecting(self): self.flag_alive = True # TODO: Add a check to ensure that start_collecting for a metric is # called only once by the client code metric_handler.initialize() self._next_run_time = getUTCmillis() + (self.ref_entity.interval * 1000) metric_handler.event_ds.put_and_notify(self)
def set_organization_group_properties(self, reg_entity_name, reg_entity_id, reg_entity_type, properties): log.info( "Organization Group Properties defined for resource {0}".format( reg_entity_name)) self.comms.send( json.dumps( self._properties(self.next_id(), reg_entity_id, reg_entity_type, getUTCmillis(), properties)))
def add_collected_data(self, collected_data): if isinstance(collected_data, list): for data_sample in collected_data: self.values.put(data_sample) return len(collected_data) elif isinstance(collected_data, tuple): self.values.put(collected_data) return 1 else: self.values.put((getUTCmillis(), collected_data)) return 1
def test_implementation_get_formatted_data_without_enclose_metatadata( self): """ Test case to check the output given by get_formatted_data method without enclosed meta_data option. RegisteredEdgeSystem->RegisteredMetric :return: None """ # Mocking the DataCenterComponent init method with mock.patch("liota.dccs.aws_iot.DataCenterComponent.__init__" ) as mocked_dcc: self.aws = AWSIoT("Mocked DCC", enclose_metadata=False) # Register the edge registered_entity = self.aws.register(self.edge_system) # Creating test Metric test_metric = Metric(name="Test_Metric", unit=None, interval=10, aggregation_size=2, sampling_function=sampling_function) registered_metric = self.aws.register(test_metric) # Creating the parent-child relationship self.aws.create_relationship(registered_entity, registered_metric) # Get current timestamp timestamp = getUTCmillis() registered_metric.values.put((timestamp, 10)) # Expected output without enclosed metadata expected_output = { "metric_name": registered_metric.ref_entity.name, "metric_data": [{ "value": 10, "timestamp": timestamp }], "unit": "null" } # Getting the parent child relationship array from registered metric formatted_data = get_formatted_data(registered_metric, False) # Convert json string to dict for the comparision formatted_json_data = json.loads(formatted_data) # Check two dicts are equal or not self.assertEqual( validate_json(formatted_json_data) == validate_json( expected_output), True, "Check implementation of get_formatted_data")
def set_system_properties(self, reg_entity_obj, system_properties): if isinstance(reg_entity_obj, RegisteredMetric): reg_entity_obj.parent.sys_properties = system_properties entity = reg_entity_obj.parent.ref_entity else: reg_entity_obj.sys_properties = system_properties entity = reg_entity_obj.ref_entity log.info("System Properties defined for resource {0}".format(entity.name)) self.comms.send(json.dumps( self._properties(self.next_id(), entity.entity_type, entity.entity_id, entity.name, getUTCmillis(), system_properties)))
def start_collecting(self): """ Start to collect data for the metric: initialize metric handler and put the metric's first event to events priority queue. :return: """ self.flag_alive = True # TODO: Add a check to ensure that start_collecting for a metric is # called only once by the client code metric_handler.initialize() self._next_run_time = getUTCmillis() + (self.ref_entity.interval * 1000) metric_handler.event_ds.put_and_notify(self)
def get_next_element_when_ready(self): """ Get next event from events priority queue when it is ready: for SystemExit event and dead event, pop from queue immediately; for other active event, wait until next run time, then pop from queue. :return: """ self.first_element_changed.acquire() try: isNotReady = True while isNotReady: if self._qsize() > 0: first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if not first_element.flag_alive: log.debug("Early termination of dead metric") first_element = self._get() break timeout = ( first_element.get_next_run_time() - getUTCmillis() ) / 1000.0 log.debug("Waiting on acquired first_element_changed LOCK " + "for: %.2f" % timeout) self.first_element_changed.wait(timeout) else: self.first_element_changed.wait() first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if (first_element.get_next_run_time() - getUTCmillis()) <= 0 \ or not first_element.flag_alive: isNotReady = False first_element = self._get() return first_element finally: self.first_element_changed.release()
def set_properties(self, reg_entity_obj, properties): # RegisteredMetric get parent's resid; RegisteredEntity gets own resid reg_entity_id = reg_entity_obj.reg_entity_id if isinstance(reg_entity_obj, RegisteredMetric): entity = reg_entity_obj.parent.ref_entity else: entity = reg_entity_obj.ref_entity log.info("Properties defined for resource {0}".format(entity.name)) self.con.send( self._properties(self.con.next_id(), reg_entity_id, entity.entity_type, getUTCmillis(), properties))
def get_next_element_when_ready(self): """ Get next event from events priority queue when it is ready: for SystemExit event and dead event, pop from queue immediately; for other active event, wait until next run time, then pop from queue. :return: """ self.first_element_changed.acquire() try: isNotReady = True while isNotReady: if self._qsize() > 0: first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if not first_element.flag_alive: log.debug("Early termination of dead metric") first_element = self._get() break timeout = (first_element.get_next_run_time() - getUTCmillis()) / 1000.0 log.debug( "Waiting on acquired first_element_changed LOCK " + "for: %.2f" % timeout) self.first_element_changed.wait(timeout) else: self.first_element_changed.wait() first_element = self.queue[0] if isinstance(first_element, SystemExit): first_element = self._get() break if (first_element.get_next_run_time() - getUTCmillis()) <= 0 \ or not first_element.flag_alive: isNotReady = False first_element = self._get() return first_element finally: self.first_element_changed.release()
def set_properties(self, reg_entity_obj, properties): """ Set Properties for Registered Entity (Edge System or Devices) :param reg_entity_obj: RegisteredEntity Object :param properties: Properties List :return: """ # RegisteredMetric get parent's resid; RegisteredEntity gets own resid reg_entity_id = reg_entity_obj.reg_entity_id if isinstance(reg_entity_obj, RegisteredMetric): entity = reg_entity_obj.parent.ref_entity else: entity = reg_entity_obj.ref_entity set_prop_resp_q = Queue.Queue() # create and add register request into req_list (waiting list) transaction_id = self._next_id() req = Request(transaction_id, set_prop_resp_q) log.debug( "Updating set properties response queue for transaction_id:{0}". format(transaction_id)) with self._req_ops_lock: self._req_dict.update({transaction_id: req}) self.comms.send( json.dumps( self._properties(transaction_id, entity.entity_type, entity.entity_id, entity.name, getUTCmillis(), properties))) response = self._handle_response(set_prop_resp_q.get(True, timeout)) if response: log.info("Properties defined for resource {0}".format(entity.name)) else: raise Exception( "Setting Properties for resource {0} failed".format( entity.name)) if entity.entity_type == "HelixGateway": with self.file_ops_lock: self._store_reg_entity_attributes("EdgeSystem", entity, reg_entity_obj.reg_entity_id, None, properties) else: # get dev_type, and prop_dict if possible with self.file_ops_lock: self._store_reg_entity_attributes("Devices", entity, reg_entity_obj.reg_entity_id, entity.entity_type, properties)
def __init__(self, filter_obj, window_size_sec): """ :param filter_obj: Filter object. :param window_size_sec: Window size in seconds. """ if not isinstance(filter_obj, Filter): raise TypeError("Filter Object is expected") if not isinstance(window_size_sec, Number) or window_size_sec < 0: log.error("window_size_sec must be a non negative number") raise ValueError("window_size_sec must be a non negative number") self.filter_obj = filter_obj self.window_size_sec = window_size_sec # To track whether at-least one sample has been passed after filtering within a window self.sample_passed = False self.next_window_time = getUTCmillis() + (self.window_size_sec * 1000)
def add_collected_data(self, collected_data): """ For the metric, add collected data into values queue. :param collected_data: collected data which may be in the format of list, tuple, and single sampled value :return: the length of added data """ if isinstance(collected_data, list): for data_sample in collected_data: self.values.put(data_sample) return len(collected_data) elif isinstance(collected_data, tuple): self.values.put(collected_data) return 1 else: self.values.put((getUTCmillis(), collected_data)) return 1
def test_implementation_get_formatted_data_with_enclose_metadata(self): """ Test case to check the implementation of get_formatted_data method with enclose_metadata option of AWSIoT class. RegisteredEdgeSystem->RegisteredMetric :return: None """ # Register the edge registered_entity = self.aws.register(self.edge_system) # Creating test Metric test_metric = Metric(name="Test_Metric", unit=None, interval=10, aggregation_size=2, sampling_function=sampling_function) registered_metric = self.aws.register(test_metric) # Creating the parent-child relationship self.aws.create_relationship(registered_entity, registered_metric) timestamp = getUTCmillis() registered_metric.values.put((timestamp, 10)) expected_output = { "edge_system_name": registered_entity.ref_entity.name, "metric_name": registered_metric.ref_entity.name, "metric_data": [{ "value": 10, "timestamp": timestamp }], "unit": "null" } # Getting the parent child relationship array from registered metric formatted_data = get_formatted_data(registered_metric, True) formatted_json_data = json.loads(formatted_data) # Check two dicts are equal or not self.assertEqual( validate_json(formatted_json_data) == validate_json( expected_output), True, "Check implementation of get_formatted_data")
def set_organization_group_properties(self, reg_entity_name, reg_entity_id, reg_entity_type, properties): log.info( "Organization Group Properties defined for resource {0}".format( reg_entity_name)) self.con.send( self._properties(self.con.next_id(), reg_entity_id, reg_entity_type, getUTCmillis(), properties)) if reg_entity_type == "HelixGateway": with self.file_ops_lock: self.store_reg_entity_attributes("EdgeSystem", reg_entity_name, reg_entity_id, None, properties) else: # get dev_type, and prop_dict if possible with self.file_ops_lock: self.store_reg_entity_attributes("Devices", reg_entity_name, reg_entity_id, reg_entity_type, properties)
def set_properties(self, reg_entity_obj, properties): # RegisteredMetric get parent's resid; RegisteredEntity gets own resid reg_entity_id = reg_entity_obj.reg_entity_id if isinstance(reg_entity_obj, RegisteredMetric): entity = reg_entity_obj.parent.ref_entity else: entity = reg_entity_obj.ref_entity log.info("Properties defined for resource {0}".format(entity.name)) self.comms.send(json.dumps( self._properties(self.next_id(), entity.entity_type, entity.entity_id, entity.name, getUTCmillis(), properties))) if entity.entity_type == "HelixGateway": with self.file_ops_lock: self.store_reg_entity_attributes("EdgeSystem", entity.name, reg_entity_obj.reg_entity_id, None, properties) else: # get dev_type, and prop_dict if possible with self.file_ops_lock: self.store_reg_entity_attributes("Devices", entity.name, reg_entity_obj.reg_entity_id, entity.entity_type, properties)
def set_properties(self, reg_entity_obj, properties): """ Set Properties for Registered Entity (Edge System or Devices) :param reg_entity_obj: RegisteredEntity Object :param properties: Properties List :return: """ # RegisteredMetric get parent's resid; RegisteredEntity gets own resid reg_entity_id = reg_entity_obj.reg_entity_id if isinstance(reg_entity_obj, RegisteredMetric): entity = reg_entity_obj.parent.ref_entity else: entity = reg_entity_obj.ref_entity set_prop_resp_q = Queue.Queue() # create and add register request into req_list (waiting list) transaction_id = self._next_id() req = Request(transaction_id, set_prop_resp_q) log.debug("Updating set properties response queue for transaction_id:{0}".format(transaction_id)) with self._req_ops_lock: self._req_dict.update({transaction_id: req}) self.comms.send(json.dumps( self._properties(transaction_id, entity.entity_type, entity.entity_id, entity.name, getUTCmillis(), properties))) response = self._handle_response(set_prop_resp_q.get(True, timeout)) if response: log.info("Properties defined for resource {0}".format(entity.name)) else: raise Exception("Setting Properties for resource {0} failed".format(entity.name)) if entity.entity_type == "HelixGateway": with self.file_ops_lock: self._store_reg_entity_attributes("EdgeSystem", entity, reg_entity_obj.reg_entity_id, None, properties) else: # get dev_type, and prop_dict if possible with self.file_ops_lock: self._store_reg_entity_attributes("Devices", entity, reg_entity_obj.reg_entity_id, entity.entity_type, properties)
def simulated_timestamp_value_sampling_function(): current_time = getUTCmillis() # Random time in the last 5 seconds, when the sample was generated. sample_generation_time = current_time - random.randint(0, 5) sample_value = random.randint(0, 20) return (sample_generation_time, sample_value)
def test_implementation_get_formatted_data_with_enclose_metadata_device_exception_flow( self): """ Test case to test the implementation of get_formatted_data method with enclose_metadata option. RegisteredEdgeSystem->RegisteredDevice->RegisteredMetric :return: None """ # Mock the get_formatted_data with mock.patch("liota.lib.utilities.dcc_utility.parse_unit" ) as mocked_parse_unit: # Method to be invoked on calling the parse_unit function mocked_parse_unit.side_effect = raise_exception # Register the edge registered_entity = self.aws.register(self.edge_system) # Creating Simulated Device test_sensor = SimulatedDevice("TestSensor") # Registering Device and creating Parent-Child relationship reg_test_sensor = self.aws.register(test_sensor) self.aws.create_relationship(registered_entity, reg_test_sensor) # Creating test Metric test_metric = Metric(name="Test_Metric", unit=ureg.degC, interval=10, aggregation_size=2, sampling_function=sampling_function) registered_metric = self.aws.register(test_metric) # Creating the parent-child relationship self.aws.create_relationship(reg_test_sensor, registered_metric) # Get current timestamp timestamp = getUTCmillis() registered_metric.values.put((timestamp, 10)) # Expected output without enclosed metadata expected_output = { "edge_system_name": registered_entity.ref_entity.name, "metric_name": registered_metric.ref_entity.name, "device_name": reg_test_sensor.ref_entity.name, "metric_data": [{ "value": 10, "timestamp": timestamp }], "unit": "null" } # Getting the parent child relationship array from registered metric formatted_data = get_formatted_data(registered_metric, True) # Convert json string to dict for the comparision formatted_json_data = json.loads(formatted_data) # Check two dicts are equal or not self.assertEqual( validate_json(formatted_json_data) == validate_json( expected_output), True, "Check implementation of get_formatted_data")
def test_implementation_get_formatted_data_with_enclose_metadata_device( self): """ Test case to test the implementation of get_formatted_data method with enclose_metadata option. RegisteredEdgeSystem->RegisteredDevice->RegisteredMetric :return: None """ # Register the edge registered_entity = self.aws.register(self.edge_system) # Creating Simulated Device test_sensor = SimulatedDevice("TestSensor") # Registering Device and creating Parent-Child relationship reg_test_sensor = self.aws.register(test_sensor) self.aws.create_relationship(registered_entity, reg_test_sensor) # Creating test Metric test_metric = Metric(name="Test_Metric", unit=ureg.degC, interval=10, aggregation_size=2, sampling_function=sampling_function) registered_metric = self.aws.register(test_metric) # Creating the parent-child relationship self.aws.create_relationship(reg_test_sensor, registered_metric) # Get current timestamp timestamp = getUTCmillis() registered_metric.values.put((timestamp, 10)) # Expected output without enclosed metadata expected_output = { "edge_system_name": registered_entity.ref_entity.name, "metric_name": registered_metric.ref_entity.name, "device_name": reg_test_sensor.ref_entity.name, "metric_data": [{ "value": 10, "timestamp": timestamp }], "unit": "null" } unit_tuple = parse_unit(ureg.degC) if unit_tuple[0] is None: # Base and Derived Units expected_output['unit'] = unit_tuple[1] else: # Prefixed or non-SI Units expected_output['unit'] = unit_tuple[0] + unit_tuple[1] # Getting the parent child relationship array from registered metric formatted_data = get_formatted_data(registered_metric, True) # Convert json string to dict for the comparision formatted_json_data = json.loads(formatted_data) # Check two dicts are equal or not self.assertEqual( validate_json(formatted_json_data) == validate_json( expected_output), True, "Check implementation of get_formatted_data")
def set_organization_group_properties(self, reg_entity_name, reg_entity_id, reg_entity_type, entity_local_uuid, properties): log.info("Organization Group Properties defined for resource {0}".format(reg_entity_name)) self.comms.send(json.dumps( self._properties(self.next_id(), reg_entity_type, entity_local_uuid, reg_entity_name, getUTCmillis(), properties)))