def check_for_wu_error(self, wu_resp): """ Checks if weather underground returned an error. If so, raises an exception. :return True if there is no exception, else raise WeatherUndergroundException """ if wu_resp is None: return True if isinstance(wu_resp, list): # history request returned fine return True parsed_json = jsonapi.loads(wu_resp) if 'response' in parsed_json and 'error' in parsed_json['response']: raise WeatherUndergroundError("{}:{}".format( parsed_json['response']['error'].get('type'), parsed_json['response']['error'].get('description'))) elif not (any(key in parsed_json for key in ['history', 'current_observation', 'hourly_forecast'])): # if there are no errors but if it doesn't contain any valid # results either then location might have been ambiguous raise WeatherUndergroundError('Ambiguous location information') else: return True
def in_loop(self, sender, **kwargs): # pylint: disable=unused-argument peer = self.peer with closing(self.in_sock) as sock: sock.bind(self.publish_address) while True: message = sock.recv_multipart() #log.debug('incoming message: {!r}'.format(message)) topic = message[0] if (topic.startswith('subscriptions/list') and topic[18:19] in ['/', '']): if len(message) > 2: del message[2:] elif len(message) == 1: message.append('') prefix = topic[19:].decode('utf-8') topics = self.pubsub.list(peer, prefix, subscribed=False).get() message.extend( topic.encode('utf-8') for _, topic, _ in topics) self.out_sock.send_multipart(message) else: message = [part.decode('utf-8') for part in message] try: topic, headers = message[:2] except (ValueError, TypeError): continue headers = jsonapi.loads(headers) message = message[2:] self.pubsub.publish(peer, topic, headers, message).get()
def log_entries(name, agent, pid, level, stream): log = logging.getLogger(name) extra = {'processName': agent, 'process': pid} for l in stream: for line in l.splitlines(): if line.startswith('{') and line.endswith('}'): try: obj = jsonapi.loads(line) try: obj['args'] = tuple(obj['args']) except (KeyError, TypeError, ValueError): pass record = logging.makeLogRecord(obj) except Exception: pass else: if record.name in log.manager.loggerDict: if not logging.getLogger( record.name).isEnabledFor(record.levelno): continue elif not log.isEnabledFor(record.levelno): continue record.remote_name, record.name = record.name, name record.__dict__.update(extra) log.handle(record) continue if line[0:1] == '<' and line[2:3] == '>' and line[1:2].isdigit(): yield _level_map.get(int(line[1]), level), line[3:] else: yield level, line
def _get_point(self, device, point_map): ''' Get value of a point_name on a device ''' result = {} query = {} point_map_obj = {} for point_names, properties in point_map.items(): point_name = properties[0] query = jsonapi.loads(self.thermostat.tstat()) if point_name in self.query_point_name: try: db = query[self.point_name_map[point_name]] result.update({point_names: str(db)}) except: result.update({point_names: str("NA")}) else: pgm, day = point_name.rsplit('_', 1) if pgm == 'heat_pgm': if day == 'week': query = self.thermostat.get_heat_pgm() result.update({point_names: str(query)}) else: query = self.thermostat.get_heat_pgm(day) result.update({point_names: str(query)}) elif pgm == 'cool_pgm': if day == 'week': query = self.thermostat.get_cool_pgm() result.update({point_names: str(query)}) else: query = self.thermostat.get_cool_pgm(day) result.update({point_names: str(query)}) return (result)
def data_received(self, peer, sender, bus, topic, headers, message): def publish_external(agent, topic, headers, message): try: _log.debug('Attempting to publish remotely {}, {}, {}'.format(topic, headers, message)) agent.vip.pubsub.publish(peer='pubsub', topic=topic, headers=headers, message=message).get(timeout=30) except: _log.debug('Data dropped {}, {}, {}'.format(topic, headers, message)) if sender == 'pubsub.compat': message = jsonapi.loads(message[0]) del(headers[headers_mod.CONTENT_TYPE]) assert isinstance(message, list) assert isinstance(message[0], dict) assert isinstance(message[1], dict) print("MESSAGE VALUES ARE: {}".format(message[0])) print("DATA VALUES ARE: {}".format(message[1])) for v in message[1].values(): assert 'tz' in v assert 'units' in v assert 'type' in v #message = [jsonapi.loads(message[0]), jsonapi.loads(message[1])] if has_point_ex: for rex in point_ex: if rex.match(topic): publish_external(self._target_platform, topic, headers, message) else: publish_external(self._target_platform, topic, headers, message)
def get_hitorian_weather(self): """ Returns historian hourly weather data provided by the api via an http request. :param station_code: station code for the area. Can be found in weather.gov.com Past_time: previous time start t :return: time of data observation as a timestamp string, data dictionary containing weather data points """ _log.debug("Collecting current weather data") url = "https://api.weather.gov/stations/{}/" \ "observations/{}".format(self.station_code, self.past_time) grequest = [ grequests.get(url, verify=requests.certs.where(), headers=self.headers, timeout=5) ] gresponse = grequests.map(grequest)[0] if gresponse is None: raise RuntimeError("get request did not return any " "response") try: response = jsonapi.loads(gresponse.content) self.properties = response["properties"] except ValueError: self.generate_response_error(url, gresponse.status_code) pass
def query_current_weather(self, location): """ Returns current hourly weather data provided by the api via an http request. :param location: currently accepts station id (K followed by 3 letters, case insensitive) or lat/long (up to 4 decimals) location dictionary formats :return: time of data observation as a timestamp string, data dictionary containing weather data points """ if location.get('station'): formatted_location = self.get_location_string(location) url = "https://api.weather.gov/stations/{}/" \ "observations/latest".format(formatted_location) else: raise ValueError('Invalid location. Expected format is:' '{"station":"station_id_value"}') grequest = [grequests.get(url, verify=requests.certs.where(), headers=self.headers, timeout=5)] gresponse = grequests.map(grequest)[0] if gresponse is None: raise RuntimeError("get request did not return any " "response") try: response = jsonapi.loads(gresponse.content) properties = response["properties"] observation_time = properties["timestamp"] return observation_time, properties except ValueError: self.generate_response_error(url, gresponse.status_code)
def extract_data_fields(self): try: data_dict = jsonapi.loads(str(self.data)) except ValueError: data_dict = None if type(data_dict) == dict: if 'method' in data_dict: self.method = str(data_dict.get('method')) if 'result' in data_dict: self.result = str(data_dict.get('result')) if 'params' in data_dict: self.params = str(data_dict.get('params')) params = data_dict.get('params') if type(params) == dict: if 'topic' in params: self.topic = str(params.get('topic')) if 'headers' in params: self.headers = str(params.get('headers')) if 'message' in params: self.message = str(params.get('message')) message_list = params.get('message') message_val = message_list[0] \ if (message_list and type(message_list) == list and len(message_list) > 0) \ else '' if message_val: self.message_value = str(message_val) elif type(params) == list and (self.sender == 'platform.driver' or self.recipient == 'platform.driver'): if len(params) > 0 and params[0]: self.device = str(params[0]) if len(params) > 1 and params[1]: self.point = str(params[1]) if len(params) > 2 and params[2] != '': self.point_value = str(params[2])
def get_current_weather(self): """ Returns current hourly weather data provided by the api via an http request. :param location: currently accepts station id (K followed by 3 letters, case insensitive) or lat/long (up to 4 decimals) location dictionary formats :return: time of data observation as a timestamp string, data dictionary containing weather data points """ _log.debug("Collecting current weather data") url = "https://api.weather.gov/stations/{}/" \ "observations/latest".format(self.station_code) grequest = [ grequests.get(url, verify=requests.certs.where(), headers=self.headers, timeout=5) ] gresponse = grequests.map(grequest)[0] if gresponse is None: raise RuntimeError("get request did not return any " "response") try: response = jsonapi.loads(gresponse.content) self.properties = response["properties"] except ValueError: self.generate_response_error(url, gresponse.status_code)
def onmessage(peer, sender, bus, topic, headers, message): alert = jsonapi.loads(message)["context"] try: alert_messages[alert] += 1 except KeyError: alert_messages[alert] = 1
def build_metadata_oadr_report(self, report): descriptions = [] for tel_vals in jsonapi.loads(report.telemetry_parameters).values(): # Rule 305: For TELEMETRY_USAGE reports, units in reportDescription.itemBase should be powerReal. if tel_vals['units'] == 'powerReal': item_base = oadr_20b.PowerRealType(itemDescription='RealPower', itemUnits='W', siScaleCode=None, powerAttributes=None) else: item_base = None min_freq, max_freq = tel_vals['min_frequency'], tel_vals[ 'max_frequency'] desc = oadr_20b.oadrReportDescriptionType( rID=tel_vals['r_id'], reportType=tel_vals['report_type'], readingType=tel_vals['reading_type'], itemBase=item_base, oadrSamplingRate=self.build_sampling_rate(min_freq, max_freq)) descriptions.append(desc) rpt_interval_duration = isodate.duration_isoformat( timedelta(seconds=report.interval_secs)) return oadr_20b.oadrReportType( duration=oadr_20b.DurationPropType(rpt_interval_duration), oadrReportDescription=descriptions, reportRequestID=None, reportSpecifierID=report.report_specifier_id, reportName=report.name, createdDateTime=utils.get_aware_utc_now())
def recv_message(self, flags=0): '''Recieve a message as (topic, headers, message) tuple.''' topic = self.recv_string(flags) headers = self.recv_string(flags) if self.rcvmore else '' headers = jsonapi.loads(headers) if headers else {} message = self.recv_multipart(flags) if self.rcvmore else [] return topic, Headers(headers), message
def build_resp_forecast(self, wu_resp): publish_items = [] if wu_resp is None: return publish_items try: parsed_json = jsonapi.loads(wu_resp) if 'hourly_forecast' in parsed_json: observations = parsed_json['hourly_forecast'] for observation in observations: parsed_values = {} parsed_values[observation_epoch] = observation['FCTTIME']['epoch'] for point in all_hourly_forecast: if point in observation: if isinstance(observation[point], dict) \ and 'english' in observation[point]: parsed_values[point] = observation[point]['english'] else: parsed_values[point] = observation[point] if parsed_values: publish_items.append(parsed_values) except Exception as e: _log.exception(e) raise return publish_items
def build_resp_history(self, wu_resp_arr): publish_items = [] if wu_resp_arr is None: return publish_items try: for wu_resp in wu_resp_arr: parsed_json = jsonapi.loads(wu_resp) if 'history' in parsed_json and 'observations' in parsed_json['history']: observations = parsed_json['history']['observations'] for observation in observations: parsed_values = {} obs_time_json = observation['utcdate'] t = datetime(int(obs_time_json['year']), int(obs_time_json['mon']), int(obs_time_json['mday']), int(obs_time_json['hour']), int(obs_time_json['min']), tzinfo=pytz.utc) # Convert time to epoch string to be consistent with current condition parsed_values[observation_epoch] = str(int((t - datetime(1970,1,1,tzinfo=pytz.utc)).total_seconds())) for point in all_history: if point in observation: parsed_values[point] = observation[point] if parsed_values: publish_items.append(parsed_values) except Exception as e: _log.exception(e) raise _log.debug("Returning history result {}".format(publish_items)) return publish_items
def transfer_file(self, peer, sender, bus, topic, headers, message): """ Pubsub interface for transferring files. The interface requires message to be a dictionary like object or a json serializable string with the following required structure: { "direction": "SENDING" "remote_path": "/remote/path/file.txt", "local_path": "/local/path/file.txt" } The above direction must be either "SENDING" or "RECEIVING". The path must be available on the host that is providing the content and will overwrite the data on the receiving side of the connection. """ enabled = self.__check_configuration__() if not enabled: return False if isinstance(message, str): try: message = jsonapi.loads(message) except JSONDecodeError: _log.error(f"Invalid json passed through string interface") return direction = message.get("direction") remote_path = message.get("remote_path") local_path = message.get("local_path") enabled = True if not remote_path: enabled = False _log.error(f"remote_path not specified in message to pub sub") if not local_path: enabled = False _log.error(f"local_path not specified in message to pub sub") if direction not in WhichWayEnum.__members__: _log.error(f"which_way must be either SENDING or RECEIVING.") enabled = False if not enabled: return if direction == WhichWayEnum.SENDING.name: success = self.__handle_scp__(WhichWayEnum.SENDING, local_path, remote_path) else: success = self.__handle_scp__(WhichWayEnum.RECEIVING, remote_path, local_path) if not success: _log.error(f"Unable to send to/recieve scp files.")
def get_topic_meta_map(self): q = "SELECT topic_id, metadata FROM " + self.meta_table + ";" rows = self.select(q, None) _log.debug("loading metadata from db") topic_meta_map = dict() for id, meta in rows: topic_meta_map[id] = jsonapi.loads(meta) if meta else None return topic_meta_map
def get_topic_meta_map(self): query = SQL('SELECT topic_id, metadata ' 'FROM {}').format(Identifier(self.meta_table)) rows = self.select(query) meta_map = { tid: jsonapi.loads(meta) if meta else None for tid, meta in rows } return meta_map
def test_discovery(scheme): vhome = create_volttron_home() # creates a vhome level key store keystore = KeyStore() serverkey = decode_key(keystore.public) # Depending upon scheme we enable/disable password jwt and certificate based jwt. if scheme == 'https': with certs_profile_1('/'.join([vhome, 'certs'])) as certs: config_params = dict(web_ssl_key=certs.server_certs[0].key_file, web_ssl_cert=certs.server_certs[0].cert_file) else: config_params = dict(web_secret_key=get_random_key()) with get_test_volttron_home(messagebus='zmq', config_params=config_params): instance_name = "booballoon" host, port = get_hostname_and_random_port() # this is the vip address address = f"tcp://{host}:{port}" def _construct_query_mock(core): """ Internal function that creates a concrete response for the data. when query('instance-name').get() is called the passed instance name is returned """ nonlocal instance_name, address kv = {"instance-name": instance_name, "addresses": [address]} return MockQuery(**kv) with mock.patch('volttron.platform.vip.agent.subsystems.query.Query', _construct_query_mock): host, port = get_hostname_and_random_port() bind_web_address = f"{scheme}://{host}:{port}" serverkey = decode_key(keystore.public) mws = MasterWebService(serverkey=serverkey, identity=MASTER_WEB, address=address, bind_web_address=bind_web_address, **config_params) mws.startupagent(sender='testweb') env = get_test_web_env("/discovery/") mock_start_response = mock.Mock() # A closingiterator is returned from the response object so we use the next # on the returned response. Then we can do json responses. response = mws.app_routing(env, mock_start_response).__next__() # load json into a dict for testing responses. response = jsonapi.loads(response.decode('utf-8')) assert response.get('instance-name') is not None assert instance_name == response.get('instance-name') assert keystore.public == response.get('serverkey') assert address == response.get('vip-address')
def get_pending_csr_requests(self): pending_csr = [] for c in os.listdir(self.csr_pending_dir): if c.endswith('.json'): if os.stat(os.path.join(self.csr_pending_dir, c)).st_size != 0: with open(os.path.join(self.csr_pending_dir, c)) as fp: pending_csr.append(jsonapi.loads(fp.read())) return pending_csr
def query(self, topic_ids, id_name_map, start=None, end=None, skip=0, agg_type=None, agg_period=None, count=None, order='FIRST_TO_LAST'): if agg_type and agg_period: table_name = agg_type + '_' + agg_period value_col = 'agg_value' else: table_name = self.data_table value_col = 'value_string' topic_id = Literal(0) query = [ SQL('''SELECT to_char(ts, 'YYYY-MM-DD"T"HH24:MI:SS.USOF:00'), ''' + value_col + ' \n' 'FROM {}\n' 'WHERE topic_id = {}').format(Identifier(table_name), topic_id) ] if start and start.tzinfo != pytz.UTC: start = start.astimezone(pytz.UTC) if end and end.tzinfo != pytz.UTC: end = end.astimezone(pytz.UTC) if start and start == end: query.append(SQL(' AND ts = {}').format(Literal(start))) else: if start: query.append(SQL(' AND ts >= {}').format(Literal(start))) if end: query.append(SQL(' AND ts < {}').format(Literal(end))) query.append( SQL('ORDER BY ts {}'.format('DESC' if order == 'LAST_TO_FIRST' else 'ASC'))) if skip or count: query.append( SQL('LIMIT {} OFFSET {}').format( Literal(None if not count or count < 0 else count), Literal(None if not skip or skip < 0 else skip))) query = SQL('\n').join(query) values = {} if value_col == 'agg_value': for topic_id._wrapped in topic_ids: name = id_name_map[topic_id.wrapped] with self.select(query, fetch_all=False) as cursor: values[name] = [(ts, value) for ts, value in cursor] else: for topic_id._wrapped in topic_ids: name = id_name_map[topic_id.wrapped] with self.select(query, fetch_all=False) as cursor: values[name] = [(ts, jsonapi.loads(value)) for ts, value in cursor] return values
def outbound_response_handler(self, ch, method, props, body): """ Message received from internal agent to send to remote agent in ZMQ VIP message format. :param ch: channel :param method: contains routing key :param props: message properties like VIP header information :param body: message :return: """ # Strip sender's identity from binding key routing_key = str(method.routing_key) platform, to_identity = routing_key.split(".", 1) platform, from_identity = props.app_id.split(".", 1) userid = props.headers.get('user', '') # Reformat message into ZMQ VIP format frames = [ to_identity, from_identity, 'VIP1', userid, props.message_id, props.type ] try: args = jsonapi.loads(body) try: # This is necessary because jsonrpc request/response is inside a list which the # ZMQ agent subsystem does not like args = jsonapi.loads(args[0]) frames.append(jsonapi.dumps(args)) except ValueError as e: if isinstance(args, list): for m in args: frames.append(m) else: frames.append(jsonapi.dumps(args)) except TypeError as e: _log.error("Invalid json format {}".format(e)) return _log.debug("Proxy ZMQ Router Outbound handler {0}, {1}".format( to_identity, args)) frames = serialize_frames(frames) try: self.zmq_router.socket.send_multipart(frames, copy=True) except ZMQError as ex: _log.error("ZMQ Error {}".format(ex))
def get_darksky_data(self, service, location, timestamp=None): """ Generic method called by the current and forecast service endpoint methods to fetch a forecast request from the Darksky API. If performance mode is set to True, the url adds exclusions for the services provided by the API that were not requested. :param service: requested service endpoint :param location: location dictionary for building url :param timestamp: timestamp of a record if this request is for the Time Machine end point :return: Darksky forecast request response """ service_json_name = '' if service in SERVICES_MAPPING: service_json_name = SERVICES_MAPPING[service]['json_name'] if "lat" in location and 'long' in location: if timestamp: timestamp = int( (timestamp.replace(tzinfo=None) - datetime.datetime.utcfromtimestamp(0)).total_seconds()) url = "https://api.darksky.net/forecast/{key}/{lat}," \ "{long},{timestamp}?units=us".format( key=self._api_key, lat=location['lat'], long=location['long'], timestamp=timestamp) else: url = "https://api.darksky.net/forecast/{key}/{lat}," \ "{long}?units=us".format( key=self._api_key, lat=location['lat'], long=location['long']) if self.performance_mode: services = ["currently", "hourly", "minutely", "daily"] if service_json_name and service_json_name in services: services.remove(service_json_name) url += "&exclude=" + ",".join(services) else: raise RuntimeError("Requested service {} is not provided" " by the Darksky API".format(service)) else: raise ValueError('Invalid location. Expected format is: ' '"{"lat": "xxx.xxxx", "long": "xxx.xxxx"}"') _log.info("requesting url: {}".format(url)) grequest = [ grequests.get(url, verify=requests.certs.where(), headers=self.headers, timeout=3) ] gresponse = grequests.map(grequest)[0] self.add_api_call() if gresponse is None: raise RuntimeError("get request did not return any " "response") try: response = jsonapi.loads(gresponse.content) return response except ValueError: self.generate_response_error(url, gresponse.status_code)
def get_topic_meta(client, topic_id): """ Execute query to return the meta dictionary of a specific topic (lowercase name) This information should take from 'meta' measurement in the InfluxDB database. """ query = 'SELECT meta_dict FROM meta WHERE topic_id=\'%s\'' % topic_id rs = client.query(query) rs = list(rs.get_points()) meta = rs[0]['meta_dict'].replace("u'", "\"").replace("'", "\"") return jsonapi.loads(meta)
def add_definitions_to_config_store(test_agent): """Add PointDefinitions and FunctionDefinitions to the mesaagent's config store.""" with open(POINT_DEFINITIONS_PATH, 'r') as f: points_json = jsonapi.loads(strip_comments(f.read())) test_agent.vip.rpc.call('config.store', 'manage_store', MESA_AGENT_ID, 'mesa_points.config', points_json, config_type='raw') with open(FUNCTION_DEFINITIONS_PATH, 'r') as f: functions_json = yaml.load(f.read()) test_agent.vip.rpc.call('config.store', 'manage_store', MESA_AGENT_ID, 'mesa_functions.config', functions_json, config_type='raw')
def onmessage(peer, sender, bus, topic, headers, message): global alert_messages alert = jsonapi.loads(message)["context"] try: alert_messages[alert] += 1 except KeyError: alert_messages[alert] = 1 print("In on message: {}".format(alert_messages))
def _handle_external_rpc_subsystem(self, message): ret_msg = dict() operation = message.args[0] rpc_msg = message.args[1] # jsonapi.loads(message.args[1]) try: method_args = rpc_msg["args"] # message.args = [method_args] message.args = method_args for idx, msg in enumerate(message.args): if isinstance(msg, str): message.args[idx] = jsonapi.loads(msg) dispatch = self._dispatcher.dispatch # _log.debug("External RPC IN message args {}".format(message)) responses = [ response for response in (dispatch(msg, message) for msg in message.args) if response ] # _log.debug("External RPC Responses {}".format(responses)) if responses: message.user = "" try: message.peer = "" message.subsystem = "external_rpc" frames = [] operation = "send_platform" frames.append(operation) msg = jsonapi.dumps( dict( to_platform=rpc_msg["from_platform"], to_peer=rpc_msg["from_peer"], from_platform=rpc_msg["to_platform"], from_peer=rpc_msg["to_peer"], args=responses, )) frames.append(msg) except KeyError: _log.error("External RPC message did not contain " "proper message format") message.args = jsonapi.dumps(ret_msg) try: self.core().connection.send_vip( peer="", subsystem="external_rpc", args=frames, msg_id=message.id, user=message.user, copy=False, ) except ZMQError as ex: _log.error("ZMQ error: {}".format(ex)) pass except KeyError: pass
def test_insert_topic_and_meta_query_should_succeed(get_container_func): container, sqlfuncts, connection_port, historian_version = get_container_func if historian_version == "<4.0.0": pytest.skip("Not relevant for historian schema before 4.0.0") topic = "football" metadata = {"units": "count"} actual_id = sqlfuncts.insert_topic(topic, metadata=metadata) assert isinstance(actual_id, int) result = get_data_in_table(connection_port, "topics")[0] assert (actual_id, topic) == result[0:2] assert metadata == jsonapi.loads(result[2])
def build_report_payload_list(self, telemetry_values): """Build a list of ReportPayloads containing current telemetry.""" report_payload_list = [] for tel_val in jsonapi.loads( self.report.telemetry_parameters).values(): payload = self.build_report_payload_float(telemetry_values, tel_val['r_id'], tel_val['method_name']) if payload: report_payload_list.append(payload) return report_payload_list
def get_auth_config_from_store(self): """ :return: Fetch currently stored auth configuration info from config store, returns empty dict if none is present """ configs = self.vip.rpc.call(CONFIGURATION_STORE, "manage_list_configs", PLATFORM_DRIVER).get(timeout=3) if self.auth_config_path in configs: return jsonapi.loads(self.vip.rpc.call( CONFIGURATION_STORE, "manage_get", PLATFORM_DRIVER, self.auth_config_path).get(timeout=3)) else: _log.warning("No Ecobee auth file found in config store") return {}
def json_string_to_dict(self, value): """ Verify if the value was converted to json string at the time of storing into db. If so, convert it back to dict and return :param value: :return: """ result_value = value if isinstance(result_value, dict) and result_value.get(_VOLTTRON_TYPE): if result_value[_VOLTTRON_TYPE] == 'json': result_value = loads(result_value['string_value']) return result_value