def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('-d', help='debug', dest='debug', action='store_true', default=False) parser.add_argument('--bootstrap_server', dest='bootstrap_server', help="comma separated values", required=True) parser.add_argument('--topics', dest='topics', help="comma separated values", required=True) parser.add_argument('--from_beginning', dest='from_beginning', action='store_true', default=False) parser.add_argument('--tdmq_url', dest='tdmq_url', required=True) parser.add_argument('--kafka_group', dest='kafka_group', default='tdm_ingestion', required=False) parser.add_argument('--tdmq_auth_token', dest='tdmq_auth_token', required=True) args = parser.parse_args() logger.info('running with args %s', args.__dict__) if (args.debug): tdm_ingestion.converters.ngsi_converter.set_log_level(logging.DEBUG) tdm_ingestion.storage.tdmq.set_log_level(logging.DEBUG) tdm_ingestion.tdmq.remote.set_log_level(logging.DEBUG) kwargs = {} if args.from_beginning: kwargs['auto.offset.reset'] = 'beginning' consumer = KafkaConsumer(args.bootstrap_server.split(','), args.topics.split(','), args.kafka_group, **kwargs) tdmq = CachedStorage(Client(args.tdmq_url, auth_token=args.tdmq_auth_token)) converter = NgsiConverter() while True: try: tdmq.write(converter.convert(consumer.poll(timeout_s=5))) except Exception as ex: logger.exception(ex)
def main(run_docker=True): try: if run_docker: docker_compose_up(docker_yaml) with open(os.path.join(DIR, '../../messages/ngsi-weather.json'), 'rb') as f: messages = json.load(f) logger.debug("Data to send %s", messages) port = get_service_port(docker_yaml, 'web', '8000') base_url = f'http://localhost:{port}/api/v0.0' for message in messages: sensor_name = NgsiConverter._get_source_id(message) try_func(send_message, 1, 10, None, 'test', message) try_func(check_timeseries, 1, 10, base_url, sensor_name, { 'after': '2000-01-01T00:00:00Z', 'before': '2100-01-01T00:00:00Z' }) except RuntimeError as e: logger.error("Test failed: %s", str(e)) else: logger.info("Test passed") finally: if run_docker: docker_compose_down(docker_yaml)
future.get(60) logger.debug("message sent") return True def increment_and_wait(counter, wait=5): time.sleep(wait) counter += 1 try: docker_compose_up(docker_yaml) with open(os.path.join(DIR, '../../messages/ngsi-weather.json'), 'rb') as f: messages = json.load(f) logger.debug("Data to send %s", messages) port = get_service_port(docker_yaml, 'web', '8000') base_url = f'http://localhost:{port}/api/v0.0' for message in messages: sensor_name = NgsiConverter._get_source_id(message) try_func(send_message, 1, 10, None, 'test', message) try_func(check_timeseries, 1, 10, base_url, sensor_name, { 'after': '2000-01-01T00:00:00Z', 'before': '2100-01-01T00:00:00Z' }) finally: docker_compose_down(docker_yaml) pass
def test_convert_device(self): self._test_convert_device(NgsiConverter())
def test_convert_energy(self): self._test_convert_energy(NgsiConverter())
def test_convert_weather(self): self._test_convert_weather(NgsiConverter())
def _test_convert_error(self, message): timeseries_list = NgsiConverter().convert(message) self.assertEqual(len(timeseries_list), 0)
def test_ngsi_convert(self): self._test_convert(NgsiConverter())
def increment_and_wait(counter, wait=5): time.sleep(wait) counter += 1 def get_sensor_id(url, sensor_name): r = requests.get(url, params={'name': sensor_name}) r.raise_for_status() return r.json()[0]['code'] with open('messages/ngsi-weather.json', 'rb') as f: data = json.load(f) port = subprocess.check_output(['docker', 'port', 'travis_web_1', '8000']).strip().split(b':')[-1].decode() base_url = f'http://localhost:{port}/api/v0.0' print(f'base_url {base_url}') _, _, _, sensor_name = NgsiConverter._get_names(data) sensor_name = f'{sensor_name}.windDirection' try_func(send_message, 1, 10, None, 'test', data) try_func(check_timeseries, 1, 10, base_url, sensor_name, { 'after': '2000-01-01T00:00:00Z', 'before': '2100-01-01T00:00:00Z' })