def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-d',
help='debug',
dest='debug',
action='store_true',
default=False)
parser.add_argument('--bootstrap_server',
dest='bootstrap_server',
help="comma separated values",
required=True)
parser.add_argument('--topics',
dest='topics',
help="comma separated values",
required=True)
parser.add_argument('--from_beginning',
dest='from_beginning',
action='store_true',
default=False)
parser.add_argument('--tdmq_url', dest='tdmq_url', required=True)
parser.add_argument('--kafka_group',
dest='kafka_group',
default='tdm_ingestion',
required=False)
parser.add_argument('--tdmq_auth_token',
dest='tdmq_auth_token',
required=True)
args = parser.parse_args()
logger.info('running with args %s', args.__dict__)
if (args.debug):
tdm_ingestion.converters.ngsi_converter.set_log_level(logging.DEBUG)
tdm_ingestion.storage.tdmq.set_log_level(logging.DEBUG)
tdm_ingestion.tdmq.remote.set_log_level(logging.DEBUG)
kwargs = {}
if args.from_beginning:
kwargs['auto.offset.reset'] = 'beginning'
consumer = KafkaConsumer(args.bootstrap_server.split(','),
args.topics.split(','), args.kafka_group,
**kwargs)
tdmq = CachedStorage(Client(args.tdmq_url,
auth_token=args.tdmq_auth_token))
converter = NgsiConverter()
while True:
try:
tdmq.write(converter.convert(consumer.poll(timeout_s=5)))
except Exception as ex:
logger.exception(ex)
def main(run_docker=True):
try:
if run_docker:
docker_compose_up(docker_yaml)
with open(os.path.join(DIR, '../../messages/ngsi-weather.json'),
'rb') as f:
messages = json.load(f)
logger.debug("Data to send %s", messages)
port = get_service_port(docker_yaml, 'web', '8000')
base_url = f'http://localhost:{port}/api/v0.0'
for message in messages:
sensor_name = NgsiConverter._get_source_id(message)
try_func(send_message, 1, 10, None, 'test', message)
try_func(check_timeseries, 1, 10, base_url, sensor_name, {
'after': '2000-01-01T00:00:00Z',
'before': '2100-01-01T00:00:00Z'
})
except RuntimeError as e:
logger.error("Test failed: %s", str(e))
else:
logger.info("Test passed")
finally:
if run_docker:
docker_compose_down(docker_yaml)
future.get(60)
logger.debug("message sent")
return True
def increment_and_wait(counter, wait=5):
time.sleep(wait)
counter += 1
try:
docker_compose_up(docker_yaml)
with open(os.path.join(DIR, '../../messages/ngsi-weather.json'), 'rb') as f:
messages = json.load(f)
logger.debug("Data to send %s", messages)
port = get_service_port(docker_yaml, 'web', '8000')
base_url = f'http://localhost:{port}/api/v0.0'
for message in messages:
sensor_name = NgsiConverter._get_source_id(message)
try_func(send_message, 1, 10, None, 'test', message)
try_func(check_timeseries, 1, 10, base_url, sensor_name,
{
'after': '2000-01-01T00:00:00Z',
'before': '2100-01-01T00:00:00Z'
})
finally:
docker_compose_down(docker_yaml)
pass
def test_convert_device(self):
self._test_convert_device(NgsiConverter())
def test_convert_energy(self):
self._test_convert_energy(NgsiConverter())
def test_convert_weather(self):
self._test_convert_weather(NgsiConverter())
def _test_convert_error(self, message):
timeseries_list = NgsiConverter().convert(message)
self.assertEqual(len(timeseries_list), 0)
def test_ngsi_convert(self):
self._test_convert(NgsiConverter())
def increment_and_wait(counter, wait=5):
time.sleep(wait)
counter += 1
def get_sensor_id(url, sensor_name):
r = requests.get(url, params={'name': sensor_name})
r.raise_for_status()
return r.json()[0]['code']
with open('messages/ngsi-weather.json', 'rb') as f:
data = json.load(f)
port = subprocess.check_output(['docker', 'port', 'travis_web_1',
'8000']).strip().split(b':')[-1].decode()
base_url = f'http://localhost:{port}/api/v0.0'
print(f'base_url {base_url}')
_, _, _, sensor_name = NgsiConverter._get_names(data)
sensor_name = f'{sensor_name}.windDirection'
try_func(send_message, 1, 10, None, 'test', data)
try_func(check_timeseries, 1, 10, base_url, sensor_name, {
'after': '2000-01-01T00:00:00Z',
'before': '2100-01-01T00:00:00Z'
})