def post(self, packet, payload):
record = packet.data
device_id = record['device_id']
component = record['name']
gid = record['gid']
id = '%s-%s-%s' % (device_id, component, gid)
log.info('====START InsertObservation id=%s' % id)
log.info('POSTing InsertObservation! try 1 - payload=%s' % payload)
statuscode, statusmessage, res = HttpOutput.post(self, packet, payload)
# InsertObservation may fail when Sensor not in SOS
# Try to do an InsertSensor and try InsertObservation again
if statuscode == 400:
log.info('No sensor for station: res=%s, will insert' % res)
insert_sensor_payload = self.create_insert_sensor_payload(packet)
log.info('POSTing InsertSensor! - payload=%s' % insert_sensor_payload)
statuscode, statusmessage, res = HttpOutput.post(self, packet, insert_sensor_payload)
if statuscode != 200:
log.warn('FAIL InsertSensor for station: rec=%s res=%s' % (str(packet.data), res))
else:
log.info('YES InsertSensor OK! id=%s POSTing InsertObservation! try 2' % id)
statuscode, statusmessage, res = HttpOutput.post(self, packet, payload)
if statuscode == 200:
log.info('YES InsertObservation! try 2 OK!! %s' % id)
else:
log.warn('FAIL InsertObservation payload=%s res=%s' % (payload, res))
elif statuscode == 200:
log.info('YES inserted Observation! try 1 id=%s' % id)
log.info('====END InsertObservation id=%s' % id)
return statuscode, statusmessage, res
def __init__(self, configdict, section):
HttpOutput.__init__(self,
configdict,
section,
consumes=FORMAT.record_array)
if self.template_file_root is None:
self.template_file_root = path.join(
path.dirname(path.realpath(__file__)), 'statemplates')
# Template file, to be used as POST body with substituted values
# TODO use Jinja2 formatting i.s.o. basic string formatting
self.entities_list = [
'thing', 'location', 'datastream', 'observation',
'observedproperty', 'sensor'
]
self.entity_templates = {}
self.sensors = {}
self.things = {}
self.locations = {}
self.observedproperties = {}
self.base_path = self.path
self.base_url = 'http://%s:%d%s' % (self.host, self.port, self.path)
self.http_session = requests.Session()
def __init__(self, configdict, section):
HttpOutput.__init__(self,
configdict,
section,
consumes=FORMAT.record_array)
# Construct write path
self.path = '/write?db=%s' % self.database
# Construct the template line to be used for each data frame
# e.g. 'joseraw,station=%s,component=%s value=%s %s'
self.tags_template = None
if self.tags_map:
self.tags = self.tags_map.keys()
self.tags_template = ''
for tag in self.tags:
self.tags_template += ',' + tag + '=%s'
self.geohash_template = None
if self.geohash_map or self.geohash_wkt_attr:
self.geohash_template = ',geohash="%s" '
self.fields = self.fields_map.keys()
self.fields_template = ' '
for field in self.fields:
self.fields_template += field + '=%s'
# will expand to e.g. joseraw,station=19,component=no2raw value=12345 1434055562000000000
# tags, fields and timestamp will be filled when creating payload
self.template_data_line = self.measurement + '%s%s%s\n'
def __init__(self, configdict, section):
HttpOutput.__init__(self, configdict, section, consumes=FORMAT.record_array)
# Template file, to be used as POST body with substituted values
self.insert_sensor_templ_path = '%s/insert-sensor.json' % self.template_file_root
self.insert_obs_templ_path ='%s/insert-observation.json' % self.template_file_root
self.proc_desc_templ_path = '%s/procedure-desc.xml' % self.template_file_root
self.insert_sensor_templ_str = None
self.insert_obs_templ_str = None
def __init__(self, configdict, section):
HttpOutput.__init__(self, configdict, section, consumes=FORMAT.record_array)
self.content_type = self.cfg.get('content_type', 'application/json;charset=UTF-8')
self.sos_request = self.cfg.get('sos_request', 'insert-observation')
# Template file, to be used as POST body with substituted values
self.template_file_ext = self.cfg.get('template_file_ext', 'json')
self.template_file_root = self.cfg.get('template_file_root', 'sostemplates')
self.template_file_path = '%s/%s.%s' % (self.template_file_root, self.sos_request, self.template_file_ext)
def __init__(self, configdict, section):
HttpOutput.__init__(self, configdict, section, consumes=FORMAT.record_array)
self.content_type = self.cfg.get('content_type', 'application/json;charset=UTF-8')
self.sos_request = self.cfg.get('sos_request', 'insert-observation')
# Template file, to be used as POST body with substituted values
self.template_file_ext = self.cfg.get('template_file_ext', 'json')
self.template_file_root = self.cfg.get('template_file_root', 'sostemplates')
self.template_file_path = '%s/%s.%s' % (self.template_file_root, self.sos_request, self.template_file_ext)
def __init__(self, configdict, section):
HttpOutput.__init__(self,
configdict,
section,
consumes=FORMAT.record_array)
# Template file, to be used as POST body with substituted values
self.insert_sensor_templ_path = '%s/insert-sensor.json' % self.template_file_root
self.insert_obs_templ_path = '%s/insert-observation.json' % self.template_file_root
self.proc_desc_templ_path = '%s/procedure-desc.xml' % self.template_file_root
self.insert_sensor_templ_str = None
self.insert_obs_templ_str = None
def __init__(self, configdict, section):
HttpOutput.__init__(self, configdict, section, consumes=FORMAT.record_array)
# Template file, to be used as POST body with substituted values
# TODO use Jinja2 formatting i.s.o. basic string formatting
self.entities_list = ['thing', 'datastream', 'observation', 'observedproperty', 'sensor']
self.entity_templates = {}
self.sensors = {}
self.things = {}
self.observedproperties = {}
self.base_path = self.path
self.base_url = 'http://%s:%d%s' % (self.host, self.port, self.path)
def __init__(self, configdict, section):
HttpOutput.__init__(self, configdict, section, consumes=FORMAT.record_array)
# Construct write path
self.path = '/write?db=%s' % self.database
# Construct the template line to be used for each data frame
# We split into separate sub-templates for tags, fields, optional geohashes
# Optional Tags template
self.tags_template = None
if self.tags_map:
self.tags = self.tags_map.keys()
self.tags_template = ''
for tag in self.tags:
if self.tags_template:
# Additional tags
self.tags_template += ',' + tag + '=%s'
else:
# First or one tag
self.tags_template = tag + '=%s'
# Required Fields template (need at least one field)
self.fields_template = None
self.fields = self.fields_map.keys()
for field in self.fields:
if self.fields_template:
# Additional fields
self.fields_template += ',' + field + '=%s'
else:
# First or one field
self.fields_template = field + '=%s'
# Optional extra geohash as tag template
if self.geohash_tag:
self.geohash_tag_template = ',' + self.geohash_tag_name + '=%s'
# Optional extra geohash as field template
if self.geohash_field:
self.geohash_field_template = ',' + self.geohash_field_name + '="%s"'
# will expand to <measurement>,<tags> <fields>
# e.g. joseraw,station=19,component=no2raw value=12345,geohash_tag=uvx53kryp 1434055562000000000
# tags, fields and timestamp will be substituted when creating payload
self.template_data_line = self.measurement + ',%s %s %d\n'
self.base_path = self.path
self.base_url = 'http://%s:%d%s' % (self.host, self.port, self.path)
self.http_session = requests.Session()
def post(self, packet, payload):
record = packet.data
device_id = record['device_id']
component = record['name']
gid = record['gid']
id = '%s-%s-%s' % (device_id, component, gid)
# insert_sensor_payload = self.create_insert_sensor_payload(packet)
# log.info('POSTing InsertSensor! - payload=%s' % insert_sensor_payload)
# statuscode, statusmessage, res = HttpOutput.post(self, packet, insert_sensor_payload)
# return statuscode, statusmessage, res
log.info('===START - POSTing InsertObservation! try 1 - id = %s', id)
statuscode, statusmessage, res = HttpOutput.post(self, packet, payload)
# InsertObservation may fail when Sensor not in SOS
# Try to do an InsertSensor and try InsertObservation again
if statuscode == 400:
log.info('No sensor for station: will insert')
insert_sensor_payload = self.create_insert_sensor_payload(packet)
log.info('POSTing InsertSensor!')
statuscode, statusmessage, res = HttpOutput.post(
self, packet, insert_sensor_payload)
if statuscode != 200:
log.warn('FAIL InsertSensor for station: rec=%s res=%s' %
(str(packet.data), res))
else:
log.info(
'YES InsertSensor OK! id=%s POSTing InsertObservation! try 2'
% id)
statuscode, statusmessage, res = HttpOutput.post(
self, packet, payload)
if statuscode == 200:
log.info('YES InsertObservation! try 2 OK!! %s' % id)
else:
log.warn('FAIL InsertObservation res=%s' % res)
elif statuscode == 200:
log.info('YES inserted Observation! try 1 id=%s' % id)
# update_sensor_desc_payload = self.create_update_sensor_desc_payload(packet)
# log.info('POSTing UpdateSensorDescription! - payload=%s' % update_sensor_desc_payload)
# statuscode, statusmessage, res = HttpOutput.post(self, packet, update_sensor_desc_payload)
# if statuscode == 200:
# log.info('OK UpdateSensorDescription for station: rec=%s res=%s' % (str(packet.data), res))
# else:
# log.warn('FAIL UpdateSensorDescription for station: rec=%s res=%s' % (str(packet.data), res))
log.info('====END - InsertObservation id=%s' % id)
return statuscode, statusmessage, res
def post_datastream(self, thing, record):
# {'device_id': 1, 'name': 'temperature', 'value': 19, 'unit': 'Celsius', 'gid': 50,
# 'time': datetime.datetime(2016, 4, 27, 5, 0, tzinfo=psycopg2.tz.FixedOffsetTimezone(offset=120, name=None)),
# 'lat': 51.472585, , 'lon': 5.671208, , 'altitude': 210, }
format_args = dict()
name = record['name']
sensor_def = SENSOR_DEFS[name]
format_args['thing_id'] = thing['@iot.id']
# Add sensor is not exists for name
sensor = self.sensors.get(name)
if not sensor:
sensor = self.post_sensor(record)
self.sensors[name] = sensor
format_args['sensor_id'] = sensor['@iot.id']
# Add observedproperty is not exists for name
observedproperty = self.observedproperties.get(name)
if not observedproperty:
observedproperty = self.post_observedproperty(record)
self.observedproperties[name] = observedproperty
format_args['observedproperty_id'] = observedproperty['@iot.id']
format_args['station_id'] = record['device_id']
format_args['name'] = record['name']
format_args['label'] = sensor_def['label']
format_args['unit'] = record['unit']
payload = self.entity_templates['datastream'].format(**format_args)
self.path = self.base_path + '/Datastreams'
statuscode, statusmessage, res = HttpOutput.post(self, None, payload)
datastream = json.loads(res)
return datastream
def __init__(self, configdict, section):
HttpOutput.__init__(self,
configdict,
section,
consumes=FORMAT.record_array)
if self.template_file_root is None:
self.template_file_root = path.join(
path.dirname(path.realpath(__file__)), 'sostemplates')
# Template file, to be used as POST body with substituted values
self.insert_sensor_templ_path = '%s/insert-sensor.json' % self.template_file_root
self.update_sensor_desc_templ_path = '%s/update-sensor-desc.json' % self.template_file_root
self.insert_obs_templ_path = '%s/insert-observation.json' % self.template_file_root
self.proc_desc_templ_path = '%s/procedure-desc.xml' % self.template_file_root
self.insert_sensor_templ_str = None
self.update_sensor_desc_templ_str = None
self.insert_obs_templ_str = None
def post_observedproperty(self, record):
format_args = dict()
format_args['name'] = record['name']
format_args['unit'] = record['unit']
payload = self.entity_templates['observedproperty'].format(**format_args)
self.path = self.base_path + '/ObservedProperties'
statuscode, statusmessage, res = HttpOutput.post(self, None, payload)
entity = json.loads(res)
return entity
def post_sensor(self, record):
format_args = dict()
format_args['name'] = record['name']
format_args['label'] = record['unit']
payload = self.entity_templates['sensor'].format(**format_args)
self.path = self.base_path + '/Sensors'
statuscode, statusmessage, res = HttpOutput.post(self, None, payload)
entity = json.loads(res)
return entity
def post_thing(self, record):
format_args = dict()
format_args['station_id'] = record['device_id']
# format_args['station_altitude'] = record['altitude']
format_args['station_lon'] = record['lon']
format_args['station_lat'] = record['lat']
payload = self.entity_templates['thing'].format(**format_args)
self.path = self.base_path + '/Things'
statuscode, statusmessage, res = HttpOutput.post(self, None, payload)
thing = json.loads(res)
return thing
def post(self, packet, payload):
# {'device_id': 1, 'name': 'temperature', 'value': 19, 'unit': 'Celsius', 'gid': 50,
# 'time': datetime.datetime(2016, 4, 27, 5, 0, tzinfo=psycopg2.tz.FixedOffsetTimezone(offset=120, name=None)),
# 'lat': 51.472585, , 'lon': 5.671208, , 'altitude': 210, }
record = packet.data
component = record['name']
device_id = str(record['device_id'])
gid = record['gid']
id = '%s-%s-%s' % (device_id, component, gid)
log.info('====START POST Observation id=%s' % id)
statuscode, statusmessage, res = HttpOutput.post(self, packet, payload)
# Check result
if statuscode in [200, 201]:
log.info('YES added Observation! id=%s status=%s' % (id, statusmessage))
else:
log.warn('FAIL POST Observation status=%d payload=%s res=%s' % (statuscode, payload, res))
log.info('====END POST Observation id=%s' % id)
return statuscode, statusmessage, res
