def get_templates(req):
"""
Fetches known templates, potentially limited by a given value. Ordering
might be user-configurable too.
:param req: The received HTTP request, as created by Flask.
:return A JSON containing pagination information and the template list
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
"""
init_tenant_context(req, db)
page_number, per_page = get_pagination(req)
page = DeviceTemplate.query.paginate(page=int(page_number),
per_page=int(per_page),
error_out=False)
templates = template_list_schema.dump(page.items).data
result = {
'pagination': {
'page': page.page,
'total': page.pages,
'has_next': page.has_next,
'next_page': page.next_num
},
'templates': templates
}
attr_format(req, result)
return result
def remove_template(template_id, token):
"""
Deletes a single template.
:param template_id: The template to be removed.
:param token: The authorization token (JWT).
:return The removed template.
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this template could not be found in
database.
:raises HTTPRequestError: If the template is being currently used by
a device.
"""
init_tenant_context(token, db)
tpl = assert_template_exists(template_id)
json_template = template_schema.dump(tpl)
try:
db.session.delete(tpl)
db.session.commit()
except IntegrityError:
raise HTTPRequestError(400, "Templates cannot be removed as they are being used by devices")
results = {
'result': 'ok',
'removed': json_template
}
return results
def create_template(req):
"""
Creates a new template.
:param req: The received HTTP request, as created by Flask.
:return The created template.
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If template attribute constraints were
violated. This might happen if two attributes have the same name, for
instance.
"""
init_tenant_context(req, db)
tpl, json_payload = parse_payload(req, template_schema)
loaded_template = DeviceTemplate(**tpl)
load_attrs(json_payload['attrs'], loaded_template, DeviceAttr, db)
db.session.add(loaded_template)
try:
db.session.commit()
LOGGER.debug(f" Created template in database")
except IntegrityError as e:
LOGGER.error(f' {e}')
raise HTTPRequestError(
400,
'Template attribute constraints are violated by the request')
results = {
'template': template_schema.dump(loaded_template),
'result': 'ok'
}
return results
def delete_all_templates(token):
"""
Deletes all templates.
:param token: The authorization token (JWT).
:raises HTTPRequestError: If this template could not be found in
database.
"""
init_tenant_context(token, db)
json_templates = []
try:
templates = db.session.query(DeviceTemplate)
for template in templates:
db.session.delete(template)
json_templates.append(template_schema.dump(template))
db.session.commit()
except IntegrityError:
raise HTTPRequestError(400, "Templates cannot be removed as they are being used by devices")
results = {
'result': 'ok',
'removed': json_templates
}
return results
def remove_template_from_device(req, device_id, template_id):
"""
Disassociates given template with device
:param req: The received HTTP request, as created by Flask.
:param device_id: The device which should be updated
:param template_id: The template to be removed from this device.
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device or template could not be found
in database.
:return A status on whether the device was updated, and the new
structure for that device.
:rtype JSON
"""
init_tenant_context(req, db)
updated_device = assert_device_exists(device_id)
relation = assert_device_relation_exists(device_id, template_id)
# Here (for now) there are no more validations to perform, as template
# removal cannot violate attribute constraints
db.session.remove(relation)
db.session.commit()
result = {
'message': 'device updated',
'device': serialize_full_device(updated_device)
}
return result
def add_template_to_device(req, device_id, template_id):
"""
Associates given template with device
:param req: The received HTTP request, as created by Flask.
:param device_id: The device which should be updated
:param template_id: The template to be added to this device.
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device or template could not be found
in database.
:return A status on whether the device was updated, and the new
structure for that device.
:rtype JSON
"""
init_tenant_context(req, db)
orm_device = assert_device_exists(device_id)
orm_template = assert_template_exists(template_id)
orm_device.templates.append(orm_template)
try:
db.session.commit()
except IntegrityError as error:
handle_consistency_exception(error)
result = {
'message': 'device updated',
'device': serialize_full_device(orm_device)
}
return result
def get_templates(req):
"""
Fetches known templates, potentially limited by a given value. Ordering
might be user-configurable too.
:param req: The received HTTP request, as created by Flask.
:return A JSON containing pagination information and the template list
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
"""
init_tenant_context(req, db)
page_number, per_page = get_pagination(req)
pagination = {'page': page_number, 'per_page': per_page, 'error_out': False}
parsed_query = []
query = req.args.getlist('attr')
for attr in query:
parsed = re.search('^(.+){1}=(.+){1}$', attr)
parsed_query.append(text("attrs.label = '{}'".format(parsed.group(1))))
parsed_query.append(text("attrs.static_value = '{}'".format(parsed.group(2))))
target_label = req.args.get('label', None)
if target_label:
parsed_query.append(text("templates.label like '%{}%'".format(target_label)))
SORT_CRITERION = {
'label': DeviceTemplate.label,
None: DeviceTemplate.id
}
sortBy = SORT_CRITERION.get(req.args.get('sortBy', None), DeviceTemplate.id)
if parsed_query:
LOGGER.debug(f" Filtering template by {parsed_query}")
page = db.session.query(DeviceTemplate) \
.join(DeviceAttr, isouter=True) \
.filter(*parsed_query) \
.order_by(sortBy) \
.paginate(**pagination)
else:
LOGGER.debug(f" Querying templates sorted by {sortBy}")
page = db.session.query(DeviceTemplate).order_by(sortBy).paginate(**pagination)
templates = []
for template in page.items:
templates.append(attr_format(req, template_schema.dump(template)))
result = {
'pagination': {
'page': page.page,
'total': page.pages,
'has_next': page.has_next,
'next_page': page.next_num
},
'templates': templates
}
return result
def get_devices(req):
"""
Fetches known devices, potentially limited by a given value. Ordering
might be user-configurable too.
:param req: The received HTTP request, as created by Flask.
:return A JSON containing pagination information and the device list
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
"""
if req.args.get('idsOnly', 'false').lower() in ['true', '1', '']:
return DeviceHandler.list_ids(req)
init_tenant_context(req, db)
page_number, per_page = get_pagination(req)
pagination = {'page': page_number, 'per_page': per_page, 'error_out': False}
parsed_query = []
query = req.args.getlist('attr')
for attr in query:
parsed = re.search('^(.+){1}=(.+){1}$', attr)
parsed_query.append("attrs.label = '{}'".format(parsed.group(1)))
parsed_query.append("attrs.static_value = '{}'".format(parsed.group(2)))
target_label = req.args.get('label', None)
if target_label:
parsed_query.append("devices.label = '{}'".format(target_label))
if len(parsed_query):
page = db.session.query(Device) \
.join(DeviceTemplateMap) \
.join(DeviceTemplate) \
.join(DeviceAttr) \
.filter(*parsed_query) \
.paginate(**pagination)
else:
page = db.session.query(Device).paginate(**pagination)
devices = []
for d in page.items:
devices.append(serialize_full_device(d))
result = {
'pagination': {
'page': page.page,
'total': page.pages,
'has_next': page.has_next,
'next_page': page.next_num
},
'devices': devices
}
return result
def delete_all_templates(req):
"""
Deletes all templates.
:param req: The received HTTP request, as created by Flask.
:raises HTTPRequestError: If this template could not be found in
database.
"""
init_tenant_context(req, db)
templates = db.session.query(DeviceTemplate)
for template in templates:
db.session.delete(template)
db.session.commit()
def list_ids(req):
"""
Fetches the list of known device ids.
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
"""
init_tenant_context(req, db)
data = []
for id in db.session.query(Device.id).all():
data.append(id[0])
return data
def import_data(data, token, content_type):
"""
Import data.
:param data: The received data HTTP request, as created by Flask.
:param token: The authorization token (JWT).
:param content_type: The content_type of request (application/json)
:return The status message.
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If import attribute constraints were
violated. This might happen if two attributes have the same name, for
instance.
"""
saved_templates = []
saved_devices = []
try:
tenant = init_tenant_context(token, db)
ImportHandler.clear_db_config(tenant)
original_req_data = copy.copy(data)
original_payload = json.loads(original_req_data)
data = ImportHandler.replace_ids_by_import_ids(data)
json_data, json_payload = parse_payload(content_type, data,
import_schema)
saved_templates = ImportHandler.save_templates(
json_data, json_payload)
saved_devices = ImportHandler.save_devices(json_data,
original_payload,
saved_templates)
ImportHandler.restore_db_config()
ImportHandler().notifies_creation_to_kafka(saved_devices, tenant)
db.session.commit()
except IntegrityError as e:
LOGGER.error(f' {e}')
db.session.rollback()
raise HTTPRequestError(
400,
'Template attribute constraints are violated by the request')
except Exception as e:
LOGGER.error(f' {e}')
db.session.rollback()
raise HTTPRequestError(400, 'Failed to import data')
finally:
db.session.close()
results = {'message': 'data imported!'}
return results
def delete_device(cls, device_id, token):
"""
Deletes a single device.
:param device_id: The device to be removed.
:param token: The authorization token (JWT).
:return The removed device.
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device could not be found in
database.
"""
tenant = init_tenant_context(token, db)
orm_device = assert_device_exists(device_id)
data = serialize_full_device(orm_device, tenant)
kafka_handler_instance = cls.kafka.getInstance(cls.kafka.kafkaNotifier)
kafka_handler_instance.remove(data, meta={"service": tenant})
db.session.delete(orm_device)
db.session.commit()
results = {'result': 'ok', 'removed_device': data}
return results
def get_by_template(req, template_id):
"""
Return a list of devices that have a particular template associated to
it
:param req: The received HTTP request, as created by Flask.
:param template_id: The template to be considered
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device or template could not be found
in database.
:return A list of devices that are associated to the selected template.
:rtype JSON
"""
tenant = init_tenant_context(req, db)
page_number, per_page = get_pagination(req)
page = (db.session.query(Device).join(DeviceTemplateMap).filter_by(
template_id=template_id).paginate(page=page_number,
per_page=per_page,
error_out=False))
devices = []
for d in page.items:
devices.append(serialize_full_device(d, tenant))
result = {
'pagination': {
'page': page.page,
'total': page.pages,
'has_next': page.has_next,
'next_page': page.next_num
},
'devices': devices
}
return result
def delete_device(req, device_id):
"""
Deletes a single device.
:param req: The received HTTP request, as created by Flask.
:param device_id: The device to be removed.
:return The removed device.
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device could not be found in
database.
"""
tenant = init_tenant_context(req, db)
orm_device = assert_device_exists(device_id)
data = serialize_full_device(orm_device, tenant)
kafka_handler = KafkaHandler()
kafka_handler.remove(data, meta={"service": tenant})
if CONFIG.orion:
subscription_handler = PersistenceHandler(service=tenant)
subscription_handler.remove(orm_device.persistence)
db.session.delete(orm_device)
db.session.commit()
results = {'result': 'ok', 'removed_device': data}
return results
def list_ids(token):
"""
Fetches the list of known device ids.
:param token: The authorization token (JWT).
:return A JSON containing a list of devices ids
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
"""
init_tenant_context(token, db)
data = []
LOGGER.debug(f" Fetching list with known devices")
for id in db.session.query(Device.id).all():
data.append(id[0])
return data
def get_template(req, template_id):
"""
Fetches a single template.
:param req: The received HTTP request, as created by Flask.
:param template_id: The requested template ID.
:return A Template
:rtype Template, as described in DatabaseModels package
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this template could not be found in
database.
"""
init_tenant_context(req, db)
tpl = assert_template_exists(template_id)
json_template = template_schema.dump(tpl)
attr_format(req, json_template)
return json_template
def get_device(req, device_id):
"""
Fetches a single device.
:param req: The received HTTP request, as created by Flask.
:param device_id: The requested device.
:return A Device
:rtype Device, as described in DatabaseModels package
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device could not be found in
database.
"""
init_tenant_context(req, db)
orm_device = assert_device_exists(device_id)
return serialize_full_device(orm_device)
def update_device(cls, params, device_id, token):
"""
Updated the information about a particular device
:param params: Parameters received from request (content_type, data)
as created by Flask
:param device_id: The device to be updated.
:param token: The authorization token (JWT).
:return The updated device.
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device could not be found in
database.
"""
try:
content_type = params.get('content_type')
data_request = params.get('data')
device_data, json_payload = parse_payload(content_type,
data_request,
device_schema)
validate_repeated_attrs(json_payload)
tenant = init_tenant_context(token, db)
old_orm_device = assert_device_exists(device_id)
db.session.delete(old_orm_device)
db.session.flush()
# handled separately by parse_template_list
device_data.pop('templates')
updated_orm_device = Device(**device_data)
parse_template_list(json_payload.get('templates', []),
updated_orm_device)
auto_create_template(json_payload, updated_orm_device)
updated_orm_device.id = device_id
updated_orm_device.updated = datetime.now()
updated_orm_device.created = old_orm_device.created
db.session.add(updated_orm_device)
db.session.commit()
except IntegrityError as error:
handle_consistency_exception(error)
except ValidationError as error:
raise HTTPRequestError(400, error.messages)
full_device = serialize_full_device(updated_orm_device, tenant)
kafka_handler_instance = cls.kafka.getInstance(cls.kafka.kafkaNotifier)
kafka_handler_instance.update(full_device, meta={"service": tenant})
result = {
'message': 'device updated',
'device': serialize_full_device(updated_orm_device, tenant)
}
return result
def update_template(req, template_id):
"""
Updates a single template.
:param req: The received HTTP request, as created by Flask.
:param template_id: The template to be updated.
:return The old version of this template (previous to the update).
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this template could not be found in
database.
"""
service = init_tenant_context(req, db)
# find old version of the template, if any
old = assert_template_exists(template_id)
# parse updated version from payload
updated, json_payload = parse_payload(req, template_schema)
old.label = updated['label']
for attr in old.attrs:
db.session.delete(attr)
for attr in json_payload['attrs']:
mapped = DeviceAttr(template=old, **attr)
db.session.add(mapped)
try:
db.session.commit()
except IntegrityError as error:
handle_consistency_exception(error)
# notify interested parties that a set of devices might have been implicitly updated
affected = db.session.query(DeviceTemplateMap) \
.filter(DeviceTemplateMap.template_id==template_id) \
.all()
affected_devices = []
for device in affected:
affected_devices.append(device.device_id)
event = {
"event": DeviceEvent.TEMPLATE,
"affected": affected_devices,
"template": template_schema.dump(old).data,
"meta": {
"service": service
}
}
send_raw(event, service)
results = {'updated': template_schema.dump(old).data, 'result': 'ok'}
return results
def configure_device(cls, params, device_id, token):
"""
Send actuation commands to the device
:param params: Parameters received from request (data)
as created by Flask
:param device_id: The device which should receive the actuation message
:param token: The authorization token (JWT).
:return A status on whether the message was sent to the device or not.
Note that this is not a guarantee that the device actually performed
what was requested.
:rtype JSON
"""
# Meta information to be published along with device actuation message
meta = {'service': ''}
invalid_attrs = []
meta['service'] = init_tenant_context(token, db)
meta['timestamp'] = int(time.time() * 1000)
orm_device = assert_device_exists(device_id)
full_device = serialize_full_device(orm_device, meta['service'])
LOGGER.debug(f" Full device: {json.dumps(full_device)}")
data = params.get('data')
payload = json.loads(data)
LOGGER.debug(f' Parsed request payload: {json.dumps(payload)}')
payload['id'] = orm_device.id
for attr in payload['attrs']:
if find_attribute(full_device, attr, 'actuator') is None:
invalid_attrs.append(attr)
if not invalid_attrs:
LOGGER.debug(f' Sending configuration message through Kafka.')
kafka_handler_instance = cls.kafka.getInstance(
cls.kafka.kafkaNotifier)
kafka_handler_instance.configure(payload, meta)
LOGGER.debug(f' Configuration sent.')
result = {f' status': 'configuration sent to device'}
else:
LOGGER.warning(
f' invalid attributes detected in command: {invalid_attrs}')
result = {
'status': 'some of the attributes are not configurable',
'attrs': invalid_attrs
}
raise HTTPRequestError(403, result)
return result
def configure_device(req, device_id):
"""
Send actuation commands to the device
:param req: The received HTTP request, as created by Flask.
:param device_id: The device which should receive the actuation message
:return A status on whether the message was sent to the device or not.
Note that this is not a guarantee that the device actually performed
what was requested.
:rtype JSON
"""
# Meta information to be published along with device actuation message
meta = {
'service': ''
}
kafka_handler = KafkaHandler()
invalid_attrs = []
meta['service'] = init_tenant_context(req, db)
orm_device = assert_device_exists(device_id)
full_device = serialize_full_device(orm_device, meta['service'])
LOGGER.debug(f" Full device: {json.dumps(full_device)}")
payload = json.loads(req.data)
LOGGER.debug(f' Parsed request payload: {json.dumps(payload)}')
payload['id'] = orm_device.id
for attr in payload['attrs']:
if find_attribute(full_device, attr, 'actuator') is None:
invalid_attrs.append(attr)
if not invalid_attrs:
LOGGER.debug(f' Sending configuration message through Kafka.')
kafka_handler.configure(payload, meta)
LOGGER.debug(f' Configuration sent.')
result = {f' status': 'configuration sent to device'}
else:
LOGGER.warning(f' invalid attributes detected in command: {invalid_attrs}')
result = {
'status': 'some of the attributes are not configurable',
'attrs': invalid_attrs
}
return result
def get_device(token, device_id, sensitive_data=False):
"""
Fetches a single device.
:param token: The authorization token (JWT).
:param device_id: The requested device.
:param sensitive_data: Informs if sensitive data like keys should be
returned
:return A Device
:rtype Device, as described in DatabaseModels package
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device could not be found in
database.
"""
tenant = init_tenant_context(token, db)
orm_device = assert_device_exists(device_id)
return serialize_full_device(orm_device, tenant, sensitive_data)
def delete_all_devices(req):
"""
Deletes all devices.
:param req: The received HTTP request, as created by Flask.
:raises HTTPRequestError: If this device could not be found in
database.
"""
tenant = init_tenant_context(req, db)
json_devices = []
devices = db.session.query(Device)
for device in devices:
db.session.delete(device)
json_devices.append(serialize_full_device(device, tenant))
db.session.commit()
results = {'result': 'ok', 'removed_devices': json_devices}
return results
def delete_all_devices(token):
"""
Deletes all devices.
:param token: The authorization token (JWT).
:raises HTTPRequestError: If this device could not be found in
database.
"""
tenant = init_tenant_context(token, db)
json_devices = []
devices = db.session.query(Device)
for device in devices:
db.session.delete(device)
json_devices.append(serialize_full_device(device, tenant))
db.session.commit()
results = {'result': 'ok', 'removed_devices': json_devices}
return results
def create_device(req):
"""
Creates and configures the given device.
:param req: The received HTTP request, as created by Flask.
:return The created device or a list of device summary. This depends on
which value the verbose (/?verbose=true) has - if true, only one device
can be created ("count" argument can't be used or - at least - it must
be exactly "1") and the full device is returned. If false, "count" can
be used with higher values and only the devices summary (a structure
containing all device IDs and their labels) is returned.
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If verbose is used with multiple device
creation request.
:raises HTTPRequestError: If count argument ("?count=X") is provided
and it is not an integer.
"""
tenant = init_tenant_context(req, db)
try:
count = int(req.args.get('count', '1'))
except ValueError as e:
LOGGER.error(e)
raise HTTPRequestError(400, "If provided, count must be integer")
c_length = len(str(count))
verbose = req.args.get('verbose', 'false') in ['true', '1', 'True']
if verbose and count != 1:
raise HTTPRequestError(
400, "Verbose can only be used for single device creation")
devices = []
# Handlers
kafka_handler = KafkaHandler()
if CONFIG.orion:
ctx_broker_handler = OrionHandler(service=tenant)
subs_handler = PersistenceHandler(service=tenant)
else:
ctx_broker_handler = None
subs_handler = None
full_device = None
for i in range(0, count):
device_data, json_payload = parse_payload(req, device_schema)
device_data['id'] = DeviceHandler.generate_device_id()
device_data['label'] = DeviceHandler.indexed_label(
count, c_length, device_data['label'], i)
# handled separately by parse_template_list
device_data.pop('templates', None)
orm_device = Device(**device_data)
parse_template_list(json_payload.get('templates', []), orm_device)
auto_create_template(json_payload, orm_device)
db.session.add(orm_device)
devices.append({
'id': device_data['id'],
'label': device_data['label']
})
full_device = serialize_full_device(orm_device, tenant)
# Updating handlers
kafka_handler.create(full_device, meta={"service": tenant})
if CONFIG.orion:
# Generating 'device type' field for history
type_descr = "template"
for dev_type in full_device['attrs'].keys():
type_descr += "_" + str(dev_type)
# TODO remove this in favor of kafka as data broker....
ctx_broker_handler.create(full_device, type_descr)
sub_id = subs_handler.create(full_device['id'], type_descr)
orm_device.persistence = sub_id
try:
db.session.commit()
except IntegrityError as error:
handle_consistency_exception(error)
if verbose:
result = {'message': 'device created', 'device': full_device}
else:
result = {'message': 'devices created', 'devices': devices}
return result
def get_templates(params, token):
"""
Fetches known templates, potentially limited by a given value. Ordering
might be user-configurable too.
:param params: Parameters received from request (page_number, per_page,
sort_by, attr, attr_type, label, attrs_format)
as created by Flask
:param token: The authorization token (JWT).
:return A JSON containing pagination information and the template list
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
"""
LOGGER.debug(f"Retrieving templates.")
LOGGER.debug(f"Initializing tenant context...")
init_tenant_context(token, db)
LOGGER.debug(f"... tenant context initialized.")
pagination = {'page': params.get('page_number'), 'per_page': params.get('per_page'), 'error_out': False}
LOGGER.debug(f"Pagination configuration is {pagination}")
parsed_query = []
query = params.get('attr')
for attr in query:
LOGGER.debug(f"Analyzing query parameter: {attr}...")
parsed = re.search('^(.+){1}=(.+){1}$', attr)
parsed_query.append(DeviceAttr.label == parsed.group(1))
parsed_query.append(DeviceAttr.static_value == parsed.group(2))
LOGGER.debug("... query parameter was added to filter list.")
query = params.get('attr_type')
for attr_type_item in query:
parsed_query.append(DeviceAttr.value_type == attr_type_item)
target_label = params.get('label')
if target_label:
LOGGER.debug(f"Adding label filter to query...")
parsed_query.append(DeviceTemplate.label.like("%{}%".format(target_label)))
LOGGER.debug(f"... filter was added to query.")
SORT_CRITERION = {
'label': DeviceTemplate.label,
None: None
}
sortBy = SORT_CRITERION.get(params.get('sortBy'), None)
LOGGER.debug(f"Sortby filter is {sortBy}")
if parsed_query:
LOGGER.debug(f" Filtering template by {parsed_query}")
# Always sort by DeviceTemplate.id
page = db.session.query(DeviceTemplate) \
.join(DeviceAttr, isouter=True) \
.filter(*parsed_query) \
.order_by(DeviceTemplate.id)
if sortBy:
page = page.order_by(sortBy)
page = page.distinct(DeviceTemplate.id)
LOGGER.debug(f"Current query: {type(page)}")
page = paginate(page, **pagination)
else:
LOGGER.debug(f" Querying templates sorted by {sortBy}")
page = db.session.query(DeviceTemplate).order_by(sortBy).paginate(**pagination)
templates = []
for template in page.items:
formatted_template = attr_format(params.get('attrs_format'), template_schema.dump(template))
LOGGER.debug(f"Adding resulting template to response...")
LOGGER.debug(f"Template is: {formatted_template['label']}")
templates.append(formatted_template)
LOGGER.debug(f"... template was added to response.")
result = {
'pagination': {
'page': page.page,
'total': page.pages,
'has_next': page.has_next,
'next_page': page.next_num
},
'templates': templates
}
LOGGER.debug(f"Full response is {result}")
return result
def update_template(cls, params, template_id, token):
"""
Updates a single template.
:param params: Parameters received from request (content_type, data)
as created by Flask
:param template_id: The template to be updated.
:param token: The authorization token (JWT).
:return The old version of this template (previous to the update).
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this template could not be found in
database.
"""
service = init_tenant_context(token, db)
content_type = params.get('content_type')
data_request = params.get('data')
# find old version of the template, if any
old = assert_template_exists(template_id)
# parse updated version from payload
updated, json_payload = parse_payload(content_type, data_request, template_schema)
LOGGER.debug(f" Current json payload: {json_payload}")
old.label = updated['label']
new = json_payload['attrs']
LOGGER.debug(f" Checking old template attributes")
def attrs_match(attr_from_db, attr_from_request):
return ((attr_from_db.label == attr_from_request["label"]) and
(attr_from_db.type == attr_from_request["type"]))
def update_attr(attrs_from_db, attrs_from_request):
attrs_from_db.value_type = attrs_from_request.get('value_type', None)
attrs_from_db.static_value = attrs_from_request.get('static_value', None)
def validate_attr(attr_from_request, is_meta):
if is_meta is False:
attr_schema.load(attr_from_request)
else:
metaattr_schema.load(attr_from_request)
def analyze_attrs(attrs_from_db, attrs_from_request, parentAttr=None):
for attr_from_db in attrs_from_db:
found = False
for idx, attr_from_request in enumerate(attrs_from_request):
validate_attr(attr_from_request, parentAttr is not None)
if attrs_match(attr_from_db, attr_from_request):
update_attr(attr_from_db, attr_from_request)
if "metadata" in attr_from_request:
analyze_attrs(attr_from_db.children, attr_from_request["metadata"], attr_from_db)
attrs_from_request.pop(idx)
found = True
break
if not found:
LOGGER.debug(f" Removing attribute {attr_from_db.label}")
db.session.delete(attr_from_db)
if parentAttr and attrs_from_request is not None:
for attr_from_request in attrs_from_request:
orm_child = DeviceAttr(parent=parentAttr, **attr_from_request)
db.session.add(orm_child)
return attrs_from_request
to_be_added = analyze_attrs(old.attrs, new)
for attr in to_be_added:
LOGGER.debug(f" Adding new attribute {attr}")
if "id" in attr:
del attr["id"]
child = DeviceAttr(template=old, **attr)
db.session.add(child)
if "metadata" in attr and attr["metadata"] is not None:
for metadata in attr["metadata"]:
LOGGER.debug(f" Adding new metadata {metadata}")
orm_child = DeviceAttr(parent=child, **metadata)
db.session.add(orm_child)
try:
LOGGER.debug(f" Commiting new data...")
refresh_template_update_column(db, old)
db.session.commit()
LOGGER.debug("... data committed.")
except IntegrityError as error:
LOGGER.debug(f" ConsistencyException was thrown.")
handle_consistency_exception(error)
# notify interested parties that a set of devices might have been implicitly updated
affected = db.session.query(DeviceTemplateMap) \
.filter(DeviceTemplateMap.template_id==template_id) \
.all()
affected_devices = []
kafka_handler_instance = cls.kafka.getInstance(cls.kafka.kafkaNotifier)
for device in affected:
orm_device = assert_device_exists(device.device_id)
kafka_handler_instance.update(serialize_full_device(orm_device, service), meta={"service": service})
affected_devices.append(device.device_id)
event = {
"event": DeviceEvent.TEMPLATE,
"data": {
"affected": affected_devices,
"template": template_schema.dump(old)
},
"meta": {"service": service}
}
kafka_handler_instance.kafkaNotifier.send_raw(event, service)
results = {
'updated': template_schema.dump(old),
'result': 'ok'
}
return results
def update_device(req, device_id):
"""
Updated the information about a particular device
:param req: The received HTTP request, as created by Flask.
:param device_id: The device to be updated.
:return The updated device.
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this device could not be found in
database.
"""
device_data, json_payload = parse_payload(req, device_schema)
tenant = init_tenant_context(req, db)
old_orm_device = assert_device_exists(device_id)
db.session.delete(old_orm_device)
db.session.flush()
# handled separately by parse_template_list
device_data.pop('templates')
updated_orm_device = Device(**device_data)
parse_template_list(json_payload.get('templates', []),
updated_orm_device)
auto_create_template(json_payload, updated_orm_device)
updated_orm_device.id = device_id
updated_orm_device.updated = datetime.now()
updated_orm_device.created = old_orm_device.created
db.session.add(updated_orm_device)
full_device = serialize_full_device(updated_orm_device, tenant)
if CONFIG.orion:
# Create subscription pointing to history service
# (STH, logstash based persister)
subs_handler = PersistenceHandler(service=tenant)
subs_handler.remove(old_orm_device.persistence)
# Generating 'device type' field for subscription request
type_descr = "template"
for dev_type in full_device['attrs'].keys():
type_descr += "_" + str(dev_type)
updated_orm_device.persistence = subs_handler.create(
device_id, type_descr)
ctx_broker_handler = OrionHandler(service=tenant)
ctx_broker_handler.update(
serialize_full_device(old_orm_device, tenant), type_descr)
kafka_handler = KafkaHandler()
kafka_handler.update(full_device, meta={"service": tenant})
try:
db.session.commit()
except IntegrityError as error:
handle_consistency_exception(error)
result = {
'message': 'device updated',
'device': serialize_full_device(updated_orm_device, tenant)
}
return result
def get_devices(req, sensitive_data=False):
"""
Fetches known devices, potentially limited by a given value. Ordering
might be user-configurable too.
:param req: The received HTTP request, as created by Flask.
:param sensitive_data: Informs if sensitive data like keys should be
returned
:return A JSON containing pagination information and the device list
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
"""
if req.args.get('idsOnly', 'false').lower() in ['true', '1', '']:
return DeviceHandler.list_ids(req)
tenant = init_tenant_context(req, db)
page_number, per_page = get_pagination(req)
pagination = {'page': page_number, 'per_page': per_page, 'error_out': False}
SORT_CRITERION = {
'label': Device.label,
None: Device.id
}
sortBy = SORT_CRITERION.get(req.args.get('sortBy', None), Device.id)
attr_filter = []
query = req.args.getlist('attr')
for attr in query:
parsed = re.search('^(.+){1}=(.+){1}$', attr)
attr = []
attr.append("attrs.label = '{}'".format(parsed.group(1)))
# static value must be the override, if any
attr.append("coalesce(overrides.static_value, attrs.static_value) = '{}'".format(parsed.group(2)))
attr_filter.append(and_(*attr))
label_filter = []
target_label = req.args.get('label', None)
if target_label:
label_filter.append("devices.label like '%{}%'".format(target_label))
template_filter = []
target_template = req.args.get('template', None)
if target_template:
template_filter.append("device_template.template_id = {}".format(target_template))
if template_filter or label_filter or attr_filter:
# find all devices that contain matching attributes (may contain devices that
# do not match all required attributes)
subquery = db.session.query(func.count(Device.id).label('count'), Device.id) \
.join(DeviceTemplateMap, isouter=True) \
.join(DeviceTemplate) \
.join(DeviceAttr, isouter=True) \
.join(DeviceOverride, (Device.id == DeviceOverride.did) & (DeviceAttr.id == DeviceOverride.aid), isouter=True) \
.filter(or_(*attr_filter)) \
.filter(*label_filter) \
.filter(*template_filter) \
.group_by(Device.id) \
.subquery()
# devices must match all supplied filters
if (len(attr_filter)):
LOGGER.debug(f" Filtering devices by {attr_filter}")
page = db.session.query(Device) \
.join(DeviceTemplateMap) \
.join(subquery, subquery.c.id == Device.id) \
.filter(subquery.c.count == len(attr_filter)) \
.filter(*label_filter) \
.filter(*template_filter) \
.order_by(sortBy) \
.paginate(**pagination)
else: # only filter by label
LOGGER.debug(f" Filtering devices by label")
page = db.session.query(Device) \
.join(DeviceTemplateMap) \
.join(subquery, subquery.c.id == Device.id) \
.filter(*label_filter) \
.filter(*template_filter) \
.order_by(sortBy) \
.paginate(**pagination)
else:
LOGGER.debug(f" Querying devices sorted by {sortBy}")
page = db.session.query(Device).order_by(sortBy).paginate(**pagination)
status_info = StatusMonitor.get_status(tenant)
devices = []
for d in page.items:
devices.append(serialize_full_device(d, tenant, sensitive_data, status_info))
result = {
'pagination': {
'page': page.page,
'total': page.pages,
'has_next': page.has_next,
'next_page': page.next_num
},
'devices': devices
}
return result
def update_template(req, template_id):
"""
Updates a single template.
:param req: The received HTTP request, as created by Flask.
:param template_id: The template to be updated.
:return The old version of this template (previous to the update).
:rtype JSON
:raises HTTPRequestError: If no authorization token was provided (no
tenant was informed)
:raises HTTPRequestError: If this template could not be found in
database.
"""
service = init_tenant_context(req, db)
# find old version of the template, if any
old = assert_template_exists(template_id)
# parse updated version from payload
updated, json_payload = parse_payload(req, template_schema)
LOGGER.debug(f" Current json payload: {json_payload}")
old.label = updated['label']
new = json_payload['attrs']
LOGGER.debug(f" Checking old template attributes")
for a in old.attrs:
LOGGER.debug(f" Checking attribute {a}...")
found = False
for idx, b in enumerate(new):
LOGGER.debug(f" Comparing against new attribute {b}")
if (a.label == b['label']) and (a.type == b['type']):
found = True
a.value_type = b.get('value_type', None)
a.static_value = b.get('static_value', None)
new.pop(idx)
LOGGER.debug(f" They match. Attribute data will be updated.")
break
if not found:
LOGGER.debug(f" No match for this attribute. It will be removed.")
db.session.delete(a)
for a in new:
LOGGER.debug(f" Adding new attribute {a}")
if "id" in a:
del a["id"]
db.session.add(DeviceAttr(template=old, **a))
try:
LOGGER.debug(f" Commiting new data...")
db.session.commit()
LOGGER.debug("... data committed.")
except IntegrityError as error:
LOGGER.debug(f" ConsistencyException was thrown.")
handle_consistency_exception(error)
# notify interested parties that a set of devices might have been implicitly updated
affected = db.session.query(DeviceTemplateMap) \
.filter(DeviceTemplateMap.template_id==template_id) \
.all()
affected_devices = []
for device in affected:
affected_devices.append(device.device_id)
event = {
"event": DeviceEvent.TEMPLATE,
"data": {
"affected": affected_devices,
"template": template_schema.dump(old)
},
"meta": {"service": service}
}
send_raw(event, service)
results = {
'updated': template_schema.dump(old),
'result': 'ok'
}
return results
