def import_data(base_url, range_from):
client = HSClient(base_url)
# sites
content = client.nav(nav_id=None)
header, data = client.parse_grid(content)
id_index = header.index('id')
nav_id_index = header.index('navId')
try:
site_tag_id = header.index('site')
except ValueError:
site_tag_id = None
sites_nav_ids = []
new_sites_ids = []
new_sites = 0
if site_tag_id is not None:
for row in data:
new_id = cleanup_id(row[id_index])
site_tag = row[site_tag_id]
if 'M' == site_tag:
sites_nav_ids.append(row[nav_id_index])
entity = Entity.objects.filter(entity_id=new_id,
m_tags__contains=['site'
]).first()
if not entity:
# create new one
kv_tags, m_tags = prepare_tags(header, row)
new_entity = Entity(entity_id=new_id,
kv_tags=kv_tags,
m_tags=m_tags)
new_entity.save()
new_sites = new_sites + 1
new_sites_ids.append(new_id)
else:
print("There is no 'site' tag in ", row)
else:
print("Cannot find 'site' in the response header")
if new_sites > 0:
print("New sites ids:", new_sites_ids)
print("Processed {0} sites, added {1} new.".format(len(data), new_sites))
# print("sites_nav_ids:", sites_nav_ids)
# equipments
total_equips = 0
new_equips = 0
new_equips_ids = []
equips_nav_ids = []
if len(sites_nav_ids) > 0:
for nav_id in sites_nav_ids:
content = client.nav(nav_id=nav_id)
header, data = client.parse_grid(content)
total_equips = total_equips + len(data)
print("{1}: Got {0} equipments".format(len(data), nav_id))
id_index = header.index('id')
nav_id_index = header.index('navId')
try:
equip_tag_id = header.index('equip')
except ValueError:
equip_tag_id = None
if equip_tag_id is not None:
for row in data:
new_id = cleanup_id(row[id_index])
equip_tag = row[equip_tag_id]
if 'M' == equip_tag:
equips_nav_ids.append(row[nav_id_index])
entity = Entity.objects.filter(entity_id=new_id,
m_tags__contains=[
'equip'
]).first()
if not entity:
# create new one
kv_tags, m_tags = prepare_tags(header, row)
new_entity = Entity(entity_id=new_id,
kv_tags=kv_tags,
m_tags=m_tags)
new_entity.save()
new_equips = new_equips + 1
new_equips_ids.append(new_id)
else:
print("There is no 'equip' tag in ", row)
else:
print("Cannot find 'equip' in the response header")
else:
print("Empty sites list")
if new_equips > 0:
print("New equipments ids:", new_equips_ids)
print("Processed {0} equipments, added {1} new.".format(
total_equips, new_equips))
# print("equip_nav_ids:", equips_nav_ids)
# points
total_points = 0
new_points = 0
new_points_ids = []
all_points_ids = []
points_nav_ids = []
if len(equips_nav_ids) > 0:
for nav_id in equips_nav_ids:
content = client.nav(nav_id=nav_id)
header, data = client.parse_grid(content)
total_points = total_points + len(data)
print("{1}: Got {0} points".format(len(data), nav_id))
try:
id_index = header.index('id')
except ValueError:
print("Cannot get id index")
continue
nav_id_index = header.index('navId')
try:
point_tag_id = header.index('point')
except ValueError:
point_tag_id = None
if point_tag_id is not None:
for row in data:
new_id = cleanup_id(row[id_index])
point_tag = row[point_tag_id]
if 'M' == point_tag:
points_nav_ids.append(row[nav_id_index])
all_points_ids.append(row[id_index])
entity = Entity.objects.filter(entity_id=new_id,
m_tags__contains=[
'point'
]).first()
if not entity:
# create new one
kv_tags, m_tags = prepare_tags(header, row)
new_entity = Entity(entity_id=new_id,
topic=row[id_index],
kv_tags=kv_tags,
m_tags=m_tags)
new_entity.save()
new_points = new_points + 1
new_points_ids.append(new_id)
else:
print("There is no 'point' tag in ", row)
else:
print("Cannot find 'point' in the response header")
else:
print("Empty equipments list")
if new_points > 0:
print("New points ids:", new_points_ids)
print("Processed {0} points, added {1} new.".format(
total_points, new_points))
# print("all_points_ids:", all_points_ids)
# print("points_nav_ids:", points_nav_ids)
# points data
topics_counter = 0
topics_list = []
crate_connection = get_crate_connection()
crate_cursor = crate_connection.cursor()
if len(all_points_ids) > 0:
for point_id in all_points_ids:
topic_data_counter = 0
sql = """INSERT INTO {0} (topic)
VALUES (%s)""".format("volttron.topic")
try:
crate_cursor.execute(
"""INSERT INTO volttron.topic (topic) VALUES (?)""",
(point_id, ))
except ProgrammingError:
# just make sure the topic exists
pass
crate_cursor.execute(
"""SELECT ts, string_value FROM "volttron"."data"
WHERE topic = ? ORDER BY ts DESC LIMIT 1;""", (point_id, ))
result = crate_cursor.fetchone()
his_range = None
if result:
ts = result[0]
t = datetime.utcfromtimestamp(ts // 1000).replace(
microsecond=ts % 1000 * 1000).replace(tzinfo=timezone.utc)
his_range = t.strftime(
'%Y-%m-%d') + ',' + datetime.utcnow().strftime('%Y-%m-%d')
else:
if range_from:
if range_from != "none":
his_range = range_from
else:
his_range = 'today'
print("Processing :", point_id, ", range: ", his_range)
content = client.his_read(id=point_id, range=his_range)
header, data = client.parse_grid(content)
if header and 'ts' in header and 'val' in header:
ts_index = header.index('ts')
val_index = header.index('val')
for item in data:
ts_str_arr = item[ts_index].split(" ")
val = item[val_index]
sql = """INSERT INTO volttron.data (double_value, source, string_value, topic, ts)
VALUES (?, ?, ?, ?, ?)"""
try:
crate_cursor.execute(sql, (
val,
'scrape',
val,
point_id,
ts_str_arr[0],
))
topic_data_counter = topic_data_counter + 1
except ProgrammingError:
# that means it already exists
pass
topics_list.append({point_id: topic_data_counter})
print("Point #{1}: Added {0} data rows.".format(
topic_data_counter, topics_counter))
topics_counter = topics_counter + 1
crate_cursor.close()
crate_connection.close()
print("Processed {0} topics.".format(topics_counter))
def test_get_current_value(self):
point = Entity()
point.entity_id = self.entity_id
point.topic = 'empty'
point.kind = 'Number'
point.unit = '°C'
current_value = utils.get_current_value(point)
self.assertIsNone(current_value)
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic
point.kind = 'Number'
point.unit = '°C'
current_value = utils.get_current_value(point)
self.assertTrue('<b>34.7</b> °C' in current_value)
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic1
point.kind = 'Bool'
current_value = utils.get_current_value(point)
self.assertTrue('<b>True</b>' in current_value)
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic
point.kind = 'Number'
point.unit = ''
current_value = utils.get_current_value(point)
self.assertTrue('<b>34.74</b>' in current_value)
def test_add_current_values(self):
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic
point.kind = 'Number'
point.unit = '°C'
point.description = 'Point description'
point.site_id = '@D'
point.equipment_id = '@D-AHU1'
point1 = Entity()
point1.entity_id = self.entity_id
point1.topic = self.topic1
point1.kind = 'Bool'
point1.unit = ''
point1.description = 'Point1 description'
point1.site_id = '@D'
point1.equipment_id = '@D-AHU2'
points = []
points.append(point)
points.append(point1)
data = utils.add_current_values(points)
self.assertIsNotNone(data)
self.assertTrue('<b>34.7</b> °C' in data[0].current_value)
self.assertTrue('<b>True</b>' in data[1].current_value)
def test_charts_for_points(self):
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic
point.kind = 'Number'
point.unit = '°C'
point.description = 'Point description'
point.site_id = '@D'
point.equipment_id = '@D-AHU1'
point1 = Entity()
point1.entity_id = self.entity_id
point1.topic = self.topic1
point1.kind = 'Bool'
point1.unit = ''
point1.description = 'Point1 description'
point1.site_id = '@D'
point1.equipment_id = '@D-AHU2'
points = []
points.append(point)
points.append(point1)
charts = utils.charts_for_points(points)
self.assertIsNotNone(charts)
self.assertEqual(len(charts), 2)
def test_get_point_values(self):
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic
point.kind = 'Number'
point.unit = '°C'
point_values = utils.get_point_values(point, '', 'avg', None)
self.assertIsNotNone(point_values)
self.assertEqual(len(point_values), 1)
point_value = point_values[0]
self.assertIsNotNone(point_value)
self.assertIn(34.74, point_value)
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic1
point.kind = 'Bool'
point_values = utils.get_point_values(point)
self.assertIsNotNone(point_values)
self.assertEqual(len(point_values), 1)
point_value = point_values[0]
self.assertIsNotNone(point_value)
self.assertIn(1, point_value)
point = Entity()
point.entity_id = self.entity_id
point.topic = self.topic
point.kind = 'String'
point.unit = ''
point_values = utils.get_point_values(point)
self.assertIsNotNone(point_values)
self.assertEqual(len(point_values), 1)
point_value = point_values[0]
self.assertIsNotNone(point_value)
self.assertIn('34.74', point_value)
def post(self, request, site_entity_id):
# get site id from request
try:
site = SiteView.objects.get(entity_id=site_entity_id)
except SiteView.DoesNotExist:
err = 'Site does not exist: {}'.format(site_entity_id)
logger.error(err)
return Response({'error': err},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# check json file attached or not
file = request.FILES.get('data', None)
if file:
if not file.name.endswith('.json'):
logger.error('File should be json')
return Response({'error': 'File should be json'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
try:
data = json.loads(file.read())
except json.JSONDecodeError:
logger.error('Invalid json format')
return Response(
{'error': traceback.format_exc()},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
# get topic data from request
data = request.data
logger.info('Read Json successfully! Start parse...')
for d_address, d_value in data.items():
device_name = None
device_id = None
device = d_value.get('device', {})
if device:
for key, value in device.items():
device_id = key
if value and value.get('objectName', None):
device_name = value.get('objectName', None)
if device_name:
device_name = device_name.strip()
# get device_name and device_id
if not device_name:
device_name = d_value.get('device_name', None)
if not device_id:
device_id = str(d_value.get('device_id', None))
if not device_id:
logger.error(
'Device id not defined. Device Address is {}'.format(
d_address))
return Response(
{'error': 'Device id not defined. Invalid json structure'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
device_description = device.get(device_id,
{}).get('description', None)
if device_description:
device_description = device_description.strip()
object_name = device.get(device_id, {}).get('objectName', None)
if object_name:
object_name = object_name.strip()
if not device_description and object_name:
device_description = object_name
for o_type, o_value in d_value.items():
if o_type not in ['device', 'device_id',
'device_name']: # detect meta data
for o_id, tags in o_value.items():
topic_name = '{site_id}/{device_id}/{o_type}/{o_id}'.format(
site_id=site_entity_id,
device_id=device_id,
o_type=o_type,
o_id=o_id)
entity_id = topic_name.replace('/', '-')
logger.info(
'Start: Create topic - {}'.format(topic_name))
# check topic exists
Topic.ensure_topic_exists(topic_name)
# get all available tags
kv_tags = dict()
kv_tags['id'] = topic_name
kv_tags['siteRef'] = site.object_id
kv_tags['bacnet_volttron_point_name'] = topic_name
kv_tags['bacnet_device_address'] = d_address
kv_tags['bacnet_device_id'] = device_id
kv_tags['bacnet_device_name'] = device_name
kv_tags[
'bacnet_device_description'] = device_description
kv_tags['bacnet_bacnet_object_type'] = o_type
kv_tags['bacnet_object_index'] = o_id
kv_tags['bacnet_prefix'] = device_id
kv_tags[
'interval'] = self.interval # default 300 for now
kv_tags['dis'] = topic_name
for t_id, t_v in tags.items():
if t_v:
t_v = t_v.replace(u"\u0000", "")
tag_name = 'bacnet_{}'.format(t_id)
kv_tags[tag_name] = t_v.encode(
'ascii', 'ignore').decode('utf-8')
self.ensure_tag_exists(tag_name)
try:
topic = Entity.objects.get(topic=topic_name)
for tag, value in topic.kv_tags.items():
topic.add_tag(tag, value, commit=False)
logger.info(
'End: Update topic - {}'.format(topic_name))
except Entity.DoesNotExist:
topic = Entity(entity_id=entity_id,
topic=topic_name,
m_tags=['point'],
kv_tags=kv_tags)
logger.info(
'End: Create topic - {}'.format(topic_name))
topic.save()
return Response({'success': 'success'})