def send_to_influx(model_data, site, timestamp, to_exclude=[],client=None, send_status=True): """ Utility function to send data to influx """ try: if client: i = client else: i = Influx() model_data_dict = model_to_dict(model_data) if to_exclude: # Remove any value we wish not to be submitted # Generally used with datetime measurement for val in to_exclude: #if to_exclude in model_data_dict.keys(): model_data_dict.pop(val) #Add our status will be used for RMC_Status if send_status: model_data_dict['status'] = 1.0 status = i.send_object_measurements(model_data_dict, timestamp=timestamp, tags={"site_id":site.id, "site_name":site.site_name}) except Exception,e: message = "Error sending to influx with exception %s in datapint %s"%(e,model_data_dict) handle_task_failure(message= message, name='send_to_influx' ,exception=e,data=model_data)
def create_test_data(site, start=None, end="now", interval=5, units='minutes' ,random=True, val=50, db='test_db'): #TODO test weekly and monthly reports _influx_db_name = db i = Influx(database=_influx_db_name) data_point_dates = generate_date_array(start=start, end=end, interval=interval, units=units) voltage_in = 220 voltage_out = 220 soc = val R1 = val R2 = val R3 = val R4 = val R5 = val print "creating %s test data points"%len(data_point_dates) print "between %s and %s "%(data_point_dates[0],data_point_dates[len(data_point_dates)-1:]) # Simulate Grid outage for time_val in data_point_dates: if random: soc = get_random_int() R1 = voltage_in * get_random_binary() R2 = get_random_interval(100,500) R3 = get_random_interval(22,28) R4 = get_random_interval(100,500) R5 = get_random_interval(100,500) dp = Data_Point.objects.create( site=site, soc = soc , battery_voltage = R3, time=time_val, AC_Voltage_in = R1, AC_Voltage_out = voltage_out, AC_input = R4, AC_output = R5, AC_output_absolute = R2, AC_Load_in = R2, AC_Load_out = R4, pv_production = R5) # Also send ton influx dp_dict = model_to_dict(dp) dp_dict.pop('time') dp_dict.pop('inverter_state') dp_dict.pop('id') i.send_object_measurements(dp_dict,timestamp=time_val.isoformat(),tags={"site_name":site.site_name}) return len(data_point_dates)
def send_to_influx(model_data, site, timestamp, to_exclude=[], client=None, send_status=True): """ Utility function to send data to influx """ try: if client: i = client else: i = Influx() model_data_dict = model_to_dict(model_data) if to_exclude: # Remove any value we wish not to be submitted # Generally used with datetime measurement for val in to_exclude: #if to_exclude in model_data_dict.keys(): model_data_dict.pop(val) #Add our status will be used for RMC_Status if send_status: model_data_dict['status'] = 1.0 status = i.send_object_measurements(model_data_dict, timestamp=timestamp, tags={ "site_id": site.id, "site_name": site.site_name }) except Exception, e: message = "Error sending to influx with exception %s in datapint %s" % ( e, model_data_dict) handle_task_failure(message=message, name='send_to_influx', exception=e, data=model_data)
def create_test_data(site, start=None, end="now", interval=5, units='minutes' , val=50, db='test_db', data={}): """ data = {'R1':[0,0,0,..],'R2':[0,0,123,12,...]...} will not generate date but use fixed data set if val is not set random data will be generated if data is not existing """ _influx_db_name = db i = Influx(database=_influx_db_name) data_point_dates = generate_date_array(start=start, end=end, interval=interval, units=units) voltage_in = 220 voltage_out = 220 soc = val R1 = val R2 = val R3 = val R4 = val R5 = val count = 0 print "creating %s test data points"%len(data_point_dates) print "between %s and %s "%(data_point_dates[0],data_point_dates[len(data_point_dates)-1:]) # Simulate Grid outage for time_val in data_point_dates: if not val: try: soc = data.get('soc',[])[count] except: soc = get_random_int() try: R1 = data.get('R1',[])[count] except: R1 = voltage_in * get_random_binary() try: R2 = data.get('R2',[])[count] except: R2 = get_random_interval(100,500) try: R3 = data.get('R3',[])[count] except: R3 = get_random_interval(22,28) try: R4 = data.get('R4',[])[count] except: R4 = get_random_interval(100,500) try: R5 = data.get('R5',[])[count] except: R5 = get_random_interval(100,500) dp = Data_Point.objects.create( site=site, soc = soc , battery_voltage = R3, time=time_val, AC_Voltage_in = R1, AC_Voltage_out = voltage_out, AC_input = R4, AC_output = R5, AC_output_absolute = R2, AC_Load_in = R2, AC_Load_out = R4, pv_production = R5) # Also send ton influx dp_dict = model_to_dict(dp) dp_dict.pop('time') dp_dict.pop('inverter_state') dp_dict.pop('id') i.send_object_measurements(dp_dict,timestamp=time_val.isoformat(),tags={"site_name":site.site_name}) count = count + 1 # Count number of outages return len(data_point_dates)
class KapacitorTestCase(TestCase): def setUp(self): # Need this to create a Site self.VRM = VRM_Account.objects.create(vrm_user_id='*****@*****.**', vrm_password="******") # Setup Influx self._influx_db_name = 'test_db' self.i = Influx(database=self._influx_db_name) try: self.i.create_database(self._influx_db_name) #Generate random data points for 24h except: self.i.delete_database(self._influx_db_name) sleep(1) self.i.create_database(self._influx_db_name) pass # Setup Kapacitor self.kap = Kapacitor() self.template_id = 'test_template' self.task_id = 'test_task' self.dj_template_name = 'alert_template' self.dbrps = [{'db': self._influx_db_name, 'rp': 'autogen'}] self.location = Geoposition(52.5, 24.3) dt = timezone.make_aware(timezone.datetime(2015, 12, 11, 22, 0)) self.site = Sesh_Site.objects.create(site_name=u"Test_aggregate", comission_date=dt, location_city=u"kigali", location_country=u"rwanda", vrm_account=self.VRM, installed_kw=123.0, position=self.location, system_voltage=12, number_of_panels=12, vrm_site_id=213, battery_bank_capacity=12321, has_genset=True, has_grid=True) #self.no_points = create_test_data(self.site, # start = self.site.comission_date, # end = dt + timedelta( hours = 48), # interval = 30, # random = False) #create test user self.test_user = Sesh_User.objects.create_user("john doe", "*****@*****.**", "asdasd12345") #assign a user to the sites assign_perm("view_Sesh_Site", self.test_user, self.site) def tearDown(self): self.i.delete_database(self._influx_db_name) self.kap.delete_template(self.template_id) self.kap.delete_task(self.task_id) pass @override_settings(INFLUX_DB='test_db') def test_template_creation(self): """ Test creating template in kapacitor """ temp_script = """ // Which measurement to consume var measurement string // Optional where filter var where_filter = lambda: TRUE // Optional list of group by dimensions var groups = [*] // Which field to process var field string // Warning criteria, has access to 'mean' field var warn lambda // Critical criteria, has access to 'mean' field var crit lambda // How much data to window var window = 5m // The slack channel for alerts var slack_channel = '#alerts' stream |from() .measurement(measurement) .where(where_filter) .groupBy(groups) |window() .period(window) .every(window) |mean(field) |alert() .warn(warn) .crit(crit) .slack() .channel(slack_channel) """ temp_id = self.template_id temp_type = 'stream' # Create template temp = self.kap.create_template(temp_id, temp_type, temp_script) self.assertTrue(temp.has_key('vars')) # Verify template creation temp_res = self.kap.get_template(temp_id) self.assertTrue(temp_res.has_key('vars')) # List template temp_res = self.kap.list_templates() self.assertTrue(temp_res.has_key('templates')) # Update Template temp_script = """ // Which measurement to consume var measurement = 'cpu' // Optional where filter var where_filter = lambda: TRUE // Optional list of group by dimensions var groups = [*] // Which field to process var field string // Warning criteria, has access to 'mean' field var warn lambda // Critical criteria, has access to 'mean' field var crit lambda // How much data to window var window = 5m // The slack channel for alerts var slack_channel = '#alerts' stream |from() .measurement(measurement) .where(where_filter) .groupBy(groups) |window() .period(window) .every(window) |mean(field) |alert() .warn(warn) .crit(crit) .slack() .channel(slack_channel) """ temp_res = self.kap.update_template(temp_id, temp_script) # Delete template self.kap.delete_template(self.template_id) def test_task_creation(self): """ Create a task and check if it actually causes an alert to trigger """ temp_script = """ stream |from() .measurement('cpu') |alert() .crit(lambda: "value" < 70) .log('/tmp/alerts.log') """ temp_id = self.template_id task_id = self.task_id # Create task temp = self.kap.create_task(task_id, dbrps=self.dbrps, script=temp_script, task_type='stream') self.assertEqual(temp['status'], 'enabled') sleep(20) for i in reversed(range(0, 5)): sleep(1) dp_dict = {'cpu': i} self.i.send_object_measurements(dp_dict, tags={"site_name": "test_site"}, database=self._influx_db_name) temp = self.kap.get_task(task_id) self.assertGreater( temp['stats']['node-stats']['alert2']['alerts_triggered'], 0) def test_task_dj_template(self): """ test task creation with django templates """ template = get_template('seshdash/kapacitor_tasks/%s.tick' % self.dj_template_name) alert_id = self.task_id alert_info = { 'field': 'cpu', 'where_filter_lambda': 'lambda: TRUE', 'error_lambda': 'lambda: \"value\" < 30', 'time_window': '5m', 'slack_channel': '#alerts' } rendered_alert = template.render(alert_info) result = self.kap.create_task(alert_id, dbrps=self.dbrps, script=rendered_alert) self.assertEquals(result['status'], 'enabled')
class KapacitorTestCase(TestCase): def setUp(self): # Need this to create a Site self.VRM = VRM_Account.objects.create(vrm_user_id='*****@*****.**',vrm_password="******") # Setup Influx self._influx_db_name = 'test_db' self.i = Influx(database=self._influx_db_name) try: self.i.create_database(self._influx_db_name) #Generate random data points for 24h except: self.i.delete_database(self._influx_db_name) sleep(1) self.i.create_database(self._influx_db_name) pass # Setup Kapacitor self.kap = Kapacitor() self.template_id = 'test_template' self.task_id = 'test_task' self.dj_template_name = 'alert_template' self.dbrps = [{'db': self._influx_db_name, 'rp':'autogen' }] self.location = Geoposition(52.5,24.3) dt = timezone.make_aware(timezone.datetime(2015, 12, 11, 22, 0)) self.site = Sesh_Site.objects.create(site_name=u"Test_aggregate", comission_date = dt, location_city = u"kigali", location_country=u"rwanda", vrm_account = self.VRM, installed_kw=123.0, position=self.location, system_voltage=12, number_of_panels=12, vrm_site_id=213, battery_bank_capacity=12321, has_genset=True, has_grid=True) #self.no_points = create_test_data(self.site, # start = self.site.comission_date, # end = dt + timedelta( hours = 48), # interval = 30, # random = False) #create test user self.test_user = Sesh_User.objects.create_user("john doe","*****@*****.**","asdasd12345") #assign a user to the sites assign_perm("view_Sesh_Site",self.test_user,self.site) def tearDown(self): self.i.delete_database(self._influx_db_name) self.kap.delete_template(self.template_id) self.kap.delete_task(self.task_id) pass @override_settings(INFLUX_DB='test_db') def test_template_creation(self): """ Test creating template in kapacitor """ temp_script = """ // Which measurement to consume var measurement string // Optional where filter var where_filter = lambda: TRUE // Optional list of group by dimensions var groups = [*] // Which field to process var field string // Warning criteria, has access to 'mean' field var warn lambda // Critical criteria, has access to 'mean' field var crit lambda // How much data to window var window = 5m // The slack channel for alerts var slack_channel = '#alerts' stream |from() .measurement(measurement) .where(where_filter) .groupBy(groups) |window() .period(window) .every(window) |mean(field) |alert() .warn(warn) .crit(crit) .slack() .channel(slack_channel) """ temp_id = self.template_id temp_type = 'stream' # Create template temp = self.kap.create_template(temp_id, temp_type, temp_script) self.assertTrue(temp.has_key('vars')) # Verify template creation temp_res = self.kap.get_template(temp_id) self.assertTrue(temp_res.has_key('vars')) # List template temp_res = self.kap.list_templates() self.assertTrue(temp_res.has_key('templates')) # Update Template temp_script = """ // Which measurement to consume var measurement = 'cpu' // Optional where filter var where_filter = lambda: TRUE // Optional list of group by dimensions var groups = [*] // Which field to process var field string // Warning criteria, has access to 'mean' field var warn lambda // Critical criteria, has access to 'mean' field var crit lambda // How much data to window var window = 5m // The slack channel for alerts var slack_channel = '#alerts' stream |from() .measurement(measurement) .where(where_filter) .groupBy(groups) |window() .period(window) .every(window) |mean(field) |alert() .warn(warn) .crit(crit) .slack() .channel(slack_channel) """ temp_res = self.kap.update_template(temp_id, temp_script) # Delete template self.kap.delete_template(self.template_id) def test_task_creation(self): """ Create a task and check if it actually causes an alert to trigger """ temp_script = """ stream |from() .measurement('cpu') |alert() .crit(lambda: "value" < 70) .log('/tmp/alerts.log') """ temp_id = self.template_id task_id = self.task_id # Create task temp = self.kap.create_task(task_id, dbrps=self.dbrps, script=temp_script, task_type='stream') self.assertEqual(temp['status'],'enabled') sleep(20) for i in reversed(range(0,5)): sleep(1) dp_dict = {'cpu': i} self.i.send_object_measurements(dp_dict, tags={"site_name":"test_site"}, database=self._influx_db_name) temp = self.kap.get_task(task_id) self.assertGreater(temp['stats']['node-stats']['alert2']['alerts_triggered'], 0) def test_task_dj_template(self): """ test task creation with django templates """ template = get_template('seshdash/kapacitor_tasks/%s.tick'%self.dj_template_name) alert_id = self.task_id alert_info ={ 'field': 'cpu', 'where_filter_lambda' : 'lambda: TRUE', 'error_lambda' : 'lambda: \"value\" < 30', 'time_window' : '5m', 'slack_channel' : '#alerts' } rendered_alert = template.render(alert_info) result = self.kap.create_task(alert_id, dbrps= self.dbrps, script=rendered_alert) self.assertEquals(result['status'], 'enabled')