def get_historical_BoM(site_pk, start_at):
"""
Get Historical Data from VRM to backfill any days
"""
i = Influx()
count = 0
site = Sesh_Site.objects.get(pk=site_pk)
site_id = site.vrm_site_id
if not site_id:
logger.info("skipping site %s has not vrm_site_id" % site)
return 0
vh_client = VictronHistoricalAPI(site.vrm_account.vrm_user_id,
site.vrm_account.vrm_password)
#site_id is a tuple
#print "getting data for siteid %s starting at %s"%(site.vrm_site_id,start_at)
data = vh_client.get_data(site_id, start_at)
logger.debug("Importing data for site:%s" % site)
for row in data:
try:
parsed_date = datetime.strptime(row.get('Date Time'),
'%Y-%m-%d %X')
date = time_utils.localize(parsed_date, tz=site.time_zone)
data_point = BoM_Data_Point(
site=site,
time=date,
soc=row.get('Battery State of Charge (System)', 0),
battery_voltage=row.get('Battery voltage', 0),
AC_input=row.get('Input power 1', 0),
AC_output=row.get('Output power 1', 0),
AC_Load_in=row.get('Input current phase 1', 0),
AC_Load_out=row.get('Output current phase 1', 0),
inverter_state=row.get('VE.Bus Error', ''),
pv_production=row.get('PV - AC-coupled on input L1',
0), # IF null need to put in 0
#TODO these need to be activated
genset_state=0,
relay_state=0,
)
date = row.get('Date Time')
with transaction.atomic():
data_point.save()
send_to_influx(data_point,
site,
date,
to_exclude=['time', 'inverter_state', 'id'],
client=i,
send_status=False)
count = count + 1
if count % 100:
logger.debug("imported %s points" % count)
#print "saved %s BoM data points"%count
except IntegrityError, e:
logger.warning("data point already exist %s" % e)
pass
except ValueError, e:
logger.warning("Invalid values in data point dropping %s" % (e))
pass
def get_aggregate_data(site,
measurement,
bucket_size='1h',
clause=None,
toSum=True,
operator='mean',
start='now'):
"""
Calculate aggregate values from Influx for provided measuruements
"""
i = Influx()
result = [0]
clause_opts = {
'negative': (lambda x: x[operator] < 0),
'positive': (lambda x: x[operator] > 0),
'zero': (lambda x: x[operator] == 0)
}
if clause and not clause in clause_opts.keys():
logger.error("unkown clause provided %s allowed %s" %
(clause, clause_opts.keys()))
return result
#get measurement values from influx
if not toSum:
operator = 'min'
aggr_results = i.get_measurement_bucket(measurement,
bucket_size,
'site_name',
site.site_name,
operator=operator,
start=start)
logger.debug("influx results %s " % (aggr_results))
#we have mean values by the hour now aggregate them
if aggr_results:
agr_value = []
if clause:
#if we have a cluase filter to apply
aggr_results = filter(clause_opts[clause], aggr_results)
if toSum:
to_sum_vals = map(lambda x: x[operator], aggr_results)
agr_value.append(sum(to_sum_vals))
result = agr_value
else:
result = aggr_results
logger.debug("Aggregating %s %s agr:%s" %
(measurement, aggr_results, agr_value))
else:
message = "No Values returned for aggregate. Check Influx Connection."
logger.warning(message)
return result
def get_weather_data(days=7, historical=False):
#TODO figure out way to get weather daya periodicall for forecast of 7 days
#get all sites
i = Influx()
sites = Sesh_Site.objects.all()
forecast_result = []
for site in sites:
forecast_client = ForecastAPI(settings.FORECAST_KEY,
site.position.latitude,
site.position.longitude)
if historical:
#Get weather dates for an extended historical interval
now = datetime.now()
days_to_get_from = now - timedelta(days)
days_arr = time_utils.get_days_interval_delta(
days_to_get_from, now)
forecast_result = forecast_client.get_historical_day_minimal(
days_arr)
else:
forecast_result = forecast_client.get_n_day_minimal_solar(days)
#we need to update values in the database
for day in forecast_result.keys():
site_weather = Site_Weather_Data.objects.filter(site=site,
date=day)
#if forecast already exists update it
if site_weather:
for point in site_weather:
point.condition = forecast_result[day]["stat"]
point.cloud_cover = forecast_result[day]["cloudcover"]
point.save()
send_to_influx(point,
site,
day,
to_exclude=['date'],
send_status=False)
else:
#else create a new object
w_data = Site_Weather_Data(
site=site,
date=day,
temp=0,
condition=forecast_result[day]["stat"],
cloud_cover=forecast_result[day]["cloudcover"],
sunrise=forecast_result[day]["sunrise"],
sunset=forecast_result[day]["sunset"])
w_data.save()
send_to_influx(w_data,
site,
day,
to_exclude=['date'],
send_status=False)
return "updated weather for %s" % sites
def setUp(self):
# Need this to create a Site
self.VRM = VRM_Account.objects.create(vrm_user_id='*****@*****.**',
vrm_password="******")
# Setup Influx
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
try:
self.i.create_database(self._influx_db_name)
#Generate random data points for 24h
except:
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
pass
# Setup Kapacitor
self.kap = Kapacitor()
self.template_id = 'test_template'
self.task_id = 'test_task'
self.dj_template_name = 'alert_template'
self.dbrps = [{'db': self._influx_db_name, 'rp': 'autogen'}]
self.location = Geoposition(52.5, 24.3)
dt = timezone.make_aware(timezone.datetime(2015, 12, 11, 22, 0))
self.site = Sesh_Site.objects.create(site_name=u"Test_aggregate",
comission_date=dt,
location_city=u"kigali",
location_country=u"rwanda",
vrm_account=self.VRM,
installed_kw=123.0,
position=self.location,
system_voltage=12,
number_of_panels=12,
vrm_site_id=213,
battery_bank_capacity=12321,
has_genset=True,
has_grid=True)
#self.no_points = create_test_data(self.site,
# start = self.site.comission_date,
# end = dt + timedelta( hours = 48),
# interval = 30,
# random = False)
#create test user
self.test_user = Sesh_User.objects.create_user("john doe",
"*****@*****.**",
"asdasd12345")
#assign a user to the sites
assign_perm("view_Sesh_Site", self.test_user, self.site)
def setUp(self):
self.VRM = VRM_Account.objects.create(vrm_user_id='*****@*****.**',
vrm_password="******")
# Setup Influx
self.i = Influx()
self.i.create_database('test_db')
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
self.no_points = 288
self.location = Geoposition(52.5, 24.3)
self.site = Sesh_Site.objects.create(site_name=u"Test_aggregate",
comission_date=timezone.datetime(
2015, 12, 11, 22, 0),
location_city=u"kigali",
location_country=u"rwanda",
vrm_account=self.VRM,
installed_kw=123.0,
position=self.location,
system_voltage=12,
number_of_panels=12,
vrm_site_id=213,
battery_bank_capacity=12321,
has_genset=True,
has_grid=True,
has_pv=True,
has_batteries=True)
self.test_user = Sesh_User.objects.create_user("john doe",
"*****@*****.**",
"asdasd12345")
#assign a user to the sites
try:
self.i.create_database(self._influx_db_name)
#Generate random data points for 24h
self.no_points = create_test_data(self.site, val=None)
except Exception, e:
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
print e
pass
def setUp(self):
self.VRM = VRM_Account.objects.create(vrm_user_id='*****@*****.**',
vrm_password="******")
self.location = Geoposition(52.5, 24.3)
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
try:
self.i.create_database(self._influx_db_name)
#Generate random data points for 24h
except:
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
pass
self.site = Sesh_Site.objects.create(site_name=u"Test_aggregate",
comission_date=timezone.datetime(
2015, 12, 11, 22, 0),
location_city=u"kigali",
location_country=u"rwanda",
vrm_account=self.VRM,
installed_kw=123.0,
position=self.location,
system_voltage=12,
number_of_panels=12,
vrm_site_id=213,
battery_bank_capacity=12321,
has_genset=True,
has_grid=True)
self.test_rmc_account = Sesh_RMC_Account.objects.create(
site=self.site, api_key='lcda5c15ae5cdsac464zx8f49asc16a')
timestamp = timezone.localtime(timezone.now()) - timedelta(minutes=60)
#print ""
#print timestamp
#timestamp = timezone.now()
self.i.insert_point(self.site, 'status', 1, time=timestamp)
sleep(1)
self.test_user = Sesh_User.objects.create_user(username="******",
email="*****@*****.**",
password="******")
#assign a user to the sites
assign_perm("view_Sesh_Site", self.test_user, self.site)
def setUp(self):
self.VRM = VRM_Account.objects.create(vrm_user_id='*****@*****.**',vrm_password="******")
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
try:
self.i.create_database(self._influx_db_name)
#Generate random data points for 24h
except:
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
pass
self.location = Geoposition(52.5,24.3)
self.site = Sesh_Site.objects.create(
site_name='test',
comission_date=timezone.now(),
location_city='Kigali',
location_country='Rwanda',
position=self.location,
installed_kw=25,
system_voltage=45,
number_of_panels=45,
battery_bank_capacity=450,
)
#create sesh rmc account
self.test_rmc_account = Sesh_RMC_Account.objects.create(site=self.site,
api_key='lcda5c15ae5cdsac464zx8f49asc16a')
#assign a user to the sites
generate_auto_rules(self.site.pk)
self.user = Sesh_User.objects.create_superuser(username='******',password='******',email='*****@*****.**')
self.test_user = Sesh_User.objects.create_user(username='******', password='******', email='*****@*****.**')
def setUp(self):
"""
Initializing
"""
# Setup Influx
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
try:
self.i.create_database(self._influx_db_name)
except:
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
pass
self.client = Client()
self.organisation = Sesh_Organisation.objects.create(
name='test_organisation')
self.user = Sesh_User.objects.create_user(
username='******',
password='******',
email='*****@*****.**',
organisation=self.organisation)
self.location = Geoposition(1, 4)
self.site = Sesh_Site.objects.create(site_name='test_site',
organisation=self.organisation,
comission_date=timezone.now(),
location_city='',
location_country='',
installed_kw='13',
position=self.location,
system_voltage=24,
number_of_panels=12,
battery_bank_capacity=1200)
self.status_card = Status_Card.objects.create(row1='AC_Load_in',
row2='AC_Load_out',
row3='AC_Voltage_in',
row4='AC_Voltage_out')
self.site.status_card = self.status_card
self.site.save()
assign_perm('view_Sesh_Site', self.user, self.site)
def setUp(self):
self.VRM = VRM_Account.objects.create(
vrm_user_id='*****@*****.**', vrm_password="******")
# Setup Influx
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
self.no_points = 288
try:
self.i.create_database(self._influx_db_name)
#Generate random data points for 24h
except Exception, e:
print e
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
pass
def setUp(self):
self.db_name = 'test_db'
self.Client = Influx()
try:
self.Client.create_database(self.db_name)
except:
self.Client.delete_database(self.db_name)
sleep(1)
self.Client.create_database(self.db_name)
pass
self.vrm = VRM_Account.objects.create(vrm_user_id='*****@*****.**',
vrm_password='******')
self.location = Geoposition(-1.5, 29.5)
#create Sesh_Site
self.site = Sesh_Site.objects.create(site_name='Mukuyu',
comission_date=timezone.datetime(
2014, 12, 12, 12, 12),
location_city='kigali',
location_country='Rwanda',
vrm_account=self.vrm,
installed_kw=123.0,
position=self.location,
system_voltage=24,
number_of_panels=12,
vrm_site_id=213,
battery_bank_capacity=1200,
has_genset=True,
has_grid=True)
#creating Site_Weather_Data
self.site_weather = Site_Weather_Data.objects.create(
site=self.site,
date=timezone.datetime(2016, 10, 10, 10, 10),
temp=0,
condition="none",
cloud_cover=1,
sunrise=timezone.now(),
sunset=timezone.now())
Sesh_User.objects.create_superuser(username='******',
password='******',
email='*****@*****.**')
def send_to_influx(model_data,
site,
timestamp,
to_exclude=[],
client=None,
send_status=True):
"""
Utility function to send data to influx
"""
try:
if client:
i = client
else:
i = Influx()
model_data_dict = model_to_dict(model_data)
if to_exclude:
# Remove any value we wish not to be submitted
# Generally used with datetime measurement
for val in to_exclude:
#if to_exclude in model_data_dict.keys():
model_data_dict.pop(val)
#Add our status will be used for RMC_Status
if send_status:
model_data_dict['status'] = 1.0
status = i.send_object_measurements(model_data_dict,
timestamp=timestamp,
tags={
"site_id": site.id,
"site_name": site.site_name
})
except Exception, e:
message = "Error sending to influx with exception %s in datapint %s" % (
e, model_data_dict)
handle_task_failure(message=message,
name='send_to_influx',
exception=e,
data=model_data)
def setUp(self):
"""
Initializing
"""
self.client = Client()
self.location = Geoposition(52.5, 24.3)
# Setup Influx
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
try:
self.i.create_database(self._influx_db_name)
except:
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
pass
self.site = Sesh_Site.objects.create(
site_name=u"Test site",
comission_date=timezone.datetime(2015, 12, 11, 22, 0),
location_city=u"kigali",
location_country=u"rwanda",
installed_kw=123.0,
position=self.location,
system_voltage=24,
number_of_panels=12,
vrm_site_id=213,
battery_bank_capacity=12321,
has_genset=True,
has_grid=True,
)
self.test_user = Sesh_User.objects.create_user(username="******",
email="*****@*****.**",
password="******")
def create_test_data(site, start=None, end="now", interval=5, units='minutes' , val=50, db='test_db', data={}):
"""
data = {'R1':[0,0,0,..],'R2':[0,0,123,12,...]...} will not generate date but use fixed data set
if val is not set random data will be generated if data is not existing
"""
_influx_db_name = db
i = Influx(database=_influx_db_name)
data_point_dates = generate_date_array(start=start, end=end, interval=interval, units=units)
voltage_in = 220
voltage_out = 220
soc = val
R1 = val
R2 = val
R3 = val
R4 = val
R5 = val
count = 0
print "creating %s test data points"%len(data_point_dates)
print "between %s and %s "%(data_point_dates[0],data_point_dates[len(data_point_dates)-1:])
# Simulate Grid outage
for time_val in data_point_dates:
if not val:
try:
soc = data.get('soc',[])[count]
except:
soc = get_random_int()
try:
R1 = data.get('R1',[])[count]
except:
R1 = voltage_in * get_random_binary()
try:
R2 = data.get('R2',[])[count]
except:
R2 = get_random_interval(100,500)
try:
R3 = data.get('R3',[])[count]
except:
R3 = get_random_interval(22,28)
try:
R4 = data.get('R4',[])[count]
except:
R4 = get_random_interval(100,500)
try:
R5 = data.get('R5',[])[count]
except:
R5 = get_random_interval(100,500)
dp = Data_Point.objects.create(
site=site,
soc = soc ,
battery_voltage = R3,
time=time_val,
AC_Voltage_in = R1,
AC_Voltage_out = voltage_out,
AC_input = R4,
AC_output = R5,
AC_output_absolute = R2,
AC_Load_in = R2,
AC_Load_out = R4,
pv_production = R5)
# Also send ton influx
dp_dict = model_to_dict(dp)
dp_dict.pop('time')
dp_dict.pop('inverter_state')
dp_dict.pop('id')
i.send_object_measurements(dp_dict,timestamp=time_val.isoformat(),tags={"site_name":site.site_name})
count = count + 1
# Count number of outages
return len(data_point_dates)
def setUp(self):
self._influx_db_name = 'test_db'
self.i = Influx(database=self._influx_db_name)
try:
self.i.create_database(self._influx_db_name)
#Generate random data points for 24h
except:
self.i.delete_database(self._influx_db_name)
sleep(1)
self.i.create_database(self._influx_db_name)
pass
self.VRM = VRM_Account.objects.create(vrm_user_id='*****@*****.**',vrm_password="******")
self.location = Geoposition(52.5,24.3)
self.site = Sesh_Site.objects.create(site_name=u"Test site",
comission_date=timezone.datetime(2015, 12, 11, 22, 0),
location_city=u"kigali",
location_country=u"rwanda",
vrm_account = self.VRM,
installed_kw=123.0,
position=self.location,
system_voltage=24,
number_of_panels=12,
vrm_site_id=213,
battery_bank_capacity=12321,
has_genset=True,
has_grid=True)
# Creating permissions for group
content_type = ContentType.objects.get_for_model(Sesh_Site)
self.permission = Permission.objects.create(codename='can_manage_sesh_site',
name='Can add Sesh Site',
content_type=content_type)
self.data_point = Data_Point.objects.create(site=self.site,
soc=10,
battery_voltage=20,
time=timezone.now(),
AC_input=0.0,
AC_output=15.0,
AC_Load_in=0.0,
AC_Load_out=-0.7)
#create sesh rmc account
self.test_rmc_account = Sesh_RMC_Account(site=self.site, api_key='lcda5c15ae5cdsac464zx8f49asc16a')
self.test_rmc_account.save()
#create rmc status
self.test_rmc_status = RMC_status.objects.create(site=self.site,
ip_address='127.0.0.1',
minutes_last_contact=100,
signal_strength=27,
data_sent_24h=12,
time=datetime.now())
self.test_rmc_status.save()
#create influx datapoint
self.influx_data_point = insert_point(self.site, 'battery_voltage', 10)
#create test user
self.test_user = Sesh_User.objects.create_user(username="******",
email="*****@*****.**",
password="******",
phone_number='250786688713',
on_call=True,
send_mail=True,
send_sms=True)
# Creating test group
self.test_organisation = Sesh_Organisation.objects.create(name='test_organisation',
send_slack=True,
slack_token=settings.SLACK_TEST_KEY)
# Creating test channels
self.test_channels = Slack_Channel.objects.create(organisation=self.test_organisation,
name='test_alerts_channel',
is_alert_channel=True)
#assign a user to the sites
assign_perm("view_Sesh_Site",self.test_user,self.site)
generate_auto_rules(self.site.pk)
influx_rule = Alert_Rule.objects.create(check_field='battery_voltage',
operator='lt',
site=self.site,
value=20)
alert.alert_generator()
self.new_influx_data_point = insert_point(self.site, 'battery_voltage', 24)
sleep(2) # Added sleep to wait for sometime until the point is written to the db
# Create data point that will silence alert
self.new_data_point = Data_Point.objects.create(site=self.site,
soc=50,
battery_voltage=24,
time=timezone.now(),
AC_input=0.0,
AC_output=15.0,
AC_Load_in=0.0,
AC_Load_out=-0.7)
self.new_rmc_status = RMC_status.objects.create(site=self.site,
ip_address='127.0.0.1',
minutes_last_contact=1,
signal_strength=27,
data_sent_24h=12,
time=datetime.now())
self.client = Client()
def setUp(self):
"""
Setting up the db
"""
# Adding an influx database used by the quick_status_icons functions used on the pages to render
self.influx_db_name = 'test_db'
self.i = Influx(database=self.influx_db_name)
try:
self.i.create_database(self.influx_db_name)
except:
# Database already exist
self.i.delete_database(self.influx_db_name)
sleep(1)
self.i.create_database(self.influx_db_name)
pass
self.client = Client()
self.organisation = Sesh_Organisation.objects.create(
name="test_org", slack_token="secret")
self.site = Sesh_Site.objects.create(
site_name='test_site1',
organisation=self.organisation,
comission_date=timezone.now(),
location_city='Kigali',
location_country='Rwanda',
position=Geoposition(12, 1),
installed_kw=25,
system_voltage=45,
number_of_panels=45,
battery_bank_capacity=450,
)
self.site2 = Sesh_Site.objects.create(
site_name='test_site2',
organisation=self.organisation,
comission_date=timezone.now(),
location_city='Kigali',
location_country='Rwanda',
position=Geoposition(12, 1),
installed_kw=25,
system_voltage=45,
number_of_panels=45,
battery_bank_capacity=450,
)
self.alert_rule = Alert_Rule.objects.create(
site=self.site,
check_field='battery_voltage',
operator='lt',
value='0',
)
self.sesh_user = Sesh_User.objects.create_superuser(
username='******',
email='*****@*****.**',
password='******',
on_call=False,
send_sms=False,
send_mail=False)
