def get_measurement_range_bucket(self, measurement_name, start, end, group_by, site=None, database=None):
"""
Returns the measurement points for a given range grouped by a given time
@param measurement_name - name of the measurement
@param start - The date to start from (type python datetime)
@param end - The time to stop at ( type python datetime)
@param group_by - The resolution of the data
@param site - instance of Sesh_Site
@param database - The influx database to use (default settings.influx_db)
"""
db = self.db
if database:
db = database
query_string = 'SELECT mean("value") FROM {measurement_name} WHERE time > {start} AND time <= {end} GROUP BY time({time})'
# This assumes the precision of influx db is set to nanoseconds
data = {'measurement_name': measurement_name,
'start': epoch_s_to_ns(get_epoch_from_datetime(start)),
'end': epoch_s_to_ns(get_epoch_from_datetime(end)),
'time': group_by}
query = query_string.format(**data)
if site:
query += " and site_name='%s'" % site.site_name
results = list(self._influx_client.query(query, database=db).get_points())
return results
def get_measurement_range(self, measurement_name, start, end, site=None, database=None):
"""
Returns the measurement points for a given range
@param measurement_name - name of the measurement
@param start - The date to start from (type python datetime)
@param end - The time to stop at (type python datetime)
@param site - An instance of a Sesh_Site Model
@param database - Influx db to use (default settings.influx_db )
"""
db = self.db
if database:
db = database
query_string = 'SELECT * FROM {measurement_name} WHERE time > {start} and time <= {end}'
# This assumes the precision of influx db is set to nanoseconds
data = {'measurement_name': measurement_name,
'start': epoch_s_to_ns(get_epoch_from_datetime(start)),
'end': epoch_s_to_ns(get_epoch_from_datetime(end))}
query = query_string.format(**data)
if site:
query += " and site_name='%s'" % site.site_name
results = list(self._influx_client.query(query, database=db).get_points())
return results
def create_alert_instance(site, rule, data_point):
if is_mysql_rule(rule):
alert_obj = Sesh_Alert.objects.create(
site=site,
alert=rule,
date=timezone.now(),
isSilence=False,
emailSent=False,
slackSent=False,
smsSent=False,
point_model=type(data_point).__name__,
point_id=str(data_point.id))
alert_obj.save()
# Set data point to point to alert
data_point.target_alert = alert_obj
data_point.save()
elif is_influx_rule(rule):
alert_obj = Sesh_Alert.objects.create(
site=site,
alert=rule,
date=timezone.now(),
isSilence=False,
emailSent=False,
slackSent=False,
smsSent=False,
point_model='influx',
point_id=get_epoch_from_datetime(parser.parse(data_point['time'])))
alert_obj.save()
return alert_obj
def create_alert_instance(site, rule, data_point):
if is_mysql_rule(rule):
alert_obj = Sesh_Alert.objects.create(
site = site,
alert=rule,
date=timezone.now(),
isSilence=False,
emailSent=False,
slackSent=False,
smsSent=False,
point_model=type(data_point).__name__,
point_id= str(data_point.id ))
alert_obj.save()
# Set data point to point to alert
data_point.target_alert = alert_obj
data_point.save()
elif is_influx_rule(rule):
alert_obj = Sesh_Alert.objects.create(
site = site,
alert = rule,
date = timezone.now(),
isSilence=False,
emailSent=False,
slackSent=False,
smsSent=False,
point_model='influx',
point_id = get_epoch_from_datetime(parser.parse(data_point['time'])))
alert_obj.save()
return alert_obj
def download_vrm_historical_data():
"""
Helper function to initiate one time download
"""
for site in Sesh_Site.objects.filter(vrm_site_is__isnull=True):
if site.vrm_site_id:
get_historical_BoM.delay(site.pk,time_utils.get_epoch_from_datetime(site.comission_date))
run_aggregate_on_historical(site.pk)
def _download_data(request):
"""
Trigger download of vrm upon loading site data
"""
sites = _get_user_sites(request)
for site in sites:
if site.import_data:
site_id = site.pk
get_historical_BoM.delay(site_id, time_utils.get_epoch_from_datetime(site.comission_date))
def download_vrm_historical_data():
"""
Helper function to initiate one time download
"""
for site in Sesh_Site.objects.filter(vrm_site_is__isnull=True):
if site.vrm_site_id:
get_historical_BoM.delay(
site.pk,
time_utils.get_epoch_from_datetime(site.comission_date))
run_aggregate_on_historical(site.pk)
def get_measurement_range_bucket(self,
measurement_name,
start,
end,
group_by,
site=None,
database=None):
"""
Returns the measurement points for a given range grouped by a given time
@param measurement_name - name of the measurement
@param start - The date to start from (type python datetime)
@param end - The time to stop at ( type python datetime)
@param group_by - The resolution of the data
@param site - instance of Sesh_Site
@param database - The influx database to use (default settings.influx_db)
"""
db = self.db
if database:
db = database
query_string = 'SELECT mean("value") FROM {measurement_name} WHERE time > {start} AND time <= {end} GROUP BY time({time})'
# This assumes the precision of influx db is set to nanoseconds
data = {
'measurement_name': measurement_name,
'start': epoch_s_to_ns(get_epoch_from_datetime(start)),
'end': epoch_s_to_ns(get_epoch_from_datetime(end)),
'time': group_by
}
query = query_string.format(**data)
if site:
query += " and site_name='%s'" % site.site_name
results = list(
self._influx_client.query(query, database=db).get_points())
return results
def get_historical_BoM(date_range=5):
"""
Get Historical Data from VRM to backfill any days
"""
datetime_now = datetime.now()
datetime_start = datetime_now - timedelta(date_range)
datetime_now_epoch = time_utils.get_epoch_from_datetime(datetime_now)
datetime_start_epoch = time_utils.get_epoch_from_datetime(datetime_start)
sites = Sesh_Site.objects.all()
count = 0
for site in sites:
v_client = VictronAPI(site.vrm_user_id,site.vrm_password)
vh_client = VictronHistoricalAPI(site.vrm_user_id,site.vrm_password)
site_id = site.vrm_site_id
for site_id in v_client.SYSTEMS_IDS:
#site_id is a tuple
data = vh_client.get_data(site_id[0],datetime_start_epoch,datetime_now_epoch)
for row in data:
data_point = BoM_Data_Point(
site = site,
time = row['Date Time'],
soc = row['Battery State of Charge (System)'],
battery_voltage = row['Battery voltage'],
AC_input = row['Input power 1'],
AC_output = row['Output power 1'],
AC_Load_in = row['Input current phase 1'],
AC_Load_out = row['Output current phase 1'],
inverter_state = row['VE.Bus Error'],
#TODO these need to be activated
genset_state = "off",
relay_state = "off",
)
data_point.save()
count = count +1
print "saved %s BoM data points"%count
def get_measurement_range(self,
measurement_name,
start,
end,
site=None,
database=None):
"""
Returns the measurement points for a given range
@param measurement_name - name of the measurement
@param start - The date to start from (type python datetime)
@param end - The time to stop at (type python datetime)
@param site - An instance of a Sesh_Site Model
@param database - Influx db to use (default settings.influx_db )
"""
db = self.db
if database:
db = database
query_string = 'SELECT * FROM {measurement_name} WHERE time > {start} and time <= {end}'
# This assumes the precision of influx db is set to nanoseconds
data = {
'measurement_name': measurement_name,
'start': epoch_s_to_ns(get_epoch_from_datetime(start)),
'end': epoch_s_to_ns(get_epoch_from_datetime(end))
}
query = query_string.format(**data)
if site:
query += " and site_name='%s'" % site.site_name
results = list(
self._influx_client.query(query, database=db).get_points())
return results
def get_enphase_daily_stats(date=None):
"""
Get enphase daily data or get aggregate data
"""
calc_range = timedelta(minutes=15)
#create enphaseAPI
sites = Sesh_Site.objects.all()
#return 'The test task executed with argument "%s" ' % param
#get dates we want to get
datetime_now = datetime.now()
datetime_start = datetime_now - timedelta(days=1)
system_results = {}
if date:
datetime_now = date
datetime_start = datetime_now - timedelta(days=1)
#turn them into epoch seconds
datetime_start_epoch = time_utils.get_epoch_from_datetime(datetime_start)
for site in sites:
en_client = EnphaseAPI(settings.ENPHASE_KEY,site.enphase_ID)
system_id = site.enphase_site_id
print "gettig stats for %s"%system_id
system_results = en_client.get_stats(system_id,start=datetime_start_epoch)
#TODO handle exception of empty result
print len(system_results['intervals'])
for interval in system_results['intervals']:
#store the data
print interval
end_time_str = time_utils.epoch_to_datetime(interval['end_at'])
system_pv_data = PV_Production_Point(
site = site,
time = end_time_str,
wh_production = interval['enwh'],
w_production = interval['powr'],
#TODO time interval shouldn't be static this needs to be calculated based on data returned,
data_duration = calc_range
)
system_pv_data.save()
return "updated enphase data %s"%site
def prep_time_series(data,field_1_y,field_2_date,field_2_y=None):
"""
Create time series data from model data to use for graphing
"""
y_data = []
y2_data = []
x_data = []
for point in data:
y_data.append(getattr(point,field_1_y))
if field_2_y:
y_data.append(getattr(point,field_2_y))
date = getattr(point,field_2_date)
#havascript expets this in milliseconds multiply by 1000
time_epoch = int(time_utils.get_epoch_from_datetime(date)) * 1000
x_data.append(time_epoch)
if field_2_y:
return y_data,y2_data,x_data
return y_data,x_data
def get_enphase_daily_stats(date=None):
"""
Get enphase daily data or get aggregate data
"""
calc_range = timedelta(minutes=15)
#create enphaseAPI
sites = Sesh_Site.objects.all()
#return 'The test task executed with argument "%s" ' % param
#get dates we want to get default this is last 24 hours
datetime_now = datetime.now()
datetime_start = datetime_now - timedelta(days=1)
system_results = {}
if date:
datetime_now = date
datetime_start = datetime_now - timedelta(days=1)
#turn them into epoch seconds
datetime_start_epoch = time_utils.get_epoch_from_datetime(datetime_start)
for site in sites:
en_client = EnphaseAPI(settings.ENPHASE_KEY, site.enphase_ID)
system_id = site.enphase_site_id
#print "gettig stats for %s"%system_id
system_results = en_client.get_stats(system_id,
start=datetime_start_epoch)
#TODO handle exception of empty result
#print len(system_results['intervals'])
for interval in system_results['intervals']:
#store the data
#print interval
end_time_str = time_utils.epoch_to_datetime(interval['end_at'])
system_pv_data = PV_Production_Point(
site=site,
time=end_time_str,
wh_production=interval['enwh'],
w_production=interval['powr'],
#TODO time interval shouldn't be static this needs to be calculated based on data returned,
data_duration=calc_range)
system_pv_data.save()
return "updated enphase data %s" % site
def graphs(request):
"""
Returns json, containing data that is used in data analysis graphs
"""
results = []
# Getting values from Post request
time = request.GET.get('time', '') # This is the time range it has to be: 24h, 7d or 30d
choices = request.GET.getlist('choice[]')
active_id = request.GET.get('active_site_id', None)
start_time = request.GET.get('start_time', datetime.now() - timedelta(weeks=1))
end_time = request.GET.get('end_time', datetime.now())
resolution = request.GET.get('resolution', '1h')
current_site = Sesh_Site.objects.filter(id=active_id).first()
if (not current_site) or current_site.organisation != request.user.organisation:
return HttpResponseBadRequest("Invalid site id, No site was found for the given site id")
time_delta_dict = {'24h':{'hours':24},
'7d': {'days':7},
'30d':{'days':30},
}
time_bucket_dict = {'24h':'1h',
'7d':'1d',
'30d':'5d',
}
# processing post request values to be used in the influx queries
for choice in choices:
choice_dict = {}
choice_dict['measurement'] = choice
#time_delta = time_delta_dict[time]
#time_bucket= time_bucket_dict[time]
choice_dict['si_unit'] = get_measurement_unit(choice)
# Gettting the values of the given element
client = Influx()
query_results = client.get_measurement_range_bucket(choice, start_time, end_time, group_by=resolution)
#looping into values
choice_data = []
for result in query_results:
choice_data_dict = {}
result_time = parser.parse(result['time'])
result_time = get_epoch_from_datetime(result_time)
if result['mean'] is not None:
result_value = round(result['mean'], 2)
else:
result_value = 0
choice_data_dict['time'] = result_time
choice_data_dict['value'] = result_value
choice_data.append([result_time, result_value])
choice_dict['data'] = choice_data
results.append(choice_dict)
return HttpResponse(json.dumps(results))