def get(self, dt):
assert isinstance(dt, datetime)
if dt.dst() is None:
dt = TIMEZONE.localize(dt)
dt = TIMEZONE.normalize(dt)
pos = bisect.bisect_left(self.coefs, Coefficent(dt, {}))
self._check_pos(pos)
return self.coefs[pos]
def get(self, dt):
assert isinstance(dt, datetime)
if dt.dst() is None:
dt = TIMEZONE.localize(dt)
dt = TIMEZONE.normalize(dt)
pos = bisect.bisect_left(self.coefs, Coefficent(dt, {}))
self._check_pos(pos)
return self.coefs[pos]
def get_station(dt):
assert isinstance(dt, datetime)
if not dt.tzinfo:
dt = TIMEZONE.localize(dt)
if TIMEZONE.normalize(dt).dst():
return 'summer'
else:
return 'winter'
def get(cls, year, month):
try:
import ssl
try:
_create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
pass
else:
ssl._create_default_https_context = _create_unverified_https_context
except ImportError:
pass
try:
cls.down_lock.acquire()
import csv
import httplib
key = '%(year)s%(month)02i' % locals()
conn = None
if key in cls._CACHE:
logger.debug('Using CACHE for REEProfile {0}'.format(key))
return cls._CACHE[key]
perff_file = 'PERFF_%(key)s.gz' % locals()
conn = httplib.HTTPSConnection(cls.HOST)
conn.request('GET', '%s/%s' % (cls.PATH, perff_file))
logger.debug('Downloading REEProfile from {0}/{1}'.format(
cls.PATH, perff_file))
r = conn.getresponse()
if r.msg.type == 'application/x-gzip':
import gzip
c = StringIO(r.read())
m = StringIO(gzip.GzipFile(fileobj=c).read())
c.close()
reader = csv.reader(m, delimiter=';')
header = True
cofs = []
coeffs_list = get_tariff_coeffs_list(year, month)
for vals in reader:
if header:
header = False
continue
if int(vals[3]) == 1:
n_hour = 1
dt = datetime(int(vals[0]), int(vals[1]), int(vals[2]))
day = TIMEZONE.localize(dt, is_dst=bool(not int(vals[4])))
day += timedelta(hours=n_hour)
n_hour += 1
cofs.append(
Coefficent(
TIMEZONE.normalize(day),
dict((k, float(vals[i]))
for i, k in enumerate(coeffs_list, 5))))
cls._CACHE[key] = cofs
return cofs
else:
raise Exception('Profiles from REE not found')
finally:
if conn is not None:
conn.close()
cls.down_lock.release()
def day_profile(self, dini, dfi, real_consumption_hourly):
assert type(
real_consumption_hourly
) == list, "Real hourly consumption for this day must be a list"
date_start = TIMEZONE.localize(dini)
date_end = TIMEZONE.localize(dini)
estimated = Profile(date_start, date_end, []).estimate(t, {'P1': 5})
def get_period_by_timestamp(self, timestamp):
"""
Gets the number of energy period
:param timestamp: datetime in format 'YYYY-MM-DD HH'
:return: period number
"""
day, hours = timestamp.split(' ')
dt_tz = TIMEZONE.normalize(TIMEZONE.localize(datetime.strptime(day, '%Y-%m-%d')) + timedelta(hours=int(hours)))
# "get_period_by_date" expects a local timestamp without timezone
# decreases 1 minute because is final datetime open interval (not included)
dt = dt_tz.replace(tzinfo=None) - timedelta(minutes=1)
return self.get_period_by_date(dt).code
def get_range(self, start, end):
assert isinstance(start, date)
assert isinstance(end, date)
start = TIMEZONE.localize(datetime(start.year, start.month, start.day,
1),
is_dst=True)
# Sum one day to get the hour 00:00 of the next day
end += timedelta(days=1)
end = TIMEZONE.localize(datetime(end.year, end.month, end.day),
is_dst=True) + timedelta(seconds=1)
pos = bisect.bisect_left(self.coefs, Coefficent(start, {}))
self._check_pos(pos)
end_pos = bisect.bisect_right(self.coefs, Coefficent(end, {}))
return self.coefs[pos:end_pos]
def get(cls, year, month):
try:
cls.down_lock.acquire()
import csv
import httplib
key = '%(year)s%(month)02i' % locals()
conn = None
if key in cls._CACHE:
logger.debug('Using CACHE for REEProfile {0}'.format(key))
return cls._CACHE[key]
perff_file = 'PERFF_%(key)s.gz' % locals()
conn = httplib.HTTPConnection(cls.HOST)
conn.request('GET', '%s/%s' % (cls.PATH, perff_file))
logger.debug('Downloading REEProfile from {0}/{1}'.format(
cls.PATH, perff_file
))
r = conn.getresponse()
if r.msg.type == 'application/x-gzip':
import gzip
c = StringIO(r.read())
m = StringIO(gzip.GzipFile(fileobj=c).read())
c.close()
reader = csv.reader(m, delimiter=';')
header = True
cofs = []
for vals in reader:
if header:
header = False
continue
if int(vals[3]) == 1:
n_hour = 1
dt = datetime(
int(vals[0]), int(vals[1]), int(vals[2])
)
day = TIMEZONE.localize(dt, is_dst=bool(not int(vals[4])))
day += timedelta(hours=n_hour)
n_hour += 1
cofs.append(Coefficent(
TIMEZONE.normalize(day), dict(
(k, float(vals[i])) for i, k in enumerate('ABCD', 5)
))
)
cls._CACHE[key] = cofs
return cofs
else:
raise Exception('Profiles from REE not found')
finally:
if conn is not None:
conn.close()
cls.down_lock.release()
def get_range(self, start, end):
assert isinstance(start, date)
assert isinstance(end, date)
start = TIMEZONE.localize(datetime(
start.year, start.month, start.day, 1), is_dst=True
)
# Sum one day to get the hour 00:00 of the next day
end += timedelta(days=1)
end = TIMEZONE.localize(datetime(
end.year, end.month, end.day), is_dst=True
) + timedelta(seconds=1)
pos = bisect.bisect_left(self.coefs, Coefficent(start, {}))
self._check_pos(pos)
end_pos = bisect.bisect_right(self.coefs, Coefficent(end, {}))
return self.coefs[pos:end_pos]
def process_consumptions(self):
"""
Process all FacturaATR of the imported F1
For each period create the related profile and estimate
Create the related Consumption objecte and save it (if more significant) to DB
"""
invoices = self.invoices
# Get all FacturaATR invoices
for factura_atr in invoices['FacturaATR']:
#self.print_invoice_summary(factura_atr)
periods, consumption_total = factura_atr.get_info_activa()
tariff_code = factura_atr.codi_tarifa
tariff_name = defs.INFO_TARIFA[tariff_code]['name']
tariff = get_tariff_by_code(tariff_name)()
# Profile and estimate Invoice range for each period
for period in periods:
assert isinstance(period.data_inici, str)
assert isinstance(period.data_final, str)
start_hour = TIMEZONE.localize(
self.convert_string_to_datetime(period.data_inici))
end_hour = TIMEZONE.localize(
self.convert_string_to_datetime(period.data_final))
measures = []
profile = Profile(start_hour, end_hour, measures)
estimation = profile.estimate(
tariff, {str(period.name): int(period.quantitat)})
logger.debug(estimation)
# For each measure of the profile create a Consumption
for measure in estimation.measures:
logger.debug(" [F1] Processing {} {} {}".format(
factura_atr.cups, measure.date, measure.measure))
consumption_from_measure = Consumption(
cups=factura_atr.cups,
hour=measure.date,
real=measure.measure,
origin=self.type)
# Save (Upsert) consumtion following strategy "importance of data"
self.save_consumption_if_needed(consumption_from_measure)
def __init__(self, start, end, measures, accumulated=None, drag_by_periods=True):
self.measures = measures[:]
self.gaps = [] # Containing the gaps and invalid measures
self.adjusted_periods = [] # If a period is adjusted
self.start_date = start
self.end_date = end
self.profile_class = REEProfile
assert type(drag_by_periods) == bool, "drag_by_periods must be a Boolean"
self.drag_by_periods = drag_by_periods
self.accumulated = Decimal(0)
if accumulated:
assert type(accumulated) == float or isinstance(accumulated, Decimal), "Provided accumulated must be a Decimal or a float"
assert accumulated < 1 and accumulated > -1, "Provided accumulated '{}' must be -1 < accumulated < 1".format(accumulated)
self.accumulated = accumulated
measures_by_date = dict(
[(m.date, m.measure) for m in measures if m.valid]
)
# End is included
while start <= end:
if measures_by_date.pop(TIMEZONE.normalize(start), None) is None:
self.gaps.append(start)
start += timedelta(hours=1)
def __init__(self,
start,
end,
measures,
accumulated=None,
drag_by_periods=True):
self.measures = measures[:]
self.gaps = [] # Containing the gaps and invalid measures
self.adjusted_periods = [] # If a period is adjusted
self.start_date = start
self.end_date = end
self.profile_class = REEProfile
assert type(
drag_by_periods) == bool, "drag_by_periods must be a Boolean"
self.drag_by_periods = drag_by_periods
self.accumulated = Decimal(0)
if accumulated:
assert type(accumulated) == float or isinstance(
accumulated,
Decimal), "Provided accumulated must be a Decimal or a float"
assert accumulated < 1 and accumulated > -1, "Provided accumulated '{}' must be -1 < accumulated < 1".format(
accumulated)
self.accumulated = accumulated
measures_by_date = dict([(m.date, m.measure) for m in measures
if m.valid])
# End is included
while start <= end:
if measures_by_date.pop(TIMEZONE.normalize(start), None) is None:
self.gaps.append(start)
start += timedelta(hours=1)
def get_range(self, start, end):
assert isinstance(start, date)
assert isinstance(end, date)
start = TIMEZONE.localize(datetime(
start.year, start.month, start.day, 1), is_dst=True
)
# Sum one day to get the hour 00:00 of the next day
end += timedelta(days=1)
end = TIMEZONE.localize(datetime(
end.year, end.month, end.day), is_dst=True
) + timedelta(seconds=1)
pos = bisect.bisect_left(self.coefs, (start, {}))
if pos == len(self.coefs):
raise ValueError('start date not found in coefficients')
end_pos = bisect.bisect_right(self.coefs, (end, {}))
return self.coefs[pos:end_pos]
def process_consumptions(self):
"""
Process all Lectures of the imported Q1
For each period create the related profile and estimate
Create the related Consumption object and save it (if more significant) to DB
"""
lectures = self.lectures
# Get all FacturaATR lectures
for lectura in lectures:
# todo fetch Tariff?
## tariff_code = factura_atr.codi_tarifa
## tariff_name = defs.INFO_TARIFA[tariff_code]['name']
## tariff = get_tariff_by_code(tariff_name)()
tariff = T20A()
start_hour = TIMEZONE.localize(self.convert_string_to_datetime(
lectura['data_inici']))
end_hour = TIMEZONE.localize(self.convert_string_to_datetime(
lectura['data_final']))
measures = []
profile = Profile(start_hour, end_hour, measures)
estimation = profile.estimate(
tariff, {str(lectura['periode']): int(lectura['consum'])})
logger.debug(estimation)
# For each measure of the profile create a Consumption
for measure in estimation.measures:
logger.debug(" [Q1] Processing {} {} {}".format(lectura[
'cups'], measure.date, measure.measure))
consumption_from_measure = Consumption(
cups=lectura['cups'],
hour=measure.date,
real=measure.measure,
origin=self.type,
time_disc=lectura['codiDH'])
# Save (Upsert) consumtion following strategy "importance of data"
self.save_consumption_if_needed(consumption_from_measure)
def get(cls, m, header):
import csv
reader = csv.reader(m, delimiter=';')
cofs = []
n_hour = 0
for vals in reader:
if header:
header = False
continue
if int(vals[3]) == 1:
n_hour = 1
dt = datetime(
int(vals[0]), int(vals[1]), int(vals[2])
)
day = TIMEZONE.localize(dt, is_dst=bool(not int(vals[4])))
day += timedelta(hours=n_hour)
n_hour += 1
cofs.append(
(TIMEZONE.normalize(day), dict(
(k, float(vals[i])) for i, k in enumerate('ABCD', 5)
))
)
return cofs
def estimate(self, tariff, balance):
assert isinstance(tariff, Tariff)
logger.debug('Estimating for tariff: {0}'.format(
tariff.code
))
measures = [x for x in self.measures if x.valid]
start = self.start_date
end = self.end_date
cofs = self.profile_class.get_range(start, end)
cofs = Coefficients(cofs)
cofs_per_period = Counter()
for gap in self.gaps:
period = tariff.get_period_by_date(gap)
gap_cof = cofs.get(gap)
cofs_per_period[period.code] += gap_cof.cof[tariff.cof]
logger.debug('Coefficients per period calculated: {0}'.format(
cofs_per_period
))
energy_per_period = self.get_estimable_consumption(tariff, balance)
energy_per_period_rem = energy_per_period.copy()
dragger = Dragger()
for idx, gap in enumerate(self.gaps):
logger.debug('Gap {0}/{1}'.format(
idx + 1, len(self.gaps)
))
drag_key = period.code
period = tariff.get_period_by_date(gap)
gap_cof = cofs.get(gap).cof[tariff.cof]
energy = energy_per_period[period.code]
# If the balance[period] < energy_profile[period] fill with 0
# the gaps
if energy < 0:
energy = 0
gap_energy = (energy * gap_cof) / cofs_per_period[period.code]
aprox = dragger.drag(gap_energy, key=drag_key)
energy_per_period_rem[period.code] -= gap_energy
logger.debug(
'Energy for hour {0} is {1}. {2} Energy {3}/{4}'.format(
gap, aprox, period.code,
energy_per_period_rem[period.code], energy
))
pos = bisect.bisect_left(measures, ProfileHour(gap, 0, True))
profile_hour = ProfileHour(TIMEZONE.normalize(gap), aprox, True)
measures.insert(pos, profile_hour)
profile = Profile(self.start_date, self.end_date, measures)
return profile
def get(self, year, month):
key = '%(year)s%(month)02i' % locals()
if key in self._CACHE:
logger.debug('Using CACHE for REEProfile {0}'.format(key))
return self._CACHE[key]
with open(self.path, 'r') as m:
reader = csv.reader(m, delimiter=';')
header = 2
cofs = []
for vals in reader:
if header:
header -= 1
continue
if int(vals[0]) != month:
continue
dt = datetime(year, int(vals[0]), int(vals[1]))
day = TIMEZONE.localize(dt) + timedelta(hours=int(vals[2]))
cofs.append(
Coefficent(
TIMEZONE.normalize(day),
dict((k, float(vals[i]))
for i, k in enumerate('ABCD', 3))))
self._CACHE[key] = cofs
return cofs
def __init__(self, start, end, measures):
self.measures = measures[:]
self.gaps = [] # Containing the gaps and invalid measures
self.start_date = start
self.end_date = end
self.profile_class = REEProfile
measures_by_date = dict(
[(m.date, m.measure) for m in measures if m.valid]
)
# End is included
while start <= end:
if measures_by_date.pop(TIMEZONE.normalize(start), None) is None:
self.gaps.append(start)
start += timedelta(hours=1)
def parse_line(self, line):
"""
Create the Consumption object from a P5D line and save it (if more significant) to DB
"""
data = line['cchval'].data
cups = data['name']
consumption = data['ai']
#ao = data['ao']
hour = TIMEZONE.localize(data['datetime'])
logger.debug(" [P5D] Processing {} {} {}".format(
cups, hour, consumption))
consumption_from_line = Consumption(cups=cups,
hour=hour,
real=consumption,
origin=self.type)
# Save (Upsert) consumption following strategy "importance of data"
self.save_consumption_if_needed(consumption_from_line)
from enerdata.datetime import datetime
from enerdata.datetime.station import get_station
from enerdata.datetime.solar_hour import convert_to_solar_hour
from enerdata.datetime.timezone import TIMEZONE
from dateutil.relativedelta import relativedelta
from expects import expect, equal, raise_error
DIFFERENCE_HOURS = {'summer': 2, 'winter': 1}
with description("The solar hour"):
with context("in Summer"):
with it("is the civil time plus 2"):
dt = TIMEZONE.localize(datetime(2019, 4, 1, 1))
final_dt = dt + relativedelta(months=3)
hours = DIFFERENCE_HOURS['summer']
while dt < final_dt:
dt = dt + relativedelta(hours=1)
solar_hour = convert_to_solar_hour(dt)
civil_hour_to_expect = dt - relativedelta(hours=hours)
assert civil_hour_to_expect.hour == solar_hour.hour
with it("is the civil time plus 1"):
dt = TIMEZONE.localize(datetime(2019, 11, 1, 1))
final_dt = dt + relativedelta(months=3)
hours = DIFFERENCE_HOURS['winter']
while dt < final_dt:
dt = dt + relativedelta(hours=1)
solar_hour = convert_to_solar_hour(dt)
civil_hour_to_expect = dt - relativedelta(hours=hours)
assert civil_hour_to_expect.hour == solar_hour.hour
def estimate(self, tariff, balance):
assert isinstance(tariff, Tariff)
logger.debug('Estimating for tariff: {0}'.format(tariff.code))
# Adapt balance for simplified T30A with just one period
if isinstance(tariff, T30A_one_period) or isinstance(
tariff, T31A_one_period):
balance = {"P1": sum([values for values in balance.values()])}
# Adapt T31A6P adding P4 to P1
if isinstance(tariff, T31A) and balance.get('P4', 0) > 0:
balance['P1'] += balance['P4']
balance['P4'] = 0
measures = [x for x in self.measures if x.valid]
start = self.start_date
end = self.end_date
# - REE cofs get from (year/month)
# - Simel cofs get from (year/month/day hour) - can't substract one day
if self.first_day_of_month or not issubclass(self.profile_class,
REEProfile):
cofs = self.profile_class.get_range(start, end)
else:
cofs = self.profile_class.get_range(start,
end - relativedelta(days=1))
cofs = Coefficients(cofs)
cofs_per_period = Counter()
for gap in self.gaps:
dt = gap - timedelta(minutes=1)
period = tariff.get_period_by_date(dt)
gap_cof = cofs.get(dt)
cofs_per_period[period.code] += gap_cof.cof[tariff.cof]
logger.debug(
'Coefficients per period calculated: {0}'.format(cofs_per_period))
energy_per_period = self.get_estimable_consumption(tariff, balance)
energy_per_period_rem = energy_per_period.copy()
dragger = Dragger()
# Initialize the Dragger with passed accumulated value
if len(self.gaps) > 0:
# Drag by_hours
if not self.drag_by_periods:
init_drag_key = "default"
else:
dt = self.gaps[0] - timedelta(minutes=1)
init_drag_key = tariff.get_period_by_date(dt).code
dragger.drag(self.accumulated, key=init_drag_key)
for idx, gap in enumerate(self.gaps):
logger.debug('Gap {0}/{1}'.format(idx + 1, len(self.gaps)))
dt = gap - timedelta(minutes=1)
period = tariff.get_period_by_date(dt)
drag_key = period.code if not self.drag_by_periods else "default"
gap_cof = cofs.get(gap).cof[tariff.cof]
energy = energy_per_period[period.code]
# If the balance[period] < energy_profile[period] fill with 0
if energy < 0:
energy = 0
try:
gap_energy = (energy *
gap_cof) / cofs_per_period[period.code]
except ZeroDivisionError as error:
gap_energy = 0
logger.debug(error)
aprox = dragger.drag(gap_energy, key=drag_key)
energy_per_period_rem[period.code] -= gap_energy
logger.debug(
'Energy for hour {0} is {1}. {2} Energy {3}/{4}'.format(
gap, aprox, period.code,
energy_per_period_rem[period.code], energy))
pos = bisect.bisect_left(measures,
ProfileHour(gap, 0, True, 0.0))
profile_hour = ProfileHour(TIMEZONE.normalize(gap), aprox,
True, dragger[drag_key])
measures.insert(pos, profile_hour)
profile = Profile(self.start_date, self.end_date, measures)
return profile
def estimate(self, tariff, balance):
assert isinstance(tariff, Tariff)
logger.debug('Estimating for tariff: {0}'.format(
tariff.code
))
# Adapt balance for simplified T30A with just one period
if isinstance(tariff, T30A_one_period) or isinstance(tariff, T31A_one_period):
balance = {
"P1": sum([values for values in balance.values()])
}
# Get balance for T31ALB
if isinstance(tariff, T31A) and tariff.low_voltage_measure:
balance = tariff.apply_31A_LB_cof(
balance, self.start_date, self.end_date
)
measures = [x for x in self.measures if x.valid]
start = self.start_date
end = self.end_date
# - REE cofs get from (year/month)
# - Simel cofs get from (year/month/day hour) - can't substract one day
if self.first_day_of_month or not issubclass(self.profile_class,
REEProfile):
cofs = self.profile_class.get_range(start, end)
else:
cofs = self.profile_class.get_range(
start, end - relativedelta(days=1)
)
cofs = Coefficients(cofs)
cofs_per_period = Counter()
for gap in self.gaps:
period = tariff.get_period_by_date(gap)
gap_cof = cofs.get(gap)
cofs_per_period[period.code] += gap_cof.cof[tariff.cof]
logger.debug('Coefficients per period calculated: {0}'.format(
cofs_per_period
))
energy_per_period = self.get_estimable_consumption(tariff, balance)
energy_per_period_rem = energy_per_period.copy()
dragger = Dragger()
# Initialize the Dragger with passed accumulated value
if len(self.gaps) > 0:
# Drag by_hours
if not self.drag_by_periods:
init_drag_key = "default"
else:
init_drag_key = tariff.get_period_by_date(self.gaps[0]).code
dragger.drag(self.accumulated, key=init_drag_key)
for idx, gap in enumerate(self.gaps):
logger.debug('Gap {0}/{1}'.format(
idx + 1, len(self.gaps)
))
period = tariff.get_period_by_date(gap)
drag_key = period.code if not self.drag_by_periods else "default"
gap_cof = cofs.get(gap).cof[tariff.cof]
energy = energy_per_period[period.code]
# If the balance[period] < energy_profile[period] fill with 0
if energy < 0:
energy = 0
gap_energy = (energy * gap_cof) / cofs_per_period[period.code]
aprox = dragger.drag(gap_energy, key=drag_key)
energy_per_period_rem[period.code] -= gap_energy
logger.debug(
'Energy for hour {0} is {1}. {2} Energy {3}/{4}'.format(
gap, aprox, period.code,
energy_per_period_rem[period.code], energy
))
pos = bisect.bisect_left(measures, ProfileHour(gap, 0, True, 0.0))
profile_hour = ProfileHour(TIMEZONE.normalize(gap), aprox, True, dragger[drag_key])
measures.insert(pos, profile_hour)
profile = Profile(self.start_date, self.end_date, measures)
return profile
from enerdata.datetime import datetime
from enerdata.datetime.timezone import TIMEZONE
from enerdata.datetime.station import *
with description('The station module'):
with it('has to return the station by date'):
assert get_station(datetime(2014, 1, 1)) == 'winter'
assert get_station(datetime(2014, 4, 1)) == 'summer'
assert get_station(datetime(2014, 10, 26, 2)) == 'winter'
assert get_station(datetime(2014, 3, 30, 2)) == 'summer'
assert get_station(TIMEZONE.localize(datetime(2014, 10, 26, 2), is_dst=True)) == 'summer'