def handle_request(mpan_cores=None):
start_year = req_int("start_year")
start_month = req_int("start_month")
start_day = req_int("start_day")
start_date_ct = ct_datetime(start_year, start_month, start_day)
finish_year = req_int("finish_year")
finish_month = req_int("finish_month")
finish_day = req_int("finish_day")
finish_date_ct = ct_datetime(finish_year, finish_month, finish_day, 23, 30)
imp_related = req_bool("imp_related")
channel_type = req_str("channel_type")
is_zipped = req_bool("is_zipped")
supply_id = req_int("supply_id") if "supply_id" in request.values else None
user = g.user
args = (
start_date_ct,
finish_date_ct,
imp_related,
channel_type,
is_zipped,
supply_id,
mpan_cores,
user,
)
threading.Thread(target=content, args=args).start()
return chellow_redirect("/downloads", 303)
def test_duos_availability_from_to(mocker):
mocker.patch("chellow.utils.root_path", 'chellow')
mocker.patch("chellow.utils.url_root", 'chellow')
print(os.getcwd())
month_from = to_utc(ct_datetime(2019, 2, 1))
month_to = to_utc(ct_datetime(2019, 2, 28, 23, 30))
ds = mocker.Mock()
ds.dno_code = '22'
ds.gsp_group_code = '_L'
ds.llfc_code = '510'
ds.is_displaced = False
ds.sc = 0
ds.supplier_bill = defaultdict(int)
ds.supplier_rate_sets = defaultdict(set)
ds.get_data_sources = mocker.Mock(return_value=[])
ds.caches = {
'dno': {
'22': {}
}
}
hh = {
'start-date': ct_datetime(2019, 2, 28, 23, 30),
'ct-decimal-hour': 23.5,
'ct-is-month-end': True,
'ct-day-of-week': 3,
'ct-year': 2019,
'ct-month': 2,
'msp-kwh': 0,
'imp-msp-kvarh': 0,
'exp-msp-kvarh': 0
}
chellow.duos.datum_2010_04_01(ds, hh)
ds.get_data_sources.assert_called_with(month_from, month_to)
def test_duos_availability_from_to(mocker):
mocker.patch("chellow.utils.root_path", "chellow")
mocker.patch("chellow.utils.url_root", "chellow")
print(os.getcwd())
month_from = to_utc(ct_datetime(2019, 2, 1))
month_to = to_utc(ct_datetime(2019, 2, 28, 23, 30))
ds = mocker.Mock()
ds.dno_code = "22"
ds.gsp_group_code = "_L"
ds.llfc_code = "510"
ds.is_displaced = False
ds.sc = 0
ds.supplier_bill = defaultdict(int)
ds.supplier_rate_sets = defaultdict(set)
ds.get_data_sources = mocker.Mock(return_value=[])
ds.caches = {"dno": {"22": {}}}
hh = {
"start-date": ct_datetime(2019, 2, 28, 23, 30),
"ct-decimal-hour": 23.5,
"ct-is-month-end": True,
"ct-day-of-week": 3,
"ct-year": 2019,
"ct-month": 2,
"msp-kwh": 0,
"imp-msp-kvarh": 0,
"exp-msp-kvarh": 0,
}
chellow.duos.datum_2010_04_01(ds, hh)
ds.get_data_sources.assert_called_with(month_from, month_to)
def test_import_Line_Loss_Factor_Class(sess):
participant_code = "CALB"
participant = Participant.insert(sess, participant_code, "AK Industries")
market_role_R = MarketRole.insert(sess, "R", "Distributor")
dno = participant.insert_party(sess, market_role_R, "WPD",
to_utc(ct_datetime(2000, 1, 1)), None, "22")
insert_voltage_levels(sess)
voltage_level = VoltageLevel.get_by_code(sess, "HV")
llfc_code = "004"
llfc_description = "PC 5-8 & HH HV"
dno.insert_llfc(
sess,
llfc_code,
llfc_description,
voltage_level,
False,
True,
to_utc(ct_datetime(1996, 1, 1)),
None,
)
sess.commit()
row = [
participant_code, "", "", llfc_code, "01/01/1996", llfc_description,
"A", ""
]
csv_reader = iter([row])
chellow.mdd_importer._import_Line_Loss_Factor_Class(sess, csv_reader)
def test_SiteSource_get_data_sources_clock_change(mocker):
mocker.patch.object(chellow.computer.SiteSource, "__init__",
lambda *x: None)
mocker.patch("chellow.computer.displaced_era")
mock_c_months_u = mocker.patch("chellow.computer.c_months_u")
ds = chellow.computer.SiteSource()
ds.forecast_date = to_utc(ct_datetime(2010, 1, 1))
ds.start_date = to_utc(ct_datetime(2008, 1, 1))
ds.finish_date = to_utc(ct_datetime(2008, 8, 31, 22, 30))
ds.stream_focus = "gen-used"
ds.sess = mocker.Mock()
ds.caches = {}
ds.site = mocker.Mock()
ds.era_maps = mocker.Mock()
ds.deltas = mocker.Mock()
start_date = to_utc(ct_datetime(2008, 4, 1))
finish_date = to_utc(ct_datetime(2008, 4, 30, 23, 30))
result = ds.get_data_sources(start_date, finish_date)
try:
next(result)
except StopIteration:
pass
mock_c_months_u.assert_called_with(finish_month=4,
finish_year=2008,
start_month=4,
start_year=2008)
def test_summertime(mocker):
start_date_ct = ct_datetime(2010, 5)
finish_date_ct = ct_datetime(2010, 5, 31, 23, 30)
start_date = to_utc(start_date_ct)
finish_date = to_utc(finish_date_ct)
contract_id = 1
user = mocker.Mock()
user.email_address = "sfreud"
mock_c_months_u = mocker.patch(
"chellow.reports.report_87.c_months_u", autospec=True
)
MockEra = mocker.patch("chellow.reports.report_87.Era", autospec=True)
MockEra.finish_date = finish_date
MockEra.start_date = start_date
mocker.patch(
"chellow.reports.report_87.contract_func",
autospec=True,
return_value=lambda: ["standing_gbp"],
)
f = StringIO()
sess = Sess([], [])
chellow.reports.report_87.create_csv(f, sess, start_date, finish_date, contract_id)
mock_c_months_u.assert_called_with(
start_year=start_date_ct.year,
start_month=start_date_ct.month,
finish_year=finish_date_ct.year,
finish_month=finish_date_ct.month,
)
def test_lafs_forecast_none(mocker, sess):
caches = {"dno": {"22": {}}}
participant = Participant.insert(sess, "CALB", "AK Industries")
market_role_R = MarketRole.insert(sess, "R", "Distributor")
dno = participant.insert_party(
sess, market_role_R, "WPD", to_utc(ct_datetime(2000, 1, 1)), None, "22"
)
insert_voltage_levels(sess)
voltage_level = VoltageLevel.get_by_code(sess, "HV")
dno.insert_llfc(
sess,
"510",
"PC 5-8 & HH HV",
voltage_level,
False,
True,
to_utc(ct_datetime(1996, 1, 1)),
None,
)
start_date = to_utc(ct_datetime(2019, 2, 28, 23, 30))
hist_date = to_utc(ct_datetime(2018, 2, 28, 23, 30))
sess.commit()
ds = mocker.Mock()
ds.forecast_date = hist_date
ds.start_date = start_date
ds.finish_date = start_date
ds.dno_code = "22"
ds.gsp_group_code = "_L"
ds.llfc_code = "510"
ds.is_displaced = False
ds.sc = 0
ds.supplier_bill = defaultdict(int)
ds.supplier_rate_sets = defaultdict(set)
ds.get_data_sources = mocker.Mock(return_value=iter([ds]))
ds.caches = caches
ds.sess = sess
hh = {
"hist-start": hist_date,
"start-date": start_date,
"ct-decimal-hour": 23.5,
"ct-is-month-end": True,
"ct-day-of-week": 3,
"ct-year": 2019,
"ct-month": 2,
"msp-kwh": 0,
"imp-msp-kvarh": 0,
"exp-msp-kvarh": 0,
}
with pytest.raises(
BadRequest,
match="Missing LAF for DNO 22 and LLFC 510 and timestamps 2019-02-28 23:30, "
"2018-02-28 23:30 and 2018-02-28 23:30",
):
chellow.duos.datum_2012_02_23(ds, hh)
def test_lafs_hist(mocker, sess):
caches = {"dno": {"22": {}}}
participant = Participant.insert(sess, "CALB", "AK Industries")
market_role_R = MarketRole.insert(sess, "R", "Distributor")
dno = participant.insert_party(
sess, market_role_R, "WPD", to_utc(ct_datetime(2000, 1, 1)), None, "22"
)
insert_voltage_levels(sess)
voltage_level = VoltageLevel.get_by_code(sess, "HV")
llfc = dno.insert_llfc(
sess,
"510",
"PC 5-8 & HH HV",
voltage_level,
False,
True,
to_utc(ct_datetime(1996, 1, 1)),
None,
)
hist_laf = 1.5
start_date = to_utc(ct_datetime(2019, 2, 28, 23, 30))
hist_date = to_utc(ct_datetime(2018, 2, 28, 23, 30))
llfc.insert_laf(sess, hist_date, hist_laf)
sess.commit()
ds = mocker.Mock()
ds.start_date = start_date
ds.finish_date = start_date
ds.dno_code = "22"
ds.gsp_group_code = "_L"
ds.llfc_code = "510"
ds.is_displaced = False
ds.sc = 0
ds.supplier_bill = defaultdict(int)
ds.supplier_rate_sets = defaultdict(set)
ds.get_data_sources = mocker.Mock(return_value=iter([ds]))
ds.caches = caches
ds.sess = sess
ds.hh_data = []
hh = {
"hist-start": hist_date,
"start-date": start_date,
"ct-decimal-hour": 23.5,
"ct-is-month-end": True,
"ct-day-of-week": 3,
"ct-year": 2019,
"ct-month": 2,
"msp-kwh": 0,
"imp-msp-kvarh": 0,
"exp-msp-kvarh": 0,
}
chellow.duos.datum_2012_02_23(ds, hh)
assert caches["dno"]["22"]["lafs"]["510"][start_date] == hist_laf
def test_duos_availability_from_to(mocker, sess):
caches = {"dno": {"22": {}}}
participant = Participant.insert(sess, "CALB", "AK Industries")
market_role_R = MarketRole.insert(sess, "R", "Distributor")
dno = participant.insert_party(
sess, market_role_R, "WPD", to_utc(ct_datetime(2000, 1, 1)), None, "22"
)
insert_voltage_levels(sess)
voltage_level = VoltageLevel.get_by_code(sess, "HV")
llfc = dno.insert_llfc(
sess,
"510",
"PC 5-8 & HH HV",
voltage_level,
False,
True,
to_utc(ct_datetime(1996, 1, 1)),
None,
)
month_from = to_utc(ct_datetime(2019, 2, 1))
month_to = to_utc(ct_datetime(2019, 2, 28, 23, 30))
for hh in hh_range(caches, month_from, month_to):
llfc.insert_laf(sess, hh, 1)
sess.commit()
ds = mocker.Mock()
ds.dno_code = "22"
ds.gsp_group_code = "_L"
ds.llfc_code = "510"
ds.is_displaced = False
ds.sc = 0
ds.supplier_bill = defaultdict(int)
ds.supplier_rate_sets = defaultdict(set)
ds.get_data_sources = mocker.Mock(return_value=[])
ds.caches = caches
ds.sess = sess
hh = {
"start-date": ct_datetime(2019, 2, 28, 23, 30),
"ct-decimal-hour": 23.5,
"ct-is-month-end": True,
"ct-day-of-week": 3,
"ct-year": 2019,
"ct-month": 2,
"msp-kwh": 0,
"imp-msp-kvarh": 0,
"exp-msp-kvarh": 0,
}
chellow.duos.datum_2010_04_01(ds, hh)
ds.get_data_sources.assert_called_with(month_from, month_to)
def test_write_sites(mocker):
sess = mocker.Mock()
caches = {}
writer = mocker.Mock()
year = 2010
year_start = to_utc(ct_datetime(year - 1, 4, 1))
year_finish = to_utc(ct_datetime(year, 3, 31, 23, 30))
month_start = to_utc(utc_datetime(year, 3, 1))
site_id = None
site = mocker.Mock()
era = mocker.Mock()
source_codes = ("gen", "gen-net")
ms = mocker.patch("chellow.reports.report_181._make_sites", autospec=True)
ms.return_value = [site]
de = mocker.patch("chellow.computer.displaced_era", autospec=True)
de.return_value = era
ss = mocker.patch("chellow.computer.SiteSource", autospec=True)
ss_instance = ss.return_value
ss_instance.hh_data = [{
"start-date": month_start,
"triad-actual-1-date": utc_datetime(year, 1, 12),
"triad-actual-1-msp-kw": 12,
"triad-actual-1-laf": 1,
"triad-actual-1-gsp-kw": 12,
"triad-actual-2-date": utc_datetime(year, 1, 12),
"triad-actual-2-msp-kw": 12,
"triad-actual-2-laf": 1,
"triad-actual-2-gsp-kw": 12,
"triad-actual-3-date": utc_datetime(year, 1, 12),
"triad-actual-3-msp-kw": 12,
"triad-actual-3-laf": 1,
"triad-actual-3-gsp-kw": 12,
"triad-actual-gsp-kw": 12,
"triad-actual-rate": 10,
"triad-actual-gbp": 120,
}]
ss_instance.supplier_bill_hhs = {month_start: {}}
mocker.patch("chellow.duos.duos_vb", autospec=True)
mocker.patch("chellow.triad.hh", autospec=True)
forecast_date = chellow.computer.forecast_date()
chellow.reports.report_181._write_sites(sess, caches, writer, year,
site_id)
ms.assert_called_once_with(sess, year_start, year_finish, site_id,
source_codes)
de.assert_called_once_with(sess, caches, site, month_start, year_finish,
forecast_date)
def test_general_import_llfc_update_valid_to_no_change(sess):
participant = Participant.insert(sess, "CALB", "AK Industries")
market_role_R = MarketRole.insert(sess, "R", "Distributor")
dno = participant.insert_party(sess, market_role_R, "WPD",
utc_datetime(2000, 1, 1), None, "10")
insert_voltage_levels(sess)
voltage_level = VoltageLevel.get_by_code(sess, "HV")
dno.insert_llfc(
sess,
"328",
"PC 5-8 & HH HV",
voltage_level,
False,
True,
to_utc(ct_datetime(2020, 4, 1)),
None,
)
sess.commit()
action = "update"
vals = [
"10",
"328",
"2020-04-01 00:00",
"Reserved EHV 33kV - Import",
"HV",
"False",
"{no change}",
"{no change}",
]
args = []
chellow.general_import.general_import_llfc(sess, action, vals, args)
def test_make_row(mocker):
llfc = mocker.Mock()
llfc.code = "651"
voltage_level = mocker.Mock()
llfc.voltage_level = voltage_level
voltage_level.code = "LV"
llfc.is_substation = False
llfc.valid_from = to_utc(ct_datetime(2020, 4, 1))
dno = mocker.Mock()
llfc.dno = dno
dno.dno_code = "22"
ss_llfc = {
"voltage_level": "HV",
"is_substation": False,
}
actual = _make_row(llfc, ss_llfc)
expected = (
"update",
"llfc",
"22",
"651",
"2020-04-01 00:00",
"{no change}",
"HV",
"{no change}",
"{no change}",
"{no change}",
)
assert actual == expected
def do_post(sess):
start_date = req_date("start", "day")
finish_year = req_int("finish_year")
finish_month = req_int("finish_month")
finish_day = req_int("finish_day")
finish_date_ct = ct_datetime(finish_year, finish_month, finish_day, 23, 30)
finish_date = to_utc(finish_date_ct)
finish_date_str = finish_date_ct.strftime("%Y%m%d%H%M")
if "site_id" in request.values:
site_id = req_int("site_id")
file_name = f"sites_hh_data_{site_id}_{finish_date_str}.csv"
args = site_id, start_date, finish_date, g.user, file_name
threading.Thread(target=site_content, args=args).start()
return chellow_redirect("/downloads", 303)
else:
typ = req_str("type")
site_codes_str = req_str("site_codes")
site_codes = site_codes_str.splitlines()
if len(site_codes) == 0:
site_codes = None
file_name = f"sites_hh_data_{finish_date_str}_filter.zip"
args = site_codes, typ, start_date, finish_date, g.user, file_name
threading.Thread(target=none_content, args=args).start()
return chellow_redirect("/downloads", 303)
def test_fetch_cvs(mocker, sess):
market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
participant = Participant.insert(sess, "CALB", "AK Industries")
participant.insert_party(sess, market_role_Z, "None core",
utc_datetime(2000, 1, 1), None, None)
properties = {
"enabled": True,
"url": "https://example.com",
}
Contract.insert_non_core(sess, "g_cv", "", properties,
to_utc(ct_datetime(2014, 6, 1)), None, {})
sess.commit()
mock_response = mocker.Mock()
with open("test/test_g_cv/cv.csv") as f:
mock_response.text = f.read()
mock_requests = mocker.patch("chellow.g_cv.requests")
mock_requests.post = mocker.Mock(return_value=mock_response)
messages = []
def log_f(msg):
messages.append(msg)
chellow.g_cv.fetch_cvs(sess, log_f)
assert messages[-1] == "Added new rate script."
def test_c_months_u():
finish_year, finish_month = 2009, 3
start, _ = next(
c_months_u(finish_year=finish_year,
finish_month=finish_month,
months=1))
assert start == to_utc(ct_datetime(finish_year, finish_month))
def test_c_months_u_start_finish():
start_year, start_month, finish_year, finish_month = 2009, 3, 2009, 4
month_list = list(
c_months_u(
start_year=start_year,
start_month=start_month,
finish_year=finish_year,
finish_month=finish_month,
))
print(month_list)
assert month_list == [
(to_utc(ct_datetime(2009, 3)), to_utc(ct_datetime(2009, 3, 31, 23,
30))),
(to_utc(ct_datetime(2009, 4)), to_utc(ct_datetime(2009, 4, 30, 23,
30))),
]
def test_SiteSource_get_data_sources(mocker):
mocker.patch.object(chellow.computer.SiteSource, "__init__",
lambda *x: None)
mocker.patch("chellow.computer.displaced_era")
ds = chellow.computer.SiteSource()
ds.forecast_date = to_utc(ct_datetime(2010, 1, 1))
ds.start_date = to_utc(ct_datetime(2008, 1, 1))
ds.finish_date = to_utc(ct_datetime(2008, 8, 31, 22, 30))
ds.stream_focus = "gen-used"
ds.sess = mocker.Mock()
ds.caches = {}
ds.site = mocker.Mock()
ds.era_maps = mocker.Mock()
ds.deltas = mocker.Mock()
start_date = to_utc(ct_datetime(2008, 7, 1))
finish_date = to_utc(ct_datetime(2008, 7, 31, 23, 30))
result = ds.get_data_sources(start_date, finish_date)
next(result)
def test_process_segment_CCD3_ro(mocker):
code = "CCD"
elements = {
"CCDE": ["3", "ADD"],
"TCOD": ["425779", "RO Mutualisation"],
"TMOD": [],
"MTNR": [],
"MLOC": ["22767395756734"],
"PRDT": [],
"PVDT": [],
"NDRP": [],
"PRRD": [],
"CONS": ["", ""],
"CONB": [],
"ADJF": ["UG"],
"CONA": [],
"BPRI": ["974"],
"NUCT": ["877457492", "KWH"],
"CSDT": ["191001"],
"CEDT": ["191101"],
"CPPU": ["748"],
"CTOT": ["76981"],
}
line = ""
issue_date = utc_datetime(2019, 9, 3)
reference = "hgtuer8"
headers = {
"issue_date": issue_date,
"reference": reference,
"bill_type_code": "N"
}
chellow.bill_parser_engie_edi._process_segment(code, elements, line,
headers)
expected_headers = {
"mpan_core": "22 7673 9575 6734",
"bill_start_date": to_utc(ct_datetime(2019, 10, 1)),
"bill_finish_date": to_utc(ct_datetime(2019, 10, 31, 23, 30)),
"issue_date": issue_date,
"reference": reference,
"bill_type_code": "N",
}
assert headers == expected_headers
def _process_month(log_f, sess, contract, latest_rs, month_start, month_finish):
latest_rs_id = latest_rs.id
log_f(
f"Checking to see if data is available from {hh_format(month_start)} "
f"to {hh_format(month_finish)} on BMRS."
)
rates = {}
month_finish_ct = to_ct(month_finish)
for d in range(month_finish_ct.day):
day_ct = ct_datetime(month_finish_ct.year, month_finish_ct.month, d + 1)
params = {
"q": f"ajax/alldata/MID/Date,SP,Provider,Price,Volume/NULL/"
f'{day_ct.strftime("%Y-%m-%d")}/ALL'
}
r = requests.get(
"https://www.bmreports.com/bmrs/", params=params, timeout=60, verify=False
)
res = r.json()
for h in res["arr"]:
dt = to_utc(day_ct + (int(h["settlementPeriod"]) - 1) * HH)
try:
rate = rates[dt]
except KeyError:
rate = rates[dt] = {}
rate[h["DataProviderId"]] = Decimal(h["MarketIndexPrice"]) / Decimal(1000)
if month_finish in rates:
log_f("The whole month's data is there.")
script = {"rates": rates}
rs = RateScript.get_by_id(sess, latest_rs_id)
contract.update_rate_script(
sess, rs, rs.start_date, month_finish, loads(rs.script)
)
contract.insert_rate_script(sess, month_start, script)
sess.commit()
log_f(f"Added a new rate script starting at {hh_format(month_start)}.")
else:
msg = "There isn't a whole month there yet."
if len(rates) > 0:
msg += " The last date is {sorted(rates.keys())[-1]}"
log_f(msg)
return None
else:
dt = to_ct(Datetime.strptime(date_str, "%d/%m/%Y"))
dt += Timedelta(hours=23, minutes=30)
return to_utc(dt)
def is_common_mtc(code):
return 499 < code < 510 or 799 < code < 1000
def do_get(sess):
return render_template("report_163.html")
VOLTAGE_MAP = {"24": {"602": {to_utc(ct_datetime(2010, 4, 1)): "LV"}}}
def _parse_Line_Loss_Factor_Class(sess, csv_reader):
VOLTAGE_LEVEL_CODES = set([v.code for v in sess.query(VoltageLevel)])
DNO_MAP = dict(
(dno.participant.code, dno)
for dno in sess.query(Party).join(MarketRole).filter(
MarketRole.code == "R").options(joinedload(Party.participant)))
rows = []
for values in csv_reader:
participant_code = values[0]
# market_role_code = values[1]
llfc_code = values[3].zfill(3)
valid_from = parse_date(values[4])
def test_to_utc():
dt_utc = to_utc(ct_datetime(2014, 9, 6, 1))
assert dt_utc == Datetime(2014, 9, 6, 0, 0, tzinfo=utc)
def test_hh_format_hh_50():
dt = to_utc(ct_datetime(2019, 10, 27, 23, 30))
actual = hh_format(dt, with_hh=True)
assert actual == ("2019-10-27 23:30", 50)
def to_date(dmy):
if len(dmy) == 0:
return None
else:
return ct_datetime(int(dmy[6:]), int(dmy[3:5]), int(dmy[:2]))
def _process_hh(ds, rate_period, est_kw, hh):
month_start, month_finish = next(
c_months_u(start_year=hh["ct-year"], start_month=hh["ct-month"]))
month_start_ct = to_ct(month_start)
if month_start_ct.month > 3:
year = month_start_ct.year
else:
year = month_start_ct.year - 1
financial_year_start = to_utc(ct_datetime(year, 4, 1))
last_financial_year_start = to_utc(ct_datetime(year - 1, 4, 1))
financial_year_finish = to_utc(ct_datetime(year + 1, 3, 31, 23, 30))
est_triad_kws = []
earliest_triad = None
for dt in get_file_rates(ds.caches, "triad_dates",
last_financial_year_start)["triad_dates"]:
triad_hh = None
earliest_triad = hh_min(earliest_triad, dt)
try:
d = next(ds.get_data_sources(dt, dt, financial_year_start))
chellow.duos.duos_vb(d)
triad_hh = d.hh_data[0]
while dt < financial_year_start:
dt += relativedelta(years=1)
for d in ds.get_data_sources(dt, dt, financial_year_start):
chellow.duos.duos_vb(d)
datum = d.hh_data[0]
triad_hh["laf"] = datum["laf"]
triad_hh["gsp-kw"] = datum["laf"] * triad_hh["msp-kw"]
except StopIteration:
triad_hh = {
"hist-start": dt,
"msp-kw": 0,
"start-date": dt,
"status": "before start of MPAN",
"laf": 1,
"gsp-kw": 0,
}
est_triad_kws.append(triad_hh)
if ds.site is None:
era = ds.supply.find_era_at(ds.sess, earliest_triad)
if (era is None
or era.get_channel(ds.sess, ds.is_import, "ACTIVE") is None
and est_kw is None):
est_kw = 0.85 * max(datum["msp-kwh"] for datum in ds.hh_data) * 2
if est_kw is not None:
for est_datum in est_triad_kws:
est_datum["msp-kw"] = est_kw
est_datum["gsp-kw"] = est_datum["msp-kw"] * est_datum["laf"]
gsp_kw = 0
for i, triad_hh in enumerate(est_triad_kws):
triad_prefix = "triad-estimate-" + str(i + 1)
hh[triad_prefix + "-date"] = triad_hh["hist-start"]
hh[triad_prefix + "-msp-kw"] = triad_hh["msp-kw"]
hh[triad_prefix + "-status"] = triad_hh["status"]
hh[triad_prefix + "-laf"] = triad_hh["laf"]
hh[triad_prefix + "-gsp-kw"] = triad_hh["gsp-kw"]
gsp_kw += triad_hh["gsp-kw"]
hh["triad-estimate-gsp-kw"] = gsp_kw / 3
polarity = "import" if ds.llfc.is_import else "export"
gsp_group_code = ds.gsp_group_code
rate = float(
get_file_rates(
ds.caches, "triad_rates",
month_start)["triad_gbp_per_gsp_kw"][polarity][gsp_group_code])
hh["triad-estimate-rate"] = rate
est_triad_gbp = hh["triad-estimate-rate"] * hh["triad-estimate-gsp-kw"]
if rate_period == "monthly":
total_intervals = 12
est_intervals = 1
hh["triad-estimate-months"] = est_intervals
else:
dt = financial_year_start
total_intervals = 0
while dt <= financial_year_finish:
total_intervals += 1
dt += relativedelta(days=1)
est_intervals = 0
for d in ds.get_data_sources(month_start, month_finish):
for h in d.hh_data:
if h["ct-decimal-hour"] == 0:
est_intervals += 1
hh["triad-estimate-days"] = est_intervals
hh["triad-estimate-gbp"] = est_triad_gbp / total_intervals * est_intervals
if hh["ct-month"] == 3:
triad_kws = []
for t_date in get_file_rates(ds.caches, "triad_dates",
month_start)["triad_dates"]:
try:
d = next(ds.get_data_sources(t_date, t_date))
if (ds.supplier_contract is None
or d.supplier_contract == ds.supplier_contract):
chellow.duos.duos_vb(d)
thh = d.hh_data[0]
else:
thh = {
"hist-start": t_date,
"msp-kw": 0,
"start-date": t_date,
"status": "before contract",
"laf": "before contract",
"gsp-kw": 0,
}
except StopIteration:
thh = {
"hist-start": t_date,
"msp-kw": 0,
"start-date": t_date,
"status": "before start of supply",
"laf": "before start of supply",
"gsp-kw": 0,
}
while t_date < financial_year_start:
t_date += relativedelta(years=1)
try:
d = next(ds.get_data_sources(t_date, t_date))
if (ds.supplier_contract is None
or d.supplier_contract == ds.supplier_contract):
chellow.duos.duos_vb(d)
thh["laf"] = d.hh_data[0]["laf"]
thh["gsp-kw"] = thh["laf"] * thh["msp-kw"]
except StopIteration:
pass
triad_kws.append(thh)
gsp_kw = 0
for i, triad_hh in enumerate(triad_kws):
pref = "triad-actual-" + str(i + 1)
hh[pref + "-date"] = triad_hh["start-date"]
hh[pref + "-msp-kw"] = triad_hh["msp-kw"]
hh[pref + "-status"] = triad_hh["status"]
hh[pref + "-laf"] = triad_hh["laf"]
hh[pref + "-gsp-kw"] = triad_hh["gsp-kw"]
gsp_kw += triad_hh["gsp-kw"]
hh["triad-actual-gsp-kw"] = gsp_kw / 3
polarity = "import" if ds.llfc.is_import else "export"
gsp_group_code = ds.gsp_group_code
tot_rate = 0
for start_date, finish_date, script in get_file_scripts("triad_rates"):
if start_date <= financial_year_finish and not hh_before(
finish_date, financial_year_start):
start_month = to_ct(start_date).month
if start_month < 4:
start_month += 12
if finish_date is None:
finish_month = 3
else:
finish_month = to_ct(finish_date).month
if finish_month < 4:
finish_month += 12
rt = get_file_rates(
ds.caches, "triad_rates", start_date
)["triad_gbp_per_gsp_kw"][polarity][gsp_group_code]
tot_rate += (finish_month - start_month + 1) * float(rt)
rate = tot_rate / 12
hh["triad-actual-rate"] = rate
hh["triad-actual-gbp"] = hh["triad-actual-rate"] * hh[
"triad-actual-gsp-kw"]
era = ds.supply.find_era_at(ds.sess, month_finish)
est_intervals = 0
interval = (relativedelta(
months=1) if rate_period == "monthly" else relativedelta(days=1))
dt = month_finish
while era is not None and dt > financial_year_start:
est_intervals += 1
dt -= interval
if hh_after(dt, era.finish_date):
era = ds.supply.find_era_at(ds.sess, dt)
if rate_period == "monthly":
hh["triad-all-estimates-months"] = est_intervals
else:
hh["triad-all-estimates-days"] = est_intervals
hh["triad-all-estimates-gbp"] = (est_triad_gbp / total_intervals *
est_intervals * -1)
def test_process_segment_CCD2(mocker):
code = "CCD"
elements = {
"CCDE": ["2", "ADD"],
"TCOD": ["584867", "AAHEDC"],
"TMOD": [],
"MTNR": [],
"MLOC": ["22767395756734"],
"PRDT": [],
"PVDT": [],
"NDRP": [],
"PRRD": [],
"CONS": ["877457492", "KWH"],
"CONB": [],
"ADJF": ["UG"],
"CONA": [],
"BPRI": ["974"],
"NUCT": ["877457492", "KWH"],
"CSDT": ["191001"],
"CEDT": ["191101"],
"CPPU": ["748"],
"CTOT": ["76981"],
}
line = ""
reference = "kdhgsf"
issue_date = utc_datetime(2019, 4, 1)
headers = {
"reference": reference,
"issue_date": issue_date,
"bill_type_code": "N"
}
bill = chellow.bill_parser_engie_edi._process_segment(
code, elements, line, headers)
expected_headers = {
"bill_type_code": "N",
"reference": reference,
"issue_date": issue_date,
"mpan_core": "22 7673 9575 6734",
"bill_start_date": to_utc(ct_datetime(2019, 10, 1)),
"bill_finish_date": to_utc(ct_datetime(2019, 10, 31, 23, 30)),
}
expected_bill = {
"bill_type_code": "N",
"reference": "kdhgsf_aahedc",
"issue_date": issue_date,
"mpan_core": "22 7673 9575 6734",
"account": "22 7673 9575 6734",
"start_date": utc_datetime(2019, 9, 30, 23, 0),
"finish_date": utc_datetime(2019, 10, 31, 23, 30),
"kwh": Decimal("0.00"),
"net": Decimal("769.81"),
"vat": 0,
"gross": Decimal("769.81"),
"breakdown": {
"raw-lines": "",
"aahedc-kwh": Decimal("877457.492"),
"aahedc-rate": [Decimal("0.00974")],
"aahedc-gbp": Decimal("769.81"),
},
"reads": [],
}
assert headers == expected_headers
assert bill == expected_bill
assert isinstance(bill["kwh"], Decimal)
assert isinstance(bill["net"], Decimal)
assert isinstance(bill["vat"], Decimal)
assert isinstance(bill["gross"], Decimal)
assert str(bill["net"]) == str(expected_bill["net"])
def test_hh_format_hh_46():
dt = to_utc(ct_datetime(2019, 3, 31, 23, 30))
actual = hh_format(dt, with_hh=True)
assert actual == ("2019-03-31 23:30", 46)
def ccl(data_source, ct_month=False):
rate_set = data_source.supplier_rate_sets['ccl-rate']
if data_source.supply.find_era_at(
data_source.sess, data_source.finish_date + HH) is None:
sup_end = data_source.finish_date
else:
sup_end = None
try:
cache = data_source.caches['ccl']
except:
data_source.caches['ccl'] = {}
cache = data_source.caches['ccl']
try:
future_funcs = data_source.caches['future_funcs']
except KeyError:
future_funcs = {}
data_source.caches['future_funcs'] = future_funcs
try:
future_funcs[ccl_contract_id]
except KeyError:
future_funcs[ccl_contract_id] = {
'start_date': None, 'func': create_future_func(1, 0)}
if data_source.bill is None:
for hh in data_source.hh_data:
if hh['ct-is-month-end'] or hh['start-date'] == sup_end:
finish_year = hh['start-date'].year
finish_month = hh['start-date'].month
kwh = 0
gbp = 0
if ct_month:
month_start = to_utc(
ct_datetime(finish_year, finish_month))
month_finish = hh['start-date']
else:
month_start = utc_datetime(finish_year, finish_month)
month_finish = month_start + relativedelta(months=1) - HH
for ds in chellow.computer.get_data_sources(
data_source, month_start, month_finish):
for datum in ds.hh_data:
try:
rate = cache[datum['start-date']]
except KeyError:
cache[datum['start-date']] = data_source.hh_rate(
ccl_contract_id, datum['start-date'],
'ccl_rate')
rate = cache[datum['start-date']]
rate_set.add(rate)
kwh += datum['msp-kwh']
gbp += datum['msp-kwh'] * rate
if kwh > 999:
hh['ccl-kwh'] = kwh
hh['ccl-gbp'] = gbp
elif data_source.is_last_bill_gen:
kwh = 0
gbp = 0
for ds in chellow.computer.get_data_sources(
data_source, data_source.bill_start, data_source.bill_finish):
for hh in ds.hh_data:
try:
rate = cache[hh['start-date']]
except KeyError:
cache[hh['start-date']] = data_source.hh_rate(
ccl_contract_id, hh['start-date'], 'ccl_rate')
rate = cache[hh['start-date']]
rate_set.add(rate)
kwh += hh['msp-kwh']
gbp += hh['msp-kwh'] * rate
hhs = (
data_source.bill_finish - data_source.bill_start).total_seconds()
if (kwh / hhs) > ((1000 * 12) / (365 * 24 * 60 * 60)):
data_source.hh_data[-1]['ccl-kwh'] = kwh
data_source.hh_data[-1]['ccl-gbp'] = gbp
def do_post(sess):
base_name = []
now = utc_datetime_now()
if "scenario_id" in request.values:
scenario_id = req_int("scenario_id")
scenario = Scenario.get_by_id(sess, scenario_id)
scenario_props = scenario.props
base_name.append(scenario.name)
start_year = scenario_props["scenario_start_year"]
start_month = scenario_props["scenario_start_month"]
start_date_ct = ct_datetime(now.year, now.month, 1)
if start_year is None:
scenario_props["scenario_start_year"] = start_date_ct.year
if start_month is None:
scenario_props["scenario_start_month"] = start_date_ct.month
else:
year = req_int("finish_year")
month = req_int("finish_month")
months = req_int("months")
start_date, _ = next(
c_months_c(finish_year=year, finish_month=month, months=months))
by_hh = req_bool("by_hh")
scenario_props = {
"scenario_start_year": start_date.year,
"scenario_start_month": start_date.month,
"scenario_duration": months,
"by_hh": by_hh,
}
base_name.append("monthly_duration")
try:
site_id = req_int("site_id") if "site_id" in request.values else None
if "site_codes" in request.values:
site_codes = req_str("site_codes").splitlines()
# Check sites codes are valid
for site_code in site_codes:
Site.get_by_code(sess, site_code)
else:
site_codes = []
if "supply_id" in request.values:
supply_id = req_int("supply_id")
else:
supply_id = None
if "compression" in request.values:
compression = req_bool("compression")
else:
compression = True
user = g.user
args = (
scenario_props,
base_name,
site_id,
supply_id,
user,
compression,
site_codes,
now,
)
threading.Thread(target=content, args=args).start()
return chellow_redirect("/downloads", 303)
except BadRequest as e:
flash(e.description)
now = Datetime.utcnow()
month_start = Datetime(now.year, now.month,
1) - relativedelta(months=1)
month_finish = Datetime(now.year, now.month, 1) - HH
return make_response(
render_template(
"ods_monthly_duration.html",
month_start=month_start,
month_finish=month_finish,
),
400,
)
def content(year, supply_id, user):
caches = {}
sess = f = writer = None
try:
sess = Session()
running_name, finished_name = chellow.dloads.make_names(
"supplies_triad.csv", user)
f = open(running_name, mode="w", newline="")
writer = csv.writer(f, lineterminator="\n")
march_start = to_utc(ct_datetime(year, 3, 1))
march_finish = to_utc(ct_datetime(year, 4, 1)) - HH
nov_start = to_utc(ct_datetime(year - 1, 11, 1))
scalar_names = {
"triad-actual-gsp-kw",
"triad-actual-gbp",
"triad-estimate-gsp-kw",
"triad-estimate-months",
"triad-estimate-gbp",
"triad-all-estimates-months",
"triad-all-estimates-gbp",
}
rate_names = {"triad-actual-rate", "triad-estimate-rate"}
for i in range(1, 4):
for p in ("triad-actual-", "triad-estimate-"):
act_pref = p + str(i) + "-"
for suf in ("msp-kw", "gsp-kw"):
scalar_names.add(act_pref + suf)
for suf in ("date", "status", "laf"):
rate_names.add(act_pref + suf)
def triad_csv(supply_source):
if supply_source is None or supply_source.mpan_core.startswith(
"99"):
return [""] * 19
chellow.duos.duos_vb(supply_source)
chellow.triad.hh(supply_source)
for hh in supply_source.hh_data:
bill_hh = supply_source.supplier_bill_hhs[hh["start-date"]]
for k in scalar_names & hh.keys():
bill_hh[k] = hh[k]
for k in rate_names & hh.keys():
bill_hh[k] = {hh[k]}
bill = reduce_bill_hhs(supply_source.supplier_bill_hhs)
values = [supply_source.mpan_core]
for i in range(1, 4):
triad_prefix = "triad-actual-" + str(i)
for suffix in [
"-date", "-msp-kw", "-status", "-laf", "-gsp-kw"
]:
values.append(csv_make_val(bill[triad_prefix + suffix]))
suffixes = ["gsp-kw", "rate", "gbp"]
values += [
csv_make_val(bill["triad-actual-" + suf]) for suf in suffixes
]
return values
writer.writerow((
"Site Code",
"Site Name",
"Supply Name",
"Source",
"Generator Type",
"Import MPAN Core",
"Import T1 Date",
"Import T1 MSP kW",
"Import T1 Status",
"Import T1 LAF",
"Import T1 GSP kW",
"Import T2 Date",
"Import T2 MSP kW",
"Import T2 Status",
"Import T2 LAF",
"Import T2 GSP kW",
"Import T3 Date",
"Import T3 MSP kW",
"Import T3 Status",
"Import T3 LAF",
"Import T3 GSP kW",
"Import GSP kW",
"Import Rate GBP / kW",
"Import GBP",
"Export MPAN Core",
"Export T1 Date",
"Export T1 MSP kW",
"Export T1 Status",
"Export T1 LAF",
"Export T1 GSP kW",
"Export T2 Date",
"Export T2 MSP kW",
"Export T2 Status",
"Export T2 LAF",
"Export T2 GSP kW",
"Export T3 Date",
"Export T3 MSP kW",
"Export T3 Status",
"Export T3 LAF",
"Export T3 GSP kW",
"Export GSP kW",
"Export Rate GBP / kW",
"Export GBP",
))
forecast_date = chellow.computer.forecast_date()
eras = _make_eras(sess, nov_start, march_finish, supply_id)
for era in eras:
site = (sess.query(Site).join(SiteEra).filter(
SiteEra.is_physical == true(), SiteEra.era == era).one())
supply = era.supply
imp_mpan_core = era.imp_mpan_core
if imp_mpan_core is None:
imp_supply_source = None
else:
imp_supply_source = chellow.computer.SupplySource(
sess, march_start, march_finish, forecast_date, era, True,
caches)
exp_mpan_core = era.exp_mpan_core
if exp_mpan_core is None:
exp_supply_source = None
else:
exp_supply_source = chellow.computer.SupplySource(
sess, march_start, march_finish, forecast_date, era, False,
caches)
gen_type = supply.generator_type
gen_type = "" if gen_type is None else gen_type.code
vals = []
for value in ([
site.code, site.name, supply.name, supply.source.code,
gen_type
] + triad_csv(imp_supply_source) + triad_csv(exp_supply_source)):
if isinstance(value, Datetime):
vals.append(hh_format(value))
else:
vals.append(str(value))
writer.writerow(vals)
# Avoid a long-running transaction
sess.rollback()
except BaseException:
msg = traceback.format_exc()
sys.stderr.write(msg)
writer.writerow([msg])
finally:
if sess is not None:
sess.close()
if f is not None:
f.close()
os.rename(running_name, finished_name)
def get_finish_date(row, name, datemode):
d = get_date_ct(row, name, datemode)
return to_utc(ct_datetime(d.year, d.month, d.day, 23, 30))
def test_to_utc():
dt_utc = to_utc(ct_datetime(2014, 9, 6, 1))
assert dt_utc == Datetime(2014, 9, 6, 0, 0, tzinfo=utc)
def test_supply(mocker, sess, client):
site = Site.insert(sess, "22488", "Water Works")
g_dn = GDn.insert(sess, "EE", "East of England")
g_ldz = g_dn.insert_g_ldz(sess, "EA")
g_exit_zone = g_ldz.insert_g_exit_zone(sess, "EA1")
insert_g_units(sess)
g_unit_M3 = GUnit.get_by_code(sess, "M3")
participant = Participant.insert(sess, "CALB", "AK Industries")
market_role_Z = MarketRole.get_by_code(sess, "Z")
participant.insert_party(
sess, market_role_Z, "None core", utc_datetime(2000, 1, 1), None, None
)
g_cv_rate_script = {
"cvs": {
"EA": {
1: {"applicable_at": utc_datetime(2020, 10, 3), "cv": 39.2000},
}
}
}
Contract.insert_non_core(
sess, "g_cv", "", {}, utc_datetime(2000, 1, 1), None, g_cv_rate_script
)
bank_holiday_rate_script = {"bank_holidays": []}
Contract.insert_non_core(
sess,
"bank_holidays",
"",
{},
utc_datetime(2000, 1, 1),
None,
bank_holiday_rate_script,
)
charge_script = """
import chellow.g_ccl
from chellow.g_engine import g_rates
from chellow.utils import reduce_bill_hhs
def virtual_bill_titles():
return [
'units_consumed', 'correction_factor', 'unit_code', 'unit_factor',
'calorific_value', 'kwh', 'gas_rate', 'gas_gbp', 'ccl_rate',
'standing_rate', 'standing_gbp', 'net_gbp', 'vat_gbp', 'gross_gbp',
'problem']
def virtual_bill(ds):
chellow.g_ccl.vb(ds)
for hh in ds.hh_data:
start_date = hh['start_date']
bill_hh = ds.bill_hhs[start_date]
bill_hh['units_consumed'] = hh['units_consumed']
bill_hh['correction_factor'] = {hh['correction_factor']}
bill_hh['unit_code'] = {hh['unit_code']}
bill_hh['unit_factor'] = {hh['unit_factor']}
bill_hh['calorific_value'] = {hh['calorific_value']}
kwh = hh['kwh']
bill_hh['kwh'] = kwh
gas_rate = float(
g_rates(ds.sess, ds.caches, db_id, start_date)['gas_rate'])
bill_hh['gas_rate'] = {gas_rate}
bill_hh['gas_gbp'] = gas_rate * kwh
bill_hh['ccl_kwh'] = kwh
ccl_rate = hh['ccl']
bill_hh['ccl_rate'] = {ccl_rate}
bill_hh['ccl_kwh'] = kwh
bill_hh['ccl_gbp'] = kwh * ccl_rate
if hh['utc_is_month_end']:
standing_rate = float(
g_rates(
ds.sess, ds.caches, db_id, start_date)['standing_rate'])
bill_hh['standing_rate'] = {standing_rate}
bill_hh['standing_gbp'] = standing_rate
if hh['utc_decimal_hour'] == 0:
pass
bill_hh['net_gbp'] = sum(
v for k, v in bill_hh.items() if k.endswith('gbp'))
bill_hh['vat_gbp'] = 0
bill_hh['gross_gbp'] = bill_hh['net_gbp'] + bill_hh['vat_gbp']
ds.bill = reduce_bill_hhs(ds.bill_hhs)
"""
g_contract_rate_script = {
"gas_rate": 0.1,
"standing_rate": 0.1,
}
g_contract = GContract.insert(
sess,
"Fusion 2020",
charge_script,
{},
utc_datetime(2000, 1, 1),
None,
g_contract_rate_script,
)
insert_g_reading_frequencies(sess)
g_reading_frequency_M = GReadingFrequency.get_by_code(sess, "M")
msn = "hgeu8rhg"
g_supply = site.insert_g_supply(
sess,
"87614362",
"main",
g_exit_zone,
utc_datetime(2010, 1, 1),
None,
msn,
1,
g_unit_M3,
g_contract,
"d7gthekrg",
g_reading_frequency_M,
)
g_batch = g_contract.insert_g_batch(sess, "b1", "Jan batch")
breakdown = {"units_consumed": 771}
insert_bill_types(sess)
bill_type_N = BillType.get_by_code(sess, "N")
insert_g_read_types(sess)
g_read_type_A = GReadType.get_by_code(sess, "A")
g_bill = g_batch.insert_g_bill(
sess,
g_supply,
bill_type_N,
"55h883",
"dhgh883",
utc_datetime(2019, 4, 3),
utc_datetime(2015, 9, 1),
utc_datetime(2015, 9, 30, 22, 30),
Decimal("45"),
Decimal("12.40"),
Decimal("1.20"),
Decimal("14.52"),
"",
breakdown,
)
g_bill.insert_g_read(
sess,
msn,
g_unit_M3,
Decimal("1"),
Decimal("37"),
Decimal("90"),
utc_datetime(2015, 9, 1),
g_read_type_A,
Decimal("890"),
utc_datetime(2015, 9, 25),
g_read_type_A,
)
sess.commit()
mock_file = StringIO()
mock_file.close = mocker.Mock()
mocker.patch(
"chellow.reports.report_g_virtual_bills_hh.open", return_value=mock_file
)
mocker.patch(
"chellow.reports.report_g_virtual_bills_hh.chellow.dloads.make_names",
return_value=("a", "b"),
)
mocker.patch("chellow.reports.report_g_virtual_bills.os.rename")
user = mocker.Mock()
g_supply_id = g_supply.id
start_date = to_utc(ct_datetime(2018, 2, 1))
finish_date = to_utc(ct_datetime(2018, 2, 1, 0, 30))
chellow.reports.report_g_virtual_bills_hh.content(
g_supply_id, start_date, finish_date, user
)
mock_file.seek(0)
table = list(csv.reader(mock_file))
expected = [
[
"MPRN",
"Site Code",
"Site Name",
"Account",
"HH Start",
"",
"",
"units_consumed",
"correction_factor",
"unit_code",
"unit_factor",
"calorific_value",
"kwh",
"gas_rate",
"gas_gbp",
"ccl_rate",
"standing_rate",
"standing_gbp",
"net_gbp",
"vat_gbp",
"gross_gbp",
"problem",
],
[
"87614362",
"22488",
"Water Works",
"d7gthekrg",
"2018-02-01 00:00",
"",
"",
"0.6944444444444444",
"1.0",
"M3",
"1.0",
"39.2",
"7.561728395061729",
"0.1",
"0.7561728395061729",
"0.00198",
"",
"",
"0.7711450617283951",
"0",
"0.7711450617283951",
"",
"ccl_gbp",
"0.014972222222222222",
"ccl_kwh",
"7.561728395061729",
],
[
"87614362",
"22488",
"Water Works",
"d7gthekrg",
"2018-02-01 00:30",
"",
"",
"0.6944444444444444",
"1.0",
"M3",
"1.0",
"39.2",
"7.561728395061729",
"0.1",
"0.7561728395061729",
"0.00198",
"",
"",
"0.7711450617283951",
"0",
"0.7711450617283951",
"",
"ccl_gbp",
"0.014972222222222222",
"ccl_kwh",
"7.561728395061729",
],
]
match_tables(table, expected)