def content(user, file_like, report_run_id):
sess = None
try:
sess = Session()
running_name, finished_name = chellow.dloads.make_names(
FNAME + ".csv", user)
f = open(running_name, mode="w", newline="")
writer = csv.writer(f, lineterminator="\n")
report_run = ReportRun.get_by_id(sess, report_run_id)
props = Contract.get_non_core_by_name(
sess, "configuration").make_properties()
_process_sites(sess, file_like, writer, props, report_run)
report_run.update("finished")
sess.commit()
except BaseException:
msg = traceback.format_exc()
if report_run is not None:
report_run.update("interrupted")
report_run.insert_row(sess, "", ["error"], {"error": msg}, {})
sess.commit()
sys.stderr.write(msg)
writer.writerow([msg])
finally:
if sess is not None:
sess.close()
if f is not None:
f.close()
os.rename(running_name, finished_name)
def do_get(sess):
show_ignored = req_bool("show_ignored")
report_run = ReportRun.insert(
sess,
FNAME,
g.user,
FNAME,
{},
)
sess.commit()
threading.Thread(target=content, args=(g.user, show_ignored, report_run.id)).start()
return chellow_redirect(f"/report_runs/{report_run.id}", 303)
def test_process_sites(sess):
site_code = "109"
Site.insert(sess, site_code, "A Site")
report_run = ReportRun.insert(sess, "asset", None, "asset", {})
sess.commit()
asset_str = "\n".join(("code,,,status", ))
file_like = StringIO(asset_str)
output = StringIO()
writer = csv.writer(output, lineterminator="\n")
props = {"asset_comparison": {"ignore_site_codes": []}}
_process_sites(sess, file_like, writer, props, report_run.id)
assert output.getvalue() == "site_code,asset_status,chellow_status\n"
def do_post(sess):
user = g.user
file_item = request.files["asset_file"]
report_run = ReportRun.insert(
sess,
FNAME,
user,
FNAME,
{
"STATUSES_ACTIVE": STATUSES_ACTIVE,
"STATUSES_INACTIVE": STATUSES_INACTIVE,
"STATUSES_IGNORE": STATUSES_IGNORE,
},
)
sess.commit()
args = user, StringIO(file_item.read().decode("utf8")), report_run.id
threading.Thread(target=content, args=args).start()
return chellow_redirect(f"/report_runs/{report_run.id}", 303)
def content(
batch_id,
bill_id,
contract_id,
start_date,
finish_date,
user,
mpan_cores,
fname_additional,
):
caches = {}
tmp_file = sess = supply_id = None
forecast_date = to_utc(Datetime.max)
try:
running_name, finished_name = chellow.dloads.make_names(
"bill_check_" + fname_additional + ".csv", user)
tmp_file = open(running_name, mode="w", newline="")
writer = csv.writer(tmp_file, lineterminator="\n")
sess = Session()
report_run = ReportRun("bill_check", user, fname_additional)
sess.add(report_run)
bills = (sess.query(Bill).order_by(
Bill.supply_id, Bill.reference).options(
joinedload(Bill.supply),
subqueryload(Bill.reads).joinedload(RegisterRead.present_type),
subqueryload(Bill.reads).joinedload(
RegisterRead.previous_type),
joinedload(Bill.batch),
))
if len(mpan_cores) > 0:
mpan_cores = list(map(parse_mpan_core, mpan_cores))
supply_ids = [
i[0] for i in sess.query(Era.supply_id).filter(
or_(
Era.imp_mpan_core.in_(mpan_cores),
Era.exp_mpan_core.in_(mpan_cores),
)).distinct()
]
bills = bills.join(Supply).filter(Supply.id.in_(supply_ids))
if batch_id is not None:
batch = Batch.get_by_id(sess, batch_id)
bills = bills.filter(Bill.batch == batch)
contract = batch.contract
elif bill_id is not None:
bill = Bill.get_by_id(sess, bill_id)
bills = bills.filter(Bill.id == bill.id)
contract = bill.batch.contract
elif contract_id is not None:
contract = Contract.get_by_id(sess, contract_id)
bills = bills.join(Batch).filter(
Batch.contract == contract,
Bill.start_date <= finish_date,
Bill.finish_date >= start_date,
)
vbf = chellow.computer.contract_func(caches, contract, "virtual_bill")
if vbf is None:
raise BadRequest("The contract " + contract.name +
" doesn't have a function virtual_bill.")
virtual_bill_titles_func = chellow.computer.contract_func(
caches, contract, "virtual_bill_titles")
if virtual_bill_titles_func is None:
raise BadRequest("The contract " + contract.name +
" doesn't have a function virtual_bill_titles.")
virtual_bill_titles = virtual_bill_titles_func()
titles = [
"batch",
"bill-reference",
"bill-type",
"bill-kwh",
"bill-net-gbp",
"bill-vat-gbp",
"bill-start-date",
"bill-finish-date",
"imp-mpan-core",
"exp-mpan-core",
"site-code",
"site-name",
"covered-from",
"covered-to",
"covered-bills",
"metered-kwh",
]
for t in virtual_bill_titles:
titles.append("covered-" + t)
titles.append("virtual-" + t)
if t.endswith("-gbp"):
titles.append("difference-" + t)
writer.writerow(titles)
bill_map = defaultdict(set, {})
for bill in bills:
bill_map[bill.supply.id].add(bill.id)
for supply_id in bill_map.keys():
_process_supply(
sess,
caches,
supply_id,
bill_map,
forecast_date,
contract,
vbf,
virtual_bill_titles,
writer,
titles,
report_run,
)
report_run.update("finished")
sess.commit()
except BadRequest as e:
if report_run is not None:
report_run.update("problem")
if supply_id is None:
prefix = "Problem: "
else:
prefix = "Problem with supply " + str(supply_id) + ":"
tmp_file.write(prefix + e.description)
except BaseException:
if report_run is not None:
report_run.update("interrupted")
if supply_id is None:
prefix = "Problem: "
else:
prefix = "Problem with supply " + str(supply_id) + ":"
msg = traceback.format_exc()
sys.stderr.write(msg + "\n")
tmp_file.write(prefix + msg)
finally:
if sess is not None:
sess.close()
tmp_file.close()
os.rename(running_name, finished_name)
def test_process(mocker, sess):
site = Site.insert(sess, "CI017", "Water Works")
market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
participant = Participant.insert(sess, "CALB", "AK Industries")
participant.insert_party(
sess, market_role_Z, "None core", utc_datetime(2000, 1, 1), None, None
)
bank_holiday_rate_script = {"bank_holidays": []}
Contract.insert_non_core(
sess,
"bank_holidays",
"",
{},
utc_datetime(2000, 1, 1),
None,
bank_holiday_rate_script,
)
market_role_X = MarketRole.insert(sess, "X", "Supplier")
market_role_M = MarketRole.insert(sess, "M", "Mop")
market_role_C = MarketRole.insert(sess, "C", "HH Dc")
market_role_R = MarketRole.insert(sess, "R", "Distributor")
participant.insert_party(
sess, market_role_M, "Fusion Mop Ltd", utc_datetime(2000, 1, 1), None, None
)
participant.insert_party(
sess, market_role_X, "Fusion Ltc", utc_datetime(2000, 1, 1), None, None
)
participant.insert_party(
sess, market_role_C, "Fusion DC", utc_datetime(2000, 1, 1), None, None
)
mop_contract = Contract.insert_mop(
sess, "Fusion", participant, "", {}, utc_datetime(2000, 1, 1), None, {}
)
dc_contract = Contract.insert_dc(
sess, "Fusion DC 2000", participant, "", {}, utc_datetime(2000, 1, 1), None, {}
)
pc = Pc.insert(sess, "00", "hh", utc_datetime(2000, 1, 1), None)
insert_cops(sess)
cop = Cop.get_by_code(sess, "5")
insert_comms(sess)
comm = Comm.get_by_code(sess, "GSM")
imp_supplier_contract = Contract.insert_supplier(
sess,
"Fusion Supplier 2000",
participant,
"",
{},
utc_datetime(2000, 1, 1),
None,
{},
)
dno = participant.insert_party(
sess, market_role_R, "WPD", utc_datetime(2000, 1, 1), None, "22"
)
meter_type = MeterType.insert(sess, "C5", "COP 1-5", utc_datetime(2000, 1, 1), None)
meter_payment_type = MeterPaymentType.insert(
sess, "CR", "Credit", utc_datetime(1996, 1, 1), None
)
Mtc.insert(
sess,
None,
"845",
"HH COP5 And Above With Comms",
False,
False,
True,
meter_type,
meter_payment_type,
0,
utc_datetime(1996, 1, 1),
None,
)
insert_voltage_levels(sess)
voltage_level = VoltageLevel.get_by_code(sess, "HV")
dno.insert_llfc(
sess,
"510",
"PC 5-8 & HH HV",
voltage_level,
False,
True,
utc_datetime(1996, 1, 1),
None,
)
dno.insert_llfc(
sess,
"521",
"Export (HV)",
voltage_level,
False,
False,
utc_datetime(1996, 1, 1),
None,
)
insert_sources(sess)
source = Source.get_by_code(sess, "net")
insert_energisation_statuses(sess)
energisation_status = EnergisationStatus.get_by_code(sess, "E")
gsp_group = GspGroup.insert(sess, "_L", "South Western")
site.insert_e_supply(
sess,
source,
None,
"Bob",
utc_datetime(2000, 1, 1),
None,
gsp_group,
mop_contract,
"773",
dc_contract,
"ghyy3",
"hgjeyhuw",
pc,
"845",
cop,
comm,
None,
energisation_status,
{},
"22 7867 6232 781",
"510",
imp_supplier_contract,
"7748",
361,
None,
None,
None,
None,
None,
)
site.insert_e_supply(
sess,
source,
None,
"Dave",
utc_datetime(2000, 1, 1),
None,
gsp_group,
mop_contract,
"773",
dc_contract,
"ghyy3",
"hgjeyhuw",
pc,
"845",
cop,
comm,
None,
energisation_status,
{},
"22 7868 6232 789",
"510",
imp_supplier_contract,
"7748",
361,
None,
None,
None,
None,
None,
)
report_run = ReportRun.insert(
sess,
"ecoes_comparison",
None,
"ecoes_comparison",
{},
)
sess.commit()
f = StringIO()
ecoes_lines = [
"titles",
",".join(
(
"2278676232781",
"address-line-1",
"address-line-2",
"address-line-3",
"address-line-4",
"address-line-5",
"address-line-6",
"address-line-7",
"address-line-8",
"address-line-9",
"post-code",
"supplier",
"registration-from",
"mtc",
"mtc-date",
"llfc",
"llfc-from",
"pc",
"",
"measurement-class",
"energisation-status",
"da",
"dc",
"mop",
"mop-appoint-date",
"gsp-group",
"gsp-effective-from",
"dno",
"msn",
"meter-install-date",
"meter-type",
"map-id",
),
),
",".join(
(
"2278686232789",
"address-line-1",
"address-line-2",
"address-line-3",
"address-line-4",
"address-line-5",
"address-line-6",
"address-line-7",
"address-line-8",
"address-line-9",
"post-code",
"supplier",
"registration-from",
"mtc",
"mtc-date",
"llfc",
"llfc-from",
"pc",
"",
"measurement-class",
"energisation-status",
"da",
"dc",
"mop",
"mop-appoint-date",
"gsp-group",
"gsp-effective-from",
"dno",
"msn",
"meter-install-date",
"meter-type",
"map-id",
),
),
]
exclude_mpan_cores = []
ignore_mpan_cores_msn = []
show_ignored = True
_process(
sess,
ecoes_lines,
exclude_mpan_cores,
ignore_mpan_cores_msn,
f,
show_ignored,
report_run,
)
expected = [
[
"mpan_core",
"mpan_core_no_spaces",
"ecoes_pc",
"chellow_pc",
"ecoes_mtc",
"chellow_mtc",
"ecoes_llfc",
"chellow_llfc",
"ecoes_ssc",
"chellow_ssc",
"ecoes_es",
"chellow_es",
"ecoes_supplier",
"chellow_supplier",
"chellow_supplier_contract_name",
"ecoes_dc",
"chellow_dc",
"ecoes_mop",
"chellow_mop",
"ecoes_gsp_group",
"chellow_gsp_group",
"ecoes_msn",
"chellow_msn",
"ecoes_msn_install_date",
"ecoes_meter_type",
"chellow_meter_type",
"ignored",
"problem",
],
[
"22 7867 6232 781",
"2278676232781",
"pc",
"00",
"mtc",
"845",
"llfc",
"510",
"",
"",
"energisation-status",
"E",
"supplier",
"CALB",
"Fusion Supplier 2000",
"dc",
"CALB",
"mop",
"CALB",
"gsp-group",
"_L",
"msn",
"hgjeyhuw",
"meter-install-date",
"meter-type",
"H",
"False",
"The energisation statuses don't match. Can't parse the PC. Can't parse "
"the MTC. The LLFCs don't match. The supplier codes don't match. The DC "
"codes don't match. The MOP codes don't match. The GSP group codes don't "
"match. The meter serial numbers don't match. The meter types don't match. "
"See https://dtc.mrasco.com/DataItem.aspx?ItemCounter=0483 ",
],
[
"22 7868 6232 789",
"2278686232789",
"pc",
"00",
"mtc",
"845",
"llfc",
"510",
"",
"",
"energisation-status",
"E",
"supplier",
"CALB",
"Fusion Supplier 2000",
"dc",
"CALB",
"mop",
"CALB",
"gsp-group",
"_L",
"msn",
"hgjeyhuw",
"meter-install-date",
"meter-type",
"H",
"False",
"The energisation statuses don't match. Can't parse the PC. Can't parse "
"the MTC. The LLFCs don't match. The supplier codes don't match. The DC "
"codes don't match. The MOP codes don't match. The GSP group codes don't "
"match. The meter serial numbers don't match. The meter types don't match. "
"See https://dtc.mrasco.com/DataItem.aspx?ItemCounter=0483 ",
],
]
assert f.getvalue() == "\n".join(",".join(line) for line in expected) + "\n"
def content(user, show_ignored, report_run_id):
sess = f = None
try:
sess = Session()
running_name, finished_name = chellow.dloads.make_names(f"{FNAME}.csv", user)
f = open(running_name, mode="w", newline="")
report_run = ReportRun.get_by_id(sess, report_run_id)
props = Contract.get_non_core_by_name(sess, "configuration").make_properties()
ECOES_KEY = "ecoes"
try:
ecoes_props = props[ECOES_KEY]
except KeyError:
raise BadRequest(
f"The property {ECOES_KEY} cannot be found in the configuration "
f"properties."
)
for key in (
"user_name",
"password",
"prefix",
"exclude_mpan_cores",
"ignore_mpan_cores_msn",
):
try:
ecoes_props[key]
except KeyError:
raise BadRequest(
f"The property {key} cannot be found in the 'ecoes' section of "
f"the configuration properties."
)
exclude_mpan_cores = ecoes_props["exclude_mpan_cores"]
ignore_mpan_cores_msn = ecoes_props["ignore_mpan_cores_msn"]
url_prefix = ecoes_props["prefix"]
proxies = props.get("proxies", {})
s = requests.Session()
s.verify = False
r = s.get(url_prefix, proxies=proxies)
data = {
"Username": ecoes_props["user_name"],
"Password": ecoes_props["password"],
}
r = s.post(url_prefix, data=data, allow_redirects=False)
r = s.get(
f"{url_prefix}NonDomesticCustomer/ExportPortfolioMPANs?fileType=csv",
proxies=proxies,
)
_process(
sess,
r.text.splitlines(True),
exclude_mpan_cores,
ignore_mpan_cores_msn,
f,
show_ignored,
report_run,
)
report_run.update("finished")
sess.commit()
except BaseException as e:
msg = traceback.format_exc()
if report_run is not None:
report_run.update("interrupted")
report_run.insert_row(sess, "", ["error"], {"error": msg}, {})
sess.commit()
sys.stderr.write(msg)
f.write(msg)
raise e
finally:
if sess is not None:
sess.close()
if f is not None:
f.close()
os.rename(running_name, finished_name)