def validate_effectivedate(cls, value: str) -> datetime:
dt = parse_date(value)
if not dt:
raise ValueError("Not a valid date: {}".format(value))
return dt
def parse_record_bom(rec: Dict) -> Dict:
"""Parse CSV records in BOM format"""
timezone = pytz.timezone(STATION_TIMEZONE_MAP[rec["station_id"]])
observation_time = parse_date(rec["observation_time"],
is_utc=True,
dayfirst=False).astimezone(timezone)
return {
"observation_time":
observation_time,
"station_id":
rec["station_id"],
"temp_apparent":
float(rec["temp_apparent"].strip())
if is_number(rec["temp_apparent"]) else None,
"temp_air":
float(rec["temp_air"].strip()) if is_number(rec["temp_air"]) else None,
"press_qnh":
None,
"wind_dir":
None,
"wind_spd":
None,
"cloud":
None,
"cloud_type":
None,
"humidity":
float(rec["humidity"].strip()) if is_number(rec["humidity"]) else None,
"wind_gust":
None,
}
def parse_dirlisting(raw_string: str) -> Dict[str, Any]:
"""
given a raw text directory listing like " Saturday 11th June 2020 6789"
will parse and return both the date and listing type
@param raw_string - the raw directory listing string
@return dict of the date in iso format and the type (file or directory)
"""
components = raw_string.split(" " * PADDING_WIDTH)
components = [i.strip() for i in components]
components = list(filter(lambda x: x != "", components))
_ltype = "dir"
if not components or len(components) < 2:
logging.debug(components)
raise Exception("Invalid line string: {}. Components are: {}".format(
raw_string, components))
if is_number(components[1]):
_ltype = "file"
dt = parse_date(components[0], network=NetworkNEM)
if type(dt) is not datetime:
raise Exception(
"{} is not a valid datetime. Original value was {}".format(
dt, components[0]))
return {
"date": dt.isoformat(),
"type": _ltype,
}
def validate_date(date_value: str) -> datetime:
dt = parse_date(date_value)
if not dt:
raise ValueError("Not a valid date: {}".format(date_value))
return dt
def process_unit_solution(table):
session = SessionLocal()
engine = get_database_engine()
if "records" not in table:
raise Exception("Invalid table no records")
records = table["records"]
records_to_store = []
records_primary_keys = []
for record in records:
trading_interval = parse_date(
record["SETTLEMENTDATE"], network=NetworkNEM, dayfirst=False
)
facility_code = normalize_duid(record["DUID"])
if not trading_interval or not facility_code:
continue
# Since this can insert 1M+ records at a time we need to
# do a separate in-memory check of primary key constraints
# better way of doing this .. @TODO
_unique_set = (trading_interval, facility_code, "NEM")
if _unique_set not in records_primary_keys:
records_to_store.append(
{
"trading_interval": trading_interval,
"facility_code": facility_code,
"eoi_quantity": float(record["INITIALMW"]),
"network_id": "NEM",
}
)
records_primary_keys.append(_unique_set)
# free
records_primary_keys = []
logger.debug("Saving %d records", len(records_to_store))
stmt = insert(FacilityScada).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
constraint="facility_scada_pkey",
set_={"eoi_quantity": stmt.excluded.eoi_quantity},
)
try:
session.execute(stmt)
session.commit()
except Exception as e:
logger.error("Error: {}".format(e))
return 0
finally:
session.close()
return len(records_to_store)
def parse(self, response: Any) -> Dict[str, Any]:
file_downloads = []
source_title = response.css("title::text").get()
download_sections = response.xpath(
"//div[@class='file-list-wrapper']/..")
if not download_sections or len(download_sections) < 1:
raise Exception(
"{} spider could not find any download sections".format(
self.name))
for download_section in download_sections:
date_text = download_section.css(
"div.field-publisheddate span::text").get()
if not date_text:
raise Exception(
"{} could not get download section published date".format(
self.name))
published_date = parse_date(date_text)
publish_link_relative = download_section.css("a::attr(href)").get()
if not publish_link_relative:
raise Exception("{} could not get rel published link".format(
self.name))
publish_link = response.urljoin(publish_link_relative)
publish_link = strip_query_string(publish_link)
download_title = download_section.css(".field-title::text").get()
download_size_raw = download_section.css(
".field-size span::text").get()
download_size = None
if download_size_raw:
download_size, _ = filesize_from_string(download_size_raw)
# create a model from the extracted fields
section_model = None
try:
section_model = AEMOFileDownloadSection(
published_date=published_date,
filename=download_title,
download_url=publish_link,
file_size=download_size,
source_url=response.url,
source_title=source_title,
)
file_downloads.append(section_model)
except ValidationError as e:
self.log("Validation error: {}".format(e), logging.ERROR)
return {"_data": file_downloads, "items": file_downloads}
def process_nem_price(table: AEMOTableSchema) -> ControllerReturn:
"""Stores the NEM price for both dispatch price and trading price"""
session = get_scoped_session()
engine = get_database_engine()
cr = ControllerReturn(total_records=len(table.records))
records_to_store = []
primary_keys = []
price_field = "price"
if table.full_name == "dispatch_price":
price_field = "price_dispatch"
for record in table.records:
# @NOTE disable pk track
trading_interval = parse_date(record["settlementdate"])
primary_key = set([trading_interval,
record["regionid"]]) # type: ignore
if primary_key in primary_keys:
continue
primary_keys.append(primary_key)
records_to_store.append({
"network_id": "NEM",
"created_by": "opennem.controllers.nem",
"network_region": record["regionid"],
"trading_interval": trading_interval,
price_field: record["rrp"],
})
cr.processed_records += 1
stmt = insert(BalancingSummary).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
index_elements=["trading_interval", "network_id", "network_region"],
set_={price_field: getattr(stmt.excluded, price_field)},
)
try:
session.execute(stmt)
session.commit()
cr.inserted_records = cr.processed_records
cr.server_latest = max(
[i["trading_interval"] for i in records_to_store])
except Exception as e:
logger.error("Error inserting NEM price records")
logger.error(e)
cr.errors = cr.processed_records
finally:
session.rollback()
session.close()
engine.dispose()
return cr
def validate_authoriseddate(cls, value: str) -> datetime:
dt = parse_date(value)
if not dt:
logger.error("Not a valid date: {}".format(value))
raise ValueError("Not a valid authoriseddate: {}".format(value))
return dt
def generate_balancing_summary(
records: List[Dict],
interval_field: str = "SETTLEMENTDATE",
network_region_field: str = "REGIONID",
price_field: Optional[str] = None,
network: NetworkSchema = NetworkNEM,
limit: int = 0,
) -> List[Dict]:
created_at = datetime.now()
primary_keys = []
return_records = []
created_by = ""
for row in records:
trading_interval = parse_date(row[interval_field],
network=network,
dayfirst=False)
network_region = None
if network_region_field and network_region_field in row:
network_region = row[network_region_field]
pk = (trading_interval, network.code, network_region)
if pk in primary_keys:
continue
primary_keys.append(pk)
price = None
if price_field and price_field in row:
price = clean_float(row[price_field])
if price:
price = str(float_to_str(price))
__rec = {
"created_by": created_by,
"created_at": created_at,
"updated_at": None,
"network_id": network.code,
"network_region": network_region,
"trading_interval": trading_interval,
"price": price,
}
return_records.append(__rec)
if limit > 0 and len(return_records) >= limit:
break
return return_records
def process_item(self, item, spider=None):
s = self.session()
csvreader = csv.DictReader(item["content"].split("\n"))
records_to_store = []
for record in csvreader:
trading_interval = parse_date(record["Trading Interval"],
dayfirst=True,
network=NetworkWEM)
if not trading_interval:
continue
records_to_store.append({
"network_id":
"WEM",
"network_region":
"WEM",
"trading_interval":
trading_interval,
"forecast_load":
record["Load Forecast (MW)"],
"generation_scheduled":
record["Scheduled Generation (MW)"],
"generation_non_scheduled":
record["Non-Scheduled Generation (MW)"],
"generation_total":
record["Total Generation (MW)"],
"price":
record["Final Price ($/MWh)"],
})
stmt = insert(BalancingSummary).values(records_to_store)
stmt.bind = self.engine
stmt = stmt.on_conflict_do_update(
constraint="balancing_summary_pkey",
set_={
"price": stmt.excluded.price,
"generation_total": stmt.excluded.generation_total,
},
)
try:
r = s.execute(stmt)
s.commit()
except Exception as e:
logger.error("Error inserting records")
logger.error(e)
finally:
s.close()
return len(records_to_store)
def process_item(self, item, spider=None):
s = SessionLocal()
csvreader = csv.DictReader(item["content"].split("\n"))
records_to_store = []
primary_keys = []
for row in csvreader:
trading_interval = parse_date(row["TRADING_DAY_INTERVAL"],
network=NetworkWEM,
dayfirst=False)
if trading_interval not in primary_keys:
forecast_load = clean_float(row["FORECAST_EOI_MW"])
records_to_store.append({
"created_by": spider.name,
"trading_interval": trading_interval,
"network_id": "WEM",
"network_region": "WEM",
"forecast_load": forecast_load,
# generation_scheduled=row["Scheduled Generation (MW)"],
# generation_total=row["Total Generation (MW)"],
"price": clean_float(row["PRICE"]),
})
primary_keys.append(trading_interval)
stmt = insert(BalancingSummary).values(records_to_store)
stmt.bind = get_database_engine()
stmt = stmt.on_conflict_do_update(
index_elements=[
"trading_interval",
"network_id",
"network_region",
],
set_={
"price": stmt.excluded.price,
"forecast_load": stmt.excluded.forecast_load,
},
)
try:
s.execute(stmt)
s.commit()
except Exception as e:
logger.error("Error inserting records")
logger.error(e)
return 0
finally:
s.close()
return len(records_to_store)
def validate_lastchanged(cls, value: Optional[str]) -> Optional[datetime]:
if not value or value == "":
return None
dt = parse_date(value)
if not dt:
logger.error("Not a valid date: {}".format(value))
raise ValueError("Not a valid lastchanged: {}".format(value))
return dt
def check_opennem_interval_delays(network_code: str) -> bool:
"""Runs periodically and alerts if there is a current delay in output of power intervals"""
network = network_from_network_code(network_code)
env = ""
if settings.debug:
env = ".dev"
url = f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json"
resp = http.get(url)
if resp.status_code != 200 or not resp.ok:
logger.error("Error retrieving: {}".format(url))
return False
resp_json = resp.json()
if "data" not in resp_json:
logger.error("Error retrieving wem power: malformed response")
return False
data = resp_json["data"]
fueltech_data = data.pop()
history_end_date = fueltech_data["history"]["last"]
history_date = parse_date(history_end_date, dayfirst=False)
if not history_date:
logger.error(
"Could not read history date for opennem interval monitor")
return False
now_date = datetime.now().astimezone(
network.get_timezone()) # type: ignore
time_delta = chop_delta_microseconds(now_date - history_date) - timedelta(
minutes=network.interval_size)
logger.debug("Live time: {}, delay: {}".format(history_date, time_delta))
alert_threshold = network.monitor_interval_alert_threshold or settings.monitor_interval_alert_threshold or 60
if time_delta > timedelta(minutes=alert_threshold):
slack_message(
f"*WARNING*: OpenNEM {network.code} interval delay on {settings.env} currently: {time_delta}.\n",
tag_users=settings.monitoring_alert_slack_user,
)
return True
class DispatchUnitSolutionSchema(MMSBase):
_interval_field = "settlementdate"
_primary_keys = ["settlementdate", "duid"]
settlementdate: datetime
duid: str
initialmw: Optional[float]
_validate_settlementdate = validator(
"settlementdate",
pre=True)(lambda x: parse_date(x, network=NetworkNEM))
_validate_duid = validator("duid")(normalize_duid)
_validate_initialmw = validator("initialmw")(clean_float)
class MMSBase(BaseModel):
_interval_field: Optional[str]
_primary_keys: Optional[List[str]] = None
class Config:
anystr_strip_whitespace = True
use_enum_values = True
arbitrary_types_allowed = True
validate_assignment = True
allow_population_by_field_name = True
alias_generator = mms_alias_generator
_validate_interval_datetime = validator(
"interval_datetime", pre=True, allow_reuse=True,
check_fields=False)(lambda x: parse_date(x, network=NetworkNEM))
_validate_settlementdate = validator(
"settlementdate", pre=True, allow_reuse=True,
check_fields=False)(lambda x: parse_date(x, network=NetworkNEM))
_validate_lastchanged = validator(
"lastchanged", pre=True, allow_reuse=True,
check_fields=False)(lambda x: parse_date(x, network=NetworkNEM))
def test_wem_settlementdate_tz(self) -> None:
subject = parse_date("2020/10/07 10:15:00",
dayfirst=False,
network=NetworkWEM)
comparator = datetime(2020,
10,
7,
10,
15,
0,
tzinfo=NetworkWEM.get_timezone())
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is True, "Date has timezone info"
def _validate_trading_interval(cls, value: Any) -> datetime:
interval_time = parse_date(
value,
dayfirst=False,
yearfirst=True,
)
if not interval_time:
raise Exception(f"Invalid APVI forecast interval: {value}")
# All APVI data is in NEM time
interval_time = interval_time.astimezone(
NetworkNEM.get_timezone()) # type: ignore
return interval_time
def check_opennem_interval_delays(network_code: str) -> bool:
network = network_from_network_code(network_code)
if settings.debug:
env = ".dev"
url = (
f"https://data.opennem.org.au/v3/stats/au/{network.code}/power/7d.json"
f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json"
)
resp = http.get(url)
if resp.status_code != 200 or not resp.ok:
logger.error("Error retrieving: {}".format(url))
return False
resp_json = resp.json()
if "data" not in resp_json:
logger.error("Error retrieving wem power: malformed response")
return False
data = resp_json["data"]
fueltech_data = data.pop()
history_end_date = fueltech_data["history"]["last"]
history_date = parse_date(history_end_date, dayfirst=False)
if not history_date:
logger.error(
"Could not read history date for opennem interval monitor")
return False
now_date = datetime.now().astimezone(network.get_timezone())
time_delta = chop_microseconds(now_date - history_date)
logger.debug("Live time: {}, delay: {}".format(history_date, time_delta))
if time_delta > timedelta(hours=3):
slack_message(
"*WARNING*: OpenNEM {} interval delay on {} currently: {}\n".
format(network.code, settings.env, time_delta))
return True
def process_meter_data_gen_duid(table):
session = SessionLocal()
engine = get_database_engine()
if "records" not in table:
raise Exception("Invalid table no records")
records = table["records"]
records_to_store = []
for record in records:
trading_interval = parse_date(
record["INTERVAL_DATETIME"], network=NetworkNEM, dayfirst=False
)
if not trading_interval:
continue
records_to_store.append(
{
"network_id": "NEM",
"trading_interval": trading_interval,
"facility_code": normalize_duid(record["DUID"]),
"eoi_quantity": record["MWH_READING"],
}
)
stmt = insert(FacilityScada).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
constraint="facility_scada_pkey",
set_={"eoi_quantity": stmt.excluded.eoi_quantity,},
)
try:
session.execute(stmt)
session.commit()
except Exception as e:
logger.error("Error inserting records")
logger.error(e)
return 0
finally:
session.close()
return len(records_to_store)
def process_pre_ap_price(table):
session = SessionLocal()
engine = get_database_engine()
if "records" not in table:
raise Exception("Invalid table no records")
records = table["records"]
records_to_store = []
for record in records:
trading_interval = parse_date(
record["SETTLEMENTDATE"], network=NetworkNEM, dayfirst=False
)
if not trading_interval:
continue
records_to_store.append(
{
"network_id": "NEM",
"network_region": record["REGIONID"],
"trading_interval": trading_interval,
"price": record["PRE_AP_ENERGY_PRICE"],
}
)
stmt = insert(BalancingSummary).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
constraint="balancing_summary_pkey",
set_={"price": stmt.excluded.price,},
)
try:
session.execute(stmt)
session.commit()
except Exception as e:
logger.error("Error inserting records")
logger.error(e)
return 0
finally:
session.close()
return len(records_to_store)
def process_item(self, item, spider=None):
session = SessionLocal()
engine = get_database_engine()
csvreader = csv.DictReader(item["content"].split("\n"))
records_to_store = []
for row in csvreader:
trading_interval = parse_date(
row["Trading Interval"], dayfirst=True, network=NetworkWEM
)
facility_code = normalize_duid(row["Facility Code"])
records_to_store.append(
{
"network_id": "WEM",
"trading_interval": trading_interval,
"facility_code": facility_code,
"eoi_quantity": row["EOI Quantity (MW)"] or None,
"generated": row["Energy Generated (MWh)"] or None,
}
)
stmt = insert(FacilityScada).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
constraint="facility_scada_pkey",
set_={
"eoi_quantity": stmt.excluded.eoi_quantity,
"generated": stmt.excluded.generated,
},
)
try:
session.execute(stmt)
session.commit()
except Exception as e:
logger.error("Error: {}".format(e))
finally:
session.close()
return len(records_to_store)
def test_nem_excel_formatted(self):
subject = parse_date("27/9/2019 2:55:00 pm")
comparator = datetime(2019, 9, 27, 14, 55, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def process_item(self, item, spider=None):
if "records" not in item:
logger.error("Invalid return response")
records = item["records"]
is_latest = False
record_date = None
if "meta" in item:
if "is_latest" in item["meta"]:
is_latest = item["meta"]["is_latest"]
if "record_date" in item["meta"]:
record_date = item["meta"]["record_date"]
if "postcode" not in records:
logger.error("No postcode data")
if "installations" not in records:
logger.error("No postcode data")
if "postcodeCapacity" not in records:
logger.error("No postcode capacity data")
postcode_gen = records["postcode"]
postcode_capacity = records["postcodeCapacity"]
installations = records["installations"]
engine = get_database_engine()
session = SessionLocal()
records_to_store = []
created_at = datetime.now()
created_by = ""
if spider and hasattr(spider, "name"):
created_by = spider.name
for record in postcode_gen:
for state, prefix in STATE_POSTCODE_PREFIXES.items():
facility_code = "{}_{}_{}".format(ROOFTOP_CODE, "apvi".upper(),
state.upper())
interval_time = parse_date(
record["ts"],
dayfirst=False,
yearfirst=True,
)
interval_time = interval_time.astimezone(
NetworkNEM.get_timezone())
generated = sum([
float(v) / 100 * postcode_capacity[k]
for k, v in record.items() if k.startswith(prefix) and v
and k in postcode_capacity and k[:2] not in WA_NON_SWIS
])
if not generated:
continue
__record = {
"created_by": created_by,
"created_at": created_at,
"network_id": "APVI",
"trading_interval": interval_time,
"facility_code": facility_code,
"generated": generated,
}
records_to_store.append(__record)
STATE_CAPACITIES = {}
if is_latest:
# temporariy only run getting capacities on latest
logger.info("Updating capacities on %s", record_date)
for postcode_prefix, capacity_val in postcode_capacity.items():
for state, prefix in STATE_POSTCODE_PREFIXES.items():
if state not in STATE_CAPACITIES:
STATE_CAPACITIES[state] = 0
if postcode_prefix.startswith(prefix):
STATE_CAPACITIES[state] += capacity_val
for state, state_capacity in STATE_CAPACITIES.items():
facility_code = "{}_{}_{}".format(ROOFTOP_CODE, "apvi".upper(),
state.upper())
state_facility: Facility = (session.query(Facility).filter_by(
code=facility_code).one_or_none())
if not state_facility:
raise Exception("Could not find rooftop facility for %s",
facility_code)
state_facility.capacity_registered = state_capacity
if state.lower() in installations:
state_number_units = installations[state.lower()]
state_facility.unit_number = state_number_units
session.add(state_facility)
session.commit()
if len(records_to_store) < 1:
return 0
stmt = insert(FacilityScada).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
index_elements=[
"trading_interval", "network_id", "facility_code",
"is_forecast"
],
set_={
"generated": stmt.excluded.generated,
"created_by": stmt.excluded.created_by,
},
)
try:
session.execute(stmt)
session.commit()
except Exception as e:
logger.error("Error: {}".format(e))
finally:
session.close()
return len(records_to_store)
def test_bom_date_utc(self):
subject = parse_date("20201008133000", dayfirst=False, is_utc=True)
comparator = datetime(2020, 10, 8, 13, 30, 0, 0, tzinfo=UTC)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is True, "Date has timezone info"
def test_nem_dispatch_interval(self):
subject = parse_date("30/9/19 4:00")
comparator = datetime(2019, 9, 30, 4, 0, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def unit_scada_generate_facility_scada(
records,
spider=None,
network: NetworkSchema = NetworkNEM,
interval_field: str = "SETTLEMENTDATE",
facility_code_field: str = "DUID",
date_format: Optional[str] = None,
power_field: Optional[str] = None,
energy_field: Optional[str] = None,
is_forecast: bool = False,
primary_key_track: bool = False,
groupby_filter: bool = True,
created_by: str = None,
limit: int = 0,
duid: str = None,
) -> List[Dict]:
created_at = datetime.now()
primary_keys = []
return_records = []
created_by = ""
if spider and hasattr(spider, "name"):
created_by = spider.name
for row in records:
trading_interval = parse_date(
row[interval_field],
network=network,
dayfirst=False,
date_format=date_format,
)
# if facility_code_field not in row:
# logger.error("Invalid row no facility_code")
# continue
facility_code = normalize_duid(row[facility_code_field])
if duid and facility_code != duid:
continue
if primary_key_track:
pkey = (trading_interval, facility_code)
if pkey in primary_keys:
continue
primary_keys.append(pkey)
generated = None
if power_field and power_field in row:
generated = clean_float(row[power_field])
if generated:
generated = float_to_str(generated)
energy = None
if energy_field and energy_field in row:
energy = clean_float(row[energy_field])
if energy:
energy = float_to_str(energy)
__rec = {
"created_by": created_by,
"created_at": created_at,
"updated_at": None,
"network_id": network.code,
"trading_interval": trading_interval,
"facility_code": facility_code,
"generated": generated,
"eoi_quantity": energy,
"is_forecast": is_forecast,
}
return_records.append(__rec)
if limit > 0 and len(return_records) >= limit:
break
if not groupby_filter:
return return_records
return_records_grouped = {}
for pk_values, rec_value in groupby(
return_records,
key=lambda r: (
r.get("network_id"),
r.get("trading_interval"),
r.get("facility_code"),
),
):
if pk_values not in return_records_grouped:
return_records_grouped[pk_values] = list(rec_value).pop()
return_records = list(return_records_grouped.values())
return return_records
def process_dispatch_interconnectorres(table: Dict, spider: Spider) -> Dict:
session = SessionLocal()
engine = get_database_engine()
if "records" not in table:
raise Exception("Invalid table no records")
records = table["records"]
records_to_store = []
for record in records:
ti_value = None
if "SETTLEMENTDATE" in record:
ti_value = record["SETTLEMENTDATE"]
if "RUN_DATETIME" in record:
ti_value = record["RUN_DATETIME"]
if not ti_value:
raise Exception("Require a trading interval")
trading_interval = parse_date(ti_value,
network=NetworkNEM,
dayfirst=False)
if not trading_interval:
continue
facility_code = normalize_duid(record["INTERCONNECTORID"])
power_value = clean_float(record["METEREDMWFLOW"])
records_to_store.append({
"network_id": "NEM",
"created_by": spider.name,
"facility_code": facility_code,
"trading_interval": trading_interval,
"generated": power_value,
})
# remove duplicates
return_records_grouped = {}
for pk_values, rec_value in groupby(
records_to_store,
key=lambda r: (
r.get("trading_interval"),
r.get("network_id"),
r.get("facility_code"),
),
):
if pk_values not in return_records_grouped:
return_records_grouped[pk_values] = list(rec_value).pop()
records_to_store = list(return_records_grouped.values())
# insert
stmt = insert(FacilityScada).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
index_elements=[
"trading_interval", "network_id", "facility_code", "is_forecast"
],
set_={"generated": stmt.excluded.generated},
)
try:
session.execute(stmt)
session.commit()
except Exception as e:
logger.error("Error inserting records")
logger.error(e)
return {"num_records": 0}
finally:
session.close()
return {"num_records": len(records_to_store)}
def test_nem_dispatch_scada_interval(self):
subject = parse_date("2020/06/01 21:35:00", dayfirst=False)
comparator = datetime(2020, 6, 1, 21, 35, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def process_trading_regionsum(table: Dict[str, Any], spider: Spider) -> Dict:
session = SessionLocal()
engine = get_database_engine()
if "records" not in table:
raise Exception("Invalid table no records")
records = table["records"]
limit = None
records_to_store = []
records_processed = 0
primary_keys = []
for record in records:
trading_interval = parse_date(
record["SETTLEMENTDATE"],
network=NetworkNEM,
dayfirst=False,
date_format="%Y/%m/%d %H:%M:%S",
)
if not trading_interval:
continue
_pk = set([trading_interval, record["REGIONID"]])
if _pk in primary_keys:
continue
primary_keys.append(_pk)
demand_total = None
if "TOTALDEMAND" in record:
demand_total = clean_float(record["TOTALDEMAND"])
records_to_store.append({
"network_id": "NEM",
"created_by": spider.name,
"network_region": record["REGIONID"],
"trading_interval": trading_interval,
"demand_total": demand_total,
})
records_processed += 1
if limit and records_processed >= limit:
logger.info("Reached limit of: {} {}".format(
limit, records_processed))
break
stmt = insert(BalancingSummary).values(records_to_store)
stmt.bind = engine
stmt = stmt.on_conflict_do_update(
index_elements=["trading_interval", "network_id", "network_region"],
set_={
"demand_total": stmt.excluded.demand_total,
},
)
try:
session.execute(stmt)
session.commit()
except Exception as e:
logger.error("Error inserting records")
logger.error(e)
return {"num_records": 0}
finally:
session.close()
return {"num_records": len(records_to_store)}
def test_nem_settlementdate(self):
subject = parse_date("2020/10/07 10:15:00", dayfirst=False)
comparator = datetime(2020, 10, 7, 10, 15, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
