def get_aemo_wem_live_facility_intervals_recent_date() -> datetime:
req = http.get(LIVE_FACILITIES, headers=REQ_HEADERS)
if req.status_code != 200:
logger.error(
"WEM live facility intervals returning non 200: {} {}".format(
LIVE_FACILITIES, req.status_code))
csv_content = req.content
csvreader = csv.DictReader(csv_content.decode("utf-8").split("\n"))
if not csvreader.fieldnames or len(csvreader.fieldnames) < 1:
logger.error(
"WEM live facility intervals returning bad CSV: {}".format(
LIVE_FACILITIES))
records = unit_scada_generate_facility_scada(
records=csvreader,
interval_field="PERIOD",
facility_code_field="FACILITY_CODE",
power_field="ACTUAL_MW",
network=NetworkWEM,
)
max_date = max([i["trading_interval"] for i in records])
return max_date
def process_item(self, item, spider=None):
if "content" not in item:
logger.error("No item content slipping store facility scada")
return item
csvreader = csv.DictReader(item["content"].split("\n"))
item["table_schema"] = FacilityScada
item["update_fields"] = ["generated", "eoi_quantity"]
item["records"] = unit_scada_generate_facility_scada(
csvreader,
spider,
interval_field="PERIOD",
facility_code_field="FACILITY_CODE",
power_field="ACTUAL_MW",
network=NetworkWEM,
)
item["content"] = None
return item
def process_item(self, item, spider=None):
if "content" not in item:
logger.error("No item content slipping store facility scada")
return item
csvreader = csv.DictReader(item["content"].split("\n"))
item["table_schema"] = FacilityScada
item["update_fields"] = ["generated", "eoi_quantity"]
item["records"] = unit_scada_generate_facility_scada(
csvreader,
spider,
interval_field="Trading Interval",
facility_code_field="Facility Code",
power_field="EOI Quantity (MW)",
energy_field="Energy Generated (MWh)",
network=NetworkWEM,
)
item["content"] = None
return item