def validate_effectivedate(cls, value: str) -> datetime: dt = parse_date(value) if not dt: raise ValueError("Not a valid date: {}".format(value)) return dt
def parse_record_bom(rec: Dict) -> Dict: """Parse CSV records in BOM format""" timezone = pytz.timezone(STATION_TIMEZONE_MAP[rec["station_id"]]) observation_time = parse_date(rec["observation_time"], is_utc=True, dayfirst=False).astimezone(timezone) return { "observation_time": observation_time, "station_id": rec["station_id"], "temp_apparent": float(rec["temp_apparent"].strip()) if is_number(rec["temp_apparent"]) else None, "temp_air": float(rec["temp_air"].strip()) if is_number(rec["temp_air"]) else None, "press_qnh": None, "wind_dir": None, "wind_spd": None, "cloud": None, "cloud_type": None, "humidity": float(rec["humidity"].strip()) if is_number(rec["humidity"]) else None, "wind_gust": None, }
def parse_dirlisting(raw_string: str) -> Dict[str, Any]: """ given a raw text directory listing like " Saturday 11th June 2020 6789" will parse and return both the date and listing type @param raw_string - the raw directory listing string @return dict of the date in iso format and the type (file or directory) """ components = raw_string.split(" " * PADDING_WIDTH) components = [i.strip() for i in components] components = list(filter(lambda x: x != "", components)) _ltype = "dir" if not components or len(components) < 2: logging.debug(components) raise Exception("Invalid line string: {}. Components are: {}".format( raw_string, components)) if is_number(components[1]): _ltype = "file" dt = parse_date(components[0], network=NetworkNEM) if type(dt) is not datetime: raise Exception( "{} is not a valid datetime. Original value was {}".format( dt, components[0])) return { "date": dt.isoformat(), "type": _ltype, }
def validate_date(date_value: str) -> datetime: dt = parse_date(date_value) if not dt: raise ValueError("Not a valid date: {}".format(date_value)) return dt
def process_unit_solution(table): session = SessionLocal() engine = get_database_engine() if "records" not in table: raise Exception("Invalid table no records") records = table["records"] records_to_store = [] records_primary_keys = [] for record in records: trading_interval = parse_date( record["SETTLEMENTDATE"], network=NetworkNEM, dayfirst=False ) facility_code = normalize_duid(record["DUID"]) if not trading_interval or not facility_code: continue # Since this can insert 1M+ records at a time we need to # do a separate in-memory check of primary key constraints # better way of doing this .. @TODO _unique_set = (trading_interval, facility_code, "NEM") if _unique_set not in records_primary_keys: records_to_store.append( { "trading_interval": trading_interval, "facility_code": facility_code, "eoi_quantity": float(record["INITIALMW"]), "network_id": "NEM", } ) records_primary_keys.append(_unique_set) # free records_primary_keys = [] logger.debug("Saving %d records", len(records_to_store)) stmt = insert(FacilityScada).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( constraint="facility_scada_pkey", set_={"eoi_quantity": stmt.excluded.eoi_quantity}, ) try: session.execute(stmt) session.commit() except Exception as e: logger.error("Error: {}".format(e)) return 0 finally: session.close() return len(records_to_store)
def parse(self, response: Any) -> Dict[str, Any]: file_downloads = [] source_title = response.css("title::text").get() download_sections = response.xpath( "//div[@class='file-list-wrapper']/..") if not download_sections or len(download_sections) < 1: raise Exception( "{} spider could not find any download sections".format( self.name)) for download_section in download_sections: date_text = download_section.css( "div.field-publisheddate span::text").get() if not date_text: raise Exception( "{} could not get download section published date".format( self.name)) published_date = parse_date(date_text) publish_link_relative = download_section.css("a::attr(href)").get() if not publish_link_relative: raise Exception("{} could not get rel published link".format( self.name)) publish_link = response.urljoin(publish_link_relative) publish_link = strip_query_string(publish_link) download_title = download_section.css(".field-title::text").get() download_size_raw = download_section.css( ".field-size span::text").get() download_size = None if download_size_raw: download_size, _ = filesize_from_string(download_size_raw) # create a model from the extracted fields section_model = None try: section_model = AEMOFileDownloadSection( published_date=published_date, filename=download_title, download_url=publish_link, file_size=download_size, source_url=response.url, source_title=source_title, ) file_downloads.append(section_model) except ValidationError as e: self.log("Validation error: {}".format(e), logging.ERROR) return {"_data": file_downloads, "items": file_downloads}
def process_nem_price(table: AEMOTableSchema) -> ControllerReturn: """Stores the NEM price for both dispatch price and trading price""" session = get_scoped_session() engine = get_database_engine() cr = ControllerReturn(total_records=len(table.records)) records_to_store = [] primary_keys = [] price_field = "price" if table.full_name == "dispatch_price": price_field = "price_dispatch" for record in table.records: # @NOTE disable pk track trading_interval = parse_date(record["settlementdate"]) primary_key = set([trading_interval, record["regionid"]]) # type: ignore if primary_key in primary_keys: continue primary_keys.append(primary_key) records_to_store.append({ "network_id": "NEM", "created_by": "opennem.controllers.nem", "network_region": record["regionid"], "trading_interval": trading_interval, price_field: record["rrp"], }) cr.processed_records += 1 stmt = insert(BalancingSummary).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( index_elements=["trading_interval", "network_id", "network_region"], set_={price_field: getattr(stmt.excluded, price_field)}, ) try: session.execute(stmt) session.commit() cr.inserted_records = cr.processed_records cr.server_latest = max( [i["trading_interval"] for i in records_to_store]) except Exception as e: logger.error("Error inserting NEM price records") logger.error(e) cr.errors = cr.processed_records finally: session.rollback() session.close() engine.dispose() return cr
def validate_authoriseddate(cls, value: str) -> datetime: dt = parse_date(value) if not dt: logger.error("Not a valid date: {}".format(value)) raise ValueError("Not a valid authoriseddate: {}".format(value)) return dt
def generate_balancing_summary( records: List[Dict], interval_field: str = "SETTLEMENTDATE", network_region_field: str = "REGIONID", price_field: Optional[str] = None, network: NetworkSchema = NetworkNEM, limit: int = 0, ) -> List[Dict]: created_at = datetime.now() primary_keys = [] return_records = [] created_by = "" for row in records: trading_interval = parse_date(row[interval_field], network=network, dayfirst=False) network_region = None if network_region_field and network_region_field in row: network_region = row[network_region_field] pk = (trading_interval, network.code, network_region) if pk in primary_keys: continue primary_keys.append(pk) price = None if price_field and price_field in row: price = clean_float(row[price_field]) if price: price = str(float_to_str(price)) __rec = { "created_by": created_by, "created_at": created_at, "updated_at": None, "network_id": network.code, "network_region": network_region, "trading_interval": trading_interval, "price": price, } return_records.append(__rec) if limit > 0 and len(return_records) >= limit: break return return_records
def process_item(self, item, spider=None): s = self.session() csvreader = csv.DictReader(item["content"].split("\n")) records_to_store = [] for record in csvreader: trading_interval = parse_date(record["Trading Interval"], dayfirst=True, network=NetworkWEM) if not trading_interval: continue records_to_store.append({ "network_id": "WEM", "network_region": "WEM", "trading_interval": trading_interval, "forecast_load": record["Load Forecast (MW)"], "generation_scheduled": record["Scheduled Generation (MW)"], "generation_non_scheduled": record["Non-Scheduled Generation (MW)"], "generation_total": record["Total Generation (MW)"], "price": record["Final Price ($/MWh)"], }) stmt = insert(BalancingSummary).values(records_to_store) stmt.bind = self.engine stmt = stmt.on_conflict_do_update( constraint="balancing_summary_pkey", set_={ "price": stmt.excluded.price, "generation_total": stmt.excluded.generation_total, }, ) try: r = s.execute(stmt) s.commit() except Exception as e: logger.error("Error inserting records") logger.error(e) finally: s.close() return len(records_to_store)
def process_item(self, item, spider=None): s = SessionLocal() csvreader = csv.DictReader(item["content"].split("\n")) records_to_store = [] primary_keys = [] for row in csvreader: trading_interval = parse_date(row["TRADING_DAY_INTERVAL"], network=NetworkWEM, dayfirst=False) if trading_interval not in primary_keys: forecast_load = clean_float(row["FORECAST_EOI_MW"]) records_to_store.append({ "created_by": spider.name, "trading_interval": trading_interval, "network_id": "WEM", "network_region": "WEM", "forecast_load": forecast_load, # generation_scheduled=row["Scheduled Generation (MW)"], # generation_total=row["Total Generation (MW)"], "price": clean_float(row["PRICE"]), }) primary_keys.append(trading_interval) stmt = insert(BalancingSummary).values(records_to_store) stmt.bind = get_database_engine() stmt = stmt.on_conflict_do_update( index_elements=[ "trading_interval", "network_id", "network_region", ], set_={ "price": stmt.excluded.price, "forecast_load": stmt.excluded.forecast_load, }, ) try: s.execute(stmt) s.commit() except Exception as e: logger.error("Error inserting records") logger.error(e) return 0 finally: s.close() return len(records_to_store)
def validate_lastchanged(cls, value: Optional[str]) -> Optional[datetime]: if not value or value == "": return None dt = parse_date(value) if not dt: logger.error("Not a valid date: {}".format(value)) raise ValueError("Not a valid lastchanged: {}".format(value)) return dt
def check_opennem_interval_delays(network_code: str) -> bool: """Runs periodically and alerts if there is a current delay in output of power intervals""" network = network_from_network_code(network_code) env = "" if settings.debug: env = ".dev" url = f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json" resp = http.get(url) if resp.status_code != 200 or not resp.ok: logger.error("Error retrieving: {}".format(url)) return False resp_json = resp.json() if "data" not in resp_json: logger.error("Error retrieving wem power: malformed response") return False data = resp_json["data"] fueltech_data = data.pop() history_end_date = fueltech_data["history"]["last"] history_date = parse_date(history_end_date, dayfirst=False) if not history_date: logger.error( "Could not read history date for opennem interval monitor") return False now_date = datetime.now().astimezone( network.get_timezone()) # type: ignore time_delta = chop_delta_microseconds(now_date - history_date) - timedelta( minutes=network.interval_size) logger.debug("Live time: {}, delay: {}".format(history_date, time_delta)) alert_threshold = network.monitor_interval_alert_threshold or settings.monitor_interval_alert_threshold or 60 if time_delta > timedelta(minutes=alert_threshold): slack_message( f"*WARNING*: OpenNEM {network.code} interval delay on {settings.env} currently: {time_delta}.\n", tag_users=settings.monitoring_alert_slack_user, ) return True
class DispatchUnitSolutionSchema(MMSBase): _interval_field = "settlementdate" _primary_keys = ["settlementdate", "duid"] settlementdate: datetime duid: str initialmw: Optional[float] _validate_settlementdate = validator( "settlementdate", pre=True)(lambda x: parse_date(x, network=NetworkNEM)) _validate_duid = validator("duid")(normalize_duid) _validate_initialmw = validator("initialmw")(clean_float)
class MMSBase(BaseModel): _interval_field: Optional[str] _primary_keys: Optional[List[str]] = None class Config: anystr_strip_whitespace = True use_enum_values = True arbitrary_types_allowed = True validate_assignment = True allow_population_by_field_name = True alias_generator = mms_alias_generator _validate_interval_datetime = validator( "interval_datetime", pre=True, allow_reuse=True, check_fields=False)(lambda x: parse_date(x, network=NetworkNEM)) _validate_settlementdate = validator( "settlementdate", pre=True, allow_reuse=True, check_fields=False)(lambda x: parse_date(x, network=NetworkNEM)) _validate_lastchanged = validator( "lastchanged", pre=True, allow_reuse=True, check_fields=False)(lambda x: parse_date(x, network=NetworkNEM))
def test_wem_settlementdate_tz(self) -> None: subject = parse_date("2020/10/07 10:15:00", dayfirst=False, network=NetworkWEM) comparator = datetime(2020, 10, 7, 10, 15, 0, tzinfo=NetworkWEM.get_timezone()) assert subject == comparator, "Parses date correctly" assert is_aware(subject) is True, "Date has timezone info"
def _validate_trading_interval(cls, value: Any) -> datetime: interval_time = parse_date( value, dayfirst=False, yearfirst=True, ) if not interval_time: raise Exception(f"Invalid APVI forecast interval: {value}") # All APVI data is in NEM time interval_time = interval_time.astimezone( NetworkNEM.get_timezone()) # type: ignore return interval_time
def check_opennem_interval_delays(network_code: str) -> bool: network = network_from_network_code(network_code) if settings.debug: env = ".dev" url = ( f"https://data.opennem.org.au/v3/stats/au/{network.code}/power/7d.json" f"https://data{env}.opennem.org.au/v3/stats/au/{network.code}/power/7d.json" ) resp = http.get(url) if resp.status_code != 200 or not resp.ok: logger.error("Error retrieving: {}".format(url)) return False resp_json = resp.json() if "data" not in resp_json: logger.error("Error retrieving wem power: malformed response") return False data = resp_json["data"] fueltech_data = data.pop() history_end_date = fueltech_data["history"]["last"] history_date = parse_date(history_end_date, dayfirst=False) if not history_date: logger.error( "Could not read history date for opennem interval monitor") return False now_date = datetime.now().astimezone(network.get_timezone()) time_delta = chop_microseconds(now_date - history_date) logger.debug("Live time: {}, delay: {}".format(history_date, time_delta)) if time_delta > timedelta(hours=3): slack_message( "*WARNING*: OpenNEM {} interval delay on {} currently: {}\n". format(network.code, settings.env, time_delta)) return True
def process_meter_data_gen_duid(table): session = SessionLocal() engine = get_database_engine() if "records" not in table: raise Exception("Invalid table no records") records = table["records"] records_to_store = [] for record in records: trading_interval = parse_date( record["INTERVAL_DATETIME"], network=NetworkNEM, dayfirst=False ) if not trading_interval: continue records_to_store.append( { "network_id": "NEM", "trading_interval": trading_interval, "facility_code": normalize_duid(record["DUID"]), "eoi_quantity": record["MWH_READING"], } ) stmt = insert(FacilityScada).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( constraint="facility_scada_pkey", set_={"eoi_quantity": stmt.excluded.eoi_quantity,}, ) try: session.execute(stmt) session.commit() except Exception as e: logger.error("Error inserting records") logger.error(e) return 0 finally: session.close() return len(records_to_store)
def process_pre_ap_price(table): session = SessionLocal() engine = get_database_engine() if "records" not in table: raise Exception("Invalid table no records") records = table["records"] records_to_store = [] for record in records: trading_interval = parse_date( record["SETTLEMENTDATE"], network=NetworkNEM, dayfirst=False ) if not trading_interval: continue records_to_store.append( { "network_id": "NEM", "network_region": record["REGIONID"], "trading_interval": trading_interval, "price": record["PRE_AP_ENERGY_PRICE"], } ) stmt = insert(BalancingSummary).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( constraint="balancing_summary_pkey", set_={"price": stmt.excluded.price,}, ) try: session.execute(stmt) session.commit() except Exception as e: logger.error("Error inserting records") logger.error(e) return 0 finally: session.close() return len(records_to_store)
def process_item(self, item, spider=None): session = SessionLocal() engine = get_database_engine() csvreader = csv.DictReader(item["content"].split("\n")) records_to_store = [] for row in csvreader: trading_interval = parse_date( row["Trading Interval"], dayfirst=True, network=NetworkWEM ) facility_code = normalize_duid(row["Facility Code"]) records_to_store.append( { "network_id": "WEM", "trading_interval": trading_interval, "facility_code": facility_code, "eoi_quantity": row["EOI Quantity (MW)"] or None, "generated": row["Energy Generated (MWh)"] or None, } ) stmt = insert(FacilityScada).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( constraint="facility_scada_pkey", set_={ "eoi_quantity": stmt.excluded.eoi_quantity, "generated": stmt.excluded.generated, }, ) try: session.execute(stmt) session.commit() except Exception as e: logger.error("Error: {}".format(e)) finally: session.close() return len(records_to_store)
def test_nem_excel_formatted(self): subject = parse_date("27/9/2019 2:55:00 pm") comparator = datetime(2019, 9, 27, 14, 55, 0) assert subject == comparator, "Parses date correctly" assert is_aware(subject) is False, "Date has no timezone info"
def process_item(self, item, spider=None): if "records" not in item: logger.error("Invalid return response") records = item["records"] is_latest = False record_date = None if "meta" in item: if "is_latest" in item["meta"]: is_latest = item["meta"]["is_latest"] if "record_date" in item["meta"]: record_date = item["meta"]["record_date"] if "postcode" not in records: logger.error("No postcode data") if "installations" not in records: logger.error("No postcode data") if "postcodeCapacity" not in records: logger.error("No postcode capacity data") postcode_gen = records["postcode"] postcode_capacity = records["postcodeCapacity"] installations = records["installations"] engine = get_database_engine() session = SessionLocal() records_to_store = [] created_at = datetime.now() created_by = "" if spider and hasattr(spider, "name"): created_by = spider.name for record in postcode_gen: for state, prefix in STATE_POSTCODE_PREFIXES.items(): facility_code = "{}_{}_{}".format(ROOFTOP_CODE, "apvi".upper(), state.upper()) interval_time = parse_date( record["ts"], dayfirst=False, yearfirst=True, ) interval_time = interval_time.astimezone( NetworkNEM.get_timezone()) generated = sum([ float(v) / 100 * postcode_capacity[k] for k, v in record.items() if k.startswith(prefix) and v and k in postcode_capacity and k[:2] not in WA_NON_SWIS ]) if not generated: continue __record = { "created_by": created_by, "created_at": created_at, "network_id": "APVI", "trading_interval": interval_time, "facility_code": facility_code, "generated": generated, } records_to_store.append(__record) STATE_CAPACITIES = {} if is_latest: # temporariy only run getting capacities on latest logger.info("Updating capacities on %s", record_date) for postcode_prefix, capacity_val in postcode_capacity.items(): for state, prefix in STATE_POSTCODE_PREFIXES.items(): if state not in STATE_CAPACITIES: STATE_CAPACITIES[state] = 0 if postcode_prefix.startswith(prefix): STATE_CAPACITIES[state] += capacity_val for state, state_capacity in STATE_CAPACITIES.items(): facility_code = "{}_{}_{}".format(ROOFTOP_CODE, "apvi".upper(), state.upper()) state_facility: Facility = (session.query(Facility).filter_by( code=facility_code).one_or_none()) if not state_facility: raise Exception("Could not find rooftop facility for %s", facility_code) state_facility.capacity_registered = state_capacity if state.lower() in installations: state_number_units = installations[state.lower()] state_facility.unit_number = state_number_units session.add(state_facility) session.commit() if len(records_to_store) < 1: return 0 stmt = insert(FacilityScada).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( index_elements=[ "trading_interval", "network_id", "facility_code", "is_forecast" ], set_={ "generated": stmt.excluded.generated, "created_by": stmt.excluded.created_by, }, ) try: session.execute(stmt) session.commit() except Exception as e: logger.error("Error: {}".format(e)) finally: session.close() return len(records_to_store)
def test_bom_date_utc(self): subject = parse_date("20201008133000", dayfirst=False, is_utc=True) comparator = datetime(2020, 10, 8, 13, 30, 0, 0, tzinfo=UTC) assert subject == comparator, "Parses date correctly" assert is_aware(subject) is True, "Date has timezone info"
def test_nem_dispatch_interval(self): subject = parse_date("30/9/19 4:00") comparator = datetime(2019, 9, 30, 4, 0, 0) assert subject == comparator, "Parses date correctly" assert is_aware(subject) is False, "Date has no timezone info"
def unit_scada_generate_facility_scada( records, spider=None, network: NetworkSchema = NetworkNEM, interval_field: str = "SETTLEMENTDATE", facility_code_field: str = "DUID", date_format: Optional[str] = None, power_field: Optional[str] = None, energy_field: Optional[str] = None, is_forecast: bool = False, primary_key_track: bool = False, groupby_filter: bool = True, created_by: str = None, limit: int = 0, duid: str = None, ) -> List[Dict]: created_at = datetime.now() primary_keys = [] return_records = [] created_by = "" if spider and hasattr(spider, "name"): created_by = spider.name for row in records: trading_interval = parse_date( row[interval_field], network=network, dayfirst=False, date_format=date_format, ) # if facility_code_field not in row: # logger.error("Invalid row no facility_code") # continue facility_code = normalize_duid(row[facility_code_field]) if duid and facility_code != duid: continue if primary_key_track: pkey = (trading_interval, facility_code) if pkey in primary_keys: continue primary_keys.append(pkey) generated = None if power_field and power_field in row: generated = clean_float(row[power_field]) if generated: generated = float_to_str(generated) energy = None if energy_field and energy_field in row: energy = clean_float(row[energy_field]) if energy: energy = float_to_str(energy) __rec = { "created_by": created_by, "created_at": created_at, "updated_at": None, "network_id": network.code, "trading_interval": trading_interval, "facility_code": facility_code, "generated": generated, "eoi_quantity": energy, "is_forecast": is_forecast, } return_records.append(__rec) if limit > 0 and len(return_records) >= limit: break if not groupby_filter: return return_records return_records_grouped = {} for pk_values, rec_value in groupby( return_records, key=lambda r: ( r.get("network_id"), r.get("trading_interval"), r.get("facility_code"), ), ): if pk_values not in return_records_grouped: return_records_grouped[pk_values] = list(rec_value).pop() return_records = list(return_records_grouped.values()) return return_records
def process_dispatch_interconnectorres(table: Dict, spider: Spider) -> Dict: session = SessionLocal() engine = get_database_engine() if "records" not in table: raise Exception("Invalid table no records") records = table["records"] records_to_store = [] for record in records: ti_value = None if "SETTLEMENTDATE" in record: ti_value = record["SETTLEMENTDATE"] if "RUN_DATETIME" in record: ti_value = record["RUN_DATETIME"] if not ti_value: raise Exception("Require a trading interval") trading_interval = parse_date(ti_value, network=NetworkNEM, dayfirst=False) if not trading_interval: continue facility_code = normalize_duid(record["INTERCONNECTORID"]) power_value = clean_float(record["METEREDMWFLOW"]) records_to_store.append({ "network_id": "NEM", "created_by": spider.name, "facility_code": facility_code, "trading_interval": trading_interval, "generated": power_value, }) # remove duplicates return_records_grouped = {} for pk_values, rec_value in groupby( records_to_store, key=lambda r: ( r.get("trading_interval"), r.get("network_id"), r.get("facility_code"), ), ): if pk_values not in return_records_grouped: return_records_grouped[pk_values] = list(rec_value).pop() records_to_store = list(return_records_grouped.values()) # insert stmt = insert(FacilityScada).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( index_elements=[ "trading_interval", "network_id", "facility_code", "is_forecast" ], set_={"generated": stmt.excluded.generated}, ) try: session.execute(stmt) session.commit() except Exception as e: logger.error("Error inserting records") logger.error(e) return {"num_records": 0} finally: session.close() return {"num_records": len(records_to_store)}
def test_nem_dispatch_scada_interval(self): subject = parse_date("2020/06/01 21:35:00", dayfirst=False) comparator = datetime(2020, 6, 1, 21, 35, 0) assert subject == comparator, "Parses date correctly" assert is_aware(subject) is False, "Date has no timezone info"
def process_trading_regionsum(table: Dict[str, Any], spider: Spider) -> Dict: session = SessionLocal() engine = get_database_engine() if "records" not in table: raise Exception("Invalid table no records") records = table["records"] limit = None records_to_store = [] records_processed = 0 primary_keys = [] for record in records: trading_interval = parse_date( record["SETTLEMENTDATE"], network=NetworkNEM, dayfirst=False, date_format="%Y/%m/%d %H:%M:%S", ) if not trading_interval: continue _pk = set([trading_interval, record["REGIONID"]]) if _pk in primary_keys: continue primary_keys.append(_pk) demand_total = None if "TOTALDEMAND" in record: demand_total = clean_float(record["TOTALDEMAND"]) records_to_store.append({ "network_id": "NEM", "created_by": spider.name, "network_region": record["REGIONID"], "trading_interval": trading_interval, "demand_total": demand_total, }) records_processed += 1 if limit and records_processed >= limit: logger.info("Reached limit of: {} {}".format( limit, records_processed)) break stmt = insert(BalancingSummary).values(records_to_store) stmt.bind = engine stmt = stmt.on_conflict_do_update( index_elements=["trading_interval", "network_id", "network_region"], set_={ "demand_total": stmt.excluded.demand_total, }, ) try: session.execute(stmt) session.commit() except Exception as e: logger.error("Error inserting records") logger.error(e) return {"num_records": 0} finally: session.close() return {"num_records": len(records_to_store)}
def test_nem_settlementdate(self): subject = parse_date("2020/10/07 10:15:00", dayfirst=False) comparator = datetime(2020, 10, 7, 10, 15, 0) assert subject == comparator, "Parses date correctly" assert is_aware(subject) is False, "Date has no timezone info"