def test_wem_settlementdate_tz(self) -> None:
subject = parse_date("2020/10/07 10:15:00",
dayfirst=False,
network=NetworkWEM)
comparator = datetime(2020,
10,
7,
10,
15,
0,
tzinfo=NetworkWEM.get_timezone())
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is True, "Date has timezone info"
def stats_factory(
stats: List[DataQueryResult],
units: UnitDefinition,
interval: TimeInterval,
period: Optional[TimePeriod] = None,
network: Optional[NetworkSchema] = None,
timezone: Optional[Union[timezone, str]] = None,
code: Optional[str] = None,
region: Optional[str] = None,
include_group_code: bool = False,
fueltech_group: Optional[bool] = False,
group_field: Optional[str] = None,
data_id: Optional[str] = None,
localize: Optional[bool] = True,
include_code: Optional[bool] = True,
) -> Optional[OpennemDataSet]:
"""
Takes a list of data query results and returns OpennemDataSets
@TODO optional groupby field
@TODO multiple groupings / slight refactor
"""
if network:
timezone = network.get_timezone()
group_codes = list(set([i.group_by for i in stats if i.group_by]))
stats_grouped = []
for group_code in group_codes:
data_grouped: Dict[datetime, Any] = dict()
for stat in stats:
if stat.group_by != group_code:
continue
if stat.interval not in data_grouped:
data_grouped[stat.interval] = None
# if stat.result:
data_grouped[stat.interval] = stat.result
data_sorted = OrderedDict(sorted(data_grouped.items()))
data_value = list(data_sorted.values())
# Skip null series
if len([i for i in data_value if i]) == 0:
continue
# @TODO possible bring this back
# Skip zero series
# if sum([i for i in data_value if i]) == 0:
# continue
# Cast trailing nulls
if not units.name.startswith("temperature") or units.cast_nulls:
data_value = cast_trailing_nulls(data_value)
# Find start/end dates
dates = list(data_grouped.keys())
if not dates:
return None
start = min(dates)
end = max(dates)
# should probably make sure these are the same TZ
if localize:
if timezone and not is_aware(start):
start = make_aware(start, timezone)
if timezone and not is_aware(end):
end = make_aware(end, timezone)
if timezone and localize and network and network.offset:
tz = pytz.FixedOffset(int(network.offset))
start = start.astimezone(tz)
end = end.astimezone(tz)
# Everything needs a timezone even flat dates
if network and timezone and not is_aware(start):
start = start.replace(tzinfo=network.get_fixed_offset())
if network and timezone and not is_aware(end):
end = end.replace(tzinfo=network.get_fixed_offset())
# free
dates = []
history = OpennemDataHistory(
start=start,
last=end,
interval=interval.interval_human,
data=data_value,
)
data = OpennemData(
data_type=units.unit_type,
units=units.unit,
# interval=interval,
# period=period,
history=history,
)
if include_code:
data.code = group_code
if network:
data.network = network.code.lower()
# *sigh* - not the most flexible model
# @TODO fix this schema and make it more flexible
if fueltech_group:
data.fuel_tech = group_code
data_comps = [
# @NOTE disable for now since FE doesn't
# support it
network.country if network else None,
network.code.lower() if network else None,
region.lower()
if region and region.lower() != network.code.lower() else None,
"fuel_tech",
group_code,
units.unit_type,
]
data.id = ".".join(i for i in data_comps if i)
# @TODO make this an alias
data.type = units.unit_type
if group_field:
group_fields = []
# setattr(data, group_field, group_code)
if network:
group_fields.append(network.country.lower())
group_fields.append(network.code.lower())
if region:
if region.lower() != network.code.lower():
group_fields.append(region.lower())
if units.name_alias:
group_fields.append(units.name_alias)
elif units.unit_type:
group_fields.append(units.unit_type)
if group_code and include_group_code:
group_fields.append(group_code)
group_fields.append(group_field)
data.id = ".".join([f for f in group_fields if f])
data.type = units.unit_type
if data_id:
data.id = data_id
if not data.id:
_id_list = []
# @NOTE disable for now since FE doesn't
# support it
# network.country if network else None,
if network:
_id_list.append(network.code.lower())
if region and (region.lower() != network.code.lower()):
_id_list.append(region.lower())
if group_code:
_id_list.append(group_code.lower())
if units and units.name_alias:
_id_list.append(units.name_alias)
elif units and units.name:
_id_list.append(units.name)
data.id = ".".join([f for f in _id_list if f])
data.type = units.unit_type
if region:
data.region = region
stats_grouped.append(data)
dt_now = datetime.now()
if network:
dt_now = dt_now.astimezone(network.get_timezone())
# @NOTE this should probably be
# country.network.region
if not code:
if network:
code = network.code
if region:
code = region
stat_set = OpennemDataSet(
type=units.unit_type,
data=stats_grouped,
created_at=dt_now,
version=get_version(),
)
if include_code:
stat_set.code = code
if network:
stat_set.network = network.code
if region:
stat_set.region = region
return stat_set
def timesince(
d: Optional[datetime] = None,
now: Optional[datetime] = None,
reversed: bool = False,
depth: int = 2,
none_value: str = "Never",
) -> str:
"""
Take two datetime objects and return the time between d and now as a nicely
formatted string, e.g. "10 minutes". If d occurs after now, return
"0 minutes".
Units used are years, months, weeks, days, hours, and minutes.
Seconds and microseconds are ignored. Up to `depth` adjacent units will be
displayed. For example, "2 weeks, 3 days" and "1 year, 3 months" are
possible outputs, but "2 weeks, 3 hours" and "1 year, 5 days" are not.
`time_strings` is an optional dict of strings to replace the default
TIME_STRINGS dict.
`depth` is an optional integer to control the number of adjacent time
units returned.
Adapted from
https://web.archive.org/web/20060617175230/http://blog.natbat.co.uk/archive/2003/Jun/14/time_since
"""
time_strings = TIME_STRINGS
if not d:
return none_value
if depth <= 0:
raise ValueError("depth must be greater than 0.")
# Convert date to datetime for comparison.
if not isinstance(d, datetime):
d = datetime(d.year, d.month, d.day)
if now and not isinstance(now, datetime):
now = datetime(now.year, now.month, now.day)
now = now or datetime.now(timezone.utc if is_aware(d) else None)
if reversed:
d, now = now, d
delta = now - d
# Deal with leapyears by subtracing the number of leapdays
leapdays = calendar.leapdays(d.year, now.year)
if leapdays != 0:
if calendar.isleap(d.year):
leapdays -= 1
elif calendar.isleap(now.year):
leapdays += 1
delta -= timedelta(leapdays)
# ignore microseconds
since = delta.days * 24 * 60 * 60 + delta.seconds
if since <= 0:
# d is in the future compared to now, stop processing.
return time_strings["minute"][0] % {"num": 0}
for i, (seconds, name) in enumerate(TIMESINCE_CHUNKS):
count = since // seconds
if count != 0:
break
else:
return time_strings["minute"][0] % {"num": 0}
result = []
current_depth = 0
while i < len(TIMESINCE_CHUNKS) and current_depth < depth:
seconds, name = TIMESINCE_CHUNKS[i]
count = since // seconds
if count == 0:
break
if count == 1:
time_string = time_strings[name][0] % {"num": count}
else:
time_string = time_strings[name][1] % {"num": count}
result.append(time_string)
since -= seconds * count
current_depth += 1
i += 1
return ", ".join(result)
def test_nem_excel_formatted(self):
subject = parse_date("27/9/2019 2:55:00 pm")
comparator = datetime(2019, 9, 27, 14, 55, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def test_bom_date_utc(self):
subject = parse_date("20201008133000", dayfirst=False, is_utc=True)
comparator = datetime(2020, 10, 8, 13, 30, 0, 0, tzinfo=UTC)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is True, "Date has timezone info"
def test_nem_settlementdate(self):
subject = parse_date("2020/10/07 10:15:00", dayfirst=False)
comparator = datetime(2020, 10, 7, 10, 15, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def test_nem_dispatch_scada_interval(self):
subject = parse_date("2020/06/01 21:35:00", dayfirst=False)
comparator = datetime(2020, 6, 1, 21, 35, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def test_nem_dispatch_interval(self):
subject = parse_date("30/9/19 4:00")
comparator = datetime(2019, 9, 30, 4, 0, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def test_wem_date_comissioned(self):
subject = parse_date("1/11/08 0:00")
comparator = datetime(2008, 11, 1, 0, 0, 0)
assert subject == comparator, "Parses date correctly"
assert is_aware(subject) is False, "Date has no timezone info"
def parse_date(
date_str: Union[str, datetime],
date_format: Optional[str] = None,
network: Optional[NetworkSchema] = None,
dayfirst: bool = True,
yearfirst: bool = False,
is_utc: bool = False,
timezone: pytimezone = None,
use_optimized: bool = True,
) -> Optional[datetime]:
dt_return = None
if isinstance(date_str, datetime):
dt_return = date_str
elif isinstance(date_str, str):
# avoid strptime if we can
try:
dt_return = datetime.fromisoformat(date_str.replace("/", "-"))
except ValueError:
pass
if not dt_return and date_format:
dt_return = datetime.strptime(date_str, date_format)
if not dt_return and use_optimized:
dt_return = optimized_data_parser(date_str)
if not dt_return:
try:
dt_return = parse(date_str,
dayfirst=dayfirst,
yearfirst=yearfirst)
except ParserError:
raise ValueError("Invalid date string passed")
else:
raise ValueError("Require a datetime or string object to parse date")
if network:
tz = network.get_timezone()
if tz:
if is_aware(dt_return):
if hasattr(tz, "localize"):
dt_return = tz.localize() # type: ignore
else:
dt_return = dt_return.replace(tzinfo=tz)
else:
dt_return = make_aware(dt_return, timezone=tz)
if is_utc:
tz = pytimezone.utc
if dt_return and is_aware(dt_return):
if tz and hasattr(tz, "localize"):
dt_return = tz.localize() # type: ignore
else:
dt_return = dt_return.replace(tzinfo=tz)
else:
dt_return = make_aware(dt_return, timezone=tz) # type: ignore
if timezone:
dt_return = make_aware(dt_return, timezone=timezone) # type: ignore
return dt_return
