def test_ts_get_krls_predictor(self): t0=api.utctime_now() ta=api.TimeAxis(t0, api.deltahours(1), 30*24) data=np.sin(np.linspace(0, 2*np.pi, ta.size())) ts_data=api.TimeSeries(ta, data, api.POINT_INSTANT_VALUE) ts=api.TimeSeries("a") try: ts.get_krls_predictor() self.fail("should not be able to get predictor for unbound") except: pass fbi=ts.find_ts_bind_info() fbi[0].ts.bind(ts_data) ts.bind_done() pred=ts.get_krls_predictor(api.deltahours(3)) ts_krls=pred.predict(ta) self.assertEqual(len(ts_krls), len(ts_data)) ts_mse=pred.mse_ts(ts_data) self.assertEqual(len(ts_mse), len(ts_data)) for i in range(len(ts_krls)): self.assertAlmostEqual(ts_krls.values[i], ts_data.values[i], places=1) self.assertAlmostEqual(ts_mse.values[i], 0, places=2) self.assertAlmostEqual(pred.predictor_mse(ts_data), 0, places=2)
def test_extend_vector_of_timeseries(self): t0=api.utctime_now() dt=api.deltahours(1) n=512 tsvector=api.TsVector() ta=api.TimeAxisFixedDeltaT(t0 + 3*n*dt, dt, 2*n) tsvector.push_back(api.TimeSeries( ta=api.TimeAxisFixedDeltaT(t0, dt, 2*n), fill_value=1.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)) tsvector.push_back(api.TimeSeries( ta=api.TimeAxisFixedDeltaT(t0 + 2*n*dt, dt, 2*n), fill_value=2.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)) extension=api.TimeSeries(ta=ta, fill_value=8.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) # extend after all time-series in the vector extended_tsvector=tsvector.extend_ts(extension) # assert first element for i in range(2*n): self.assertEqual(extended_tsvector[0](t0 + i*dt), 1.0) for i in range(n): self.assertTrue(math.isnan(extended_tsvector[0](t0 + (2*n + i)*dt))) for i in range(2*n): self.assertEqual(extended_tsvector[0](t0 + (3*n + i)*dt), 8.0) # assert second element for i in range(2*n): self.assertEqual(extended_tsvector[1](t0 + (2*n + i)*dt), 2.0) for i in range(n): self.assertEqual(extended_tsvector[1](t0 + (4*n + i)*dt), 8.0) tsvector_2=api.TsVector() tsvector_2.push_back(api.TimeSeries( ta=api.TimeAxisFixedDeltaT(t0 + 2*n*dt, dt, 4*n), fill_value=10.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)) tsvector_2.push_back(api.TimeSeries( ta=api.TimeAxisFixedDeltaT(t0 + 4*n*dt, dt, 4*n), fill_value=20.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)) # extend each element in tsvector by the corresponding element in tsvector_2 extended_tsvector=tsvector.extend_ts(tsvector_2) # assert first element for i in range(2*n): self.assertEqual(extended_tsvector[0](t0 + i*dt), 1.0) for i in range(4*n): self.assertEqual(extended_tsvector[0](t0 + (2*n + i)*dt), 10.0) # assert second element for i in range(2*n): self.assertEqual(extended_tsvector[1](t0 + (2*n + i)*dt), 2.0) for i in range(4*n): self.assertEqual(extended_tsvector[1](t0 + (4*n + i)*dt), 20.0)
def test_trim_day(self): t = api.utctime_now() td = self.std.trim(t, api.Calendar.DAY) c = self.std.calendar_units(td) a = self.std.calendar_units(t) self.assertEqual(c.second, 0, 'incorrect seconds should be 0') self.assertEqual(c.minute, 0, 'trim day should set minutes to 0') self.assertEqual(c.hour, 0, 'trim day should set hours to 0') self.assertEqual(a.year, c.year, 'trim day Should leave same year') self.assertEqual(a.month, c.month, 'trim day Should leave month') self.assertEqual(a.day, c.day, 'should leave same day')
def test_basic_timeseries_math_operations(self): """ Test that timeseries functionality is exposed, and briefly verify correctness of operators (the shyft core do the rest of the test job, not repeated here). """ c=api.Calendar() t0=api.utctime_now() dt=api.deltahours(1) n=240 ta=api.TimeAxis(t0, dt, n) a=api.TimeSeries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) self.assertTrue(a) # should evaluate to true b=api.TimeSeries(ta=ta, fill_value=1.0, point_fx=api.point_interpretation_policy.POINT_INSTANT_VALUE) b.fill(2.0) # demo how to fill a point ts self.assertAlmostEqual((1.0 - b).values.to_numpy().max(), -1.0) self.assertAlmostEqual((b - 1.0).values.to_numpy().max(), 1.0) c=a + b*3.0 - a/2.0 # operator + * - / d=-a # unary minus e=a.average(ta) # average f=api.max(c, 300.0) g=api.min(c, -300.0) # h = a.max(c, 300) # class static method not supported h=c.max(300.0) k=c.min(-300) self.assertEqual(a.size(), n) self.assertEqual(b.size(), n) self.assertEqual(c.size(), n) self.assertAlmostEqual(c.value(0), 3.0 + 2.0*3.0 - 3.0/2.0) # 7.5 for i in range(n): self.assertAlmostEqual(c.value(i), a.value(i) + b.value(i)*3.0 - a.value(i)/2.0, delta=0.0001) self.assertAlmostEqual(d.value(i), - a.value(i), delta=0.0001) self.assertAlmostEqual(e.value(i), a.value(i), delta=0.00001) self.assertAlmostEqual(f.value(i), +300.0, delta=0.00001) self.assertAlmostEqual(h.value(i), +300.0, delta=0.00001) self.assertAlmostEqual(g.value(i), -300.0, delta=0.00001) self.assertAlmostEqual(k.value(i), -300.0, delta=0.00001) # now some more detailed tests for setting values b.set(0, 3.0) self.assertAlmostEqual(b.value(0), 3.0) # 3.0 + 3 * 3 - 3.0/2.0 self.assertAlmostEqual(c.value(1), 7.5, delta=0.0001) # 3 + 3*3 - 1.5 = 10.5 self.assertAlmostEqual(c.value(0), 10.5, delta=0.0001) # 3 + 3*3 - 1.5 = 10.5
def test_timeseries_vector(self): c = api.Calendar() t0 = api.utctime_now() dt = api.deltahours(1) n = 240 ta = api.Timeaxis(t0, dt, n) a = api.Timeseries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) b = api.Timeseries(ta=ta, fill_value=2.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) v = api.TsVector() v.append(a) v.append(b) self.assertEqual(len(v), 2) self.assertAlmostEqual(v[0].value(0), 3.0, "expect first ts to be 3.0") aa = api.Timeseries(ta=a.time_axis, values=a.values, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) # copy construct (really copy the values!) a.fill(1.0) self.assertAlmostEqual(v[0].value(0), 1.0, "expect first ts to be 1.0, because the vector keeps a reference ") self.assertAlmostEqual(aa.value(0), 3.0)
def test_timeseries_vector(self): c=api.Calendar() t0=api.utctime_now() dt=api.deltahours(1) n=240 ta=api.TimeAxisFixedDeltaT(t0, dt, n) a=api.TimeSeries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) b=api.TimeSeries(ta=ta, fill_value=2.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) v=api.TsVector() v.append(a) v.append(b) self.assertEqual(len(v), 2) self.assertAlmostEqual(v[0].value(0), 3.0, "expect first ts to be 3.0") aa=api.TimeSeries(ta=a.time_axis, values=a.values, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) # copy construct (really copy the values!) a.fill(1.0) self.assertAlmostEqual(v[0].value(0), 1.0, "expect first ts to be 1.0, because the vector keeps a reference ") self.assertAlmostEqual(aa.value(0), 3.0)
def test_basic_timeseries_math_operations(self): """ Test that timeseries functionality is exposed, and briefly verify correctness of operators (the shyft core do the rest of the test job, not repeated here). """ c = api.Calendar() t0 = api.utctime_now() dt = api.deltahours(1) n = 240 ta = api.Timeaxis2(t0, dt, n) a = api.Timeseries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) b = api.Timeseries(ta=ta, fill_value=1.0) b.fill(2.0) # demo how to fill a point ts c = a + b * 3.0 - a / 2.0 # operator + * - / d = -a # unary minus e = a.average(ta) # average f = api.max(c, 300.0) g = api.min(c, -300.0) h = a.max(c, 300) k = a.min(c, -300) self.assertEqual(a.size(), n) self.assertEqual(b.size(), n) self.assertEqual(c.size(), n) self.assertAlmostEqual(c.value(0), 3.0 + 2.0 * 3.0 - 3.0 / 2.0) # 7.5 for i in range(n): self.assertAlmostEqual(c.value(i), a.value(i) + b.value(i) * 3.0 - a.value(i) / 2.0, delta=0.0001) self.assertAlmostEqual(d.value(i), - a.value(i), delta=0.0001) self.assertAlmostEqual(e.value(i), a.value(i), delta=0.00001) self.assertAlmostEqual(f.value(i), +300.0, delta=0.00001) self.assertAlmostEqual(h.value(i), +300.0, delta=0.00001) self.assertAlmostEqual(g.value(i), -300.0, delta=0.00001) self.assertAlmostEqual(k.value(i), -300.0, delta=0.00001) # now some more detailed tests for setting values b.set(0, 3.0) self.assertAlmostEqual(b.value(0), 3.0) # 3.0 + 3 * 3 - 3.0/2.0 self.assertAlmostEqual(c.value(1), 7.5, delta=0.0001) # 3 + 3*3 - 1.5 = 10.5 self.assertAlmostEqual(c.value(0), 10.5, delta=0.0001) # 3 + 3*3 - 1.5 = 10.5
def test_krls_ts(self): t0=api.utctime_now() ta=api.TimeAxis(t0, api.deltahours(1), 30*24) data=np.sin(np.linspace(0, 2*np.pi, ta.size())) ts_data=api.TimeSeries(ta, data, api.POINT_INSTANT_VALUE) ts=api.TimeSeries("a") ts_krls=ts.krls_interpolation(api.deltahours(3)) ts_krls_blob=ts_krls.serialize() ts2_krls=api.TimeSeries.deserialize(ts_krls_blob) self.assertTrue(ts2_krls.needs_bind()) fbi=ts2_krls.find_ts_bind_info() self.assertEqual(len(fbi), 1) fbi[0].ts.bind(ts_data) ts2_krls.bind_done() self.assertFalse(ts2_krls.needs_bind()) self.assertEqual(len(ts2_krls), len(ts_data)) for i in range(len(ts2_krls)): self.assertAlmostEqual(ts2_krls.values[i], ts_data.values[i], places=1)
def __init__(self, *args, **kwargs): super(DtssTestCase, self).__init__(*args, **kwargs) self.callback_count = 0 self.find_count = 0 self.ts_infos = TsInfoVector() self.rd_throws = False self.cache_reads = False self.cache_dtss = None utc = Calendar() t_now = utctime_now() self.stored_tsv = list() for i in range(30): self.ts_infos.append( TsInfo(name=fake_store_url('{0}'.format(i)), point_fx=point_fx.POINT_AVERAGE_VALUE, delta_t=deltahours(1), olson_tz_id='', data_period=UtcPeriod(utc.time(2017, 1, 1), utc.time(2018, 1, 1)), created=t_now, modified=t_now))
def test_rating_curve_ts(self): t0=api.utctime_now() ta=api.TimeAxis(t0, api.deltaminutes(30), 48*2) data=np.linspace(0, 10, ta.size()) ts=api.TimeSeries(ta, data, api.POINT_INSTANT_VALUE) rcf1=api.RatingCurveFunction() rcf1.add_segment(0, 1, 0, 1) rcf1.add_segment(api.RatingCurveSegment(5, 2, 0, 1)) rcf2=api.RatingCurveFunction() rcf2.add_segment(0, 3, 0, 1) rcf2.add_segment(api.RatingCurveSegment(8, 4, 0, 1)) rcp=api.RatingCurveParameters() rcp.add_curve(t0, rcf1) rcp.add_curve(t0 + api.deltahours(24), rcf2) sts=api.TimeSeries("a") rcsts=sts.rating_curve(rcp) rcsts_blob=rcsts.serialize() rcsts_2=api.TimeSeries.deserialize(rcsts_blob) self.assertTrue(rcsts_2.needs_bind()) fbi=rcsts_2.find_ts_bind_info() self.assertEqual(len(fbi), 1) fbi[0].ts.bind(ts) rcsts_2.bind_done() self.assertFalse(rcsts_2.needs_bind()) self.assertEqual(len(rcsts_2), len(ts)) for i in range(rcsts_2.size()): expected=(1*ts.get(i).v if ts.get(i).v < 5 else 2*ts.get(i).v) if ts.get(i).t < t0 + api.deltahours(24) else ( 3*ts.get(i).v if ts.get(i).v < 8 else 4*ts.get(i).v) self.assertEqual(rcsts_2.get(i).t, ts.get(i).t) self.assertEqual(rcsts_2.get(i).v, expected)
def start_date(self): utc = api.Calendar() today = utc.trim(api.utctime_now(), api.Calendar.DAY) return today - api.Calendar.DAY # yesterday
def test_a_time_series_vector(self): c=api.Calendar() t0=api.utctime_now() dt=api.deltahours(1) n=240 ta=api.TimeAxisFixedDeltaT(t0, dt, n) a=api.TimeSeries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) b=api.TimeSeries(ta=ta, fill_value=2.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) c=api.TimeSeries(ta=ta, fill_value=10.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) v=api.TsVector() v.append(a) v.append(b) self.assertEqual(len(v), 2) self.assertAlmostEqual(v[0].value(0), 3.0, "expect first ts to be 3.0") aa=api.TimeSeries(ta=a.time_axis, values=a.values, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) # copy construct (really copy the values!) a.fill(1.0) self.assertAlmostEqual(v[0].value(0), 1.0, "expect first ts to be 1.0, because the vector keeps a reference ") self.assertAlmostEqual(aa.value(0), 3.0) vt=v.values_at(t0).to_numpy() self.assertEqual(len(vt), len(v)) v1=v[0:1] self.assertEqual(len(v1), 1) self.assertAlmostEqual(v1[0].value(0), 1.0) v_clone=api.TsVector(v) self.assertEqual(len(v_clone), len(v)) del v_clone[-1] self.assertEqual(len(v_clone), 1) self.assertEqual(len(v), 2) v_slice_all=v.slice(api.IntVector()) v_slice_1=v.slice(api.IntVector([1])) v_slice_12=v.slice(api.IntVector([0, 1])) self.assertEqual(len(v_slice_all), 2) self.assertEqual(len(v_slice_1), 1) self.assertAlmostEqual(v_slice_1[0].value(0), 2.0) self.assertEqual(len(v_slice_12), 2) self.assertAlmostEqual(v_slice_12[0].value(0), 1.0) # multiplication by scalar v_x_2a=v*2.0 v_x_2b=2.0*v for i in range(len(v)): self.assertAlmostEqual(v_x_2a[i].value(0), 2*v[i].value(0)) self.assertAlmostEqual(v_x_2b[i].value(0), 2*v[i].value(0)) # division by scalar v_d_a=v/3.0 v_d_b=3.0/v for i in range(len(v)): self.assertAlmostEqual(v_d_a[i].value(0), v[i].value(0)/3.0) self.assertAlmostEqual(v_d_b[i].value(0), 3.0/v[i].value(0)) # addition by scalar v_a_a=v + 3.0 v_a_b=3.0 + v for i in range(len(v)): self.assertAlmostEqual(v_a_a[i].value(0), v[i].value(0) + 3.0) self.assertAlmostEqual(v_a_b[i].value(0), 3.0 + v[i].value(0)) # sub by scalar v_s_a=v - 3.0 v_s_b=3.0 - v for i in range(len(v)): self.assertAlmostEqual(v_s_a[i].value(0), v[i].value(0) - 3.0) self.assertAlmostEqual(v_s_b[i].value(0), 3.0 - v[i].value(0)) # multiplication vector by ts v_x_ts=v*c ts_x_v=c*v for i in range(len(v)): self.assertAlmostEqual(v_x_ts[i].value(0), v[i].value(0)*c.value(0)) self.assertAlmostEqual(ts_x_v[i].value(0), c.value(0)*v[i].value(0)) # division vector by ts v_d_ts=v/c ts_d_v=c/v for i in range(len(v)): self.assertAlmostEqual(v_d_ts[i].value(0), v[i].value(0)/c.value(0)) self.assertAlmostEqual(ts_d_v[i].value(0), c.value(0)/v[i].value(0)) # add vector by ts v_a_ts=v + c ts_a_v=c + v for i in range(len(v)): self.assertAlmostEqual(v_a_ts[i].value(0), v[i].value(0) + c.value(0)) self.assertAlmostEqual(ts_a_v[i].value(0), c.value(0) + v[i].value(0)) # sub vector by ts v_s_ts=v - c ts_s_v=c - v for i in range(len(v)): self.assertAlmostEqual(v_s_ts[i].value(0), v[i].value(0) - c.value(0)) self.assertAlmostEqual(ts_s_v[i].value(0), c.value(0) - v[i].value(0)) # vector mult vector va=v vb=2.0*v v_m_v=va*vb self.assertEqual(len(v_m_v), len(va)) for i in range(len(va)): self.assertAlmostEqual(v_m_v[i].value(0), va[i].value(0)*vb[i].value(0)) # vector div vector v_d_v=va/vb self.assertEqual(len(v_d_v), len(va)) for i in range(len(va)): self.assertAlmostEqual(v_d_v[i].value(0), va[i].value(0)/vb[i].value(0)) # vector add vector v_a_v=va + vb self.assertEqual(len(v_a_v), len(va)) for i in range(len(va)): self.assertAlmostEqual(v_a_v[i].value(0), va[i].value(0) + vb[i].value(0)) # vector sub vector v_s_v=va - vb self.assertEqual(len(v_s_v), len(va)) for i in range(len(va)): self.assertAlmostEqual(v_s_v[i].value(0), va[i].value(0) - vb[i].value(0)) # vector unary minus v_u=- va self.assertEqual(len(v_u), len(va)) for i in range(len(va)): self.assertAlmostEqual(v_u[i].value(0), -va[i].value(0)) # integral functions, just to verify exposure works, and one value is according to spec. ta2=api.TimeAxis(t0, dt*24, n//24) v_avg=v.average(ta2) v_int=v.integral(ta2) v_acc=v.accumulate(ta2) v_sft=v.time_shift(dt*24) self.assertIsNotNone(v_avg) self.assertIsNotNone(v_int) self.assertIsNotNone(v_acc) self.assertIsNotNone(v_sft) self.assertAlmostEqual(v_avg[0].value(0), 1.0) self.assertAlmostEqual(v_int[0].value(0), 86400.0) self.assertAlmostEqual(v_acc[0].value(0), 0.0) self.assertAlmostEqual(v_sft[0].time(0), t0 + dt*24) # min/max functions min_v_double=va.min(-1000.0) max_v_double=va.max(1000.0) self.assertAlmostEqual(min_v_double[0].value(0), -1000.0) self.assertAlmostEqual(max_v_double[0].value(0), +1000.0) min_v_double=api.min(va, -1000.0) max_v_double=api.max(va, +1000.0) self.assertAlmostEqual(min_v_double[0].value(0), -1000.0) self.assertAlmostEqual(max_v_double[0].value(0), +1000.0) # c = 10.0 c1000=100.0*c min_v_double=va.min(-c1000) max_v_double=va.max(c1000) self.assertAlmostEqual(min_v_double[0].value(0), -c1000.value(0)) self.assertAlmostEqual(max_v_double[0].value(0), c1000.value(0)) min_v_double=api.min(va, -c1000) max_v_double=api.max(va, c1000) self.assertAlmostEqual(min_v_double[0].value(0), -c1000.value(0)) self.assertAlmostEqual(max_v_double[0].value(0), c1000.value(0)) v1000=va*1000.0 min_v_double=va.min(-v1000) max_v_double=va.max(v1000) self.assertAlmostEqual(min_v_double[0].value(0), -v1000[0].value(0)) self.assertAlmostEqual(max_v_double[0].value(0), v1000[0].value(0)) min_v_double=api.min(va, -v1000) max_v_double=api.max(va, v1000) self.assertAlmostEqual(min_v_double[0].value(0), -v1000[0].value(0)) self.assertAlmostEqual(max_v_double[0].value(0), v1000[0].value(0)) # finally, test that exception is raised if we try to multiply two unequal sized vectors try: x=v_clone*va self.assertTrue(False, 'We expected exception for unequal sized ts-vector op') except RuntimeError as re: pass # also test that empty vector + vector -> vector etc. va_2=va + api.TsVector() va_3=api.TsVector() + va va_4=va - api.TsVector() va_5=api.TsVector() - va va_x=api.TsVector() + api.TsVector() self.assertEqual(len(va_2), len(va)) self.assertEqual(len(va_3), len(va)) self.assertEqual(len(va_4), len(va)) self.assertEqual(len(va_5), len(va)) self.assertEqual(not va_x, True) self.assertEqual(not va_2, False) va_2_ok=False va_x_ok=True if va_2: va_2_ok=True if va_x: va_x_ok=False self.assertTrue(va_2_ok) self.assertTrue(va_x_ok)
def test_ts_extend(self): t0=api.utctime_now() dt=api.deltahours(1) n=512 ta_a=api.TimeAxisFixedDeltaT(t0, dt, 2*n) ta_b=api.TimeAxisFixedDeltaT(t0 + n*dt, dt, 2*n) ta_c=api.TimeAxisFixedDeltaT(t0 + 2*n*dt, dt, 2*n) ta_d=api.TimeAxisFixedDeltaT(t0 + 3*n*dt, dt, 2*n) a=api.TimeSeries(ta=ta_a, fill_value=1.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) b=api.TimeSeries(ta=ta_b, fill_value=2.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) c=api.TimeSeries(ta=ta_c, fill_value=4.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) d=api.TimeSeries(ta=ta_d, fill_value=8.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) # default behavior: extend from end of a ac=a.extend(c) for i in range(2*n): # valus from first ts self.assertEqual(ac(t0 + i*dt), 1.0) for i in range(2*n): # values from extension ts self.assertEqual(ac(t0 + (i + 2*n)*dt), 4.0) # default behavior: extend from end of a, fill gap with nan ad=a.extend(d) for i in range(2*n): # values from first self.assertEqual(ad(t0 + i*dt), 1.0) for i in range(n): # gap self.assertTrue(math.isnan(ad(t0 + (i + 2*n)*dt))) for i in range(2*n): # extension self.assertEqual(ad(t0 + (i + 3*n)*dt), 8.0) # split at the first value of d instead of last of c cd=c.extend(d, split_policy=api.extend_split_policy.RHS_FIRST) for i in range(n): # first, only until the extension start self.assertEqual(cd(t0 + (2*n + i)*dt), 4.0) for i in range(2*n): # extension self.assertEqual(cd(t0 + (3*n + i)*dt), 8.0) # split at a given time step, and extend the last value through the gap ac=a.extend(c, split_policy=api.extend_split_policy.AT_VALUE, split_at=(t0 + dt*n//2), fill_policy=api.extend_fill_policy.USE_LAST) for i in range(n//2): # first, only until the given split value self.assertEqual(ac(t0 + i*dt), 1.0) for i in range(3*n//2): # gap, uses last value before gap self.assertEqual(ac(t0 + (n//2 + i)*dt), 1.0) for i in range(2*n): # extension self.assertEqual(ac(t0 + (2*n + i)*dt), 4.0) # split at the beginning of the ts to extend when the extension start before it cb=c.extend(b, split_policy=api.extend_split_policy.AT_VALUE, split_at=(t0 + 2*n*dt)) for i in range(n): # don't extend before self.assertTrue(math.isnan(cb(t0 + (n + i)*dt))) for i in range(n): # we split at the beginning => only values from extension self.assertEqual(cb(t0 + (2*n + i)*dt), 2.0) for i in range(n): # no values after extension self.assertTrue(math.isnan(cb(t0 + (3*n + i)*dt))) # extend with ts starting after the end, fill the gap with a given value ad=a.extend(d, fill_policy=api.extend_fill_policy.FILL_VALUE, fill_value=5.5) for i in range(2*n): # first self.assertEqual(ad(t0 + i*dt), 1.0) for i in range(n): # gap, filled with 5.5 self.assertEqual(ad(t0 + (2*n + i)*dt), 5.5) for i in range(2*n): # extension self.assertEqual(ad(t0 + (3*n + i)*dt), 8.0) # check extend with more exotic combination of time-axis(we had an issue with this..) a=api.TimeSeries(api.TimeAxis(0, 1, 10), fill_value=1.0, point_fx=api.POINT_AVERAGE_VALUE) b=api.TimeSeries(api.TimeAxis(api.Calendar(), 0, 1, 20), fill_value=2.0, point_fx=api.POINT_AVERAGE_VALUE) ab=a.extend(b) ba=b.extend(a, split_policy=api.extend_split_policy.AT_VALUE, split_at=a.time_axis.time(5)) self.assertAlmostEqual(ab.value(0), 1.0) self.assertAlmostEqual(ab.value(11), 2.0) self.assertAlmostEqual(ba.value(0), 2.0) self.assertAlmostEqual(ab.value(7), 1.0)
from statkraft.ltm.io.run_repository import RunRepository import shyft.api as sa from statkraft.ltm.scripting import plot_ts, plot_percentiles import matplotlib.pyplot as plt t0 = sa.utctime_now() - sa.Calendar.WEEK rr = RunRepository() run_id = rr.find_closest_operational_run(t0) run = rr.recreate(run_id=run_id) sts = run.model.areas["Sorland"].aggregated_hydro.bypass time_axis = sa.TimeAxis(run.start_utc, sa.Calendar.DAY, 100) ts_vec = sts.mean(time_axis=time_axis, unit='MW', years=[1931, 1945]) plot_ts(ts_vec)
import matplotlib.pyplot as plt import shyft.api as sa from statkraft.ltm.io.run_repository import RunRepository from statkraft.ltm.scripting import plot_ts calendar = sa.Calendar("Europe/Oslo") utc_start = calendar.time(2018, 7, 1) time_axis = sa.TimeAxis(calendar, utc_start, calendar.QUARTER, 1) now = sa.utctime_now() then = now - calendar.WEEK rr = RunRepository() rid1 = rr.find_closest_operational_run(now) rid2 = rr.find_closest_operational_run(then) run1 = rr.recreate(run_id=rid1) run2 = rr.recreate(run_id=rid2) sp1 = run1.model.market.areas["NO2"].power_price.mean(time_axis=time_axis) sp2 = run2.model.market.areas["NO2"].power_price.mean(time_axis=time_axis) plot_ts(sp1 - sp2) plt.legend([f"NO2 price difference: run[{rid1}] - run[{rid2}]"])
def setUp(self): self.c=api.Calendar() self.d=api.deltahours(1) self.n=24 self.t= self.c.trim(api.utctime_now(),self.d) self.ta=api.Timeaxis(self.t,self.d,self.n)
from statkraft.ltm.io.run_repository import RunRepository import shyft.api as sa from statkraft.ltm.state import quantity from statkraft.ltm.scripting import plot_ts import matplotlib.pyplot as plt from statkraft.ltm.io.converter import to_pandas rr = RunRepository() t0 = sa.utctime_now() - sa.Calendar.DAY * 2 res = rr.search(labels=["operational", "norway"], created_from=t0) areas = ["NO1", "NO2", "NO5"] tsv = quantity(sa.TsVector(), "GWh") tot_cons_list = [] legend_list = [] for key in res.keys(): run_info = res[key] run = rr.recreate(run_id=key) legend_list.append(key) time_axis = sa.TimeAxis(sa.utctime_now(), sa.Calendar.DAY, 365) tot_cons = quantity(sa.TsVector(), "GWh") for key1 in run.model.market.areas.keys(): this_area = run.model.market.areas[key1] if key1 in areas: cons = this_area.consumption.mean(unit="GWh", time_axis=time_axis) tot_cons += cons tot_cons_list.append(tot_cons) diff_cons = tot_cons_list[0] - tot_cons_list[1] tsv.extend(diff_cons.magnitude)
def test_speed(self): """ the purpose of this test is to figure out the speed characteristics of qm. testcases of interest is 12 (4x3) forecast typical arome 4 times a day, use last 3 days, 1-3 hour dt, 14 days ahead 100 historical scenarios time-axis wanted is next 14 days plus historical scenarios 3..60 weeks ahead expected performance: the first 14 days includes sorting 10 forcasts, 100 historical pr. timestep. the period after should be close to 'memcpy' performance. """ # Arrange the inputs utc = Calendar() n_hist_ts = 100 n_fc_days = 14 n_hist_days = n_fc_days + 360 n_fc_ts = 10 t0 = utc.time(2017, 1, 1, 0, 0, 0) def generate_ts(ta: TimeAxis, n_fc: int) -> TsVector: fx_avg = ts_point_fx.POINT_AVERAGE_VALUE r = TsVector() w = 2 * 3.14 / len(ta) for i in range(n_fc): a = np.random.ranf() * 20 - 10.0 b = np.random.ranf() * 5.0 v = dv([a + b * math.sin(w * i) for i in range(len(ta))]) r.append(TimeSeries(ta, v, fx_avg)) return r ta_hist = TimeAxis(t0, deltahours(1), 24 * n_hist_days) historical_scenario_ts = generate_ts(ta_hist, n_hist_ts) fc_set = TsVectorSet() fc_weight = dv() n_fc_sets = 4 * 2 fc_every_dt = deltahours(6) # six hours between each arome fc. dt_fc = deltahours(1) for i in range(n_fc_sets): t0_fc = t0 + fc_every_dt * i fc_set.append( generate_ts(TimeAxis(t0_fc, dt_fc, 24 * n_fc_days), n_fc_ts)) fc_weight.append(float(3 + i)) # interpolation_start= no_utctime # Act interpolated_quantiles = False qm_end_idx1 = 24 * (n_fc_days - 2) qm_end_idx2 = 24 * (n_fc_days - 1) n_ts = 0 n_v = 0 print(r"n_days\ttime_used[s]\n") tot_seconds = 0.0 for h_days in range(n_fc_days + 10, n_hist_days, 30): ta_qm = TimeAxis(t0 + n_fc_sets * fc_every_dt, dt_fc, 24 * h_days) a0 = utctime_now() result = quantile_map_forecast(fc_set, fc_weight, historical_scenario_ts, ta_qm, ta_qm.time(qm_end_idx1), ta_qm.time(qm_end_idx2), interpolated_quantiles) self.assertIsNotNone(result) n_ts += len(result) n_v += len(result[0]) * len(result) a1 = utctime_now() tot_seconds += float(a1 - a0) print(f' {h_days}\t{float(a1-a0)}') print( f'Total of {n_ts} ts, was forecasted, number values produced {n_v/1000000} Mpts, Mb/s={8.0*n_v/1000000/tot_seconds}' )
import socket from datetime import datetime import logging import os import traceback from shyft.api import utctime_now # To time the reading from SMG from modules import r2_and_regression as reg from modules import read_and_setup as rs start_time = utctime_now() logging_path = '../logging' log_file = os.path.join( logging_path, # folder with log files '{}#{}#{}#{}.log'.format( os.path.splitext(os.path.basename(__file__))[0], # script file name socket.gethostname().lower(), # host_name datetime.now().strftime('%Y%m%dT%H%M%S'), # timestamp os.getpid() # process ID )) logging.basicConfig( filename=log_file, level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logging.info('autojob started.') try: auto_input = {} auto_input['tilsig'] = rs.read_and_setup('tilsig')
import matplotlib.pyplot as plt import shyft.api as sa from statkraft.ltm.io.run_repository import RunRepository from statkraft.ltm.scripting import plot_ts from statkraft.ltm.state import quantity rr = RunRepository() rid = rr.find_closest_operational_run(sa.utctime_now()) run = rr.recreate(run_id=rid) time_axis = sa.TimeAxis(run.start_utc, sa.Calendar.DAY, 52 * 7) tsv = quantity(sa.TsVector(), "EUR/MWh") legend_list = [] for area_name, area in run.model.areas.items(): legend_list.append(area_name) tsv.extend(area.power_price.mean(time_axis=time_axis).magnitude) plot_ts(tsv) plt.legend(legend_list) plt.show()
from statkraft.ltm.io.run_repository import RunRepository import shyft.api as sa from statkraft.ltm.state import quantity from statkraft.ltm.io.converter import to_pandas rr = RunRepository() t0 = sa.utctime_now() - sa.Calendar.DAY run_id = rr.find_closest_operational_run(t0) run = rr.recreate(run_id=run_id) areas = ["SE1", "SE2", "SE3", "SE4"] energy_inflow = quantity(sa.TsVector(), "GWh") time_axis = sa.TimeAxis(sa.utctime_now(), sa.Calendar.MONTH, 24) years = run.run_info.scenario_years for key in run.model.areas.keys(): area = run.model.areas[key] if area.market_price_area != None and area.market_price_area.name in areas: storable_inflow = area.aggregated_hydro.storable_inflow( unit="GWh", time_axis=time_axis) non_storable_inflow = area.aggregated_hydro.nonstorable_inflow( unit="GWh", time_axis=time_axis) bypass = area.aggregated_hydro.bypass(unit="GWh", time_axis=time_axis) spill = area.aggregated_hydro.spillage(unit="GWh", time_axis=time_axis) energy_inflow += storable_inflow + non_storable_inflow - bypass - spill df, pip = to_pandas(energy_inflow) df = df.magnitude df.columns = years + [df.columns[-1]] df.to_csv("inflow_sweden.csv")
def test_utctime_now(self): a = api.utctime_now() x = dt.datetime.utcnow() b = self.utc.time(api.YMDhms(x.year, x.month, x.day, x.hour, x.minute, x.second)) self.assertLess(abs(a - b), 2, 'Should be less than 2 seconds')
from statkraft.ltm.io.run_repository import RunRepository import shyft.api as sa from statkraft.ltm.scripting import plot_ts, plot_percentiles from matplotlib import pyplot as plt from statkraft.ltm.io.converter import to_pandas import pandas as pd rr = RunRepository() t0 = sa.utctime_now() run_id = rr.find_closest_operational_run(t0) run = rr.recreate(run_id = run_id) time_axis = sa.TimeAxis(run.start_utc, sa.Calendar.DAY,365) prices = {} df_tot = pd.DataFrame() for area_name, area in run.model.areas.items(): df, pip = to_pandas(area.power_price.mean(time_axis = time_axis)) prices[area_name] = df.magnitude
obs_series_f = "P:/projects/config_auto/observed_inflow_series.yml" forecast_period=864000 # seconds in 10days enki_root=os.path.join("P:/","projects") config_file = os.path.join(enki_root, "config_auto","runner_configurations_auto.yaml") # Using python datetime #current_d = datetime.utcnow() #log.write(current_d.strftime('%Y%m%dT%H%M')+'\n') #current_d = current_d.replace(hour=0, minute=0, second=0, microsecond=0)-timedelta(hours=1) #current_d_timestamp = int(round((current_d - datetime.utcfromtimestamp(0)).total_seconds())) # Using enki Calendar cal=api.Calendar() current_d_timestamp = cal.trim(api.utctime_now(),api.Calendar.DAY)-3600 #current_d = cal.toString(current_d_timestamp) current_d = datetime.utcfromtimestamp(current_d_timestamp) #log.write(cal.toString(current_d_timestamp)+'\n') #log.close() for model_name in model_names: for run_mode in run_modes: if(run_mode=='observed'): config_name=model_name+'_observed' with open(config_file) as f: dct = yaml.load(f) val=dct[config_name] #init_state_timestamp=int(round((val['start_datetime'] - datetime.utcfromtimestamp(0)).total_seconds())) # Python datetime
from shyft import shyftdata_dir from shyft import api # points as in c++ test print(" --------------------------------------------------------- ") print(" --- Simple point test --- ") print(" --- just some calculation to check api --- ") print(" --------------------------------------------------------- ") utc = api.Calendar( ) # provides shyft build-in functionality for date/time handling radparam = api.RadiationParameter(0.2, 1.0) radcal = api.RadiationCalculator(radparam) radresponse = api.RadiationResponse() time = api.utctime_now() radcal.net_radiation(radresponse, 40.4, time, 0.0, 0.0, 20.0, 40.0, 150.0, 0.0) print(radresponse.net) pmparam = api.PenmanMonteithParameter(0.12, 2.0, 2.0, 144.0, 0) pmcalculator = api.PenmanMonteithCalculator(pmparam) pmresponse = api.PenmanMonteithResponse() temperature = 20.0 rhumidity = 40.0 elevation = 150.0 windspeed = 1.0 pmcalculator.reference_evapotranspiration(pmresponse, api.deltahours(24), radresponse.net, temperature, temperature, rhumidity, elevation,
def run_regression(auto_input, variables: list = ['magasin', 'tilsig'], regions: list = [ 'NO1', 'NO2', 'NO3', 'NO4', 'NO5', 'SE1', 'SE2', 'SE3', 'SE4' ], jupyter: bool = False, backup: bool = False, loop: bool = False) -> None: """This function is the head function for the regression and it also deals with the outputs. Args: variable: Must be either 'magasin' or 'tilsig' regions: Must be one or more of the default regions jupyter: Set to Tru if the code is runned on Jupyter Notebooks backup: Set to True if you would rather use the backup input variables (input_variables_backup.txt) than the automatically updated variables from the last tuning (input_variables_from_tuning.txt). loop: Set to True if you want to do a Tuning to update input_variavles_from_tuning.txt and run the loop. This takes approximately half an hour. Tuning example: >> run_regression(auto_input, loop=True) Updating SMG on Jupyter Notebooks exaple: >> run_regression(auto_input, jupyter=True) """ start_tuning = utctime_now() for variable in variables: if not variable in ['magasin', 'tilsig']: sys.exit("Variable must be either 'tilsig' or 'magasin'") df_week, MagKap, period, forecast_time, read_start = auto_input[ variable] reg_end = ( pd.to_datetime(time.strftime(forecast_time), format="%Y.%m.%d") - Timedelta(days=7)).strftime('%Y.%m.%d') if (0 <= today.weekday() <= 1) or ( today.weekday() == 2 and today.hour < 14): # True for tipping last_forecast = forecast_time else: last_forecast = reg_end df_cleaned = deletingNaNs(df_week.loc[:last_forecast]) if loop: if variable == 'tilsig': print( '\n\n---------------------------------------------------------------' ) print( ' TILSIG ' ) print( '---------------------------------------------------------------\n' ) max_kandidater = 196 min_kandidater = 1 else: print( '\n\n---------------------------------------------------------------' ) print( ' MAGASIN ' ) print( '---------------------------------------------------------------\n' ) max_kandidater = 171 min_kandidater = 1 max_weeks = 230 min_weeks = 16 print('max ant. kandidater: {}, min ant. kandidater: {}'.format( max_kandidater, min_kandidater)) print('max ant. uker: {}, min ant. uker: {}'.format( max_weeks, min_weeks)) for region in regions: print( '---------------------------------------------------------------' ) print( ' {} ' .format(region)) print( '---------------------------------------------------------------' ) if not region in [ 'NO1', 'NO2', 'NO3', 'NO4', 'NO5', 'SE1', 'SE2', 'SE3', 'SE4' ]: sys.exit( "Region must one out of: 'NO1', 'NO2', 'NO3', 'NO4', 'NO5', 'SE1', 'SE2', 'SE3', 'SE4'" ) start_time_loop = utctime_now() fasit, fasit_key = make_fasit(variable, region, reg_end, period) print('Fasit er lest inn.\n') if fasit[fasit_key].isnull().any(): print( 'OBS: Det mangler verdier på fasiten! Går videre til neste region i loopen..' ) continue sorted_r2 = get_R2_sorted(variable, df_cleaned, fasit, fasit_key) if loop: max_p = 0.025 # First loop: Tuning number of candidates for best possible R2 combined df_ant_kandidater = pd.DataFrame(columns=columns) for antall in range(min_kandidater, max_kandidater + 1, 1): if antall > len(sorted_r2): chosen_r2 = sorted_r2 else: chosen_r2 = sorted_r2[:antall] output = make_estimate(df_cleaned, fasit, fasit_key, last_forecast, first_period, max_p, chosen_r2, loop=True) df_ant_kandidater = df_ant_kandidater.append( { columns[0]: output[0], columns[1]: output[1], columns[2]: output[2], columns[3]: output[3], columns[4]: output[4], columns[5]: output[5], columns[6]: output[6] }, ignore_index=True) if antall > len(sorted_r2): print( 'Feilmelding: Ønsket antall kandidater overskrider maks (%i).\n' % len(sorted_r2)) break idx_max = df_ant_kandidater.r2_samlet.idxmax(skipna=True) ant_kandidater_beste = int( df_ant_kandidater.ant_kandidater.values[idx_max]) print('Beste ant_kandidater loop 1: ', ant_kandidater_beste) # Second loop: tuning length of the short regression for best possible R2 combined, using the best number of # candidates found in the First loop. df_short_period = pd.DataFrame(columns=columns) for short_period in range(min_weeks, max_weeks + 1, 1): short_period = int(short_period) final_chosen_r2 = sorted_r2[:ant_kandidater_beste] output = make_estimate(df_cleaned, fasit, fasit_key, last_forecast, short_period, max_p, final_chosen_r2, loop=True) df_short_period = df_short_period.append( { columns[0]: output[0], columns[1]: output[1], columns[2]: output[2], columns[3]: output[3], columns[4]: output[4], columns[5]: output[5], columns[6]: output[6] }, ignore_index=True) idx_max = df_short_period.r2_samlet.idxmax(skipna=True) short_period_beste = int( df_short_period.short_period.values[idx_max]) print('Beste short_period loop 2: ', short_period_beste) # Getting the best input variables from loop and write to input_variables_from_tuning.txt df_all_methods = pd.concat( [df_ant_kandidater, df_short_period], ignore_index=True, sort=False) idx_max = df_all_methods.r2_samlet.idxmax(skipna=True) ant_kandidater_beste = int( df_all_methods.ant_kandidater.values[idx_max]) chosen_r2_beste = sorted_r2[:ant_kandidater_beste] short_period_beste = df_all_methods.short_period.values[ idx_max] ws.write_input_variables_to_file(region, variable, max_p, ant_kandidater_beste, short_period_beste) print('\nTuning for regionen tok %.0f minutter. \n' % ((utctime_now() - start_time_loop) / 60)) else: # getting the best variables from input_variables_from_tuning.txt or input_variables_backup.txr short_period_beste, max_p, ant_kandidater_beste, input_file = rs.get_input_variables_from_file( variable, region, backup) chosen_r2_beste = sorted_r2[:ant_kandidater_beste] print("Input variables was read from: ", input_file) # Show results input1 = make_estimate(df_cleaned, fasit, fasit_key, last_forecast, short_period_beste, max_p, chosen_r2_beste, loop=False) input2 = fasit_key, ant_kandidater_beste, max_p, reg_end, read_start if not loop: # Write results from the regression to SMG. fasit, long_results, short_results, df_tot, chosen_p, chosen_r2, r2_modelled, prediction, tipping_df, short_period, nb_weeks_tipping = input1 # write to SMG: ws.write_SMG_regresjon(variable, region, tipping_df) # write to SMG, virtual: ws.write_V_SMG_Regresjon(short_results, chosen_p, fasit_key, r2_modelled, MagKap) if jupyter: ws.show_result_jupyter(input1, input2) else: ws.show_result(input1, input2) print('---------------------------------------------------------------') print(' SLUTT ') print('---------------------------------------------------------------') print( '\nRegresjon for alle regioner og variabler brukte totalt %.0f minutter. \n' % ((utctime_now() - start_tuning) / 60))
def setUp(self): self.c=api.Calendar() self.d=api.deltahours(1) self.n=24 self.t=self.c.trim(api.utctime_now(), self.d) self.ta=api.TimeAxisFixedDeltaT(self.t, self.d, self.n)
def t_now(self): return int(sa.utctime_now())