def api_fio_json(fio_result_id, job_id, log_type): start_frame = request.args.get('start_frame') end_frame = request.args.get('end_frame') granularity = request.args.get('granularity', '1S') io_type = request.args.get('io_type') fio_data = FioResult(DATA_PATH, fio_result_id) if start_frame and end_frame: data_frame = fio_data.to_dataframe(job_id, log_type, io_type) if data_frame is None: return jsonify(dict(error='404')) # return error 404 with # html status code 200 # so we can catch this error # directly in frontend # without triggering errors. data_frame = data_frame[(data_frame.index > Milli(start_frame)) & (data_frame.index < Milli(end_frame))] if "iops" not in log_type: data_frame[1] = data_frame[1].astype(float).values / 1000 data_frame[1] = data_frame[1].round(2) if granularity: data_frame = data_frame.\ resample(granularity, label='right', closed='right').mean().\ fillna('None') return jsonify(dict( x=data_frame.index.astype('timedelta64[s]').values.tolist(), y=data_frame[1].values.tolist() )) return jsonify(fio_data.to_json(job_id, log_type))
def test_MillisecondTimestampArithmetic(): assert_offset_equal( Milli(), Timestamp("2010-01-01"), Timestamp("2010-01-01 00:00:00.001") ) assert_offset_equal( Milli(-1), Timestamp("2010-01-01 00:00:00.001"), Timestamp("2010-01-01") )
def api_fio_json_combined(fio_result_id, log_type): start_frame = request.args.get('start_frame') end_frame = request.args.get('end_frame') granularity = request.args.get('granularity', '1S') io_type = request.args.get('io_type') fio_data = FioResult(DATA_PATH, fio_result_id) fio_results = [] for fio_group_report in fio_data.group_reports: for job in fio_group_report.jobs: fio_result = fio_data.to_dataframe(job.job_id, log_type, io_type) if fio_result is not None: if "iops" not in log_type: fio_result[1] = fio_result[1].astype(float).values / 1000 fio_result[1] = fio_result[1].round(2) fio_result = fio_result.resample(granularity, label='right', closed='right').mean() fio_results.append(fio_result) if not fio_results: return jsonify(dict(error='404')) # return error 404 with # html status code 200 # so we can catch this error # directly in frontend # without triggering errors. data_frame = pd.concat(fio_results, join='inner') data_frame = data_frame.sort_index() data_frame = data_frame.resample(granularity, label='right', closed='right') if "lat" in log_type: data_frame = data_frame.mean() else: data_frame = data_frame.sum() data_frame = data_frame.fillna('None') if start_frame and end_frame: data_frame = data_frame[(data_frame.index > Milli(start_frame)) & (data_frame.index < Milli(end_frame))] return jsonify(dict(x=data_frame.index.astype('timedelta64[s]') .values.tolist(), y=data_frame[1].values.tolist()))
def test_resolution(self): def _assert_less(ts1, ts2): val1 = self.dtc.convert(ts1, None, None) val2 = self.dtc.convert(ts2, None, None) if not val1 < val2: raise AssertionError(f"{val1} is not less than {val2}.") # Matplotlib's time representation using floats cannot distinguish # intervals smaller than ~10 microsecond in the common range of years. ts = Timestamp("2012-1-1") _assert_less(ts, ts + Second()) _assert_less(ts, ts + Milli()) _assert_less(ts, ts + Micro(50))
# Offset names ("time rules") and related functions from pandas.tseries.offsets import (Day, BDay, Hour, Minute, Second, Milli, Week, Micro, MonthEnd, MonthBegin, BMonthBegin, BMonthEnd, YearBegin, YearEnd, BYearBegin, BYearEnd, QuarterBegin, QuarterEnd, BQuarterBegin, BQuarterEnd) _offset_map = { 'D' : Day(), 'B' : BDay(), 'H' : Hour(), 'T' : Minute(), 'S' : Second(), 'L' : Milli(), 'U' : Micro(), None : None, # Monthly - Calendar 'M' : MonthEnd(), 'MS' : MonthBegin(), # Monthly - Business 'BM' : BMonthEnd(), 'BMS' : BMonthBegin(), # Annual - Calendar 'A-JAN' : YearEnd(month=1), 'A-FEB' : YearEnd(month=2), 'A-MAR' : YearEnd(month=3),
'AS': 'AS-JAN', # YearBegin(month=1), 'BA': 'BA-DEC', # BYearEnd(month=12), 'BAS': 'BAS-JAN', # BYearBegin(month=1), 'Min': 'T', 'min': 'T', 'ms': 'L', 'us': 'U', 'ns': 'N' } _name_to_offset_map = { 'days': Day(1), 'hours': Hour(1), 'minutes': Minute(1), 'seconds': Second(1), 'milliseconds': Milli(1), 'microseconds': Micro(1), 'nanoseconds': Nano(1) } _INVALID_FREQ_ERROR = "Invalid frequency: {0}" def to_offset(freqstr): """ Return DateOffset object from string representation or Timedelta object Examples -------- >>> to_offset('5Min')
def test_Millisecond(): assert_offset_equal(Milli(), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 1000)) assert_offset_equal(Milli(-1), datetime(2010, 1, 1, 0, 0, 0, 1000), datetime(2010, 1, 1)) assert_offset_equal(Milli(2), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 2000)) assert_offset_equal(2 * Milli(), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 2000)) assert_offset_equal(-1 * Milli(), datetime(2010, 1, 1, 0, 0, 0, 1000), datetime(2010, 1, 1)) assert Milli(3) + Milli(2) == Milli(5) assert Milli(3) - Milli(2) == Milli()
_offset_map: Dict[str, DateOffset] = {} def get_period_alias(offset_str: str) -> Optional[str]: """ Alias to closest period strings BQ->Q etc. """ return _offset_to_period_map.get(offset_str, None) _name_to_offset_map = { "days": Day(1), "hours": Hour(1), "minutes": Minute(1), "seconds": Second(1), "milliseconds": Milli(1), "microseconds": Micro(1), "nanoseconds": Nano(1), } def to_offset(freq) -> Optional[DateOffset]: """ Return DateOffset object from string or tuple representation or datetime.timedelta object. Parameters ---------- freq : str, tuple, datetime.timedelta, DateOffset or None Returns
class TestDateTimeConverter: @pytest.fixture def dtc(self): return converter.DatetimeConverter() def test_convert_accepts_unicode(self, dtc): r1 = dtc.convert("12:22", None, None) r2 = dtc.convert("12:22", None, None) assert r1 == r2, "DatetimeConverter.convert should accept unicode" def test_conversion(self, dtc): rs = dtc.convert(["2012-1-1"], None, None)[0] xp = dates.date2num(datetime(2012, 1, 1)) assert rs == xp rs = dtc.convert("2012-1-1", None, None) assert rs == xp rs = dtc.convert(date(2012, 1, 1), None, None) assert rs == xp rs = dtc.convert("2012-1-1", None, None) assert rs == xp rs = dtc.convert(Timestamp("2012-1-1"), None, None) assert rs == xp # also testing datetime64 dtype (GH8614) rs = dtc.convert("2012-01-01", None, None) assert rs == xp rs = dtc.convert("2012-01-01 00:00:00+0000", None, None) assert rs == xp rs = dtc.convert( np.array(["2012-01-01 00:00:00+0000", "2012-01-02 00:00:00+0000"]), None, None, ) assert rs[0] == xp # we have a tz-aware date (constructed to that when we turn to utc it # is the same as our sample) ts = Timestamp("2012-01-01").tz_localize("UTC").tz_convert( "US/Eastern") rs = dtc.convert(ts, None, None) assert rs == xp rs = dtc.convert(ts.to_pydatetime(), None, None) assert rs == xp rs = dtc.convert(Index([ts - Day(1), ts]), None, None) assert rs[1] == xp rs = dtc.convert(Index([ts - Day(1), ts]).to_pydatetime(), None, None) assert rs[1] == xp def test_conversion_float(self, dtc): rtol = 0.5 * 10**-9 rs = dtc.convert(Timestamp("2012-1-1 01:02:03", tz="UTC"), None, None) xp = converter.dates.date2num(Timestamp("2012-1-1 01:02:03", tz="UTC")) tm.assert_almost_equal(rs, xp, rtol=rtol) rs = dtc.convert(Timestamp("2012-1-1 09:02:03", tz="Asia/Hong_Kong"), None, None) tm.assert_almost_equal(rs, xp, rtol=rtol) rs = dtc.convert(datetime(2012, 1, 1, 1, 2, 3), None, None) tm.assert_almost_equal(rs, xp, rtol=rtol) def test_conversion_outofbounds_datetime(self, dtc): # 2579 values = [date(1677, 1, 1), date(1677, 1, 2)] rs = dtc.convert(values, None, None) xp = converter.dates.date2num(values) tm.assert_numpy_array_equal(rs, xp) rs = dtc.convert(values[0], None, None) xp = converter.dates.date2num(values[0]) assert rs == xp values = [datetime(1677, 1, 1, 12), datetime(1677, 1, 2, 12)] rs = dtc.convert(values, None, None) xp = converter.dates.date2num(values) tm.assert_numpy_array_equal(rs, xp) rs = dtc.convert(values[0], None, None) xp = converter.dates.date2num(values[0]) assert rs == xp @pytest.mark.parametrize( "time,format_expected", [ (0, "00:00"), # time2num(datetime.time.min) (86399.999999, "23:59:59.999999"), # time2num(datetime.time.max) (90000, "01:00"), (3723, "01:02:03"), (39723.2, "11:02:03.200"), ], ) def test_time_formatter(self, time, format_expected): # issue 18478 result = converter.TimeFormatter(None)(time) assert result == format_expected @pytest.mark.parametrize("freq", ("B", "L", "S")) def test_dateindex_conversion(self, freq, dtc): rtol = 10**-9 dateindex = tm.makeDateIndex(k=10, freq=freq) rs = dtc.convert(dateindex, None, None) xp = converter.dates.date2num(dateindex._mpl_repr()) tm.assert_almost_equal(rs, xp, rtol=rtol) @pytest.mark.parametrize("offset", [Second(), Milli(), Micro(50)]) def test_resolution(self, offset, dtc): # Matplotlib's time representation using floats cannot distinguish # intervals smaller than ~10 microsecond in the common range of years. ts1 = Timestamp("2012-1-1") ts2 = ts1 + offset val1 = dtc.convert(ts1, None, None) val2 = dtc.convert(ts2, None, None) if not val1 < val2: raise AssertionError(f"{val1} is not less than {val2}.") def test_convert_nested(self, dtc): inner = [Timestamp("2017-01-01"), Timestamp("2017-01-02")] data = [inner, inner] result = dtc.convert(data, None, None) expected = [dtc.convert(x, None, None) for x in data] assert (np.array(result) == expected).all()
def test_ms_vs_MS(self): left = get_offset('ms') right = get_offset('MS') self.assertEqual(left, Milli()) self.assertEqual(right, MonthBegin())
def import_trajectory(filepath, delim_whitespace=False, interval=0, interp=False, is_utc=False, columns=None, skiprows=None, timeformat='sow'): """ Read and parse ASCII trajectory data in a comma-delimited format. Parameters ---------- filepath : str or File-like object. Filesystem path to trajectory data file delim_whitespace : bool interval : float, Optional Output data rate. Default behavior is to infer the rate. interp : Union[List[str], List[int]], Optional Gaps in data will be filled with interpolated values. List of column indices (list of ints) or list of column names (list of strs) to interpolate. Default behavior is not to interpolate. is_utc : bool, Optional Indicates that the timestamps are UTC. The index datetimes will be shifted to remove the GPS-UTC leap second offset. columns : List[str] Strings to use as the column names. If none supplied (default), columns will be determined based on timeformat skiprows : Union[None, Iterable, int, Callable], Optional Line numbers to skip (0-indexed) or number of lines to skip (int) at the start of the file. If callable, the callable function will be evaluated against the row indices, returning True if the row should be skipped and False otherwise. An example of a valid callable argument would be lambda x: x in [0, 2]. timeformat : str 'sow' | 'hms' | 'serial' Default: 'hms' Indicates the time format to expect. The 'sow' format requires a field named 'week' with the GPS week, and a field named 'sow' with the GPS seconds of week. The 'hms' format requires a field named 'mdy' with the date in the format 'MM/DD/YYYY', and a field named 'hms' with the time in the format 'HH:MM:SS.SSS'. The 'serial' format (not yet implemented) requires a field named 'datenum' with the serial date number. Returns ------- DataFrame Pandas DataFrame of ingested Trajectory data. """ df = pd.read_csv(filepath, delim_whitespace=delim_whitespace, header=None, engine='c', na_filter=False, skiprows=skiprows) # assumed position of these required fields if columns is None: if timeformat.lower() == 'sow': columns = ['week', 'sow', 'lat', 'long', 'ell_ht'] elif timeformat.lower() == 'hms': columns = ['mdy', 'hms', 'lat', 'long', 'ell_ht'] elif timeformat.lower() == 'serial': columns = ['datenum', 'lat', 'long', 'ell_ht'] else: raise ValueError('timeformat value {fmt!r} not recognized'.format( fmt=timeformat)) # 'None' indicates a not-needed field # if a field is after all non-essentials, and is not named, it will be removed if len(df.columns) > len(columns): columns.extend([None] * (len(df.columns) - len(columns))) # drop unwanted columns drop_list = list() for idx, val in enumerate(columns): if val is None: drop_list.append(idx) columns = [x for x in columns if x is not None] if drop_list: df.drop(df.columns[drop_list], axis=1, inplace=True) df.columns = columns # create index if timeformat == 'sow': df.index = convert_gps_time(df['week'], df['sow'], format='datetime') df.index = df.index.round(Milli()) df.drop(['sow', 'week'], axis=1, inplace=True) elif timeformat == 'hms': df.index = pd.to_datetime(df['mdy'].str.strip() + df['hms'].str.strip(), format="%m/%d/%Y%H:%M:%S.%f") df.drop(['mdy', 'hms'], axis=1, inplace=True) elif timeformat == 'serial': raise NotImplementedError # df.index = datenum_to_datetime(df['datenum']) # remove leap second if is_utc: # TO DO: Check dates at beginning and end to determine whether a leap second was added in the middle of the survey. shift = leap_seconds(datetime=df.index[0]) df.index = df.index.shift(-shift, freq='S') # set or infer the interval # TO DO: Need to infer interval for both cases to know whether resample if interval > 0: offset_str = '{:d}U'.format(int(interval * 1e6)) else: offset_str = '100000U' # fill gaps with NaNs new_index = pd.date_range(df.index[0], df.index[-1], freq=offset_str) df = df.reindex(new_index) if interp: numeric = df.select_dtypes(include=[np.number]) numeric = numeric.interpolate(method='time') # replace columns for col in numeric.columns: df[col] = numeric[col] return df
def _dateparse(self, time): return pd.Timedelta(Milli(time))