def api_fio_json(fio_result_id, job_id, log_type):
start_frame = request.args.get('start_frame')
end_frame = request.args.get('end_frame')
granularity = request.args.get('granularity', '1S')
io_type = request.args.get('io_type')
fio_data = FioResult(DATA_PATH, fio_result_id)
if start_frame and end_frame:
data_frame = fio_data.to_dataframe(job_id, log_type, io_type)
if data_frame is None:
return jsonify(dict(error='404'))
# return error 404 with
# html status code 200
# so we can catch this error
# directly in frontend
# without triggering errors.
data_frame = data_frame[(data_frame.index > Milli(start_frame)) &
(data_frame.index < Milli(end_frame))]
if "iops" not in log_type:
data_frame[1] = data_frame[1].astype(float).values / 1000
data_frame[1] = data_frame[1].round(2)
if granularity:
data_frame = data_frame.\
resample(granularity, label='right', closed='right').mean().\
fillna('None')
return jsonify(dict(
x=data_frame.index.astype('timedelta64[s]').values.tolist(),
y=data_frame[1].values.tolist()
))
return jsonify(fio_data.to_json(job_id, log_type))
def test_MillisecondTimestampArithmetic():
assert_offset_equal(
Milli(), Timestamp("2010-01-01"), Timestamp("2010-01-01 00:00:00.001")
)
assert_offset_equal(
Milli(-1), Timestamp("2010-01-01 00:00:00.001"), Timestamp("2010-01-01")
)
def api_fio_json_combined(fio_result_id, log_type):
start_frame = request.args.get('start_frame')
end_frame = request.args.get('end_frame')
granularity = request.args.get('granularity', '1S')
io_type = request.args.get('io_type')
fio_data = FioResult(DATA_PATH, fio_result_id)
fio_results = []
for fio_group_report in fio_data.group_reports:
for job in fio_group_report.jobs:
fio_result = fio_data.to_dataframe(job.job_id, log_type, io_type)
if fio_result is not None:
if "iops" not in log_type:
fio_result[1] = fio_result[1].astype(float).values / 1000
fio_result[1] = fio_result[1].round(2)
fio_result = fio_result.resample(granularity, label='right',
closed='right').mean()
fio_results.append(fio_result)
if not fio_results:
return jsonify(dict(error='404'))
# return error 404 with
# html status code 200
# so we can catch this error
# directly in frontend
# without triggering errors.
data_frame = pd.concat(fio_results, join='inner')
data_frame = data_frame.sort_index()
data_frame = data_frame.resample(granularity, label='right',
closed='right')
if "lat" in log_type:
data_frame = data_frame.mean()
else:
data_frame = data_frame.sum()
data_frame = data_frame.fillna('None')
if start_frame and end_frame:
data_frame = data_frame[(data_frame.index > Milli(start_frame)) &
(data_frame.index < Milli(end_frame))]
return jsonify(dict(x=data_frame.index.astype('timedelta64[s]')
.values.tolist(), y=data_frame[1].values.tolist()))
def test_resolution(self):
def _assert_less(ts1, ts2):
val1 = self.dtc.convert(ts1, None, None)
val2 = self.dtc.convert(ts2, None, None)
if not val1 < val2:
raise AssertionError(f"{val1} is not less than {val2}.")
# Matplotlib's time representation using floats cannot distinguish
# intervals smaller than ~10 microsecond in the common range of years.
ts = Timestamp("2012-1-1")
_assert_less(ts, ts + Second())
_assert_less(ts, ts + Milli())
_assert_less(ts, ts + Micro(50))
# Offset names ("time rules") and related functions
from pandas.tseries.offsets import (Day, BDay, Hour, Minute, Second, Milli,
Week, Micro, MonthEnd, MonthBegin,
BMonthBegin, BMonthEnd, YearBegin, YearEnd,
BYearBegin, BYearEnd, QuarterBegin,
QuarterEnd, BQuarterBegin, BQuarterEnd)
_offset_map = {
'D' : Day(),
'B' : BDay(),
'H' : Hour(),
'T' : Minute(),
'S' : Second(),
'L' : Milli(),
'U' : Micro(),
None : None,
# Monthly - Calendar
'M' : MonthEnd(),
'MS' : MonthBegin(),
# Monthly - Business
'BM' : BMonthEnd(),
'BMS' : BMonthBegin(),
# Annual - Calendar
'A-JAN' : YearEnd(month=1),
'A-FEB' : YearEnd(month=2),
'A-MAR' : YearEnd(month=3),
'AS': 'AS-JAN', # YearBegin(month=1),
'BA': 'BA-DEC', # BYearEnd(month=12),
'BAS': 'BAS-JAN', # BYearBegin(month=1),
'Min': 'T',
'min': 'T',
'ms': 'L',
'us': 'U',
'ns': 'N'
}
_name_to_offset_map = {
'days': Day(1),
'hours': Hour(1),
'minutes': Minute(1),
'seconds': Second(1),
'milliseconds': Milli(1),
'microseconds': Micro(1),
'nanoseconds': Nano(1)
}
_INVALID_FREQ_ERROR = "Invalid frequency: {0}"
def to_offset(freqstr):
"""
Return DateOffset object from string representation or
Timedelta object
Examples
--------
>>> to_offset('5Min')
def test_Millisecond():
assert_offset_equal(Milli(), datetime(2010, 1, 1),
datetime(2010, 1, 1, 0, 0, 0, 1000))
assert_offset_equal(Milli(-1), datetime(2010, 1, 1, 0, 0, 0, 1000),
datetime(2010, 1, 1))
assert_offset_equal(Milli(2), datetime(2010, 1, 1),
datetime(2010, 1, 1, 0, 0, 0, 2000))
assert_offset_equal(2 * Milli(), datetime(2010, 1, 1),
datetime(2010, 1, 1, 0, 0, 0, 2000))
assert_offset_equal(-1 * Milli(), datetime(2010, 1, 1, 0, 0, 0, 1000),
datetime(2010, 1, 1))
assert Milli(3) + Milli(2) == Milli(5)
assert Milli(3) - Milli(2) == Milli()
_offset_map: Dict[str, DateOffset] = {}
def get_period_alias(offset_str: str) -> Optional[str]:
"""
Alias to closest period strings BQ->Q etc.
"""
return _offset_to_period_map.get(offset_str, None)
_name_to_offset_map = {
"days": Day(1),
"hours": Hour(1),
"minutes": Minute(1),
"seconds": Second(1),
"milliseconds": Milli(1),
"microseconds": Micro(1),
"nanoseconds": Nano(1),
}
def to_offset(freq) -> Optional[DateOffset]:
"""
Return DateOffset object from string or tuple representation
or datetime.timedelta object.
Parameters
----------
freq : str, tuple, datetime.timedelta, DateOffset or None
Returns
class TestDateTimeConverter:
@pytest.fixture
def dtc(self):
return converter.DatetimeConverter()
def test_convert_accepts_unicode(self, dtc):
r1 = dtc.convert("12:22", None, None)
r2 = dtc.convert("12:22", None, None)
assert r1 == r2, "DatetimeConverter.convert should accept unicode"
def test_conversion(self, dtc):
rs = dtc.convert(["2012-1-1"], None, None)[0]
xp = dates.date2num(datetime(2012, 1, 1))
assert rs == xp
rs = dtc.convert("2012-1-1", None, None)
assert rs == xp
rs = dtc.convert(date(2012, 1, 1), None, None)
assert rs == xp
rs = dtc.convert("2012-1-1", None, None)
assert rs == xp
rs = dtc.convert(Timestamp("2012-1-1"), None, None)
assert rs == xp
# also testing datetime64 dtype (GH8614)
rs = dtc.convert("2012-01-01", None, None)
assert rs == xp
rs = dtc.convert("2012-01-01 00:00:00+0000", None, None)
assert rs == xp
rs = dtc.convert(
np.array(["2012-01-01 00:00:00+0000", "2012-01-02 00:00:00+0000"]),
None,
None,
)
assert rs[0] == xp
# we have a tz-aware date (constructed to that when we turn to utc it
# is the same as our sample)
ts = Timestamp("2012-01-01").tz_localize("UTC").tz_convert(
"US/Eastern")
rs = dtc.convert(ts, None, None)
assert rs == xp
rs = dtc.convert(ts.to_pydatetime(), None, None)
assert rs == xp
rs = dtc.convert(Index([ts - Day(1), ts]), None, None)
assert rs[1] == xp
rs = dtc.convert(Index([ts - Day(1), ts]).to_pydatetime(), None, None)
assert rs[1] == xp
def test_conversion_float(self, dtc):
rtol = 0.5 * 10**-9
rs = dtc.convert(Timestamp("2012-1-1 01:02:03", tz="UTC"), None, None)
xp = converter.dates.date2num(Timestamp("2012-1-1 01:02:03", tz="UTC"))
tm.assert_almost_equal(rs, xp, rtol=rtol)
rs = dtc.convert(Timestamp("2012-1-1 09:02:03", tz="Asia/Hong_Kong"),
None, None)
tm.assert_almost_equal(rs, xp, rtol=rtol)
rs = dtc.convert(datetime(2012, 1, 1, 1, 2, 3), None, None)
tm.assert_almost_equal(rs, xp, rtol=rtol)
def test_conversion_outofbounds_datetime(self, dtc):
# 2579
values = [date(1677, 1, 1), date(1677, 1, 2)]
rs = dtc.convert(values, None, None)
xp = converter.dates.date2num(values)
tm.assert_numpy_array_equal(rs, xp)
rs = dtc.convert(values[0], None, None)
xp = converter.dates.date2num(values[0])
assert rs == xp
values = [datetime(1677, 1, 1, 12), datetime(1677, 1, 2, 12)]
rs = dtc.convert(values, None, None)
xp = converter.dates.date2num(values)
tm.assert_numpy_array_equal(rs, xp)
rs = dtc.convert(values[0], None, None)
xp = converter.dates.date2num(values[0])
assert rs == xp
@pytest.mark.parametrize(
"time,format_expected",
[
(0, "00:00"), # time2num(datetime.time.min)
(86399.999999, "23:59:59.999999"), # time2num(datetime.time.max)
(90000, "01:00"),
(3723, "01:02:03"),
(39723.2, "11:02:03.200"),
],
)
def test_time_formatter(self, time, format_expected):
# issue 18478
result = converter.TimeFormatter(None)(time)
assert result == format_expected
@pytest.mark.parametrize("freq", ("B", "L", "S"))
def test_dateindex_conversion(self, freq, dtc):
rtol = 10**-9
dateindex = tm.makeDateIndex(k=10, freq=freq)
rs = dtc.convert(dateindex, None, None)
xp = converter.dates.date2num(dateindex._mpl_repr())
tm.assert_almost_equal(rs, xp, rtol=rtol)
@pytest.mark.parametrize("offset", [Second(), Milli(), Micro(50)])
def test_resolution(self, offset, dtc):
# Matplotlib's time representation using floats cannot distinguish
# intervals smaller than ~10 microsecond in the common range of years.
ts1 = Timestamp("2012-1-1")
ts2 = ts1 + offset
val1 = dtc.convert(ts1, None, None)
val2 = dtc.convert(ts2, None, None)
if not val1 < val2:
raise AssertionError(f"{val1} is not less than {val2}.")
def test_convert_nested(self, dtc):
inner = [Timestamp("2017-01-01"), Timestamp("2017-01-02")]
data = [inner, inner]
result = dtc.convert(data, None, None)
expected = [dtc.convert(x, None, None) for x in data]
assert (np.array(result) == expected).all()
def test_ms_vs_MS(self):
left = get_offset('ms')
right = get_offset('MS')
self.assertEqual(left, Milli())
self.assertEqual(right, MonthBegin())
def import_trajectory(filepath,
delim_whitespace=False,
interval=0,
interp=False,
is_utc=False,
columns=None,
skiprows=None,
timeformat='sow'):
"""
Read and parse ASCII trajectory data in a comma-delimited format.
Parameters
----------
filepath : str or File-like object.
Filesystem path to trajectory data file
delim_whitespace : bool
interval : float, Optional
Output data rate. Default behavior is to infer the rate.
interp : Union[List[str], List[int]], Optional
Gaps in data will be filled with interpolated values. List of
column indices (list of ints) or list of column names (list of strs)
to interpolate. Default behavior is not to interpolate.
is_utc : bool, Optional
Indicates that the timestamps are UTC. The index datetimes will be
shifted to remove the GPS-UTC leap second offset.
columns : List[str]
Strings to use as the column names.
If none supplied (default), columns will be determined based on
timeformat
skiprows : Union[None, Iterable, int, Callable], Optional
Line numbers to skip (0-indexed) or number of lines to skip (int) at
the start of the file. If callable, the callable function will be
evaluated against the row indices, returning True if the row should
be skipped and False otherwise. An example of a valid callable argument
would be lambda x: x in [0, 2].
timeformat : str
'sow' | 'hms' | 'serial' Default: 'hms'
Indicates the time format to expect. The 'sow' format requires a field
named 'week' with the GPS week, and a field named 'sow' with the GPS
seconds of week. The 'hms' format requires a field named 'mdy' with the
date in the format 'MM/DD/YYYY', and a field named 'hms' with the time
in the format 'HH:MM:SS.SSS'. The 'serial' format (not yet implemented)
requires a field named 'datenum' with the serial date number.
Returns
-------
DataFrame
Pandas DataFrame of ingested Trajectory data.
"""
df = pd.read_csv(filepath,
delim_whitespace=delim_whitespace,
header=None,
engine='c',
na_filter=False,
skiprows=skiprows)
# assumed position of these required fields
if columns is None:
if timeformat.lower() == 'sow':
columns = ['week', 'sow', 'lat', 'long', 'ell_ht']
elif timeformat.lower() == 'hms':
columns = ['mdy', 'hms', 'lat', 'long', 'ell_ht']
elif timeformat.lower() == 'serial':
columns = ['datenum', 'lat', 'long', 'ell_ht']
else:
raise ValueError('timeformat value {fmt!r} not recognized'.format(
fmt=timeformat))
# 'None' indicates a not-needed field
# if a field is after all non-essentials, and is not named, it will be removed
if len(df.columns) > len(columns):
columns.extend([None] * (len(df.columns) - len(columns)))
# drop unwanted columns
drop_list = list()
for idx, val in enumerate(columns):
if val is None:
drop_list.append(idx)
columns = [x for x in columns if x is not None]
if drop_list:
df.drop(df.columns[drop_list], axis=1, inplace=True)
df.columns = columns
# create index
if timeformat == 'sow':
df.index = convert_gps_time(df['week'], df['sow'], format='datetime')
df.index = df.index.round(Milli())
df.drop(['sow', 'week'], axis=1, inplace=True)
elif timeformat == 'hms':
df.index = pd.to_datetime(df['mdy'].str.strip() +
df['hms'].str.strip(),
format="%m/%d/%Y%H:%M:%S.%f")
df.drop(['mdy', 'hms'], axis=1, inplace=True)
elif timeformat == 'serial':
raise NotImplementedError
# df.index = datenum_to_datetime(df['datenum'])
# remove leap second
if is_utc:
# TO DO: Check dates at beginning and end to determine whether a leap second was added in the middle of the survey.
shift = leap_seconds(datetime=df.index[0])
df.index = df.index.shift(-shift, freq='S')
# set or infer the interval
# TO DO: Need to infer interval for both cases to know whether resample
if interval > 0:
offset_str = '{:d}U'.format(int(interval * 1e6))
else:
offset_str = '100000U'
# fill gaps with NaNs
new_index = pd.date_range(df.index[0], df.index[-1], freq=offset_str)
df = df.reindex(new_index)
if interp:
numeric = df.select_dtypes(include=[np.number])
numeric = numeric.interpolate(method='time')
# replace columns
for col in numeric.columns:
df[col] = numeric[col]
return df
def _dateparse(self, time):
return pd.Timedelta(Milli(time))
