def test_encode_cdftt2000():
x = cdfepoch.encode(186999622360321123)
assert x == '2005-12-04T20:19:18.176321123'
y = cdfepoch.encode([500000000100, 123456789101112131],
iso_8601=False)
assert y[0] == '01-Jan-2000 12:07:15.816.000.100'
assert y[1] == '30-Nov-2003 09:32:04.917.112.131'
def test_encode_cdfepoch():
x = cdfepoch.encode([62285326000000.0, 62985326000000.0])
assert x[0] == '1973-09-28T23:26:40.000'
assert x[1] == '1995-12-04T19:53:20.000'
y = cdfepoch.encode(62975326000002.0, iso_8601=False)
assert y == '11-Aug-1995 02:06:40.002'
def test_parse_cdftt2000():
input_time = 131415926535793238
x = cdfepoch.encode(input_time)
assert x == "2004-03-01T12:24:22.351793238"
parsed = cdfepoch.parse(x)
assert parsed == input_time
assert cdfepoch().to_datetime(parsed) == [datetime(2004, 3, 1, 12, 24, 22, 351793)]
def test_parse_cdfepoch16():
input_time = 53467976543.0 + 543218654100j
x = cdfepoch.encode(input_time)
assert x == "1694-05-01T07:42:23.543218654100"
parsed = cdfepoch.parse(x)
assert parsed == input_time
assert cdfepoch().to_datetime(parsed) == [datetime(1694, 5, 1, 7, 42, 23, 543218)]
def test_compute_cdftt2000(dtime):
random_time = [dtime.year, dtime.month, dtime.day,
dtime.hour, dtime.minute, dtime.second,
dtime.microsecond // 1000, # Millisecond
randint(0, 999), # Microsecond
randint(0, 999), # Nanosecond
]
x = cdfepoch.breakdown(cdfepoch.compute(random_time))
for i, t in enumerate(x):
assert t == random_time[i], f'Time {random_time} was not equal to {x}'
def test_unixtime_roundtrip(tzone):
_environ = os.environ.copy()
try:
os.environ['TZ'] = tzone
y, m, d = 2000, 1, 1
epoch = cdfepoch.compute_tt2000([[y, m, d]])
unixtime = cdfepoch.unixtime(epoch)
assert unixtime == [946684800.0]
finally:
os.environ.clear()
os.environ.update(_environ)
def test_compute_cdfepoch(dtime):
'''
Using random numbers for the compute tests
'''
random_time = [dtime.year, dtime.month, dtime.day,
dtime.hour, dtime.minute, dtime.second,
dtime.microsecond // 1000]
x = cdfepoch.breakdown(cdfepoch.compute(random_time))
i = 0
for t in x:
assert t == random_time[i], f'Time {random_time} was not equal to {x}'
i += 1
def test_compute_cdfepoch16(dtime):
random_time = [dtime.year, dtime.month, dtime.day,
dtime.hour, dtime.minute, dtime.second,
dtime.microsecond // 1000, # Millisecond
randint(0, 999), # Microsecond
randint(0, 999), # Nanosecond
randint(0, 999), # Picosecond
]
x = cdfepoch.breakdown(cdfepoch.compute(random_time))
i = 0
for t in x:
assert t == random_time[i], f'Time {random_time} was not equal to {x}'
i += 1
def test_findepochrange_cdfepoch():
start_time = "2013-12-01T12:24:22.000"
end_time = "2014-12-01T12:24:22.000"
x = cdfepoch.parse([start_time, end_time])
time_array = np.arange(x[0], x[1], step=1000000)
test_start = [2014, 8, 1, 8, 1, 54, 123]
test_end = [2018, 1, 1, 1, 1, 1, 1]
index = cdfepoch.findepochrange(time_array, starttime=test_start, endtime=test_end)
# Test that the test_start is less than the first index, but more than one less
assert time_array[index[0]] >= cdfepoch.compute(test_start)
assert time_array[index[0]-1] <= cdfepoch.compute(test_start)
assert time_array[index[-1]] <= cdfepoch.compute(test_end)
def test_encode_cdfepoch16():
'''
cdf_encode_epoch16(dcomplex(63300946758.000000, 176214648000.00000)) in IDL
returns 04-Dec-2005 20:39:28.176.214.654.976
However, I believe this IDL routine is bugged. This website:
https://www.epochconverter.com/seconds-days-since-y0
shows a correct answer.
'''
x = cdfepoch.encode(np.complex128(63300946758.000000 + 176214648000.00000j))
assert x == '2005-12-04T20:19:18.176214648000'
y = cdfepoch.encode(np.complex128([33300946758.000000 + 106014648000.00000j,
61234543210.000000 + 000011148000.00000j]),
iso_8601=False)
assert y[0] == '07-Apr-1055 14:59:18.106.014.648.000'
assert y[1] == '12-Jun-1940 03:20:10.000.011.148.000'
def test_compute_cdftt2000():
random_time = []
random_time.append(randint(0, 2018)) # Year
random_time.append(randint(1, 12)) # Month
random_time.append(randint(1, 28)) # Date
random_time.append(randint(0, 23)) # Hour
random_time.append(randint(0, 59)) # Minute
random_time.append(randint(0, 59)) # Second
random_time.append(randint(0, 999)) # Millisecond
random_time.append(randint(0, 999)) # Microsecond
random_time.append(randint(0, 999)) # Nanosecond
x = cdfepoch.breakdown(cdfepoch.compute(random_time))
i = 0
for t in x:
assert t == random_time[i], 'Time {} was not equal to {}'.format(
random_time, x)
i += 1
def test_compute_cdftt2000():
random_time = []
# These are the supported years for CDF files; see
# https://spdf.gsfc.nasa.gov/pub/software/cdf/doc/cdf371/cdf371ug.pdf
# page 55
random_time.append(randint(1709, 2292)) # Year
random_time.append(randint(1, 12)) # Month
random_time.append(randint(1, 28)) # Date
random_time.append(randint(0, 23)) # Hour
random_time.append(randint(0, 59)) # Minute
random_time.append(randint(0, 59)) # Second
random_time.append(randint(0, 999)) # Millisecond
random_time.append(randint(0, 999)) # Microsecond
random_time.append(randint(0, 999)) # Nanosecond
x = cdfepoch.breakdown(cdfepoch.compute(random_time))
i = 0
for t in x:
assert t == random_time[i], 'Time {} was not equal to {}'.format(
random_time, x)
i += 1
def test_breakdown_cdftt2000():
x = cdfepoch.breakdown(123456789101112131)
assert x[0] == 2003
assert x[1] == 11
assert x[2] == 30
assert x[3] == 9
assert x[4] == 32
assert x[5] == 4
assert x[6] == 917
assert x[7] == 112
assert x[8] == 131
def test_breakdown_cdfepoch16():
x = cdfepoch.breakdown(np.complex128(63300946758.000000 + 176214648000.00000j))
assert x[0] == 2005
assert x[1] == 12
assert x[2] == 4
assert x[3] == 20
assert x[4] == 19
assert x[5] == 18
assert x[6] == 176
assert x[7] == 214
assert x[8] == 648
assert x[9] == 000
def get_dataset_raw(self, keys, WFR_file_id=0):
""" Convert the raw data to dict format """
self.epoch = None
o = {"Epoch": []}
for f in self.files["file_objects"]:
dates = [
dt.datetime(i[0], i[1], i[2], i[3], i[4], i[5])
for i in CDFepoch.breakdown(f.varget("Epoch"))
]
o["Epoch"].extend(dates)
if self.verbose: print(f.cdf_info())
for key in keys:
if key not in o.keys(): o[key] = f.varget(key)[:]
else: o[key] = np.concatenate(o[key], f.varget(key)[:])
if WFR_file_id is not None: o["WFR"] = self.get_WFR_info(WFR_file_id)
self.epoch = o["Epoch"]
return o
def test_breakdown_cdfepoch():
x = cdfepoch.breakdown([62285326000000.0, 62985326000000.0])
# First in the array
assert x[0][0] == 1973
assert x[0][1] == 9
assert x[0][2] == 28
assert x[0][3] == 23
assert x[0][4] == 26
assert x[0][5] == 40
assert x[0][6] == 0
# Second in the array
assert x[1][0] == 1995
assert x[1][1] == 12
assert x[1][2] == 4
assert x[1][3] == 19
assert x[1][4] == 53
assert x[1][5] == 20
assert x[1][6] == 0
def test_findepochrange_cdftt2000():
start_time = "2004-03-01T12:24:22.351793238"
end_time = "2004-03-01T12:28:22.351793238"
x = cdfepoch.parse([start_time, end_time])
time_array = np.arange(x[0], x[1], step=1000000)
test_start = [2004, 3, 1, 12, 25, 54, 123, 111, 98]
test_end = [2004, 3, 1, 12, 26, 4, 123, 456, 789]
index = cdfepoch.findepochrange(time_array, starttime=test_start, endtime=test_end)
# Test that the test_start is less than the first index, but more than one less
assert time_array[index[0]] >= cdfepoch.compute(test_start)
assert time_array[index[0]-1] <= cdfepoch.compute(test_start)
assert time_array[index[-1]] <= cdfepoch.compute(test_end)
assert time_array[index[-1]+1] >= cdfepoch.compute(test_end)
def test_findepochrange_cdfepoch16():
start_time = "1978-03-10T03:24:22.351793238462"
end_time = "1978-06-13T01:28:22.338327950466"
x = cdfepoch.parse([start_time, end_time])
first_int_step = int((x[1].real - x[0].real) / 1000)
second_int_step = int((x[1].imag - x[0].imag) / 1000)
time_array = []
for i in range(0, 1000):
time_array.append(x[0]+complex(first_int_step*i, second_int_step*i))
test_start = [1978, 6, 10, 3, 24, 22, 351, 793, 238, 462]
test_end = [1978, 6, 12, 23, 11, 1, 338, 341, 416, 466]
index = cdfepoch.findepochrange(time_array, starttime=test_start, endtime=test_end)
# Test that the test_start is less than the first index, but more than one less
assert time_array[index[0]].real >= cdfepoch.compute(test_start).real
assert time_array[index[0]-1].real <= cdfepoch.compute(test_start).real
assert time_array[index[-1]].real <= cdfepoch.compute(test_end).real
assert time_array[index[-1]+1].real >= cdfepoch.compute(test_end).real
def test_unixtime():
x = cdfepoch.unixtime([500000000100, 123456789101112131])
assert x[0] == 946728435.816
assert x[1] == 1070184724.917112
def _convert_cdf_time_types(data,
atts,
properties,
to_datetime=False,
to_unixtime=False):
'''
# Converts CDF time types into either datetime objects, unixtime, or nothing
# If nothing, ALL CDF_EPOCH16 types are converted to CDF_EPOCH, because xarray can't handle int64s
'''
if not hasattr(data, '__len__'):
data = [data]
if to_datetime and to_unixtime:
print(
"Cannot convert to both unixtime and datetime. Continuing with conversion to unixtime."
)
to_datetime = False
# Convert all data in the "data" variable to unixtime or datetime if needed
data_type = properties['Data_Type_Description']
if len(data) == 0 or data_type not in ('CDF_EPOCH', 'CDF_EPOCH16',
'CDF_TIME_TT2000'):
new_data = data
else:
if to_datetime:
new_data = cdfepoch.to_datetime(data)
if 'UNITS' in atts:
atts['UNITS']['Data'] = 'Datetime (UTC)'
elif to_unixtime:
new_data = cdfepoch.unixtime(data)
if 'UNITS' in atts:
atts['UNITS']['Data'] = 'seconds'
else:
if data_type == 'CDF_EPOCH16':
new_data = cdfepoch.compute(cdfepoch.breakdown(data)[0:7])
else:
new_data = data
# Convert all the attributes in the "atts" dictionary to unixtime or datetime if needed
new_atts = {}
for att in atts:
data_type = atts[att]['Data_Type']
data = atts[att]['Data']
if not hasattr(data, '__len__'):
data = [data]
if len(data) == 0 or data_type not in ('CDF_EPOCH', 'CDF_EPOCH16',
'CDF_TIME_TT2000'):
new_atts[att] = data
else:
if to_datetime:
new_atts[att] = cdfepoch.to_datetime(data)
elif to_unixtime:
new_atts[att] = cdfepoch.unixtime(data)
else:
if data_type == 'CDF_EPOCH16':
new_atts[att] = cdfepoch.compute(
cdfepoch.breakdown(data)[0:7])
else:
new_atts[att] = data
return new_data, new_atts
def test_parse_cdfepoch():
x = cdfepoch.encode(62567898765432.0)
assert x == "1982-09-12T11:52:45.432"
parsed = cdfepoch.parse(x)
assert parsed == approx(62567898765432.0)
def time_epoch_encode(self, to_np):
cdfepoch.encode(self.epochs)
def time_epoch_to_datetime_tt2000(self, to_np):
cdfepoch.to_datetime(self.epochs_tt2000)
def time_epoch_to_datetime(self, to_np):
cdfepoch.to_datetime(self.epochs)
def read_cdf(fname):
"""
Read a CDF file that follows the ISTP/IACG guidelines.
Parameters
----------
fname : path-like
Location of single CDF file to read.
Returns
-------
list[GenericTimeSeries]
A list of time series objects, one for each unique time index within
the CDF file.
References
----------
Space Physics Guidelines for CDF https://spdf.gsfc.nasa.gov/sp_use_of_cdf.html
"""
cdf = cdflib.CDF(str(fname))
# Extract the time varying variables
cdf_info = cdf.cdf_info()
meta = cdf.globalattsget()
all_var_keys = cdf_info['rVariables'] + cdf_info['zVariables']
var_attrs = {key: cdf.varattsget(key) for key in all_var_keys}
# Get keys that depend on time
var_keys = [var for var in var_attrs if 'DEPEND_0' in var_attrs[var]]
# Get unique time index keys
time_index_keys = sorted(set([var_attrs[var]['DEPEND_0'] for var in var_keys]))
all_ts = []
# For each time index, construct a GenericTimeSeries
for index_key in time_index_keys:
try:
index = cdf.varget(index_key)
except ValueError:
# Empty index for cdflib >= 0.3.20
continue
if index is None:
# Empty index for cdflib <0.3.20
continue
# TODO: use to_astropy_time() instead here when we drop pandas in timeseries
index = CDFepoch.to_datetime(index)
df = pd.DataFrame(index=pd.DatetimeIndex(name=index_key, data=index))
units = {}
for var_key in sorted(var_keys):
attrs = var_attrs[var_key]
if attrs['DEPEND_0'] != index_key:
continue
# Get data
if cdf.varinq(var_key)['Last_Rec'] == -1:
log.debug(f'Skipping {var_key} in {fname} as it has zero elements')
continue
data = cdf.varget(var_key)
# Get units
if 'UNITS' in attrs:
unit_str = attrs['UNITS']
try:
unit = u.Unit(unit_str)
except ValueError:
if unit_str in _known_units:
unit = _known_units[unit_str]
else:
warn_user(f'astropy did not recognize units of "{unit_str}". '
'Assigning dimensionless units. '
'If you think this unit should not be dimensionless, '
'please raise an issue at https://github.com/sunpy/sunpy/issues')
unit = u.dimensionless_unscaled
else:
warn_user(f'No units provided for variable "{var_key}". '
'Assigning dimensionless units.')
unit = u.dimensionless_unscaled
if data.ndim == 2:
# Multiple columns, give each column a unique label
for i, col in enumerate(data.T):
df[var_key + f'_{i}'] = col
units[var_key + f'_{i}'] = unit
else:
# Single column
df[var_key] = data
units[var_key] = unit
all_ts.append(GenericTimeSeries(data=df, units=units, meta=meta))
if not len(all_ts):
log.debug(f'No data found in file {fname}')
return all_ts
def read_cdf(fname):
"""
Read a CDF file that follows the ISTP/IACG guidelines.
Parameters
----------
fname : path-like
Location of single CDF file to read.
Returns
-------
list[GenericTimeSeries]
A list of time series objects, one for each unique time index within
the CDF file.
References
----------
Space Physics Guidelines for CDF https://spdf.gsfc.nasa.gov/sp_use_of_cdf.html
"""
cdf = cdflib.CDF(str(fname))
# Extract the time varying variables
cdf_info = cdf.cdf_info()
meta = cdf.globalattsget()
all_var_keys = cdf_info['rVariables'] + cdf_info['zVariables']
var_attrs = {key: cdf.varattsget(key) for key in all_var_keys}
# Get keys that depend on time
var_keys = [var for var in var_attrs if 'DEPEND_0' in var_attrs[var]]
# Get unique time index keys
time_index_keys = sorted(
set([var_attrs[var]['DEPEND_0'] for var in var_keys]))
all_ts = []
# For each time index, construct a GenericTimeSeries
for index_key in time_index_keys:
index = cdf.varget(index_key)
# TODO: use to_astropy_time() instead here when we drop pandas in timeseries
index = CDFepoch.to_datetime(index)
df = pd.DataFrame(index=pd.DatetimeIndex(name=index_key, data=index))
units = {}
for var_key in sorted(var_keys):
attrs = var_attrs[var_key]
if attrs['DEPEND_0'] != index_key:
continue
# Get data
data = cdf.varget(var_key)
# Get units
unit = attrs['UNITS']
if unit in ['None', '', 'unitless']:
unit = u.dimensionless_unscaled
else:
unit = u.Unit(unit)
if data.ndim == 2:
# Multiple columns, give each column a unique label
for i, col in enumerate(data.T):
df[var_key + f'_{i}'] = col
units[var_key + f'_{i}'] = unit
else:
# Single column
df[var_key] = data
units[var_key] = unit
all_ts.append(GenericTimeSeries(data=df, units=units, meta=meta))
return all_ts