def test_catalog():
"""Request for catalog returns correct status and first dataset"""
meta = hapi(server)
assert meta['status'] == {
'code': 1200,
'message': 'OK'
} and meta['catalog'][0]['id'] == 'dataset1'
def test_cache_short():
# Compare read with empty cache with read with hot cache and usecache=True
dataset = 'dataset1'
start = '1970-01-01'
stop = '1970-01-01T00:00:03'
opts = {'logging': False, 'cachedir': '/tmp/hapi-data', 'cache': True}
opts['usecache'] = False
clearcache(opts)
data, meta = hapi(server, dataset, 'scalarint,vectorint', start, stop,
**opts)
opts['usecache'] = True
data2, meta2 = hapi(server, dataset, 'scalarint,vectorint', start, stop,
**opts)
assert np.array_equal(data, data2)
def test_subset_short():
dataset = 'dataset1'
start = '1970-01-01'
stop = '1970-01-01T00:00:03'
opts = {'logging': False, 'cachedir': '/tmp/hapi-data', 'cache': True}
opts['usecache'] = False
# Request two subsets with empty cache. Common parts should be same.
clearcache(opts)
data, meta = hapi(server, dataset, 'scalarint', start, stop, **opts)
clearcache(opts)
data2, meta2 = hapi(server, dataset, 'scalarint,vectorint', start, stop,
**opts)
ok = np.array_equal(data['Time'], data2['Time'])
ok = ok and np.array_equal(data['scalarint'], data2['scalarint'])
assert ok
# Request all parameters and single parameter. Common parameter should be same.
clearcache(opts)
data, meta = hapi(server, dataset, '', start, stop, **opts)
clearcache(opts)
data2, meta2 = hapi(server, dataset, 'vectorint', start, stop, **opts)
ok = np.array_equal(data['Time'], data2['Time'])
ok = ok and np.array_equal(data['vectorint'], data2['vectorint'])
assert ok
opts['usecache'] = True
# Request two subsets with hot cache. Common parts should be same.
data, meta = hapi(server, dataset, 'scalarint', start, stop, **opts)
data2, meta2 = hapi(server, dataset, 'scalarint,vectorint', start, stop,
**opts)
ok = np.array_equal(data['Time'], data2['Time'])
ok = ok and np.array_equal(data['scalarint'], data2['scalarint'])
assert ok
# Request all parameters and single parameter with hot cache. Common parameter should be same.
clearcache(opts)
data, meta = hapi(server, dataset, '', start, stop, **opts)
clearcache(opts)
data2, meta2 = hapi(server, dataset, 'vectorint', start, stop, **opts)
ok = np.array_equal(data['Time'], data2['Time'])
ok = ok and np.array_equal(data['vectorint'], data2['vectorint'])
assert ok
def test_parameter():
"""Request for dataset returns correct parameter metadata"""
meta = hapi(server, 'dataset1')
pklFile = 'test_parameter.pkl'
pklFile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data',
pklFile)
if not os.path.isfile(pklFile):
writepickle(pklFile, meta)
assert True
return
else:
metatest = readpickle(pklFile)
assert DeepDiff(meta, metatest) == {}
def test_bad_parameter():
"""Correct error when given nonexistent parameter name"""
with pytest.raises(Exception):
hapi(server, 'dataset1', 'scalarx')
def test_bad_dataset_name():
"""Correct error when given nonexistent dataset name"""
with pytest.raises(Exception):
hapi(server, 'dataset1x')
def test_bad_server_url():
"""Correct error when given bad URL"""
with pytest.raises(Exception):
hapi(serverbad, {'logging': True})
def test_server_list():
"""Test that specifying a server returns something."""
assert hapi(server) is not None
def test_hapi():
"""Test that a call with no parameters returns something."""
assert hapi() is not None
def hapiplot(*args, **kwargs):
"""Plot response from HAPI server.
Demos
-----
<https://github.com/hapi-server/client-python/blob/master/hapiclient/plot/hapiplot_test.py>
Usage
-----
data, meta = hapiplot(server, dataset, params, start, stop, **kwargs)
or
meta = hapiplot(data, meta, **kwargs)
where data and meta are return values from `hapi()`.
All parameters are plotted. If a parameter has a bins attribute,
it is plotted using `heatmap()`. Otherwise, it is plotted using
`timeseries()`.
Returns
-------
`data` is the same as that returned from `hapi()`.
`meta` is the same as that returned from `hapi()` with the additon of
meta['parameters'][i]['hapiplot']['figure'] is a reference to the
figure (e.g., plt.gcf()). Usage example:
>>> fig = meta['parameters'][i]['hapiplot']['figure']
>>> fig.set_facecolor('blue')
>>> fig.axes[0].set_ylabel('new y-label')
>>> fig.axes[0].set_title('new title\\nsubtitle\\nsubtitle')
>>> fig.tight_layout()
meta['parameters'][i]['hapiplot']['colorbar'] is a reference to the
colorbar on the figure (if parameter plotted as a heatmap)
meta['parameters'][i]['hapiplot']['image'] is PNG, PDF, or SVG data
and is included only if `returnimage=True`. Usage example:
>>> img = meta['parameters'][i]['hapiplot']['image']
>>> Image.open(io.BytesIO(img)).show()
>>> # or
>>> f = open('/tmp/a.png', 'wb')
>>> f.write(img)
>>> f.close()
See Also
---------
hapi: Get data from a HAPI server
timeseries: Used by `hapiplot()` to HAPI parameters with no `bins`
heatmap: Used by `hapiplot()` to HAPI parameters with `bins`
<https://github.com/hapi-server/client-python-notebooks>
kwargs
------
* logging: [False] Display console messages
* usecache: [True] Use cached data
* tsopts: {} kwargs for the `timeseries()` function
* hmopts: {} kwargs for the `heatmap()` function
Other kwargs
------------
* returnimage: [False] If True, `hapiplot()` returns binary image data
* returnformat: [png], svg, or pdf
* cachedir: Directory to store images. Default is hapiclient.hapi.cachedir()
* useimagecache: [True] Used cached image (when returnimage=True)
* saveimage: [False] Save image to `cachedir`
* saveformat: [png], svg, or pdf
Example
--------
>>> server = 'http://hapi-server.org/servers/TestData/hapi'
>>> dataset = 'dataset1'
>>> start = '1970-01-01T00:00:00'
>>> stop = '1970-01-02T00:00:00'
>>> params = 'scalar,vector'
>>> opts = {'logging': True}
>>>
>>> from hapiclient import hapiplot
>>> hapiplot(server, dataset, params, start, stop, **opts)
>>>
>>> # or
>>>
>>> from hapiclient import hapi, hapiplot
>>> data, meta = hapi(server, dataset, params, start, stop, **opts)
>>> hapiplot(data, meta, **opts)
"""
if len(args) == 5:
# For consistency with gallery and autoplot functions, allow useage of
# hapiplot(server, dataset, parameters, start, stop, **kwargs)
from hapiclient.hapi import hapiopts
from hapiclient.hapi import hapi
kwargs_allowed = hapiopts()
kwargs_reduced = {}
# Extract hapi() options from kwargs
for key, value in kwargs.items():
if key in kwargs_allowed:
kwargs_reduced[key] = value
data, meta = hapi(args[0], args[1], args[2], args[3], args[4],
**kwargs_reduced)
meta = hapiplot(data, meta, **kwargs)
return data, meta
else:
data = args[0]
meta = args[1]
# Default options
opts = {
'logging': False,
'saveimage': False,
'returnimage': False,
'usecache': True,
'useimagecache': True,
'cachedir': cachedir(),
'backend': 'default',
'style': 'fast',
'title': '',
'ztitle': '',
'xlabel': '',
'ylabel': '',
'zlabel': '',
'logx': False,
'logy': False,
'logz': False,
'tsopts': {},
'hmopts': {},
'backend': 'default',
'rcParams': {
'savefig.dpi': 144,
'savefig.format': 'png',
'savefig.bbox': 'tight',
'savefig.transparent': False,
'figure.max_open_warning': 50,
'figure.figsize': (7, 3),
'figure.dpi': 144,
'axes.titlesize': 10,
"font.family": "serif",
"font.serif": rcParams['font.serif'],
"font.weight": "normal"
},
'_rcParams': {
'figure.bbox': 'standard'
}
}
# Override defaults
opts = setopts(opts, kwargs)
from hapiclient import __version__
log('Running hapi.py version %s' % __version__, opts)
# _rcParams are not actually rcParams:
#'figure.bbox': 'standard',
# Set to 'tight' to have fig.tight_layout() called before figure shown.
if opts["saveimage"]:
# Create cache directory
dir = cachedir(opts['cachedir'], meta['x_server'])
if not os.path.exists(dir): os.makedirs(dir)
# Convert from NumPy array of byte literals to NumPy array of
# datetime objects.
timename = meta['parameters'][0]['name']
Time = hapitime2datetime(data[timename])
if len(meta["parameters"]) == 1:
a = 0 # Time is only parameter
else:
a = 1 # Time plus another parameter
for i in range(a, len(meta["parameters"])):
meta["parameters"][i]['hapiplot'] = {}
name = meta["parameters"][i]["name"]
# Return cached image (case where we are returning binary image data)
# imagepath() options. Only need filename under these conditions.
if opts['saveimage'] or (opts['returnimage']
and opts['useimagecache']):
# Will use given rc style parameters and style name to generate file name.
# Assumes rc parameters of style and hapiplot defaults never change.
styleParams = {}
fmt = opts['rcParams']['savefig.format']
if 'rcParams' in kwargs:
styleParams = kwargs['rcParams']
if 'savefig.format' in kwargs['rcParams']:
kwargs['rcParams']['savefig.format']
fnameimg = imagepath(meta, i, opts['cachedir'], styleParams, fmt)
if opts['useimagecache'] and opts['returnimage'] and os.path.isfile(
fnameimg):
log('Returning cached binary image data in ' + fnameimg, opts)
meta["parameters"][i]['hapiplot']['imagefile'] = fnameimg
with open(fnameimg, "rb") as f:
meta["parameters"][i]['hapiplot']['image'] = f.read()
continue
name = meta["parameters"][i]["name"]
log("Plotting parameter '%s'" % name, opts)
if len(data[name].shape) > 3:
# TODO: Implement more than 2 dimensions?
warning(
'Parameter ' + name +
' has size with more than 2 dimensions. Plotting first two only.'
)
continue
# If parameter has a size with two elements, e.g., [N1, N2]
# create N2 plots.
if len(data[name].shape) == 3: # shape = (Time, N1, N2)
nplts = data[name].shape[1]
if opts['returnimage']:
warning(
'Only returning first image for parameter with size[1] > 1.'
)
nplts = 1
for j in range(nplts):
timename = meta['parameters'][0]['name']
# Name to indicate what is plotted
name_new = name + "[:," + str(j) + "]"
# Reduced data ND Array
datar = np.ndarray(shape=(data[name].shape[0]),
dtype=[(timename, data.dtype[timename]),
(name_new, data[name].dtype.str,
data.dtype[name].shape[1])])
datar[timename] = data[timename]
datar[name_new] = data[name][:, j]
# Copy metadata to create a reduced metadata object
metar = meta.copy() # Shallow copy
metar["parameters"] = []
# Create parameters array with elements of Time parameter ...
metar["parameters"].append(meta["parameters"][0])
# .... and this parameter
metar["parameters"].append(meta["parameters"][i].copy())
# Give new name to indicate it is a subset of full parameter
metar["parameters"][1]['name'] = name_new
metar["parameters"][1]['name_orig'] = name
# New size is N1
metar["parameters"][1]['size'] = [
meta["parameters"][i]['size'][1]
]
if 'units' in metar["parameters"][1]:
if type(meta["parameters"][i]['units']
) == str or meta["parameters"][i]['units'] == None:
# Same units applies to all dimensions
metar["parameters"][1]["units"] = meta["parameters"][
i]['units']
else:
metar["parameters"][1]["units"] = meta["parameters"][
i]['units'][j]
if 'label' in metar["parameters"][1]:
if type(meta["parameters"][i]['label']) == str:
# Same label applies to all dimensions
metar["parameters"][1]["label"] = meta["parameters"][
i]['label']
else:
metar["parameters"][1]["label"] = meta["parameters"][
i]['label'][j]
# Extract bins corresponding to jth column of data[name]
if 'bins' in metar["parameters"][1]:
metar["parameters"][1]['bins'] = []
metar["parameters"][1]['bins'].append(
meta["parameters"][i]['bins'][j])
# rcParams is modified by setopts to have all rcParams.
# reset to original passed rcParams so that imagepath
# computes file name based on rcParams passed to hapiplot.
if 'rcParams' in kwargs:
opts['rcParams'] = kwargs['rcParams']
metar = hapiplot(datar, metar, **opts)
meta["parameters"][i]['hapiplot'] = metar["parameters"][i][
'hapiplot']
return meta
if 'name_orig' in meta["parameters"][i]:
title = meta["x_server"] + "\n" + meta["x_dataset"] + " | " + meta[
"parameters"][i]['name_orig']
else:
title = meta["x_server"] + "\n" + meta["x_dataset"] + " | " + name
as_heatmap = False
if 'size' in meta['parameters'][
i] and meta['parameters'][i]['size'][0] > 10:
as_heatmap = True
if 'bins' in meta['parameters'][i]:
as_heatmap = True
if 'units' in meta["parameters"][i] and type(
meta["parameters"][i]["units"]) == list:
if as_heatmap:
warning(
"Not plotting %s as heatmap because components have different units."
% meta["parameters"][i]["name"])
as_heatmap = False
if as_heatmap:
# Plot as heatmap
hmopts = {
'returnimage': opts['returnimage'],
'transparent': opts['rcParams']['savefig.transparent']
}
if meta["parameters"][i]["type"] == "string":
warning(
"Plots for only types double, integer, and isotime implemented. Not plotting %s."
% meta["parameters"][i]["name"])
continue
z = np.asarray(data[name])
if 'fill' in meta["parameters"][i] and meta["parameters"][i][
'fill']:
if meta["parameters"][i]["type"] == 'integer':
z = z.astype('<f8', copy=False)
z = fill2nan(z, meta["parameters"][i]['fill'])
if 'bins' in meta['parameters'][i]:
ylabel = meta["parameters"][i]['bins'][0]["name"] + " [" + meta[
"parameters"][i]['bins'][0]["units"] + "]"
else:
ylabel = "col %d" % i
units = meta["parameters"][i]["units"]
nl = ""
if len(name) + len(units) > 30:
nl = "\n"
zlabel = name + nl + " [" + units + "]"
if 'bins' in meta['parameters'][i]:
if 'ranges' in meta["parameters"][i]['bins'][0]:
bins = np.array(meta["parameters"][i]['bins'][0]["ranges"])
else:
bins = np.array(
meta["parameters"][i]['bins'][0]["centers"])
else:
bins = np.arange(meta['parameters'][i]['size'][0])
dt = np.diff(Time)
dtu = np.unique(dt)
if len(dtu) > 1:
#warning('Time values are not uniformly spaced. Bin width for '
# 'time will be based on time separation of consecutive time values.')
if False and 'cadence' in meta:
# Cadence != time bin width in general, so don't do this.
# See https://github.com/hapi-server/data-specification/issues/75
# Kept for future reference when Parameter.bin.window or
# Parameter.bin.windowWidth is added to spec.
import isodate
dt = isodate.parse_duration(meta['cadence'])
if 'timeStampLocation' in meta:
if meta['timeStampLocation'].lower() == "begin":
Time = np.vstack((Time, Time + dt))
if meta['timeStampLocation'].lower() == "end":
Time = np.vstack((Time - dt, Time))
if meta['timeStampLocation'].lower() == "center":
Time = np.vstack((Time - dt / 2, Time + dt / 2))
else:
# Default is center
Time = np.vstack((Time - dt / 2, Time + dt / 2))
Time = np.transpose(Time)
elif 'timeStampLocation' in meta:
if meta['timeStampLocation'].lower() == "begin":
Time = np.append(Time, Time[-1] + dtu[0])
if meta['timeStampLocation'].lower() == "end":
Time = Time - dtu[0]
Time = np.append(Time, Time[-1] + dtu[0])
if opts['xlabel'] != '' and 'xlabel' not in opts['hmopts']:
hmopts['xlabel'] = opts['xlabel']
opts['hmopts']['ylabel'] = ylabel
if opts['ylabel'] != '' and 'ylabel' not in opts['hmopts']:
hmopts['ylabel'] = opts['ylabel']
opts['hmopts']['title'] = title
if opts['title'] != '' and 'title' not in opts['hmopts']:
hmopts['title'] = opts['title']
opts['hmopts']['zlabel'] = zlabel
if opts['zlabel'] != '' and 'zlabel' not in opts['hmopts']:
hmopts['zlabel'] = opts['zlabel']
if False:
opts['hmopts']['ztitle'] = ztitle
if opts['ztitle'] != '' and 'ztitle' not in opts['hmopts']:
hmopts['ztitle'] = opts['ztitle']
if opts['logx'] is not False:
hmopts['logx'] = True
if opts['logy'] is not False:
hmopts['logy'] = True
if opts['logz'] is not False:
hmopts['logz'] = True
for key, value in opts['hmopts'].items():
hmopts[key] = value
with rc_context(rc=opts['rcParams']):
fig, cb = heatmap(Time, bins, np.transpose(z), **hmopts)
meta["parameters"][i]['hapiplot']['figure'] = fig
meta["parameters"][i]['hapiplot']['colorbar'] = cb
else:
tsopts = {
'logging': opts['logging'],
'returnimage': opts['returnimage'],
'transparent': opts['rcParams']['savefig.transparent']
}
ptype = meta["parameters"][i]["type"]
if ptype == "isotime":
y = hapitime2datetime(data[name])
elif ptype == 'string':
y = data[name].astype('U')
else:
y = np.asarray(data[name])
if 'fill' in meta["parameters"][i] and meta["parameters"][i][
'fill']:
if ptype == 'isotime' or ptype == 'string':
Igood = y != meta["parameters"][i]['fill']
# Note that json reader returns fill to U not b.
Nremoved = data[name].size - Igood.size
if Nremoved > 0:
# TODO: Implement masking so connected line plots will
# show gaps as they do for NaN values.
warning('Parameter ' + name + ' is of type ' + ptype +
' and has ' + str(Nremoved) +
' fill value(s). Masking is not implemented, '
'so removing fill elements before plotting.')
Time = Time[Igood]
y = y[Igood]
if ptype == 'integer':
y = y.astype('<f8', copy=False)
if ptype == 'integer' or ptype == 'double':
y = fill2nan(y, meta["parameters"][i]['fill'])
units = None
if 'units' in meta["parameters"][i] and meta["parameters"][i][
'units']:
units = meta["parameters"][i]["units"]
nl = ""
if type(units) == str:
if len(name) + len(units) > 30:
nl = "\n" # TODO: Automatically figure out when this is needed.
ylabel = name
if units is not None and type(units) is not list:
ylabel = name + nl + " [" + units + "]"
if type(units) == list:
ylabel = name
if not 'legendlabels' in opts['tsopts']:
legendlabels = []
if 'size' in meta['parameters'][i]:
for l in range(0, meta['parameters'][i]['size'][0]):
bin_label = ''
bin_name = ''
col_name = ''
if 'bins' in meta['parameters'][i]:
bin_name = meta['parameters'][i]['bins'][0]['name']
if 'label' in meta['parameters'][i]['bins'][0]:
if type(meta['parameters'][i]['bins'][0]
['label']) == str:
bin_name = meta['parameters'][i]['bins'][
0]['label']
else:
bin_name = meta['parameters'][i]['bins'][
0]['label'][l]
sep = ''
if 'centers' in meta['parameters'][i]['bins'][
0] and 'ranges' in meta['parameters'][i][
'bins'][0]:
bin_name = bin_name + ' bin with'
sep = ';'
bin_label = ''
if 'units' in meta['parameters'][i]['bins'][0]:
bin_units = meta['parameters'][i]['bins'][0][
'units']
if type(bin_units) == list:
if type(bin_units[l]) == str:
bin_units = ' [' + bin_units[l] + ']'
elif bin_units[l] == None:
bin_units = ' []'
else:
bin_units = ''
else:
if type(bin_units) == str:
bin_units = ' [' + bin_units + ']'
else:
bin_units = ''
if 'centers' in meta['parameters'][i]['bins'][0]:
if meta['parameters'][i]['bins'][0]['centers'][
l] is not None:
bin_label = bin_label + ' center = ' + str(
meta['parameters'][i]['bins'][0]
['centers'][l]) + bin_units
else:
bin_label = bin_label + ' center = None'
if 'ranges' in meta['parameters'][i]['bins'][0]:
if type(meta['parameters'][i]['bins'][0]
['ranges'][l]) == list:
bin_label = bin_label + sep + ' range = [' + str(
meta['parameters'][i]['bins'][0]
['ranges'][l][0]) + ', ' + str(
meta['parameters'][i]['bins'][0]
['ranges'][l][1]) + ']' + bin_units
else:
bin_label = bin_label + sep + ' range = [None]'
if bin_label != '':
bin_label = 'bin:' + bin_label
col_name = bin_name + '#%d' % l
if col_name == '':
col_name = 'col #%d' % l
if 'label' in meta['parameters'][i]:
#print(meta)
#print(meta['parameters'][i]['label'])
if type(meta['parameters'][i]['label']) == list:
col_name = meta['parameters'][i]['label'][l]
if type(units) == list:
if len(units) == 1:
legendlabels.append(col_name + ' [' +
units[0] + '] ' +
bin_label)
elif type(units[l]) == str:
legendlabels.append(col_name + ' [' +
units[l] + '] ' +
bin_label)
elif units[l] == None:
legendlabels.append(col_name + ' [] ' +
bin_label)
else:
legendlabels.append(col_name + ' ' + bin_label)
else:
# Units are on y label
legendlabels.append(col_name + ' ' + bin_label)
tsopts['legendlabels'] = legendlabels
# If xlabel in opts and opts['tsopts'], warn?
if opts['xlabel'] != '' and 'xlabel' not in opts['tsopts']:
tsopts['xlabel'] = opts['xlabel']
tsopts['ylabel'] = ylabel
if opts['ylabel'] != '' and 'ylabel' not in opts['tsopts']:
tsopts['ylabel'] = opts['ylabel']
tsopts['title'] = title
if opts['title'] != '' and 'title' not in opts['tsopts']:
tsopts['title'] = opts['title']
if opts['logx'] is not False and 'logx' not in opts['tsopts']:
tsopts['logx'] = True
if opts['logy'] is not False and 'logy' not in opts['tsopts']:
tsopts['logy'] = True
# Apply tsopts
for key, value in opts['tsopts'].items():
tsopts[key] = value
with rc_context(rc=opts['rcParams']):
fig = timeseries(Time, y, **tsopts)
meta["parameters"][i]['hapiplot']['figure'] = fig
if opts['saveimage']:
log('Writing %s' % fnameimg, opts)
meta["parameters"][i]['hapiplot']['imagefile'] = fnameimg
else:
from io import BytesIO
fnameimg = BytesIO()
if opts['returnimage']:
with rc_context(rc=opts['rcParams']):
fig.canvas.print_figure(fnameimg)
if opts['saveimage']:
with open(fnameimg, mode='rb') as f:
meta["parameters"][i]['hapiplot']['image'] = f.read()
else:
meta["parameters"][i]['hapiplot']['image'] = fnameimg.getvalue(
)
else:
with rc_context(rc=opts['rcParams']):
fig.savefig(fnameimg)
# Two calls to fig.tight_layout() may be needed b/c of bug in PyQt:
# https://github.com/matplotlib/matplotlib/issues/10361
if opts['_rcParams']['figure.bbox'] == 'tight':
fig.tight_layout()
return meta
def read_STEREO(download_start, download_end, spacecraft):
server = 'https://cdaweb.gsfc.nasa.gov/hapi'
dataset = 'ST' + spacecraft + '_L2_MAGPLASMA_1M'
start = download_start
stop = download_end
parameters = 'BTOTAL,BFIELDRTN,Np,Vp,Tp,Vth,Beta,Total_Pressure,Cone_Angle,Magnetic_Pressure,Dynamic_Pressure,Vp_RTN,HEE,R'
#parameters = 'BTOTAL,BFIELDRTN,Np,Vp,Tp,Vth,Beta,Total_Pressure,Cone_Angle,Magnetic_Pressure,Dynamic_Pressure,Vp_RTN,HEE,R'
opts = {'logging': True, 'use_cache': True}
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
aux_date_time = [str(data['Time'][i]) for i in range(0, len(data['Time']))]
date_time = [
datetime.strptime(aux_date_time[i], "b'%Y-%m-%dT%H:%M:%S.%fZ'")
for i in range(0, len(aux_date_time))
]
#save_time = [datetime.strftime(date_time[0],"%Y%m%d")]
#save_time = save_time[0] #+ '_' + str(dfpar.ICME_doystart[index])
HEEQx = data['HEE'][:, 0]
HEEQy = data['HEE'][:, 1]
HEEQz = data['HEE'][:, 2]
Angle = [
round(math.atan2(HEEQy[index], HEEQx[index]) * 180 / math.pi, 1)
for index in range(len(HEEQx))
]
Latitude = [
round(HEEQz[index] * 180 / math.pi, 1) for index in range(len(HEEQz))
]
d = {
'date_time': date_time,
'B': data['BTOTAL'],
'Bx': data['BFIELDRTN'][:, 0],
'By': data['BFIELDRTN'][:, 1],
'Bz': data['BFIELDRTN'][:, 2],
'Np': data['Np'],
'Vp': data['Vp'],
'Tp': data['Tp'],
'Vth': data['Vth'],
'beta': data['Beta'],
'Ptot': data['Total_Pressure'],
'Vsw': data['Vp_RTN'],
'Angle': Angle,
'Latitude': Latitude,
'HEEx': HEEQx,
'HEEy': HEEQy,
'HEEz': HEEQz,
'Radi': data['R']
}
#'Latitude':Latitude,'Radi':data['R']}
df = pd.DataFrame(data=d)
aux_datetitle = []
aux_xticks = []
aux_doy = []
aux_ddoy = []
aux_year = []
aux_day = []
aux_hour = []
aux_minute = []
for index in range(0, len(date_time)):
aux_datetitle.append(date_time[index].strftime('%b %-d, %Y %H:%M:%S'))
aux_xticks.append(str(date_time[index].strftime('%b %-d')))
year = date_time[index].timetuple(
).tm_year #year, day, hour, minute, sec as floats
day = date_time[index].timetuple().tm_mday
hour = date_time[index].timetuple().tm_hour
minute = date_time[index].timetuple().tm_min
sec = date_time[index].timetuple().tm_sec
day_fraction = float(hour * 3600 + minute * 60 + sec) / float(
24 * 3600)
aux_doy.append(date_time[index].timetuple().tm_yday) #doy calculation
aux_ddoy.append(
float(date_time[index].timetuple().tm_yday) +
day_fraction) #decimal doy calculation
aux_year.append(float(year))
aux_day.append(float(day))
aux_hour.append(float(hour))
aux_minute.append(float(minute))
aux_xticks = [(str(round(aux_ddoy[i], 1)) + '\n' + str(aux_xticks[i]))
for i in range(0, len(aux_doy))
] #date format for xticks axis
df['doy'] = copy.deepcopy(aux_doy) #new columns to DataFrame
df['ddoy'] = copy.deepcopy(aux_ddoy)
df['datetime'] = copy.deepcopy(aux_datetitle)
df['minute'] = copy.deepcopy(aux_minute)
df['hour'] = copy.deepcopy(aux_hour)
df['day'] = copy.deepcopy(aux_day)
df['year'] = copy.deepcopy(aux_year)
df['xplot'] = copy.deepcopy(aux_xticks)
filter_vals = (df.B > 1e-10) & (df.Vth < 90) & (df.Vth > 0)
df = df[filter_vals]
return df
if False:
import numpy as np
from matplotlib import pyplot as plt
from hapiclient.hapi import hapi
from hapiclient.hapiplot import hapitime2datetime
from hapiclient.util.datetick import datetick
server = 'http://hapi-server.org/servers/TestData/hapi'
dataset = 'dataset1'
start = '1970-01-01Z'
stop = '1970-01-01T00:00:11Z'
parameters = 'spectra'
opts = {'logging': True, 'use_cache': True}
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
Time = hapitime2datetime(data['Time'])
Cl = np.array(meta['parameters'][1]['bins'][0]['centers'], dtype='d')
Rl = Time
M = data['spectra']
fig = plt.gcf()
ax = plt.gca()
#im = ax.pcolormesh(Rl, Cl, np.transpose(M))
im, ax, cb = plotmatrix(Cl, Rl, np.transpose(M))
cb.set_label('z')
fig = plt.gcf()
datetick('x', (fig, ax))
def readcompare(server, dataset, parameters, run, opts):
# Note that for this dataset, there are differences in
# the numeric values that seem not to be due to issues
# with the reader. This needs investigation.
dataset = 'dataset1'
start = '1970-01-01'
allpass = True
if run == 'short':
stop = '1970-01-01T00:00:03' # Returns 3 time values
if run == 'long':
stop= '1970-01-02T00:00:00' # Returns 86400 time values
# Checks that all four read methods give same result.
# Does not check that an individual read is correct.
# Do this manually.
opts['format'] = 'csv'
print('\nParameter(s) = %s; run = %s. cache = %s; usecache = %s' % (parameters, run, opts['cache'], opts['usecache']))
if opts['cache']:
print('__________________________________________')
print('Method read/parse download ')
print('__________________________________________')
else:
print('__________________________________________')
print('Method create buff download/parse')
print('__________________________________________')
opts['method'] = 'numpynolength'
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
info = requestinfo(server, dataset, parameters, start, stop, opts['cachedir'])
print('numpy no length %5.2f ms %8.4f s' % (1000.*info['x_readTime'], info['x_downloadTime']))
datalast = data
opts['method'] = 'pandasnolength'
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
info = requestinfo(server, dataset, parameters, start, stop, opts['cachedir'])
if np.array_equal(data, datalast):
print('pandas no length %5.2f ms %8.4f s' % (1000.*info['x_readTime'], info['x_downloadTime']))
else:
print('pandas no length %5.2f ms %8.4f s (diffs in float values <= 1e-15)' % (1000.*info['x_readTime'], info['x_downloadTime']))
allpass = comparisonOK(data, datalast)
datalast = data
opts['method'] = 'numpy'
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
info = requestinfo(server, dataset, parameters, start, stop, opts['cachedir'])
if np.array_equal(data, datalast):
print('numpy %5.2f ms %8.4f s' % (1000.*info['x_readTime'], info['x_downloadTime']))
else:
print('numpy %5.2f ms %8.4f s (diffs in float values <= 1e-15)' % (1000.*info['x_readTime'], info['x_downloadTime']))
allpass = comparisonOK(data, datalast)
datalast = data
opts['method'] = 'pandas'
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
info = requestinfo(server, dataset, parameters, start, stop, opts['cachedir'])
if np.array_equal(data, datalast):
print('pandas %5.2f ms %8.4f s' % (1000.*info['x_readTime'], info['x_downloadTime']))
else:
print('pandas %5.2f ms %8.4f s (diffs in float values <= 1e-15)' % (1000.*info['x_readTime'], info['x_downloadTime']))
allpass = comparisonOK(data, datalast)
datalast = data
opts['format'] = 'binary'
opts['method'] = '' # Ignored
data, meta = hapi(server, dataset, parameters, start, stop, **opts)
info = requestinfo(server, dataset, parameters, start, stop, opts['cachedir'])
if np.array_equal(data, datalast):
print('binary %5.2f ms %8.4f s' % (1000.*info['x_readTime'], info['x_downloadTime']))
else:
print('binary %5.2f ms %8.4f s (diffs in float values <= 1e-15)' % (1000.*info['x_readTime'], info['x_downloadTime']))
allpass = comparisonOK(data, datalast)
return allpass