def __init__(self, url):
""" Constructor, reads in a file from the url provided
@param url: OpenNDAP url for the NetCDF file
"""
self.url = url
# Let's open the file using OpenNDAP
f = open_url(url)
# Pull out the attributes, and column names
self.attributes = f.attributes['NC_GLOBAL']
self.columns = f.keys()
if 'time' in f:
# There's a time dimension in this dataset, so let's grab the possible
# values out
try:
time_units = f['time'].attributes['units']
self.time_values = [parse(t, time_units) for t in f['time']]
except IndexError:
# Not a parseable time format
pass
del f
def load_GEFS_data(directory, fx_names, file_sub_str):
"""Loads a list of GEFS files merging them into model format. """
blocks = OrderedDict()
for i, f in enumerate(fx_names):
ds = load_GEFS_file(directory, fx_names[i], file_sub_str)
X_ds = ds.variables.values()[-1][:]
if i == 0:
n_samples = X_ds.shape[0]
# (n_samples, n_features, n_ensemble, n_hours, lat, long)
shape = (n_samples, len(fx_names)) + tuple(X_ds.shape[1:])
print 'create X with shape', shape
X = np.empty(shape, dtype=np.float32)
X[:, i, :, :, :] = X_ds
blocks['nm'] = X
time = ds.variables['time']
dates = pd.DatetimeIndex([coards.parse(t, time.units) for t in time])
blocks['date'] = dates
X = StructuredArray(blocks)
X.lat = ds.variables['lat'][:]
X.lon = ds.variables['lon'][:]
X.fx_name = OrderedDict()
X.fx_name['nm'] = fx_names
X.fx_name['date'] = ['date']
return X
def get_time(grid, dataset):
for dim in grid.maps.values():
if ' since ' in dim.attributes.get('units', ''):
try:
return [coards.parse(value, dim.units) for value in np.asarray(dim)]
except:
pass
return None
def _get_millis_since_epoch(self, intervals):
"""
Convert a timestamp into milliseconds since the Unix epoch (needed for Flot)
:param intervals: The time interval to convert (eg number of seconds since)
:return: milliseconds since the Unix epoch
"""
epoch = datetime.datetime.utcfromtimestamp(0)
time = parse(intervals, self._time_units)
delta = time - epoch
return delta.total_seconds() * 1000
def get_time(grid, dataset):
for dim in grid.maps.values():
if ' since ' in dim.attributes.get('units', ''):
try:
return [
coards.parse(value, dim.units) for value in np.asarray(dim)
]
except:
pass
return None
def _get_data_start_date(self):
"""
Get the start date of the data as a datetime (rather than as seconds after some arbitrary time)
:return: Datetime representing the data start date
"""
return parse(0, self._time_units)
