def addElementDatasets(manager, station, element, data, first_hour, last_hour,
**kwargs):
# create element group
attrs = {}
attrs['created'] = manager._timestamp()
attrs['description'] = kwargs.get('description',
ELEMENTS[element].description)
attrs['first_hour'] = dateAsTuple(first_hour, True)
attrs['frequency'] = 'hour'
attrs['interval'] = 1
attrs['last_hour'] = dateAsTuple(last_hour, True)
print '\n creating %s element group' % element
manager.createGroup(element, attrs)
# generate the dates array and use it to create a dataset
dataset_name = '%s.date' % element
print ' creating %s dataset' % dataset_name
manager.createDataset(dataset_name,
generateHoursArray(first_hour, last_hour), attrs)
manager.setDatasetAttribute(dataset_name, 'description',
'Year,Month,Day,Hour')
manager.setDatasetAttribute(dataset_name, 'date_formula', DATE_FORMULA)
# create value dataset
dataset_name = '%s.value' % element
for attr_name, attr_value in kwargs.items():
if attr_name not in attrs: attrs[attr_name] = attr_value
attrs['value_type'] = ELEMENTS[element].value_type
print ' creating %s dataset' % dataset_name
data = transformData(station, element, data)
manager.createDataset(dataset_name, data, attrs)
def getStationData(self, date, sensor, station, padding=0):
ucan = HourlyDataConnection(days_per_request=2)
if station['sid'] != self.data_cache['stn_sid']:
ucan_start, ucan_end = ucan.getValidDatetimeRange(station, sensor)
self.data_cache['ucan_start'] = ucan_start
self.data_cache['ucan_end'] = ucan_end
self.data_cache['stn_sid'] = station['sid']
else:
ucan_start = self.data_cache['ucan_start']
ucan_end = self.data_cache['ucan_end']
start_time, end_time = self._getStartEndTime(date)
if padding > 0:
hours = relativedelta(hours=padding)
start_time -= hours
end_time += hours
if start_time < ucan_start or start_time > ucan_end: return None
if end_time > ucan_end or end_time < ucan_start: return None
start_time = dateAsTuple(start_time, True)
end_time = dateAsTuple(end_time, True)
_start_date_, _end_date_, data =\
ucan.getData(station, sensor, start_time, end_time)
del ucan
data = N.array(data)
data[N.where(N.isinf(data))] = N.nan
return data, start_time, end_time
def getStationData(factory, station, _date_, dataset_name, cushion=0):
_datetime_ = asDatetime(_date_)
start_time = (_datetime_ - relativedelta(days=1)) + relativedelta(hours=8)
end_time = _datetime_ + relativedelta(hours=7)
if cushion != 0:
start_time -= relativedelta(hours=cushion)
end_time += relativedelta(hours=cushion)
start_time = dateAsTuple(start_time, True)
end_time = dateAsTuple(end_time, True)
ucan = HourlyDataConnection(days_per_request=1)
_start_date_, _end_date_, data =\
ucan.getData(station, dataset_name, start_time, end_time)
data = N.array(data)
data[N.where(N.isinf(data))] = N.nan
return datetime(*_start_date_), datetime(*_end_date_), data
def getData(self,
station,
dataset_name,
start_time,
end_time,
debug=False):
dtype,missing,units,tsv_name,tsv_units =\
getTsVarType(station,dataset_name)
try:
return self.ucan.getHourlyData(station, tsv_name, dtype, tsv_units,
missing,
dateAsTuple(start_time, True),
dateAsTuple(end_time, True), debug)
except Exception as e:
errmsg = "Exception occurred while requesting data for '%s'."
detail = errmsg % dataset_name
reportLastException(None, None, detail)
raise explainException(e, station, detail)
def getStationPrecip(factory, station, _date_, cushion=0, test_run=False):
_datetime_ = asDatetime(_date_)
start_time = _datetime_ - ONE_DAY + relativedelta(hours=7)
end_time = _datetime_ + relativedelta(hours=7)
if cushion != 0:
start_time -= relativedelta(hours=cushion)
end_time += relativedelta(hours=cushion)
if test_run:
if station['sid'] == STATION_CACHE['sid']:
data = STATION_CACHE['data']
dates = STATION_CACHE['dates']
else:
# hourly data file must already exist
filepath = factory.getFilepathForUcanid(station['ucanid'], 'hours')
if not os.path.exists(filepath):
print SKIP_MSG % station
print 'Hourly data file does not exist : %s' % filepath
return None
manager = HDF5DataFileManager(filepath, 'r')
data = manager.getData('pcpn.value')
STATION_CACHE['data'] = data
dates = manager.getData('pcpn.date')
STATION_CACHE['dates'] = dates
STATION_CACHE['sid'] = station['sid']
_start_time = dateAsInt(start_time, True)
_end_time = dateAsInt(end_time, True)
indexes = N.where((dates > (_start_time - 1)) & (dates < _end_time))
if len(indexes[0]) > 0: return data[indexes]
return None
else:
start_time = dateAsTuple(start_time, True)
end_time = dateAsTuple(end_time, True)
ucan = HourlyDataConnection(days_per_request=1)
_start_date_, _end_date_, precip =\
ucan.getData(station, 'pcpn', start_time, end_time)
precip = N.array(precip)
precip[N.where(N.isinf(precip))] = N.nan
return precip
def getTimeSpan(self, date):
_time = list(dateAsTuple(date)[:3])
_time.append(self.first_hour_in_day)
start_time = asDatetime(_time)
end_time = start_time + ONE_DAY
return start_time, end_time