def generateYearsArray(start_year, end_year, **kwargs):
first_year = asDatetime(start_year)
last_year = asDatetime(end_year)
interval = relativedelta(years=int(kwargs.get('interval', 1)))
date_format = kwargs.get('date_format', None)
as_numpy = kwargs.get('as_numpy', True)
years = []
date = first_year
if date_format in (datetime, object, 'object'):
while date <= last_year:
years.append(date)
date += interval
dtype = 'object'
as_numpy = False
elif date_format in (int, 'int'):
while date <= last_year:
years.append(date.year)
date += interval
dtype = 'i2'
else:
if date_format is None: date_format = '%Y'
while date <= last_year:
years.append(date.strftime(date_format))
date += interval
dtype = '|S%d' % len(years[0])
if as_numpy: return N.array(years, dtype=dtype)
else: return tuple(years)
def _loadDataGridAttributes_(self):
if hasattr(self, 'start_date') \
and isinstance(self.start_date, basestring):
self.start_date = asDatetime(self.start_date)
if hasattr(self, 'end_date') \
and isinstance(self.end_date, basestring):
self.end_date = asDatetime(self.end_date)
def listDates(self, interval_start, interval_end, interval=None):
if interval is None: num_days = relativedelta(days=self.interval)
else: num_days = relativedelta(days=interval)
dates = []
date = asDatetime(interval_start)
while date <= asDatetime(interval_end):
dates.append(date)
date += num_days
return tuple(dates)
def listDates(self, interval_start, interval_end, interval=None):
if interval is None: _interval = self.interval
else: _interval = interval
dates = []
date = asDatetime(interval_start)
while date <= asDatetime(interval_end):
dates.append(date)
date = monthsInterval(date, _interval)
return tuple(dates)
def setTimeSpan(self, iter_start, iter_end):
if isinstance(iter_start, (list, tuple)) and len(iter_start) == 2:
iter_start = (iter_start[0], iter_start[1],
lastDayOfMonth(*iter_start))
self.iter_start = asDatetime(iter_start)
if isinstance(iter_end, (list, tuple)) and len(iter_end) == 2:
iter_end = (iter_end[0], iter_end[1], lastDayOfMonth(*iter_end))
self.iter_end = asDatetime(iter_end)
def _validTimes(self, start_time, end_time):
""" returns tuple containing index of start_time and index of end_time
"""
if start_time is None or start_time == ':':
if end_time is None: return self.base_time, self.base_time
elif end_time == ':': return self.base_time, self.last_time
return self.base_time, asDatetime(end_time)
else:
_start_time = asDatetime(start_time)
if end_time is None: return _start_time, _start_time
elif end_time == ':': return _start_time, self.last_time
return _start_time, asDatetime(end_time)
def _setLastValidDate(self, dataset_path, start_date, data):
if data.ndim == 2: num_days = 1
else: num_days = data.shape[0]
end_date = start_date + relativedelta(days=num_days - 1)
last_date_str = \
self.getDatasetAttribute(dataset_path, 'last_valid_date', None)
if isinstance(end_date, datetime):
end_is_later = end_date > asDatetime(last_date_str)
else:
end_is_later = asDatetime(end_date) > asDatetime(last_date_str)
if last_date_str is None or end_is_later:
self.setDatasetAttribute(dataset_path, 'last_valid_date',
end_date.strftime('%Y-%m-%d'))
def _datasetEndDate(self, dataset):
if 'end_date' in dataset.attrs.keys():
return asDatetime(dataset.end_date)
elif 'end_day' in attrs:
return datetime.datetime(self._targetYear(), *attrs['end_day'])
else:
return self._fileEndDate()
def _indexForTime(_time):
delta = asDatetime(_time) - self.base_time
hours = (delta.days * 24) + (delta.seconds / 3600)
indx = hours / interval
over = hours % interval
if over > 0: return indx + 1
else: return indx
def index(self, date_time):
_date_time = asDatetime(date_time)
if _date_time == self.base_time: return 0
delta = _date_time - self.base_time
hours = (delta.days * 24) + (delta.seconds / 3600)
if (hours % self.interval) == 0: return hours / self.interval
else: return (hours + self.interval) / self.interval
def _indexForTime(_time):
date_time = asDatetime(_time)
years = date_time.year - self.base_time.year
months = years * 12
if date_time.month != 12: months -= (12 - date_time.month)
if self.base_time.month != 1:
months -= self.base_time.month - 1
return months
def mapWorkingDir(date, variety, map_key, map_type=None, test_path=False):
if '.' in map_key: map_group = map_key.split('.')[0]
else: map_group = map_key
target_year = targetYearFromDate(asDatetime(date))
variety_dir = varietyWorkingDir(target_year, 'grape', variety, test_path)
working_dir = os.path.join(variety_dir, 'maps', nameToDirpath(map_group))
if not os.path.exists(working_dir): os.makedirs(working_dir)
return working_dir
def _datasetStartDoy(self, dataset):
attrs = dict(dataset.attrs)
if hasattr(self, 'start_doy'): return int(attrs['start_doy'])
elif 'start_day' in attrs:
date = datetime.date(self._targetYear(), *attrs['start_day'])
return date.timetuple().tm_yday
elif 'start_date' in attrs:
return asDatetime(attrs['start_date']).timetuple().tm_yday
else: return self._fileStartDoy()
def __init__(self,
time_series,
start_time=None,
end_time=None,
interval=None,
duration=None):
""" handles instances of the TimeSeriesData class or it's
subclassses
"""
if start_time is None: iter_start = time_series.common_start_time
else: iter_start = asDatetime(start_time)
if end_time is None: iter_end = time_series.common_end_time
else: iter_end = asDatetime(end_time)
if interval is None: interval = time_series.interval
if duration is None: duration = time_series.interval
self.iterator = timeIterator(time_series.frequency, iter_start,
iter_end, interval, duration)
def webMapFilepath(date,
variety,
model_name,
map_group,
map_type,
config_key='web_maps'):
target_year = targetYearFromDate(asDatetime(date))
webmaps_dir = webMapsDir(target_year, variety)
filename = \
webMapFilename(date, variety, model_name, map_group, map_type, config_key)
return os.path.join(webmaps_dir, filename)
def generateHoursArray(start_hour, end_hour, **kwargs):
first_hour = asDatetime(start_hour)
last_hour = asDatetime(end_hour)
interval = relativedelta(hours=int(kwargs.get('interval', 1)))
date_format = kwargs.get('date_format', None)
as_numpy = kwargs.get('as_numpy', True)
hours = []
hour = first_hour
if date_format in (datetime, object, 'object'):
while hour <= last_hour:
hours.append(hour)
hour += interval
dtype = 'object'
as_numpy = False
elif date_format in (int, 'int'):
while hour <= last_hour:
hours.append((hour.year * 1000000) + (hour.month * 10000) +
(hour.day * 100) + hour.hour)
hour += interval
dtype = N.dtype(int)
elif date_format == 'tuple':
while hour <= last_hour:
hours.append((hour.year, hour.month, hour.day, hour.hour))
hour += interval
dtype = N.dtype({
'names': ['year', 'month', 'day', 'hour'],
'formats': ['i2', 'i2', 'i2', 'i2']
})
else:
if date_format is None: date_format = '%Y-%m-%d-%H'
while hour <= last_hour:
hours.append(hour.strftime(date_format))
hour += interval
dtype = '|S%d' % len(hours[0])
if as_numpy: return N.array(hours, dtype=dtype)
else: return tuple(hours)
def _loadManagerAttributes_(self):
groups, datasets = self.getFileHierarchy(grouped=True)
self._dataset_names = list(datasets)
self._group_names = list(groups)
attributes = self.getFileAttributes()
for attr_name, attr_value in attributes.items():
if attr_name in ('created', 'updated'):
try:
self.__dict__[attr_name] = asDatetime(attr_value)
except:
self.__dict__[attr_name] = attr_value
else: self.__dict__[attr_name] = attr_value
def __init__(self,
time_series,
start_time=None,
end_time=None,
interval=None,
duration=None):
""" handles instances of the MultiTimeSeries class or it's
subclassses
"""
#TODO : handle 2D arrays
if start_time is None: iter_start = time_series.common_start_time
else: iter_start = asDatetime(start_time)
if end_time is None: iter_end = time_series.common_end_time
else: iter_end = asDatetime(end_time)
if interval is None: interval = time_series.interval
if duration is None: duration = time_series.interval
self.dataset_names = time_series.dataset_names
self.indexers = time_series.indexers
self.iterator = timeIterator(time_series.frequency, iter_start,
iter_end, interval, duration)
def generateDaysArray(start_date, end_date, **kwargs):
first_date = asDatetime(start_date)
last_date = asDatetime(end_date)
interval = relativedelta(days=int(kwargs.get('interval', 1)))
date_format = kwargs.get('date_format', None)
as_numpy = kwargs.get('as_numpy', True)
days = []
date = first_date
if date_format in (datetime, object, 'object'):
while date <= last_date:
days.append(date)
date += interval
dtype = 'object'
as_numpy = False
elif date_format in (int, 'int'):
while date <= date_hour:
days.append((date.year * 10000) + (date.month * 100) + date.day)
date += interval
dtype = int
elif date_format == 'tuple':
while date <= last_date:
days.append((date.year, date.month, date.day))
date += interval
dtype = N.dtype({
'names': ['year', 'month', 'day'],
'formats': ['i2', 'i2', 'i2']
})
else:
if date_format is None: date_format = '%Y-%m-%d'
while date <= last_date:
days.append(date.strftime(date_format))
date += interval
dtype = '|S%d' % len(days[0])
if as_numpy: return N.array(days, dtype=dtype)
else: return tuple(days)
def mapFilepath(date,
variety,
model_name,
map_group,
map_type,
lo_gdd_th=None,
hi_gdd_th=None,
test_path=False):
target_year = targetYearFromDate(asDatetime(date))
map_dir = mapWorkingDir(target_year, variety, model_name, map_group,
map_type, lo_gdd_th, hi_gdd_th, test_path)
filename = mapFilename(date, variety, model_name, map_group, map_type,
lo_gdd_th, hi_gdd_th, test_path)
return os.path.join(map_dir, filename)
def compareDates(self, date1, op, date2):
if isinstance(date1, datetime.datetime):
if isinstance(date2, datetime.datetime):
return self._compareDates(date1, op, date2)
else: return self._compareDates(date1, op, asDatetime(date2))
elif isinstance(date1, datetime.date):
if isinstance(date2, datetime.date):
return self._compareDates(date1, op, date2)
else: return self._compareDates(date1, op, asDatetimeDate(date2))
elif isinstance(date1, (tuple, basestring)):
return self._compareDates(asDatetimeDate(date1), op, date2)
else:
errmsg = 'Invalid type for argument "date1" : %s'
raise TypeError, errmsg % str(type(date1))
def index(self, date_time):
_date_time = asDatetime(date_time)
_base_time = self.base_time
if _date_time.year == _base_time.year\
and _date_time.month == _base_time.month:
return 0
_interval = self.interval
# index may be negative if date_time is earlier than base_time
years = _date_time.year - _base_time.year
if years == 0: # both in same year
months = _date_time.month - _base_time.month
else: # different years
months = (years * 12) + (_date_time.month - _base_time.month)
if (months % _interval) == 0: return months / _interval
else: return (months + _interval) / _interval
def unpackQueryResults(self, query_result, elems, meta):
# unpack the grids
if isinstance(elems, basestring): _elems = elems.split(',')
else: _elems = elems
data = [ ]
for result in query_result['data']:
date = [asDatetime(result[0]),]
for indx, grid in enumerate(result[1:]):
elem_name = _elems[indx]
date.append(self.unpackGrid(elem_name, grid))
data.append(date)
if len(data) > 1:
# convert list of (date, elem, ...) tuples to date tuple,
# and a data tuple for each element
data = zip(*data)
# convert data tuples to arrays
for indx, grid in enumerate(data[1:],start=1):
data[indx] = N.array(grid)
else:
date = data[0][0]
data = [grid for grid in data[0][1:]]
data.insert(0,(date,))
data_dict = { 'dates': data[0], }
for indx, elem_name in enumerate(_elems, start=1):
data_dict[elem_name] = data[indx]
if meta is not None:
meta_dict = query_result['meta']
if 'elev' in meta_dict:
meta_dict['elev'] = self.unpackGrid('elev', meta_dict['elev'])
if 'lat' in meta_dict:
meta_dict['lat'] = self.unpackGrid('lat', meta_dict['lat'])
meta_dict['lon'] = self.unpackGrid('lon', meta_dict['lon'])
data_dict.update(meta_dict)
return data_dict
ndfd_bbox.append(region_bbox[1] - lat_offset)
ndfd_bbox.append(region_bbox[2] + lon_offset)
ndfd_bbox.append(region_bbox[3] + lat_offset)
print 'NDFD bounding box', ndfd_bbox
#for grib in gribs.read():
# print '\n', grib, '\n', grib.keys()
print 'exploring grib variable "%s"' % grib_variable
grib = gribs.select(name=var_name_map[grib_variable])
print '\n', grib
print '\nin grib 0 :'
message = grib[message_num]
print ' "dataDate"', message.dataDate
fcast_hour = message['hour']
print ' "fcast_hour"', fcast_hour
print ' "forecastTime"', message['forecastTime']
fcast_time = asDatetime(message.dataDate) + relativedelta(hours=fcast_hour)
print 'forecast datetime', fcast_time
lat, lon = message.latlons()
print ' grib lat', lat.shape, lat.min(), lat.max()
print ' grib lon', lon.shape, lon.min(), lon.max()
maxt = message.values
print ' grib maxt', maxt.shape, maxt.min(), maxt.max()
if region.name == 'conus':
ndfd_lat = lat
ndfd_lon = lon
ndfd_maxt = maxt
else:
indexes = N.where(((lat > ndfd_bbox[1]) & (lat < ndfd_bbox[3]))
& ((lon > ndfd_bbox[0]) & (lon < ndfd_bbox[2])))
min_y, max_y = indexes[0].min(), indexes[0].max()
datasets['Carolina.L43H86.gdd.chill_mask'].update(time_attrs)
datasets['Carolina.L43H86.stage.index'].update(time_attrs)
datasets['Carolina.L43H86.kill.level'].update(time_attrs)
prov_attrs = factory.tool.datasets.provenance.attrs
datasets['Carolina.L43H86.gdd.provenance'] = deepcopy(prov_attrs)
datasets['Carolina.L43H86.stage.provenance'] = deepcopy(prov_attrs)
datasets['Carolina.L43H86.kill.provenance'] = deepcopy(prov_attrs)
# get last valid from from frost's variety file
frost_reader = Hdf5DateGridFileReader(frost_filepath)
last_valid_date = frost_reader.dateAttribute('Carolina.L43H86.gdd.accumulated',
'last_valid_date')
# start date for this build is one day after last valid date
start_date = last_valid_date + ONE_DAY
start_datetime = asDatetime(start_date)
frost_reader.close()
# get mint and key dates
temps_reader = Hdf5DateGridFileReader(temps_filepath)
if debug: print 'temps file groups :', temps_reader.listGroups()
try:
end_date = temps_reader.dateAttribute('temps.mint', 'last_valid_date')
mint = temps_reader.dateSlice('temps.mint', start_date, end_date)
except:
end_date = temps_reader.dateAttribute('reported.mint', 'last_valid_date')
mint = temps_reader.dateSlice('reported.mint', start_date, end_date)
temps_reader.close()
del temps_reader
# create a chill file reader
def _datasetStartDate(self, dataset):
if 'start_date' in dataset.attrs.key():
return asDatetime(dataset.start_date)
else:
return self._fileStartDate()
def _indexForDate(self, dataset_name, date):
return (asDatetime(date) - self.start_date).days
day = int(date_args[5])
end_date = datetime(year,month,day)
else:
errmsg = 'Invalid number of date arguments (%d).' % date_args
raise ValueError, errmsg
target_year = factory.getTargetYear(start_date)
else:
errmsg = 'Invalid number of date arguments (%d).' % date_args
raise ValueError, errmsg
reader = factory.getVarietyReader(target_year, variety)
description = reader.getGroupAttribute(group_name, 'description').title()
if start_date is None:
start_date_str = reader.getDatasetAttribute(prov_dataset_name, 'start_date')
start_date = asDatetime(start_date_str)
if end_date is None:
last_date = reader.getDatasetAttribute(prov_dataset_name, 'last_valid_date')
end_date = asDatetime(last_date)
print variety.description.center(44)
start_index, end_index = \
reader.indexesFromDates(prov_dataset_name, start_date, end_date)
provenance = reader.getDataset(prov_dataset_name)
names = provenance.dtype.names
print (DESCRIPTION % description).center(44)
print (DATES % (start_date_str, last_date)).center(44)
print '\n %s %s %s %s diff' % names[:4]
print '========== ====== ====== ===== ====='
def index(self, date_time):
_date_time = asDatetime(date_time)
if _date_time == self.base_time: return 0
delta = _date_time - self.base_time
if (delta.days % self.interval) == 0: return delta.days / self.interval
else: return (delta.days + self.interval) / self.interval
def __init__(self, base_time, interval):
self.base_time = asDatetime(base_time)
self.interval = interval