def generateYearsArray(start_year, end_year, **kwargs): first_year = asDatetime(start_year) last_year = asDatetime(end_year) interval = relativedelta(years=int(kwargs.get('interval', 1))) date_format = kwargs.get('date_format', None) as_numpy = kwargs.get('as_numpy', True) years = [] date = first_year if date_format in (datetime, object, 'object'): while date <= last_year: years.append(date) date += interval dtype = 'object' as_numpy = False elif date_format in (int, 'int'): while date <= last_year: years.append(date.year) date += interval dtype = 'i2' else: if date_format is None: date_format = '%Y' while date <= last_year: years.append(date.strftime(date_format)) date += interval dtype = '|S%d' % len(years[0]) if as_numpy: return N.array(years, dtype=dtype) else: return tuple(years)
def _loadDataGridAttributes_(self): if hasattr(self, 'start_date') \ and isinstance(self.start_date, basestring): self.start_date = asDatetime(self.start_date) if hasattr(self, 'end_date') \ and isinstance(self.end_date, basestring): self.end_date = asDatetime(self.end_date)
def listDates(self, interval_start, interval_end, interval=None): if interval is None: num_days = relativedelta(days=self.interval) else: num_days = relativedelta(days=interval) dates = [] date = asDatetime(interval_start) while date <= asDatetime(interval_end): dates.append(date) date += num_days return tuple(dates)
def listDates(self, interval_start, interval_end, interval=None): if interval is None: _interval = self.interval else: _interval = interval dates = [] date = asDatetime(interval_start) while date <= asDatetime(interval_end): dates.append(date) date = monthsInterval(date, _interval) return tuple(dates)
def setTimeSpan(self, iter_start, iter_end): if isinstance(iter_start, (list, tuple)) and len(iter_start) == 2: iter_start = (iter_start[0], iter_start[1], lastDayOfMonth(*iter_start)) self.iter_start = asDatetime(iter_start) if isinstance(iter_end, (list, tuple)) and len(iter_end) == 2: iter_end = (iter_end[0], iter_end[1], lastDayOfMonth(*iter_end)) self.iter_end = asDatetime(iter_end)
def _validTimes(self, start_time, end_time): """ returns tuple containing index of start_time and index of end_time """ if start_time is None or start_time == ':': if end_time is None: return self.base_time, self.base_time elif end_time == ':': return self.base_time, self.last_time return self.base_time, asDatetime(end_time) else: _start_time = asDatetime(start_time) if end_time is None: return _start_time, _start_time elif end_time == ':': return _start_time, self.last_time return _start_time, asDatetime(end_time)
def _setLastValidDate(self, dataset_path, start_date, data): if data.ndim == 2: num_days = 1 else: num_days = data.shape[0] end_date = start_date + relativedelta(days=num_days - 1) last_date_str = \ self.getDatasetAttribute(dataset_path, 'last_valid_date', None) if isinstance(end_date, datetime): end_is_later = end_date > asDatetime(last_date_str) else: end_is_later = asDatetime(end_date) > asDatetime(last_date_str) if last_date_str is None or end_is_later: self.setDatasetAttribute(dataset_path, 'last_valid_date', end_date.strftime('%Y-%m-%d'))
def _datasetEndDate(self, dataset): if 'end_date' in dataset.attrs.keys(): return asDatetime(dataset.end_date) elif 'end_day' in attrs: return datetime.datetime(self._targetYear(), *attrs['end_day']) else: return self._fileEndDate()
def _indexForTime(_time): delta = asDatetime(_time) - self.base_time hours = (delta.days * 24) + (delta.seconds / 3600) indx = hours / interval over = hours % interval if over > 0: return indx + 1 else: return indx
def index(self, date_time): _date_time = asDatetime(date_time) if _date_time == self.base_time: return 0 delta = _date_time - self.base_time hours = (delta.days * 24) + (delta.seconds / 3600) if (hours % self.interval) == 0: return hours / self.interval else: return (hours + self.interval) / self.interval
def _indexForTime(_time): date_time = asDatetime(_time) years = date_time.year - self.base_time.year months = years * 12 if date_time.month != 12: months -= (12 - date_time.month) if self.base_time.month != 1: months -= self.base_time.month - 1 return months
def mapWorkingDir(date, variety, map_key, map_type=None, test_path=False): if '.' in map_key: map_group = map_key.split('.')[0] else: map_group = map_key target_year = targetYearFromDate(asDatetime(date)) variety_dir = varietyWorkingDir(target_year, 'grape', variety, test_path) working_dir = os.path.join(variety_dir, 'maps', nameToDirpath(map_group)) if not os.path.exists(working_dir): os.makedirs(working_dir) return working_dir
def _datasetStartDoy(self, dataset): attrs = dict(dataset.attrs) if hasattr(self, 'start_doy'): return int(attrs['start_doy']) elif 'start_day' in attrs: date = datetime.date(self._targetYear(), *attrs['start_day']) return date.timetuple().tm_yday elif 'start_date' in attrs: return asDatetime(attrs['start_date']).timetuple().tm_yday else: return self._fileStartDoy()
def __init__(self, time_series, start_time=None, end_time=None, interval=None, duration=None): """ handles instances of the TimeSeriesData class or it's subclassses """ if start_time is None: iter_start = time_series.common_start_time else: iter_start = asDatetime(start_time) if end_time is None: iter_end = time_series.common_end_time else: iter_end = asDatetime(end_time) if interval is None: interval = time_series.interval if duration is None: duration = time_series.interval self.iterator = timeIterator(time_series.frequency, iter_start, iter_end, interval, duration)
def webMapFilepath(date, variety, model_name, map_group, map_type, config_key='web_maps'): target_year = targetYearFromDate(asDatetime(date)) webmaps_dir = webMapsDir(target_year, variety) filename = \ webMapFilename(date, variety, model_name, map_group, map_type, config_key) return os.path.join(webmaps_dir, filename)
def generateHoursArray(start_hour, end_hour, **kwargs): first_hour = asDatetime(start_hour) last_hour = asDatetime(end_hour) interval = relativedelta(hours=int(kwargs.get('interval', 1))) date_format = kwargs.get('date_format', None) as_numpy = kwargs.get('as_numpy', True) hours = [] hour = first_hour if date_format in (datetime, object, 'object'): while hour <= last_hour: hours.append(hour) hour += interval dtype = 'object' as_numpy = False elif date_format in (int, 'int'): while hour <= last_hour: hours.append((hour.year * 1000000) + (hour.month * 10000) + (hour.day * 100) + hour.hour) hour += interval dtype = N.dtype(int) elif date_format == 'tuple': while hour <= last_hour: hours.append((hour.year, hour.month, hour.day, hour.hour)) hour += interval dtype = N.dtype({ 'names': ['year', 'month', 'day', 'hour'], 'formats': ['i2', 'i2', 'i2', 'i2'] }) else: if date_format is None: date_format = '%Y-%m-%d-%H' while hour <= last_hour: hours.append(hour.strftime(date_format)) hour += interval dtype = '|S%d' % len(hours[0]) if as_numpy: return N.array(hours, dtype=dtype) else: return tuple(hours)
def _loadManagerAttributes_(self): groups, datasets = self.getFileHierarchy(grouped=True) self._dataset_names = list(datasets) self._group_names = list(groups) attributes = self.getFileAttributes() for attr_name, attr_value in attributes.items(): if attr_name in ('created', 'updated'): try: self.__dict__[attr_name] = asDatetime(attr_value) except: self.__dict__[attr_name] = attr_value else: self.__dict__[attr_name] = attr_value
def __init__(self, time_series, start_time=None, end_time=None, interval=None, duration=None): """ handles instances of the MultiTimeSeries class or it's subclassses """ #TODO : handle 2D arrays if start_time is None: iter_start = time_series.common_start_time else: iter_start = asDatetime(start_time) if end_time is None: iter_end = time_series.common_end_time else: iter_end = asDatetime(end_time) if interval is None: interval = time_series.interval if duration is None: duration = time_series.interval self.dataset_names = time_series.dataset_names self.indexers = time_series.indexers self.iterator = timeIterator(time_series.frequency, iter_start, iter_end, interval, duration)
def generateDaysArray(start_date, end_date, **kwargs): first_date = asDatetime(start_date) last_date = asDatetime(end_date) interval = relativedelta(days=int(kwargs.get('interval', 1))) date_format = kwargs.get('date_format', None) as_numpy = kwargs.get('as_numpy', True) days = [] date = first_date if date_format in (datetime, object, 'object'): while date <= last_date: days.append(date) date += interval dtype = 'object' as_numpy = False elif date_format in (int, 'int'): while date <= date_hour: days.append((date.year * 10000) + (date.month * 100) + date.day) date += interval dtype = int elif date_format == 'tuple': while date <= last_date: days.append((date.year, date.month, date.day)) date += interval dtype = N.dtype({ 'names': ['year', 'month', 'day'], 'formats': ['i2', 'i2', 'i2'] }) else: if date_format is None: date_format = '%Y-%m-%d' while date <= last_date: days.append(date.strftime(date_format)) date += interval dtype = '|S%d' % len(days[0]) if as_numpy: return N.array(days, dtype=dtype) else: return tuple(days)
def mapFilepath(date, variety, model_name, map_group, map_type, lo_gdd_th=None, hi_gdd_th=None, test_path=False): target_year = targetYearFromDate(asDatetime(date)) map_dir = mapWorkingDir(target_year, variety, model_name, map_group, map_type, lo_gdd_th, hi_gdd_th, test_path) filename = mapFilename(date, variety, model_name, map_group, map_type, lo_gdd_th, hi_gdd_th, test_path) return os.path.join(map_dir, filename)
def compareDates(self, date1, op, date2): if isinstance(date1, datetime.datetime): if isinstance(date2, datetime.datetime): return self._compareDates(date1, op, date2) else: return self._compareDates(date1, op, asDatetime(date2)) elif isinstance(date1, datetime.date): if isinstance(date2, datetime.date): return self._compareDates(date1, op, date2) else: return self._compareDates(date1, op, asDatetimeDate(date2)) elif isinstance(date1, (tuple, basestring)): return self._compareDates(asDatetimeDate(date1), op, date2) else: errmsg = 'Invalid type for argument "date1" : %s' raise TypeError, errmsg % str(type(date1))
def index(self, date_time): _date_time = asDatetime(date_time) _base_time = self.base_time if _date_time.year == _base_time.year\ and _date_time.month == _base_time.month: return 0 _interval = self.interval # index may be negative if date_time is earlier than base_time years = _date_time.year - _base_time.year if years == 0: # both in same year months = _date_time.month - _base_time.month else: # different years months = (years * 12) + (_date_time.month - _base_time.month) if (months % _interval) == 0: return months / _interval else: return (months + _interval) / _interval
def unpackQueryResults(self, query_result, elems, meta): # unpack the grids if isinstance(elems, basestring): _elems = elems.split(',') else: _elems = elems data = [ ] for result in query_result['data']: date = [asDatetime(result[0]),] for indx, grid in enumerate(result[1:]): elem_name = _elems[indx] date.append(self.unpackGrid(elem_name, grid)) data.append(date) if len(data) > 1: # convert list of (date, elem, ...) tuples to date tuple, # and a data tuple for each element data = zip(*data) # convert data tuples to arrays for indx, grid in enumerate(data[1:],start=1): data[indx] = N.array(grid) else: date = data[0][0] data = [grid for grid in data[0][1:]] data.insert(0,(date,)) data_dict = { 'dates': data[0], } for indx, elem_name in enumerate(_elems, start=1): data_dict[elem_name] = data[indx] if meta is not None: meta_dict = query_result['meta'] if 'elev' in meta_dict: meta_dict['elev'] = self.unpackGrid('elev', meta_dict['elev']) if 'lat' in meta_dict: meta_dict['lat'] = self.unpackGrid('lat', meta_dict['lat']) meta_dict['lon'] = self.unpackGrid('lon', meta_dict['lon']) data_dict.update(meta_dict) return data_dict
ndfd_bbox.append(region_bbox[1] - lat_offset) ndfd_bbox.append(region_bbox[2] + lon_offset) ndfd_bbox.append(region_bbox[3] + lat_offset) print 'NDFD bounding box', ndfd_bbox #for grib in gribs.read(): # print '\n', grib, '\n', grib.keys() print 'exploring grib variable "%s"' % grib_variable grib = gribs.select(name=var_name_map[grib_variable]) print '\n', grib print '\nin grib 0 :' message = grib[message_num] print ' "dataDate"', message.dataDate fcast_hour = message['hour'] print ' "fcast_hour"', fcast_hour print ' "forecastTime"', message['forecastTime'] fcast_time = asDatetime(message.dataDate) + relativedelta(hours=fcast_hour) print 'forecast datetime', fcast_time lat, lon = message.latlons() print ' grib lat', lat.shape, lat.min(), lat.max() print ' grib lon', lon.shape, lon.min(), lon.max() maxt = message.values print ' grib maxt', maxt.shape, maxt.min(), maxt.max() if region.name == 'conus': ndfd_lat = lat ndfd_lon = lon ndfd_maxt = maxt else: indexes = N.where(((lat > ndfd_bbox[1]) & (lat < ndfd_bbox[3])) & ((lon > ndfd_bbox[0]) & (lon < ndfd_bbox[2]))) min_y, max_y = indexes[0].min(), indexes[0].max()
datasets['Carolina.L43H86.gdd.chill_mask'].update(time_attrs) datasets['Carolina.L43H86.stage.index'].update(time_attrs) datasets['Carolina.L43H86.kill.level'].update(time_attrs) prov_attrs = factory.tool.datasets.provenance.attrs datasets['Carolina.L43H86.gdd.provenance'] = deepcopy(prov_attrs) datasets['Carolina.L43H86.stage.provenance'] = deepcopy(prov_attrs) datasets['Carolina.L43H86.kill.provenance'] = deepcopy(prov_attrs) # get last valid from from frost's variety file frost_reader = Hdf5DateGridFileReader(frost_filepath) last_valid_date = frost_reader.dateAttribute('Carolina.L43H86.gdd.accumulated', 'last_valid_date') # start date for this build is one day after last valid date start_date = last_valid_date + ONE_DAY start_datetime = asDatetime(start_date) frost_reader.close() # get mint and key dates temps_reader = Hdf5DateGridFileReader(temps_filepath) if debug: print 'temps file groups :', temps_reader.listGroups() try: end_date = temps_reader.dateAttribute('temps.mint', 'last_valid_date') mint = temps_reader.dateSlice('temps.mint', start_date, end_date) except: end_date = temps_reader.dateAttribute('reported.mint', 'last_valid_date') mint = temps_reader.dateSlice('reported.mint', start_date, end_date) temps_reader.close() del temps_reader # create a chill file reader
def _datasetStartDate(self, dataset): if 'start_date' in dataset.attrs.key(): return asDatetime(dataset.start_date) else: return self._fileStartDate()
def _indexForDate(self, dataset_name, date): return (asDatetime(date) - self.start_date).days
day = int(date_args[5]) end_date = datetime(year,month,day) else: errmsg = 'Invalid number of date arguments (%d).' % date_args raise ValueError, errmsg target_year = factory.getTargetYear(start_date) else: errmsg = 'Invalid number of date arguments (%d).' % date_args raise ValueError, errmsg reader = factory.getVarietyReader(target_year, variety) description = reader.getGroupAttribute(group_name, 'description').title() if start_date is None: start_date_str = reader.getDatasetAttribute(prov_dataset_name, 'start_date') start_date = asDatetime(start_date_str) if end_date is None: last_date = reader.getDatasetAttribute(prov_dataset_name, 'last_valid_date') end_date = asDatetime(last_date) print variety.description.center(44) start_index, end_index = \ reader.indexesFromDates(prov_dataset_name, start_date, end_date) provenance = reader.getDataset(prov_dataset_name) names = provenance.dtype.names print (DESCRIPTION % description).center(44) print (DATES % (start_date_str, last_date)).center(44) print '\n %s %s %s %s diff' % names[:4] print '========== ====== ====== ===== ====='
def index(self, date_time): _date_time = asDatetime(date_time) if _date_time == self.base_time: return 0 delta = _date_time - self.base_time if (delta.days % self.interval) == 0: return delta.days / self.interval else: return (delta.days + self.interval) / self.interval
def __init__(self, base_time, interval): self.base_time = asDatetime(base_time) self.interval = interval