def fill_gaps(self, begin=None, end=None): """ Detects gaps in data and tries to fill them by downloading and resampling the data within these periods. Parameters ---------- begin : datetime Begin date of intervall to check, defaults to None. end : datetime End date of intervall to check, defaults to None. """ gaps = [] for region in self.valid_regions: if self.regions is not None: if region not in self.regions: continue _, _, period = nc.get_properties(self.src_file[region]) if begin is None: if self.begin_date < self.dest_start_date: begin = self.dest_start_date else: begin = self.begin_date if end is None: end = period[1] drange = dt.get_dtindex(self.dest_temp_res, begin, end) for date in drange: nonans = [] for var in self.get_variables(): img, _, _, _ = self.read_img(date, region, var) if np.nanmean(img) is not np.ma.masked: nonans.append(1) if len(nonans) == 0: if date not in gaps: gaps.append(date) if len(gaps) == 0: print '[INFO] No gaps found.' else: print('[INFO] Found ' + str(len(gaps)) + ' gap(s), attempt to fill..') for date in gaps: if self.dest_temp_res in ['day', 'daily']: begin = date end = date else: begin, end = dt.check_period_boundaries( self.dest_temp_res, date) self.download_and_resample(begin=begin, end=end)
def fill_gaps(self, begin=None, end=None): """ Detects gaps in data and tries to fill them by downloading and resampling the data within these periods. Parameters ---------- begin : datetime Begin date of intervall to check, defaults to None. end : datetime End date of intervall to check, defaults to None. """ gaps = [] for region in self.valid_regions: if self.regions is not None: if region not in self.regions: continue _, _, period = nc.get_properties(self.src_file[region]) if begin is None: if self.begin_date < self.dest_start_date: begin = self.dest_start_date else: begin = self.begin_date if end is None: end = period[1] drange = dt.get_dtindex(self.dest_temp_res, begin, end) for date in drange: nonans = [] for var in self.get_variables(): img, _, _, _ = self.read_img(date, region, var) if np.nanmean(img) is not np.ma.masked: nonans.append(1) if len(nonans) == 0: if date not in gaps: gaps.append(date) if len(gaps) == 0: print '[INFO] No gaps found.' else: print('[INFO] Found ' + str(len(gaps)) + ' gap(s), attempt to fill..') for date in gaps: if self.dest_temp_res in ['day', 'daily']: begin = date end = date else: begin, end = dt.check_period_boundaries(self.dest_temp_res, date) self.download_and_resample(begin=begin, end=end)
def _check_begin_end(self, begin, end): """ Checks begin and end date and returns valid dates. Parameters ---------- begin : datetime Begin date to check. end : datetime End date to check. Returns ------- begin : datetime Begin date. end : datetime End date. """ if begin is None: if self.dest_start_date < self.begin_date: begin = self.begin_date else: begin = self.dest_start_date if begin < self._get_download_date(): begin = self._get_download_date() # start one period earlier to close possible gaps begin = begin - timedelta(days=1) begin, _ = dt.check_period_boundaries(self.dest_temp_res, begin) if begin < self.begin_date: begin = self.begin_date if begin < self.dest_start_date: begin = self.dest_start_date if end is None: end = datetime.now() return begin, end