Exemplo n.º 1
0
    def _resample_spatial(self,
                          region,
                          begin,
                          end,
                          delete_rawdata,
                          shapefile=None):
        """Helper method that calls spatial resampling routines.

        Parameters:
        region : str
            FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
        begin : datetime
            Start date of resampling
        end : datetime
            End date of resampling
        delete_rawdata : bool
            True if original downloaded files should be deleted after
            resampling
        """

        dest_file = self._get_tmp_filepath('spatial', region)

        dirList = os.listdir(self.rawdata_path)
        dirList.sort()

        if region == 'global':
            grid = gr.RegularGrid(sp_res=self.dest_sp_res)
        else:
            grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)

        for item in dirList:

            src_file = os.path.join(self.rawdata_path, item)

            fdate = get_file_date(item, self.filedate)

            if begin is not None:
                if fdate < begin:
                    continue

            if end is not None:
                if fdate > end:
                    continue

            if check_compressed(src_file):
                dirname = os.path.splitext(item)[0]
                dirpath = os.path.join(self.rawdata_path, dirname)
                unpack(src_file)
                src_file = select_file(os.listdir(dirpath))
                src_file = os.path.join(dirpath, src_file)

            if begin is not None:
                if fdate < begin:
                    if check_compressed(item):
                        shutil.rmtree(
                            os.path.join(self.rawdata_path,
                                         os.path.splitext(item)[0]))
                    continue
            if end is not None:
                if fdate > end:
                    if check_compressed(item):
                        shutil.rmtree(
                            os.path.join(self.rawdata_path,
                                         os.path.splitext(item)[0]))
                    continue

            print '.',

            try:
                image, _, _, _, timestamp, metadata = \
                    resample_to_shape(src_file, region, self.dest_sp_res, grid,
                                      self.name, self.nan_value,
                                      self.dest_nan_value, self.variables,
                                      shapefile)
            except ValueError:
                print "[INFO] no data available for that region."
                return "[INFO] no data available for that region."

            if timestamp is None:
                timestamp = get_file_date(item, self.filedate)

            if self.temp_res == self.dest_temp_res:
                filename = (region + '_' + str(self.dest_sp_res) + '_' +
                            str(self.dest_temp_res) + '.nc')
                dfile = os.path.join(self.data_path, filename)
                nc.save_image(image, timestamp, region, metadata, dfile,
                              self.dest_start_date, self.dest_sp_res,
                              self.dest_nan_value, shapefile,
                              self.dest_temp_res)
            else:
                nc.write_tmp_file(image, timestamp, region, metadata,
                                  dest_file, self.dest_start_date,
                                  self.dest_sp_res, self.dest_nan_value,
                                  shapefile)

            # deletes unpacked files if existing
            if check_compressed(item):
                shutil.rmtree(
                    os.path.join(self.rawdata_path,
                                 os.path.splitext(item)[0]))

        print ''
Exemplo n.º 2
0
    def _resample_spatial(self, region, begin, end, delete_rawdata,
                          shapefile=None):
        """Helper method that calls spatial resampling routines.

        Parameters:
        region : str
            FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
        begin : datetime
            Start date of resampling
        end : datetime
            End date of resampling
        delete_rawdata : bool
            True if original downloaded files should be deleted after
            resampling
        """

        dest_file = self._get_tmp_filepath('spatial', region)

        dirList = os.listdir(self.rawdata_path)
        dirList.sort()

        if region == 'global':
            grid = gr.RegularGrid(sp_res=self.dest_sp_res)
        else:
            grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)

        for item in dirList:

            src_file = os.path.join(self.rawdata_path, item)

            fdate = get_file_date(item, self.filedate)

            if begin is not None:
                if fdate < begin:
                    continue

            if end is not None:
                if fdate > end:
                    continue

            if check_compressed(src_file):
                dirname = os.path.splitext(item)[0]
                dirpath = os.path.join(self.rawdata_path, dirname)
                unpack(src_file)
                src_file = select_file(os.listdir(dirpath))
                src_file = os.path.join(dirpath, src_file)

            if begin is not None:
                if fdate < begin:
                    if check_compressed(item):
                        shutil.rmtree(os.path.join(self.rawdata_path,
                                                   os.path.splitext(item)[0]))
                    continue
            if end is not None:
                if fdate > end:
                    if check_compressed(item):
                        shutil.rmtree(os.path.join(self.rawdata_path,
                                                   os.path.splitext(item)[0]))
                    continue

            print '.',

            image, _, _, _, timestamp, metadata = \
                resample_to_shape(src_file, region, self.dest_sp_res, grid,
                                  self.name, self.nan_value,
                                  self.dest_nan_value, self.variables,
                                  shapefile)

            if timestamp is None:
                timestamp = get_file_date(item, self.filedate)

            if self.temp_res == self.dest_temp_res:
                filename = (region + '_' + str(self.dest_sp_res) + '_'
                            + str(self.dest_temp_res) + '.nc')
                dfile = os.path.join(self.data_path, filename)
                nc.save_image(image, timestamp, region, metadata, dfile,
                              self.dest_start_date, self.dest_sp_res,
                              self.dest_nan_value, shapefile,
                              self.dest_temp_res)
            else:
                nc.write_tmp_file(image, timestamp, region, metadata,
                                  dest_file, self.dest_start_date,
                                  self.dest_sp_res, self.dest_nan_value,
                                  shapefile)

            # deletes unpacked files if existing
            if check_compressed(item):
                shutil.rmtree(os.path.join(self.rawdata_path,
                                           os.path.splitext(item)[0]))

        print ''
Exemplo n.º 3
0
    def test_select_file(self):
        fname = 'test.nc'
        flist = ff.select_file(self.filelist)

        assert flist == fname