Exemplo n.º 1
0
    def test_unpack(self):
        # create txt-file
        txt1 = open(os.path.join(curpath(), 'data', 'test1.txt'), 'w')
        txt1.close()

        # create first zip archive
        ziparc = zipfile.ZipFile(os.path.join(curpath(), 'data',
                                              'ziparc.zip'), 'w')

        # write txt-file and folder to zip archive
        ziparc.write(os.path.join(curpath(), 'data', 'test1.txt'))
        ziparc.close()

        # 'create' second txt-file
        os.rename(os.path.join(curpath(), 'data', 'test1.txt'),
                  os.path.join(curpath(), 'data', 'test2.txt'))

        # create second zip archive
        ziparc2 = zipfile.ZipFile(os.path.join(curpath(), 'data',
                                               'ziparc2.zip'), 'w')

        # write second txt-file and first zip archive to second zip archive
        ziparc2.write(os.path.join(curpath(), 'data', 'test2.txt'))
        ziparc2.write(os.path.join(curpath(), 'data', 'ziparc.zip'))
        ziparc2.close()

        dirlen = 2
        up.unpack(ziparc2.filename, self.outpath)

        unpack_len = len(os.listdir(self.outpath))

        assert unpack_len == dirlen

        shutil.rmtree(self.outpath)
        os.remove(os.path.join(curpath(), 'data', ziparc.filename))
        os.remove(os.path.join(curpath(), 'data', ziparc2.filename))
Exemplo n.º 2
0
    def test_unpack(self):
        # create txt-file
        txt1 = open(os.path.join(curpath(), 'data', 'test1.txt'), 'w')
        txt1.close()

        # create first zip archive
        ziparc = zipfile.ZipFile(os.path.join(curpath(), 'data', 'ziparc.zip'),
                                 'w')

        # write txt-file and folder to zip archive
        ziparc.write(os.path.join(curpath(), 'data', 'test1.txt'))
        ziparc.close()

        # 'create' second txt-file
        os.rename(os.path.join(curpath(), 'data', 'test1.txt'),
                  os.path.join(curpath(), 'data', 'test2.txt'))

        # create second zip archive
        ziparc2 = zipfile.ZipFile(
            os.path.join(curpath(), 'data', 'ziparc2.zip'), 'w')

        # write second txt-file and first zip archive to second zip archive
        ziparc2.write(os.path.join(curpath(), 'data', 'test2.txt'))
        ziparc2.write(os.path.join(curpath(), 'data', 'ziparc.zip'))
        ziparc2.close()

        dirlen = 2
        up.unpack(ziparc2.filename, self.outpath)

        unpack_len = len(os.listdir(self.outpath))

        assert unpack_len == dirlen

        shutil.rmtree(self.outpath)
        os.remove(os.path.join(curpath(), 'data', ziparc.filename))
        os.remove(os.path.join(curpath(), 'data', ziparc2.filename))
Exemplo n.º 3
0
    def _resample_spatial(self, region, begin, end, delete_rawdata,
                          shapefile=None):
        """Helper method that calls spatial resampling routines.

        Parameters:
        region : str
            FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
        begin : datetime
            Start date of resampling
        end : datetime
            End date of resampling
        delete_rawdata : bool
            True if original downloaded files should be deleted after
            resampling
        """

        dest_file = self._get_tmp_filepath('spatial', region)

        dirList = os.listdir(self.rawdata_path)
        dirList.sort()

        if region == 'global':
            grid = gr.RegularGrid(sp_res=self.dest_sp_res)
        else:
            grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)

        for item in dirList:

            src_file = os.path.join(self.rawdata_path, item)

            fdate = get_file_date(item, self.filedate)

            if begin is not None:
                if fdate < begin:
                    continue

            if end is not None:
                if fdate > end:
                    continue

            if check_compressed(src_file):
                dirname = os.path.splitext(item)[0]
                dirpath = os.path.join(self.rawdata_path, dirname)
                unpack(src_file)
                src_file = select_file(os.listdir(dirpath))
                src_file = os.path.join(dirpath, src_file)

            if begin is not None:
                if fdate < begin:
                    if check_compressed(item):
                        shutil.rmtree(os.path.join(self.rawdata_path,
                                                   os.path.splitext(item)[0]))
                    continue
            if end is not None:
                if fdate > end:
                    if check_compressed(item):
                        shutil.rmtree(os.path.join(self.rawdata_path,
                                                   os.path.splitext(item)[0]))
                    continue

            print '.',

            image, _, _, _, timestamp, metadata = \
                resample_to_shape(src_file, region, self.dest_sp_res, grid,
                                  self.name, self.nan_value,
                                  self.dest_nan_value, self.variables,
                                  shapefile)

            if timestamp is None:
                timestamp = get_file_date(item, self.filedate)

            if self.temp_res == self.dest_temp_res:
                filename = (region + '_' + str(self.dest_sp_res) + '_'
                            + str(self.dest_temp_res) + '.nc')
                dfile = os.path.join(self.data_path, filename)
                nc.save_image(image, timestamp, region, metadata, dfile,
                              self.dest_start_date, self.dest_sp_res,
                              self.dest_nan_value, shapefile,
                              self.dest_temp_res)
            else:
                nc.write_tmp_file(image, timestamp, region, metadata,
                                  dest_file, self.dest_start_date,
                                  self.dest_sp_res, self.dest_nan_value,
                                  shapefile)

            # deletes unpacked files if existing
            if check_compressed(item):
                shutil.rmtree(os.path.join(self.rawdata_path,
                                           os.path.splitext(item)[0]))

        print ''
Exemplo n.º 4
0
    def _resample_spatial(self,
                          region,
                          begin,
                          end,
                          delete_rawdata,
                          shapefile=None):
        """Helper method that calls spatial resampling routines.

        Parameters:
        region : str
            FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
        begin : datetime
            Start date of resampling
        end : datetime
            End date of resampling
        delete_rawdata : bool
            True if original downloaded files should be deleted after
            resampling
        """

        dest_file = self._get_tmp_filepath('spatial', region)

        dirList = os.listdir(self.rawdata_path)
        dirList.sort()

        if region == 'global':
            grid = gr.RegularGrid(sp_res=self.dest_sp_res)
        else:
            grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)

        for item in dirList:

            src_file = os.path.join(self.rawdata_path, item)

            fdate = get_file_date(item, self.filedate)

            if begin is not None:
                if fdate < begin:
                    continue

            if end is not None:
                if fdate > end:
                    continue

            if check_compressed(src_file):
                dirname = os.path.splitext(item)[0]
                dirpath = os.path.join(self.rawdata_path, dirname)
                unpack(src_file)
                src_file = select_file(os.listdir(dirpath))
                src_file = os.path.join(dirpath, src_file)

            if begin is not None:
                if fdate < begin:
                    if check_compressed(item):
                        shutil.rmtree(
                            os.path.join(self.rawdata_path,
                                         os.path.splitext(item)[0]))
                    continue
            if end is not None:
                if fdate > end:
                    if check_compressed(item):
                        shutil.rmtree(
                            os.path.join(self.rawdata_path,
                                         os.path.splitext(item)[0]))
                    continue

            print '.',

            try:
                image, _, _, _, timestamp, metadata = \
                    resample_to_shape(src_file, region, self.dest_sp_res, grid,
                                      self.name, self.nan_value,
                                      self.dest_nan_value, self.variables,
                                      shapefile)
            except ValueError:
                print "[INFO] no data available for that region."
                return "[INFO] no data available for that region."

            if timestamp is None:
                timestamp = get_file_date(item, self.filedate)

            if self.temp_res == self.dest_temp_res:
                filename = (region + '_' + str(self.dest_sp_res) + '_' +
                            str(self.dest_temp_res) + '.nc')
                dfile = os.path.join(self.data_path, filename)
                nc.save_image(image, timestamp, region, metadata, dfile,
                              self.dest_start_date, self.dest_sp_res,
                              self.dest_nan_value, shapefile,
                              self.dest_temp_res)
            else:
                nc.write_tmp_file(image, timestamp, region, metadata,
                                  dest_file, self.dest_start_date,
                                  self.dest_sp_res, self.dest_nan_value,
                                  shapefile)

            # deletes unpacked files if existing
            if check_compressed(item):
                shutil.rmtree(
                    os.path.join(self.rawdata_path,
                                 os.path.splitext(item)[0]))

        print ''