def test_create_ndvi_worldgrid(tempdir): """ This is really an integration test because it tests the following : - Creates of a NDVI worldgrid from HDF files - Writes the NDVI fractions containing the HDF data - Loads (using geographical coordinates) the area corresponding to the HDF from the created jgrid - Verify that the jgrid data matches the HDF data This require that jgrid read/write work properly as well as all the jgrid georeferencing """ script = os.path.join(test_utils.get_rastercube_dir(), 'scripts', 'create_ndvi_worldgrid.py') worldgrid = tempdir ndvi_dir = os.path.join(worldgrid, 'ndvi') qa_dir = os.path.join(worldgrid, 'qa') dates_csv = os.path.join(utils.get_data_dir(), '1_manual', 'ndvi_dates.2.csv') tile = 'h29v07' print 'dates_csv : ', dates_csv cmd = [ sys.executable, script, '--tile=%s' % tile, '--noconfirm', '--worldgrid=%s' % worldgrid, '--frac_ndates=1', '--dates_csv=%s' % dates_csv ] output = subprocess.check_output(cmd) # Verify that the header has the correct dates ndvi_header = jgrid.load(ndvi_dir) qa_header = jgrid.load(qa_dir) dates = [utils.format_date(ts) for ts in ndvi_header.timestamps_ms] assert dates == ['2000_02_18', '2000_03_05'] assert ndvi_header.num_dates_fracs == 2 # Load the HDF and the corresponding date from the jgrid and # check for consistency hdf_fname = os.path.join(utils.get_modis_hdf_dir(), '2000', 'MOD13Q1.A2000065.h29v07.005.2008238013448.hdf') f = modis.ModisHDF(hdf_fname) ndvi_ds = f.load_gdal_dataset(modis.MODIS_NDVI_DATASET_NAME) hdf_ndvi = ndvi_ds.ReadAsArray() hdf_qa = f.load_gdal_dataset(modis.MODIS_QA_DATASET_NAME).ReadAsArray() # Load from the jgrid using the lat/lng polygon of the HDF file # This means we also test georeferencing hdf_poly = gdal_utils.latlng_bounding_box_from_ds(ndvi_ds) xy_from, qa, qa_mask, ndvi, ndvi_mask = \ jgrid_utils.load_poly_latlng_from_multi_jgrids( [qa_header, ndvi_header], hdf_poly) assert ndvi.shape[:2] == hdf_ndvi.shape # Verify that the jgrid ndvi and the HDF ndvi store the same values assert_array_equal(hdf_ndvi, ndvi[:, :, 1]) assert_array_equal(hdf_qa, qa[:, :, 1])
def truncate_frac(frac_num, ndvi_root, qa_root): """ Given a frac_num, will truncate the first hdfs file to have a size of (frac_width, frac_height, frac_ndates) which should correspond to (400, 400, 200) """ _start = time.time() ndvi_header = jgrid.load(ndvi_root) qa_header = jgrid.load(qa_root) frac_d = 0 frac_id = (frac_num, frac_d) try: ndvi = jgrid.read_frac(ndvi_header.frac_fname(frac_id)) qa = jgrid.read_frac(qa_header.frac_fname(frac_id)) except ValueError: print 'Fraction', frac_num, 'is corrupted!' print 'Solve the problem for frac_num:', frac_num, 'frac_d:', frac_d, 'and execute the script again' return # At this point, we just have to truncate the array if ndvi is not None: if ndvi.shape[2] > ndvi_header.frac_ndates: ndvi = ndvi[:, :, 0:ndvi_header.frac_ndates] ndvi_header.write_frac(frac_id, ndvi) else: print frac_num, ': NDVI already OK' else: print frac_num, ': NDVI is None' if qa is not None: if qa.shape[2] > qa_header.frac_ndates: qa = qa[:, :, 0:qa_header.frac_ndates] qa_header.write_frac(frac_id, qa) else: print frac_num, ': QA already OK' else: print frac_num, ': QA is None' print 'Processed %d, took %.02f [s]' % (frac_num, time.time() - _start) sys.stdout.flush()
def truncate_frac(frac_num, ndvi_root, qa_root): """ Given a frac_num, will truncate the first hdfs file to have a size of (frac_width, frac_height, frac_ndates) which should correspond to (400, 400, 200) """ _start = time.time() ndvi_header = jgrid.load(ndvi_root) qa_header = jgrid.load(qa_root) frac_d = 0 frac_id = (frac_num, frac_d) try: ndvi = jgrid.read_frac(ndvi_header.frac_fname(frac_id)) qa = jgrid.read_frac(qa_header.frac_fname(frac_id)) except ValueError: print 'Fraction', frac_num, 'is corrupted!' print 'Solve the problem for frac_num:', frac_num, 'frac_d:', frac_d, 'and execute the script again' return # At this point, we just have to truncate the array if ndvi is not None: if ndvi.shape[2] > ndvi_header.frac_ndates: ndvi = ndvi[:, :, 0:ndvi_header.frac_ndates] ndvi_header.write_frac(frac_id, ndvi) else: print frac_num, ': NDVI already OK' else: print frac_num, ': NDVI is None' if qa is not None: if qa.shape[2] > qa_header.frac_ndates: qa = qa[:, :, 0:qa_header.frac_ndates] qa_header.write_frac(frac_id, qa) else: print frac_num, ': QA already OK' else: print frac_num, ': QA is None' print 'Processed %d, took %.02f [s]' % (frac_num, time.time() - _start) sys.stdout.flush()
def assert_grids_same(root1, root2): """ Asserts that two jgrids are the same (same header, same data). This assert that the two grids have the same x/y chunking, but NOT necessarily the same date chunking """ h1 = jgrid.load(root1) h2 = jgrid.load(root2) assert np.all(h1.timestamps_ms == h2.timestamps_ms) assert np.all(h1.shape == h2.shape) assert h1.num_fracs == h2.num_fracs fracs1 = h1.list_available_fracnums() fracs2 = h2.list_available_fracnums() assert np.all(fracs1 == fracs2) for frac_num in fracs1: data1 = h1.load_frac_by_num(frac_num) data2 = h2.load_frac_by_num(frac_num) assert_array_equal(data1, data2)
def assert_grids_same(root1, root2): """ Asserts that two jgrids are the same (same header, same data). This assert that the two grids have the same x/y chunking, but NOT necessarily the same date chunking """ h1 = jgrid.load(root1) h2 = jgrid.load(root2) assert np.all(h1.timestamps_ms == h2.timestamps_ms) assert np.all(h1.shape == h2.shape) assert h1.num_fracs == h2.num_fracs fracs1 = h1.list_available_fracnums() fracs2 = h2.list_available_fracnums() assert np.all(fracs1 == fracs2) for frac_num in fracs1: data1 = h1.load_frac_by_num(frac_num) data2 = h2.load_frac_by_num(frac_num) assert_array_equal(data1, data2)
print 'Processed %d, took %.02f [s]' % (frac_num, time.time() - _start) sys.stdout.flush() if __name__ == '__main__': args = parser.parse_args() worldgrid = args.worldgrid ndvi_root = os.path.join(worldgrid, 'ndvi') qa_root = os.path.join(worldgrid, 'qa') nworkers = args.nworkers fraction = args.fraction assert jgrid.Header.exists(ndvi_root) print 'Reading headers from HDFS...' ndvi_header = jgrid.load(ndvi_root) qa_header = jgrid.load(qa_root) assert np.all(ndvi_header.timestamps_ms == qa_header.timestamps_ms) # -- Figure out the fractions we have to update print 'Looking for available fractions in HDFS...' fractions = ndvi_header.list_available_fracnums() if len(fractions) == 0: print 'No fractions to process - terminating' sys.exit(0) assert np.all(fractions == qa_header.list_available_fracnums()) if fraction is None:
def test_create_ndvi_worldgrid(tempdir): """ This is really an integration test because it tests the following : - Creates of a NDVI worldgrid from HDF files - Writes the NDVI fractions containing the HDF data - Loads (using geographical coordinates) the area corresponding to the HDF from the created jgrid - Verify that the jgrid data matches the HDF data This require that jgrid read/write work properly as well as all the jgrid georeferencing """ script = os.path.join(test_utils.get_rastercube_dir(), 'scripts', 'create_ndvi_worldgrid.py') worldgrid = tempdir ndvi_dir = os.path.join(worldgrid, 'ndvi') qa_dir = os.path.join(worldgrid, 'qa') dates_csv = os.path.join(utils.get_data_dir(), '1_manual', 'ndvi_dates.2.csv') tile = 'h29v07' print 'dates_csv : ', dates_csv cmd = [ sys.executable, script, '--tile=%s' % tile, '--noconfirm', '--worldgrid=%s' % worldgrid, '--frac_ndates=1', '--dates_csv=%s' % dates_csv ] output = subprocess.check_output(cmd) # Verify that the header has the correct dates ndvi_header = jgrid.load(ndvi_dir) qa_header = jgrid.load(qa_dir) dates = [utils.format_date(ts) for ts in ndvi_header.timestamps_ms] assert dates == ['2000_02_18', '2000_03_05'] assert ndvi_header.num_dates_fracs == 2 # Load the HDF and the corresponding date from the jgrid and # check for consistency hdf_fname = os.path.join(utils.get_modis_hdf_dir(), '2000', 'MOD13Q1.A2000065.h29v07.005.2008238013448.hdf') f = modis.ModisHDF(hdf_fname) ndvi_ds = f.load_gdal_dataset(modis.MODIS_NDVI_DATASET_NAME) hdf_ndvi = ndvi_ds.ReadAsArray() hdf_qa = f.load_gdal_dataset( modis.MODIS_QA_DATASET_NAME).ReadAsArray() # Load from the jgrid using the lat/lng polygon of the HDF file # This means we also test georeferencing hdf_poly = gdal_utils.latlng_bounding_box_from_ds(ndvi_ds) xy_from, qa, qa_mask, ndvi, ndvi_mask = \ jgrid_utils.load_poly_latlng_from_multi_jgrids( [qa_header, ndvi_header], hdf_poly) assert ndvi.shape[:2] == hdf_ndvi.shape # Verify that the jgrid ndvi and the HDF ndvi store the same values assert_array_equal(hdf_ndvi, ndvi[:, :, 1]) assert_array_equal(hdf_qa, qa[:, :, 1])
return ndvi, qa if __name__ == '__main__': args = parser.parse_args() frac_num = args.fraction frac_d = args.fraction_part frac_id = (frac_num, frac_d) modis_dir = utils.get_modis_hdf_dir() worldgrid = args.worldgrid ndvi_root = os.path.join(worldgrid, 'ndvi') qa_root = os.path.join(worldgrid, 'qa') assert jgrid.Header.exists(ndvi_root) ndvi_header = jgrid.load(ndvi_root) qa_header = jgrid.load(qa_root) fname = ndvi_header.frac_fname(frac_id) if not io.fs_exists(fname): print 'The selected fraction does not exist in HDFS' exit(0) assert np.all(ndvi_header.timestamps_ms == qa_header.timestamps_ms) # Select dates for the requested fraction_part start_date_i = ndvi_header.frac_ndates * frac_d end_date_i = np.amin([len(ndvi_header.timestamps_ms) - start_date_i, ndvi_header.frac_ndates]) selected_dates = ndvi_header.timestamps_ms[start_date_i:end_date_i] modgrid = grids.MODISGrid()
def complete_frac(frac_num, ndvi_root, qa_root, frac_tilename, tilename_fileindex): """ Given a frac_num, will make sure it contains data for all dates in ndvi_header.timestamps_ms """ _start = time.time() modgrid = grids.MODISGrid() ndvi_header = jgrid.load(ndvi_root) qa_header = jgrid.load(qa_root) tilename = frac_tilename[frac_num] tile_h, tile_v = modis.parse_tilename(tilename) hdf_files = tilename_fileindex[tilename] d_from = 0 d_to = ndvi_header.shape[2] // ndvi_header.frac_ndates + 1 frac_id = None frac_d = None # Find the most recent existing fraction and the most recent timestamp for frac_d in range(d_from, d_to)[::-1]: frac_id = (frac_num, frac_d) fname = ndvi_header.frac_fname(frac_id) if io.fs_exists(fname): break assert frac_id is not None assert frac_d is not None # Read the data of the most recent fraction in HDFS ndvi = jgrid.read_frac(ndvi_header.frac_fname(frac_id)) qa = jgrid.read_frac(qa_header.frac_fname(frac_id)) assert ndvi.shape == qa.shape # Compute the index of the last date in HDFS most_recent_t = frac_d * ndvi_header.frac_ndates + ndvi.shape[2] i_range, j_range = modgrid.get_cell_indices_in_tile( frac_num, tile_h, tile_v) # At this point, we just have to complete with the missing dates for t in range(most_recent_t, len(ndvi_header.timestamps_ms)): ts = ndvi_header.timestamps_ms[t] fname = hdf_files[ts] new_ndvi, new_qa = read_ndvi_qa(fname, i_range, j_range) if ndvi.shape[2] == ndvi_header.frac_ndates: # Write a complete fraction frac_id = (frac_num, frac_d) ndvi_header.write_frac(frac_id, ndvi) qa_header.write_frac(frac_id, qa) # Prepare variables for a new fraction frac_d += 1 ndvi = new_ndvi[:, :, None] qa = new_qa[:, :, None] else: # TODO: If we end up completing multiple dates, we could preallocate # But for now, this is unlikely (we'll complete with the most # recent data) ndvi = np.concatenate([ndvi, new_ndvi[:, :, None]], axis=2) qa = np.concatenate([qa, new_qa[:, :, None]], axis=2) assert ndvi.shape == qa.shape # Write last incomplete fraction frac_id = (frac_num, frac_d) ndvi_header.write_frac(frac_id, ndvi) qa_header.write_frac(frac_id, qa) print 'Processed %d, appended %d dates, took %.02f [s]' % ( frac_num, len(ndvi_header.timestamps_ms) - most_recent_t, time.time() - _start) sys.stdout.flush()
def complete_frac(frac_num, ndvi_root, qa_root, frac_tilename, tilename_fileindex): """ Given a frac_num, will make sure it contains data for all dates in ndvi_header.timestamps_ms """ _start = time.time() modgrid = grids.MODISGrid() ndvi_header = jgrid.load(ndvi_root) qa_header = jgrid.load(qa_root) tilename = frac_tilename[frac_num] tile_h, tile_v = modis.parse_tilename(tilename) hdf_files = tilename_fileindex[tilename] d_from = 0 d_to = ndvi_header.shape[2] // ndvi_header.frac_ndates + 1 frac_id = None frac_d = None # Find the most recent existing fraction and the most recent timestamp for frac_d in range(d_from, d_to)[::-1]: frac_id = (frac_num, frac_d) fname = ndvi_header.frac_fname(frac_id) if io.fs_exists(fname): break assert frac_id is not None assert frac_d is not None # Read the data of the most recent fraction in HDFS ndvi = jgrid.read_frac(ndvi_header.frac_fname(frac_id)) qa = jgrid.read_frac(qa_header.frac_fname(frac_id)) assert ndvi.shape == qa.shape # Compute the index of the last date in HDFS most_recent_t = frac_d * ndvi_header.frac_ndates + ndvi.shape[2] i_range, j_range = modgrid.get_cell_indices_in_tile(frac_num, tile_h, tile_v) # At this point, we just have to complete with the missing dates for t in range(most_recent_t, len(ndvi_header.timestamps_ms)): ts = ndvi_header.timestamps_ms[t] fname = hdf_files[ts] new_ndvi, new_qa = read_ndvi_qa(fname, i_range, j_range) if ndvi.shape[2] == ndvi_header.frac_ndates: # Write a complete fraction frac_id = (frac_num, frac_d) ndvi_header.write_frac(frac_id, ndvi) qa_header.write_frac(frac_id, qa) # Prepare variables for a new fraction frac_d += 1 ndvi = new_ndvi[:, :, None] qa = new_qa[:, :, None] else: # TODO: If we end up completing multiple dates, we could preallocate # But for now, this is unlikely (we'll complete with the most # recent data) ndvi = np.concatenate([ndvi, new_ndvi[:, :, None]], axis=2) qa = np.concatenate([qa, new_qa[:, :, None]], axis=2) assert ndvi.shape == qa.shape # Write last incomplete fraction frac_id = (frac_num, frac_d) ndvi_header.write_frac(frac_id, ndvi) qa_header.write_frac(frac_id, qa) print 'Processed %d, appended %d dates, took %.02f [s]' % ( frac_num, len(ndvi_header.timestamps_ms) - most_recent_t, time.time() - _start ) sys.stdout.flush()