Exemple #1
0
                                                                                     t12_K,
                                                                                     t37_K,
                                                                                     t_clim_K,
                                                                                     sun_zenith_angle,
                                                                                     sat_zenith_angle)

                            if np.isnan(st_truth_K):
                                # No need to do more for this pixel, if the output is not a number.
                                continue

                            swath_input_id = db.insert_swath_values(
                                str(avhrr_model.satellite_id),
                                surface_temp=st_truth_K, # float(true_st_K),
                                t_11=float(t11_K),
                                t_12=float(t12_K),
                                sat_zenith_angle=sat_zenith_angle,
                                sun_zenith_angle=sun_zenith_angle,
                                cloudmask=int(avhrr_model.cloudmask[row_index, col_index]),
                                swath_datetime=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                                lat=float(lat),
                                lon=float(lon)
                                )
                            
             
                            # Do the perturbations...
                            for i in range(int(args["--number-of-perturbations"])):
                                perturbed_t11_K = random.gauss(t11_K, sigma_11)
                                perturbed_t12_K = random.gauss(t12_K, sigma_12)
                                perturbed_t37_K = random.gauss(t37_K, sigma_37) \
                                    if t37_K is None or np.isnan(t37_K) else np.NaN

                                # Missing climatology
def populate_from_files(database_filename,
                        avhrr_filename,
                        sun_sat_angle_filename,
                        cloudmask_filename,
                        sea_ice_fraction_data_directory,
                        number_of_perturbations,
                        run_in_parallel=False):
    """
    Populate the database with perturbed values.
    """
    LOG.info("db_filename:                      %s" % (database_filename))
    LOG.info("avhrr_filename:                   %s" % (avhrr_filename))
    LOG.info("sunsatangle_filename:             %s" % (sunsatangle_filename))
    LOG.info("cloudmask_filename:               %s" % (cloudmask_filename))
    LOG.info("sea_ice_fraction_data_directory:  %s" %
             (sea_ice_fraction_data_directory))

    # Reading in the input file.
    # The file is cached, so that when the values are read, they are read
    # from memory, and not from the file system. This speeds up the
    # calculations.
    with models.avhrr_hdf5.Hdf5(avhrr_filename, sun_sat_angle_filename,
                                cloudmask_filename) as avhrr_model:
        LOG.info(avhrr_model)
        assert (avhrr_model.lat.shape == avhrr_model.lon.shape)

        # Get the sigma values based on the satellite id.
        sigmas = eustace.sigmas.get_sigmas(avhrr_model.satellite_id)
        LOG.info(sigmas)

        sea_ice_fractions = get_sea_ice_fractions(
            sea_ice_fraction_data_directory, avhrr_filename)
        if sea_ice_fractions is not None:
            assert (avhrr_model.lat.shape == sea_ice_fractions.shape)

        # Some book keeping...
        total_perturbed_st_count = 0
        counter = 0

        # Set the random seed, so that the results are the same
        # the next time the exact same system is is run.
        random.seed(1)

        output_queue = mp.Queue()
        number_of_cpus = mp.cpu_count()
        number_of_processes_started = 0
        number_of_processes_finished = 0

        # Book keeping.
        start_time = datetime.datetime.now()

        # Using the coefficients based on the satellite id.
        with eustace.coefficients.Coefficients(
                avhrr_model.satellite_id) as coeff:
            ## Using a ram disk speeds up the calculations, quite a lot.
            ## Creating ramdisk:
            # mkdir /tmp/ramdisk
            #
            ## A 3Gb ram disk.
            # mount -t tmpfs -o size=3072m tmpfs /tmp/ramdisk
            #
            ## or
            #
            ## For a 12Gb ram disk.
            # mount -t tmpfs -o size=$((12 * 1024))m tmpfs /tmp/ramdisk
            #
            #
            ## Defining the database.
            with eustace.db.Db(database_filename) as db:
                # Rows.
                for row_index in np.arange(avhrr_model.lon.shape[0]):
                    # Some diagnostics while running.
                    LOG.info(
                        "ROW: %i.   total st_count: %i.   total_time: %s.   sts./sec: %f"
                        % (row_index, total_perturbed_st_count,
                           str(datetime.datetime.now() - start_time),
                           (total_perturbed_st_count /
                            (datetime.datetime.now() -
                             start_time).total_seconds())))

                    # Cols.
                    for col_index in np.arange(avhrr_model.lon.shape[1]):
                        counter += 1

                        # Reading in the values.
                        cloudmask = avhrr_model.cloudmask[row_index, col_index]
                        if cloudmask != 1 and cloudmask != 4:
                            LOG.debug("Bad cloudmask: %i" % (cloudmask))
                            continue

                        # T11 is channel 4.
                        t11_K = avhrr_model.ch4[row_index, col_index]

                        # T12 is channel 5.
                        t12_K = avhrr_model.ch5[row_index, col_index]

                        # T37 is channel 3b.
                        t37_K = avhrr_model.ch3b[row_index, col_index]

                        if np.isnan(t11_K) or np.isnan(t12_K):
                            # t11 and t12 are both needed for all calculations.
                            # Is something wrong if they are both missing?
                            # Consider what to do.
                            raise RuntimeException("Missing T11 or T12")

                        # Angles.
                        sun_zenith_angle = float(
                            avhrr_model.sun_zenith_angle[row_index, col_index])
                        sat_zenith_angle = float(
                            avhrr_model.sat_zenith_angle[row_index, col_index])

                        # Missing climatology. Using t11_K in stead.
                        t_clim_K = t11_K

                        # Lat / lon.
                        lat = avhrr_model.lat[row_index, col_index]
                        lon = avhrr_model.lon[row_index, col_index]
                        if lat is None or np.isnan(
                                lat) or lon is None or np.isnan(lon):
                            continue

                        # Pick algorithm.
                        algorithm = eustace.surface_temperature.select_surface_temperature_algorithm(
                            sun_zenith_angle, t11_K, t37_K)

                        # Calculate the temperature.
                        st_truth_K = eustace.surface_temperature.get_surface_temperature(
                            algorithm, coeff, t11_K, t12_K, t37_K, t_clim_K,
                            sun_zenith_angle, sat_zenith_angle)

                        if np.isnan(st_truth_K):
                            # No need to do more for this pixel, if the output is not a number.
                            continue

                        if sea_ice_fractions is not None:
                            if sea_ice_fractions[row_index][
                                    col_index] is None or np.isnan(
                                        sea_ice_fractions[row_index]
                                        [col_index]):
                                sea_ice_fraction = None
                            else:
                                sea_ice_fraction = float(
                                    sea_ice_fractions[row_index][col_index])
                        else:
                            sea_ice_fraction = None

                        swath_input_id = db.insert_swath_values(
                            str(avhrr_model.satellite_id),
                            surface_temp=st_truth_K,  # float(true_st_K),
                            t_11=float(t11_K),
                            t_12=float(t12_K),
                            sat_zenith_angle=sat_zenith_angle,
                            sun_zenith_angle=sun_zenith_angle,
                            cloudmask=int(avhrr_model.cloudmask[row_index,
                                                                col_index]),
                            swath_datetime=avhrr_model.swath_datetime,
                            lat=float(lat),
                            lon=float(lon),
                            sea_ice_fraction=sea_ice_fraction)

                        if not run_in_parallel:
                            # WARNING!
                            # If the number of perturbations is a small number, it is much faster
                            # to run sequencially!!
                            perturbations = eustace.surface_temperature.get_n_perturbed_temeratures(
                                coeff,
                                number_of_perturbations,
                                t11_K,
                                t12_K,
                                t37_K,
                                t_clim_K,
                                sigmas["sigma_11"],
                                sigmas["sigma_12"],
                                sigmas["sigma_37"],
                                sun_zenith_angle,
                                sat_zenith_angle,
                                random_seed=counter)
                            num_inserted = db.insert_many_perturbations(
                                swath_input_id, perturbations)
                            total_perturbed_st_count += num_inserted

                        else:
                            # This starts a process running a number of perturbations
                            # and inserts the result in the in the output queue.
                            perturbate_in_parallel(output_queue,
                                                   swath_input_id,
                                                   coeff,
                                                   number_of_perturbations,
                                                   t11_K,
                                                   t12_K,
                                                   t37_K,
                                                   t_clim_K,
                                                   sigmas["sigma_11"],
                                                   sigmas["sigma_12"],
                                                   sigmas["sigma_37"],
                                                   sun_zenith_angle,
                                                   sat_zenith_angle,
                                                   random_seed=counter)
                            number_of_processes_started += 1

                            if number_of_processes_started > number_of_cpus:
                                # Get will wait forever, for the process to finish.
                                swath_input_id, perturbations = output_queue.get(
                                )
                                number_of_processes_finished += 1
                                num_inserted = db.insert_many_perturbations(
                                    swath_input_id, perturbations)
                                total_perturbed_st_count += num_inserted

                if run_in_parallel:
                    while number_of_processes_started > number_of_processes_finished:
                        swath_input_id, perturbations = output_queue.get()
                        number_of_processes_finished += 1
                        db.insert_many_perturbations(swath_input_id,
                                                     perturbations)

                # FIN.
                LOG.info("Finished perturbing '%s'." %
                         (avhrr_model.avhrr_filename))
def populate_from_files(database_filename, avhrr_filename, sun_sat_angle_filename,
                        cloudmask_filename, sea_ice_fraction_data_directory,
                        number_of_perturbations, run_in_parallel = False
                        ):
    """
    Populate the database with perturbed values.
    """
    LOG.info("db_filename:                      %s" % (database_filename))
    LOG.info("avhrr_filename:                   %s" % (avhrr_filename))
    LOG.info("sunsatangle_filename:             %s" % (sunsatangle_filename))
    LOG.info("cloudmask_filename:               %s" % (cloudmask_filename))
    LOG.info("sea_ice_fraction_data_directory:  %s" % (sea_ice_fraction_data_directory))

    # Reading in the input file.
    # The file is cached, so that when the values are read, they are read
    # from memory, and not from the file system. This speeds up the
    # calculations.
    with models.avhrr_hdf5.Hdf5(avhrr_filename,
                                sun_sat_angle_filename,
                                cloudmask_filename) as avhrr_model:
        LOG.info(avhrr_model)
        assert(avhrr_model.lat.shape == avhrr_model.lon.shape)
        
        # Get the sigma values based on the satellite id.
        sigmas = eustace.sigmas.get_sigmas(avhrr_model.satellite_id)
        LOG.info(sigmas)

        sea_ice_fractions = get_sea_ice_fractions(sea_ice_fraction_data_directory, avhrr_filename)
        if sea_ice_fractions is not None:
            assert(avhrr_model.lat.shape == sea_ice_fractions.shape)

        # Some book keeping...
        total_perturbed_st_count = 0
        counter = 0
        
        # Set the random seed, so that the results are the same
        # the next time the exact same system is is run.
        random.seed(1)

        output_queue = mp.Queue()
        number_of_cpus = mp.cpu_count()
        number_of_processes_started = 0
        number_of_processes_finished = 0

        # Book keeping.
        start_time = datetime.datetime.now()

        # Using the coefficients based on the satellite id.
        with eustace.coefficients.Coefficients(avhrr_model.satellite_id) as coeff:
            ## Using a ram disk speeds up the calculations, quite a lot.
            ## Creating ramdisk:
            # mkdir /tmp/ramdisk
            #
            ## A 3Gb ram disk.
            # mount -t tmpfs -o size=3072m tmpfs /tmp/ramdisk
            #
            ## or
            #
            ## For a 12Gb ram disk.
            # mount -t tmpfs -o size=$((12 * 1024))m tmpfs /tmp/ramdisk
            #
            #
            ## Defining the database.
            with eustace.db.Db(database_filename) as db:
                # Rows.
                for row_index in np.arange(avhrr_model.lon.shape[0]):
                    # Some diagnostics while running.
                    LOG.info("ROW: %i.   total st_count: %i.   total_time: %s.   sts./sec: %f" %
                             (row_index, total_perturbed_st_count,
                              str(datetime.datetime.now() - start_time),
                              (total_perturbed_st_count / (datetime.datetime.now() -
                                                           start_time).total_seconds())))

                    # Cols.
                    for col_index in np.arange(avhrr_model.lon.shape[1]):
                        counter += 1

                        # Reading in the values.
                        cloudmask = avhrr_model.cloudmask[row_index, col_index] 
                        if cloudmask != 1 and cloudmask != 4:
                            LOG.debug("Bad cloudmask: %i" % (cloudmask))
                            continue

                        # T11 is channel 4.
                        t11_K = avhrr_model.ch4[row_index, col_index]

                        # T12 is channel 5.
                        t12_K = avhrr_model.ch5[row_index, col_index]

                        # T37 is channel 3b.
                        t37_K = avhrr_model.ch3b[row_index, col_index]

                        if np.isnan(t11_K) or np.isnan(t12_K):
                            # t11 and t12 are both needed for all calculations.
                            # Is something wrong if they are both missing?
                            # Consider what to do.
                            raise RuntimeException("Missing T11 or T12")

                        # Angles.
                        sun_zenith_angle = float(avhrr_model.sun_zenith_angle[row_index, col_index])
                        sat_zenith_angle = float(avhrr_model.sat_zenith_angle[row_index, col_index])

                        # Missing climatology. Using t11_K in stead.
                        t_clim_K = t11_K

                        # Lat / lon.
                        lat = avhrr_model.lat[row_index, col_index]
                        lon = avhrr_model.lon[row_index, col_index]
                        if lat is None or np.isnan(lat) or lon is None or np.isnan(lon):
                            continue

                        # Pick algorithm.
                        algorithm = eustace.surface_temperature.select_surface_temperature_algorithm(
                            sun_zenith_angle,
                            t11_K,
                            t37_K)

                        # Calculate the temperature.
                        st_truth_K = eustace.surface_temperature.get_surface_temperature(algorithm,
                                                                                         coeff,
                                                                                         t11_K,
                                                                                         t12_K,
                                                                                         t37_K,
                                                                                         t_clim_K,
                                                                                         sun_zenith_angle,
                                                                                         sat_zenith_angle)

                        if np.isnan(st_truth_K):
                            # No need to do more for this pixel, if the output is not a number.
                            continue

                        if sea_ice_fractions is not None:
                            if sea_ice_fractions[row_index][col_index] is None or np.isnan(sea_ice_fractions[row_index][col_index]):
                                sea_ice_fraction = None
                            else:
                                sea_ice_fraction = float(sea_ice_fractions[row_index][col_index])
                        else:
                            sea_ice_fraction = None


                        swath_input_id = db.insert_swath_values(
                            str(avhrr_model.satellite_id),
                            surface_temp=st_truth_K, # float(true_st_K),
                            t_11=float(t11_K),
                            t_12=float(t12_K),
                            sat_zenith_angle=sat_zenith_angle,
                            sun_zenith_angle=sun_zenith_angle,
                            cloudmask=int(avhrr_model.cloudmask[row_index, col_index]),
                            swath_datetime=avhrr_model.swath_datetime,
                            lat=float(lat),
                            lon=float(lon),
                            sea_ice_fraction=sea_ice_fraction
                            )


                        if not run_in_parallel:
                            # WARNING!
                            # If the number of perturbations is a small number, it is much faster
                            # to run sequencially!!
                            perturbations = eustace.surface_temperature.get_n_perturbed_temeratures(coeff,
                                                                                                    number_of_perturbations,
                                                                                                    t11_K,
                                                                                                    t12_K,
                                                                                                    t37_K,
                                                                                                    t_clim_K,
                                                                                                    sigmas["sigma_11"],
                                                                                                    sigmas["sigma_12"],
                                                                                                    sigmas["sigma_37"],
                                                                                                    sun_zenith_angle,
                                                                                                    sat_zenith_angle,
                                                                                                    random_seed=counter)
                            num_inserted = db.insert_many_perturbations(swath_input_id, perturbations)
                            total_perturbed_st_count += num_inserted

                        else:
                            # This starts a process running a number of perturbations
                            # and inserts the result in the in the output queue.
                            perturbate_in_parallel(output_queue,
                                                   swath_input_id,
                                                   coeff,
                                                   number_of_perturbations,
                                                   t11_K,
                                                   t12_K,
                                                   t37_K,
                                                   t_clim_K,
                                                   sigmas["sigma_11"],
                                                   sigmas["sigma_12"],
                                                   sigmas["sigma_37"],
                                                   sun_zenith_angle,
                                                   sat_zenith_angle,
                                                   random_seed=counter)
                            number_of_processes_started += 1

                            if number_of_processes_started > number_of_cpus:
                                # Get will wait forever, for the process to finish.
                                swath_input_id, perturbations = output_queue.get()
                                number_of_processes_finished += 1
                                num_inserted = db.insert_many_perturbations(swath_input_id, perturbations)
                                total_perturbed_st_count += num_inserted


                if run_in_parallel:
                    while number_of_processes_started > number_of_processes_finished:
                        swath_input_id, perturbations = output_queue.get()
                        number_of_processes_finished += 1
                        db.insert_many_perturbations(swath_input_id, perturbations)

                # FIN.
                LOG.info("Finished perturbing '%s'." % (avhrr_model.avhrr_filename))