コード例 #1
0
def get_pvdaq_data(sysid=2,
                   api_key='DEMO_KEY',
                   year=2011,
                   delim=',',
                   standardize=True):
    """
    This fuction queries one or more years of raw PV system data from NREL's PVDAQ data service:
            https://maps.nrel.gov/pvdaq/
    """
    # Force year to be a list of integers
    ti = time()
    try:
        year = int(year)
    except TypeError:
        year = [int(yr) for yr in year]
    else:
        year = [year]
    # Each year must queries separately, so iterate over the years and generate a list of dataframes.
    df_list = []
    it = 0
    for yr in year:
        progress(it, len(year), 'querying year {}'.format(year[it]))
        req_params = {'api_key': api_key, 'system_id': sysid, 'year': yr}
        base_url = 'https://developer.nrel.gov/api/pvdaq/v3/data_file?'
        param_list = [
            str(item[0]) + '=' + str(item[1]) for item in req_params.items()
        ]
        req_url = base_url + '&'.join(param_list)
        response = requests.get(req_url)
        if int(response.status_code) != 200:
            print('\n error: ', response.status_code)
            return
        df = pd.read_csv(StringIO(response.text), delimiter=delim)
        df_list.append(df)
        it += 1
    tf = time()
    progress(it, len(year),
             'queries complete in {:.1f} seconds       '.format(tf - ti))
    # concatenate the list of yearly data frames
    df = pd.concat(df_list, axis=0, sort=True)
    if standardize:
        df = standardize_time_axis(df,
                                   datetimekey='Date-Time',
                                   timeindex=False)
    return df
コード例 #2
0
 def signal_separation(self, verbose=True):
     normed_data = self.normed_data
     W1 = self.W1
     W2 = self.W2
     ti = time()
     counter = 0
     total = normed_data.shape[1]
     signal1 = np.zeros((normed_data.shape[1], W1.shape[1]))
     signal2 = np.zeros((normed_data.shape[1], W2.shape[1]))
     for y in list(normed_data.T):
         if verbose:
             progress(counter, total)
         r1, r2 = estimate_parameters(y, mat1=W1, mat2=W2)
         signal1[counter, :] = r1
         signal2[counter, :] = r2
         counter += 1
     if verbose:
         progress(counter, total, 'total_time: {:.2f} minutes'.format(
             (time() - ti) / 60))
     self.signal1 = signal1
     self.signal2 = signal2
     self.N_ = self.signal1.shape[1]
コード例 #3
0
    def _minimize_objective(self,
                            l_cs_value,
                            r_cs_value,
                            beta_value,
                            component_r0,
                            weights,
                            mu_l=None,
                            mu_r=None,
                            tau=None,
                            exit_criterion_epsilon=1e-3,
                            max_iteration=100,
                            is_degradation_calculated=True,
                            max_degradation=None,
                            min_degradation=None,
                            non_neg_constraints=True,
                            verbose=True,
                            bootstraps=None):
        left_matrix_minimization = self._get_left_matrix_minimization(
            weights, tau, mu_l, non_neg_constraints=non_neg_constraints)
        right_matrix_minimization = self._get_right_matrix_minimization(
            weights,
            tau,
            mu_r,
            non_neg_constraints=non_neg_constraints,
            is_degradation_calculated=is_degradation_calculated,
            max_degradation=max_degradation,
            min_degradation=min_degradation)
        ti = time()
        objective_values = self._calculate_objective(mu_l,
                                                     mu_r,
                                                     tau,
                                                     l_cs_value,
                                                     r_cs_value,
                                                     beta_value,
                                                     weights,
                                                     sum_components=False)
        if verbose:
            ps = 'Starting at Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
            ps += ' f3: {:.3e}, f4: {:.3e}'
            print(
                ps.format(np.sum(objective_values), objective_values[0],
                          objective_values[1], objective_values[2],
                          objective_values[3]))
        improvement = np.inf
        old_objective_value = np.sum(objective_values)
        iteration = 0
        f1_last = objective_values[0]

        tol_schedule = []  #np.logspace(-4, -8, 6)

        while improvement >= exit_criterion_epsilon:
            try:
                tol = tol_schedule[iteration]
            except IndexError:
                tol = 1e-8

            self._store_minimization_state_data(mu_l, mu_r, tau, l_cs_value,
                                                r_cs_value, beta_value,
                                                component_r0)

            try:
                if self.__left_first:
                    if verbose:
                        print('    Minimizing left matrix')
                    l_cs_value, r_cs_value, beta_value\
                        = left_matrix_minimization.minimize(
                            l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
                    if verbose:
                        print('    Minimizing right matrix')
                    l_cs_value, r_cs_value, beta_value\
                        = right_matrix_minimization.minimize(
                            l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
                else:
                    if verbose:
                        print('    Minimizing right matrix')
                    l_cs_value, r_cs_value, beta_value\
                        = right_matrix_minimization.minimize(
                            l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
                    if verbose:
                        print('    Minimizing left matrix')
                    l_cs_value, r_cs_value, beta_value\
                        = left_matrix_minimization.minimize(
                            l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
            except cvx.SolverError:
                if self.__left_first:
                    if verbose:
                        print(
                            'Solver failed! Starting over and reversing minimization order.'
                        )
                    self.__left_first = False
                    iteration = 0
                    l_cs_value = self._decomposition.matrix_l0
                    r_cs_value = self._decomposition.matrix_r0
                    component_r0 = self._obtain_initial_component_r0(
                        verbose=verbose)
                    continue
                else:
                    if verbose:
                        print('Solver failing again! Exiting...')
                    self._state_data.is_solver_error = True
                    break
            except ProblemStatusError as e:
                if verbose:
                    print(e)
                if self.__left_first:
                    if verbose:
                        print(
                            'Starting over and reversing minimization order.')
                    self.__left_first = False
                    iteration = 0
                    l_cs_value = self._decomposition.matrix_l0
                    r_cs_value = self._decomposition.matrix_r0
                    component_r0 = self._obtain_initial_component_r0(
                        verbose=verbose)
                    continue
                else:
                    if verbose:
                        print('Exiting...')
                    self._state_data.is_problem_status_error = True
                    break

            component_r0 = r_cs_value[0, :]

            objective_values = self._calculate_objective(mu_l,
                                                         mu_r,
                                                         tau,
                                                         l_cs_value,
                                                         r_cs_value,
                                                         beta_value,
                                                         weights,
                                                         sum_components=False)
            new_objective_value = np.sum(objective_values)
            improvement = ((old_objective_value - new_objective_value) * 1. /
                           old_objective_value)
            old_objective_value = new_objective_value
            iteration += 1
            if verbose:
                ps = '{} - Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
                ps += ' f3: {:.3e}, f4: {:.3e}'
                print(
                    ps.format(iteration, new_objective_value,
                              objective_values[0], objective_values[1],
                              objective_values[2], objective_values[3]))
            if objective_values[0] > f1_last:
                self._state_data.f1_increase = True
                if verbose:
                    print('Caution: residuals increased')
            if improvement < 0:
                if verbose:
                    print('Caution: objective increased.')
                self._state_data.obj_increase = True
                improvement *= -1
            if objective_values[3] > 1e2:
                if self.__left_first:
                    if verbose:
                        print(
                            'Bad trajectory detected. Starting over and reversing minimization order.'
                        )
                    self.__left_first = False
                    iteration = 0
                    l_cs_value = self._decomposition.matrix_l0
                    r_cs_value = self._decomposition.matrix_r0
                    component_r0 = self._obtain_initial_component_r0(
                        verbose=verbose)
                else:
                    if verbose:
                        print('Algorithm Failed!')
                    improvement = 0
            if iteration >= max_iteration:
                if verbose:
                    print(
                        'Reached iteration limit. Previous improvement: {:.2f}%'
                        .format(improvement * 100))
                improvement = 0.

            self._store_minimization_state_data(mu_l, mu_r, tau, l_cs_value,
                                                r_cs_value, beta_value,
                                                component_r0)

        # except cvx.SolverError:
        #     if self.__left_first:
        #         if verbose:
        #             print('solver failed! Starting over and reversing minimization order.')
        #
        #     self._state_data.is_solver_error = True
        # except ProblemStatusError as e:
        #     if verbose:
        #         print(e)
        #     self._state_data.is_problem_status_error = True

        tf = time()
        if verbose:
            print('Minimization complete in {:.2f} minutes'.format(
                (tf - ti) / 60.))
        self._analyze_residuals(l_cs_value, r_cs_value, weights)
        self._keep_result_variables_as_properties(l_cs_value, r_cs_value,
                                                  beta_value)
        if bootstraps is not None:
            if verbose:
                print('Running bootstrap analysis...')
            ti = time()
            self._bootstrap_samples = defaultdict(dict)
            for ix in range(bootstraps):
                # resample the days with non-zero weights only
                bootstrap_weights = resample_index(length=np.sum(
                    weights > 1e-1))
                new_weights = np.zeros_like(weights)
                new_weights[weights > 1e-1] = bootstrap_weights
                new_weights = np.multiply(weights, new_weights)
                left_matrix_minimization.update_weights(new_weights)
                right_matrix_minimization.update_weights(new_weights)
                l_cs_value = self._l_cs_value
                r_cs_value = self._r_cs_value
                beta_value = self._beta_value
                # ti = time()
                objective_values = self._calculate_objective(
                    mu_l,
                    mu_r,
                    tau,
                    l_cs_value,
                    r_cs_value,
                    beta_value,
                    new_weights,
                    sum_components=False)
                if verbose:
                    progress(ix,
                             bootstraps,
                             status=' {:.2f} minutes'.format(
                                 (time() - ti) / 60))
                    # ps = 'Bootstrap Sample {}\n'.format(ix)
                    # ps += 'Starting at Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
                    # ps += ' f3: {:.3e}, f4: {:.3e}'
                    # print(ps.format(
                    #     np.sum(objective_values), objective_values[0],
                    #     objective_values[1], objective_values[2],
                    #     objective_values[3]
                    # ))
                improvement = np.inf
                old_objective_value = np.sum(objective_values)
                iteration = 0
                f1_last = objective_values[0]

                tol_schedule = []  # np.logspace(-4, -8, 6)

                while improvement >= exit_criterion_epsilon:
                    try:
                        tol = tol_schedule[iteration]
                    except IndexError:
                        tol = 1e-8

                    # self._store_minimization_state_data(mu_l, mu_r, tau,
                    #                                     l_cs_value, r_cs_value,
                    #                                     beta_value,
                    #                                     component_r0)

                    try:
                        if self.__left_first:
                            # if verbose:
                            # print('    Minimizing left matrix')
                            l_cs_value, r_cs_value, beta_value \
                                = left_matrix_minimization.minimize(
                                l_cs_value, r_cs_value, beta_value,
                                component_r0, tol=tol)
                            # if verbose:
                            # print('    Minimizing right matrix')
                            l_cs_value, r_cs_value, beta_value \
                                = right_matrix_minimization.minimize(
                                l_cs_value, r_cs_value, beta_value,
                                component_r0, tol=tol)
                        else:
                            # if verbose:
                            # print('    Minimizing right matrix')
                            l_cs_value, r_cs_value, beta_value \
                                = right_matrix_minimization.minimize(
                                l_cs_value, r_cs_value, beta_value,
                                component_r0, tol=tol)
                            # if verbose:
                            # print('    Minimizing left matrix')
                            l_cs_value, r_cs_value, beta_value \
                                = left_matrix_minimization.minimize(
                                l_cs_value, r_cs_value, beta_value,
                                component_r0, tol=tol)
                    except cvx.SolverError:
                        if self.__left_first:
                            if verbose:
                                print(
                                    'Solver failed! Starting over and reversing minimization order.'
                                )
                            self.__left_first = False
                            iteration = 0
                            l_cs_value = self._decomposition.matrix_l0
                            r_cs_value = self._decomposition.matrix_r0
                            component_r0 = self._obtain_initial_component_r0(
                                verbose=verbose)
                            continue
                        else:
                            if verbose:
                                print('Solver failing again! Exiting...')
                            self._state_data.is_solver_error = True
                            break
                    except ProblemStatusError as e:
                        if verbose:
                            print(e)
                        if self.__left_first:
                            if verbose:
                                print(
                                    'Starting over and reversing minimization order.'
                                )
                            self.__left_first = False
                            iteration = 0
                            l_cs_value = self._decomposition.matrix_l0
                            r_cs_value = self._decomposition.matrix_r0
                            component_r0 = self._obtain_initial_component_r0(
                                verbose=verbose)
                            continue
                        else:
                            if verbose:
                                print('Exiting...')
                            self._state_data.is_problem_status_error = True
                            break

                    component_r0 = r_cs_value[0, :]

                    objective_values = self._calculate_objective(
                        mu_l,
                        mu_r,
                        tau,
                        l_cs_value,
                        r_cs_value,
                        beta_value,
                        new_weights,
                        sum_components=False)
                    new_objective_value = np.sum(objective_values)
                    improvement = (
                        (old_objective_value - new_objective_value) * 1. /
                        old_objective_value)
                    old_objective_value = new_objective_value
                    iteration += 1
                    # if verbose:
                    # ps = '{} - Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
                    # ps += ' f3: {:.3e}, f4: {:.3e}'
                    # print(ps.format(
                    #     iteration, new_objective_value,
                    #     objective_values[0],
                    #     objective_values[1], objective_values[2],
                    #     objective_values[3]
                    # ))
                    if objective_values[0] > f1_last:
                        self._state_data.f1_increase = True
                        if verbose:
                            print('Caution: residuals increased')
                    if improvement < 0:
                        if verbose:
                            print('Caution: objective increased.')
                        self._state_data.obj_increase = True
                        improvement *= -1
                    if objective_values[3] > 1e2:
                        if self.__left_first:
                            if verbose:
                                print(
                                    'Bad trajectory detected. Starting over and reversing minimization order.'
                                )
                            self.__left_first = False
                            iteration = 0
                            l_cs_value = self._decomposition.matrix_l0
                            r_cs_value = self._decomposition.matrix_r0
                            component_r0 = self._obtain_initial_component_r0(
                                verbose=verbose)
                        else:
                            if verbose:
                                print('Algorithm Failed!')
                            improvement = 0
                    if iteration >= max_iteration:
                        if verbose:
                            print(
                                'Reached iteration limit. Previous improvement: {:.2f}%'
                                .format(improvement * 100))
                        improvement = 0.
                # tf = time()
                # if verbose:
                #     print('Bootstrap {} complete in {:.2f} minutes'.format(
                #           ix, (tf - ti) / 60.))
                self._bootstrap_samples[ix]['L'] = l_cs_value
                self._bootstrap_samples[ix]['R'] = r_cs_value
                self._bootstrap_samples[ix]['beta'] = beta_value
            if verbose:
                progress(bootstraps,
                         bootstraps,
                         status=' {:.2f} minutes'.format((time() - ti) / 60))