def get_pvdaq_data(sysid=2,
api_key='DEMO_KEY',
year=2011,
delim=',',
standardize=True):
"""
This fuction queries one or more years of raw PV system data from NREL's PVDAQ data service:
https://maps.nrel.gov/pvdaq/
"""
# Force year to be a list of integers
ti = time()
try:
year = int(year)
except TypeError:
year = [int(yr) for yr in year]
else:
year = [year]
# Each year must queries separately, so iterate over the years and generate a list of dataframes.
df_list = []
it = 0
for yr in year:
progress(it, len(year), 'querying year {}'.format(year[it]))
req_params = {'api_key': api_key, 'system_id': sysid, 'year': yr}
base_url = 'https://developer.nrel.gov/api/pvdaq/v3/data_file?'
param_list = [
str(item[0]) + '=' + str(item[1]) for item in req_params.items()
]
req_url = base_url + '&'.join(param_list)
response = requests.get(req_url)
if int(response.status_code) != 200:
print('\n error: ', response.status_code)
return
df = pd.read_csv(StringIO(response.text), delimiter=delim)
df_list.append(df)
it += 1
tf = time()
progress(it, len(year),
'queries complete in {:.1f} seconds '.format(tf - ti))
# concatenate the list of yearly data frames
df = pd.concat(df_list, axis=0, sort=True)
if standardize:
df = standardize_time_axis(df,
datetimekey='Date-Time',
timeindex=False)
return df
def signal_separation(self, verbose=True):
normed_data = self.normed_data
W1 = self.W1
W2 = self.W2
ti = time()
counter = 0
total = normed_data.shape[1]
signal1 = np.zeros((normed_data.shape[1], W1.shape[1]))
signal2 = np.zeros((normed_data.shape[1], W2.shape[1]))
for y in list(normed_data.T):
if verbose:
progress(counter, total)
r1, r2 = estimate_parameters(y, mat1=W1, mat2=W2)
signal1[counter, :] = r1
signal2[counter, :] = r2
counter += 1
if verbose:
progress(counter, total, 'total_time: {:.2f} minutes'.format(
(time() - ti) / 60))
self.signal1 = signal1
self.signal2 = signal2
self.N_ = self.signal1.shape[1]
def _minimize_objective(self,
l_cs_value,
r_cs_value,
beta_value,
component_r0,
weights,
mu_l=None,
mu_r=None,
tau=None,
exit_criterion_epsilon=1e-3,
max_iteration=100,
is_degradation_calculated=True,
max_degradation=None,
min_degradation=None,
non_neg_constraints=True,
verbose=True,
bootstraps=None):
left_matrix_minimization = self._get_left_matrix_minimization(
weights, tau, mu_l, non_neg_constraints=non_neg_constraints)
right_matrix_minimization = self._get_right_matrix_minimization(
weights,
tau,
mu_r,
non_neg_constraints=non_neg_constraints,
is_degradation_calculated=is_degradation_calculated,
max_degradation=max_degradation,
min_degradation=min_degradation)
ti = time()
objective_values = self._calculate_objective(mu_l,
mu_r,
tau,
l_cs_value,
r_cs_value,
beta_value,
weights,
sum_components=False)
if verbose:
ps = 'Starting at Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
ps += ' f3: {:.3e}, f4: {:.3e}'
print(
ps.format(np.sum(objective_values), objective_values[0],
objective_values[1], objective_values[2],
objective_values[3]))
improvement = np.inf
old_objective_value = np.sum(objective_values)
iteration = 0
f1_last = objective_values[0]
tol_schedule = [] #np.logspace(-4, -8, 6)
while improvement >= exit_criterion_epsilon:
try:
tol = tol_schedule[iteration]
except IndexError:
tol = 1e-8
self._store_minimization_state_data(mu_l, mu_r, tau, l_cs_value,
r_cs_value, beta_value,
component_r0)
try:
if self.__left_first:
if verbose:
print(' Minimizing left matrix')
l_cs_value, r_cs_value, beta_value\
= left_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
if verbose:
print(' Minimizing right matrix')
l_cs_value, r_cs_value, beta_value\
= right_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
else:
if verbose:
print(' Minimizing right matrix')
l_cs_value, r_cs_value, beta_value\
= right_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
if verbose:
print(' Minimizing left matrix')
l_cs_value, r_cs_value, beta_value\
= left_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value, component_r0, tol=tol)
except cvx.SolverError:
if self.__left_first:
if verbose:
print(
'Solver failed! Starting over and reversing minimization order.'
)
self.__left_first = False
iteration = 0
l_cs_value = self._decomposition.matrix_l0
r_cs_value = self._decomposition.matrix_r0
component_r0 = self._obtain_initial_component_r0(
verbose=verbose)
continue
else:
if verbose:
print('Solver failing again! Exiting...')
self._state_data.is_solver_error = True
break
except ProblemStatusError as e:
if verbose:
print(e)
if self.__left_first:
if verbose:
print(
'Starting over and reversing minimization order.')
self.__left_first = False
iteration = 0
l_cs_value = self._decomposition.matrix_l0
r_cs_value = self._decomposition.matrix_r0
component_r0 = self._obtain_initial_component_r0(
verbose=verbose)
continue
else:
if verbose:
print('Exiting...')
self._state_data.is_problem_status_error = True
break
component_r0 = r_cs_value[0, :]
objective_values = self._calculate_objective(mu_l,
mu_r,
tau,
l_cs_value,
r_cs_value,
beta_value,
weights,
sum_components=False)
new_objective_value = np.sum(objective_values)
improvement = ((old_objective_value - new_objective_value) * 1. /
old_objective_value)
old_objective_value = new_objective_value
iteration += 1
if verbose:
ps = '{} - Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
ps += ' f3: {:.3e}, f4: {:.3e}'
print(
ps.format(iteration, new_objective_value,
objective_values[0], objective_values[1],
objective_values[2], objective_values[3]))
if objective_values[0] > f1_last:
self._state_data.f1_increase = True
if verbose:
print('Caution: residuals increased')
if improvement < 0:
if verbose:
print('Caution: objective increased.')
self._state_data.obj_increase = True
improvement *= -1
if objective_values[3] > 1e2:
if self.__left_first:
if verbose:
print(
'Bad trajectory detected. Starting over and reversing minimization order.'
)
self.__left_first = False
iteration = 0
l_cs_value = self._decomposition.matrix_l0
r_cs_value = self._decomposition.matrix_r0
component_r0 = self._obtain_initial_component_r0(
verbose=verbose)
else:
if verbose:
print('Algorithm Failed!')
improvement = 0
if iteration >= max_iteration:
if verbose:
print(
'Reached iteration limit. Previous improvement: {:.2f}%'
.format(improvement * 100))
improvement = 0.
self._store_minimization_state_data(mu_l, mu_r, tau, l_cs_value,
r_cs_value, beta_value,
component_r0)
# except cvx.SolverError:
# if self.__left_first:
# if verbose:
# print('solver failed! Starting over and reversing minimization order.')
#
# self._state_data.is_solver_error = True
# except ProblemStatusError as e:
# if verbose:
# print(e)
# self._state_data.is_problem_status_error = True
tf = time()
if verbose:
print('Minimization complete in {:.2f} minutes'.format(
(tf - ti) / 60.))
self._analyze_residuals(l_cs_value, r_cs_value, weights)
self._keep_result_variables_as_properties(l_cs_value, r_cs_value,
beta_value)
if bootstraps is not None:
if verbose:
print('Running bootstrap analysis...')
ti = time()
self._bootstrap_samples = defaultdict(dict)
for ix in range(bootstraps):
# resample the days with non-zero weights only
bootstrap_weights = resample_index(length=np.sum(
weights > 1e-1))
new_weights = np.zeros_like(weights)
new_weights[weights > 1e-1] = bootstrap_weights
new_weights = np.multiply(weights, new_weights)
left_matrix_minimization.update_weights(new_weights)
right_matrix_minimization.update_weights(new_weights)
l_cs_value = self._l_cs_value
r_cs_value = self._r_cs_value
beta_value = self._beta_value
# ti = time()
objective_values = self._calculate_objective(
mu_l,
mu_r,
tau,
l_cs_value,
r_cs_value,
beta_value,
new_weights,
sum_components=False)
if verbose:
progress(ix,
bootstraps,
status=' {:.2f} minutes'.format(
(time() - ti) / 60))
# ps = 'Bootstrap Sample {}\n'.format(ix)
# ps += 'Starting at Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
# ps += ' f3: {:.3e}, f4: {:.3e}'
# print(ps.format(
# np.sum(objective_values), objective_values[0],
# objective_values[1], objective_values[2],
# objective_values[3]
# ))
improvement = np.inf
old_objective_value = np.sum(objective_values)
iteration = 0
f1_last = objective_values[0]
tol_schedule = [] # np.logspace(-4, -8, 6)
while improvement >= exit_criterion_epsilon:
try:
tol = tol_schedule[iteration]
except IndexError:
tol = 1e-8
# self._store_minimization_state_data(mu_l, mu_r, tau,
# l_cs_value, r_cs_value,
# beta_value,
# component_r0)
try:
if self.__left_first:
# if verbose:
# print(' Minimizing left matrix')
l_cs_value, r_cs_value, beta_value \
= left_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value,
component_r0, tol=tol)
# if verbose:
# print(' Minimizing right matrix')
l_cs_value, r_cs_value, beta_value \
= right_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value,
component_r0, tol=tol)
else:
# if verbose:
# print(' Minimizing right matrix')
l_cs_value, r_cs_value, beta_value \
= right_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value,
component_r0, tol=tol)
# if verbose:
# print(' Minimizing left matrix')
l_cs_value, r_cs_value, beta_value \
= left_matrix_minimization.minimize(
l_cs_value, r_cs_value, beta_value,
component_r0, tol=tol)
except cvx.SolverError:
if self.__left_first:
if verbose:
print(
'Solver failed! Starting over and reversing minimization order.'
)
self.__left_first = False
iteration = 0
l_cs_value = self._decomposition.matrix_l0
r_cs_value = self._decomposition.matrix_r0
component_r0 = self._obtain_initial_component_r0(
verbose=verbose)
continue
else:
if verbose:
print('Solver failing again! Exiting...')
self._state_data.is_solver_error = True
break
except ProblemStatusError as e:
if verbose:
print(e)
if self.__left_first:
if verbose:
print(
'Starting over and reversing minimization order.'
)
self.__left_first = False
iteration = 0
l_cs_value = self._decomposition.matrix_l0
r_cs_value = self._decomposition.matrix_r0
component_r0 = self._obtain_initial_component_r0(
verbose=verbose)
continue
else:
if verbose:
print('Exiting...')
self._state_data.is_problem_status_error = True
break
component_r0 = r_cs_value[0, :]
objective_values = self._calculate_objective(
mu_l,
mu_r,
tau,
l_cs_value,
r_cs_value,
beta_value,
new_weights,
sum_components=False)
new_objective_value = np.sum(objective_values)
improvement = (
(old_objective_value - new_objective_value) * 1. /
old_objective_value)
old_objective_value = new_objective_value
iteration += 1
# if verbose:
# ps = '{} - Objective: {:.3e}, f1: {:.3e}, f2: {:.3e},'
# ps += ' f3: {:.3e}, f4: {:.3e}'
# print(ps.format(
# iteration, new_objective_value,
# objective_values[0],
# objective_values[1], objective_values[2],
# objective_values[3]
# ))
if objective_values[0] > f1_last:
self._state_data.f1_increase = True
if verbose:
print('Caution: residuals increased')
if improvement < 0:
if verbose:
print('Caution: objective increased.')
self._state_data.obj_increase = True
improvement *= -1
if objective_values[3] > 1e2:
if self.__left_first:
if verbose:
print(
'Bad trajectory detected. Starting over and reversing minimization order.'
)
self.__left_first = False
iteration = 0
l_cs_value = self._decomposition.matrix_l0
r_cs_value = self._decomposition.matrix_r0
component_r0 = self._obtain_initial_component_r0(
verbose=verbose)
else:
if verbose:
print('Algorithm Failed!')
improvement = 0
if iteration >= max_iteration:
if verbose:
print(
'Reached iteration limit. Previous improvement: {:.2f}%'
.format(improvement * 100))
improvement = 0.
# tf = time()
# if verbose:
# print('Bootstrap {} complete in {:.2f} minutes'.format(
# ix, (tf - ti) / 60.))
self._bootstrap_samples[ix]['L'] = l_cs_value
self._bootstrap_samples[ix]['R'] = r_cs_value
self._bootstrap_samples[ix]['beta'] = beta_value
if verbose:
progress(bootstraps,
bootstraps,
status=' {:.2f} minutes'.format((time() - ti) / 60))