def agent(): delta_t = 1e-4 nomFreq = 50 nomVoltPeak = 230 * 1.414 iLimit = 30 DroopGain = 40000.0 # W/Hz QDroopGain = 1000.0 # VAR/V mutable_params = dict(voltP=MutableFloat(25e-3), voltI=MutableFloat(60)) ctrl = [] # Voltage PI parameters for the current sourcing inverter voltage_dqp_iparams = PI_params(kP=mutable_params['voltP'], kI=mutable_params['voltI'], limits=(-iLimit, iLimit)) # Current PI parameters for the voltage sourcing inverter current_dqp_iparams = PI_params(kP=0.012, kI=90, limits=(-1, 1)) # Droop of the active power Watt/Hz, delta_t droop_param = DroopParams(DroopGain, 0.005, nomFreq) # Droop of the reactive power VAR/Volt Var.s/Volt qdroop_param = DroopParams(QDroopGain, 0.002, nomVoltPeak) ctrl.append( MultiPhaseDQ0PIPIController(voltage_dqp_iparams, current_dqp_iparams, delta_t, droop_param, qdroop_param, name='master')) # Discrete controller implementation for a DQ based Current controller for the current sourcing inverter # Current PI parameters for the current sourcing inverter current_dqp_iparams = PI_params(kP=0.005, kI=200, limits=(-1, 1)) # PI params for the PLL in the current forming inverter pll_params = PLLParams(kP=10, kI=200, limits=(-10000, 10000), f_nom=nomFreq) # Droop of the active power Watts/Hz, W.s/Hz droop_param = InverseDroopParams(DroopGain, 0, nomFreq, tau_filt=0.04) # Droop of the reactive power VAR/Volt Var.s/Volt qdroop_param = InverseDroopParams(100, 0, nomVoltPeak, tau_filt=0.01) ctrl.append( MultiPhaseDQCurrentController(current_dqp_iparams, pll_params, delta_t, iLimit, droop_param, qdroop_param, name='slave')) # validate that parameters can be changed later on agent = StaticControlAgent( ctrl, { 'master': [[f'lc1.inductor{i + 1}.i' for i in range(3)], [f'lc1.capacitor{i + 1}.v' for i in range(3)]], 'slave': [[f'lcl1.inductor{i + 1}.i' for i in range(3)], [f'lcl1.capacitor{i + 1}.v' for i in range(3)], np.zeros(3)] }) return mutable_params, agent
# expanding points eventually. # The following variable is multiplied with the first performance of the initial set by the factor below: explore_threshold = 0 # Factor to multiply with the initial reward to give back an abort_reward-times higher negative reward in case of # limit exceeded abort_reward = -10 * j_min # Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers # Choose all droop parameter as mutable parameters mutable_params = dict(pDroop_master=MutableFloat(30000.0), pDroop_slave=MutableFloat(30000.0), qDroop_master=MutableFloat(QDroopGain), qDroop_slave=MutableFloat(10)) # Define the droop parameters for the inverter of the active power Watt/Hz (DroopGain), delta_t (0.005) used for # the filter and the nominal frequency # Droop controller used to calculate the virtual frequency drop due to load changes droop_param_master = DroopParams(mutable_params['pDroop_master'], 0.005, freq_nom) # droop parameter used to react on frequency drop droop_param_slave = InverseDroopParams(mutable_params['pDroop_slave'], delta_t, freq_nom, tau_filt=0.04) # Droop characteristic for the reactive power VAR/Volt Var.s/Volt # qDroop parameter used for virtual voltage drop qdroop_param_master = DroopParams(mutable_params['qDroop_master'], 0.002, v_nom) # Droop characteristic for the reactive power VAR/Volt Var.s/Volt qdroop_param_slave = InverseDroopParams(mutable_params['qDroop_slave'], delta_t, v_nom, tau_filt=0.01) ###############
# Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers # Choose Kp and Ki for the current and voltage controller as mutable parameters # mutable_params = dict(currentP=MutableFloat(10e-3), currentI=MutableFloat(10), voltageP=MutableFloat(0.05), # voltageI=MutableFloat(75)) # Vdc = 600 V mutable_params = dict(currentP=MutableFloat(0.04), currentI=MutableFloat(11.8), voltageP=MutableFloat(0.0175), voltageI=MutableFloat(12)) voltage_dqp_iparams = PI_params(kP=mutable_params['voltageP'], kI=mutable_params['voltageI'], limits=(-iLimit, iLimit)) current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=mutable_params['currentI'], limits=(-1, 1)) # Best set from paper III-D # Define the droop parameters for the inverter of the active power Watt/Hz (DroopGain), delta_t (0.005) used for the # filter and the nominal frequency # Droop controller used to calculate the virtual frequency drop due to load changes
# The algorithm will not try to expand any points that are below this threshold. This makes the algorithm stop # expanding points eventually. # The following variable is multiplied with the first performance of the initial set by the factor below: explore_threshold = 0 # Factor to multiply with the initial reward to give back an abort_reward-times higher negative reward in case of # limit exceeded abort_reward = -10 * j_min # Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers # Choose Kp and Ki for the current and voltage controller as mutable parameters mutable_params = dict(currentP=MutableFloat(10e-3), currentI=MutableFloat(10), voltageP=MutableFloat(25e-3), voltageI=MutableFloat(60)) voltage_dqp_iparams = PI_params(kP=mutable_params['voltageP'], kI=mutable_params['voltageI'], limits=(-i_lim, i_lim)) current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=mutable_params['currentI'], limits=(-1, 1)) # Define the droop parameters for the inverter of the active power Watt/Hz (DroopGain), delta_t (0.005) used for the # filter and the nominal frequency # Droop controller used to calculate the virtual frequency drop due to load changes droop_param = DroopParams(DroopGain, 0.005, net.freq_nom) # Define the Q-droop parameters for the inverter of the reactive power VAR/Volt, delta_t (0.002) used for the # filter and the nominal voltage qdroop_param = DroopParams(QDroopGain, 0.002, net.v_nom)
abort_reward = 10 * j_min # Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers mutable_params = None current_dqp_iparams = None if adjust == 'Kp': # mutable_params = parameter (Kp gain of the current controller of the inverter) to be optimized using # the SafeOpt algorithm mutable_params = dict(currentP=MutableFloat(5e-3)) # Define the PI parameters for the current controller of the inverter current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=115, limits=(-1, 1)) # For 1D example, if Ki should be adjusted elif adjust == 'Ki': mutable_params = dict(currentI=MutableFloat(10)) current_dqp_iparams = PI_params(kP=10e-3, kI=mutable_params['currentI'], limits=(-1, 1)) # For 2D example, choose Kp and Ki as mutable parameters elif adjust == 'Kpi':
# Factor to multiply with the initial reward to give back an abort_reward-times higher negative reward in case of # limit exceeded abort_reward = 10 * j_min # 10 # Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers mutable_params = None current_dqp_iparams = None if adjust == 'Kp': # mutable_params = parameter (Kp gain of the current controller of the inverter) to be optimized using # the SafeOpt algorithm mutable_params = dict(currentP=MutableFloat(5e-3)) # Define the PI parameters for the current controller of the inverter current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=115, limits=(-1, 1)) # For 1D example, if Ki should be adjusted elif adjust == 'Ki': mutable_params = dict(currentI=MutableFloat(10)) current_dqp_iparams = PI_params(kP=10e-3, kI=mutable_params['currentI'], limits=(-1, 1)) # For 2D example, choose Kp and Ki as mutable parameters elif adjust == 'Kpi': mutable_params = dict(currentP=MutableFloat(15e-3), currentI=MutableFloat(10)) # setP= 10 e^-3 and setQ=10 current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=mutable_params['currentI'], limits=(-1, 1)) # Define a current sourcing inverter as master inverter using the pi and droop parameters from above
# Factor to multiply with the initial reward to give back an abort_reward-times higher negative reward in case of # limit exceeded # has to be negative due to normalized performance (regarding J_init = 1) abort_reward = 100 * j_min # Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers mutable_params = None current_dqp_iparams = None if adjust == 'Kp': # mutable_params = parameter (Kp gain of the current controller of the inverter) to be optimized using # the SafeOpt algorithm mutable_params = dict(currentP=MutableFloat(0.04)) # Define the PI parameters for the current controller of the inverter current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=12, limits=(-1, 1)) # For 1D example, if Ki should be adjusted elif adjust == 'Ki': mutable_params = dict(currentI=MutableFloat(5)) current_dqp_iparams = PI_params(kP=0.005, kI=mutable_params['currentI'], limits=(-1, 1)) # For 2D example, choose Kp and Ki as mutable parameters elif adjust == 'Kpi': mutable_params = dict(currentP=MutableFloat(0.04), currentI=MutableFloat(11.8)) current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=mutable_params['currentI'], limits=(-1, 1))
explore_threshold = 0 # Factor to multiply with the initial reward to give back an abort_reward-times higher negative reward in case of # limit exceeded abort_reward = 100 * j_min # Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers # Choose Kp and Ki for the current and voltage controller as mutable parameters mutable_params = dict(voltageP=MutableFloat(0.0175), voltageI=MutableFloat(12)) # 300Hz mutable_params = dict(voltageP=MutableFloat(0.2), voltageI=MutableFloat(550)) # 300Hz voltage_dqp_iparams = PI_params(kP=mutable_params['voltageP'], kI=mutable_params['voltageI'], limits=(-iLimit, iLimit)) kp_c = 0.04 ki_c = 11.8 current_dqp_iparams = PI_params(kP=kp_c, kI=ki_c, limits=(-1, 1)) # Current controller values # Define the droop parameters for the inverter of the active power Watt/Hz (DroopGain), delta_t (0.005) used for the # filter and the nominal frequency
def run_experiment(len_kp, len_ki): if isfile(f'{save_folder}/{len_kp:.4f},{len_ki:.4f}.txt'): with open(f'{save_folder}/{len_kp:.4f},{len_ki:.4f}.txt', 'r') as f: return strtobool(f.read().strip()) rew = Reward(i_limit=iLimit, i_nominal=iNominal, mu_c=mu, max_episode_steps=max_episode_steps, obs_dict=[[f'lc.inductor{k}.i' for k in '123'], 'master.phase', [f'master.SPI{k}' for k in 'dq0']]) ##################################### # Definitions for the GP prior_mean = 0 # 2 # mean factor of the GP prior mean which is multiplied with the first performance of the # initial set noise_var = 0.001 # 0.001 ** 2 # measurement noise sigma_omega prior_var = 2 # prior variance of the GP bounds = None lengthscale = None if adjust == 'Kp': bounds = [(0.0001, 0.1)] # bounds on the input variable Kp lengthscale = [ .025 ] # length scale for the parameter variation [Kp] for the GP # For 1D example, if Ki should be adjusted if adjust == 'Ki': bounds = [(0, 20)] # bounds on the input variable Ki lengthscale = [ 10 ] # length scale for the parameter variation [Ki] for the GP # For 2D example, choose Kp and Ki as mutable parameters (below) and define bounds and lengthscale for both of them if adjust == 'Kpi': bounds = [(0.001, 0.07), (2, 150)] lengthscale = [0.012, 30.] df_len = pd.DataFrame({ 'lengthscale': lengthscale, 'bounds': bounds, 'balanced_load': balanced_load, 'barrier_param_mu': mu }) # The performance should not drop below the safe threshold, which is defined by the factor safe_threshold times # the initial performance: safe_threshold = 0.8 means. Performance measurement for optimization are seen as # unsafe, if the new measured performance drops below 20 % of the initial performance of the initial safe (!) # parameter set safe_threshold = 0 j_min = cal_j_min(phase_shift, amp_dev) # Used for normalization # The algorithm will not try to expand any points that are below this threshold. This makes the algorithm stop # expanding points eventually. # The following variable is multiplied with the first performance of the initial set by the factor below: explore_threshold = 0 # Factor to multiply with the initial reward to give back an abort_reward-times higher negative reward in case of # limit exceeded # has to be negative due to normalized performance (regarding J_init = 1) abort_reward = 100 * j_min # Definition of the kernel kernel = GPy.kern.Matern32(input_dim=len(bounds), variance=prior_var, lengthscale=lengthscale, ARD=True) ##################################### # Definition of the controllers mutable_params = None current_dqp_iparams = None if adjust == 'Kp': # mutable_params = parameter (Kp gain of the current controller of the inverter) to be optimized using # the SafeOpt algorithm mutable_params = dict(currentP=MutableFloat(0.04)) # Define the PI parameters for the current controller of the inverter current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=12, limits=(-1, 1)) # For 1D example, if Ki should be adjusted elif adjust == 'Ki': mutable_params = dict(currentI=MutableFloat(5)) current_dqp_iparams = PI_params(kP=0.005, kI=mutable_params['currentI'], limits=(-1, 1)) # For 2D example, choose Kp and Ki as mutable parameters elif adjust == 'Kpi': mutable_params = dict(currentP=MutableFloat(0.04), currentI=MutableFloat(11.8)) current_dqp_iparams = PI_params(kP=mutable_params['currentP'], kI=mutable_params['currentI'], limits=(-1, 1)) # Define a current sourcing inverter as master inverter using the pi and droop parameters from above ctrl = MultiPhaseDQCurrentSourcingController(current_dqp_iparams, delta_t, undersampling=undersample, name='master', f_nom=net.freq_nom) i_ref = MutableParams([MutableFloat(f) for f in i_ref1]) ##################################### # Definition of the optimization agent # The agent is using the SafeOpt algorithm by F. Berkenkamp (https://arxiv.org/abs/1509.01066) in this example # Arguments described above # History is used to store results agent = SafeOptAgent( mutable_params, abort_reward, j_min, kernel, dict(bounds=bounds, noise_var=noise_var, prior_mean=prior_mean, safe_threshold=safe_threshold, explore_threshold=explore_threshold), [ctrl], dict(master=[[f'lc.inductor{k}.i' for k in '123'], i_ref]), history=FullHistory(), ) ##################################### # Definition of the environment using a FMU created by OpenModelica # (https://www.openmodelica.org/) # Using an inverter supplying a load # - using the reward function described above as callable in the env # - viz_cols used to choose which measurement values should be displayed (here, only the 3 currents across the # inductors of the inverters are plotted. Labels and grid is adjusted using the PlotTmpl (For more information, # see UserGuide) # - inputs to the models are the connection points to the inverters (see user guide for more details) # - model outputs are the the 3 currents through the inductors and the 3 voltages across the capacitors if include_simulate: # Defining unbalanced loads sampling from Gaussian distribution with sdt = 0.2*mean # r_load = Load(R, 0.1 * R, balanced=balanced_load, tolerance=0.1) # l_load = Load(L, 0.1 * L, balanced=balanced_load, tolerance=0.1) # i_noise = Noise([0, 0, 0], [0.0023, 0.0015, 0.0018], 0.0005, 0.32) # if no noise should be included: r_load = Load(R, 0 * R, balanced=balanced_load) l_load = Load(L, 0 * L, balanced=balanced_load) def reset_loads(): r_load.reset() l_load.reset() plotter = PlotManager(agent, save_results=save_results, save_folder=save_folder, show_plots=show_plots) def ugly_foo(t): if t >= .05: i_ref[:] = i_ref2 else: i_ref[:] = i_ref1 return partial(l_load.give_value, n=2)(t) env = gym.make( 'openmodelica_microgrid_gym:ModelicaEnv_test-v1', # reward_fun=Reward().rew_fun, reward_fun=rew.rew_fun_c, # time_step=delta_t, viz_cols=[ PlotTmpl([[f'lc.inductor{i}.i' for i in '123'], [f'master.SPI{i}' for i in 'abc']], callback=plotter.xylables_i_abc, color=[['b', 'r', 'g'], ['b', 'r', 'g']], style=[[None], ['--']]), PlotTmpl([[f'master.m{i}' for i in 'abc']], callback=lambda fig: plotter.update_axes( fig, title='Simulation', ylabel='$m_{\mathrm{abc}}\,/\,\mathrm{}$')), PlotTmpl([[f'master.CVI{i}' for i in 'dq0'], [f'master.SPI{i}' for i in 'dq0']], callback=plotter.xylables_i_dq0, color=[['b', 'r', 'g'], ['b', 'r', 'g']], style=[[None], ['--']]) ], log_level=logging.INFO, viz_mode='episode', max_episode_steps=max_episode_steps, model_params={ 'lc.resistor1.R': partial(r_load.give_value, n=0), 'lc.resistor2.R': partial(r_load.give_value, n=1), 'lc.resistor3.R': partial(r_load.give_value, n=2), 'lc.inductor1.L': partial(l_load.give_value, n=0), 'lc.inductor2.L': partial(l_load.give_value, n=1), 'lc.inductor3.L': ugly_foo }, model_path='../../omg_grid/grid.paper.fmu', # model_path='../omg_grid/omg_grid.Grids.Paper_SC.fmu', net=net, history=FullHistory(), action_time_delay=1 * undersample) runner = MonteCarloRunner(agent, env) runner.run(num_episodes, n_mc=n_MC, visualise=True, prepare_mc_experiment=reset_loads) with open(f'{save_folder}/{len_kp:.4f},{len_ki:.4f}.txt', 'w') as f: print(f'{agent.unsafe}', file=f) return agent.unsafe