def moving_pole_angle_controller(ctr, pole_angle_ref, state, gains, slow,
figures, serror, sfactor, plot):
action = 1
cart_position, cart_velocity, pole_angle, pole_velocity = ou.get_obs(state)
pole_angle_gain, pole_velocity_gain, cart_position_gain, cart_velocity_gain = ou.get_gains(
gains)
pole_velocity_ref, pole_angle_error = proportional(pole_angle_ref,
pole_angle,
pole_angle_gain)
cart_position_ref, pole_velocity_error = integrator(
pole_velocity_ref, pole_velocity, pole_velocity_gain, slow,
cart_position)
cart_velocity_ref, cart_position_error = proportional(
cart_position_ref, cart_position, cart_position_gain)
power, cart_velocity_error = proportional(cart_velocity_ref, cart_velocity,
cart_velocity_gain)
if power >= 0:
action = 0
error = root_sum_squared_error([
pole_angle_error, pole_velocity_error, cart_position_error,
cart_velocity_error
])
serror = rm.smooth(error, serror, sfactor)
if plot:
figures.add_points(ctr, pole_angle_ref, pole_angle, pole_velocity_ref,
pole_velocity, cart_position_ref, cart_position,
cart_velocity_ref, cart_velocity, action, error,
serror)
return action, serror
def __call__(self, model):
current_loss = self.lossfn(model.outputs, model(model.inputs))
if self.initial_loss == None:
self.initial_loss = current_loss
#print(current_loss.numpy())
if self.previous_loss > 0:
self.current_slope=current_loss-self.previous_loss
else:
self.current_slope=0
self.slope_error=self.ref_slope-self.current_slope
if self.slope_error < 0:
self.slope_error_accumulator += self.slope_error
if self.slope_error_accumulator < self.tumble_threshold :
self.dWeights = np.random.uniform(-1,1,self.nweights)
self.updates = sigmoid( current_loss *self.learning_rate * self.dWeights, self.sigmoid_range, self.sigmoid_scale)
self.slope_error_accumulator=0
if current_loss< self.initial_loss:
self.ref_slope = self.initial_ref_slope * current_loss / self.initial_loss
self.tumble_threshold = self.initial_tumble_threshold * current_loss / self.initial_loss
if self.previous_loss >= 0:
if self.dlsmooth==None:
self.dlsmooth = self.current_slope
else:
self.dlsmooth = smooth( self.current_slope, self.dlsmooth, self.dlsmoothfactor)
self.previous_loss = current_loss
return self.updates
def __call__(self, model):
current_loss = self.lossfn(model.outputs, model(model.inputs))
#print(current_loss.numpy())
self.add_period_loss(current_loss)
if self.ctr % self.period ==0:
self.current_historical_loss = self.get_mean_loss()
#print("m", self.current_historical_loss.numpy(), self.previous_historical_loss)
if self.current_historical_loss >= self.previous_historical_loss or self.previous_historical_loss <0 :
self.dWeights = np.random.uniform(-1,1,self.nweights)
self.updates = sigmoid( self.current_historical_loss *self.learning_rate * self.dWeights, self.sigmoid_range, self.sigmoid_scale)
#self.updates = [sigmoid( self.current_historical_loss *self.learning_rate * self.dWeights[0], self.sigmoid_range, self.sigmoid_scale),
# sigmoid( self.current_historical_loss *self.learning_rate * self.dWeights[1], self.sigmoid_range, self.sigmoid_scale)]
if self.previous_historical_loss >= 0:
self.dl = self.current_historical_loss-self.previous_historical_loss
if self.dlsmooth==None:
self.dlsmooth = self.dl
else:
self.dlsmooth = smooth( self.dl, self.dlsmooth, self.dlsmoothfactor)
self.previous_historical_loss = self.current_historical_loss
#else:
# self.updates=np.zeros(self.nweights)
self.previous_loss = current_loss
self.ctr+=1
return self.updates
def get_action(self, power):
action = 0
power = rm.smooth(power, self.prev_power, 0.75)
self.prev_power = power
if power >= 0:
action = 1
return action
def add_error_data(self, current, target):
self.error = self.loss_fn(current, target)
self.mean = self.mean + (self.error - self.mean) / self.n
self.n += 1
if self.global_error == None:
self.global_error = self.error
else:
self.global_error = rm.smooth(self.error, self.global_error,
self.loss_smooth)
#print(self.error, self.global_error)
return self.global_error
def controller(ctr, pole_angle_ref, state, gains, figures, serror,
sum_error_ma, sfactor, prev_power):
action = 1
cart_position, cart_velocity, pole_angle, pole_velocity = ou.get_obs(state)
pole_angle_gain, pole_velocity_gain, cart_position_gain, cart_velocity_gain = ou.get_gains(
gains)
pole_velocity_ref, pole_angle_error = proportional(pole_angle_ref,
pole_angle,
pole_angle_gain)
cart_position_ref, pole_velocity_error = proportional(
pole_velocity_ref, pole_velocity, pole_velocity_gain)
cart_velocity_ref, cart_position_error = proportional(
cart_position_ref, cart_position, cart_position_gain)
power, cart_velocity_error = proportional(cart_velocity_ref, cart_velocity,
cart_velocity_gain)
print(power, end=" ")
#power = prev_power + 0.05 * power
power = rm.smooth(power, prev_power, 0.8)
print(power)
if power >= 0:
action = 0
sum_error = pole_angle_error + pole_velocity_error + cart_position_error + cart_velocity_error
sum_error_ma = rm.smooth(sum_error, sum_error_ma, sfactor)
error = root_sum_squared_error([
pole_angle_error, pole_velocity_error, cart_position_error,
cart_velocity_error
])
serror = rm.smooth(error, serror, sfactor)
figures.add_points(ctr, pole_angle_ref, pole_angle, pole_velocity_ref,
pole_velocity, cart_position_ref, cart_position,
cart_velocity_ref, cart_velocity, action, sum_error_ma,
serror)
return action, serror, sum_error_ma, power
def moving_controller(ctr, pole_position_ref, state, gains, slow, figures,
position_figure, serror, sfactor, scale, prev_power,
plot, cart_position_ref):
action = 1
cart_position, cart_velocity, pole_angle, pole_velocity = ou.get_obs(state)
print(
f'{pole_angle:.3f} {pole_velocity:.3f} {cart_position:.3f} {cart_velocity:.3f}'
)
"""
test=0.2
cart_position=test
cart_velocity=test
pole_angle=test
pole_velocity=test
"""
pole_position_gain, pole_angle_gain, pole_velocity_gain, cart_position_gain, cart_velocity_gain = ou.get_gains(
gains)
pole_position = cart_position + math.sin(pole_angle)
pole_angle_ref, pole_position_error = sigmoid(pole_position_ref,
pole_position,
pole_position_gain, scale)
pole_velocity_ref, pole_angle_error = proportional(pole_angle_ref,
pole_angle,
pole_angle_gain)
cart_position_ref, pole_velocity_error = integrator(
pole_velocity_ref, pole_velocity, pole_velocity_gain, slow,
cart_position)
#cart_position_ref,pole_velocity_error=integrator(pole_velocity_ref, pole_velocity, pole_velocity_gain, slow, cart_position_ref)
cart_velocity_ref, cart_position_error = proportional(
cart_position_ref, cart_position, cart_position_gain)
power, cart_velocity_error = proportional(cart_velocity_ref, cart_velocity,
cart_velocity_gain)
power = rm.smooth(power, prev_power, 0.75)
print(
f'pole_position {pole_position_ref:.3f} {pole_position:.3f} {pole_position_error:.3f} {pole_angle_ref:.3f}'
)
print(
f'pole_angle {pole_angle_ref:.3f} {pole_angle:.3f} {pole_angle_error:.3f} {pole_velocity_ref:.3f}'
)
print(
f'pole_velocity {pole_velocity_ref:.3f} {pole_velocity:.3f} {pole_velocity_error:.3f} {cart_position_ref:.3f} **'
)
print(
f'cart_position {cart_position_ref:.3f} {cart_position:.3f} {cart_position_error:.3f} {cart_velocity_ref:.3f}'
)
print(
f'cart_velocity {cart_velocity_ref:.3f} {cart_velocity:.3f} {cart_velocity_error:.3f} {power:.3f}'
)
if power >= 0:
action = 0
print(action)
error = root_sum_squared_error([
pole_angle_error, pole_velocity_error, cart_position_error,
cart_velocity_error
])
serror = rm.smooth(error, serror, sfactor)
if plot:
if type(figures) == go.FigureWidget:
add_cartpolepoints_to_widget(figures, 2, ctr, pole_angle_ref,
pole_angle, pole_velocity_ref,
pole_velocity, cart_position_ref,
cart_position, cart_velocity_ref,
cart_velocity, action, error, serror)
else:
figures.add_points(ctr, pole_angle_ref, pole_angle,
pole_velocity_ref, pole_velocity,
cart_position_ref, cart_position,
cart_velocity_ref, cart_velocity, action, error,
serror)
if type(position_figure) == go.FigureWidget:
add_cartpole_positions_to_widget(position_figure, ctr,
pole_position_ref, pole_position)
else:
position_figure.add_points(ctr, pole_position_ref, pole_position)
return action, serror, pole_position, power, cart_position_ref