def __init__(self, Testing=False):
        self.testing = Testing
        if not os.path.exists("./models/controller"):
            os.makedirs("./models/controller")

        self.actor = Actor(state_dim, action_dim, BATCH_SIZE, TAU, LRA)

        try:
            self.actor.model.load_state_dict(
                torch.load("./models/controller/actor.pt"))
            self.actor.target_model.load_state_dict(
                torch.load("./models/controller/actor.pt"))
            print("Load actor model successfully")
        except:
            print("Cannot find actor weights in this directory")

        if self.testing is False:
            self.buff = ReplayBuffer(BUFFER_SIZE)
            self.OU = OU()
            self.critic = Critic(state_dim, action_dim, BATCH_SIZE, TAU, LRC)
            try:
                self.critic.model.load_state_dict(
                    torch.load("./models/controller/critic.pt"))
                self.critic.target_model.load_state_dict(
                    torch.load("./models/controller/critic.pt"))
                print("Load critic model successfully")
            except:
                print("Cannot find critic weights in this directory")
        else:
            self.actor.model.eval()
class ddpgAgent():
    def __init__(self, Testing=False):
        self.testing = Testing
        if not os.path.exists("./models/controller"):
            os.makedirs("./models/controller")

        self.actor = Actor(state_dim, action_dim, BATCH_SIZE, TAU, LRA)

        try:
            self.actor.model.load_state_dict(
                torch.load("./models/controller/actor.pt"))
            self.actor.target_model.load_state_dict(
                torch.load("./models/controller/actor.pt"))
            print("Load actor model successfully")
        except:
            print("Cannot find actor weights in this directory")

        if self.testing is False:
            self.buff = ReplayBuffer(BUFFER_SIZE)
            self.OU = OU()
            self.critic = Critic(state_dim, action_dim, BATCH_SIZE, TAU, LRC)
            try:
                self.critic.model.load_state_dict(
                    torch.load("./models/controller/critic.pt"))
                self.critic.target_model.load_state_dict(
                    torch.load("./models/controller/critic.pt"))
                print("Load critic model successfully")
            except:
                print("Cannot find critic weights in this directory")
        else:
            self.actor.model.eval()

    def loadTestModels(self, actor):
        current_actor = "./models/controller/intermittent/" + actor
        try:
            self.actor.model.load_state_dict(torch.load(current_actor))
            self.actor.target_model.load_state_dict(torch.load(current_actor))
            print("Load intermittent actor model successfully", current_actor)
        except:
            print("Cannot find intermittent actor weights in this directory")
        self.actor.model.eval()
        return

    def getAction(self, state, epsilon):
        action = np.zeros([1, action_dim])
        noise = np.zeros([1, action_dim])
        with torch.no_grad():
            action_original = self.actor.model(
                self.to_tensor(state.reshape(1, state.shape[0])))
        action_original = self.to_array(action_original)
        if self.testing is False:
            #noise[0][0] = (1.0-float(self.testing)) * max(epsilon, 0) * self.OU.function(action_original[0][0], 0.2, 1.00, 0.10)
            noise[0][0] = max(epsilon, 0) * self.OU.function(
                action_original[0][0], 0, 0.60, 0.10)
            #print('noise is :',noise[0][0],'action is:',action_original)
        action[0][0] = action_original[0][0] + noise[0][0]
        if action[0][0] < 0.0:
            action[0][0] = 0.0
        if action[0][0] > 1.0:
            action[0][0] = 1.0
        #print("NN Controller: {:5.4f}, Noise NN Controller: {:5.4f}".format(action_original[0][0], action[0][0]))
        return action

    def storeTrajectory(self, s, a, r, s_, done):
        self.buff.add(s, a[0], r, s_, done)

    def learn(self):
        batch = self.buff.getBatch(BATCH_SIZE)
        states = np.asarray([e[0] for e in batch])
        actions = np.asarray([e[1] for e in batch])
        rewards = np.asarray([e[2] for e in batch])
        new_states = np.asarray([e[3] for e in batch])
        dones = np.asarray([e[4] for e in batch])
        y_t = np.asarray([e[1] for e in batch])

        with torch.no_grad():
            target_q_values = self.critic.target_model([
                self.to_tensor(new_states),
                self.actor.target_model(self.to_tensor(new_states))
            ])
        target_q_values = self.to_array(target_q_values)

        for k in range(len(batch)):
            if dones[k]:
                y_t[k] = rewards[k]
            else:
                y_t[k] = rewards[k] + GAMMA * target_q_values[k]

        # train the critic model
        self.critic.model.zero_grad()
        q_values = self.critic.model(
            [self.to_tensor(states),
             self.to_tensor(actions)])
        self.critic.train(q_values, self.to_tensor(y_t))

        # train the actor model
        self.actor.model.zero_grad()
        policy_loss = -self.critic.model(
            [self.to_tensor(states),
             self.actor.model(self.to_tensor(states))])
        self.actor.train(policy_loss)

        self.actor.target_train()
        self.critic.target_train()

    def save_model(self):
        print("Saving model now...")
        torch.save(self.actor.model.state_dict(),
                   "./models/controller/actor.pt")
        torch.save(self.critic.model.state_dict(),
                   "./models/controller/critic.pt")

    def save_intermittent_model(self, episode):
        print("Saving intermittent model now...")
        actor_name = './models/controller/intermittent/' + 'actor' + str(
            episode) + '.pt'
        critic_name = './models/controller/intermittent/' + 'critic' + str(
            episode) + '.pt'
        torch.save(self.actor.model.state_dict(), actor_name)
        torch.save(self.critic.model.state_dict(), critic_name)

    def to_tensor(self, numpy_array):
        return torch.from_numpy(numpy_array).float().cuda()

    def to_array(self, torch_tensor):
        return torch_tensor.cpu().float().numpy()