Пример #1
0
def main():
    cfg = ConfigParser()
    cfg.read('config.ini')

    IP = cfg.get('server', 'ip')
    PORT = cfg.getint('server', 'port')
    FILE = cfg.get('file', 'file')
    SIZE = cfg.getint('env', 'buffer_size')
    TIME = cfg.getfloat('env', 'time')

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((IP, PORT))
    fd = sock.fileno()
    io = io_thread(sock=sock, filename=FILE, buffer_size=SIZE)
    mpsched.persist_state(fd)

    io.start()
    my_env = env(fd=fd, buff_size=SIZE, time=TIME, k=4, l=0.01, n=0.03, p=0.05)

    state = my_env.reset()
    while True:
        action = []
        state_nxt, reward, count, recv_buff_size, done = my_env.step(action)
        if done:
            break
        print(reward)
        print(recv_buff_size)
    print(count)

    io.join()
Пример #2
0
    def __init__(self, fd, cfg, memory, explore=True):
        threading.Thread.__init__(self)
        self.fd = fd
        self.cfg = cfg
        self.memory = memory
        self.explore = explore
        self.agent = torch.load(cfg.get('nafcnn', 'agent'))
        self.ounoise = OUNoise(action_dimension=1)
        mpsched.persist_state(fd)

        self.env = Env(fd=self.fd,
                       time=self.cfg.getfloat('env', 'time'),
                       k=self.cfg.getint('env', 'k'),
                       alpha=self.cfg.getfloat('env', 'alpha'),
                       b=self.cfg.getfloat('env', 'b'),
                       c=self.cfg.getfloat('env', 'c'))
Пример #3
0
    def reset(self):
        mpsched.persist_state(self.fd)
        time.sleep(1)
        self.last = mpsched.get_sub_info(self.fd)

        for i in range(self.k):
            subs = mpsched.get_sub_info(self.fd)
            for j in range(len(subs)):
                 self.tp[j].append(subs[j][0] - self.last[j][0])
                 self.rtt[j].append(subs[j][1] - self.last[j][1])
                 self.cwnd[j].append(subs[j][2])
            self.last = subs
            time.sleep(self.time)
        mate = mpsched.get_meta_info(self.fd)
        self.recv_buff_size = mate[0]
        self.rr = mate[1]
        return [self.tp[0] + self.rtt[0] + self.cwnd[0] + [self.recv_buff_size, self.rr], self.tp[1] + self.rtt[1] + self.cwnd[1]+ [self.recv_buff_size, self.rr]]
Пример #4
0
    def reset(self):
        mpsched.persist_state(self.fd)
        """time.sleep()"""
        self.last = mpsched.get_sub_info(self.fd)
        self.rr.append(self.last[0][3])
        self.rr.append(self.last[1][3])
        self.in_flight.append(self.last[0][4])
        self.in_flight.append(self.last[1][4])

        for i in range(self.k):
            subs = mpsched.get_sub_info(self.fd)
            for j in range(len(subs)):
                self.tp[j].append(subs[j][0] - self.last[j][0])
                self.rtt[j].append(subs[j][1])
                self.dRtt[j].append(subs[j][1] - self.last[j][1])
                self.cwnd[j].append(subs[j][2])
                self.rr[j] = subs[j][3]
                self.in_flight[j] = subs[j][4]
            self.last = subs
            time.sleep(self.time)

        return [self.tp[0]+self.rtt[0]+self.cwnd[0]+[self.rr[0], self.in_flight[0]], self.tp[1]+self.rtt[1]+self.cwnd[1]+[self.rr[1], self.in_flight[1]]]
Пример #5
0
def main():
    cfg = ConfigParser()
    cfg.read('config.ini')

    IP = cfg.get('server', 'ip')
    PORT = cfg.getint('server', 'port')
    FILE = cfg.get('file', 'file')
    SIZE = cfg.getint('env', 'buffer_size')
    TIME = cfg.getfloat('env', 'time')
    EPISODE = cfg.getint('env', 'episode')

    parser = argparse.ArgumentParser(description='PyTorch REINFORCE example')

    parser.add_argument('--gamma',
                        type=float,
                        default=0.99,
                        metavar='G',
                        help='discount factor for reward (default: 0.99)')
    parser.add_argument('--tau',
                        type=float,
                        default=0.001,
                        metavar='G',
                        help='discount factor for model (default: 0.001)')

    parser.add_argument('--noise_scale',
                        type=float,
                        default=0.3,
                        metavar='G',
                        help='initial noise scale (default: 0.3)')
    parser.add_argument('--final_noise_scale',
                        type=float,
                        default=0.3,
                        metavar='G',
                        help='final noise scale (default: 0.3)')
    parser.add_argument('--exploration_end',
                        type=int,
                        default=100,
                        metavar='N',
                        help='number of episodes with noise (default: 100)')

    parser.add_argument('--hidden_size',
                        type=int,
                        default=128,
                        metavar='N',
                        help='number of hidden size (default: 128)')
    parser.add_argument('--replay_size',
                        type=int,
                        default=1000000,
                        metavar='N',
                        help='size of replay buffer (default: 1000000)')
    parser.add_argument('--updates_per_step',
                        type=int,
                        default=5,
                        metavar='N',
                        help='model updates per simulator step (default: 5)')
    parser.add_argument('--batch_size',
                        type=int,
                        default=64,
                        metavar='N',
                        help='batch size (default: 128)')

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((IP, PORT))
    fd = sock.fileno()
    my_env = env(fd=fd, buff_size=SIZE, time=TIME, k=8, l=0.01, n=0.03, p=0.05)
    mpsched.persist_state(fd)

    args = parser.parse_args()
    agent = NAF_CNN(args.gamma, args.tau, args.hidden_size,
                    my_env.observation_space.shape[0], my_env.action_space)
    memory = ReplayMemory(args.replay_size)
    ounoise = OUNoise(my_env.action_space.shape[0])

    rewards = []
    times = []
    for i_episode in range(EPISODE):
        if (i_episode < 0.9 * EPISODE):  # training
            io = io_thread(sock=sock, filename=FILE, buffer_size=SIZE)
            io.start()

            state = my_env.reset()

            ounoise.scale = (args.noise_scale - args.final_noise_scale) * max(
                0, args.exploration_end -
                i_episode) / args.exploration_end + args.final_noise_scale
            ounoise.reset()
            print(state)
            episode_reward = 0
            while True:
                state = torch.FloatTensor(state)
                #print("state: {}\n ounoise: {}".format(state, ounoise.scale))
                action = agent.select_action(state, ounoise)
                #print("action: {}".format(action))
                next_state, reward, count, recv_buff_size, done = my_env.step(
                    action)
                #print("buff size: ",recv_buff_size)
                #print("reward: ", reward)
                episode_reward += reward

                action = torch.FloatTensor(action)
                mask = torch.Tensor([not done])
                next_state = torch.FloatTensor(next_state)
                reward = torch.FloatTensor([float(reward)])
                memory.push(state, action, mask, next_state, reward)

                state = next_state

                if len(memory) > args.batch_size * 5:
                    for _ in range(args.updates_per_step):
                        transitions = memory.sample(args.batch_size)
                        batch = Transition(*zip(*transitions))
                        #print("update",10*'--')
                        agent.update_parameters(batch)

                if done:
                    break
            rewards.append(episode_reward)
            io.join()
        else:  # testing
            io = io_thread(sock=sock, filename=FILE, buffer_size=SIZE)
            io.start()
            state = my_env.reset()
            episode_reward = 0
            start_time = time.time()
            while True:
                state = torch.FloatTensor(state)
                #print("state: {}\n".format(state))
                action = agent.select_action(state)
                #print("action: {}".format(action))
                next_state, reward, count, done = my_env.step(action)
                episode_reward += reward
                state = next_state

                if done:
                    break
            rewards.append(episode_reward)
            times.append(str(time.time() - start_time) + "\n")
            io.join()
        #print("Episode: {}, noise: {}, reward: {}, average reward: {}".format(i_episode, ounoise.scale, rewards[-1], np.mean(rewards[-100:])))
        fo = open("times.txt", "w")
        fo.writelines(lines)
        fo.close()

    sock.close()