def main(): agent = RLAgent('agent', decisionFrequency=10.0, defaultSpeed=4, defaultAltitude=6, yawRate=60) agent.defineState(orientation=getOrientation, position=getPosition, angularVelocity=getAngularVelocity, linearVelocity=getVelocity, linearAcceleration=getLinearAcceleration, angularAcceleration=getAngularAcceleration) agent.setRl(partial(flightLogger, dataset='datasets/' + 'replay.csv')) agent.start() agent.join()
def main(): model = VelocityModel( regressionModel=joblib.load('models/gradient-m.model'), frequency=10.0) agent = RLAgent('agent', decisionFrequency=20.0, defaultSpeed=4, defaultAltitude=6, yawRate=60, alternativeModel=model, maxDepth=math.inf, initialState=None) agent.setRl(verifyModel) agent.start() agent.join()
def monteCarlo(agent, maxDepth=3, trials=12, frequency=10): PROCESSES = 4 model = VelocityModel( regressionModel=joblib.load('models/gradient-m.model'), frequency=frequency) actions = np.array(agent.getActions()) initialState, isTerminal = agent.getState(), 0 jobs = [None] * len(actions) * trials while bool(isTerminal) is False: initialState = agent.getState() qs = {i: [] for i in actions} for index, a in enumerate(np.repeat(actions, trials)): virtualAgent, isTerminal = RLAgent( 'virtual', alternativeModel=model, decisionFrequency=math.inf, maxDepth=maxDepth, initialState=initialState), False virtualAgent.setReward(reward) virtualAgent.goal = agent.getGoal() virtualAgent.goalMargins = agent.getGoalMargins() virtualAgent.setRl( partial(monteCarloSearch, actions=getRandomActions(a, actions, maxDepth))) jobs[index] = virtualAgent pool = Pool(8) results = [pool.apply_async(job.run) for job in jobs] for result in results: action, score = result.get() qs[action].append(score) pool.close() pool.join() yield actions[np.argmax([np.average(qs[a]) for a in actions])] r, nextState, isTerminal = (yield) f = 1 / (nextState.lastUpdate - initialState.lastUpdate) # correct for deviations from desired freq. model.frequency = f agent.logger.info(f) yield
def __init__(self): # Utilクラスインスタンス生成 self.util = RLUtil.RLUtil() # 実行する課題を設定 self.env = gym.make(self.util.getENV()) # 課題の状態と行動の数を設定 self.num_states = self.env.observation_space.shape[0] self.num_actions = self.env.action_space.n # このとき、cartpoleの行動(left/right)の2を取得 # Agentクラスのインスタンスを生成 self.agent = RLAgent.RLAgent(self.num_states, self.num_actions) # デバイス情報の設定 d = self.util.getDEVICE() self.device = torch.device(d)
import numpy as np import tensorflow as tf import math import cPickle import graph import RLAgent import httpConec as hC import matplotlib.pyplot as plt import geneTopo topo, hosts, nodes, links = geneTopo.getDCtopo() ag = RLAgent.PGNAgent(links, nodes * 2 + 1) atest = graph.graf(nodes, links, initopo=topo, inihost=hosts) atest.initial() #atest.printTopo() batch_size = 16 batch_number = 0 total_episodes = 640 episode_number = 0 valid_action = 0 invalid_action = 0 xs, ys, rs = [], [], [] epslon = 0.9 plt_x = range(total_episodes / batch_size) plt_y = [] r_batch = [] valid_action_combo = 0 gradBuffer = ag.sess.run(ag.tvars)
import game import RLAgent import Multigamer #Multi Game Simulation agent = RLAgent.RandomSnakeAgent() #Mutligamer attributes: # - numberOfParallelGames (no default) # - RLAgent (no default) # - numberOfTrials (default = 1) # - numberOfIterations (default=1) # - reward (default=100) # - cost (default=1) # - dim (default=(250,250)) gamer = Multigamer.multiGamer(10, agent, 5, 25) gamer.runSimulation() results = gamer.getResults() ## Single Game Simulation (with GUI) # snake = game.Snake() # foodSpawner = game.FoodSpawner() # snakeGame = game.SnakeGame(snake, foodSpawner, 100, 1, True, (150,150)) # results = snakeGame.simulate(agent,5,3) ## Single Game Manual Play (with GUI) # snake = game.Snake() # foodSpawner = game.FoodSpawner() # #snakeGame.play() # #score, movements, food = snakeGame.getResults()
def test(self, code, day=252, year=1, test_year=1, full_year=5, test_month=1, full_month=5, category='d', eps=0.05): now = datetime.now() now = now.strftime('%Y%m%d') path = '.\\DB\\CSV\\daily\\' if category == 'm': path = '.\\DB\\CSV\\min\\' day = 20 day = day * 390 test_year = test_month full_year = full_month now = now + '_m' filename = code + '_ch.csv' chartfile = code + '.csv' if not os.path.exists(path + filename): self.logging.info('파일 생성') self.pre.change_csv(code, category=category) if not os.path.exists('.\\Test\\re_' + now): os.makedirs('.\\Test\\re_' + now) self.logging.info('테스트 데이터 분리') df_chart = pd.read_csv(path + chartfile) if category == 'm': df_chart = self.pre.min_preprocessing(df_chart) df_chart_split = df_chart.iloc[-1 * day * year * test_year:] df_chart_split.to_csv('.\\Test\\re_' + now + '\\chart.csv', index=False) df_chart = self.pre.change_feature(df_chart) df_test = df_chart.iloc[-1 * day * year * test_year:] df_prev = df_chart.iloc[-1 * day * year * test_year - 120:-1 * day * year * test_year] step = len(df_test) env = RLEnvTrain.RLEnv(df_test) agent = RLAgent.Agent(gamma=0.98, eps_start=eps, eps_end=0.01, eps_decay_steps=800, eps_exponential_decay=0.99, replay_capacity=int(1e6), batch_size=step - 1, tau=10, code=code, V_nn='DNN', P_nn='CNN', method='A2C') # policy value A2C reward_list = [] action_List = [] quant_list = [] stock_cnt_list = [] obs_list = [] obs = env.reset() agent.reset() if not os.path.isdir('.\\Test\\re_' + now + '\\'): os.makedirs('.\\Test\\re_' + now + '\\') df_prev = df_prev data = obs.reshape(1, -1) data = pd.DataFrame(data, columns=df_chart.columns) df_prev = pd.concat([df_prev, data], ignore_index=True) obs = self.pre.add_feature(df_prev) step = len(df_test) self.logging.info('테스트 시작') for i in tqdm(range(step)): # 관측 데이터로 예측한 가치신경망, 정책 신경망 예측값 value_per, policy_per = agent.predict_action_per(obs) action = agent.policy(value_per, policy_per) # 현재 잔고 및 주식 보유 수량 init_cash, stock_cnt = env.init_cash, env.total_stock # 현재 주식 가격 cu_price = obs[1] # close quant = agent.decide_quant(action, value_per, policy_per, init_cash, cu_price) # 매도 매수 가능 한 경우 확인 action, quant = env.validation_(action, quant, cu_price, stock_cnt) next_obs, reward, done, info = env.next_step(action, quant) if next_obs is not None: data = next_obs.reshape(1, -1) data = pd.DataFrame(data, columns=df_prev.columns) df_prev = pd.concat([df_prev, data], ignore_index=True) next_obs = self.pre.add_feature(df_prev) reward_list.append(reward) action_List.append(action) quant_list.append(quant) stock_cnt_list.append(stock_cnt) obs_list.append(obs) obs = next_obs df_reward = pd.DataFrame(reward_list) df_action = pd.DataFrame(action_List) df_quant = pd.DataFrame(quant_list) df_stock_cnt = pd.DataFrame(stock_cnt_list) df_obs = pd.DataFrame(obs_list) df_reward.to_csv('.\\Test\\re_' + now + '\\reward.csv', index=False) df_action.to_csv('.\\Test\\re_' + now + '\\action.csv', index=False) df_quant.to_csv('.\\Test\\re_' + now + '\\quant.csv', index=False) df_stock_cnt.to_csv('.\\Test\\re_' + now + '\\stock_cnt.csv', index=False) df_obs.to_csv('.\\Test\\re_' + now + '\\obs.csv', index=False) self.logging.info(f'reward : {reward} | 테스트 끝')
import pandas as pd import numpy as np import RLEnvTrain, RLAgent import time from tqdm import tqdm df = pd.read_csv('.\\DB\\CSV\\daily\\DA000020_ch.csv') env = RLEnvTrain.RLEnv(df) agent = RLAgent.Agent() for k in range(100): obs = env.reset() for i in tqdm(range(1000)): quant = 1 # 매수매도 수량 action = agent.policy(obs) price = obs[1] if not env.validation_(action, quant, price): action = 0 quant = 0 next_obs, reward, done, info = env.next_step(action, quant) agent.memorize_transition(obs, action, reward, next_obs, 0.0 if done else 1.0) if agent.train: agent.experience_replay() if done: break
import numpy as np import RLEnvTrain, RLAgent from tqdm import tqdm from datetime import datetime os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import tensorflow as tf tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) df = pd.read_csv('.\\DB\\CSV\\daily\\DA000020_ch.csv') df['profit'] = 0 df_obs = df.iloc[119:int(len(df) * 0.6), :].copy() df_obs = df_obs.reset_index() df_obs = df_obs.drop(['index'], axis=1) env = RLEnvTrain.RLEnv(df_obs) agent = RLAgent.Agent(batch_size=400) reward_list = [] action_List = [] quant_list = [] re_list = [] for k in range(100): # 학습 날짜 시간 디렉토리 now = datetime.now() now = now.strftime('%Y%m%d_%H%M%S') if not os.path.isdir('.\\reward\\re_' + now + '\\'): os.mkdir('.\\reward\\re_' + now + '\\') obs = env.reset() sub_action_list = [] sub_quant_list = [] sub_re_list = []
import numpy as np import tensorflow as tf import config import csv import graph import RLAgent import httpConec2 as hC import matplotlib.pyplot as plt from matplotlib.pyplot import draw topo,hosts,nodes,hostnum,links=config.topo print "host num:",hostnum print "nodes num:",nodes ag=RLAgent.PGNAgent(links,(nodes-hostnum)*2+1) atest=graph.graf(nodes,links,initopo=topo,inihost=hosts) atest.initial() atest.printTopo() stepnum=2000 batch_size=config.batchsize batch_sum=0 episode_number=0 valid_action=0 total_episodes=config.episodes xs,ys,rs=[],[],[] epslon=config.explore_rate plt_ANPB=[] plt_ENPB=[] plt_RNPB=[] plt_rb=[] overlink_record=[] rsp=hC.sendTopo(atest.E,atest.host)
f'데이터셋 분리 Train : {len(df_train)} | Test : {len(df_test)} | Set-up : {len(df_prev)}학습을 시작합니다.' ) step = len(df_train) logging.info(f'한 학습당 step : {step}') #step = 10 logging.info(f'학습 환경 구성') env = RLEnvTrain.RLEnv(df_train) logging.info(f'학습 에이전트 구성') agent = RLAgent.Agent(gamma=0.98, eps_start=0.8, eps_end=0.01, eps_decay_steps=800, eps_exponential_decay=0.99, replay_capacity=int(1e6), batch_size=step - 1, tau=10, code=code, V_nn='DNN', P_nn='CNN', method='A2C', tick='m') #policy value A2C reward_list = [] action_List = [] quant_list = [] re_list = [] stock_cnt_list = [] logging.info('10번의 학습 시작') for k in range(5):
env.reset() now_obs = env.observe() # 초기화 후 관측치 print('reset_state', now_obs) for step in range(1, episode + 1): action = agent.eps_policy(now_obs) #행동 done, reward = agent.do_action(action, now_obs) # 액션 실행 next_obs = env.observe() agent.memorize(now_obs, action, reward, next_obs, 0.0 if done == 0 else 1.0) if agent.train: agent.experience_replay() if done == 0: break now_obs = next_obs end = time.time() print(end - start) return os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' df = pd.read_csv('.\\DB\\CSV\\daily\\DA000020.csv') df.sort_values(by='date', inplace=True) env = RLTrainEnv.RLtradingTestEnv(df) agent = RLAgent.RLAgent(env) agent.set_cash(1000000 * len(df)) train_model(env, episode=4096)
def main(): agent = RLAgent('agent', decisionFrequency=10.0, defaultSpeed=4, defaultAltitude=20, yawRate=70) # callbacks will be called in the order they were specified, beware of order of execution (if any state parameter is # dependant on another) # state is lazily updated by the environment as the agent needs it , agent always get the freshest estimate of the # state, state updates are done by the environment in a rate that corresponds to agent decision making freq. agent.defineState(orientation=getOrientation, angularVelocity=getAngularVelocity, linearVelocity=getVelocity, position=getPosition) agent.setRl(monteCarlo) agent.setReward(reward) agent.setGoal(position=np.array([-40, -50, 0])) agent.setGoalMargins(position=np.array([0.5, 0.5, math.inf])) agent.start() agent.join()