def main():
client_id = Utils.connectToVREP()
# Define RL elements
environment = StandingUpEnvironment(client_id)
task = StandingUpTask(environment)
temperature = 2
ttable_path = os.path.join(BASE_DIR, 't-table.pkl')
qtable_path = os.path.join(BASE_DIR,
'q-table-{}.pkl'.format(Q_TABLE_VERSION))
dtmc_generator = DTMCGenerator(ttable_path, qtable_path, temperature)
dtmc_generator.load_policy(POLICY_PREFIX + 'sm10-policy.pkl', BASE_DIR)
with open(ttable_path, 'rb') as file:
ttable = pickle.load(file)
trans_prob_dict = {}
for key, value in ttable.items():
new_key = (key[0], key[1])
v = trans_prob_dict.get(new_key, [])
v.append((key[2], value))
trans_prob_dict[new_key] = v
while True:
n1 = dt.datetime.now()
state = task.getObservation()[0]
action = numpy.argmax(dtmc_generator.Q[state])
print('trans probs {}'.format(trans_prob_dict.get((state, action))))
action = select_action(dtmc_generator.policy, state, 'argmax')
print('State {} Action {} Prob {}'.format(
state, action, dtmc_generator.policy[state][action]))
print('pol trans probs {}'.format(trans_prob_dict.get(
(state, action))))
if action == 729:
task.reset()
else:
task.performAction(action)
successors = dtmc_generator.get_successor_states(state, action)
new_state = task.getObservation()[0]
found = False
for state, prob in successors:
if new_state == state:
print('{} state found, prob {}'.format(state, prob))
found = True
if not found:
print('{} state not found! successors: {}'.format(
new_state, successors))
n2 = dt.datetime.now()
print('elapsed time: {} s'.format((n2 - n1).microseconds / 1e6))
Utils.endVREP()
def run(self):
proc = Utils.exec_vrep(self.port)
time.sleep(10)
# connect to V-REP server
try:
client_id = Utils.connectToVREP(self.port)
env = StandingUpEnvironment(client_id)
task = StandingUpTask(env)
counter = 0
while counter < MAX_ITERATIONS:
state = task.getObservation()[0]
action = select_action(self.policy, state, 'prob')
# print('State {} Action {} Prob {}'.format(state, action, self.policy[state][action]))
if action == 729:
if state == task.state_mapper.goal_state:
self.logger.info('Goal!')
elif state == task.state_mapper.fallen_state:
self.logger.info('Fallen!')
elif state == task.state_mapper.too_far_state:
self.logger.info('Far!')
elif state == task.state_mapper.self_collided_state:
self.logger.info('Collided!')
else:
self.logger.info(state)
counter += 1
print('Iteration {}'.format(counter))
task.reset()
else:
task.performAction(action)
finally:
proc.kill()
def run(self):
try:
proc = Utils.exec_vrep(self.port)
time.sleep(60)
client_id = Utils.connectToVREP(self.port)
environment = StandingUpEnvironment(client_id)
task = StandingUpTask(environment)
for episode in range(self.n_episodes):
old_state = current_state = task.getObservation()[0]
action = self.select_action(self.policy, current_state)
print('State {} Action {} Prob {}'.format(
current_state, action, self.policy[current_state][action]))
task.performAction(action)
while action != 729:
old_state = current_state
# Test to verify Monitor capability,, #Pk:added more actions
#if action == 579 or action == 337:
# current_state = task.state_mapper.self_collided_state
#else:
current_state = task.getObservation()[0]
self.t_table.incrementValue(
(old_state, action, current_state))
action = self.select_action(self.policy, current_state)
print('State {} Action {} Prob {}'.format(
current_state, action,
self.policy[current_state][action]))
task.performAction(action)
task.reset()
if current_state == task.state_mapper.goal_state:
self.counters['goal'] += 1
elif current_state == task.state_mapper.fallen_state:
self.counters['fallen'] += 1
elif current_state == task.state_mapper.too_far_state:
self.counters['far'] += 1
elif current_state == task.state_mapper.self_collided_state:
self.counters['collided'] += 1
else:
self.counters['unknown'] += 1
finally:
self.barrier.wait()
Utils.endVREP(client_id)
proc.kill()
def main():
client_id = Utils.connectToVREP()
environment = StandingUpEnvironment(client_id)
task = StandingUpTask(environment)
#print('Initial State: ')
#print(environment.bioloid.read_state())
trajectory_data = []
for i in range(50):
print('Iteration {}'.format(i))
trajectory = []
for action in Utils.standingUpActions:
observation = task.getObservation()[0]
state_vector = task.env.bioloid.read_state()
action = Utils.vecToInt(action)
task.performAction(action)
reward = task.getReward()
trajectory.append({
'state': observation,
'state_vector': state_vector,
'action': action,
'reward': reward,
'full_state': task.env.bioloid.read_full_state()
})
trajectory_data.append(trajectory)
observation = task.getObservation()[0]
state_vector = task.env.bioloid.read_state()
trajectory.append({
'state': observation,
'state_vector': state_vector,
'action': -1,
'reward': 0,
'full_state': task.env.bioloid.read_full_state()
})
task.reset()
with open('../data/trajectory.pkl', 'wb') as file:
pickle.dump(trajectory_data, file)
Utils.endVREP()
def main():
client_id = Utils.connectToVREP()
environment = StandingUpEnvironment(client_id)
task = StandingUpTask(environment)
while True:
action_str = input("Insert next action: ")
action = [int(x) for x in action_str.split(' ')]
observation = task.getObservation()
print(task.current_sensors)
a = Utils.vecToInt(action)
task.performAction(a)
task.getReward()
print('self-collided: ' + str(environment.bioloid.is_fallen()))
print('is-fallen: ' + str(environment.bioloid.is_fallen()))
environment.reset()
def main():
client_id = Utils.connectToVREP()
environment = StandingUpEnvironment(client_id)
task = StandingUpTask(environment)
state_vector_length = len(environment.bioloid.read_state())
n = int(input('Number of iterations: '))
workbook = xlsxwriter.Workbook('data/reports/trajectory-trials.xls')
worksheets = []
for i in range(len(Utils.standingUpActions)):
worksheets.append(workbook.add_worksheet('t' + str(i + 1)))
for i in range(n):
print('Iteration ' + str(i + 1))
print('Initial State: ')
# task.getObservation()
print(task.state_mapper.sd.discretize(environment.getSensors()))
print(task.getObservation()[0])
for j, action in enumerate(Utils.standingUpActions):
environment.performAction(action)
state_vector = environment.getSensors()
discretized_state = task.state_mapper.sd.discretize(state_vector)
for k, s in enumerate(discretized_state):
worksheets[j].write(i, k, s)
state_n = task.update_current_state()
if state_n != task.state_mapper.goal_state:
state_distance = euclidean(
task.state_mapper.state_space[state_n], discretized_state)
else:
state_distance = 0
goal_distance = task.state_mapper.get_goal_distance(
discretized_state)
print(discretized_state)
print('---------------------')
worksheets[j].write(i, state_vector_length + 1, state_n)
worksheets[j].write(i, state_vector_length + 2, state_distance)
worksheets[j].write(i, state_vector_length + 3, goal_distance)
environment.reset()
res_worksheet = workbook.add_worksheet('Results')
row = 0
for i in range(len(Utils.standingUpActions)):
sheet_name = 't' + str(i + 1)
res_worksheet.write(row, 0, sheet_name)
res_worksheet.write(row, 1, 'mean')
res_worksheet.write(row + 1, 1, 'var')
for j in range(state_vector_length):
col_name = chr(ord('A') + j)
data_range = sheet_name + '.' + col_name + '1:' + col_name + str(n)
# TODO: check why range is made lowercase :/
res_worksheet.write_formula(row, 2 + j,
'=AVERAGE(' + data_range + ')')
res_worksheet.write_formula(row + 1, 2 + j,
'=VAR.P(' + data_range + ')')
row += 2
workbook.close()