def act_on_environment(self, request, context):
if not env.feedback or not env.feedback[2]:
action = unpack_for_grpc(request.SerializedEntity)
env.next_score()
env.feedback = env.env.step(action)
if env.feedback[2]:
finalize(env)
return evaluation_pb2.Package(
SerializedEntity=pack_for_grpc({
"feedback": env.feedback,
"current_score": env.score,
}))
def act_on_environment(self, request, context):
global EVALUATION_COMPLETED
if not env.feedback or not env.feedback[2]:
action = unpack_for_grpc(request.SerializedEntity)
env.next_score()
env.feedback = env.env.step(action)
if env.feedback[2]:
if not LOCAL_EVALUATION:
update_submission_result(env, self.challenge_pk, self.phase_pk,
self.submission_pk)
else:
print("Final Score: {0}".format(env.score))
print("Stopping Evaluation!")
EVALUATION_COMPLETED = True
return evaluation_pb2.Package(
SerializedEntity=pack_for_grpc({
"feedback": env.feedback,
"current_score": env.score,
}))
def remote_ep_over(stub):
res_env = unpack_for_grpc(
stub.episode_over(evaluation_pb2.Package()).SerializedEntity
)
return res_env["episode_over"]
def remote_evaluate(
self, agent: Agent, num_episodes: Optional[int] = None
):
# The modules imported below are specific to habitat-challenge remote evaluation.
# These modules are not part of the habitat-api repository.
import evaluation_pb2
import evaluation_pb2_grpc
import evalai_environment_habitat
import grpc
import pickle
import time
time.sleep(60)
def pack_for_grpc(entity):
return pickle.dumps(entity)
def unpack_for_grpc(entity):
return pickle.loads(entity)
def remote_ep_over(stub):
res_env = unpack_for_grpc(
stub.episode_over(evaluation_pb2.Package()).SerializedEntity
)
return res_env["episode_over"]
env_address_port = os.environ.get("EVALENV_ADDPORT", "localhost:8085")
channel = grpc.insecure_channel(env_address_port)
stub = evaluation_pb2_grpc.EnvironmentStub(channel)
base_num_episodes = unpack_for_grpc(
stub.num_episodes(evaluation_pb2.Package()).SerializedEntity
)
num_episodes = base_num_episodes["num_episodes"]
agg_metrics: Dict = defaultdict(float)
count_episodes = 0
while count_episodes < num_episodes:
agent.reset()
res_env = unpack_for_grpc(
stub.reset(evaluation_pb2.Package()).SerializedEntity
)
while not remote_ep_over(stub):
obs = res_env["observations"]
action = agent.act(obs)
res_env = unpack_for_grpc(
stub.act_on_environment(
evaluation_pb2.Package(
SerializedEntity=pack_for_grpc(action)
)
).SerializedEntity
)
metrics = unpack_for_grpc(
stub.get_metrics(
evaluation_pb2.Package(
SerializedEntity=pack_for_grpc(action)
)
).SerializedEntity
)
for m, v in metrics["metrics"].items():
agg_metrics[m] += v
count_episodes += 1
avg_metrics = {k: v / count_episodes for k, v in agg_metrics.items()}
stub.evalai_update_submission(evaluation_pb2.Package())
return avg_metrics
def get_action_space(self, request, context):
challenge = habitat.Challenge()
agent = unpack_for_grpc(request.SerializedEntity)
challenge.submit(agent)
return evaluation_pb2.Package(SerializedEntity=agent)
def pack_for_grpc(entity):
return pickle.dumps(entity)
def unpack_for_grpc(entity):
return pickle.loads(entity)
done = None
while not done:
base = unpack_for_grpc(
stub.act_on_environment(
evaluation_pb2.Package(
SerializedEntity=pack_for_grpc(1))).SerializedEntity)
time.sleep(1)
done = base["feedback"][2]
print(base["feedback"])
print(done)
# a = stub.act_on_environment(evaluation_pb2.Package(SerializedEntity=pack_for_grpc("ABC")))
print(base)
# action_space = base.get("action_space")
# environment = base.get("environment")
# print(environment)
# feedback = unpack_for_grpc(
# stub.get_action_space(evaluation_pb2.Package(SerializedEntity="1".encode()))
# )
def get_action_space(self, request, context):
message = pack_for_grpc(env.get_action_space())
return evaluation_pb2.Package(SerializedEntity=message)
