def get_action(self, action_by_agent, event):
vessel_snapshots = self.env.snapshot_list["vessels"]
action_info = list(action_by_agent.values())[0]
model_action = action_info[0] if isinstance(action_info,
tuple) else action_info
scope, tick, port, vessel = event.action_scope, event.tick, event.port_idx, event.vessel_idx
zero_action_idx = len(
self.action_space) / 2 # index corresponding to value zero.
vessel_space = vessel_snapshots[tick:vessel:self.vessel_attributes][
2] if self.finite_vessel_space else float("inf")
early_discharge = vessel_snapshots[tick:vessel:"early_discharge"][
0] if self.has_early_discharge else 0
percent = abs(self.action_space[model_action])
if model_action < zero_action_idx:
action_type = ActionType.LOAD
actual_action = min(round(percent * scope.load), vessel_space)
elif model_action > zero_action_idx:
action_type = ActionType.DISCHARGE
plan_action = percent * (scope.discharge +
early_discharge) - early_discharge
actual_action = round(plan_action) if plan_action > 0 else round(
percent * scope.discharge)
else:
actual_action, action_type = 0, None
return {port: Action(vessel, port, actual_action, action_type)}
def __call__(self, model_action, decision_event, snapshot_list):
scope = decision_event.action_scope
tick = decision_event.tick
port_idx = decision_event.port_idx
vessel_idx = decision_event.vessel_idx
port_empty = snapshot_list["ports"][
tick:port_idx:["empty", "full", "on_shipper", "on_consignee"]][0]
vessel_remaining_space = snapshot_list["vessels"][
tick:vessel_idx:["empty", "full", "remaining_space"]][2]
early_discharge = snapshot_list["vessels"][
tick:vessel_idx:"early_discharge"][0]
assert 0 <= model_action < len(self._action_space)
if model_action < self._zero_action_index:
actual_action = max(
round(self._action_space[model_action] * port_empty),
-vessel_remaining_space)
elif model_action > self._zero_action_index:
plan_action = self._action_space[model_action] * (
scope.discharge + early_discharge) - early_discharge
actual_action = (round(plan_action) if plan_action > 0 else round(
self._action_space[model_action] * scope.discharge))
else:
actual_action = 0
return Action(vessel_idx, port_idx, actual_action)
def zero_play(**args):
env = Env(**args)
_, pa, is_done = env.step(None)
while not is_done:
action = Action(pa.vessel_idx, pa.port_idx, 0)
r, pa, is_done = env.step(action)
return env.snapshot_list
def decision_cnt_analysis(env, pv=False, buffer_size=8):
if not pv:
decision_cnt = [buffer_size] * len(env.node_name_mapping["static"])
r, pa, is_done = env.step(None)
while not is_done:
decision_cnt[pa.port_idx] += 1
action = Action(pa.vessel_idx, pa.port_idx, 0)
r, pa, is_done = env.step(action)
else:
decision_cnt = OrderedDict()
r, pa, is_done = env.step(None)
while not is_done:
if (pa.port_idx, pa.vessel_idx) not in decision_cnt:
decision_cnt[pa.port_idx, pa.vessel_idx] = buffer_size
else:
decision_cnt[pa.port_idx, pa.vessel_idx] += 1
action = Action(pa.vessel_idx, pa.port_idx, 0)
r, pa, is_done = env.step(action)
env.reset()
return decision_cnt
def random_shortage(env, tick, action_dim=21):
_, pa, is_done = env.step(None)
node_cnt = len(env.summary["node_mapping"]["ports"])
while not is_done:
"""
load, discharge = pa.action_scope.load, pa.action_scope.discharge
action_idx = np.random.randint(action_dim) - zero_idx
if action_idx < 0:
actual_action = int(1.0*action_idx/zero_idx*load)
else:
actual_action = int(1.0*action_idx/zero_idx*discharge)
"""
action = Action(pa.vessel_idx, pa.port_idx, 0)
r, pa, is_done = env.step(action)
shs = env.snapshot_list["ports"][tick -
1:list(range(node_cnt)):"acc_shortage"]
fus = env.snapshot_list["ports"][tick -
1:list(range(node_cnt)):"acc_fulfillment"]
env.reset()
return fus - shs, np.sum(shs + fus)
def test_early_discharge(self) -> None:
for backend_name in backends_to_test:
self._init_env(backend_name)
metric, decision_event, is_done = self._env.step(None)
assert isinstance(decision_event, DecisionEvent)
self.assertEqual(decision_event.action_scope.load, 1240)
self.assertEqual(decision_event.action_scope.discharge, 0)
self.assertEqual(decision_event.early_discharge, 0)
decision_event = pickle.loads(
pickle.dumps(decision_event)) # Test serialization
load_action = Action(vessel_idx=decision_event.vessel_idx,
port_idx=decision_event.port_idx,
quantity=1201,
action_type=ActionType.LOAD)
discharge_action = Action(vessel_idx=decision_event.vessel_idx,
port_idx=decision_event.port_idx,
quantity=1,
action_type=ActionType.DISCHARGE)
metric, decision_event, is_done = self._env.step(
[load_action, discharge_action])
history = []
while not is_done:
metric, decision_event, is_done = self._env.step(None)
assert decision_event is None or isinstance(
decision_event, DecisionEvent)
if decision_event is not None and decision_event.vessel_idx == 35:
v = self._business_engine._vessels[35]
history.append((v.full, v.empty, v.early_discharge))
hard_coded_benchmark = [(465, 838, 362), (756, 547, 291),
(1261, 42, 505), (1303, 0, 42),
(1303, 0, 0), (1303, 0, 0), (803, 0, 0)]
self.assertListEqual(history, hard_coded_benchmark)
#
payload_detail_benchmark = {
'ORDER': ['tick', 'src_port_idx', 'dest_port_idx', 'quantity'],
'RETURN_FULL': ['src_port_idx', 'dest_port_idx', 'quantity'],
'VESSEL_ARRIVAL': ['port_idx', 'vessel_idx'],
'LOAD_FULL': ['port_idx', 'vessel_idx'],
'DISCHARGE_FULL':
['vessel_idx', 'port_idx', 'from_port_idx', 'quantity'],
'PENDING_DECISION': [
'tick', 'port_idx', 'vessel_idx', 'snapshot_list',
'action_scope', 'early_discharge'
],
'LOAD_EMPTY':
['port_idx', 'vessel_idx', 'action_type', 'quantity'],
'DISCHARGE_EMPTY':
['port_idx', 'vessel_idx', 'action_type', 'quantity'],
'VESSEL_DEPARTURE': ['port_idx', 'vessel_idx'],
'RETURN_EMPTY': ['port_idx', 'quantity']
}
self.assertTrue(
compare_dictionary(
self._business_engine.get_event_payload_detail(),
payload_detail_benchmark))
port_number = self._business_engine._data_cntr.port_number
self.assertListEqual(self._business_engine.get_agent_idx_list(),
list(range(port_number)))
# Query environment summary, which includes business instances, intra-instance attributes, etc.
print(env.summary)
for ep in range(2):
# Gym-like step function.
metrics, decision_event, is_done = env.step(None)
while not is_done:
past_week_ticks = [
x for x in range(max(decision_event.tick -
7, 0), decision_event.tick)
]
decision_port_idx = decision_event.port_idx
intr_port_infos = ["booking", "empty", "shortage"]
# Query the decision port booking, empty container inventory, shortage information in the past week
past_week_info = env.snapshot_list["ports"][
past_week_ticks:decision_port_idx:intr_port_infos]
dummy_action = Action(decision_event.vessel_idx,
decision_event.port_idx, 0, ActionType.LOAD)
# Drive environment with dummy action (no repositioning)
metrics, decision_event, is_done = env.step(dummy_action)
# Query environment business metrics at the end of an episode,
# it is your optimized object (usually includes multi-target).
print(f"ep: {ep}, environment metrics: {env.metrics}")
env.reset()
# Usage:
# 1. Only push speicified (1st for this example) environment, leave others behind
# if decision_event:
# env0_dec: DecisionEvent = decision_event[0]
# # 1.1 After 1st environment is done, then others will push forward.
# if env0_dec:
# ss0 = env.snapshot_list["vessels"][env0_dec.tick:env0_dec.vessel_idx:"remaining_space"]
# action = {0: Action(env0_dec.vessel_idx, env0_dec.port_idx, -env0_dec.action_scope.load)}
# 2. Only pass action to 1st environment (give None to other environments),
# but keep pushing all the environment, until the end
if decision_event:
env0_dec: DecisionEvent = decision_event[0]
if env0_dec:
ss0 = env.snapshot_list["vessels"][env0_dec.tick:env0_dec.vessel_idx:"remaining_space"]
action = [None] * env.batch_number
# with a list of action, will push all environment to next step
action[0] = Action(env0_dec.vessel_idx, env0_dec.port_idx, -env0_dec.action_scope.load)
metrics, decision_event, is_done = env.step(action)
print("Final tick for each environment:", env.tick)
print("Final frame index for each environment:", env.frame_index)
env.reset()
def single_player_worker(index, config, exp_idx_mapping, pipe, action_io,
exp_output):
"""The A2C worker function to collect experience.
Args:
index (int): The process index counted from 0.
config (dict): It is a dottable dictionary that stores the configuration of the simulation, state_shaper and
postprocessing shaper.
exp_idx_mapping (dict): The key is agent code and the value is the starting index where the experience is stored
in the experience batch.
pipe (Pipe): The pipe instance for communication with the main process.
action_io (SharedStructure): The shared memory to hold the state information that the main process uses to
generate an action.
exp_output (SharedStructure): The shared memory to transfer the experience list to the main process.
"""
env = Env(**config.env.param)
fix_seed(env, config.env.seed)
static_code_list, dynamic_code_list = list(env.summary["node_mapping"]["ports"].values()), \
list(env.summary["node_mapping"]["vessels"].values())
# Create gnn_state_shaper without consuming any resources.
gnn_state_shaper = GNNStateShaper(
static_code_list,
dynamic_code_list,
config.env.param.durations,
config.model.feature,
tick_buffer=config.model.tick_buffer,
max_value=env.configs["total_containers"])
gnn_state_shaper.compute_static_graph_structure(env)
action_io_np = action_io.structuralize()
action_shaper = DiscreteActionShaper(config.model.action_dim)
exp_shaper = ExperienceShaper(static_code_list,
dynamic_code_list,
config.env.param.durations,
gnn_state_shaper,
scale_factor=config.env.return_scaler,
time_slot=config.training.td_steps,
discount_factor=config.training.gamma,
idx=index,
shared_storage=exp_output.structuralize(),
exp_idx_mapping=exp_idx_mapping)
i = 0
while pipe.recv() == "reset":
env.reset()
r, decision_event, is_done = env.step(None)
j = 0
logs = []
while not is_done:
model_input = gnn_state_shaper(decision_event, env.snapshot_list)
action_io_np["v"][:, index] = model_input["v"]
action_io_np["p"][:, index] = model_input["p"]
action_io_np["vo"][index] = model_input["vo"]
action_io_np["po"][index] = model_input["po"]
action_io_np["vedge"][index] = model_input["vedge"]
action_io_np["pedge"][index] = model_input["pedge"]
action_io_np["ppedge"][index] = model_input["ppedge"]
action_io_np["mask"][index] = model_input["mask"]
action_io_np["pid"][index] = decision_event.port_idx
action_io_np["vid"][index] = decision_event.vessel_idx
pipe.send("features")
model_action = pipe.recv()
env_action = action_shaper(decision_event, model_action)
exp_shaper.record(decision_event=decision_event,
model_action=model_action,
model_input=model_input)
logs.append([
index, decision_event.tick, decision_event.port_idx,
decision_event.vessel_idx, model_action, env_action,
decision_event.action_scope.load,
decision_event.action_scope.discharge
])
action = Action(decision_event.vessel_idx, decision_event.port_idx,
env_action)
r, decision_event, is_done = env.step(action)
j += 1
action_io_np["sh"][index] = compute_shortage(
env.snapshot_list, config.env.param.durations, static_code_list)
i += 1
pipe.send("done")
gnn_state_shaper.end_ep_callback(env.snapshot_list)
# Organize and synchronize exp to shared memory.
exp_shaper(env.snapshot_list)
exp_shaper.reset()
logs = np.array(logs, dtype=np.float)
pipe.send(logs)
durations=durations)
# Query environment summary, which includes business instances, intra-instance attributes, etc.
print(env.summary)
for ep in range(2):
# Gym-like step function
metrics, decision_event, is_done = env.step(None)
while not is_done:
past_week_ticks = [
x for x in range(decision_event.tick - 7, decision_event.tick)
]
decision_port_idx = decision_event.port_idx
intr_port_infos = ["booking", "empty", "shortage"]
# Query the decision port booking, empty container inventory, shortage information in the past week
past_week_info = env.snapshot_list["ports"][
past_week_ticks:decision_port_idx:intr_port_infos]
dummy_action = Action(decision_event.vessel_idx,
decision_event.port_idx, 0)
# Drive environment with dummy action (no repositioning)
metrics, decision_event, is_done = env.step(dummy_action)
# Query environment business metrics at the end of an episode,
# it is your optimized object (usually includes multi-target).
print(f"ep: {ep}, environment metrics: {env.metrics}")
env.reset()
# To reset environmental data before starting a new experiment.
env.reset()
for ep in range(NUM_EPISODE):
# Tell streamit we are in a new episode.
streamit.episode(ep)
# Gym-like step function.
metrics, decision_event, is_done = env.step(None)
while not is_done:
action_scope = decision_event.action_scope
to_discharge = action_scope.discharge > 0 and randint(0, 1) > 0
random_action = Action(
decision_event.vessel_idx, decision_event.port_idx,
randint(
0, action_scope.discharge
if to_discharge else action_scope.load),
ActionType.DISCHARGE if to_discharge else ActionType.LOAD)
# Drive environment with dummy action (no repositioning)
metrics, decision_event, is_done = env.step(random_action)
# Query environment business metrics at the end of an episode,
# it is your optimized object (usually includes multi-target).
print(f"ep: {ep}, environment metrics: {env.metrics}")
env.reset()