def power(x, n):
"""Calculate exponentiation of `x` raised to the `n` power.
Args:
x (number): Base number.
n (number): Exponent.
Returns:
number: Result of calculation.
Example:
>>> power(5, 2)
25
>>> power(12.5, 3)
1953.125
See Also:
- :func:`power` (main definition)
- :func:`pow_` (alias)
.. versionadded:: 2.1.0
"""
if pyd.is_number(x):
result = pow(x, n)
elif pyd.is_list(x):
result = pyd.map_(x, lambda item: pow(item, n))
else:
result = None
return result
def __init__(self,
spec,
info_space,
body,
a=None,
agent_space=None,
global_nets=None):
self.spec = spec
self.info_space = info_space
self.a = a or 0 # for compatibility with agent_space
self.agent_spec = spec['agent'][self.a]
self.name = self.agent_spec['name']
assert not ps.is_list(
global_nets
), f'single agent global_nets must be a dict, got {global_nets}'
if agent_space is None: # singleton mode
self.body = body
body.agent = self
MemoryClass = getattr(memory, ps.get(self.agent_spec,
'memory.name'))
self.body.memory = MemoryClass(self.agent_spec['memory'],
self.body)
AlgorithmClass = getattr(algorithm,
ps.get(self.agent_spec, 'algorithm.name'))
self.algorithm = AlgorithmClass(self, global_nets)
else:
self.space_init(agent_space, body, global_nets)
logger.info(util.self_desc(self))
def resolve_aeb(spec):
'''
Resolve an experiment spec into the full list of points (coordinates) in AEB space.
@param {dict} spec An experiment spec.
@returns {list} aeb_list Resolved array of points in AEB space.
@example
spec = spec_util.get('base.json', 'general_inner')
aeb_list = spec_util.resolve_aeb(spec)
# => [(0, 0, 0), (0, 0, 1), (1, 1, 0), (1, 1, 1)]
'''
agent_num = len(spec['agent'])
env_num = len(spec['env'])
ae_product = _.get(spec, 'body.product')
body_num = _.get(spec, 'body.num')
body_num_list = body_num if _.is_list(body_num) else [body_num] * env_num
aeb_list = []
if ae_product == 'outer':
for e in range(env_num):
sub_aeb_list = list(itertools.product(
range(agent_num), [e], range(body_num_list[e])))
aeb_list.extend(sub_aeb_list)
elif ae_product == 'inner':
for a, e in zip(range(agent_num), range(env_num)):
sub_aeb_list = list(
itertools.product([a], [e], range(body_num_list[e])))
aeb_list.extend(sub_aeb_list)
else: # custom AEB, body_num is a aeb_list
aeb_list = [tuple(aeb) for aeb in body_num]
aeb_list.sort()
assert is_aeb_compact(
aeb_list), 'Failed check: for a, e, uniq count == len (shape), and for each a,e hash, b uniq count == b len (shape)'
return aeb_list
def resolve_aeb(spec):
'''
Resolve an experiment spec into the full list of points (coordinates) in AEB space.
@param {dict} spec An experiment spec.
@returns {list} aeb_list Resolved array of points in AEB space.
@example
spec = spec_util.get('base.json', 'general_inner')
aeb_list = spec_util.resolve_aeb(spec)
# => [(0, 0, 0), (0, 0, 1), (1, 1, 0), (1, 1, 1)]
'''
agent_num = len(spec['agent'])
env_num = len(spec['env'])
ae_product = ps.get(spec, 'body.product')
body_num = ps.get(spec, 'body.num')
body_num_list = body_num if ps.is_list(body_num) else [body_num] * env_num
aeb_list = []
if ae_product == 'outer':
for e in range(env_num):
sub_aeb_list = list(itertools.product(range(agent_num), [e], range(body_num_list[e])))
aeb_list.extend(sub_aeb_list)
elif ae_product == 'inner':
for a, e in zip(range(agent_num), range(env_num)):
sub_aeb_list = list(itertools.product([a], [e], range(body_num_list[e])))
aeb_list.extend(sub_aeb_list)
else: # custom AEB, body_num is a aeb_list
aeb_list = [tuple(aeb) for aeb in body_num]
aeb_list.sort()
assert is_aeb_compact(aeb_list), 'Failed check: for a, e, uniq count == len (shape), and for each a,e hash, b uniq count == b len (shape)'
return aeb_list
def power(x, n):
"""Calculate exponentiation of `x` raised to the `n` power.
Args:
x (number): Base number.
n (number): Exponent.
Returns:
number: Result of calculation.
Example:
>>> power(5, 2)
25
>>> power(12.5, 3)
1953.125
See Also:
- :func:`power` (main definition)
- :func:`pow_` (alias)
.. versionadded:: 2.1.0
"""
if pyd.is_number(x):
result = pow(x, n)
elif pyd.is_list(x):
result = pyd.map_(x, lambda item: pow(item, n))
else:
result = None
return result
def round_(x, precision=0):
"""Round number to precision.
Args:
x (number): Number to round.
precision (int, optional): Rounding precision. Defaults to ``0``.
Returns:
int: Rounded number.
Example:
>>> round_(3.275) == 3.0
True
>>> round_(3.275, 1) == 3.3
True
See Also:
- :func:`round_` (main definition)
- :func:`curve` (alias)
.. versionadded:: 2.1.0
"""
rounder = pyd.partial_right(round, precision)
if pyd.is_number(x):
result = rounder(x)
elif pyd.is_list(x):
# pylint: disable=unnecessary-lambda
result = pyd.map_(x, lambda item: rounder(item))
else:
result = None
return result
def build_model_tails(self, out_dim, out_layer_activation):
'''Build each model_tail. These are stored as Sequential models in model_tails'''
if not ps.is_list(out_layer_activation):
out_layer_activation = [out_layer_activation] * len(out_dim)
model_tails = nn.ModuleList()
if ps.is_empty(self.tail_hid_layers):
for out_d, out_activ in zip(out_dim, out_layer_activation):
tail = net_util.build_fc_model(
[self.body_hid_layers[-1], out_d], out_activ)
model_tails.append(tail)
else:
assert len(self.tail_hid_layers) == len(
out_dim
), 'Hydra tail hid_params inconsistent with number out dims'
for out_d, out_activ, hid_layers in zip(out_dim,
out_layer_activation,
self.tail_hid_layers):
dims = hid_layers
model_tail = net_util.build_fc_model(
dims, self.hid_layers_activation)
tail_out = net_util.build_fc_model([dims[-1], out_d],
out_activ)
model_tail.add_module(str(len(model_tail)), tail_out)
model_tails.append(model_tail)
return model_tails
def power(x, n):
"""Calculate exponentiation of `x` raised to the `n` power.
Args:
x (number): Base number.
n (number): Exponent.
Returns:
number: Result of calculation.
Example:
>>> power(5, 2)
25
>>> power(12.5, 3)
1953.125
.. versionadded:: 2.1.0
.. versionchanged:: 4.0.0
Removed alias ``pow_``.
"""
if pyd.is_number(x):
result = pow(x, n)
elif pyd.is_list(x):
result = [pow(item, n) for item in x]
else:
result = None
return result
def __init__(self, spec, aeb_space, global_nets=None):
self.spec = spec
self.aeb_space = aeb_space
aeb_space.agent_space = self
self.info_space = aeb_space.info_space
self.aeb_shape = aeb_space.aeb_shape
assert not ps.is_dict(
global_nets
), f'multi agent global_nets must be a list of dicts, got {global_nets}'
assert ps.is_list(self.spec['agent'])
self.agents = []
for a in range(len(self.spec['agent'])):
body_a = self.aeb_space.body_space.get(a=a)
if global_nets is not None:
agent_global_nets = global_nets[a]
else:
agent_global_nets = None
agent = Agent(self.spec,
self.info_space,
body=body_a,
a=a,
agent_space=self,
global_nets=agent_global_nets)
self.agents.append(agent)
logger.info(util.self_desc(self))
def to_torch_batch(batch, device, is_episodic):
'''Mutate a batch (dict) to make its values from numpy into PyTorch tensor'''
for k in batch:
if is_episodic: # for episodic format
batch[k] = np.concatenate(batch[k])
elif ps.is_list(batch[k]):
batch[k] = np.array(batch[k])
batch[k] = torch.from_numpy(batch[k].astype(np.float32)).to(device)
return batch
def check_comp_spec(comp_spec, comp_spec_format):
'''Base method to check component spec'''
for spec_k, spec_format_v in comp_spec_format.items():
comp_spec_v = comp_spec[spec_k]
if ps.is_list(spec_format_v):
v_set = spec_format_v
assert comp_spec_v in v_set, f'Component spec value {ps.pick(comp_spec, spec_k)} needs to be one of {util.to_json(v_set)}'
else:
v_type = spec_format_v
assert isinstance(comp_spec_v, v_type), f'Component spec {ps.pick(comp_spec, spec_k)} needs to be of type: {v_type}'
def get_out_dim(body, add_critic=False):
'''Construct the NetClass out_dim for a body according to is_discrete, action_type, and whether to add a critic unit'''
policy_out_dim = get_policy_out_dim(body)
if add_critic:
if ps.is_list(policy_out_dim):
out_dim = policy_out_dim + [1]
else:
out_dim = [policy_out_dim, 1]
else:
out_dim = policy_out_dim
return out_dim
def check_comp_spec(comp_spec, comp_spec_format):
'''Base method to check component spec'''
for spec_k, spec_format_v in comp_spec_format.items():
comp_spec_v = comp_spec[spec_k]
if _.is_list(spec_format_v):
v_set = spec_format_v
assert comp_spec_v in v_set, f'Component spec value {_.pick(comp_spec, spec_k)} needs to be one of {util.to_json(v_set)}'
else:
v_type = spec_format_v
assert isinstance(
comp_spec_v, v_type), f'Component spec {_.pick(comp_spec, spec_k)} needs to be of type: {v_type}'
def guard_multi_pdparams(pdparams, body):
'''Guard pdparams for multi action'''
action_dim = body.action_dim
is_multi_action = ps.is_iterable(action_dim)
if is_multi_action:
assert ps.is_list(pdparams)
pdparams = [t.clone() for t in pdparams] # clone for grad safety
assert len(pdparams) == len(action_dim), pdparams
# transpose into (batch_size, [action_dims])
pdparams = [list(torch.split(t, action_dim, dim=0)) for t in torch.cat(pdparams, dim=1)]
return pdparams
def get_policy_out_dim(body):
'''Helper method to construct the policy network out_dim for a body according to is_discrete, action_type'''
if body.is_discrete:
if body.action_type == 'multi_discrete':
assert ps.is_list(body.action_dim), body.action_dim
policy_out_dim = body.action_dim
else:
assert ps.is_integer(body.action_dim), body.action_dim
policy_out_dim = body.action_dim
else:
if body.action_type == 'multi_continuous':
assert ps.is_list(body.action_dim), body.action_dim
raise NotImplementedError('multi_continuous not supported yet')
else:
assert ps.is_integer(body.action_dim), body.action_dim
if body.action_dim == 1:
policy_out_dim = 2 # singleton stay as int
else:
policy_out_dim = body.action_dim * [2]
return policy_out_dim
def check_body_spec(spec):
'''Base method to check body spec for multi-agent multi-env'''
ae_product = ps.get(spec, 'body.product')
body_num = ps.get(spec, 'body.num')
if ae_product == 'outer':
pass
elif ae_product == 'inner':
agent_num = len(spec['agent'])
env_num = len(spec['env'])
assert agent_num == env_num, 'Agent and Env spec length must be equal for body `inner` product. Given {agent_num}, {env_num}'
else: # custom
assert ps.is_list(body_num)
def check_body_spec(spec):
'''Base method to check body spec for AEB space resolution'''
ae_product = ps.get(spec, 'body.product')
body_num = ps.get(spec, 'body.num')
if ae_product == 'outer':
pass
elif ae_product == 'inner':
agent_num = len(spec['agent'])
env_num = len(spec['env'])
assert agent_num == env_num, 'Agent and Env spec length must be equal for body `inner` product. Given {agent_num}, {env_num}'
else: # custom AEB
assert ps.is_list(body_num)
def check_comp_spec(comp_spec, comp_spec_format):
'''Base method to check component spec'''
for spec_k, spec_format_v in comp_spec_format.items():
comp_spec_v = comp_spec[spec_k]
if ps.is_list(spec_format_v):
v_set = spec_format_v
assert comp_spec_v in v_set, f'Component spec value {ps.pick(comp_spec, spec_k)} needs to be one of {util.to_json(v_set)}'
else:
v_type = spec_format_v
assert isinstance(comp_spec_v, v_type), f'Component spec {ps.pick(comp_spec, spec_k)} needs to be of type: {v_type}'
if isinstance(v_type, tuple) and int in v_type and isinstance(comp_spec_v, float):
# cast if it can be int
comp_spec[spec_k] = int(comp_spec_v)
def update_path(obj, callback, keys, default=None):
"""Update the value of an object described by `keys` using `callback`. If
any part of the object path doesn't exist, it will be created with
`default`. The callback is invoked with the last key value of `obj`:
``(value)``
Args:
obj (list|dict): Object to modify.
callback (callable): Function that returns updated value.
keys (list): A list of string keys that describe the object path to
modify.
default (mixed, optional): Default value to assign if path part is not
set. Defaults to ``{}`` if `obj` is a ``dict`` or ``[]`` if `obj`
is a ``list``.
Returns:
mixed: Updated `obj`.
Example:
>>> update_path({}, lambda value: value, ['a', 'b'])
{'a': {'b': None}}
>>> update_path([], lambda value: value, [0, 0])
[[None]]
.. versionadded:: 2.0.0
"""
# pylint: disable=redefined-outer-name
if default is None:
default = {} if isinstance(obj, dict) else []
if not pyd.is_list(keys):
keys = [keys]
last_key = pyd.last(keys)
obj = clone_deep(obj)
target = obj
for key in pyd.initial(keys):
set_item(target, key, clone_deep(default), allow_override=False)
try:
target = target[key]
except TypeError:
target = target[int(key)]
set_item(target, last_key, callback(get_item(target,
last_key,
default=None)))
return obj
def update_path(obj, callback, keys, default=None):
"""Update the value of an object described by `keys` using `callback`. If
any part of the object path doesn't exist, it will be created with
`default`. The callback is invoked with the last key value of `obj`:
``(value)``
Args:
obj (list|dict): Object to modify.
callback (function): Function that returns updated value.
keys (list): A list of string keys that describe the object path to
modify.
default (mixed, optional): Default value to assign if path part is not
set. Defaults to ``{}`` if `obj` is a ``dict`` or ``[]`` if `obj`
is a ``list``.
Returns:
mixed: Updated `obj`.
Example:
>>> update_path({}, lambda value: value, ['a', 'b'])
{'a': {'b': None}}
>>> update_path([], lambda value: value, [0, 0])
[[None]]
.. versionadded:: 2.0.0
"""
# pylint: disable=redefined-outer-name
if default is None:
default = {} if isinstance(obj, dict) else []
if not pyd.is_list(keys):
keys = [keys]
last_key = pyd.last(keys)
obj = clone_deep(obj)
target = obj
for key in pyd.initial(keys):
set_item(target, key, clone_deep(default), allow_override=False)
try:
target = target[key]
except TypeError:
target = target[int(key)]
set_item(target, last_key, callback(get_item(target,
last_key,
default=None)))
return obj
def __init__(self, spec, body, global_nets=None):
self.spec = spec
self.agent_spec = spec['agent'][0] # idx 0 for single-agent
self.name = self.agent_spec['name']
assert not ps.is_list(global_nets), f'single agent global_nets must be a dict, got {global_nets}'
# set components
self.body = body
body.agent = self
MemoryClass = getattr(memory, ps.get(self.agent_spec, 'memory.name'))
self.body.memory = MemoryClass(self.agent_spec['memory'], self.body)
AlgorithmClass = getattr(algorithm, ps.get(self.agent_spec, 'algorithm.name'))
self.algorithm = AlgorithmClass(self, global_nets)
logger.info(util.self_desc(self))
def __init__(self, spec, body, a=None, global_nets=None):
self.spec = spec
self.a = a or 0 # for compatibility with agent_space
self.agent_spec = spec['agent'][self.a]
self.name = self.agent_spec['name']
assert not ps.is_list(
global_nets
), f'single agent global_nets must be a dict, got {global_nets}'
self.nlu = None
if 'nlu' in self.agent_spec:
params = deepcopy(ps.get(self.agent_spec, 'nlu'))
NluClass = getattr(nlu, params.pop('name'))
self.nlu = NluClass(**params)
self.dst = None
if 'dst' in self.agent_spec:
params = deepcopy(ps.get(self.agent_spec, 'dst'))
DstClass = getattr(dst, params.pop('name'))
self.dst = DstClass(**params)
if 'word_dst' in self.agent_spec:
params = deepcopy(ps.get(self.agent_spec, 'word_dst'))
DstClass = getattr(word_dst, params.pop('name'))
self.dst = DstClass(**params)
self.state_encoder = None
if 'state_encoder' in self.agent_spec:
params = deepcopy(ps.get(self.agent_spec, 'state_encoder'))
StateEncoderClass = getattr(state_encoder, params.pop('name'))
self.state_encoder = StateEncoderClass(**params)
self.action_decoder = None
if 'action_decoder' in self.agent_spec:
params = deepcopy(ps.get(self.agent_spec, 'action_decoder'))
ActionDecoderClass = getattr(action_decoder, params.pop('name'))
self.action_decoder = ActionDecoderClass(**params)
self.nlg = None
if 'nlg' in self.agent_spec:
params = deepcopy(ps.get(self.agent_spec, 'nlg'))
NlgClass = getattr(nlg, params.pop('name'))
self.nlg = NlgClass(**params)
self.body = body
body.agent = self
AlgorithmClass = getattr(algorithm,
ps.get(self.agent_spec, 'algorithm.name'))
self.algorithm = AlgorithmClass(self, global_nets)
if ps.get(self.agent_spec, 'memory'):
MemoryClass = getattr(memory, ps.get(self.agent_spec,
'memory.name'))
self.body.memory = MemoryClass(self.agent_spec['memory'],
self.body)
self.warmup_epi = ps.get(self.agent_spec, 'algorithm.warmup_epi') or -1
self.body.state, self.body.encoded_state, self.body.action = None, None, None
logger.info(util.self_desc(self))
def calc_pdparam(self, x, net=None):
'''
The pdparam will be the logits for discrete prob. dist., or the mean and std for continuous prob. dist.
'''
out = super().calc_pdparam(x, net=net)
if self.shared:
assert ps.is_list(out), f'Shared output should be a list [pdparam, v]'
if len(out) == 2: # single policy
pdparam = out[0]
else: # multiple-task policies, still assumes 1 value
pdparam = out[:-1]
self.v_pred = out[-1].view(-1) # cache for loss calc to prevent double-pass
else: # out is pdparam
pdparam = out
return pdparam
def flatten_dict(obj, delim='.'):
'''Missing pydash method to flatten dict'''
nobj = {}
for key, val in obj.items():
if ps.is_dict(val) and not ps.is_empty(val):
strip = flatten_dict(val, delim)
for k, v in strip.items():
nobj[key + delim + k] = v
elif ps.is_list(val) and not ps.is_empty(val) and ps.is_dict(val[0]):
for idx, v in enumerate(val):
nobj[key + delim + str(idx)] = v
if ps.is_object(v):
nobj = flatten_dict(nobj, delim)
else:
nobj[key] = val
return nobj
def __init__(self, agent, global_nets=None):
'''
@param {*} agent is the container for algorithm and related components, and interfaces with env.
'''
self.agent = agent
if ps.is_list(global_nets): # multiagent
self.global_nets = global_nets[agent.a]
else:
self.global_nets = global_nets or {}
self.algorithm_spec = agent.agent_spec['algorithm']
self.name = self.algorithm_spec['name']
self.memory_spec = agent.agent_spec['memory']
self.net_spec = agent.agent_spec['net']
self.body = self.agent.body
self.init_algorithm_params()
self.init_nets()
logger.info(util.self_desc(self))
def build_conv_layers(self, conv_hid_layers):
'''
Builds all of the convolutional layers in the network and store in a Sequential model
'''
conv_layers = []
in_d = 1 # input channel
for i, hid_layer in enumerate(conv_hid_layers):
hid_layer = [tuple(e) if ps.is_list(e) else e for e in hid_layer] # guard list-to-tuple
# hid_layer = out_d, kernel, stride, padding, dilation
conv_layers.append(nn.Conv2d(in_d, *hid_layer))
conv_layers.append(net_util.get_activation_fn(self.hid_layers_activation))
# Don't include batch norm in the first layer
if self.batch_norm and i != 0:
conv_layers.append(nn.BatchNorm2d(in_d))
in_d = hid_layer[0] # update to out_d
conv_model = nn.Sequential(*conv_layers)
return conv_model
def get_policy_out_dim(body):
'''Helper method to construct the policy network out_dim for a body according to is_discrete, action_type'''
action_dim = body.action_dim
if body.is_discrete:
if body.action_type == 'multi_discrete':
assert ps.is_list(action_dim), action_dim
policy_out_dim = action_dim
else:
assert ps.is_integer(action_dim), action_dim
policy_out_dim = action_dim
else:
assert ps.is_integer(action_dim), action_dim
if action_dim == 1: # single action, use [loc, scale]
policy_out_dim = 2
else: # multi-action, use [locs], [scales]
policy_out_dim = [action_dim, action_dim]
return policy_out_dim
def __init__(self, spec, aeb_space, global_nets=None):
self.spec = spec
self.aeb_space = aeb_space
aeb_space.agent_space = self
self.info_space = aeb_space.info_space
self.aeb_shape = aeb_space.aeb_shape
assert ps.is_list(self.spec['agent'])
self.agents = []
for a in range(len(self.spec['agent'])):
body_a = self.aeb_space.body_space.get(a=a)
agent = Agent(self.spec,
self.info_space,
body=body_a,
a=a,
agent_space=self,
global_nets=global_nets)
self.agents.append(agent)
logger.info(util.self_desc(self))
def run_trial_test_dist(spec_file, spec_name=False):
spec = spec_util.get(spec_file, spec_name)
spec = spec_util.override_spec(spec, 'test')
spec_util.tick(spec, 'trial')
spec['meta']['distributed'] = 'synced'
spec['meta']['max_session'] = 2
trial = Trial(spec)
# manually run the logic to obtain global nets for testing to ensure global net gets updated
global_nets = trial.init_global_nets()
# only test first network
if ps.is_list(global_nets): # multiagent only test first
net = list(global_nets[0].values())[0]
else:
net = list(global_nets.values())[0]
session_metrics_list = trial.parallelize_sessions(global_nets)
trial_metrics = analysis.analyze_trial(spec, session_metrics_list)
trial.close()
assert isinstance(trial_metrics, dict)
def call_hook(self, path, *args):
f = self.get_hook(path)
if f is None:
return []
#TODO implement here
if pydash.is_list(f):
def cb(h):
pass
results = pydash.map_(f, cb)
if type(f) is 'function':
return []
else:
print('unknown hook type')
return []
def run_trial_test_dist(spec_file, spec_name=False):
spec = spec_util.get(spec_file, spec_name)
spec = spec_util.override_test_spec(spec)
info_space = InfoSpace()
info_space.tick('trial')
spec['meta']['distributed'] = True
spec['meta']['max_session'] = 2
trial = Trial(spec, info_space)
# manually run the logic to obtain global nets for testing to ensure global net gets updated
global_nets = trial.init_global_nets()
# only test first network
if ps.is_list(global_nets): # multiagent only test first
net = list(global_nets[0].values())[0]
else:
net = list(global_nets.values())[0]
session_datas = trial.parallelize_sessions(global_nets)
trial.session_data_dict = {data.index[0]: data for data in session_datas}
trial_data = analysis.analyze_trial(trial)
trial.close()
assert isinstance(trial_data, pd.DataFrame)
def __init__(self, spec, info_space, global_nets=None):
self.spec = spec
self.info_space = info_space
self.index = self.info_space.get('session')
util.set_session_logger(self.spec, self.info_space, logger)
self.data = None
# init singleton agent and env
self.env = make_env(self.spec)
body = Body(self.env, self.spec['agent'])
util.set_rand_seed(self.info_space.get_random_seed(), self.env)
util.try_set_cuda_id(self.spec, self.info_space)
assert not ps.is_list(
global_nets
), f'single agent global_nets must be a dict, got {global_nets}'
self.agent = Agent(self.spec,
self.info_space,
body=body,
global_nets=global_nets)
enable_aeb_space(self) # to use lab's data analysis framework
logger.info(util.self_desc(self))
logger.info(f'Initialized session {self.index}')
def init_nets(self):
'''
Initialize the neural networks used to learn the actor and critic from the spec
Below we automatically select an appropriate net based on two different conditions
1. If the action space is discrete or continuous action
- Networks for continuous action spaces have two heads and return two values, the first is a tensor containing the mean of the action policy, the second is a tensor containing the std deviation of the action policy. The distribution is assumed to be a Gaussian (Normal) distribution.
- Networks for discrete action spaces have a single head and return the logits for a categorical probability distribution over the discrete actions
2. If the actor and critic are separate or share weights
- If the networks share weights then the single network returns a list.
- Continuous action spaces: The return list contains 3 elements: The first element contains the mean output for the actor (policy), the second element the std dev of the policy, and the third element is the state-value estimated by the network.
- Discrete action spaces: The return list contains 2 element. The first element is a tensor containing the logits for a categorical probability distribution over the actions. The second element contains the state-value estimated by the network.
3. If the network type is feedforward, convolutional, or recurrent
- Feedforward and convolutional networks take a single state as input and require an OnPolicyReplay or OnPolicyBatchReplay memory
- Recurrent networks take n states as input and require an OnPolicySeqReplay or OnPolicySeqBatchReplay memory
'''
net_type = self.net_spec['type']
# options of net_type are {MLPNet, ConvNet, RecurrentNet} x {Shared, Separate}
in_dim = self.body.state_dim
if self.body.is_discrete:
if 'Shared' in net_type:
self.share_architecture = True
out_dim = [self.body.action_dim, 1]
else:
assert 'Separate' in net_type
self.share_architecture = False
out_dim = self.body.action_dim
critic_out_dim = 1
else:
if 'Shared' in net_type:
self.share_architecture = True
out_dim = [self.body.action_dim, self.body.action_dim, 1]
else:
assert 'Separate' in net_type
self.share_architecture = False
out_dim = [self.body.action_dim, self.body.action_dim]
critic_out_dim = 1
self.net_spec['type'] = net_type = net_type.replace('Shared', '').replace('Separate', '')
if 'MLP' in net_type and ps.is_list(out_dim) and len(out_dim) > 1:
self.net_spec['type'] = 'MLPHeterogenousTails'
actor_net_spec = self.net_spec.copy()
critic_net_spec = self.net_spec.copy()
for k in self.net_spec:
if 'actor_' in k:
actor_net_spec[k.replace('actor_', '')] = actor_net_spec.pop(k)
critic_net_spec.pop(k)
if 'critic_' in k:
critic_net_spec[k.replace('critic_', '')] = critic_net_spec.pop(k)
actor_net_spec.pop(k)
NetClass = getattr(net, self.net_spec['type'])
# properly set net_spec and action_dim for actor, critic nets
if self.share_architecture:
# net = actor_critic as one
self.net = NetClass(actor_net_spec, self, in_dim, out_dim)
self.net_names = ['net']
else:
# main net = actor
self.net = NetClass(actor_net_spec, self, in_dim, out_dim)
if critic_net_spec['use_same_optim']:
critic_net_spec = actor_net_spec
self.critic = NetClass(critic_net_spec, self, in_dim, critic_out_dim)
self.net_names = ['net', 'critic']
self.post_init_nets()
def update_with(obj, path, updater, customizer=None):
"""This method is like :func:`update` except that it accepts customizer
which is invoked to produce the objects of path. If customizer returns
``None``, path creation is handled by the method instead. The customizer is
invoked with three arguments: ``(nested_value, key, nested_object)``.
Args:
obj (list|dict): Object to modify.
path (str|list): A string or list of keys that describe the object path
to modify.
updater (function): Function that returns updated value.
customizer (function, optional): The function to customize assigned
values.
Returns:
mixed: Updated `obj`.
Warning:
`obj` is modified in place.
Example:
>>> update_with({}, '[0][1]', lambda: 'a', lambda: {})
{0: {1: 'a'}}
.. versionadded:: 4.0.0
"""
if not callable(updater):
updater = pyd.constant(updater)
if customizer is not None and not callable(customizer):
call_customizer = partial(callit, clone, customizer, argcount=1)
elif customizer:
call_customizer = partial(callit, customizer,
argcount=getargcount(customizer, maxargs=3))
else:
call_customizer = None
default_type = dict if isinstance(obj, dict) else list
tokens = to_path_tokens(path)
if not pyd.is_list(tokens): # pragma: no cover
tokens = [tokens]
last_key = pyd.last(tokens)
if isinstance(last_key, PathToken):
last_key = last_key.key
target = obj
for idx, token in enumerate(pyd.initial(tokens)):
if isinstance(token, PathToken):
key = token.key
default_factory = pyd.get(tokens,
[idx + 1, 'default_factory'],
default=default_type)
else:
key = token
default_factory = default_type
obj_val = base_get(target, key, default=None)
path_obj = None
if call_customizer:
path_obj = call_customizer(obj_val, key, target)
if path_obj is None:
path_obj = default_factory()
base_set(target, key, path_obj, allow_override=False)
try:
target = target[key]
except TypeError as exc: # pragma: no cover
try:
target = target[int(key)]
_failed = False
except Exception:
_failed = True
if _failed:
raise TypeError('Unable to update object at index {!r}. {}'
.format(key, exc))
value = base_get(target, last_key, default=None)
base_set(target, last_key, callit(updater, value))
return obj
def cast_list(val):
'''missing pydash method to cast value as list'''
if ps.is_list(val):
return val
else:
return [val]
def unset(obj, path):
"""Removes the property at `path` of `obj`.
Note:
Only ``list``, ``dict``, or objects with a ``pop()`` method can be
unset by this function.
Args:
obj (mixed): The object to modify.
path (mixed): The path of the property to unset.
Returns:
bool: Whether the property was deleted.
Warning:
`obj` is modified in place.
Example:
>>> obj = {'a': [{'b': {'c': 7}}]}
>>> unset(obj, 'a[0].b.c')
True
>>> obj
{'a': [{'b': {}}]}
>>> unset(obj, 'a[0].b.c')
False
"""
tokens = to_path_tokens(path)
if not pyd.is_list(tokens): # pragma: no cover
tokens = [tokens]
last_key = pyd.last(tokens)
if isinstance(last_key, PathToken):
last_key = last_key.key
target = obj
for idx, token in enumerate(pyd.initial(tokens)):
if isinstance(token, PathToken):
key = token.key
else:
key = token
try:
try:
target = target[key]
except TypeError:
target = target[int(key)]
except Exception:
target = NoValue
if target is NoValue:
break
did_unset = False
if target is not NoValue:
try:
try:
target.pop(last_key)
did_unset = True
except TypeError:
target.pop(int(last_key))
did_unset = True
except Exception:
pass
return did_unset
def test_cast_list(test_list, test_str):
assert ps.is_list(test_list)
assert ps.is_list(util.cast_list(test_list))
assert not ps.is_list(test_str)
assert ps.is_list(util.cast_list(test_str))
def update_with(obj, path, updater, customizer=None): # noqa: C901
"""
This method is like :func:`update` except that it accepts customizer which is invoked to produce
the objects of path. If customizer returns ``None``, path creation is handled by the method
instead. The customizer is invoked with three arguments: ``(nested_value, key, nested_object)``.
Args:
obj (list|dict): Object to modify.
path (str|list): A string or list of keys that describe the object path to modify.
updater (callable): Function that returns updated value.
customizer (callable, optional): The function to customize assigned values.
Returns:
mixed: Updated `obj`.
Warning:
`obj` is modified in place.
Example:
>>> update_with({}, '[0][1]', lambda: 'a', lambda: {})
{0: {1: 'a'}}
.. versionadded:: 4.0.0
"""
if not callable(updater):
updater = pyd.constant(updater)
if customizer is not None and not callable(customizer):
call_customizer = partial(callit, clone, customizer, argcount=1)
elif customizer:
call_customizer = partial(callit,
customizer,
argcount=getargcount(customizer, maxargs=3))
else:
call_customizer = None
default_type = dict if isinstance(obj, dict) else list
tokens = to_path_tokens(path)
if not pyd.is_list(tokens): # pragma: no cover
tokens = [tokens]
last_key = pyd.last(tokens)
if isinstance(last_key, PathToken):
last_key = last_key.key
target = obj
for idx, token in enumerate(pyd.initial(tokens)):
if isinstance(token, PathToken):
key = token.key
default_factory = pyd.get(tokens, [idx + 1, "default_factory"],
default=default_type)
else:
key = token
default_factory = default_type
obj_val = base_get(target, key, default=None)
path_obj = None
if call_customizer:
path_obj = call_customizer(obj_val, key, target)
if path_obj is None:
path_obj = default_factory()
base_set(target, key, path_obj, allow_override=False)
try:
target = base_get(target, key, default=None)
except TypeError as exc: # pragma: no cover
try:
target = target[int(key)]
_failed = False
except Exception:
_failed = True
if _failed:
raise TypeError(
"Unable to update object at index {!r}. {}".format(
key, exc))
value = base_get(target, last_key, default=None)
base_set(target, last_key, callit(updater, value))
return obj
def test_cast_list(test_list, test_str):
assert _.is_list(test_list)
assert _.is_list(util.cast_list(test_list))
assert not _.is_list(test_str)
assert _.is_list(util.cast_list(test_str))
def test_is_list(case, expected):
assert _.is_list(case) == expected
def unset(obj, path): # noqa: C901
"""
Removes the property at `path` of `obj`.
Note:
Only ``list``, ``dict``, or objects with a ``pop()`` method can be unset by this function.
Args:
obj (mixed): The object to modify.
path (mixed): The path of the property to unset.
Returns:
bool: Whether the property was deleted.
Warning:
`obj` is modified in place.
Example:
>>> obj = {'a': [{'b': {'c': 7}}]}
>>> unset(obj, 'a[0].b.c')
True
>>> obj
{'a': [{'b': {}}]}
>>> unset(obj, 'a[0].b.c')
False
"""
tokens = to_path_tokens(path)
if not pyd.is_list(tokens): # pragma: no cover
tokens = [tokens]
last_key = pyd.last(tokens)
if isinstance(last_key, PathToken):
last_key = last_key.key
target = obj
for token in pyd.initial(tokens):
if isinstance(token, PathToken):
key = token.key
else:
key = token
try:
try:
target = target[key]
except TypeError:
target = target[int(key)]
except Exception:
target = NoValue
if target is NoValue:
break
did_unset = False
if target is not NoValue:
try:
try:
target.pop(last_key)
did_unset = True
except TypeError:
target.pop(int(last_key))
did_unset = True
except Exception:
pass
return did_unset