def make_model(self):
# Create placeholders.
self._placeholders['target_values'] = tf.placeholder(tf.float32, [len(self._params['task_ids']), None], name='target_values')
self._placeholders['target_mask'] = tf.placeholder(tf.float32, [len(self._params['task_ids']), None], name='target_mask')
self._placeholders['num_graphs'] = tf.placeholder(tf.int64, [], name='num_graphs')
self._placeholders['out_layer_dropout_keep_prob'] = tf.placeholder(tf.float32, [], name='out_layer_dropout_keep_prob')
with tf.variable_scope('graph_model'):
self.prepare_model()
self._ops['final_node_representations'] = self.compute_final_node_representations()
# Compute the squared loss and absolute difference for each prediction task.
self._ops['losses'] = []
for (internal_id, task_id) in enumerate(self._params['task_ids']):
with tf.variable_scope('out_layer_task_%d' % task_id):
task_target_mask = self._placeholders['target_mask'][internal_id, :]
task_target_num = tf.reduce_sum(task_target_mask) + utils.SMALL_NUMBER
if self._params['local_regression']:
with tf.variable_scope('local_regression'):
self._weights['local_regression_transform_task_%d' % task_id] = utils.MLP(
self._params['hidden_size'], 1, [], self._placeholders['out_layer_dropout_keep_prob'])
computed_values = self.local_regression(self._ops['final_node_representations'],
self._weights['local_regression_transform_task_%d' % task_id])
# Computed values has shape [b, v].
diff = computed_values - tf.expand_dims(self._placeholders['target_values'][internal_id, :], axis=-1)
# TODO: Handle multiple tasks. Ignore this for now.
self._ops['accuracy_task_%d' % task_id] = tf.reduce_sum(tf.reduce_mean(tf.abs(diff), axis=1)) / task_target_num
task_loss = tf.reduce_sum(tf.reduce_mean(0.5 * tf.square(diff), axis=1)) / task_target_num
else:
with tf.variable_scope('regression_gate'):
self._weights['regression_gate_task_%d' % task_id] = utils.MLP(
2 * self._params['hidden_size'], 1, [], self._placeholders['out_layer_dropout_keep_prob'])
with tf.variable_scope('regression'):
self._weights['regression_transform_task_%d' % task_id] = utils.MLP(
self._params['hidden_size'], 1, [], self._placeholders['out_layer_dropout_keep_prob'])
computed_values = self.global_regression(self._ops['final_node_representations'],
self._weights['regression_gate_task_%d' % task_id],
self._weights['regression_transform_task_%d' % task_id])
diff = computed_values - self._placeholders['target_values'][internal_id, :]
diff = diff * task_target_mask # Mask out unused values
self._ops['accuracy_task_%d' % task_id] = tf.reduce_sum(tf.abs(diff)) / task_target_num
task_loss = tf.reduce_sum(0.5 * tf.square(diff)) / task_target_num
self._predictions = computed_values
self._targets = self._placeholders['target_values'][internal_id, :]
# Normalise loss to account for fewer task-specific examples in batch:
task_loss = task_loss * (1.0 / (self._params['task_sample_ratios'].get(task_id) or 1.0))
self._ops['losses'].append(task_loss)
self._ops['loss'] = tf.reduce_sum(self._ops['losses'])
def __init__(self, config):
self.config = config
hidden_size_orig = self.config['hidden_size_orig']
h_size = self.config['gnn_h_size']
num_edge_types = self.config['num_edge_types']
self.weights = {}
self.weights['mapping'] = utils.MLP(hidden_size_orig, h_size, self.config['embedding_layer']['mapping_dims'], 'relu', 'mapping')
def __init__(self,
hidden_sizes: Sequence[int],
output_size: int,
dropout: float,
final_activation: Optional[Any] = None):
super(_MlpWithProjector, self).__init__()
self.mlp = utils.MLP(hidden_sizes,
dropout=dropout,
final_activation=final_activation)
self.projector = tf.keras.layers.Dense(output_size)
def __init__(self, config):
self.config = config
h_size = self.config['gnn_h_size']
h_size_orig = self.config['hidden_size_orig']
m_size = self.config['gnn_m_size']
self.weights = {}
if self.config['use_node_values'] == 1:
self.weights['mlp_f_m'] = utils.MLP(
h_size + 1, h_size * m_size,
self.config['prediction_cell']['mlp_f_m_dims'],
self.config['prediction_cell']['mlp_f_m_activation'],
'mlp_regression_transform')
self.weights['mlp_g_m'] = utils.MLP(
h_size + 1 + h_size_orig, h_size * m_size,
self.config['prediction_cell']['mlp_g_m_dims'],
self.config['prediction_cell']['mlp_g_m_activation'],
'mlp_regression_gate')
else:
self.weights['mlp_f_m'] = utils.MLP(
h_size, h_size * m_size,
self.config['prediction_cell']['mlp_f_m_dims'],
self.config['prediction_cell']['mlp_f_m_activation'],
'mlp_regression_transform')
self.weights['mlp_g_m'] = utils.MLP(
h_size + h_size_orig, h_size * m_size,
self.config['prediction_cell']['mlp_g_m_dims'],
self.config['prediction_cell']['mlp_g_m_activation'],
'mlp_regression_gate')
self.weights['mlp_reduce'] = utils.MLP(
h_size * m_size, h_size * m_size,
self.config['prediction_cell']['mlp_reduce_dims'],
self.config['prediction_cell']['mlp_reduce_activation'],
'mlp_reduce')
offset = 0
if config['with_aux_in'] == 1:
offset = 2
self.weights['mlp_reduce_after_aux_in_1'] = utils.MLP(
h_size * m_size + offset, self.config['prediction_cell']
['mlp_reduce_after_aux_in_1_out_dim'],
self.config['prediction_cell']['mlp_reduce_after_aux_in_1_dims'],
self.config['prediction_cell']
['mlp_reduce_after_aux_in_1_activation'],
'mlp_reduce_after_aux_in_1')
self.weights['mlp_reduce_after_aux_in_2'] = utils.MLP(
self.config['prediction_cell']
['mlp_reduce_after_aux_in_1_out_dim'],
self.config['prediction_cell']
['mlp_reduce_after_aux_in_2_out_dim'],
self.config['prediction_cell']['mlp_reduce_after_aux_in_2_dims'],
self.config['prediction_cell']
['mlp_reduce_after_aux_in_2_activation'],
'mlp_reduce_after_aux_in_2')
def test_mlp_with_final_activitation(self):
output_sizes = [5, 6]
final_activation = tf.keras.layers.ReLU()
test_model = utils.MLP(
output_sizes=output_sizes, final_activation=final_activation)
input_tensor = tf.keras.Input(shape=(8))
output_tensor = test_model(input_tensor)
expected_output_shape = [None, 6]
self.assertEqual(expected_output_shape, output_tensor.shape.as_list())
self.assertEqual(tf.float32, output_tensor.dtype)
def __init__(
self,
hparams: Union[ModelParams, dict, None] = None,
**kwargs,
):
super().__init__()
if hparams is None:
hparams = self.params(**kwargs)
elif isinstance(hparams, dict):
hparams = self.params(**hparams, **kwargs)
if isinstance(self.hparams, AttributeDict):
self.hparams.update(AttributeDict(attr.asdict(hparams)))
else:
self.hparams = AttributeDict(attr.asdict(hparams))
# Check for configuration issues
if (hparams.gather_keys_for_queue and not hparams.shuffle_batch_norm
and not hparams.encoder_arch.startswith("ws_")):
warnings.warn(
"Configuration suspicious: gather_keys_for_queue without shuffle_batch_norm or weight standardization"
)
some_negative_examples = hparams.use_negative_examples_from_batch or hparams.use_negative_examples_from_queue
if hparams.loss_type == "ce" and not some_negative_examples:
warnings.warn(
"Configuration suspicious: cross entropy loss without negative examples"
)
# Create encoder model
self.model = utils.get_encoder(hparams.encoder_arch,
hparams.dataset_name)
# Create dataset
self.dataset = utils.get_moco_dataset(hparams)
if hparams.use_lagging_model:
# "key" function (no grad)
self.lagging_model = copy.deepcopy(self.model)
for param in self.lagging_model.parameters():
param.requires_grad = False
else:
self.lagging_model = None
self.projection_model = utils.MLP(
hparams.embedding_dim,
hparams.dim,
hparams.mlp_hidden_dim,
num_layers=hparams.projection_mlp_layers,
normalization=get_mlp_normalization(hparams),
weight_standardization=hparams.use_mlp_weight_standardization,
)
self.prediction_model = utils.MLP(
hparams.dim,
hparams.dim,
hparams.mlp_hidden_dim,
num_layers=hparams.prediction_mlp_layers,
normalization=get_mlp_normalization(hparams, prediction=True),
weight_standardization=hparams.use_mlp_weight_standardization,
)
if hparams.use_lagging_model:
# "key" function (no grad)
self.lagging_projection_model = copy.deepcopy(
self.projection_model)
for param in self.lagging_projection_model.parameters():
param.requires_grad = False
else:
self.lagging_projection_model = None
# this classifier is used to compute representation quality each epoch
self.sklearn_classifier = LogisticRegression(max_iter=100,
solver="liblinear")
if hparams.use_negative_examples_from_queue:
# create the queue
self.register_buffer("queue", torch.randn(hparams.dim, hparams.K))
self.queue = torch.nn.functional.normalize(self.queue, dim=0)
self.register_buffer("queue_ptr", torch.zeros(1, dtype=torch.long))
else:
self.queue = None
def __init__(self, config: model_config.VanillaLinearVAECellConfig):
self._gumbel_softmax_label_adjustment_multiplier = config.gumbel_softmax_label_adjustment_multiplier
vocab_embeddings_initializer = None
if config.shared_bert_embedding:
shared_embedding_layer = _BERT(
config.max_seq_length,
bert_config=configs.BertConfig(
**config.shared_bert_embedding_config),
trainable=config.trainable_embedding)
else:
# If word_embedding_path is specified, use the embedding size of the
# pre-trained embeddings.
if config.word_embedding_path:
with tf.io.gfile.GFile(config.word_embedding_path,
'rb') as embedding_file:
word_embedding = np.load(embedding_file)
embedding_vocab_size, embed_size = word_embedding.shape
if config.vocab_size != embedding_vocab_size:
raise ValueError(
'Expected consistent vocab size between vocab.txt and the '
'embedding, found {} and {}.'.format(
embedding_vocab_size, config.vocab_size))
config.embed_size = embed_size
vocab_embeddings_initializer = (
tf.keras.initializers.Constant(word_embedding))
if config.shared_embedding:
shared_embedding_layer = _Embedding(
INPUT_ID_NAME,
config.vocab_size,
config.embed_size,
embeddings_initializer=vocab_embeddings_initializer,
input_length=config.max_seq_length,
trainable=config.trainable_embedding)
else:
shared_embedding_layer = None
encoder = DualRNNEncoder(
vocab_size=config.vocab_size,
embed_size=config.embed_size,
max_seq_length=config.max_seq_length,
hidden_size=config.encoder_hidden_size,
num_layers=config.num_ecnoder_rnn_layers,
dropout=config.dropout,
cell_type=config.encoder_cell_type,
embeddings_initializer=vocab_embeddings_initializer,
shared_embedding_layer=shared_embedding_layer,
trainable_embedding=config.trainable_embedding)
sampler = utils.GumbelSoftmaxSampler(config.temperature, hard=False)
decoder = DualRNNDecoder(
vocab_size=config.vocab_size,
embed_size=config.embed_size,
max_seq_length=config.max_seq_length - 1,
hidden_size=config.decoder_hidden_size,
# Hardcoded to be 1 layer to align with pytorch version. Otherwise, we
# need to define the initial state for each layer in
# _prepare_decoder_initial_state and change _post_process_decoder_state
num_layers=1,
dropout=config.dropout,
cell_type=config.decoder_cell_type,
embeddings_initializer=vocab_embeddings_initializer,
shared_embedding_layer=shared_embedding_layer,
trainable_embedding=config.trainable_embedding,
return_state=True)
state_updater = _VanillaStateUpdater(config.state_updater_cell_type,
config.num_states, config.dropout)
self.encoder_output_projector = _VanillaEncoderOutputProjector(
hidden_sizes=list(config.encoder_projection_sizes),
output_size=config.num_states,
dropout=config.dropout)
self.sample_post_processor = utils.MLP(
config.sampler_post_processor_output_sizes, dropout=config.dropout)
self.shared_embedding_layer = shared_embedding_layer
super(VanillaLinearVAECell, self).__init__(encoder=encoder,
sampler=sampler,
decoder=decoder,
state_updater=state_updater)
def __init__(self, hidden_sizes: Sequence[int], output_size: int,
dropout: float):
super(_VanillaEncoderOutputProjector, self).__init__()
self.mlp = utils.MLP(hidden_sizes, dropout=dropout)
self.project_layer = tf.keras.layers.Dense(output_size)
def main(cfg):
from omegaconf import OmegaConf
attach_state = cfg.attach_state
from_pixels = cfg.from_pixels
encoder_type = cfg.encoder_type
if cfg.user_config:
print("+++++++++++++++++ Using user specified config")
cfg = OmegaConf.load(cfg.user_config)
cfg.attach_state = attach_state
cfg.from_pixels = from_pixels
cfg.encoder_type = encoder_type
print("+++++++++++++++++ Configuration : \n", cfg)
expert_path = home + "/pytorch_sac/expert/" + cfg.env + "_state"
print("+++++++++++++++++ Expert Path : ", expert_path)
actor_path = expert_path + "/actor.pt"
env = utils.make_env(cfg) # Make env based on cfg.
#if cfg.frame_stack = True:
# self.env = utils.FrameStack(self.env, k=3)
cfg.agent.params.obs_dim = env.observation_space.shape[0]
if attach_state:
cfg.agent.params.obs_dim += get_env_state_dim(cfg)
cfg.agent.params.action_dim = env.action_space.shape[0]
cfg.agent.params.action_range = [
float(env.action_space.low.min()),
float(env.action_space.high.max())
]
agent = hydra.utils.instantiate(cfg.agent)
print("Observation Dimension : ", cfg.agent.params.obs_dim)
conf = OmegaConf.load(expert_path + '/config.yaml')
assert conf.env == cfg.env
conf.agent.params.action_dim = env.action_space.shape[0]
conf.agent.params.action_range = [
float(env.action_space.low.min()),
float(env.action_space.high.max())
]
conf.agent.params.obs_dim = get_env_state_dim(conf)
agent_expert = hydra.utils.instantiate(conf.agent)
agent_expert.actor.load_state_dict(torch.load(actor_path))
#video_recorder = VideoRecorder(None)
#print("DATASET CAPACITY : 1000000")
start_ind = 0
end_ind = 1000000
load_start_time = time.time()
data = torch.load(home + "/pytorch_sac/Data/" + cfg.env + "_" +
cfg.encoder_type + str(start_ind) + "_" + str(end_ind) +
".pt")
print(data[0].shape)
print(data[1].shape)
print(data[2].shape)
print(data[3].shape)
print("Time to load the data : ", time.time() - load_start_time)
dataset = Dataset((data[0].shape[1], ), (data[1].shape[1], ),
env.action_space.shape, int(end_ind - start_ind),
torch.device("cuda"))
buffer_insert_start_time = time.time()
for i in range(data[0].shape[0]):
obs = data[0][i]
state = data[1][i]
action_expert = data[2][i]
reward = data[3][i]
done = data[4][i]
dataset.add(obs, state, action_expert, reward, done)
print("Time taken to add into buffer : ",
time.time() - buffer_insert_start_time)
recon_update_steps = 10000
recon_batch_size = 1024
recon_eval_freq = 100
loss_fn = nn.MSELoss()
model = utils.MLP(cfg.agent.params.obs_dim, 1024,
conf.agent.params.obs_dim, 2).to(cfg.device)
print("Learning Rate : ", cfg.agent.params.actor_lr)
optimizer = torch.optim.Adam(model.parameters(),
lr=cfg.agent.params.actor_lr,
betas=cfg.agent.params.actor_betas)
recon_steps = 0
start_recon_time = time.time()
for i in range(recon_update_steps):
obses, states, actions_expert, _, _ = dataset.sample(recon_batch_size)
if attach_state:
obses = torch.cat((obses, states), axis=1)
if not cfg.from_pixels:
obses = states
loss = update_model_recon(model, obses, states, loss_fn, optimizer)
recon_steps += 1
if recon_steps % recon_eval_freq == 0:
#average_ep_reward = evaluate(env, agent, cfg, attach_state)
print("Step : ", recon_steps, " Loss : ", loss.data,
" Time taken : ",
time.time() - start_recon_time)
#print("Average Episode Reward : ", average_ep_reward)
print("Total time taken : ", time.time() - load_start_time)
states, states_recon = evaluate(env, agent_expert, model, cfg,
attach_state)
states_np = np.array(states)
states_recon_np = np.array(states_recon)
print(states_np.shape)
print(states_recon_np.shape)
np.savetxt(home + "/pytorch_sac/images/" + cfg.env + "_" +
cfg.encoder_type + "_" + 'State.csv',
states_np,
delimiter=',')
np.savetxt(home + "/pytorch_sac/images/" + cfg.env + "_" +
cfg.encoder_type + "_" + 'State_recon.csv',
states_recon_np,
delimiter=',')