def parse_1_set_of_kwargs(env_instance,
kwargs_to_parse,
method_name,
repeated_key,
only_repeated,
old_arguments=None):
# print('\n\n_parse_1_set_of_kwargs method:\nkwargs_to_parse:\n', kwargs_to_parse, '\nmethod_name:\n',
# method_name, '\nrepeated_key:\n', repeated_key, '\nonly_repeated:\n', only_repeated, '\nold_arguments:\n',
# old_arguments)
# print("(parse_1_set_of_kwargs)kwargs_to_parse:", kwargs_to_parse)
kwargs_to_parse = construct(kwargs_to_parse)
process_abbreviations(env_instance, kwargs_to_parse, method_name)
# print("(parse_1_set_of_kwargs)kwargs_to_parse:", kwargs_to_parse)
# print("(parse_1_set_of_kwargs)kwargs_to_parse['valid_batch_kwargs']:", kwargs_to_parse['valid_batch_kwargs'])
if old_arguments is None:
current_arguments = env_instance.get_default_method_parameters(
method_name)
if only_repeated:
current_arguments = current_arguments[repeated_key]
else:
current_arguments = construct(old_arguments)
# print("(parse_1_set_of_kwargs)current_arguments:", current_arguments)
for key, value in kwargs_to_parse.items():
paste_into_nested_structure(current_arguments, key, value)
#print('current_arguments:\n', current_arguments)
return current_arguments
def create_one_combination_insertions(insert_template, hp_combination):
# print('\ncreate_one_combination_insertions')
combination_insertions = construct(insert_template)
for hp_name, value in hp_combination.items():
# print('hp_name:', hp_name)
# print('value:', value)
insert_key = (hp_name[0], hp_name[1], hp_name[2])
single_tmpl = combination_insertions[insert_key]
# print('single_tmpl:', single_tmpl)
# print('hp_name[0]:', hp_name[0])
# print('hp_name[3]:', hp_name[3])
if hp_name[0] == 'additional_placeholder':
if hp_name[3] is not None:
single_tmpl['paste']['value'][hp_name[3]] = value
else:
single_tmpl['paste']['value'] = value
else:
if hp_name[3] is not None:
single_tmpl['paste'][hp_name[3]] = value
else:
single_tmpl['paste'] = value
if single_tmpl['share'] is not None:
share = single_tmpl['share']
if share['hp_type'] == 'additional_placeholder':
share['paste']['value']['value'] = value
else:
share['paste']['value'] = value
# print('\n')
return combination_insertions
def process_other_hp_specs_absences(other_hps):
other_hps = construct(other_hps)
new_other_hps = dict()
for hp_name, hp_values_and_specs in other_hps.items():
if isinstance(hp_values_and_specs, list):
new_other_hps.update(
process_other_hp_dictionary_absence(hp_name,
hp_values_and_specs))
else:
if 'list_indices' not in hp_values_and_specs:
hp_values_and_specs['list_indices'] = None
elif isinstance(hp_values_and_specs['list_indices'], int):
hp_values_and_specs['list_indices'] = [
hp_values_and_specs['list_indices']
]
if 'hp_type' not in hp_values_and_specs:
hp_values_and_specs['hp_type'] = 'additional_placeholder'
if 'controller' not in hp_values_and_specs:
hp_values_and_specs['controller'] = True
if 'type' not in hp_values_and_specs:
if hp_values_and_specs['controller']:
hp_values_and_specs['type'] = 'fixed'
else:
hp_values_and_specs['type'] = None
if 'fixed' not in hp_values_and_specs:
if hp_values_and_specs['controller']:
hp_values_and_specs['fixed'] = dict()
else:
hp_values_and_specs['fixed'] = None
hp_values_and_specs['share'] = None
new_other_hps[hp_name] = hp_values_and_specs
return new_other_hps
def process_build_hp_abbreviations(build_hps):
build_hps = construct(build_hps)
new_build_hps = dict()
for hp_name, hp_values_and_specs in build_hps.items():
if isinstance(hp_values_and_specs, list):
new_build_hps.update(
process_build_hp_text_abbreviation(hp_name,
hp_values_and_specs))
else:
if 'hp_type' not in hp_values_and_specs:
hp_values_and_specs['hp_type'] = 'build_hp'
if 'list_indices' not in hp_values_and_specs:
hp_values_and_specs['list_indices'] = None
elif isinstance(hp_values_and_specs['list_indices'], int):
hp_values_and_specs['list_indices'] = [
hp_values_and_specs['list_indices']
]
if 'share' not in hp_values_and_specs:
hp_values_and_specs['share'] = None
hp_values_and_specs['controller'] = False
hp_values_and_specs['type'] = None
hp_values_and_specs['fixed'] = None
if hp_values_and_specs['share'] is not None:
if 'direction' not in hp_values_and_specs['share']:
default = spot_direction(hp_name)
if spot_direction(hp_name) is None:
hp_values_and_specs['share'][
'direction'] = 'batch_kwarg'
else:
hp_values_and_specs['share']['direction'] = default
if 'controller' not in hp_values_and_specs['share']:
hp_values_and_specs['share']['controller'] = True
new_build_hps[hp_name] = hp_values_and_specs
new_build_hps = reshape_1_index_hps(new_build_hps)
return new_build_hps
def expand_varying_entry(hps):
hps = construct(hps)
new_hps = dict()
for hp_name, hp_values_and_specs in hps.items():
if hp_values_and_specs['controller'] and isinstance(hp_values_and_specs['varying'], list):
hp_values_and_specs['varying'] = {'value': hp_values_and_specs['varying']}
new_hps[hp_name] = hp_values_and_specs
return new_hps
def process_abbreviation_in_1_entry(key, value, method_name):
new_value = construct(value)
if key == 'stop':
if isinstance(value, int):
new_value = {'type': 'limit_steps', 'limit': value}
set_controller_name_in_specs(new_value, 'stop')
if key == 'batch_size':
if isinstance(value, int):
new_value = {'type': 'fixed', 'value': value}
set_controller_name_in_specs(new_value, 'batch_size')
if key == 'num_unrollings':
if isinstance(value, int):
new_value = {'type': 'fixed', 'value': value}
set_controller_name_in_specs(new_value, 'num_unrollings')
if key == 'checkpoint_steps':
if isinstance(value, list):
new_value = {'type': 'true_on_steps', 'steps': value}
elif isinstance(value, int):
new_value = {'type': 'periodic_truth', 'period': value}
else:
new_value = {'type': 'always_false'}
set_controller_name_in_specs(new_value, 'checkpoint_steps')
if key == 'learning_rate':
set_controller_name_in_specs(new_value, 'learning_rate')
if key == 'debug':
if isinstance(value, int):
new_value = {'type': 'true_on_steps', 'steps': [value]}
else:
new_value = None
set_controller_name_in_specs(new_value, 'debug')
if key == 'opt_inf_stop':
if isinstance(value, int):
new_value = {'type': 'limit_steps', 'limit': value}
set_controller_name_in_specs(new_value,
'optimizer_inference_num_steps')
if key == 'reset_period':
if isinstance(value, int):
new_value = {'type': 'periodic_truth', 'period': value}
set_controller_name_in_specs(new_value,
'optimizer_inference_num_steps')
if method_name == 'train' or method_name == 'train_optimizer':
if key == 'additions_to_feed_dict' or key == 'opt_inf_additions_to_feed_dict':
# print('inside additions_to_feed_dict shortcuts processing')
if new_value is not None:
for addition in new_value:
# print('addition:', addition)
if not isinstance(addition['value'], dict):
# print('Removing of shortcut is happening now')
addition['value'] = {
'type': 'fixed',
'value': addition['value']
}
# print("addition['value']:", addition['value'])
set_controller_name_in_specs(addition['value'],
addition['placeholder'])
return new_value
def create_change_tracking_specifications(specifications):
if isinstance(specifications, list):
old_specs = construct(specifications)
if isinstance(specifications, dict):
old_specs = [dict(specifications)]
new_specs = dict()
new_specs['old_specs'] = old_specs
new_specs['type'] = 'changes_detector'
return new_specs
def _optimizer_core(self, optimizer_ins, state, gpu_idx, permute=True):
new_state = construct(state)
# print(state)
for ok, ov in optimizer_ins.items():
with tf.name_scope(ok):
for inp_name in self._selected:
if inp_name in ov:
with tf.name_scope(inp_name):
ov[inp_name + '_pr'], new_state[ok][
inp_name] = self._apply_net(
ov[inp_name], state[ok][inp_name],
self._opt_trainable[ok][inp_name])
# for ok, ov in optimizer_ins.items():
# for ik, iv in ov.items():
# print()
# print(ok, ik)
# print(iv)
return optimizer_ins, new_state
def _apply_res_layer(self, ins, res_vars, rnn_part, scope):
with tf.name_scope(scope):
outs = construct(ins)
rnn_parts = list()
for layer_idx in range(self._pupil_net_size['num_layers']):
if layer_idx == 0:
previous_layer_tensors = []
else:
previous_layer_tensors = [
ins['layer_%s' % (layer_idx - 1)]['o'],
ins['layer_%s' % (layer_idx - 1)]['sigma']
]
if layer_idx == self._pupil_net_size['num_layers'] - 1:
next_layer_tensors = []
else:
next_layer_tensors = [
ins['layer_%s' % (layer_idx + 1)]['o'],
ins['layer_%s' % (layer_idx + 1)]['sigma']
]
layer_name = 'layer_%s' % layer_idx
core_inps = [
*previous_layer_tensors,
ins[layer_name]['o'],
ins[layer_name]['sigma'],
*next_layer_tensors
]
target = [
ins[layer_name]['o'],
ins[layer_name]['sigma']
]
o, sigma, lstm_rnn_part = self._apply_res_core(
res_vars[layer_name], core_inps, rnn_part, target,
layer_name, self._pupil_dims['layers'][layer_idx])
rnn_parts.append(lstm_rnn_part)
outs[layer_name]['o'] = o
outs[layer_name]['sigma'] = sigma
# print("(ResNet4Lstm._apply_res_layer)emb_rnn_part:", emb_rnn_part)
# print("(ResNet4Lstm._apply_res_layer)lstm_rnn_parts:", lstm_rnn_parts)
# print("(ResNet4Lstm._apply_res_layer)output_rnn_parts:", output_rnn_parts)
return outs, sum(rnn_parts)
def configure_args_for_launches(env_instance, args_for_launches, shares, model='pupil'):
args_for_launches_to_be_used = construct(args_for_launches)
parsed = list()
for to_be_used in args_for_launches_to_be_used:
with_shares = apply_shares(to_be_used, shares)
if model == 'pupil':
one_parsed = parse_train_method_arguments(env_instance,
[],
with_shares,
set_passed_parameters_as_default=False)
else:
one_parsed = parse_train_optimizer_method_arguments(
env_instance,
[],
with_shares,
set_passed_parameters_as_default=False
)
start_specs = one_parsed['start_specs']
run_specs_set = one_parsed['run']
del one_parsed['session_specs']
parsed.append((start_specs, run_specs_set))
return parsed
def _apply_layer(self, ins, vars, scope):
with tf.name_scope(scope):
outs = construct(ins)
if self._emb_layer_is_present:
target = [
ins['embedding_layer']['o_c'],
ins['embedding_layer']['sigma_c']
]
o, sigma = self._apply_core(
vars['embedding_layer'], target,
'embedding_layer', self._pupil_dims['embedding_layer'])
outs['embedding_layer']['o_c'] = o
outs['embedding_layer']['sigma_c'] = sigma
for layer_idx in range(self._pupil_net_size['num_layers']):
layer_name = 'lstm_layer_%s' % layer_idx
target = [
ins[layer_name]['o_c'],
ins[layer_name]['sigma_c']
]
o, sigma = self._apply_core(
vars[layer_name], target,
layer_name, self._pupil_dims['lstm_layers'][layer_idx])
outs[layer_name]['o_c'] = o
outs[layer_name]['sigma_c'] = sigma
for layer_idx in range(self._pupil_net_size['num_output_layers']):
target = [
ins['output_layer_%s' % layer_idx]['o_c'],
ins['output_layer_%s' % layer_idx]['sigma_c']
]
layer_name = 'output_layer_%s' % layer_idx
o, sigma = self._apply_core(
vars[layer_name], target,
layer_name, self._pupil_dims['output_layers'][layer_idx])
layer_name = 'output_layer_%s' % layer_idx
outs[layer_name]['o_c'] = o
outs[layer_name]['sigma_c'] = sigma
return outs
def create_1_set_of_args_for_launches(kwargs, one_set_of_args_insertions):
kwargs = construct(kwargs)
for one_hp_insertion in one_set_of_args_insertions:
kwargs = insert_not_build_hp(kwargs, one_hp_insertion)
return kwargs
def _apply_res_layer(self, ins, res_vars, scope):
with tf.name_scope(scope):
outs = construct(ins)
if self._emb_layer_is_present:
core_inps = [
ins['embedding_layer']['o_c'],
ins['embedding_layer']['sigma_c'],
ins['lstm_layer_0']['o_c'],
ins['lstm_layer_0']['sigma_c']
]
target = [
ins['embedding_layer']['o_c'],
ins['embedding_layer']['sigma_c']
]
o, sigma = self._apply_res_core(
res_vars['embedding_layer'], core_inps, target,
'embedding_layer', self._pupil_dims['embedding_layer'])
outs['embedding_layer']['o_c'] = o
outs['embedding_layer']['sigma_c'] = sigma
for layer_idx in range(self._pupil_net_size['num_layers']):
if layer_idx == 0:
if self._emb_layer_is_present:
previous_layer_tensors = [
ins['embedding_layer']['o_c'],
ins['embedding_layer']['sigma_c']
]
else:
previous_layer_tensors = []
else:
previous_layer_tensors = [
ins['lstm_layer_%s' % (layer_idx - 1)]['o_c'],
ins['lstm_layer_%s' % (layer_idx - 1)]['sigma_c']
]
if layer_idx == self._pupil_net_size['num_layers'] - 1:
next_layer_tensors = [
ins['output_layer_0']['o_c'],
ins['output_layer_0']['sigma_c']
]
else:
next_layer_tensors = [
ins['lstm_layer_%s' % (layer_idx + 1)]['o_c'],
ins['lstm_layer_%s' % (layer_idx + 1)]['sigma_c']
]
layer_name = 'lstm_layer_%s' % layer_idx
core_inps = [
*previous_layer_tensors,
ins[layer_name]['o_c'],
ins[layer_name]['sigma_c'],
self._pad(ins[layer_name]['o_c'], -1),
self._pad(ins[layer_name]['sigma_c'], -1),
self._pad(ins[layer_name]['o_c'], 1),
self._pad(ins[layer_name]['sigma_c'], 1),
*next_layer_tensors
]
target = [
ins[layer_name]['o_c'],
ins[layer_name]['sigma_c']
]
o, sigma = self._apply_res_core(
res_vars[layer_name], core_inps, target,
layer_name, self._pupil_dims['lstm_layers'][layer_idx])
outs[layer_name]['o_c'] = o
outs[layer_name]['sigma_c'] = sigma
for layer_idx in range(self._pupil_net_size['num_output_layers']):
if layer_idx == 0:
previous_layer_tensors = [
ins['lstm_layer_%s' % (self._pupil_net_size['num_layers'] - 1)]['o_c'],
ins['lstm_layer_%s' % (self._pupil_net_size['num_layers'] - 1)]['sigma_c']
]
else:
previous_layer_tensors = [
ins['output_layer_%s' % (layer_idx - 1)]['o_c'],
ins['output_layer_%s' % (layer_idx - 1)]['sigma_c']
]
if layer_idx == self._pupil_net_size['num_output_layers'] - 1:
next_layer_tensors = []
else:
next_layer_tensors = [
ins['output_layer_%s' % (layer_idx + 1)]['o_c'],
ins['output_layer_%s' % (layer_idx + 1)]['sigma_c']
]
core_inps = [
*previous_layer_tensors,
ins['output_layer_%s' % layer_idx]['o_c'],
ins['output_layer_%s' % layer_idx]['sigma_c'],
*next_layer_tensors
]
target = [
ins['output_layer_%s' % layer_idx]['o_c'],
ins['output_layer_%s' % layer_idx]['sigma_c']
]
layer_name = 'output_layer_%s' % layer_idx
o, sigma = self._apply_res_core(
res_vars[layer_name], core_inps, target,
layer_name, self._pupil_dims['output_layers'][layer_idx])
layer_name = 'output_layer_%s' % layer_idx
outs[layer_name]['o_c'] = o
outs[layer_name]['sigma_c'] = sigma
return outs
def _apply_res_layer(self, ins, res_vars, rnn_part, scope):
with tf.name_scope(scope):
outs = construct(ins)
if self._emb_layer_is_present:
core_inps = [
ins['embedding_layer']['o_c'],
ins['embedding_layer']['sigma_c'],
ins['lstm_layer_0']['o_c'], ins['lstm_layer_0']['sigma_c']
]
target = [
ins['embedding_layer']['o_c'],
ins['embedding_layer']['sigma_c']
]
# print('(ResNet4Lstm._apply_res_layer)core_inps:', core_inps)
# print('(ResNet4Lstm._apply_res_layer)res_vars:', res_vars)
o, sigma, emb_rnn_part = self._apply_res_core(
res_vars['embedding_layer'], core_inps, rnn_part, target,
'embedding_layer', self._pupil_dims['embedding_layer'])
outs['embedding_layer']['o_c'] = o
outs['embedding_layer']['sigma_c'] = sigma
else:
emb_rnn_part = 0
lstm_rnn_parts = list()
for layer_idx in range(self._pupil_net_size['num_layers']):
if layer_idx == 0:
if self._emb_layer_is_present:
previous_layer_tensors = [
ins['embedding_layer']['o_c'],
ins['embedding_layer']['sigma_c']
]
else:
previous_layer_tensors = []
else:
previous_layer_tensors = [
ins['lstm_layer_%s' % (layer_idx - 1)]['o_c'],
ins['lstm_layer_%s' % (layer_idx - 1)]['sigma_c']
]
if layer_idx == self._pupil_net_size['num_layers'] - 1:
next_layer_tensors = [
ins['output_layer_0']['o_c'],
ins['output_layer_0']['sigma_c']
]
else:
next_layer_tensors = [
ins['lstm_layer_%s' % (layer_idx + 1)]['o_c'],
ins['lstm_layer_%s' % (layer_idx + 1)]['sigma_c']
]
layer_name = 'lstm_layer_%s' % layer_idx
core_inps = [
*previous_layer_tensors, ins[layer_name]['o_c'],
ins[layer_name]['sigma_c'],
self._pad(ins[layer_name]['o_c'], -1),
self._pad(ins[layer_name]['sigma_c'], -1),
self._pad(ins[layer_name]['o_c'], 1),
self._pad(ins[layer_name]['sigma_c'],
1), *next_layer_tensors
]
target = [ins[layer_name]['o_c'], ins[layer_name]['sigma_c']]
o, sigma, lstm_rnn_part = self._apply_res_core(
res_vars[layer_name], core_inps, rnn_part, target,
layer_name, self._pupil_dims['lstm_layers'][layer_idx])
lstm_rnn_parts.append(lstm_rnn_part)
outs[layer_name]['o_c'] = o
outs[layer_name]['sigma_c'] = sigma
output_rnn_parts = list()
for layer_idx in range(self._pupil_net_size['num_output_layers']):
if layer_idx == 0:
previous_layer_tensors = [
ins['lstm_layer_%s' %
(self._pupil_net_size['num_layers'] - 1)]['o_c'],
ins['lstm_layer_%s' %
(self._pupil_net_size['num_layers'] -
1)]['sigma_c']
]
else:
previous_layer_tensors = [
ins['output_layer_%s' % (layer_idx - 1)]['o_c'],
ins['output_layer_%s' % (layer_idx - 1)]['sigma_c']
]
if layer_idx == self._pupil_net_size['num_output_layers'] - 1:
next_layer_tensors = []
else:
next_layer_tensors = [
ins['output_layer_%s' % (layer_idx + 1)]['o_c'],
ins['output_layer_%s' % (layer_idx + 1)]['sigma_c']
]
core_inps = [
*previous_layer_tensors,
ins['output_layer_%s' % layer_idx]['o_c'],
ins['output_layer_%s' % layer_idx]['sigma_c'],
*next_layer_tensors
]
target = [
ins['output_layer_%s' % layer_idx]['o_c'],
ins['output_layer_%s' % layer_idx]['sigma_c']
]
layer_name = 'output_layer_%s' % layer_idx
# print("(ResNet4Lstm._apply_res_layer)layer_idx:", layer_idx)
# print("(ResNet4Lstm._apply_res_layer)core_inps:", core_inps)
# print("(ResNet4Lstm._apply_res_layer)rnn_part:", rnn_part)
o, sigma, output_rnn_part = self._apply_res_core(
res_vars[layer_name], core_inps, rnn_part, target,
layer_name, self._pupil_dims['output_layers'][layer_idx])
output_rnn_parts.append(output_rnn_part)
layer_name = 'output_layer_%s' % layer_idx
outs[layer_name]['o_c'] = o
outs[layer_name]['sigma_c'] = sigma
# print("(ResNet4Lstm._apply_res_layer)emb_rnn_part:", emb_rnn_part)
# print("(ResNet4Lstm._apply_res_layer)lstm_rnn_parts:", lstm_rnn_parts)
# print("(ResNet4Lstm._apply_res_layer)output_rnn_parts:", output_rnn_parts)
return outs, emb_rnn_part + sum(lstm_rnn_parts + output_rnn_parts)