def parse_1_set_of_kwargs(env_instance, kwargs_to_parse, method_name, repeated_key, only_repeated, old_arguments=None): # print('\n\n_parse_1_set_of_kwargs method:\nkwargs_to_parse:\n', kwargs_to_parse, '\nmethod_name:\n', # method_name, '\nrepeated_key:\n', repeated_key, '\nonly_repeated:\n', only_repeated, '\nold_arguments:\n', # old_arguments) # print("(parse_1_set_of_kwargs)kwargs_to_parse:", kwargs_to_parse) kwargs_to_parse = construct(kwargs_to_parse) process_abbreviations(env_instance, kwargs_to_parse, method_name) # print("(parse_1_set_of_kwargs)kwargs_to_parse:", kwargs_to_parse) # print("(parse_1_set_of_kwargs)kwargs_to_parse['valid_batch_kwargs']:", kwargs_to_parse['valid_batch_kwargs']) if old_arguments is None: current_arguments = env_instance.get_default_method_parameters( method_name) if only_repeated: current_arguments = current_arguments[repeated_key] else: current_arguments = construct(old_arguments) # print("(parse_1_set_of_kwargs)current_arguments:", current_arguments) for key, value in kwargs_to_parse.items(): paste_into_nested_structure(current_arguments, key, value) #print('current_arguments:\n', current_arguments) return current_arguments
def create_one_combination_insertions(insert_template, hp_combination): # print('\ncreate_one_combination_insertions') combination_insertions = construct(insert_template) for hp_name, value in hp_combination.items(): # print('hp_name:', hp_name) # print('value:', value) insert_key = (hp_name[0], hp_name[1], hp_name[2]) single_tmpl = combination_insertions[insert_key] # print('single_tmpl:', single_tmpl) # print('hp_name[0]:', hp_name[0]) # print('hp_name[3]:', hp_name[3]) if hp_name[0] == 'additional_placeholder': if hp_name[3] is not None: single_tmpl['paste']['value'][hp_name[3]] = value else: single_tmpl['paste']['value'] = value else: if hp_name[3] is not None: single_tmpl['paste'][hp_name[3]] = value else: single_tmpl['paste'] = value if single_tmpl['share'] is not None: share = single_tmpl['share'] if share['hp_type'] == 'additional_placeholder': share['paste']['value']['value'] = value else: share['paste']['value'] = value # print('\n') return combination_insertions
def process_other_hp_specs_absences(other_hps): other_hps = construct(other_hps) new_other_hps = dict() for hp_name, hp_values_and_specs in other_hps.items(): if isinstance(hp_values_and_specs, list): new_other_hps.update( process_other_hp_dictionary_absence(hp_name, hp_values_and_specs)) else: if 'list_indices' not in hp_values_and_specs: hp_values_and_specs['list_indices'] = None elif isinstance(hp_values_and_specs['list_indices'], int): hp_values_and_specs['list_indices'] = [ hp_values_and_specs['list_indices'] ] if 'hp_type' not in hp_values_and_specs: hp_values_and_specs['hp_type'] = 'additional_placeholder' if 'controller' not in hp_values_and_specs: hp_values_and_specs['controller'] = True if 'type' not in hp_values_and_specs: if hp_values_and_specs['controller']: hp_values_and_specs['type'] = 'fixed' else: hp_values_and_specs['type'] = None if 'fixed' not in hp_values_and_specs: if hp_values_and_specs['controller']: hp_values_and_specs['fixed'] = dict() else: hp_values_and_specs['fixed'] = None hp_values_and_specs['share'] = None new_other_hps[hp_name] = hp_values_and_specs return new_other_hps
def process_build_hp_abbreviations(build_hps): build_hps = construct(build_hps) new_build_hps = dict() for hp_name, hp_values_and_specs in build_hps.items(): if isinstance(hp_values_and_specs, list): new_build_hps.update( process_build_hp_text_abbreviation(hp_name, hp_values_and_specs)) else: if 'hp_type' not in hp_values_and_specs: hp_values_and_specs['hp_type'] = 'build_hp' if 'list_indices' not in hp_values_and_specs: hp_values_and_specs['list_indices'] = None elif isinstance(hp_values_and_specs['list_indices'], int): hp_values_and_specs['list_indices'] = [ hp_values_and_specs['list_indices'] ] if 'share' not in hp_values_and_specs: hp_values_and_specs['share'] = None hp_values_and_specs['controller'] = False hp_values_and_specs['type'] = None hp_values_and_specs['fixed'] = None if hp_values_and_specs['share'] is not None: if 'direction' not in hp_values_and_specs['share']: default = spot_direction(hp_name) if spot_direction(hp_name) is None: hp_values_and_specs['share'][ 'direction'] = 'batch_kwarg' else: hp_values_and_specs['share']['direction'] = default if 'controller' not in hp_values_and_specs['share']: hp_values_and_specs['share']['controller'] = True new_build_hps[hp_name] = hp_values_and_specs new_build_hps = reshape_1_index_hps(new_build_hps) return new_build_hps
def expand_varying_entry(hps): hps = construct(hps) new_hps = dict() for hp_name, hp_values_and_specs in hps.items(): if hp_values_and_specs['controller'] and isinstance(hp_values_and_specs['varying'], list): hp_values_and_specs['varying'] = {'value': hp_values_and_specs['varying']} new_hps[hp_name] = hp_values_and_specs return new_hps
def process_abbreviation_in_1_entry(key, value, method_name): new_value = construct(value) if key == 'stop': if isinstance(value, int): new_value = {'type': 'limit_steps', 'limit': value} set_controller_name_in_specs(new_value, 'stop') if key == 'batch_size': if isinstance(value, int): new_value = {'type': 'fixed', 'value': value} set_controller_name_in_specs(new_value, 'batch_size') if key == 'num_unrollings': if isinstance(value, int): new_value = {'type': 'fixed', 'value': value} set_controller_name_in_specs(new_value, 'num_unrollings') if key == 'checkpoint_steps': if isinstance(value, list): new_value = {'type': 'true_on_steps', 'steps': value} elif isinstance(value, int): new_value = {'type': 'periodic_truth', 'period': value} else: new_value = {'type': 'always_false'} set_controller_name_in_specs(new_value, 'checkpoint_steps') if key == 'learning_rate': set_controller_name_in_specs(new_value, 'learning_rate') if key == 'debug': if isinstance(value, int): new_value = {'type': 'true_on_steps', 'steps': [value]} else: new_value = None set_controller_name_in_specs(new_value, 'debug') if key == 'opt_inf_stop': if isinstance(value, int): new_value = {'type': 'limit_steps', 'limit': value} set_controller_name_in_specs(new_value, 'optimizer_inference_num_steps') if key == 'reset_period': if isinstance(value, int): new_value = {'type': 'periodic_truth', 'period': value} set_controller_name_in_specs(new_value, 'optimizer_inference_num_steps') if method_name == 'train' or method_name == 'train_optimizer': if key == 'additions_to_feed_dict' or key == 'opt_inf_additions_to_feed_dict': # print('inside additions_to_feed_dict shortcuts processing') if new_value is not None: for addition in new_value: # print('addition:', addition) if not isinstance(addition['value'], dict): # print('Removing of shortcut is happening now') addition['value'] = { 'type': 'fixed', 'value': addition['value'] } # print("addition['value']:", addition['value']) set_controller_name_in_specs(addition['value'], addition['placeholder']) return new_value
def create_change_tracking_specifications(specifications): if isinstance(specifications, list): old_specs = construct(specifications) if isinstance(specifications, dict): old_specs = [dict(specifications)] new_specs = dict() new_specs['old_specs'] = old_specs new_specs['type'] = 'changes_detector' return new_specs
def _optimizer_core(self, optimizer_ins, state, gpu_idx, permute=True): new_state = construct(state) # print(state) for ok, ov in optimizer_ins.items(): with tf.name_scope(ok): for inp_name in self._selected: if inp_name in ov: with tf.name_scope(inp_name): ov[inp_name + '_pr'], new_state[ok][ inp_name] = self._apply_net( ov[inp_name], state[ok][inp_name], self._opt_trainable[ok][inp_name]) # for ok, ov in optimizer_ins.items(): # for ik, iv in ov.items(): # print() # print(ok, ik) # print(iv) return optimizer_ins, new_state
def _apply_res_layer(self, ins, res_vars, rnn_part, scope): with tf.name_scope(scope): outs = construct(ins) rnn_parts = list() for layer_idx in range(self._pupil_net_size['num_layers']): if layer_idx == 0: previous_layer_tensors = [] else: previous_layer_tensors = [ ins['layer_%s' % (layer_idx - 1)]['o'], ins['layer_%s' % (layer_idx - 1)]['sigma'] ] if layer_idx == self._pupil_net_size['num_layers'] - 1: next_layer_tensors = [] else: next_layer_tensors = [ ins['layer_%s' % (layer_idx + 1)]['o'], ins['layer_%s' % (layer_idx + 1)]['sigma'] ] layer_name = 'layer_%s' % layer_idx core_inps = [ *previous_layer_tensors, ins[layer_name]['o'], ins[layer_name]['sigma'], *next_layer_tensors ] target = [ ins[layer_name]['o'], ins[layer_name]['sigma'] ] o, sigma, lstm_rnn_part = self._apply_res_core( res_vars[layer_name], core_inps, rnn_part, target, layer_name, self._pupil_dims['layers'][layer_idx]) rnn_parts.append(lstm_rnn_part) outs[layer_name]['o'] = o outs[layer_name]['sigma'] = sigma # print("(ResNet4Lstm._apply_res_layer)emb_rnn_part:", emb_rnn_part) # print("(ResNet4Lstm._apply_res_layer)lstm_rnn_parts:", lstm_rnn_parts) # print("(ResNet4Lstm._apply_res_layer)output_rnn_parts:", output_rnn_parts) return outs, sum(rnn_parts)
def configure_args_for_launches(env_instance, args_for_launches, shares, model='pupil'): args_for_launches_to_be_used = construct(args_for_launches) parsed = list() for to_be_used in args_for_launches_to_be_used: with_shares = apply_shares(to_be_used, shares) if model == 'pupil': one_parsed = parse_train_method_arguments(env_instance, [], with_shares, set_passed_parameters_as_default=False) else: one_parsed = parse_train_optimizer_method_arguments( env_instance, [], with_shares, set_passed_parameters_as_default=False ) start_specs = one_parsed['start_specs'] run_specs_set = one_parsed['run'] del one_parsed['session_specs'] parsed.append((start_specs, run_specs_set)) return parsed
def _apply_layer(self, ins, vars, scope): with tf.name_scope(scope): outs = construct(ins) if self._emb_layer_is_present: target = [ ins['embedding_layer']['o_c'], ins['embedding_layer']['sigma_c'] ] o, sigma = self._apply_core( vars['embedding_layer'], target, 'embedding_layer', self._pupil_dims['embedding_layer']) outs['embedding_layer']['o_c'] = o outs['embedding_layer']['sigma_c'] = sigma for layer_idx in range(self._pupil_net_size['num_layers']): layer_name = 'lstm_layer_%s' % layer_idx target = [ ins[layer_name]['o_c'], ins[layer_name]['sigma_c'] ] o, sigma = self._apply_core( vars[layer_name], target, layer_name, self._pupil_dims['lstm_layers'][layer_idx]) outs[layer_name]['o_c'] = o outs[layer_name]['sigma_c'] = sigma for layer_idx in range(self._pupil_net_size['num_output_layers']): target = [ ins['output_layer_%s' % layer_idx]['o_c'], ins['output_layer_%s' % layer_idx]['sigma_c'] ] layer_name = 'output_layer_%s' % layer_idx o, sigma = self._apply_core( vars[layer_name], target, layer_name, self._pupil_dims['output_layers'][layer_idx]) layer_name = 'output_layer_%s' % layer_idx outs[layer_name]['o_c'] = o outs[layer_name]['sigma_c'] = sigma return outs
def create_1_set_of_args_for_launches(kwargs, one_set_of_args_insertions): kwargs = construct(kwargs) for one_hp_insertion in one_set_of_args_insertions: kwargs = insert_not_build_hp(kwargs, one_hp_insertion) return kwargs
def _apply_res_layer(self, ins, res_vars, scope): with tf.name_scope(scope): outs = construct(ins) if self._emb_layer_is_present: core_inps = [ ins['embedding_layer']['o_c'], ins['embedding_layer']['sigma_c'], ins['lstm_layer_0']['o_c'], ins['lstm_layer_0']['sigma_c'] ] target = [ ins['embedding_layer']['o_c'], ins['embedding_layer']['sigma_c'] ] o, sigma = self._apply_res_core( res_vars['embedding_layer'], core_inps, target, 'embedding_layer', self._pupil_dims['embedding_layer']) outs['embedding_layer']['o_c'] = o outs['embedding_layer']['sigma_c'] = sigma for layer_idx in range(self._pupil_net_size['num_layers']): if layer_idx == 0: if self._emb_layer_is_present: previous_layer_tensors = [ ins['embedding_layer']['o_c'], ins['embedding_layer']['sigma_c'] ] else: previous_layer_tensors = [] else: previous_layer_tensors = [ ins['lstm_layer_%s' % (layer_idx - 1)]['o_c'], ins['lstm_layer_%s' % (layer_idx - 1)]['sigma_c'] ] if layer_idx == self._pupil_net_size['num_layers'] - 1: next_layer_tensors = [ ins['output_layer_0']['o_c'], ins['output_layer_0']['sigma_c'] ] else: next_layer_tensors = [ ins['lstm_layer_%s' % (layer_idx + 1)]['o_c'], ins['lstm_layer_%s' % (layer_idx + 1)]['sigma_c'] ] layer_name = 'lstm_layer_%s' % layer_idx core_inps = [ *previous_layer_tensors, ins[layer_name]['o_c'], ins[layer_name]['sigma_c'], self._pad(ins[layer_name]['o_c'], -1), self._pad(ins[layer_name]['sigma_c'], -1), self._pad(ins[layer_name]['o_c'], 1), self._pad(ins[layer_name]['sigma_c'], 1), *next_layer_tensors ] target = [ ins[layer_name]['o_c'], ins[layer_name]['sigma_c'] ] o, sigma = self._apply_res_core( res_vars[layer_name], core_inps, target, layer_name, self._pupil_dims['lstm_layers'][layer_idx]) outs[layer_name]['o_c'] = o outs[layer_name]['sigma_c'] = sigma for layer_idx in range(self._pupil_net_size['num_output_layers']): if layer_idx == 0: previous_layer_tensors = [ ins['lstm_layer_%s' % (self._pupil_net_size['num_layers'] - 1)]['o_c'], ins['lstm_layer_%s' % (self._pupil_net_size['num_layers'] - 1)]['sigma_c'] ] else: previous_layer_tensors = [ ins['output_layer_%s' % (layer_idx - 1)]['o_c'], ins['output_layer_%s' % (layer_idx - 1)]['sigma_c'] ] if layer_idx == self._pupil_net_size['num_output_layers'] - 1: next_layer_tensors = [] else: next_layer_tensors = [ ins['output_layer_%s' % (layer_idx + 1)]['o_c'], ins['output_layer_%s' % (layer_idx + 1)]['sigma_c'] ] core_inps = [ *previous_layer_tensors, ins['output_layer_%s' % layer_idx]['o_c'], ins['output_layer_%s' % layer_idx]['sigma_c'], *next_layer_tensors ] target = [ ins['output_layer_%s' % layer_idx]['o_c'], ins['output_layer_%s' % layer_idx]['sigma_c'] ] layer_name = 'output_layer_%s' % layer_idx o, sigma = self._apply_res_core( res_vars[layer_name], core_inps, target, layer_name, self._pupil_dims['output_layers'][layer_idx]) layer_name = 'output_layer_%s' % layer_idx outs[layer_name]['o_c'] = o outs[layer_name]['sigma_c'] = sigma return outs
def _apply_res_layer(self, ins, res_vars, rnn_part, scope): with tf.name_scope(scope): outs = construct(ins) if self._emb_layer_is_present: core_inps = [ ins['embedding_layer']['o_c'], ins['embedding_layer']['sigma_c'], ins['lstm_layer_0']['o_c'], ins['lstm_layer_0']['sigma_c'] ] target = [ ins['embedding_layer']['o_c'], ins['embedding_layer']['sigma_c'] ] # print('(ResNet4Lstm._apply_res_layer)core_inps:', core_inps) # print('(ResNet4Lstm._apply_res_layer)res_vars:', res_vars) o, sigma, emb_rnn_part = self._apply_res_core( res_vars['embedding_layer'], core_inps, rnn_part, target, 'embedding_layer', self._pupil_dims['embedding_layer']) outs['embedding_layer']['o_c'] = o outs['embedding_layer']['sigma_c'] = sigma else: emb_rnn_part = 0 lstm_rnn_parts = list() for layer_idx in range(self._pupil_net_size['num_layers']): if layer_idx == 0: if self._emb_layer_is_present: previous_layer_tensors = [ ins['embedding_layer']['o_c'], ins['embedding_layer']['sigma_c'] ] else: previous_layer_tensors = [] else: previous_layer_tensors = [ ins['lstm_layer_%s' % (layer_idx - 1)]['o_c'], ins['lstm_layer_%s' % (layer_idx - 1)]['sigma_c'] ] if layer_idx == self._pupil_net_size['num_layers'] - 1: next_layer_tensors = [ ins['output_layer_0']['o_c'], ins['output_layer_0']['sigma_c'] ] else: next_layer_tensors = [ ins['lstm_layer_%s' % (layer_idx + 1)]['o_c'], ins['lstm_layer_%s' % (layer_idx + 1)]['sigma_c'] ] layer_name = 'lstm_layer_%s' % layer_idx core_inps = [ *previous_layer_tensors, ins[layer_name]['o_c'], ins[layer_name]['sigma_c'], self._pad(ins[layer_name]['o_c'], -1), self._pad(ins[layer_name]['sigma_c'], -1), self._pad(ins[layer_name]['o_c'], 1), self._pad(ins[layer_name]['sigma_c'], 1), *next_layer_tensors ] target = [ins[layer_name]['o_c'], ins[layer_name]['sigma_c']] o, sigma, lstm_rnn_part = self._apply_res_core( res_vars[layer_name], core_inps, rnn_part, target, layer_name, self._pupil_dims['lstm_layers'][layer_idx]) lstm_rnn_parts.append(lstm_rnn_part) outs[layer_name]['o_c'] = o outs[layer_name]['sigma_c'] = sigma output_rnn_parts = list() for layer_idx in range(self._pupil_net_size['num_output_layers']): if layer_idx == 0: previous_layer_tensors = [ ins['lstm_layer_%s' % (self._pupil_net_size['num_layers'] - 1)]['o_c'], ins['lstm_layer_%s' % (self._pupil_net_size['num_layers'] - 1)]['sigma_c'] ] else: previous_layer_tensors = [ ins['output_layer_%s' % (layer_idx - 1)]['o_c'], ins['output_layer_%s' % (layer_idx - 1)]['sigma_c'] ] if layer_idx == self._pupil_net_size['num_output_layers'] - 1: next_layer_tensors = [] else: next_layer_tensors = [ ins['output_layer_%s' % (layer_idx + 1)]['o_c'], ins['output_layer_%s' % (layer_idx + 1)]['sigma_c'] ] core_inps = [ *previous_layer_tensors, ins['output_layer_%s' % layer_idx]['o_c'], ins['output_layer_%s' % layer_idx]['sigma_c'], *next_layer_tensors ] target = [ ins['output_layer_%s' % layer_idx]['o_c'], ins['output_layer_%s' % layer_idx]['sigma_c'] ] layer_name = 'output_layer_%s' % layer_idx # print("(ResNet4Lstm._apply_res_layer)layer_idx:", layer_idx) # print("(ResNet4Lstm._apply_res_layer)core_inps:", core_inps) # print("(ResNet4Lstm._apply_res_layer)rnn_part:", rnn_part) o, sigma, output_rnn_part = self._apply_res_core( res_vars[layer_name], core_inps, rnn_part, target, layer_name, self._pupil_dims['output_layers'][layer_idx]) output_rnn_parts.append(output_rnn_part) layer_name = 'output_layer_%s' % layer_idx outs[layer_name]['o_c'] = o outs[layer_name]['sigma_c'] = sigma # print("(ResNet4Lstm._apply_res_layer)emb_rnn_part:", emb_rnn_part) # print("(ResNet4Lstm._apply_res_layer)lstm_rnn_parts:", lstm_rnn_parts) # print("(ResNet4Lstm._apply_res_layer)output_rnn_parts:", output_rnn_parts) return outs, emb_rnn_part + sum(lstm_rnn_parts + output_rnn_parts)