def train_on_files(self, shot_files, use_shots, all_machines, verbose=False): conf = self.conf all_signals = conf['paths']['all_signals'] shot_list = ShotList() shot_list.load_from_shot_list_files_objects(shot_files, all_signals) shot_list_picked = shot_list.random_sublist(use_shots) previously_saved, machines_saved = self.previously_saved_stats() machines_to_compute = all_machines - machines_saved recompute = conf['data']['recompute_normalization'] if recompute: machines_to_compute = all_machines previously_saved = False if not previously_saved or len(machines_to_compute) > 0: if previously_saved: self.load_stats(verbose=True) print('computing normalization for machines {}'.format( machines_to_compute)) use_cores = max(1, mp.cpu_count() - 2) pool = mp.Pool(use_cores) print('running in parallel on {} processes'.format( pool._processes)) start_time = time.time() for (i, stats) in enumerate( pool.imap_unordered(self.train_on_single_shot, shot_list_picked)): # for (i,stats) in # enumerate(map(self.train_on_single_shot,shot_list_picked)): if stats.machine in machines_to_compute: self.incorporate_stats(stats) self.machines.add(stats.machine) sys.stdout.write('\r' + '{}/{}'.format(i, len(shot_list_picked))) pool.close() pool.join() print( '\nFinished Training Normalizer on ', '{} files in {} seconds'.format(len(shot_list_picked), time.time() - start_time)) self.save_stats(verbose=True) else: self.load_stats(verbose=verbose) # print representation of trained Normalizer to stdout: # Machine, NormalizerName, per-signal normalization stats/params if verbose: g.print_unique(self)
def guarantee_preprocessed(conf, verbose=False): pp = Preprocessor(conf) if pp.all_are_preprocessed(): if verbose: g.print_unique("shots already processed.") (shot_list_train, shot_list_validate, shot_list_test) = pp.load_shotlists() else: if verbose: g.print_unique("preprocessing all shots...") # , end='') pp.clean_shot_lists() shot_list = pp.preprocess_all() shot_list.sort() shot_list_train, shot_list_test = shot_list.split_train_test(conf) # num_shots = len(shot_list_train) + len(shot_list_test) validation_frac = conf['training']['validation_frac'] if validation_frac <= 0.05: if verbose: g.print_unique('Setting validation to a minimum of 0.05') validation_frac = 0.05 shot_list_train, shot_list_validate = shot_list_train.split_direct( 1.0 - validation_frac, do_shuffle=True) pp.save_shotlists(shot_list_train, shot_list_validate, shot_list_test) shot_list_train, shot_list_validate, shot_list_test = apply_bleed_in( conf, shot_list_train, shot_list_validate, shot_list_test) if verbose: print_shot_list_sizes(shot_list_train, shot_list_validate, shot_list_test) g.print_unique("...done") # g.print_unique("...printing test shot list:") # for s in shot_list_test: # g.print_unique(str(s.number)) return shot_list_train, shot_list_validate, shot_list_test
def fetch_jet_data(signal_path, shot_num, c): found = False time = np.array([0]) ydata = None data = np.array([0]) try: data = c.get('_sig=jet("{}/",{})'.format(signal_path, shot_num)).data() if np.ndim(data) == 2: data = np.transpose(data) time = c.get('_sig=dim_of(jet("{}/",{}),1)'.format( signal_path, shot_num)).data() ydata = c.get('_sig=dim_of(jet("{}/",{}),0)'.format( signal_path, shot_num)).data() else: time = c.get('_sig=dim_of(jet("{}/",{}))'.format( signal_path, shot_num)).data() found = True except Exception as e: g.print_unique(e) sys.stdout.flush() # pass return time, data, ydata, found
def print_shot_list_sizes(shot_list_train, shot_list_validate, shot_list_test=None): nshots = len(shot_list_train) + len(shot_list_validate) nshots_disrupt = (shot_list_train.num_disruptive() + shot_list_validate.num_disruptive()) if shot_list_test is not None: nshots += len(shot_list_test) nshots_disrupt += shot_list_test.num_disruptive() g.print_unique('total: {} shots, {} disruptive'.format( nshots, nshots_disrupt)) g.print_unique('training: {} shots, {} disruptive'.format( len(shot_list_train), shot_list_train.num_disruptive())) g.print_unique('validate: {} shots, {} disruptive'.format( len(shot_list_validate), shot_list_validate.num_disruptive())) if shot_list_test is not None: g.print_unique('testing: {} shots, {} disruptive'.format( len(shot_list_test), shot_list_test.num_disruptive())) return
from plasma.preprocessor.normalize import (AveragingVarNormalizer as Normalizer) else: print('unkown normalizer. exiting') exit(1) # set PRNG seed, unique for each worker, based on MPI task index for # reproducible shuffling in guranteed_preprocessed() and training steps np.random.seed(g.task_index) random.seed(g.task_index) only_predict = len(sys.argv) > 1 custom_path = None if only_predict: custom_path = sys.argv[1] g.print_unique("predicting using path {}".format(custom_path)) ##################################################### # NORMALIZATION # ##################################################### normalizer = Normalizer(conf) if g.task_index == 0: # make sure preprocessing has been run, and results are saved to files # if not, only master MPI rank spawns thread pool to perform preprocessing (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) # similarly, train normalizer (if necessary) w/ master MPI rank only normalizer.train() # verbose=False only suppresses if purely loading g.comm.Barrier() g.print_unique("begin preprocessor+normalization (all MPI ranks)...") # second call has ALL MPI ranks load preprocessed shots from .npz files
from plasma.preprocessor.normalize import (AveragingVarNormalizer as Normalizer) else: print('unkown normalizer. exiting') exit(1) # set PRNG seed, unique for each worker, based on MPI task index for # reproducible shuffling in guranteed_preprocessed() and training steps np.random.seed(g.task_index) random.seed(g.task_index) only_predict = len(sys.argv) > 1 custom_path = None if only_predict: custom_path = sys.argv[1] g.print_unique("predicting using path {}".format(custom_path)) ##################################################### # NORMALIZATION # ##################################################### normalizer = Normalizer(conf) if g.task_index == 0: # make sure preprocessing has been run, and results are saved to files # if not, only master MPI rank spawns thread pool to perform preprocessing (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) # similarly, train normalizer (if necessary) w/ master MPI rank only normalizer.train() # verbose=False only suppresses if purely loading g.comm.Barrier() g.print_unique("begin preprocessor+normalization (all MPI ranks)...") # second call has ALL MPI ranks load preprocessed shots from .npz files
def parameters(input_file): """Parse yaml file of configuration parameters.""" # TODO(KGF): the following line imports TensorFlow as a Keras backend # by default (absent env variable KERAS_BACKEND and/or config file # $HOME/.keras/keras.json) "from plasma.conf import conf" # via "import keras.backend as K" in targets.py from plasma.models.targets import (HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget) with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file, Loader=yaml.SafeLoader) params['user_name'] = getpass.getuser() output_path = params['fs_path'] + "/" + params['user_name'] base_path = output_path params['paths']['base_path'] = base_path if isinstance(params['paths']['signal_prepath'], list): print( 'reading from multiple data folder!**********************************************' ) params['paths']['signal_prepath'] = [ base_path + s for s in params['paths']['signal_prepath'] ] else: params['paths']['signal_prepath'] = base_path + params['paths'][ 'signal_prepath'] params['paths']['shot_list_dir'] = (base_path + params['paths']['shot_list_dir']) params['paths']['output_path'] = output_path if params['paths']['data'] == 'd3d_data_gar18': h = myhash_signals(sig.all_signals_gar18.values()) if params['paths']['data'] == 'd3d_data_n1rms': h = myhash_signals(sig.all_signals_n1rms.values()) elif params['paths']['data'] == 'd3d_data_garbage': h = myhash_signals(sig.all_signals_gar18.values()) * 2 elif params['paths']['data'] == 'd3d_data_real_time': h = myhash_signals(sig.all_signals_real_time.values()) elif params['paths']['data'] == 'd3d_data_real_time_0D': h = myhash_signals(sig.all_signals_real_time_0D.values()) else: h = myhash_signals(sig.all_signals.values()) params['paths']['global_normalizer_path'] = ( output_path + '/normalization/normalization_signal_group_{}.npz'.format(h)) if params['training']['hyperparam_tuning']: # params['paths']['saved_shotlist_path'] = # './normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( './normalization/normalization_signal_group_{}.npz'.format(h)) params['paths']['model_save_path'] = './model_checkpoints/' params['paths']['csvlog_save_path'] = './csv_logs/' params['paths']['results_prepath'] = './results/' else: # params['paths']['saved_shotlist_path'] = output_path + # '/normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( params['paths']['global_normalizer_path']) params['paths']['model_save_path'] = (output_path + '/model_checkpoints/') params['paths']['csvlog_save_path'] = output_path + '/csv_logs/' params['paths']['results_prepath'] = output_path + '/results/' params['paths']['tensorboard_save_path'] = ( output_path + params['paths']['tensorboard_save_path']) params['paths']['saved_shotlist_path'] = ( params['paths']['base_path'] + '/processed_shotlists/' + params['paths']['data'] + '/shot_lists_signal_group_{}.npz'.format(h)) params['paths']['processed_prepath'] = (output_path + '/processed_shots/' + 'signal_group_{}/'.format(h)) # ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': MaxHingeTarget.fac = params['data']['positive_example_penalty'] params['data']['target'] = MaxHingeTarget elif params['target'] == 'binary': params['data']['target'] = BinaryTarget elif params['target'] == 'ttd': params['data']['target'] = TTDTarget elif params['target'] == 'ttdinv': params['data']['target'] = TTDInvTarget elif params['target'] == 'ttdlinear': params['data']['target'] = TTDLinearTarget else: g.print_unique('Unkown type of target. Exiting') exit(1) # params['model']['output_activation'] = # params['data']['target'].activation # binary crossentropy performs slightly better? # params['model']['loss'] = params['data']['target'].loss # signals if params['paths']['data'] in ['d3d_data_gar18', 'd3d_data_garbage']: params['paths']['all_signals_dict'] = sig.all_signals_gar18 elif params['paths']['data'] == 'd3d_data_n1rms': params['paths']['all_signals_dict'] = sig.all_signals_n1rms elif params['paths']['data'] == 'd3d_data_real_time': params['paths']['all_signals_dict'] = sig.all_signals_real_time elif params['paths']['data'] == 'd3d_data_real_time_0D': params['paths']['all_signals_dict'] = sig.all_signals_real_time_0D else: params['paths']['all_signals_dict'] = sig.all_signals # assert order # q95, li, ip, lm, betan, energy, dens, pradcore, pradedge, pin, # pechin, torquein, ipdirect, etemp_profile, edens_profile # shot lists jet_carbon_wall = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], ['CWall_clear.txt', 'CFC_unint.txt'], 'jet carbon wall data') jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint.txt', 'BeWall_clear.txt'], 'jet iter like wall data') jet_iterlike_wall_late = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint_late.txt', 'ILW_clear_late.txt'], 'Late jet iter like wall data') # jet_iterlike_wall_full = ShotListFiles( # sig.jet, params['paths']['shot_list_dir'], # ['ILW_unint_full.txt', 'ILW_clear_full.txt'], # 'Full jet iter like wall data') jenkins_jet_carbon_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_CWall_clear.txt', 'jenkins_CFC_unint.txt'], 'Subset of jet carbon wall data for Jenkins tests') jenkins_jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_ILW_unint.txt', 'jenkins_BeWall_clear.txt'], 'Subset of jet iter like wall data for Jenkins tests') jet_full = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], [ 'ILW_unint.txt', 'BeWall_clear.txt', 'CWall_clear.txt', 'CFC_unint.txt' ], 'jet full data') # d3d_10000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_10000.txt', 'd3d_disrupt_10000.txt'], # 'd3d data 10000 ND and D shots') # d3d_1000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_1000.txt', 'd3d_disrupt_1000.txt'], # 'd3d data 1000 ND and D shots') # d3d_100 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_100.txt', 'd3d_disrupt_100.txt'], # 'd3d data 100 ND and D shots') d3d_full = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['d3d_clear_data_avail.txt', 'd3d_disrupt_data_avail.txt'], 'd3d data since shot 125500') d3d_full_new = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['shots_since_2016_clear.txt', 'shots_since_2016_disrupt.txt'], 'd3d data since shot 125500') d3d_jenkins = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['jenkins_d3d_clear.txt', 'jenkins_d3d_disrupt.txt'], 'Subset of d3d data for Jenkins test') # d3d_jb_full = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['shotlist_JaysonBarr_clear.txt', # 'shotlist_JaysonBarr_disrupt.txt'], # 'd3d shots since 160000-170000') # nstx_full = ShotListFiles( # nstx, params['paths']['shot_list_dir'], # ['disrupt_nstx.txt'], 'nstx shots (all are disruptive') if params['paths']['data'] == 'jet_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_data_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_data_1D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_1D elif params['paths']['data'] == 'jet_data_late': params['paths']['shot_files'] = [jet_iterlike_wall_late] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_data_carbon_to_late_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_data_temp_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } elif params['paths']['data'] == 'jet_data_dens_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } elif params['paths']['data'] == 'jet_carbon_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_mixed_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jenkins_jet': params['paths']['shot_files'] = [jenkins_jet_carbon_wall] params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals # jet data but with fully defined signals elif params['paths']['data'] == 'jet_data_fully_defined': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals # jet data but with fully defined signals elif params['paths']['data'] == 'jet_data_fully_defined_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] in ['d3d_data_gar18', 'd3d_data_garbage']: params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95t': sig.q95t, 'lit': sig.lit, 'ipt': sig.ipt, 'lmt': sig.lmt, 'betant': sig.betant, 'energyt': sig.energyt, 'denst': sig.denst, 'pradcoret': sig.pradcoret, 'pradedget': sig.pradedget, 'pint': sig.pint, 'torqueint': sig.torqueint, 'ipdirectt': sig.ipdirectt, 'iptargett': sig.iptargett, 'iperrt': sig.iperrt, 'etemp_profilet': sig.etemp_profilet, 'edens_profilet': sig.edens_profilet, } elif params['paths']['data'] in ['d3d_data_n1rms']: params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95t': sig.q95, 'lit': sig.li, 'ipt': sig.ip, 'lmt': sig.lm, 'betant': sig.betan, 'energyt': sig.energy, 'denst': sig.dens, 'pradcoret': sig.pradcore, 'pradedget': sig.pradedge, 'pint': sig.pin, 'torqueint': sig.torquein, 'ipdirectt': sig.ipdirect, 'iptargett': sig.iptarget, 'iperrt': sig.iperr, 'etemp_profilet': sig.etemp_profile, 'edens_profilet': sig.edens_profile, 'n1_rms': sig.n1_rms, } elif params['paths']['data'] == 'd3d_data_new': params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_data_real_time': params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95_EFITRT1': sig.q95_EFITRT1, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_data_real_time_0D': params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95_EFITRT1': sig.q95_EFITRT1, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'vd': sig.vd, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, } elif params['paths']['data'] == 'd3d_data_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_data_all_profiles': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, 'itemp_profile': sig.itemp_profile, 'zdens_profile': sig.zdens_profile, 'trot_profile': sig.trot_profile, 'pthm_profile': sig.pthm_profile, 'neut_profile': sig.neut_profile, 'q_profile': sig.q_profile, 'bootstrap_current_profile': sig.bootstrap_current_profile, 'q_psi_profile': sig.q_psi_profile, } elif params['paths']['data'] == 'd3d_data_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, } elif params['paths']['data'] == 'd3d_data_all': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.d3d_signals elif params['paths']['data'] == 'jenkins_d3d': params['paths']['shot_files'] = [d3d_jenkins] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_data_fully_defined': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_data_fully_defined_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data_temp_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } # fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data_dens_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } # fully_defined_signals_0D # cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_jet_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_late_jet_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_data_0D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_to_jet_data_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'jet_to_d3d_data_1D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D elif params['paths']['data'] == 'd3d_to_jet_data_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D else: g.print_unique("Unknown dataset {}".format( params['paths']['data'])) exit(1) if len(params['paths']['specific_signals']): for s in params['paths']['specific_signals']: if s not in params['paths']['use_signals_dict'].keys(): g.print_unique( "Signal {} is not fully defined for {} machine. ", "Skipping...".format( s, params['paths']['data'].split("_")[0])) params['paths']['specific_signals'] = list( filter( lambda x: x in params['paths']['use_signals_dict'].keys(), params['paths']['specific_signals'])) selected_signals = { k: params['paths']['use_signals_dict'][k] for k in params['paths']['specific_signals'] } params['paths']['use_signals'] = sort_by_channels( list(selected_signals.values())) else: # default case params['paths']['use_signals'] = sort_by_channels( list(params['paths']['use_signals_dict'].values())) params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) g.print_unique( "Selected signals (determines which signals are used" + " for training):\n{}".format(params['paths']['use_signals'])) params['paths']['shot_files_all'] = ( params['paths']['shot_files'] + params['paths']['shot_files_test']) params['paths']['all_machines'] = list( set([file.machine for file in params['paths']['shot_files_all']])) # type assertations assert (isinstance(params['data']['signal_to_augment'], str) or isinstance(params['data']['signal_to_augment'], None)) assert isinstance(params['data']['augment_during_training'], bool) return params
def mpi_train(conf, shot_list_train, shot_list_validate, loader, callbacks_list=None, shot_list_test=None): loader.set_inference_mode(False) # TODO(KGF): this is not defined in conf.yaml, but added to processed dict # for the first time here: conf['num_workers'] = g.comm.Get_size() specific_builder = builder.ModelBuilder(conf) if g.tf_ver >= parse_version('1.14.0'): # Internal TensorFlow flags, subject to change (v1.14.0+ only?) try: from tensorflow.python.util import module_wrapper as depr except ImportError: from tensorflow.python.util import deprecation_wrapper as depr # depr._PRINT_DEPRECATION_WARNINGS = False # does nothing depr._PER_MODULE_WARNING_LIMIT = 0 # Suppresses warnings from "keras/backend/tensorflow_backend.py" # except: "Rate should be set to `rate = 1 - keep_prob`" # Also suppresses warnings from "keras/optimizers.py # does NOT suppresses warn from "/tensorflow/python/ops/math_grad.py" else: # TODO(KGF): next line suppresses ALL info and warning messages, # not just deprecation warnings... tf.logging.set_verbosity(tf.logging.ERROR) # TODO(KGF): for TF>v1.13.0 (esp v1.14.0), this next line prompts a ton of # deprecation warnings with externally-packaged Keras, e.g.: # WARNING:tensorflow:From .../keras/backend/tensorflow_backend.py:174: # The name tf.get_default_session is deprecated. # Please use tf.compat.v1.get_default_session instead. train_model = specific_builder.build_model(False) # Cannot fix these Keras internals via "import tensorflow.compat.v1 as tf" # # TODO(KGF): note, these are different than C-based info diagnostics e.g.: # 2019-11-06 18:27:31.698908: I ... dynamic library libcublas.so.10 # which are NOT suppressed by set_verbosity. See top level __init__.py # load the latest epoch we did. Returns 0 if none exist yet e = specific_builder.load_model_weights(train_model) e_old = e num_epochs = conf['training']['num_epochs'] lr_decay = conf['model']['lr_decay'] batch_size = conf['training']['batch_size'] lr = conf['model']['lr'] clipnorm = conf['model']['clipnorm'] warmup_steps = conf['model']['warmup_steps'] # TODO(KGF): rename as "num_iter_minimum" or "min_steps_per_epoch" num_batches_minimum = conf['training']['num_batches_minimum'] if 'adam' in conf['model']['optimizer']: optimizer = MPIAdam(lr=lr) elif (conf['model']['optimizer'] == 'sgd' or conf['model']['optimizer'] == 'tf_sgd'): optimizer = MPISGD(lr=lr) elif 'momentum_sgd' in conf['model']['optimizer']: optimizer = MPIMomentumSGD(lr=lr) else: print("Optimizer not implemented yet") exit(1) g.print_unique('{} epoch(s) left to go'.format(num_epochs - e)) batch_generator = partial(loader.training_batch_generator_partial_reset, shot_list=shot_list_train) g.print_unique("warmup steps = {}".format(warmup_steps)) mpi_model = MPIModel(train_model, optimizer, g.comm, batch_generator, batch_size, lr=lr, warmup_steps=warmup_steps, num_batches_minimum=num_batches_minimum, conf=conf) mpi_model.compile(conf['model']['optimizer'], clipnorm, conf['data']['target'].loss) tensorboard = None if g.task_index == 0: tensorboard_save_path = conf['paths']['tensorboard_save_path'] write_grads = conf['callbacks']['write_grads'] tensorboard = TensorBoard(log_dir=tensorboard_save_path, histogram_freq=1, write_graph=True, write_grads=write_grads) tensorboard.set_model(mpi_model.model) # TODO(KGF): check addition of TF model summary write added from fork fr = open('model_architecture.log', 'a') ori = sys.stdout sys.stdout = fr mpi_model.model.summary() sys.stdout = ori fr.close() mpi_model.model.summary() if g.task_index == 0: callbacks = mpi_model.build_callbacks(conf, callbacks_list) callbacks.set_model(mpi_model.model) callback_metrics = conf['callbacks']['metrics'] callbacks.set_params({ 'epochs': num_epochs, 'metrics': callback_metrics, 'batch_size': batch_size, }) callbacks.on_train_begin() if conf['callbacks']['mode'] == 'max': best_so_far = -np.inf cmp_fn = max else: best_so_far = np.inf cmp_fn = min while e < num_epochs: g.write_unique('\nBegin training from epoch {:.2f}/{}'.format( e, num_epochs)) if g.task_index == 0: callbacks.on_epoch_begin(int(round(e))) mpi_model.set_lr(lr * lr_decay**e) # KGF: core work of loop performed in next line (step, ave_loss, curr_loss, num_so_far, effective_epochs) = mpi_model.train_epoch() e = e_old + effective_epochs g.write_unique('Finished training of epoch {:.2f}/{}\n'.format( e, num_epochs)) # TODO(KGF): add diagnostic about "saving to epoch X"? loader.verbose = False # True during the first iteration if g.task_index == 0: specific_builder.save_model_weights(train_model, int(round(e))) if conf['training']['no_validation']: break epoch_logs = {} g.write_unique('Begin evaluation of epoch {:.2f}/{}\n'.format( e, num_epochs)) # TODO(KGF): flush output/ MPI barrier? # g.flush_all_inorder() # TODO(KGF): is there a way to avoid Keras.Models.load_weights() # repeated calls throughout mpi_make_pred*() fn calls? _, _, _, roc_area, loss = mpi_make_predictions_and_evaluate( conf, shot_list_validate, loader) if conf['training']['ranking_difficulty_fac'] != 1.0: (_, _, _, roc_area_train, loss_train) = mpi_make_predictions_and_evaluate( conf, shot_list_train, loader) batch_generator = partial( loader.training_batch_generator_partial_reset, shot_list=shot_list_train) mpi_model.batch_iterator = batch_generator mpi_model.batch_iterator_func.__exit__() mpi_model.num_so_far_accum = mpi_model.num_so_far_indiv mpi_model.set_batch_iterator_func() if ('monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']): times = conf['callbacks']['monitor_times'] areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_validate, loader, times) epoch_str = 'epoch {}, '.format(int(round(e))) g.write_unique(epoch_str + ' '.join([ 'val_roc_{} = {}'.format(t, roc) for t, roc in zip(times, areas) ]) + '\n') if shot_list_test is not None: areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_test, loader, times) g.write_unique(epoch_str + ' '.join([ 'test_roc_{} = {}'.format(t, roc) for t, roc in zip(times, areas) ]) + '\n') epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']], best_so_far) stop_training = False g.flush_all_inorder() if g.task_index == 0: print('=========Summary======== for epoch {:.2f}'.format(e)) print('Training Loss numpy: {:.3e}'.format(ave_loss)) print('Validation Loss: {:.3e}'.format(loss)) print('Validation ROC: {:.4f}'.format(roc_area)) if conf['training']['ranking_difficulty_fac'] != 1.0: print('Training Loss: {:.3e}'.format(loss_train)) print('Training ROC: {:.4f}'.format(roc_area_train)) print('======================== ') callbacks.on_epoch_end(int(round(e)), epoch_logs) if hasattr(mpi_model.model, 'stop_training'): stop_training = mpi_model.model.stop_training # only save model weights if quantity we are tracking is improving if best_so_far != epoch_logs[conf['callbacks']['monitor']]: if ('monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']): print("No improvement, saving model weights anyways") else: print("Not saving model weights") specific_builder.delete_model_weights( train_model, int(round(e))) # tensorboard val_generator = partial(loader.training_batch_generator, shot_list=shot_list_validate)() val_steps = 1 tensorboard.on_epoch_end(val_generator, val_steps, int(round(e)), epoch_logs) stop_training = g.comm.bcast(stop_training, root=0) g.write_unique('Finished evaluation of epoch {:.2f}/{}'.format( e, num_epochs)) # TODO(KGF): compare to old diagnostic: # g.write_unique("end epoch {}".format(e_old)) if stop_training: g.write_unique("Stopping training due to early stopping") break if g.task_index == 0: callbacks.on_train_end() tensorboard.on_train_end() mpi_model.close()
def train_epoch(self): ''' Perform distributed mini-batch SGD for one epoch. It takes the batch iterator function and a NN model from MPIModel object, fetches mini-batches in a while-loop until number of samples seen by the ensemble of workers (num_so_far) exceeds the training dataset size (num_total). NOTE: "sample" = "an entire shot" within this description During each iteration, the gradient updates (deltas) and the loss are calculated for each model replica in the ensemble, weights are averaged over ensemble, and the new weights are set. It performs calls to: MPIModel.get_deltas, MPIModel.set_new_weights methods Argument list: Empty Returns: - step: final iteration number - ave_loss: model loss averaged over iterations within this epoch - curr_loss: training loss averaged over replicas at final iteration - num_so_far: the cumulative number of samples seen by the ensemble of replicas up to the end of the final iteration (step) of this epoch Intermediate outputs and logging: debug printout of task_index (MPI), epoch number, number of samples seen to a current epoch, average training loss ''' verbose = False first_run = True step = 0 loss_averager = Averager() t_start = time.time() timeline_prof = False if (self.conf is not None and conf['training']['timeline_prof']): timeline_prof = True step_limit = 0 if (self.conf is not None and conf['training']['step_limit'] > 0): step_limit = conf['training']['step_limit'] batch_iterator_func = self.batch_iterator_func num_total = 1 ave_loss = -1 curr_loss = -1 t0 = 0 t1 = 0 t2 = 0 while ((self.num_so_far - self.epoch * num_total) < num_total or step < self.num_batches_minimum): if step_limit > 0 and step > step_limit: print('reached step limit') break try: (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, num_total, is_warmup_period) = next(batch_iterator_func) except StopIteration: g.print_unique("Resetting batch iterator.") self.num_so_far_accum = self.num_so_far_indiv self.set_batch_iterator_func() batch_iterator_func = self.batch_iterator_func (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, num_total, is_warmup_period) = next(batch_iterator_func) self.num_so_far_indiv = self.num_so_far_accum + num_so_far_curr # if batches_to_reset: # self.model.reset_states(batches_to_reset) warmup_phase = (step < self.warmup_steps and self.epoch == 0) num_replicas = 1 if warmup_phase else self.num_replicas self.num_so_far = self.mpi_sum_scalars(self.num_so_far_indiv, num_replicas) # run the model once to force compilation. Don't actually use these # values. if first_run: first_run = False t0_comp = time.time() # print('input_dimension:',batch_xs.shape) # print('output_dimension:',batch_ys.shape) _, _ = self.train_on_batch_and_get_deltas( batch_xs, batch_ys, verbose) self.comm.Barrier() sys.stdout.flush() # TODO(KGF): check line feed/carriage returns around this g.print_unique( '\nCompilation finished in {:.2f}s'.format(time.time() - t0_comp)) t_start = time.time() sys.stdout.flush() if np.any(batches_to_reset): reset_states(self.model, batches_to_reset) if ('noise' in self.conf['training'].keys() and self.conf['training']['noise'] is not False): batch_xs = self.add_noise(batch_xs) t0 = time.time() deltas, loss = self.train_on_batch_and_get_deltas( batch_xs, batch_ys, verbose) t1 = time.time() if not is_warmup_period: self.set_new_weights(deltas, num_replicas) t2 = time.time() write_str_0 = self.calculate_speed(t0, t1, t2, num_replicas) curr_loss = self.mpi_average_scalars(1.0 * loss, num_replicas) # g.print_unique(self.model.get_weights()[0][0][:4]) loss_averager.add_val(curr_loss) ave_loss = loss_averager.get_ave() eta = self.estimate_remaining_time( t0 - t_start, self.num_so_far - self.epoch * num_total, num_total) write_str = ( '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], '.format( self.task_index, step, eta, 1.0 * self.num_so_far, num_total) + 'loss: {:.5f} [{:.5f}] | '.format(ave_loss, curr_loss) + 'walltime: {:.4f} | '.format(time.time() - self.start_time)) g.write_unique(write_str + write_str_0) if timeline_prof: # dump profile tl = timeline.Timeline(self.run_metadata.step_stats) ctf = tl.generate_chrome_trace_format() # dump file per iteration with open('timeline_%s.json' % step, 'w') as f: f.write(ctf) step += 1 else: g.write_unique('\r[{}] warmup phase, num so far: {}'.format( self.task_index, self.num_so_far)) effective_epochs = 1.0 * self.num_so_far / num_total epoch_previous = self.epoch self.epoch = effective_epochs g.write_unique( # TODO(KGF): "a total of X epochs within this session" ? '\nFinished training epoch {:.2f} '.format(self.epoch) # TODO(KGF): "precisely/exactly X epochs just passed"? + 'during this session ({:.2f} epochs passed)'.format(self.epoch - epoch_previous) # '\nEpoch {:.2f} finished training ({:.2f} epochs passed)'.format( # 1.0 * self.epoch, self.epoch - epoch_previous) + ' in {:.2f} seconds\n'.format(t2 - t_start)) return (step, ave_loss, curr_loss, self.num_so_far, effective_epochs)
def fetch_d3d_data(signal_path, shot, c=None): tree, signal = get_tree_and_tag_no_backslash(signal_path) if tree is None: signal = c.get('findsig("' + signal + '",_fstree)').value tree = c.get('_fstree').value # if c is None: # c = MDSplus.Connection('atlas.gat.com') # Retrieve data found = False xdata = np.array([0]) ydata = None data = np.array([0]) # Retrieve data from MDSplus (thin) # first try, retrieve directly from tree andsignal def get_units(str): units = c.get('units_of(' + str + ')').data() if units == '' or units == ' ': units = c.get('units(' + str + ')').data() return units try: c.openTree(tree, shot) data = c.get('_s = ' + signal).data() # data_units = c.get('units_of(_s)').data() rank = np.ndim(data) found = True except Exception as e: g.print_unique(e) sys.stdout.flush() pass # Retrieve data from PTDATA if node not found if not found: # g.print_unique("not in full path {}".format(signal)) data = c.get('_s = ptdata2("' + signal + '",' + str(shot) + ')').data() if len(data) != 1: rank = np.ndim(data) found = True # Retrieve data from Pseudo-pointname if not in ptdata if not found: # g.print_unique("not in PTDATA {}".format(signal)) data = c.get('_s = pseudo("' + signal + '",' + str(shot) + ')').data() if len(data) != 1: rank = np.ndim(data) found = True # this means the signal wasn't found if not found: g.print_unique("No such signal: {}".format(signal)) pass # get time base if found: if rank > 1: xdata = c.get('dim_of(_s,1)').data() ydata = c.get('dim_of(_s)').data() else: xdata = c.get('dim_of(_s)').data() # MDSplus seems to return 2-D arrays transposed. Change them back. if np.ndim(data) == 2: data = np.transpose(data) if np.ndim(ydata) == 2: ydata = np.transpose(ydata) if np.ndim(xdata) == 2: xdata = np.transpose(xdata) # print(' GADATA Retrieval Time : ', time.time() - t0) xdata = xdata * 1e-3 # time is measued in ms return xdata, data, ydata, found
'pradedget': pradedget, 'pint': pint, 'torqueint': torqueint, 'ipdirectt': ipdirectt, 'iptargett': iptargett, 'iperrt': iperrt, 'etemp_profilet': etemp_profilet, 'edens_profilet': edens_profilet, } # for actual data analysis, use: # all_signals_restricted = [q95, li, ip, energy, lm, dens, pradcore, pradtot, # pin, etemp_profile, edens_profile] all_signals_restricted = all_signals g.print_unique('All signals (determines which signals are downloaded' ' & preprocessed):') g.print_unique(all_signals.values()) fully_defined_signals = { sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machines(all_machines)) } fully_defined_signals_0D = { sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machines(all_machines) and sig.num_channels == 1) } fully_defined_signals_1D = { sig_name: sig for (sig_name, sig) in all_signals_restricted.items()
def train_epoch(self): ''' The purpose of the method is to perform distributed mini-batch SGD for one epoch. It takes the batch iterator function and a NN model from MPIModel object, fetches mini-batches in a while-loop until number of samples seen by the ensemble of workers (num_so_far) exceeds the training dataset size (num_total). During each iteration, the gradient updates (deltas) and the loss are calculated for each model replica in the ensemble, weights are averaged over ensemble, and the new weights are set. It performs calls to: MPIModel.get_deltas, MPIModel.set_new_weights methods Argument list: Empty Returns: - step: epoch number - ave_loss: training loss averaged over replicas - curr_loss: - num_so_far: the number of samples seen by ensemble of replicas to a current epoch (step) Intermediate outputs and logging: debug printout of task_index (MPI), epoch number, number of samples seen to a current epoch, average training loss ''' verbose = False first_run = True step = 0 loss_averager = Averager() t_start = time.time() batch_iterator_func = self.batch_iterator_func num_total = 1 ave_loss = -1 curr_loss = -1 t0 = 0 t1 = 0 t2 = 0 while ((self.num_so_far - self.epoch * num_total) < num_total or step < self.num_batches_minimum): try: (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, num_total, is_warmup_period) = next(batch_iterator_func) except StopIteration: g.print_unique("Resetting batch iterator.") self.num_so_far_accum = self.num_so_far_indiv self.set_batch_iterator_func() batch_iterator_func = self.batch_iterator_func (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, num_total, is_warmup_period) = next(batch_iterator_func) self.num_so_far_indiv = self.num_so_far_accum + num_so_far_curr # if batches_to_reset: # self.model.reset_states(batches_to_reset) warmup_phase = (step < self.warmup_steps and self.epoch == 0) num_replicas = 1 if warmup_phase else self.num_replicas self.num_so_far = self.mpi_sum_scalars(self.num_so_far_indiv, num_replicas) # run the model once to force compilation. Don't actually use these # values. if first_run: first_run = False t0_comp = time.time() _, _ = self.train_on_batch_and_get_deltas( batch_xs, batch_ys, verbose) self.comm.Barrier() sys.stdout.flush() # TODO(KGF): check line feed/carriage returns around this g.print_unique( '\nCompilation finished in {:.2f}s'.format(time.time() - t0_comp)) t_start = time.time() sys.stdout.flush() if np.any(batches_to_reset): reset_states(self.model, batches_to_reset) t0 = time.time() deltas, loss = self.train_on_batch_and_get_deltas( batch_xs, batch_ys, verbose) t1 = time.time() if not is_warmup_period: self.set_new_weights(deltas, num_replicas) t2 = time.time() write_str_0 = self.calculate_speed(t0, t1, t2, num_replicas) curr_loss = self.mpi_average_scalars(1.0 * loss, num_replicas) # g.print_unique(self.model.get_weights()[0][0][:4]) loss_averager.add_val(curr_loss) ave_loss = loss_averager.get_val() eta = self.estimate_remaining_time( t0 - t_start, self.num_so_far - self.epoch * num_total, num_total) write_str = ( '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], '.format( self.task_index, step, eta, 1.0 * self.num_so_far, num_total) + 'loss: {:.5f} [{:.5f}] | '.format(ave_loss, curr_loss) + 'walltime: {:.4f} | '.format(time.time() - self.start_time)) g.write_unique(write_str + write_str_0) step += 1 else: g.write_unique('\r[{}] warmup phase, num so far: {}'.format( self.task_index, self.num_so_far)) effective_epochs = 1.0 * self.num_so_far / num_total epoch_previous = self.epoch self.epoch = effective_epochs g.write_unique( # TODO(KGF): "a total of X epochs within this session" ? '\nFinished training epoch {:.2f} '.format(self.epoch) # TODO(KGF): "precisely/exactly X epochs just passed"? + 'during this session ({:.2f} epochs passed)'.format(self.epoch - epoch_previous) # '\nEpoch {:.2f} finished training ({:.2f} epochs passed)'.format( # 1.0 * self.epoch, self.epoch - epoch_previous) + ' in {:.2f} seconds\n'.format(t2 - t_start)) return (step, ave_loss, curr_loss, self.num_so_far, effective_epochs)
def parameters(input_file): """Parse yaml file of configuration parameters.""" # TODO(KGF): the following line imports TensorFlow as a Keras backend # by default (absent env variable KERAS_BACKEND and/or config file # $HOME/.keras/keras.json) "from plasma.conf import conf" # via "import keras.backend as K" in targets.py from plasma.models.targets import (HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget) with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file, Loader=yaml.SafeLoader) params['user_name'] = getpass.getuser() base_path = params['fs_path'] output_path = os.path.join(base_path, params['user_name']) # TODO(KGF): this next line should be deleted at some pt, breaking BC base_path = output_path print(output_path) # TODO(KGF): allow for completely indpendent save/output_path vs. base_path # configured in conf.yaml. don't assume username subdirectory, or pwd # save_path = os.environ.get("PWD") params['paths']['base_path'] = base_path params['paths']['output_path'] = output_path if isinstance(params['paths']['signal_prepath'], list): g.print_unique('Reading from multiple data folders!') params['paths']['signal_prepath'] = [ base_path + s for s in params['paths']['signal_prepath'] ] else: params['paths']['signal_prepath'] = ( base_path + params['paths']['signal_prepath']) params['paths']['shot_list_dir'] = (base_path + params['paths']['shot_list_dir']) # See notes in data/signals.py for details on signal tols relative to # t_disrupt. The following 2x dataset definitions permit progressively # worse signal quality when preprocessing the shots and omitting some if params['paths']['data'] == 'd3d_data_max_tol': # let signals terminate up to 29 ms before t_disrupt on D3D h = myhash_signals(sig.all_signals_max_tol.values()) elif params['paths']['data'] == 'd3d_data_garbage': # let up to 3x signals disappear at any time before t_disrupt # (and NaNs?) # ----- # temp workaround for identical signal dictionary (but different # omit criteria in shots.py Shot.get_signals_and_times_from_file()) # ---> 2x hash int # TODO(KGF): not robust; create reproducible specification and # recording of signal filtering procedure h = myhash_signals(sig.all_signals_max_tol.values()) * 2 else: h = myhash_signals(sig.all_signals.values()) params['paths']['global_normalizer_path'] = ( output_path + '/normalization/normalization_signal_group_{}.npz'.format(h)) if params['training']['hyperparam_tuning']: # params['paths']['saved_shotlist_path'] = # './normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( './normalization/normalization_signal_group_{}.npz'.format(h)) params['paths']['model_save_path'] = './model_checkpoints/' params['paths']['csvlog_save_path'] = './csv_logs/' params['paths']['results_prepath'] = './results/' else: # params['paths']['saved_shotlist_path'] = output_path + # '/normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( params['paths']['global_normalizer_path']) params['paths']['model_save_path'] = (output_path + '/model_checkpoints/') params['paths']['csvlog_save_path'] = output_path + '/csv_logs/' params['paths']['results_prepath'] = output_path + '/results/' params['paths']['tensorboard_save_path'] = ( output_path + params['paths']['tensorboard_save_path']) params['paths']['saved_shotlist_path'] = ( params['paths']['base_path'] + '/processed_shotlists/' + params['paths']['data'] + '/shot_lists_signal_group_{}.npz'.format(h)) params['paths']['processed_prepath'] = (output_path + '/processed_shots/' + 'signal_group_{}/'.format(h)) # ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': MaxHingeTarget.fac = params['data']['positive_example_penalty'] params['data']['target'] = MaxHingeTarget elif params['target'] == 'binary': params['data']['target'] = BinaryTarget elif params['target'] == 'ttd': params['data']['target'] = TTDTarget elif params['target'] == 'ttdinv': params['data']['target'] = TTDInvTarget elif params['target'] == 'ttdlinear': params['data']['target'] = TTDLinearTarget else: # TODO(KGF): "Target" base class is unused here g.print_unique('Unknown type of target. Exiting') exit(1) # params['model']['output_activation'] = # params['data']['target'].activation # binary crossentropy performs slightly better? # params['model']['loss'] = params['data']['target'].loss # signals if params['paths']['data'] in ['d3d_data_max_tol', 'd3d_data_garbage']: params['paths']['all_signals_dict'] = sig.all_signals_max_tol else: params['paths']['all_signals_dict'] = sig.all_signals # assert order # q95, li, ip, lm, betan, energy, dens, pradcore, pradedge, pin, # pechin, torquein, ipdirect, etemp_profile, edens_profile # shot lists jet_carbon_wall = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], ['CWall_clear.txt', 'CFC_unint.txt'], 'jet carbon wall data') jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint.txt', 'BeWall_clear.txt'], 'jet iter like wall data') jet_iterlike_wall_late = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint_late.txt', 'ILW_clear_late.txt'], 'Late jet iter like wall data') # jet_iterlike_wall_full = ShotListFiles( # sig.jet, params['paths']['shot_list_dir'], # ['ILW_unint_full.txt', 'ILW_clear_full.txt'], # 'Full jet iter like wall data') jenkins_jet_carbon_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_CWall_clear.txt', 'jenkins_CFC_unint.txt'], 'Subset of jet carbon wall data for Jenkins tests') jenkins_jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_ILW_unint.txt', 'jenkins_BeWall_clear.txt'], 'Subset of jet iter like wall data for Jenkins tests') jet_full = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], [ 'ILW_unint.txt', 'BeWall_clear.txt', 'CWall_clear.txt', 'CFC_unint.txt' ], 'jet full data') # d3d_10000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_10000.txt', 'd3d_disrupt_10000.txt'], # 'd3d data 10000 ND and D shots') # d3d_1000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_1000.txt', 'd3d_disrupt_1000.txt'], # 'd3d data 1000 ND and D shots') # d3d_100 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_100.txt', 'd3d_disrupt_100.txt'], # 'd3d data 100 ND and D shots') d3d_full = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['d3d_clear_data_avail.txt', 'd3d_disrupt_data_avail.txt'], 'd3d data since shot 125500') # to 168555 # superset of d3d_full added in 2019 from C. Rea: d3d_full_2019 = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['d3d_clear_since_2016.txt', 'd3d_disrupt_since_2016.txt'], 'd3d data since shot 125500') # to 180847 d3d_jenkins = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['jenkins_d3d_clear.txt', 'jenkins_d3d_disrupt.txt'], 'Subset of d3d data for Jenkins test') # TODO(KGF): currently unused shot list files in project directory # /tigress/FRNN/shot_lists/: # d3d_clear.txt : 40560, 168554 # d3d_disrupt : 100000, 168555 # TODO(KGF): should /tigress/FRNN/shot_lists/ be organized into subdirs # like the original repo directory data/shot_lists/d3d/, jet/, nstx/ ? # d3d_jb_full = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['shotlist_JaysonBarr_clear.txt', # 'shotlist_JaysonBarr_disrupt.txt'], # 'd3d shots since 160000-170000') # nstx_full = ShotListFiles( # nstx, params['paths']['shot_list_dir'], # ['disrupt_nstx.txt'], 'nstx shots (all are disruptive') # ================== # JET DATASETS # ================== if params['paths']['data'] == 'jet_all': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_1D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_1D elif params['paths']['data'] == 'jet_late': params['paths']['shot_files'] = [jet_iterlike_wall_late] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_carbon_to_late_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_temp_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } elif params['paths']['data'] == 'jet_dens_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } elif params['paths']['data'] == 'jet_carbon_all': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_mixed_all': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jenkins_jet': params['paths']['shot_files'] = [jenkins_jet_carbon_wall] params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals # JET data but with fully defined signals elif params['paths']['data'] == 'jet_fully_defined': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals # JET data but with fully defined signals elif params['paths']['data'] == 'jet_fully_defined_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D # ================== # D3D DATASETS # ================== elif params['paths']['data'] == 'd3d_all': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] in [ 'd3d_data_max_tol', 'd3d_data_garbage' ]: params['paths']['shot_files'] = [d3d_full_2019] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95t': sig.q95t, 'lit': sig.lit, 'ipt': sig.ipt, 'lmt': sig.lmt, 'betant': sig.betant, 'energyt': sig.energyt, 'denst': sig.denst, 'pradcoret': sig.pradcoret, 'pradedget': sig.pradedget, 'pint': sig.pint, 'torqueint': sig.torqueint, 'ipdirectt': sig.ipdirectt, 'iptargett': sig.iptargett, 'iperrt': sig.iperrt, 'etemp_profilet': sig.etemp_profilet, 'edens_profilet': sig.edens_profilet, } elif params['paths']['data'] == 'd3d_2019': params['paths']['shot_files'] = [d3d_full_2019] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_all_profiles': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, 'itemp_profile': sig.itemp_profile, 'zdens_profile': sig.zdens_profile, 'trot_profile': sig.trot_profile, 'pthm_profile': sig.pthm_profile, 'neut_profile': sig.neut_profile, 'q_profile': sig.q_profile, 'bootstrap_current_profile': sig.bootstrap_current_profile, 'q_psi_profile': sig.q_psi_profile, } elif params['paths']['data'] == 'd3d_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, } # TODO(KGF): rename. Unlike JET, there are probably differences between # sig.d3d_signals and the manually-defined sigs in above d3d_all # elif params['paths']['data'] == 'd3d_all': # params['paths']['shot_files'] = [d3d_full] # params['paths']['shot_files_test'] = [] # params['paths']['use_signals_dict'] = sig.d3d_signals elif params['paths']['data'] == 'jenkins_d3d': params['paths']['shot_files'] = [d3d_jenkins] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_fully_defined': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_fully_defined_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_temp_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } # fully_defined_signals_0D elif params['paths']['data'] == 'd3d_dens_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } # fully_defined_signals_0D # ====================== # CROSS-MACHINE DATASETS # ====================== elif params['paths']['data'] == 'jet_to_d3d_all': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_jet_all': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_late_jet': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_0D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_to_jet_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'jet_to_d3d_1D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D elif params['paths']['data'] == 'd3d_to_jet_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D else: g.print_unique("Unknown dataset {}".format( params['paths']['data'])) exit(1) if len(params['paths']['specific_signals']): for s in params['paths']['specific_signals']: if s not in params['paths']['use_signals_dict'].keys(): g.print_unique( "Signal {} is not fully defined for {} machine. ", "Skipping...".format( s, params['paths']['data'].split("_")[0])) params['paths']['specific_signals'] = list( filter( lambda x: x in params['paths']['use_signals_dict'].keys(), params['paths']['specific_signals'])) selected_signals = { k: params['paths']['use_signals_dict'][k] for k in params['paths']['specific_signals'] } params['paths']['use_signals'] = sort_by_channels( list(selected_signals.values())) else: # default case params['paths']['use_signals'] = sort_by_channels( list(params['paths']['use_signals_dict'].values())) params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) g.print_unique( "Selected signals (determines which signals are used" + " for training):\n{}".format(params['paths']['use_signals'])) params['paths']['shot_files_all'] = ( params['paths']['shot_files'] + params['paths']['shot_files_test']) params['paths']['all_machines'] = list( set([file.machine for file in params['paths']['shot_files_all']])) # type assertations assert (isinstance(params['data']['signal_to_augment'], str) or isinstance(params['data']['signal_to_augment'], None)) assert isinstance(params['data']['augment_during_training'], bool) return params
def print_summary(self, action='loaded'): g.print_unique( '{} normalization data from {} shots ( {} disruptive )'.format( action, self.num_processed, self.num_disruptive))
from plasma.conf_parser import parameters import os import errno import plasma.global_vars as g # TODO(KGF): this conf.py feels like an unnecessary level of indirection if g.conf_file is not None: g.print_unique(f"Loading configuration from {g.conf_file}") conf = parameters(g.conf_file) elif os.path.exists( os.path.join(os.path.abspath(os.path.dirname(__file__)), '../examples/conf.yaml')): conf = parameters( os.path.join(os.path.abspath(os.path.dirname(__file__)), '../examples/conf.yaml')) elif os.path.exists('./conf.yaml'): conf = parameters('./conf.yaml') elif os.path.exists('./examples/conf.yaml'): conf = parameters('./examples/conf.yaml') elif os.path.exists('../examples/conf.yaml'): conf = parameters('../examples/conf.yaml') else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), 'conf.yaml')