def parameters(input_file): """Parse yaml file of configuration parameters.""" # TODO(KGF): the following line imports TensorFlow as a Keras backend # by default (absent env variable KERAS_BACKEND and/or config file # $HOME/.keras/keras.json) "from plasma.conf import conf" # via "import keras.backend as K" in targets.py from plasma.models.targets import (HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget) with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file, Loader=yaml.SafeLoader) params['user_name'] = getpass.getuser() base_path = params['fs_path'] output_path = os.path.join(base_path, params['user_name']) # TODO(KGF): this next line should be deleted at some pt, breaking BC base_path = output_path print(output_path) # TODO(KGF): allow for completely indpendent save/output_path vs. base_path # configured in conf.yaml. don't assume username subdirectory, or pwd # save_path = os.environ.get("PWD") params['paths']['base_path'] = base_path params['paths']['output_path'] = output_path if isinstance(params['paths']['signal_prepath'], list): g.print_unique('Reading from multiple data folders!') params['paths']['signal_prepath'] = [ base_path + s for s in params['paths']['signal_prepath'] ] else: params['paths']['signal_prepath'] = ( base_path + params['paths']['signal_prepath']) params['paths']['shot_list_dir'] = (base_path + params['paths']['shot_list_dir']) # See notes in data/signals.py for details on signal tols relative to # t_disrupt. The following 2x dataset definitions permit progressively # worse signal quality when preprocessing the shots and omitting some if params['paths']['data'] == 'd3d_data_max_tol': # let signals terminate up to 29 ms before t_disrupt on D3D h = myhash_signals(sig.all_signals_max_tol.values()) elif params['paths']['data'] == 'd3d_data_garbage': # let up to 3x signals disappear at any time before t_disrupt # (and NaNs?) # ----- # temp workaround for identical signal dictionary (but different # omit criteria in shots.py Shot.get_signals_and_times_from_file()) # ---> 2x hash int # TODO(KGF): not robust; create reproducible specification and # recording of signal filtering procedure h = myhash_signals(sig.all_signals_max_tol.values()) * 2 else: h = myhash_signals(sig.all_signals.values()) params['paths']['global_normalizer_path'] = ( output_path + '/normalization/normalization_signal_group_{}.npz'.format(h)) if params['training']['hyperparam_tuning']: # params['paths']['saved_shotlist_path'] = # './normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( './normalization/normalization_signal_group_{}.npz'.format(h)) params['paths']['model_save_path'] = './model_checkpoints/' params['paths']['csvlog_save_path'] = './csv_logs/' params['paths']['results_prepath'] = './results/' else: # params['paths']['saved_shotlist_path'] = output_path + # '/normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( params['paths']['global_normalizer_path']) params['paths']['model_save_path'] = (output_path + '/model_checkpoints/') params['paths']['csvlog_save_path'] = output_path + '/csv_logs/' params['paths']['results_prepath'] = output_path + '/results/' params['paths']['tensorboard_save_path'] = ( output_path + params['paths']['tensorboard_save_path']) params['paths']['saved_shotlist_path'] = ( params['paths']['base_path'] + '/processed_shotlists/' + params['paths']['data'] + '/shot_lists_signal_group_{}.npz'.format(h)) params['paths']['processed_prepath'] = (output_path + '/processed_shots/' + 'signal_group_{}/'.format(h)) # ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': MaxHingeTarget.fac = params['data']['positive_example_penalty'] params['data']['target'] = MaxHingeTarget elif params['target'] == 'binary': params['data']['target'] = BinaryTarget elif params['target'] == 'ttd': params['data']['target'] = TTDTarget elif params['target'] == 'ttdinv': params['data']['target'] = TTDInvTarget elif params['target'] == 'ttdlinear': params['data']['target'] = TTDLinearTarget else: # TODO(KGF): "Target" base class is unused here g.print_unique('Unknown type of target. Exiting') exit(1) # params['model']['output_activation'] = # params['data']['target'].activation # binary crossentropy performs slightly better? # params['model']['loss'] = params['data']['target'].loss # signals if params['paths']['data'] in ['d3d_data_max_tol', 'd3d_data_garbage']: params['paths']['all_signals_dict'] = sig.all_signals_max_tol else: params['paths']['all_signals_dict'] = sig.all_signals # assert order # q95, li, ip, lm, betan, energy, dens, pradcore, pradedge, pin, # pechin, torquein, ipdirect, etemp_profile, edens_profile # shot lists jet_carbon_wall = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], ['CWall_clear.txt', 'CFC_unint.txt'], 'jet carbon wall data') jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint.txt', 'BeWall_clear.txt'], 'jet iter like wall data') jet_iterlike_wall_late = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint_late.txt', 'ILW_clear_late.txt'], 'Late jet iter like wall data') # jet_iterlike_wall_full = ShotListFiles( # sig.jet, params['paths']['shot_list_dir'], # ['ILW_unint_full.txt', 'ILW_clear_full.txt'], # 'Full jet iter like wall data') jenkins_jet_carbon_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_CWall_clear.txt', 'jenkins_CFC_unint.txt'], 'Subset of jet carbon wall data for Jenkins tests') jenkins_jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_ILW_unint.txt', 'jenkins_BeWall_clear.txt'], 'Subset of jet iter like wall data for Jenkins tests') jet_full = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], [ 'ILW_unint.txt', 'BeWall_clear.txt', 'CWall_clear.txt', 'CFC_unint.txt' ], 'jet full data') # d3d_10000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_10000.txt', 'd3d_disrupt_10000.txt'], # 'd3d data 10000 ND and D shots') # d3d_1000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_1000.txt', 'd3d_disrupt_1000.txt'], # 'd3d data 1000 ND and D shots') # d3d_100 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_100.txt', 'd3d_disrupt_100.txt'], # 'd3d data 100 ND and D shots') d3d_full = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['d3d_clear_data_avail.txt', 'd3d_disrupt_data_avail.txt'], 'd3d data since shot 125500') # to 168555 # superset of d3d_full added in 2019 from C. Rea: d3d_full_2019 = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['d3d_clear_since_2016.txt', 'd3d_disrupt_since_2016.txt'], 'd3d data since shot 125500') # to 180847 d3d_jenkins = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['jenkins_d3d_clear.txt', 'jenkins_d3d_disrupt.txt'], 'Subset of d3d data for Jenkins test') # TODO(KGF): currently unused shot list files in project directory # /tigress/FRNN/shot_lists/: # d3d_clear.txt : 40560, 168554 # d3d_disrupt : 100000, 168555 # TODO(KGF): should /tigress/FRNN/shot_lists/ be organized into subdirs # like the original repo directory data/shot_lists/d3d/, jet/, nstx/ ? # d3d_jb_full = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['shotlist_JaysonBarr_clear.txt', # 'shotlist_JaysonBarr_disrupt.txt'], # 'd3d shots since 160000-170000') # nstx_full = ShotListFiles( # nstx, params['paths']['shot_list_dir'], # ['disrupt_nstx.txt'], 'nstx shots (all are disruptive') # ================== # JET DATASETS # ================== if params['paths']['data'] == 'jet_all': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_1D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_1D elif params['paths']['data'] == 'jet_late': params['paths']['shot_files'] = [jet_iterlike_wall_late] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_carbon_to_late_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_temp_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } elif params['paths']['data'] == 'jet_dens_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } elif params['paths']['data'] == 'jet_carbon_all': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_mixed_all': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jenkins_jet': params['paths']['shot_files'] = [jenkins_jet_carbon_wall] params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals # JET data but with fully defined signals elif params['paths']['data'] == 'jet_fully_defined': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals # JET data but with fully defined signals elif params['paths']['data'] == 'jet_fully_defined_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D # ================== # D3D DATASETS # ================== elif params['paths']['data'] == 'd3d_all': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] in [ 'd3d_data_max_tol', 'd3d_data_garbage' ]: params['paths']['shot_files'] = [d3d_full_2019] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95t': sig.q95t, 'lit': sig.lit, 'ipt': sig.ipt, 'lmt': sig.lmt, 'betant': sig.betant, 'energyt': sig.energyt, 'denst': sig.denst, 'pradcoret': sig.pradcoret, 'pradedget': sig.pradedget, 'pint': sig.pint, 'torqueint': sig.torqueint, 'ipdirectt': sig.ipdirectt, 'iptargett': sig.iptargett, 'iperrt': sig.iperrt, 'etemp_profilet': sig.etemp_profilet, 'edens_profilet': sig.edens_profilet, } elif params['paths']['data'] == 'd3d_2019': params['paths']['shot_files'] = [d3d_full_2019] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_all_profiles': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, 'itemp_profile': sig.itemp_profile, 'zdens_profile': sig.zdens_profile, 'trot_profile': sig.trot_profile, 'pthm_profile': sig.pthm_profile, 'neut_profile': sig.neut_profile, 'q_profile': sig.q_profile, 'bootstrap_current_profile': sig.bootstrap_current_profile, 'q_psi_profile': sig.q_psi_profile, } elif params['paths']['data'] == 'd3d_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, } # TODO(KGF): rename. Unlike JET, there are probably differences between # sig.d3d_signals and the manually-defined sigs in above d3d_all # elif params['paths']['data'] == 'd3d_all': # params['paths']['shot_files'] = [d3d_full] # params['paths']['shot_files_test'] = [] # params['paths']['use_signals_dict'] = sig.d3d_signals elif params['paths']['data'] == 'jenkins_d3d': params['paths']['shot_files'] = [d3d_jenkins] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_fully_defined': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_fully_defined_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_temp_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } # fully_defined_signals_0D elif params['paths']['data'] == 'd3d_dens_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } # fully_defined_signals_0D # ====================== # CROSS-MACHINE DATASETS # ====================== elif params['paths']['data'] == 'jet_to_d3d_all': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_jet_all': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_late_jet': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_0D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_to_jet_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'jet_to_d3d_1D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D elif params['paths']['data'] == 'd3d_to_jet_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D else: g.print_unique("Unknown dataset {}".format( params['paths']['data'])) exit(1) if len(params['paths']['specific_signals']): for s in params['paths']['specific_signals']: if s not in params['paths']['use_signals_dict'].keys(): g.print_unique( "Signal {} is not fully defined for {} machine. ", "Skipping...".format( s, params['paths']['data'].split("_")[0])) params['paths']['specific_signals'] = list( filter( lambda x: x in params['paths']['use_signals_dict'].keys(), params['paths']['specific_signals'])) selected_signals = { k: params['paths']['use_signals_dict'][k] for k in params['paths']['specific_signals'] } params['paths']['use_signals'] = sort_by_channels( list(selected_signals.values())) else: # default case params['paths']['use_signals'] = sort_by_channels( list(params['paths']['use_signals_dict'].values())) params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) g.print_unique( "Selected signals (determines which signals are used" + " for training):\n{}".format(params['paths']['use_signals'])) params['paths']['shot_files_all'] = ( params['paths']['shot_files'] + params['paths']['shot_files_test']) params['paths']['all_machines'] = list( set([file.machine for file in params['paths']['shot_files_all']])) # type assertations assert (isinstance(params['data']['signal_to_augment'], str) or isinstance(params['data']['signal_to_augment'], None)) assert isinstance(params['data']['augment_during_training'], bool) return params
def parameters(input_file): """Parse yaml file of configuration parameters.""" # TODO(KGF): the following line imports TensorFlow as a Keras backend # by default (absent env variable KERAS_BACKEND and/or config file # $HOME/.keras/keras.json) "from plasma.conf import conf" # via "import keras.backend as K" in targets.py from plasma.models.targets import (HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget) with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file, Loader=yaml.SafeLoader) params['user_name'] = getpass.getuser() output_path = params['fs_path'] + "/" + params['user_name'] base_path = output_path params['paths']['base_path'] = base_path if isinstance(params['paths']['signal_prepath'], list): print( 'reading from multiple data folder!**********************************************' ) params['paths']['signal_prepath'] = [ base_path + s for s in params['paths']['signal_prepath'] ] else: params['paths']['signal_prepath'] = base_path + params['paths'][ 'signal_prepath'] params['paths']['shot_list_dir'] = (base_path + params['paths']['shot_list_dir']) params['paths']['output_path'] = output_path if params['paths']['data'] == 'd3d_data_gar18': h = myhash_signals(sig.all_signals_gar18.values()) if params['paths']['data'] == 'd3d_data_n1rms': h = myhash_signals(sig.all_signals_n1rms.values()) elif params['paths']['data'] == 'd3d_data_garbage': h = myhash_signals(sig.all_signals_gar18.values()) * 2 elif params['paths']['data'] == 'd3d_data_real_time': h = myhash_signals(sig.all_signals_real_time.values()) elif params['paths']['data'] == 'd3d_data_real_time_0D': h = myhash_signals(sig.all_signals_real_time_0D.values()) else: h = myhash_signals(sig.all_signals.values()) params['paths']['global_normalizer_path'] = ( output_path + '/normalization/normalization_signal_group_{}.npz'.format(h)) if params['training']['hyperparam_tuning']: # params['paths']['saved_shotlist_path'] = # './normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( './normalization/normalization_signal_group_{}.npz'.format(h)) params['paths']['model_save_path'] = './model_checkpoints/' params['paths']['csvlog_save_path'] = './csv_logs/' params['paths']['results_prepath'] = './results/' else: # params['paths']['saved_shotlist_path'] = output_path + # '/normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( params['paths']['global_normalizer_path']) params['paths']['model_save_path'] = (output_path + '/model_checkpoints/') params['paths']['csvlog_save_path'] = output_path + '/csv_logs/' params['paths']['results_prepath'] = output_path + '/results/' params['paths']['tensorboard_save_path'] = ( output_path + params['paths']['tensorboard_save_path']) params['paths']['saved_shotlist_path'] = ( params['paths']['base_path'] + '/processed_shotlists/' + params['paths']['data'] + '/shot_lists_signal_group_{}.npz'.format(h)) params['paths']['processed_prepath'] = (output_path + '/processed_shots/' + 'signal_group_{}/'.format(h)) # ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': MaxHingeTarget.fac = params['data']['positive_example_penalty'] params['data']['target'] = MaxHingeTarget elif params['target'] == 'binary': params['data']['target'] = BinaryTarget elif params['target'] == 'ttd': params['data']['target'] = TTDTarget elif params['target'] == 'ttdinv': params['data']['target'] = TTDInvTarget elif params['target'] == 'ttdlinear': params['data']['target'] = TTDLinearTarget else: g.print_unique('Unkown type of target. Exiting') exit(1) # params['model']['output_activation'] = # params['data']['target'].activation # binary crossentropy performs slightly better? # params['model']['loss'] = params['data']['target'].loss # signals if params['paths']['data'] in ['d3d_data_gar18', 'd3d_data_garbage']: params['paths']['all_signals_dict'] = sig.all_signals_gar18 elif params['paths']['data'] == 'd3d_data_n1rms': params['paths']['all_signals_dict'] = sig.all_signals_n1rms elif params['paths']['data'] == 'd3d_data_real_time': params['paths']['all_signals_dict'] = sig.all_signals_real_time elif params['paths']['data'] == 'd3d_data_real_time_0D': params['paths']['all_signals_dict'] = sig.all_signals_real_time_0D else: params['paths']['all_signals_dict'] = sig.all_signals # assert order # q95, li, ip, lm, betan, energy, dens, pradcore, pradedge, pin, # pechin, torquein, ipdirect, etemp_profile, edens_profile # shot lists jet_carbon_wall = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], ['CWall_clear.txt', 'CFC_unint.txt'], 'jet carbon wall data') jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint.txt', 'BeWall_clear.txt'], 'jet iter like wall data') jet_iterlike_wall_late = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['ILW_unint_late.txt', 'ILW_clear_late.txt'], 'Late jet iter like wall data') # jet_iterlike_wall_full = ShotListFiles( # sig.jet, params['paths']['shot_list_dir'], # ['ILW_unint_full.txt', 'ILW_clear_full.txt'], # 'Full jet iter like wall data') jenkins_jet_carbon_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_CWall_clear.txt', 'jenkins_CFC_unint.txt'], 'Subset of jet carbon wall data for Jenkins tests') jenkins_jet_iterlike_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], ['jenkins_ILW_unint.txt', 'jenkins_BeWall_clear.txt'], 'Subset of jet iter like wall data for Jenkins tests') jet_full = ShotListFiles(sig.jet, params['paths']['shot_list_dir'], [ 'ILW_unint.txt', 'BeWall_clear.txt', 'CWall_clear.txt', 'CFC_unint.txt' ], 'jet full data') # d3d_10000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_10000.txt', 'd3d_disrupt_10000.txt'], # 'd3d data 10000 ND and D shots') # d3d_1000 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_1000.txt', 'd3d_disrupt_1000.txt'], # 'd3d data 1000 ND and D shots') # d3d_100 = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['d3d_clear_100.txt', 'd3d_disrupt_100.txt'], # 'd3d data 100 ND and D shots') d3d_full = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['d3d_clear_data_avail.txt', 'd3d_disrupt_data_avail.txt'], 'd3d data since shot 125500') d3d_full_new = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['shots_since_2016_clear.txt', 'shots_since_2016_disrupt.txt'], 'd3d data since shot 125500') d3d_jenkins = ShotListFiles( sig.d3d, params['paths']['shot_list_dir'], ['jenkins_d3d_clear.txt', 'jenkins_d3d_disrupt.txt'], 'Subset of d3d data for Jenkins test') # d3d_jb_full = ShotListFiles( # sig.d3d, params['paths']['shot_list_dir'], # ['shotlist_JaysonBarr_clear.txt', # 'shotlist_JaysonBarr_disrupt.txt'], # 'd3d shots since 160000-170000') # nstx_full = ShotListFiles( # nstx, params['paths']['shot_list_dir'], # ['disrupt_nstx.txt'], 'nstx shots (all are disruptive') if params['paths']['data'] == 'jet_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_data_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_data_1D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals_1D elif params['paths']['data'] == 'jet_data_late': params['paths']['shot_files'] = [jet_iterlike_wall_late] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_data_carbon_to_late_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_data_temp_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } elif params['paths']['data'] == 'jet_data_dens_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } elif params['paths']['data'] == 'jet_carbon_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_mixed_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jenkins_jet': params['paths']['shot_files'] = [jenkins_jet_carbon_wall] params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.jet_signals # jet data but with fully defined signals elif params['paths']['data'] == 'jet_data_fully_defined': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals # jet data but with fully defined signals elif params['paths']['data'] == 'jet_data_fully_defined_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] in ['d3d_data_gar18', 'd3d_data_garbage']: params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95t': sig.q95t, 'lit': sig.lit, 'ipt': sig.ipt, 'lmt': sig.lmt, 'betant': sig.betant, 'energyt': sig.energyt, 'denst': sig.denst, 'pradcoret': sig.pradcoret, 'pradedget': sig.pradedget, 'pint': sig.pint, 'torqueint': sig.torqueint, 'ipdirectt': sig.ipdirectt, 'iptargett': sig.iptargett, 'iperrt': sig.iperrt, 'etemp_profilet': sig.etemp_profilet, 'edens_profilet': sig.edens_profilet, } elif params['paths']['data'] in ['d3d_data_n1rms']: params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95t': sig.q95, 'lit': sig.li, 'ipt': sig.ip, 'lmt': sig.lm, 'betant': sig.betan, 'energyt': sig.energy, 'denst': sig.dens, 'pradcoret': sig.pradcore, 'pradedget': sig.pradedge, 'pint': sig.pin, 'torqueint': sig.torquein, 'ipdirectt': sig.ipdirect, 'iptargett': sig.iptarget, 'iperrt': sig.iperr, 'etemp_profilet': sig.etemp_profile, 'edens_profilet': sig.edens_profile, 'n1_rms': sig.n1_rms, } elif params['paths']['data'] == 'd3d_data_new': params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_data_real_time': params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95_EFITRT1': sig.q95_EFITRT1, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_data_real_time_0D': params['paths']['shot_files'] = [d3d_full_new] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95_EFITRT1': sig.q95_EFITRT1, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'vd': sig.vd, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, } elif params['paths']['data'] == 'd3d_data_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } elif params['paths']['data'] == 'd3d_data_all_profiles': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': sig.ipdirect, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, 'itemp_profile': sig.itemp_profile, 'zdens_profile': sig.zdens_profile, 'trot_profile': sig.trot_profile, 'pthm_profile': sig.pthm_profile, 'neut_profile': sig.neut_profile, 'q_profile': sig.q_profile, 'bootstrap_current_profile': sig.bootstrap_current_profile, 'q_psi_profile': sig.q_psi_profile, } elif params['paths']['data'] == 'd3d_data_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, } elif params['paths']['data'] == 'd3d_data_all': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.d3d_signals elif params['paths']['data'] == 'jenkins_d3d': params['paths']['shot_files'] = [d3d_jenkins] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': sig.q95, 'li': sig.li, 'ip': sig.ip, 'lm': sig.lm, 'betan': sig.betan, 'energy': sig.energy, 'dens': sig.dens, 'pradcore': sig.pradcore, 'pradedge': sig.pradedge, 'pin': sig.pin, 'torquein': sig.torquein, 'ipdirect': sig.ipdirect, 'iptarget': sig.iptarget, 'iperr': sig.iperr, 'etemp_profile': sig.etemp_profile, 'edens_profile': sig.edens_profile, } # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_data_fully_defined': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals # jet data but with fully defined signals elif params['paths']['data'] == 'd3d_data_fully_defined_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data_temp_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'etemp_profile': sig.etemp_profile } # fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data_dens_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'edens_profile': sig.edens_profile } # fully_defined_signals_0D # cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_jet_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_late_jet_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_data_0D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_to_jet_data_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'jet_to_d3d_data_1D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D elif params['paths']['data'] == 'd3d_to_jet_data_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D else: g.print_unique("Unknown dataset {}".format( params['paths']['data'])) exit(1) if len(params['paths']['specific_signals']): for s in params['paths']['specific_signals']: if s not in params['paths']['use_signals_dict'].keys(): g.print_unique( "Signal {} is not fully defined for {} machine. ", "Skipping...".format( s, params['paths']['data'].split("_")[0])) params['paths']['specific_signals'] = list( filter( lambda x: x in params['paths']['use_signals_dict'].keys(), params['paths']['specific_signals'])) selected_signals = { k: params['paths']['use_signals_dict'][k] for k in params['paths']['specific_signals'] } params['paths']['use_signals'] = sort_by_channels( list(selected_signals.values())) else: # default case params['paths']['use_signals'] = sort_by_channels( list(params['paths']['use_signals_dict'].values())) params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) g.print_unique( "Selected signals (determines which signals are used" + " for training):\n{}".format(params['paths']['use_signals'])) params['paths']['shot_files_all'] = ( params['paths']['shot_files'] + params['paths']['shot_files_test']) params['paths']['all_machines'] = list( set([file.machine for file in params['paths']['shot_files_all']])) # type assertations assert (isinstance(params['data']['signal_to_augment'], str) or isinstance(params['data']['signal_to_augment'], None)) assert isinstance(params['data']['augment_during_training'], bool) return params
def parameters(input_file): """Parse yaml file of configuration parameters.""" from plasma.models.targets import HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file) params['user_name'] = getpass.getuser() output_path = params['fs_path'] + "/" + params['user_name'] base_path = output_path params['paths']['base_path'] = base_path params['paths'][ 'signal_prepath'] = base_path + params['paths']['signal_prepath'] params['paths'][ 'shot_list_dir'] = base_path + params['paths']['shot_list_dir'] params['paths']['output_path'] = output_path h = get_unique_signal_hash(all_signals.values()) params['paths'][ 'global_normalizer_path'] = output_path + '/normalization/normalization_signal_group_{}.npz'.format( h) if params['training']['hyperparam_tuning']: # params['paths']['saved_shotlist_path'] = './normalization/shot_lists.npz' params['paths'][ 'normalizer_path'] = './normalization/normalization_signal_group_{}.npz'.format( h) params['paths']['model_save_path'] = './model_checkpoints/' params['paths']['csvlog_save_path'] = './csv_logs/' params['paths']['results_prepath'] = './results/' else: # params['paths']['saved_shotlist_path'] = output_path +'/normalization/shot_lists.npz' params['paths']['normalizer_path'] = params['paths'][ 'global_normalizer_path'] params['paths'][ 'model_save_path'] = output_path + '/model_checkpoints/' params['paths']['csvlog_save_path'] = output_path + '/csv_logs/' params['paths']['results_prepath'] = output_path + '/results/' params['paths']['tensorboard_save_path'] = output_path + params[ 'paths']['tensorboard_save_path'] params['paths']['saved_shotlist_path'] = params['paths'][ 'base_path'] + '/processed_shotlists/' + params['paths'][ 'data'] + '/shot_lists_signal_group_{}.npz'.format(h) params['paths'][ 'processed_prepath'] = output_path + '/processed_shots/' + 'signal_group_{}/'.format( h) #ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': MaxHingeTarget.fac = params['data']['positive_example_penalty'] params['data']['target'] = MaxHingeTarget elif params['target'] == 'binary': params['data']['target'] = BinaryTarget elif params['target'] == 'ttd': params['data']['target'] = TTDTarget elif params['target'] == 'ttdinv': params['data']['target'] = TTDInvTarget elif params['target'] == 'ttdlinear': params['data']['target'] = TTDLinearTarget else: print('Unkown type of target. Exiting') exit(1) #params['model']['output_activation'] = params['data']['target'].activation #binary crossentropy performs slightly better? #params['model']['loss'] = params['data']['target'].loss #signals params['paths']['all_signals_dict'] = all_signals #assert order q95,li,ip,lm,betan,energy,dens,pradcore,pradedge,pin,pechin,torquein,ipdirect,etemp_profile,edens_profile #shot lists jet_carbon_wall = ShotListFiles(jet, params['paths']['shot_list_dir'], ['CWall_clear.txt', 'CFC_unint.txt'], 'jet carbon wall data') jet_iterlike_wall = ShotListFiles( jet, params['paths']['shot_list_dir'], ['ILW_unint.txt', 'BeWall_clear.txt'], 'jet iter like wall data') jenkins_jet_carbon_wall = ShotListFiles( jet, params['paths']['shot_list_dir'], ['jenkins_CWall_clear.txt', 'jenkins_CFC_unint.txt'], 'Subset of jet carbon wall data for Jenkins tests') jenkins_jet_iterlike_wall = ShotListFiles( jet, params['paths']['shot_list_dir'], ['jenkins_ILW_unint.txt', 'jenkins_BeWall_clear.txt'], 'Subset of jet iter like wall data for Jenkins tests') jet_full = ShotListFiles(jet, params['paths']['shot_list_dir'], [ 'ILW_unint.txt', 'BeWall_clear.txt', 'CWall_clear.txt', 'CFC_unint.txt' ], 'jet full data') d3d_10000 = ShotListFiles( d3d, params['paths']['shot_list_dir'], ['d3d_clear_10000.txt', 'd3d_disrupt_10000.txt'], 'd3d data 10000 ND and D shots') d3d_1000 = ShotListFiles( d3d, params['paths']['shot_list_dir'], ['d3d_clear_1000.txt', 'd3d_disrupt_1000.txt'], 'd3d data 1000 ND and D shots') d3d_100 = ShotListFiles(d3d, params['paths']['shot_list_dir'], ['d3d_clear_100.txt', 'd3d_disrupt_100.txt'], 'd3d data 100 ND and D shots') d3d_full = ShotListFiles( d3d, params['paths']['shot_list_dir'], ['d3d_clear_data_avail.txt', 'd3d_disrupt_data_avail.txt'], 'd3d data since shot 125500') d3d_jenkins = ShotListFiles( d3d, params['paths']['shot_list_dir'], ['jenkins_d3d_clear.txt', 'jenkins_d3d_disrupt.txt'], 'Subset of d3d data for Jenkins test') d3d_jb_full = ShotListFiles(d3d, params['paths']['shot_list_dir'], [ 'shotlist_JaysonBarr_clear.txt', 'shotlist_JaysonBarr_disrupt.txt' ], 'd3d shots since 160000-170000') nstx_full = ShotListFiles(nstx, params['paths']['shot_list_dir'], ['disrupt_nstx.txt'], 'nstx shots (all are disruptive') if params['paths']['data'] == 'jet_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals elif params['paths']['data'] == 'jet_data_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals_0D elif params['paths']['data'] == 'jet_data_1D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals_1D elif params['paths']['data'] == 'jet_carbon_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = jet_signals elif params['paths']['data'] == 'jet_mixed_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = jet_signals elif params['paths']['data'] == 'jenkins_jet': params['paths']['shot_files'] = [jenkins_jet_carbon_wall] params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals elif params['paths'][ 'data'] == 'jet_data_fully_defined': #jet data but with fully defined signals params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals elif params['paths'][ 'data'] == 'jet_data_fully_defined_0D': #jet data but with fully defined signals params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': q95, 'li': li, 'ip': ip, 'lm': lm, 'betan': betan, 'energy': energy, 'dens': dens, 'pradcore': pradcore, 'pradedge': pradedge, 'pin': pin, 'torquein': torquein, 'ipdirect': ipdirect, 'iptarget': iptarget, 'iperr': iperr, 'etemp_profile': etemp_profile, 'edens_profile': edens_profile } elif params['paths']['data'] == 'd3d_data_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': ipdirect, 'etemp_profile': etemp_profile, 'edens_profile': edens_profile } elif params['paths']['data'] == 'd3d_data_all_profiles': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'ipdirect': ipdirect, 'etemp_profile': etemp_profile, 'edens_profile': edens_profile, 'itemp_profile': itemp_profile, 'zdens_profile': zdens_profile, 'trot_profile': trot_profile, 'pthm_profile': pthm_profile, 'neut_profile': neut_profile, 'q_profile': q_profile, 'bootstrap_current_profile': bootstrap_current_profile, 'q_psi_profile': q_psi_profile } elif params['paths']['data'] == 'd3d_data_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': q95, 'li': li, 'ip': ip, 'lm': lm, 'betan': betan, 'energy': energy, 'dens': dens, 'pradcore': pradcore, 'pradedge': pradedge, 'pin': pin, 'torquein': torquein, 'ipdirect': ipdirect, 'iptarget': iptarget, 'iperr': iperr } elif params['paths']['data'] == 'd3d_data_all': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = d3d_signals elif params['paths']['data'] == 'jenkins_d3d': params['paths']['shot_files'] = [d3d_jenkins] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { 'q95': q95, 'li': li, 'ip': ip, 'lm': lm, 'betan': betan, 'energy': energy, 'dens': dens, 'pradcore': pradcore, 'pradedge': pradedge, 'pin': pin, 'torquein': torquein, 'ipdirect': ipdirect, 'iptarget': iptarget, 'iperr': iperr, 'etemp_profile': etemp_profile, 'edens_profile': edens_profile } elif params['paths'][ 'data'] == 'd3d_data_fully_defined': #jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = fully_defined_signals elif params['paths'][ 'data'] == 'd3d_data_fully_defined_0D': #jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = fully_defined_signals_0D #cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = fully_defined_signals elif params['paths']['data'] == 'd3d_to_jet_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_data_0D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = fully_defined_signals_0D elif params['paths']['data'] == 'd3d_to_jet_data_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals_0D elif params['paths']['data'] == 'jet_to_d3d_data_1D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = fully_defined_signals_1D elif params['paths']['data'] == 'd3d_to_jet_data_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals_1D else: print("Unkown data set {}".format(params['paths']['data'])) exit(1) if len(params['paths']['specific_signals']): for sig in params['paths']['specific_signals']: if sig not in params['paths']['use_signals_dict'].keys(): print( "Signal {} is not fully defined for {} machine. Skipping..." .format(sig, params['paths']['data'].split("_")[0])) params['paths']['specific_signals'] = list( filter( lambda x: x in params['paths']['use_signals_dict'].keys(), params['paths']['specific_signals'])) selected_signals = { k: params['paths']['use_signals_dict'][k] for k in params['paths']['specific_signals'] } params['paths']['use_signals'] = sort_by_channels( list(selected_signals.values())) else: #default case params['paths']['use_signals'] = sort_by_channels( list(params['paths']['use_signals_dict'].values())) params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) print( "Selected signals (determines which signals training is run on):\n{}" .format(params['paths']['use_signals'])) params['paths']['shot_files_all'] = params['paths'][ 'shot_files'] + params['paths']['shot_files_test'] params['paths']['all_machines'] = list( set([file.machine for file in params['paths']['shot_files_all']])) #type assertations assert type(params['data']['signal_to_augment']) == str or type( params['data']['signal_to_augment']) == None assert type(params['data']['augment_during_training']) == bool return params