if only_predict: custom_path = sys.argv[1] shot_num = int(sys.argv[2]) print("predicting using path {} on shot {}".format(custom_path, shot_num)) assert(only_predict) ##################################################### # NORMALIZATION # ##################################################### # TODO(KGF): identical in at least 3x files in examples/ # make sure preprocessing has been run, and is saved as a file if task_index == 0: # TODO(KGF): check tuple unpack (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) comm.Barrier() (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) shot_list = sum([l.filter_by_number([shot_num]) for l in [shot_list_train, shot_list_validate, shot_list_test]], ShotList()) assert(len(shot_list) == 1) # for s in shot_list.shots: # s.restore() def chunks(l, n): """Yield successive n-sized chunks from l.""" return[l[i:i + n] for i in range(0, len(l), n)]
only_predict = len(sys.argv) > 1 custom_path = None if only_predict: custom_path = sys.argv[1] g.print_unique("predicting using path {}".format(custom_path)) ##################################################### # NORMALIZATION # ##################################################### normalizer = Normalizer(conf) if g.task_index == 0: # make sure preprocessing has been run, and results are saved to files # if not, only master MPI rank spawns thread pool to perform preprocessing (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) # similarly, train normalizer (if necessary) w/ master MPI rank only normalizer.train() # verbose=False only suppresses if purely loading g.comm.Barrier() g.print_unique("begin preprocessor+normalization (all MPI ranks)...") # second call has ALL MPI ranks load preprocessed shots from .npz files (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf, verbose=True) # second call to normalizer training normalizer.conf['data']['recompute_normalization'] = False normalizer.train(verbose=True) # KGF: may want to set it back... # normalizer.conf['data']['recompute_normalization'] = conf['data']['recompute_normalization'] # noqa loader = Loader(conf, normalizer) g.print_unique("...done")
random.seed(task_index) if task_index == 0: pprint(conf) only_predict = len(sys.argv) > 1 custom_path = None if only_predict: custom_path = sys.argv[1] print("predicting using path {}".format(custom_path)) assert(only_predict) ##################################################### ####################Normalization#################### ##################################################### if task_index == 0: #make sure preprocessing has been run, and is saved as a file shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) comm.Barrier() shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) def chunks(l, n): """Yield successive n-sized chunks from l.""" return[ l[i:i + n] for i in range(0, len(l), n)] def hide_signal_data(shot,t=0,sigs_to_hide=None): for sig in shot.signals: if sigs_to_hide is None or (sigs_to_hide is not None and sig in sigs_to_hide): shot.signals_dict[sig][t:,:] = shot.signals_dict[sig][t,:] def create_shot_list_tmp(original_shot,time_points,sigs=None): shot_list_tmp = ShotList()
random.seed(task_index) if task_index == 0: pprint(conf) # Wouter: I adapted this to make the first argument the config file. only_predict = len(sys.argv) > 2 custom_path = None if only_predict: custom_path = sys.argv[2] print("predicting using path {}".format(custom_path)) ##################################################### ####################Normalization#################### ##################################################### if task_index == 0: #make sure preprocessing has been run, and is saved as a file shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed( conf) comm.Barrier() shot_list_train, shot_list_validate, shot_list_test = guarantee_preprocessed( conf) print("normalization", end='') normalizer = Normalizer(conf) normalizer.train() loader = Loader(conf, normalizer) print("...done") #ensure training has a separate random seed for every worker np.random.seed(task_index) random.seed(task_index) if not only_predict: mpi_train(conf, shot_list_train, shot_list_validate, loader)
from plasma.preprocessor.preprocess import guarantee_preprocessed import random import numpy as np from plasma.conf import conf from pprint import pprint pprint(conf) ##################################################### # PREPROCESSING # ##################################################### np.random.seed(0) random.seed(0) guarantee_preprocessed(conf, verbose=True)
from plasma.preprocessor.preprocess import guarantee_preprocessed import random import numpy as np from plasma.conf import conf from pprint import pprint pprint(conf) ##################################################### # PREPROCESSING # ##################################################### np.random.seed(0) random.seed(0) guarantee_preprocessed(conf)
from __future__ import print_function import os import sys import time import datetime import random import numpy as np from plasma.conf import conf from pprint import pprint pprint(conf) from plasma.preprocessor.preprocess import guarantee_preprocessed ##################################################### ####################PREPROCESSING#################### ##################################################### np.random.seed(0) random.seed(0) guarantee_preprocessed(conf)