def set_gpus(n_gpus=gpu_settings["n_gpus"], min_vram=gpu_settings["min_vram"], split_gpu_into=gpu_settings["split_gpu_into"]): ''' Configures the GPUs to be allocated for training, preferring the GPUs with most free VRAM. :param n_gpus: How many physical GPUs to allocate for this training process. Set to 0 to run on CPU. min_memory: How much free VRAM each physical GPU has to have. Too low value causes an error if the GPU runs out of memory when training. This prevents TensorFlow from allocating all of the memory on GPU to the process. split_into: How many logical GPUs to split each physical GPU into. This can speed up the training due to distributed training. Each physical GPU has to have min_memory * split_into VRAM available or an error is raised. :return None ''' if n_gpus == 0: environ['CUDA_VISIBLE_DEVICES'] = '' gpu_stats = GPUStatCollection.new_query() gpu_ids = map(lambda gpu: int(gpu.entry['index']), gpu_stats) gpu_freemem = map( lambda gpu: float(gpu.entry['memory.total'] - gpu.entry['memory.used'] ), gpu_stats) pairs = list(zip(gpu_ids, gpu_freemem)) valid_pairs = [ pair for pair in pairs if pair[1] >= min_vram * split_gpu_into ] if len(valid_pairs) < n_gpus: raise ValueError( f"Not enough valid GPUs detected. Check if the machine has at least {n_gpus} GPUs with at least {min_vram * split_gpu_into}MB free VRAM or set a lower --n_gpus value" ) sorted_indices = list(argsort([mem[1] for mem in valid_pairs]))[::-1] sorted_pairs = [valid_pairs[i] for i in sorted_indices] if n_gpus != 0: print( f"Setting {n_gpus} physical GPUs split into {n_gpus * split_gpu_into} logical GPUs with {min_vram}MB VRAM each for this training" ) else: print("Training on CPU") environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' devices = ",".join([str(pair[0]) for pair in sorted_pairs[:n_gpus]]) environ['CUDA_VISIBLE_DEVICES'] = devices if split_gpu_into > 1: physical_devices = tf_config.list_physical_devices('GPU') for device in physical_devices: tf_config.set_logical_device_configuration(device, [ tf_config.LogicalDeviceConfiguration(memory_limit=min_vram) for _ in range(split_gpu_into) ])
def __init__(self, model_parameters, hidden_dim, seq_len, n_seq, gamma): physical_devices = tfconfig.list_physical_devices('GPU') if len(physical_devices) > 0: try: tfconfig.experimental.set_memory_growth( physical_devices[0], True) except: # Invalid device or cannot modify virtual devices once initialized. pass self.seq_len = seq_len self.n_seq = n_seq self.hidden_dim = hidden_dim self.gamma = gamma super().__init__(model_parameters)
def __init__( self, model_parameters ): gpu_devices = tfconfig.list_physical_devices('GPU') if len(gpu_devices) > 0: try: tfconfig.experimental.set_memory_growth(gpu_devices[0], True) except: # Invalid device or cannot modify virtual devices once initialized. pass self._model_parameters = model_parameters [self.batch_size, self.lr, self.beta_1, self.beta_2, self.noise_dim, self.data_dim, self.layers_dim] = model_parameters self.define_gan()
def prep_gpu(distribution=None): print(f"\n!--PREPPING GPU--! with stratagy ") #traceback.print_stack() if distribution == None: gpus = list_physical_devices('GPU') if gpus: try: # Currently, memory growth needs to be the same across GPUs for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) logical_gpus = list_logical_devices('GPU') print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") except RuntimeError as e: # Memory growth must be set before GPUs have been initialized print(e) raise print() return
def call(self, inputs, training=True): """Normal call.""" pad_size0 = self.units - 1 - self.non_causal pad_size1 = self.non_causal padded_input = tf.pad(inputs, [[0, 0], [pad_size0, pad_size1], [0, 0]]) # SVD inserting DEBUG = False if DEBUG: print("Banks:", self.banks) print("Units:", self.units) print("N_inputs: ", self.n_inputs) if config.list_physical_devices('GPU') or \ (self.n_inputs == padded_input.shape[-1]): # this implementation does not evaluate on a CPU if mapping subsets of # the input into the different FIR filters. transposed = tf.transpose(tf.reverse(self.coefficients, axis=[-1])) Y = tf.nn.conv1d(padded_input, transposed, stride=1, padding='VALID') if DEBUG: print("padded input: ", padded_input.shape) print("transposed kernel: ", transposed.shape) print("Y: ", Y.shape) return Y # alternative, kludgy implementation, evaluate each filter separately on the # corresponding subset of inputs, concatenate outputs when done transposed = tf.transpose(tf.reverse(self.coefficients, axis=[-1])) if DEBUG: print("padded input: ", padded_input.shape) print("transposed kernel: ", transposed.shape) L = [] for i in range(transposed.shape[2]): W = transposed.shape[1] A = padded_input[:, :, (i*W):((i+1)*W)] B = transposed[:, :, i:(i+1)] L.append(tf.nn.conv1d(A, B, stride=1, padding='VALID')) Y = tf.concat(L, axis=2) if DEBUG: print("L[0]: ", L[0].shape) print("Y: ", Y.shape) return Y
def prep_gpu(distribution=None): import tensorflow as tf try: from tensorflow.config import list_physical_devices, list_logical_devices except ImportError: from tensorflow.config.experimental import list_physical_devices, list_logical_devices print(f"\n!--PREPPING GPU--! ") if distribution is None: gpus = list_physical_devices('GPU') if gpus: try: # Currently, memory growth needs to be the same across GPUs for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) logical_gpus = list_logical_devices('GPU') print(len(gpus), 'Physical GPUs,', len(logical_gpus), 'Logical GPUs') except RuntimeError as e: # Memory growth must be set before GPUs have been initialized print(e) raise return
def _setup_gpu_environment() -> None: """Sets configuration for TensorFlow GPU environment based on env variable.""" gpu_memory_config = os.getenv(ENV_GPU_CONFIG) if not gpu_memory_config: return # Import from tensorflow only if necessary (environment variable was set) from tensorflow import config as tf_config parsed_gpu_config = _parse_gpu_config(gpu_memory_config) physical_gpus = tf_config.list_physical_devices("GPU") # Logic taken from https://www.tensorflow.org/guide/gpu if physical_gpus: for gpu_id, gpu_id_memory in parsed_gpu_config.items(): _allocate_gpu_memory(physical_gpus[gpu_id], gpu_id_memory) else: rasa.shared.utils.io.raise_warning( f"You have an environment variable '{ENV_GPU_CONFIG}' set but no GPUs were " f"detected to configure.")
from tensorflow.keras.preprocessing.text import Tokenizer from tensorflow.keras.preprocessing.sequence import pad_sequences from tensorflow.keras.models import Sequential, load_model from tensorflow.keras.utils import to_categorical from tensorflow.keras.optimizers import Adam from tensorflow.keras import layers from tensorflow.keras import callbacks from tensorflow import config physical_devices = config.list_physical_devices('GPU') config.experimental.set_memory_growth(physical_devices[0], enable=True) import numpy as np import matplotlib.pyplot as plt from os import path import time def plot_history(history, acc_name='keras_accuracy.png', loss_name='keras_loss.png'): plt.clf() accuracy = history['accuracy'] val_accuracy = history['val_accuracy'] epochs = range(1, len(accuracy) + 1) plt.plot(epochs, accuracy, 'tab:blue', label='Training Accuracy') plt.plot(epochs, val_accuracy, 'tab:orange', label='Validation Accuracy') plt.title('Training and Validation Accuracy') plt.xlabel('Epochs')
def gpu_info(): return list_physical_devices('GPU')
def evaluate_books( self, books, checkpoint, cachefile=None, output_individual_voters=False, n_confusions=10, silent=True, ): keras.backend.clear_session() if type(books) == str: books = [books] if type(checkpoint) == str: checkpoint = [checkpoint] checkpoint = [ (cp if cp.endswith(".json") else cp + ".json") for cp in checkpoint ] checkpoint = glob_all(checkpoint) checkpoint = [cp[:-5] for cp in checkpoint] if cachefile is None: cachefile = self.cachefile lids = list( lids_from_books(books, cachefile, complete_only=True, skip_commented=True) ) data = Nsh5(cachefile=cachefile, lines=lids) predparams = PredictorParams() predparams.device.gpus = [n for n, _ in enumerate(list_physical_devices("GPU"))] predparams.silent = silent predictor = MultiPredictor.from_paths( checkpoints=checkpoint, voter_params=VoterParams(), predictor_params=predparams, ) newprcs = [] for prc in predictor.data.params.pre_proc.processors: prc = deepcopy(prc) if isinstance(prc, FinalPreparationProcessorParams): prc.normalize, prc.invert, prc.transpose = False, False, True newprcs.append(prc) elif isinstance(prc, PrepareSampleProcessorParams): newprcs.append(prc) predictor.data.params.pre_proc.processors = newprcs do_prediction = predictor.predict(data) all_voter_sentences = {} all_prediction_sentences = {} for s in do_prediction: _, (_, prediction), _ = s.inputs, s.outputs, s.meta sentence = prediction.sentence if prediction.voter_predictions is not None and output_individual_voters: for i, p in enumerate(prediction.voter_predictions): if i not in all_prediction_sentences: all_prediction_sentences[i] = {} all_prediction_sentences[i][s.meta["id"]] = p.sentence all_voter_sentences[s.meta["id"]] = sentence # evaluation from calamari_ocr.ocr.evaluator import Evaluator, EvaluatorParams evaluator_params = EvaluatorParams( setup=predparams.pipeline, progress_bar=True, skip_empty_gt=True, ) evaluator = Evaluator(evaluator_params, predictor.data) evaluator.preload_gt(gt_dataset=data, progress_bar=True) def single_evaluation(label, predicted_sentences): r = evaluator.evaluate( gt_data=evaluator.preloaded_gt, pred_data=predicted_sentences ) print("=================") print(f"Evaluation result of {label}") print("=================") print("") print( "Got mean normalized label error rate of {:.2%} ({} errs, {} total chars, {} sync errs)".format( r["avg_ler"], r["total_char_errs"], r["total_chars"], r["total_sync_errs"], ) ) print() print() # sort descending print_confusions(r, n_confusions) return r full_evaluation = {} for id, data in [ (str(i), sent) for i, sent in all_prediction_sentences.items() ] + [("voted", all_voter_sentences)]: full_evaluation[id] = {"eval": single_evaluation(id, data), "data": data} if not predparams.silent: print(full_evaluation) return full_evaluation
def predict_books( self, books, checkpoint, cachefile=None, pageupload=True, text_index=1, pred_all=False, ): keras.backend.clear_session() if type(books) == str: books = [books] if type(checkpoint) == str: checkpoint = [checkpoint] checkpoint = [ (cp if cp.endswith(".json") else cp + ".json") for cp in checkpoint ] checkpoint = glob_all(checkpoint) checkpoint = [cp[:-5] for cp in checkpoint] if cachefile is None: cachefile = self.cachefile verbose = False lids = list( lids_from_books( books, cachefile, complete_only=False, skip_commented=False, new_only=not pred_all, ) ) data = Nsh5(cachefile=cachefile, lines=lids) predparams = PredictorParams() predparams.device.gpus = [n for n, _ in enumerate(list_physical_devices("GPU"))] predictor = MultiPredictor.from_paths( checkpoints=checkpoint, voter_params=VoterParams(), predictor_params=predparams, ) newprcs = [] for prc in predictor.data.params.pre_proc.processors: prc = deepcopy(prc) if isinstance(prc, FinalPreparationProcessorParams): prc.normalize, prc.invert, prc.transpose = False, False, True newprcs.append(prc) elif isinstance(prc, PrepareSampleProcessorParams): newprcs.append(prc) predictor.data.params.pre_proc.processors = newprcs do_prediction = predictor.predict(data) pipeline = predictor.data.get_or_create_pipeline( predictor.params.pipeline, data ) reader = pipeline.reader() if len(reader) == 0: raise Exception( "Empty dataset provided. Check your lines (got {})!".format(lids) ) avg_sentence_confidence = 0 n_predictions = 0 reader.prepare_store() samples = [] sentences = [] # output the voted results to the appropriate files for s in do_prediction: _, (_, prediction), meta = s.inputs, s.outputs, s.meta sample = reader.sample_by_id(meta["id"]) n_predictions += 1 sentence = prediction.sentence avg_sentence_confidence += prediction.avg_char_probability if verbose: lr = "\u202A\u202B" logger.info( "{}: '{}{}{}'".format( meta["id"], lr[get_base_level(sentence)], sentence, "\u202C" ) ) samples.append(sample) sentences.append(sentence) reader.store_text(sentence, sample, output_dir=None, extension=None) logger.info( "Average sentence confidence: {:.2%}".format( avg_sentence_confidence / n_predictions ) ) if pageupload: ocrdata = {} for lname, text in reader.predictions.items(): _, b, p, ln = lname.split("/") if b not in ocrdata: ocrdata[b] = {} if p not in ocrdata[b]: ocrdata[b][p] = {} ocrdata[b][p][ln] = text data = {"ocrdata": ocrdata, "index": text_index} self.get_session().post( self.baseurl + "/_ocrdata", data=gzip.compress(json.dumps(data).encode("utf-8")), headers={ "Content-Type": "application/json;charset=UTF-8", "Content-Encoding": "gzip", }, ) logger.info("Results uploaded") else: reader.store() logger.info("All prediction files written")
def cftrain_books( self, books, n_folds=5, cachefile=None, name="models", tempdir=None, keep_temporary_files=False, max_parallel_models=-1, skip_commented=True, validation_split_ratio=1, bidi="", n_augmentations=0, ema_decay=0.0, train_verbose=1, debug=False, epochs=100, whitelist="", keep_loaded_codec=False, preload=True, weights=[], ensemble=0, ): keras.backend.clear_session() if isinstance(weights, str): weights = [weights] if isinstance(books, str): books = [books] if cachefile is None: cachefile = self.cachefile if max_parallel_models < 1: max_parallel_models = n_folds lids = list( lids_from_books( books, cachefile, complete_only=True, skip_commented=skip_commented ) ) train = Nsh5(cachefile=cachefile, lines=lids) cfparams = CrossFoldTrainerParams() cfparams.weights = weights cfparams.temporary_dir = tempdir cfparams.best_models_dir = name cfparams.n_folds = n_folds cfparams.keep_temporary_files = False cfparams.max_parallel_models = max_parallel_models newprcs = [] for prc in cfparams.trainer.scenario.data.pre_proc.processors: prc = deepcopy(prc) if PipelineMode.TRAINING in prc.modes: if isinstance(prc, FinalPreparationProcessorParams): prc.normalize, prc.invert, prc.transpose = False, False, True elif isinstance(prc, AugmentationProcessorParams): prc.n_augmentations = n_augmentations elif not isinstance(prc, PrepareSampleProcessorParams): prc.modes = set() newprcs.append(prc) cfparams.trainer.scenario.data.pre_proc.processors = newprcs cfparams.trainer.device.gpus = [ n for n, _ in enumerate(list_physical_devices("GPU")) ] cfparams.trainer.gen.train = train if bidi: for prc in cfparams.trainer.scenario.data.post_proc.processors_of_type( BidiTextProcessorParams ): prc.bidi_direction = BidiDirection.RTL cfparams.trainer.epochs = epochs cfparams.trainer.codec.keep_loaded = keep_loaded_codec cfparams.trainer.gen.train.preload = preload cfparams.trainer.scenario.model.ensemble = ensemble cfparams.trainer.ema_decay = ema_decay cfparams.trainer.scenario.data.__post_init__() cfparams.trainer.scenario.__post_init__() cfparams.trainer.__post_init__() print(cfparams.to_json()) with open("debug.json", "w") as f: f.write(cfparams.to_json()) trainer = CrossFoldTrainer(cfparams) return trainer.run()
def train_books( self, books, cachefile=None, name="model", skip_commented=True, validation_split_ratio=1, bidi="", n_augmentations=0, ema_decay=0.0, train_verbose=1, debug=False, epochs=100, whitelist="", keep_loaded_codec=False, preload=True, weights=None, ensemble=0, ): keras.backend.clear_session() if isinstance(books, str): books = [books] if cachefile is None: cachefile = self.cachefile p = CalamariScenario.default_trainer_params() lids = list( lids_from_books( books, cachefile, complete_only=True, skip_commented=skip_commented ) ) train = Nsh5(cachefile=cachefile, lines=lids) newprcs = [] for prc in p.scenario.data.pre_proc.processors: prc = deepcopy(prc) if PipelineMode.TRAINING in prc.modes: if isinstance(prc, FinalPreparationProcessorParams): prc.normalize, prc.invert, prc.transpose = False, False, True elif isinstance(prc, AugmentationProcessorParams): prc.n_augmentations = n_augmentations elif not isinstance(prc, PrepareSampleProcessorParams): prc.modes -= (PipelineMode.TRAINING, PipelineMode.EVALUATION) newprcs.append(prc) p.scenario.data.pre_proc.processors = newprcs p.device.gpus = [n for n, _ in enumerate(list_physical_devices("GPU"))] if validation_split_ratio < 1: p.gen = CalamariSplitTrainerPipelineParams( validation_split_ratio=validation_split_ratio, train=train ) else: p.gen = CalamariTrainOnlyPipelineParams(train=train) if bidi: for prc in p.scenario.data.post_proc.processors_of_type( BidiTextProcessorParams ): prc.bidi_direction = BidiDirection.RTL p.epochs = epochs p.codec.keep_loaded = keep_loaded_codec p.gen.train.preload = preload p.warmstart.model = weights p.scenario.model.ensemble = ensemble p.ema_decay = ema_decay p.scenario.data.__post_init__() p.scenario.__post_init__() p.__post_init__() p.output_dir = name trainer = p.scenario.cls().create_trainer(p) return trainer.train()
""" LogName = __name__ # 日誌名稱設為模組名稱 RefPath = '' # 參考路徑 if __name__ == "__main__": # executed RefPath = os.path.dirname(os.getcwd()) LogName = None # 當此腳本被執行時為根日誌(不設名稱) else: # imported modulepath = os.path.dirname(__file__) RefPath = os.path.dirname(modulepath) Logger = myLog.Log(myParams.DefaultRootLogLevel, LogName) # 將運算裝置寫入log try: Logger.info(" *** Physical devices for DRL computing *** ") Logger.info('{}'.join(map(str, list_physical_devices("CPU")))) Logger.info('{}'.join(map(str, list_physical_devices("GPU")))) Logger.info(" *** (A blank line above means no GPU used.) *** \n") except: Logger.warning("Call tf.config.list_physical_devices runs into trouble!") Model_Path = RefPath + '\\' + myParams.AI_Foldername # 存放模型參數 if not os.path.isdir(Model_Path): os.mkdir(Model_Path) # Global constants Agent_List = ["DQN", "DDQN", "DDDQN"] Train_Test = "train" Symbol = '' Act_List = ["sigmoid", "softmax", "tanh", "relu", "leakyrelu", "selu", "elu"] LeakyAlpha = 0.3 # leaky slope, keras default = 0.3
import os import numpy as np import matplotlib.pyplot as plt import tensorflow_datasets as tfds from tensorflow.data import Dataset from tensorflow.keras.optimizers import Adam from tensorflow.data.experimental import AUTOTUNE from tensorflow.keras.callbacks import ModelCheckpoint from tensorflow.config.experimental import set_memory_growth from tensorflow.config import list_physical_devices # Failed to get convolution algorithm physical_devices = list_physical_devices('GPU') set_memory_growth(physical_devices[0], True) # Hyperparameters from cycle.config import ORIG_IMG_SIZE from cycle.config import INPUT_IMG_SIZE from cycle.config import BUFFER_SIZE from cycle.config import BATCH_SIZE from cycle import CycleGAN from cycle.callbacks import GANMonitor from cycle.loss import generator_loss_fn from cycle.loss import discriminator_loss_fn from cycle.generator import get_resnet_generator from cycle.discriminator import get_discriminator from cycle.preprocessing import preprocess_train_image from cycle.preprocessing import preprocess_test_image
def wrapper(*args, **kwds): if config.list_physical_devices('GPU'): return f(*args, **kwds) else: print(f'No GPUs detected, skipping "{f.__name__}".') return True