def set_gpus(n_gpus=gpu_settings["n_gpus"],
min_vram=gpu_settings["min_vram"],
split_gpu_into=gpu_settings["split_gpu_into"]):
'''
Configures the GPUs to be allocated for training, preferring the GPUs with most free VRAM.
:param
n_gpus: How many physical GPUs to allocate for this training process. Set to 0 to run on CPU.
min_memory: How much free VRAM each physical GPU has to have. Too low value causes an error if the GPU runs out of memory when training.
This prevents TensorFlow from allocating all of the memory on GPU to the process.
split_into: How many logical GPUs to split each physical GPU into. This can speed up the training due to distributed training.
Each physical GPU has to have min_memory * split_into VRAM available or an error is raised.
:return
None
'''
if n_gpus == 0:
environ['CUDA_VISIBLE_DEVICES'] = ''
gpu_stats = GPUStatCollection.new_query()
gpu_ids = map(lambda gpu: int(gpu.entry['index']), gpu_stats)
gpu_freemem = map(
lambda gpu: float(gpu.entry['memory.total'] - gpu.entry['memory.used']
), gpu_stats)
pairs = list(zip(gpu_ids, gpu_freemem))
valid_pairs = [
pair for pair in pairs if pair[1] >= min_vram * split_gpu_into
]
if len(valid_pairs) < n_gpus:
raise ValueError(
f"Not enough valid GPUs detected. Check if the machine has at least {n_gpus} GPUs with at least {min_vram * split_gpu_into}MB free VRAM or set a lower --n_gpus value"
)
sorted_indices = list(argsort([mem[1] for mem in valid_pairs]))[::-1]
sorted_pairs = [valid_pairs[i] for i in sorted_indices]
if n_gpus != 0:
print(
f"Setting {n_gpus} physical GPUs split into {n_gpus * split_gpu_into} logical GPUs with {min_vram}MB VRAM each for this training"
)
else:
print("Training on CPU")
environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
devices = ",".join([str(pair[0]) for pair in sorted_pairs[:n_gpus]])
environ['CUDA_VISIBLE_DEVICES'] = devices
if split_gpu_into > 1:
physical_devices = tf_config.list_physical_devices('GPU')
for device in physical_devices:
tf_config.set_logical_device_configuration(device, [
tf_config.LogicalDeviceConfiguration(memory_limit=min_vram)
for _ in range(split_gpu_into)
])
def __init__(self, model_parameters, hidden_dim, seq_len, n_seq, gamma):
physical_devices = tfconfig.list_physical_devices('GPU')
if len(physical_devices) > 0:
try:
tfconfig.experimental.set_memory_growth(
physical_devices[0], True)
except:
# Invalid device or cannot modify virtual devices once initialized.
pass
self.seq_len = seq_len
self.n_seq = n_seq
self.hidden_dim = hidden_dim
self.gamma = gamma
super().__init__(model_parameters)
def __init__(
self,
model_parameters
):
gpu_devices = tfconfig.list_physical_devices('GPU')
if len(gpu_devices) > 0:
try:
tfconfig.experimental.set_memory_growth(gpu_devices[0], True)
except:
# Invalid device or cannot modify virtual devices once initialized.
pass
self._model_parameters = model_parameters
[self.batch_size, self.lr, self.beta_1, self.beta_2, self.noise_dim,
self.data_dim, self.layers_dim] = model_parameters
self.define_gan()
def prep_gpu(distribution=None):
print(f"\n!--PREPPING GPU--! with stratagy ")
#traceback.print_stack()
if distribution == None:
gpus = list_physical_devices('GPU')
if gpus:
try:
# Currently, memory growth needs to be the same across GPUs
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus),
"Logical GPUs")
except RuntimeError as e:
# Memory growth must be set before GPUs have been initialized
print(e)
raise
print()
return
def call(self, inputs, training=True):
"""Normal call."""
pad_size0 = self.units - 1 - self.non_causal
pad_size1 = self.non_causal
padded_input = tf.pad(inputs, [[0, 0], [pad_size0, pad_size1], [0, 0]])
# SVD inserting
DEBUG = False
if DEBUG:
print("Banks:", self.banks)
print("Units:", self.units)
print("N_inputs: ", self.n_inputs)
if config.list_physical_devices('GPU') or \
(self.n_inputs == padded_input.shape[-1]):
# this implementation does not evaluate on a CPU if mapping subsets of
# the input into the different FIR filters.
transposed = tf.transpose(tf.reverse(self.coefficients, axis=[-1]))
Y = tf.nn.conv1d(padded_input, transposed, stride=1, padding='VALID')
if DEBUG:
print("padded input: ", padded_input.shape)
print("transposed kernel: ", transposed.shape)
print("Y: ", Y.shape)
return Y
# alternative, kludgy implementation, evaluate each filter separately on the
# corresponding subset of inputs, concatenate outputs when done
transposed = tf.transpose(tf.reverse(self.coefficients, axis=[-1]))
if DEBUG:
print("padded input: ", padded_input.shape)
print("transposed kernel: ", transposed.shape)
L = []
for i in range(transposed.shape[2]):
W = transposed.shape[1]
A = padded_input[:, :, (i*W):((i+1)*W)]
B = transposed[:, :, i:(i+1)]
L.append(tf.nn.conv1d(A, B, stride=1, padding='VALID'))
Y = tf.concat(L, axis=2)
if DEBUG:
print("L[0]: ", L[0].shape)
print("Y: ", Y.shape)
return Y
def prep_gpu(distribution=None):
import tensorflow as tf
try:
from tensorflow.config import list_physical_devices, list_logical_devices
except ImportError:
from tensorflow.config.experimental import list_physical_devices, list_logical_devices
print(f"\n!--PREPPING GPU--! ")
if distribution is None:
gpus = list_physical_devices('GPU')
if gpus:
try:
# Currently, memory growth needs to be the same across GPUs
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = list_logical_devices('GPU')
print(len(gpus), 'Physical GPUs,', len(logical_gpus), 'Logical GPUs')
except RuntimeError as e:
# Memory growth must be set before GPUs have been initialized
print(e)
raise
return
def _setup_gpu_environment() -> None:
"""Sets configuration for TensorFlow GPU environment based on env variable."""
gpu_memory_config = os.getenv(ENV_GPU_CONFIG)
if not gpu_memory_config:
return
# Import from tensorflow only if necessary (environment variable was set)
from tensorflow import config as tf_config
parsed_gpu_config = _parse_gpu_config(gpu_memory_config)
physical_gpus = tf_config.list_physical_devices("GPU")
# Logic taken from https://www.tensorflow.org/guide/gpu
if physical_gpus:
for gpu_id, gpu_id_memory in parsed_gpu_config.items():
_allocate_gpu_memory(physical_gpus[gpu_id], gpu_id_memory)
else:
rasa.shared.utils.io.raise_warning(
f"You have an environment variable '{ENV_GPU_CONFIG}' set but no GPUs were "
f"detected to configure.")
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import layers
from tensorflow.keras import callbacks
from tensorflow import config
physical_devices = config.list_physical_devices('GPU')
config.experimental.set_memory_growth(physical_devices[0], enable=True)
import numpy as np
import matplotlib.pyplot as plt
from os import path
import time
def plot_history(history,
acc_name='keras_accuracy.png',
loss_name='keras_loss.png'):
plt.clf()
accuracy = history['accuracy']
val_accuracy = history['val_accuracy']
epochs = range(1, len(accuracy) + 1)
plt.plot(epochs, accuracy, 'tab:blue', label='Training Accuracy')
plt.plot(epochs, val_accuracy, 'tab:orange', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.xlabel('Epochs')
def gpu_info():
return list_physical_devices('GPU')
def evaluate_books(
self,
books,
checkpoint,
cachefile=None,
output_individual_voters=False,
n_confusions=10,
silent=True,
):
keras.backend.clear_session()
if type(books) == str:
books = [books]
if type(checkpoint) == str:
checkpoint = [checkpoint]
checkpoint = [
(cp if cp.endswith(".json") else cp + ".json") for cp in checkpoint
]
checkpoint = glob_all(checkpoint)
checkpoint = [cp[:-5] for cp in checkpoint]
if cachefile is None:
cachefile = self.cachefile
lids = list(
lids_from_books(books, cachefile, complete_only=True, skip_commented=True)
)
data = Nsh5(cachefile=cachefile, lines=lids)
predparams = PredictorParams()
predparams.device.gpus = [n for n, _ in enumerate(list_physical_devices("GPU"))]
predparams.silent = silent
predictor = MultiPredictor.from_paths(
checkpoints=checkpoint,
voter_params=VoterParams(),
predictor_params=predparams,
)
newprcs = []
for prc in predictor.data.params.pre_proc.processors:
prc = deepcopy(prc)
if isinstance(prc, FinalPreparationProcessorParams):
prc.normalize, prc.invert, prc.transpose = False, False, True
newprcs.append(prc)
elif isinstance(prc, PrepareSampleProcessorParams):
newprcs.append(prc)
predictor.data.params.pre_proc.processors = newprcs
do_prediction = predictor.predict(data)
all_voter_sentences = {}
all_prediction_sentences = {}
for s in do_prediction:
_, (_, prediction), _ = s.inputs, s.outputs, s.meta
sentence = prediction.sentence
if prediction.voter_predictions is not None and output_individual_voters:
for i, p in enumerate(prediction.voter_predictions):
if i not in all_prediction_sentences:
all_prediction_sentences[i] = {}
all_prediction_sentences[i][s.meta["id"]] = p.sentence
all_voter_sentences[s.meta["id"]] = sentence
# evaluation
from calamari_ocr.ocr.evaluator import Evaluator, EvaluatorParams
evaluator_params = EvaluatorParams(
setup=predparams.pipeline,
progress_bar=True,
skip_empty_gt=True,
)
evaluator = Evaluator(evaluator_params, predictor.data)
evaluator.preload_gt(gt_dataset=data, progress_bar=True)
def single_evaluation(label, predicted_sentences):
r = evaluator.evaluate(
gt_data=evaluator.preloaded_gt, pred_data=predicted_sentences
)
print("=================")
print(f"Evaluation result of {label}")
print("=================")
print("")
print(
"Got mean normalized label error rate of {:.2%} ({} errs, {} total chars, {} sync errs)".format(
r["avg_ler"],
r["total_char_errs"],
r["total_chars"],
r["total_sync_errs"],
)
)
print()
print()
# sort descending
print_confusions(r, n_confusions)
return r
full_evaluation = {}
for id, data in [
(str(i), sent) for i, sent in all_prediction_sentences.items()
] + [("voted", all_voter_sentences)]:
full_evaluation[id] = {"eval": single_evaluation(id, data), "data": data}
if not predparams.silent:
print(full_evaluation)
return full_evaluation
def predict_books(
self,
books,
checkpoint,
cachefile=None,
pageupload=True,
text_index=1,
pred_all=False,
):
keras.backend.clear_session()
if type(books) == str:
books = [books]
if type(checkpoint) == str:
checkpoint = [checkpoint]
checkpoint = [
(cp if cp.endswith(".json") else cp + ".json") for cp in checkpoint
]
checkpoint = glob_all(checkpoint)
checkpoint = [cp[:-5] for cp in checkpoint]
if cachefile is None:
cachefile = self.cachefile
verbose = False
lids = list(
lids_from_books(
books,
cachefile,
complete_only=False,
skip_commented=False,
new_only=not pred_all,
)
)
data = Nsh5(cachefile=cachefile, lines=lids)
predparams = PredictorParams()
predparams.device.gpus = [n for n, _ in enumerate(list_physical_devices("GPU"))]
predictor = MultiPredictor.from_paths(
checkpoints=checkpoint,
voter_params=VoterParams(),
predictor_params=predparams,
)
newprcs = []
for prc in predictor.data.params.pre_proc.processors:
prc = deepcopy(prc)
if isinstance(prc, FinalPreparationProcessorParams):
prc.normalize, prc.invert, prc.transpose = False, False, True
newprcs.append(prc)
elif isinstance(prc, PrepareSampleProcessorParams):
newprcs.append(prc)
predictor.data.params.pre_proc.processors = newprcs
do_prediction = predictor.predict(data)
pipeline = predictor.data.get_or_create_pipeline(
predictor.params.pipeline, data
)
reader = pipeline.reader()
if len(reader) == 0:
raise Exception(
"Empty dataset provided. Check your lines (got {})!".format(lids)
)
avg_sentence_confidence = 0
n_predictions = 0
reader.prepare_store()
samples = []
sentences = []
# output the voted results to the appropriate files
for s in do_prediction:
_, (_, prediction), meta = s.inputs, s.outputs, s.meta
sample = reader.sample_by_id(meta["id"])
n_predictions += 1
sentence = prediction.sentence
avg_sentence_confidence += prediction.avg_char_probability
if verbose:
lr = "\u202A\u202B"
logger.info(
"{}: '{}{}{}'".format(
meta["id"], lr[get_base_level(sentence)], sentence, "\u202C"
)
)
samples.append(sample)
sentences.append(sentence)
reader.store_text(sentence, sample, output_dir=None, extension=None)
logger.info(
"Average sentence confidence: {:.2%}".format(
avg_sentence_confidence / n_predictions
)
)
if pageupload:
ocrdata = {}
for lname, text in reader.predictions.items():
_, b, p, ln = lname.split("/")
if b not in ocrdata:
ocrdata[b] = {}
if p not in ocrdata[b]:
ocrdata[b][p] = {}
ocrdata[b][p][ln] = text
data = {"ocrdata": ocrdata, "index": text_index}
self.get_session().post(
self.baseurl + "/_ocrdata",
data=gzip.compress(json.dumps(data).encode("utf-8")),
headers={
"Content-Type": "application/json;charset=UTF-8",
"Content-Encoding": "gzip",
},
)
logger.info("Results uploaded")
else:
reader.store()
logger.info("All prediction files written")
def cftrain_books(
self,
books,
n_folds=5,
cachefile=None,
name="models",
tempdir=None,
keep_temporary_files=False,
max_parallel_models=-1,
skip_commented=True,
validation_split_ratio=1,
bidi="",
n_augmentations=0,
ema_decay=0.0,
train_verbose=1,
debug=False,
epochs=100,
whitelist="",
keep_loaded_codec=False,
preload=True,
weights=[],
ensemble=0,
):
keras.backend.clear_session()
if isinstance(weights, str):
weights = [weights]
if isinstance(books, str):
books = [books]
if cachefile is None:
cachefile = self.cachefile
if max_parallel_models < 1:
max_parallel_models = n_folds
lids = list(
lids_from_books(
books, cachefile, complete_only=True, skip_commented=skip_commented
)
)
train = Nsh5(cachefile=cachefile, lines=lids)
cfparams = CrossFoldTrainerParams()
cfparams.weights = weights
cfparams.temporary_dir = tempdir
cfparams.best_models_dir = name
cfparams.n_folds = n_folds
cfparams.keep_temporary_files = False
cfparams.max_parallel_models = max_parallel_models
newprcs = []
for prc in cfparams.trainer.scenario.data.pre_proc.processors:
prc = deepcopy(prc)
if PipelineMode.TRAINING in prc.modes:
if isinstance(prc, FinalPreparationProcessorParams):
prc.normalize, prc.invert, prc.transpose = False, False, True
elif isinstance(prc, AugmentationProcessorParams):
prc.n_augmentations = n_augmentations
elif not isinstance(prc, PrepareSampleProcessorParams):
prc.modes = set()
newprcs.append(prc)
cfparams.trainer.scenario.data.pre_proc.processors = newprcs
cfparams.trainer.device.gpus = [
n for n, _ in enumerate(list_physical_devices("GPU"))
]
cfparams.trainer.gen.train = train
if bidi:
for prc in cfparams.trainer.scenario.data.post_proc.processors_of_type(
BidiTextProcessorParams
):
prc.bidi_direction = BidiDirection.RTL
cfparams.trainer.epochs = epochs
cfparams.trainer.codec.keep_loaded = keep_loaded_codec
cfparams.trainer.gen.train.preload = preload
cfparams.trainer.scenario.model.ensemble = ensemble
cfparams.trainer.ema_decay = ema_decay
cfparams.trainer.scenario.data.__post_init__()
cfparams.trainer.scenario.__post_init__()
cfparams.trainer.__post_init__()
print(cfparams.to_json())
with open("debug.json", "w") as f:
f.write(cfparams.to_json())
trainer = CrossFoldTrainer(cfparams)
return trainer.run()
def train_books(
self,
books,
cachefile=None,
name="model",
skip_commented=True,
validation_split_ratio=1,
bidi="",
n_augmentations=0,
ema_decay=0.0,
train_verbose=1,
debug=False,
epochs=100,
whitelist="",
keep_loaded_codec=False,
preload=True,
weights=None,
ensemble=0,
):
keras.backend.clear_session()
if isinstance(books, str):
books = [books]
if cachefile is None:
cachefile = self.cachefile
p = CalamariScenario.default_trainer_params()
lids = list(
lids_from_books(
books, cachefile, complete_only=True, skip_commented=skip_commented
)
)
train = Nsh5(cachefile=cachefile, lines=lids)
newprcs = []
for prc in p.scenario.data.pre_proc.processors:
prc = deepcopy(prc)
if PipelineMode.TRAINING in prc.modes:
if isinstance(prc, FinalPreparationProcessorParams):
prc.normalize, prc.invert, prc.transpose = False, False, True
elif isinstance(prc, AugmentationProcessorParams):
prc.n_augmentations = n_augmentations
elif not isinstance(prc, PrepareSampleProcessorParams):
prc.modes -= (PipelineMode.TRAINING, PipelineMode.EVALUATION)
newprcs.append(prc)
p.scenario.data.pre_proc.processors = newprcs
p.device.gpus = [n for n, _ in enumerate(list_physical_devices("GPU"))]
if validation_split_ratio < 1:
p.gen = CalamariSplitTrainerPipelineParams(
validation_split_ratio=validation_split_ratio, train=train
)
else:
p.gen = CalamariTrainOnlyPipelineParams(train=train)
if bidi:
for prc in p.scenario.data.post_proc.processors_of_type(
BidiTextProcessorParams
):
prc.bidi_direction = BidiDirection.RTL
p.epochs = epochs
p.codec.keep_loaded = keep_loaded_codec
p.gen.train.preload = preload
p.warmstart.model = weights
p.scenario.model.ensemble = ensemble
p.ema_decay = ema_decay
p.scenario.data.__post_init__()
p.scenario.__post_init__()
p.__post_init__()
p.output_dir = name
trainer = p.scenario.cls().create_trainer(p)
return trainer.train()
"""
LogName = __name__ # 日誌名稱設為模組名稱
RefPath = '' # 參考路徑
if __name__ == "__main__": # executed
RefPath = os.path.dirname(os.getcwd())
LogName = None # 當此腳本被執行時為根日誌(不設名稱)
else: # imported
modulepath = os.path.dirname(__file__)
RefPath = os.path.dirname(modulepath)
Logger = myLog.Log(myParams.DefaultRootLogLevel, LogName)
# 將運算裝置寫入log
try:
Logger.info(" *** Physical devices for DRL computing *** ")
Logger.info('{}'.join(map(str, list_physical_devices("CPU"))))
Logger.info('{}'.join(map(str, list_physical_devices("GPU"))))
Logger.info(" *** (A blank line above means no GPU used.) *** \n")
except:
Logger.warning("Call tf.config.list_physical_devices runs into trouble!")
Model_Path = RefPath + '\\' + myParams.AI_Foldername # 存放模型參數
if not os.path.isdir(Model_Path): os.mkdir(Model_Path)
# Global constants
Agent_List = ["DQN", "DDQN", "DDDQN"]
Train_Test = "train"
Symbol = ''
Act_List = ["sigmoid", "softmax", "tanh", "relu", "leakyrelu", "selu", "elu"]
LeakyAlpha = 0.3 # leaky slope, keras default = 0.3
import os
import numpy as np
import matplotlib.pyplot as plt
import tensorflow_datasets as tfds
from tensorflow.data import Dataset
from tensorflow.keras.optimizers import Adam
from tensorflow.data.experimental import AUTOTUNE
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.config.experimental import set_memory_growth
from tensorflow.config import list_physical_devices
# Failed to get convolution algorithm
physical_devices = list_physical_devices('GPU')
set_memory_growth(physical_devices[0], True)
# Hyperparameters
from cycle.config import ORIG_IMG_SIZE
from cycle.config import INPUT_IMG_SIZE
from cycle.config import BUFFER_SIZE
from cycle.config import BATCH_SIZE
from cycle import CycleGAN
from cycle.callbacks import GANMonitor
from cycle.loss import generator_loss_fn
from cycle.loss import discriminator_loss_fn
from cycle.generator import get_resnet_generator
from cycle.discriminator import get_discriminator
from cycle.preprocessing import preprocess_train_image
from cycle.preprocessing import preprocess_test_image
def wrapper(*args, **kwds):
if config.list_physical_devices('GPU'):
return f(*args, **kwds)
else:
print(f'No GPUs detected, skipping "{f.__name__}".')
return True
