def __init__(self,features,items,strategy='uniform',alpha=None, bias=None):
'''
Class Initializer.
Arguments:
:param strategy: how we sample the new negative labels
options are "popular", and "uniform", or None
:param alpha: how to flatten the unigram distribution we sample from
0.0 corresponds to uniform, 1.0 corresponds to complete popular sampling.
:param features: which features are being sampled?
expects a list of features, which can be multidimensional
(e.g. you can pass 'household_id', or ['city,'state'] as a single element of the list of all features to sample
:param items: possible values of sampled feature; again a list of combinations
that can be multidimensional depending on the corresponding feature. should be a list of lists of tuples,
but if single-dimensional could also be a list of lists of objects as long as they are hashable
sample_bias: constant term added to count of labels in data for sampling from unigram
'''
self.df = None
self.strategy_name = strategy
if alpha is not None:
self.alpha=alpha
elif str.lower(self.strategy_name)=="uniform":
self.alpha=0.
elif str.lower(self.strategy_name)=="popular":
self.alpha=1.
if bias is None:
self.bias=10
else:
self.bias=bias
self.features=features
self.items= items
assert isinstance(self.alpha, float), "Must provide sampling strategy or alpha term that is a float"
assert isinstance(self.features, list), "Must provide features as a list (even if its just one)"
Callback.__init__(self)
def __init__(self, my_model, train_p, test_p, lc_file_name=None):
Callback.__init__(self)
self.my_model = my_model
self.test = test_p
self.train = train_p
self.lc_file = None
self.lc_file_name = lc_file_name
def __init__(self, predit_funct=None):
Callback.__init__(self)
# output_notebook()
self.loss = np.array([])
self.psnrs = np.array([])
output_server("line")
self.imagew = 512
self.min_loss = 10000
self.predit_funct = predit_funct
self.p = figure()
self.p2 = figure()
self.x = np.array([])
self.y = np.array([])
self.bx = np.array([])
self.by = np.array([])
self.cx = np.array([])
self.epochNo = 0
self.p.line(self.x, self.y, name='line', color="tomato", line_width=2)
self.p.line(self.bx, self.by, name='batch_line', color="blue", line_width=2)
self.p2.line(self.cx, self.psnrs, name='psnr', color="green", line_width=2)
show(self.p)
# show(self.p2)
# self.p2 = figure(x_range=[0, self.imagew], y_range=[0, self.imagew])
# self.p2.image_rgba(name='image', image=[np.array((self.imagew, self.imagew), dtype='uint32')], x=0, y=0, dw=self.imagew, dh=self.imagew)
# show(self.p2)
self.psnr = 0
def __init__(self, dataset_id):
Callback.__init__(self)
self.seen = 0
self.dataset_id = dataset_id
self.samples = 1000
self.oldperc = 0
def __init__(self,
test_generator,
test_batches,
weights_file,
history_file,
n=10):
"""
Constructor for the TrackTestHistoryAndModel callback.
:param test_generator: generator, iterator that generates batched test data image patches.
:param test_batches: int, number of complete batches in the generator.
:param weights_file: str, target path for the model weights/parameters to be stored in HDF5 format.
:param history_file: str, target path for the history CSV file.
:param n: int, marks after how many epochs the model is being evaluated on the test data.
"""
Callback.__init__(self)
self.test_generator = test_generator
self.test_batches = test_batches
self.weights_file = weights_file
self.history_file = history_file
self.n = n
self.accuracies = []
self.losses = []
def __init__(self,
only_save_weight: bool = True,
*models: Tuple[Model, PathLike]):
Callback.__init__(self)
self._only_save_weight = only_save_weight
self._models = models
self._best = np.inf
def __init__(self, X_test, Y_test, x_aug, y_aug, indices):
Callback.__init__(self)
self.X_test = X_test
self.Y_test = Y_test
self.x_aug = x_aug
self.y_aug = y_aug
self.indices = indices
def __init__(self, predit_funct=None):
Callback.__init__(self)
# output_notebook()
self.loss = np.array([])
self.psnrs = np.array([])
output_server("line")
self.imagew = 512
self.min_loss = 10000
self.predit_funct = predit_funct
self.p = figure()
self.p2 = figure()
self.x = np.array([])
self.y = np.array([])
self.bx = np.array([])
self.by = np.array([])
self.cx = np.array([])
self.epochNo = 0
self.p.line(self.x, self.y, name='line', color="tomato", line_width=2)
self.p.line(self.bx,
self.by,
name='batch_line',
color="blue",
line_width=2)
self.p2.line(self.cx,
self.psnrs,
name='psnr',
color="green",
line_width=2)
show(self.p)
# show(self.p2)
# self.p2 = figure(x_range=[0, self.imagew], y_range=[0, self.imagew])
# self.p2.image_rgba(name='image', image=[np.array((self.imagew, self.imagew), dtype='uint32')], x=0, y=0, dw=self.imagew, dh=self.imagew)
# show(self.p2)
self.psnr = 0
def __init__(self, X_test, Y_test, x_aug, y_aug, indices):
Callback.__init__(self)
self.X_test = X_test
self.Y_test = to_label(Y_test[0])
self.x_aug = x_aug
self.y_aug = to_label(y_aug[0])
self.indices = indices
def __init__(self,
dev_data,
maxlen,
trained_model_path,
trigger_model,
object_model,
subject_model,
loc_model,
time_model,
negative_model,
train_model,
max_learning_rate=extract_train_config.learning_rate,
min_learning_rate=extract_train_config.min_learning_rate):
Callback.__init__(self, )
self.best = 0.
self.passed = 0
self.maxlen = maxlen
self.dev_data = dev_data
# 训练后模型路径
self.trained_mdoel_path = trained_model_path
# 学习率
self.max_learning_rate = max_learning_rate
self.min_learning_rate = min_learning_rate
# 论元模型
self.trigger_model = trigger_model
self.object_model = object_model
self.subject_model = subject_model
self.negative_model = negative_model
self.time_model = time_model
self.loc_model = loc_model
# 主模型
self.train_model = train_model
def __init__(self,
model,
tf_record_db,
checkpoint_save_path=None,
log_save_path=None,
batch_size=32):
"""
Args:
model: instance of @keras.model.Model. The model to be evaluate. The model's input must be a @keras.layers.Input layer.
tf_record_db: instance of @tfrecords_db.TfRecordDB.
Kwargs:
checkpoint_save_path: file to save the checkpoint.
log_save_path: file to save the log file.
batch_size: the evaluation batch size.
"""
Callback.__init__(self)
self._model = model
self._tf_record_db = tf_record_db
self._checkpoint_save_path = checkpoint_save_path
self._log_save_path = log_save_path
self._batch_size = batch_size
self._model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
def __init__(self):
"""
Constructor.
"""
Callback.__init__(self)
self.losses = []
return
def __init__(self,
epochs,
X_test_claims,
y_test,
loss_filename,
epsilon=0.0,
min_epoch=15,
X_test_nt=None):
self.epochs = epochs
self.patience = 2
self.counter = 0
self.prev_score = 0
self.epsilon = epsilon
self.loss_filename = loss_filename
self.min_epoch = min_epoch
self.X_test_nt = X_test_nt
#self.print_train_f1 = print_train_f1
#self.X_train_claims = X_train_claims
#self.X_train_orig_docs = X_train_orig_docs
#self.X_train_evid = X_train_evid
#self.y_train = y_train
self.X_test_claims = X_test_claims
# self.X_test_orig_docs = X_test_orig_docs
self.y_test = y_test
Callback.__init__(self)
def __init__(self, filepath, filepathmean, epoch_start=0, do_mean=True):
Callback.__init__(self)
self.filepath = filepath
self.filepathmean = filepathmean
self.headers = True
self.keys = []
self.epoch_start = epoch_start
self.do_mean = do_mean
def __init__(self):
Callback.__init__(self)
self.epoch_acc_dic = {}
self.epoch_loss_dic = {}
self.epoch_val_acc_dic = {}
self.epoch_val_loss_dic = {}
self.epoch_time_dic = {}
self.starttime = None
def __init__(self, name, peptides, mhc_seqs, labels, weights, predictor):
Callback.__init__(self)
self.name = name
self.peptides = peptides
self.mhc_seqs = mhc_seqs
self.labels = labels
self.weights = weights
self.predictor = predictor
self.aucs = defaultdict(list)
def __init__(self, x_test, y_test, save_file_weights, save_file_hist, n=10):
Callback.__init__(self)
self.x_test = x_test
self.y_test = y_test
self.save_file_weights = save_file_weights
self.save_file_hist = save_file_hist
self.n = n
self.test_acc = []
self.test_loss = []
def __init__(self, config):
Callback.__init__(self)
self.model_dir = os.path.join(FRCNN_MODELS_DIR, config.model_name)
with open(os.path.join(self.model_dir, 'config.json'), 'w') as file:
json.dump(config,
file,
default=lambda o: o.__dict__,
indent=4,
separators=(',', ': '))
def __init__(self,
print_fcn=print,
format_epoch="Epoch: {} - {}",
format_keyvalue="{}: {:0.4f}",
format_separator=" - "):
Callback.__init__(self)
self.print_fcn = print_fcn
self.format_epoch = format_epoch
self.format_keyvalue = format_keyvalue
self.format_separator = format_separator
def __init__(self, test_data, train_data, is_bidirectional, infeat,
outfeat, model_type):
Callback.__init__(self)
self.test_data = test_data
self.train_data = train_data
self.count = 0
self.idx = 1 if is_bidirectional else 5
self.infeat = infeat
self.outfeat = outfeat
self.description = model_type
def __init__(self,
tensorboard_callback,
trace_level=tf.RunOptions.FULL_TRACE):
"""initialise the baseclass,
keep a reference to the tensorboard callback (we access the tensorboard writer)
create run_options and run_metadata objects to record data."""
Callback.__init__(self)
self.tensorboard_callback = tensorboard_callback
self.run_options = tf.RunOptions(trace_level=trace_level)
self.run_metadata = tf.RunMetadata()
def __init__(self, out_file):
"""
Constructor for the TrackTrainHistory callback.
:param out_file: str, target location of the history CSV file.
"""
Callback.__init__(self)
self.accuracies = []
self.losses = []
self.out_file = out_file
def __init__(self, training, network, count_mode='samples'):
# FIXME[question]: can we use real python multiple inheritance here?
# (that is just super().__init__(*args, **kwargs))
BaseTrainer.__init__(self, training, network)
Callback.__init__(self)
if count_mode == 'samples':
self.use_steps = False
elif count_mode == 'steps':
self.use_steps = True
else:
raise ValueError('Unknown `count_mode`: ' + str(count_mode))
def __init__(self,
train_generator,
val_generator,
summary_path,
loss,
model=None):
Callback.__init__(self)
self.train_generator = train_generator
self.val_generator = val_generator
self.summary_path = summary_path
self.loss = loss
def __init__(self,
x_data,
y_data,
model_filename,
eval_fun=None,
eval_every_n_epochs=1,
save_best=True):
"""
Produces an instance of a keras Callback. It allows for running a set of routines when a
number of training epochs has elapsed.
Each routine accepts the X and Y data and the best performing model up to the current
epoch. The best performing model is evaluated by the validation accuracy (or the train
accuracy if no validation is done). It allows for you to evaluate you model during
training and print performance reports of the model. it also allows you to have snapshots
of your model during training.
Args:
x_data: The list of X data
y_data: The list of Y data
model_filename: The filename to use for saving the model snapshots
eval_fun: The list of functions to run
eval_every_n_epochs: The number of epochs after each to run the routines
save_best: Set True to save the model snapshots
"""
Callback.__init__(self)
self.save_best = save_best
self.eval_every_n_epochs = eval_every_n_epochs
self.eval_fun = eval_fun
if eval_fun is None:
self.eval_fun = []
elif not isinstance(eval_fun, list) and not isinstance(
eval_fun, tuple):
self.eval_fun = [eval_fun]
else:
self.eval_fun = eval_fun
if not isinstance(x_data, list) and not isinstance(x_data, tuple):
self.x_test = [x_data]
else:
self.x_test = x_data
if not isinstance(y_data, list) and not isinstance(y_data, tuple):
self.y_test = [y_data]
else:
self.y_test = y_data
if not isinstance(self.y_test, type(self.x_test)) or not isinstance(
self.eval_fun, type(self.x_test)):
assert AssertionError('Given types for x_y or incorrect')
self.best_val_loss = None
self.model_filename = model_filename
self.best_val_model = None
self.update_since_last = False
def __init__(self, name, checkpoint_path, net_name="NET", print_fcn=print):
Callback.__init__(self)
self.print_fcn = print_fcn
self.checkpoint_path = checkpoint_path
self.log_file_name = "logs/" + str(
name
) + ".txt" #_" + strftime("%Y-%m-%d_%H:%M:%S", gmtime()) + ".txt")
log_file = open(self.log_file_name, "a")
log_file.write("------")
log_file.write("NEW NET: " + net_name)
log_file.write("------")
log_file.close()
self.best_val_loss = 20
def __init__(self, target, save_monitor='val_loss', mode='desc'):
Callback.__init__(self)
self._target = target
self._save_monitor = save_monitor
self._mode = mode
if mode == 'desc':
self._best = float('inf')
elif mode == 'asc':
self._best = -float('inf')
else:
raise ValueError(
"Invalid mode '%s'. Mode must be either 'asc' or 'desc'.")
self._mode_map = {'asc': self.is_asc, 'desc': self.is_desc}
def __init__(self,
validation,
train,
test,
every_n=25,
verbose=0,
folder_test="",
network_name=""):
Callback.__init__(self)
self.every_n = every_n
self.verbose = verbose
self.network_name = network_name
self.folder_test = folder_test
self.validation = validation
self.train = train
self.test = test
def __init__(self, name, fig_title, url):
"""
fig_title: Figure Title
url : str, optional
Url of the bokeh-server. Ex: when starting the bokeh-server with
``bokeh-server --ip 0.0.0.0`` at ``alice``, server_url should be
``http://alice:5006``. When not specified the default configured
by ``bokeh_server`` in ``.blocksrc`` will be used. Defaults to
``http://localhost:5006/``.
Reference: mila-udem/blocks-extras
"""
Callback.__init__(self)
self.name = name
self.fig_title = fig_title
self.plots = []
output_server(name, url=url)
cursession().publish()
def __init__(self, dataset, epochs, n_classes, batch_size, h5_model_path,
fold_stats_path):
# allocating inputs as properties
self.dataset = dataset
self.epochs = epochs
self.n_classes = n_classes
self.batch_size = batch_size
self.h5_model_path = h5_model_path
self.fold_stats_path = fold_stats_path
# pre-allocating train, valid and test dicts with loss and conf_mtx
self.train = {'loss': [], 'acc': []}
self.valid = {'loss': [], 'acc': []}
# pre-allocating best (to track the best net configuration)
self.best = {'loss': float('inf'), 'acc': 0, 'idx': None}
# calling the super class constructor
Callback.__init__(self)
def __init__(self,cv_number):
Callback.__init__(self)
self.cv_number = cv_number
def __init__(self, batch_size, **kwargs):
Callback.__init__(self, **kwargs)
self.batch_size = batch_size
def __init__(self, test_data):
Callback.__init__(self)
self.test_data = test_data
self.count = 0
def __init__(self, val = False):
Callback.__init__(self)
self.val = val
self.losses = []
self.batch_losses = []
self.batch_accs = []
def __init__(self, W, transpose=False):
Callback.__init__(self)
self.W = W
self.W_shape = self.W.get_value().shape
self.transpose = transpose
