def benchmark_split_merge_tokenizer(self):
if FLAGS.ragged_vs_dense:
return
random_seed.set_seed(5)
char_splits = self._get_char_level_splits()
if not context.executing_eagerly():
# Evaluate splits as their shape cannot be infered in graph mode
# and are needed for mapping
with session.Session() as sess:
sess.run(self.iterator.initializer)
char_splits = sess.run(char_splits)
def randomize_splits(inputs):
return random_ops.random_uniform(inputs.shape,
maxval=2,
dtype=dtypes.int32)
labels = ragged_functional_ops.map_flat_values(randomize_splits,
char_splits)
if not context.executing_eagerly():
# Evaluate labels computation to exclude these steps from op benchmarking
with session.Session() as sess:
labels = sess.run(labels)
tokenizer = text_ops.SplitMergeTokenizer()
self._run(tokenizer, {"labels": labels})
def setUp(self):
self.conv2d_mod = Conv2DModulated(filters=10,
kernel_size=3,
lr_multiplier=1.0,
gain=1.0,
modulate=True,
demodulate=True)
random_seed.set_seed(42)
def __init__(self,
experiment_id: str = None,
verbose: bool = True,
out_directory: str = None,
auto_datetime_directory: bool = True,
set_seeds: bool = True,
suppress_warnings: bool = True):
self.experiment_id = experiment_id
self.experiment_start = datetime.datetime.now()
self.out_directory = out_directory
self.verbose = verbose
self.run_count = 0
self.run_start = None
self.run_id = None
if set_seeds:
seed(1)
set_seed(1)
if self.experiment_id is None:
caller_frame = inspect.stack()[1]
caller_filename = caller_frame.filename
self.experiment_id = os.path.splitext(
os.path.basename(caller_filename))[0]
if self.out_directory is None:
experiment_directory = pathlib.Path(__file__).parent.absolute()
self.out_directory = os.path.abspath(
os.path.join(experiment_directory,
f'../../out/{self.experiment_id}/'))
if auto_datetime_directory:
self.out_directory = os.path.join(
self.out_directory,
self.experiment_start.strftime('%Y_%m_%d/%H_%M_%S'))
if os.path.isdir(self.out_directory):
shutil.rmtree(self.out_directory)
if not os.path.isdir(self.out_directory):
os.makedirs(self.out_directory)
self.print(f'Created new experiment "{self.experiment_id}"')
self.log('timing.log', f'experiment_start={self.experiment_start}')
try:
self.log('git/hash', git_hash())
self.log('git/state.diff', git_diff())
except:
pass # not a git repo
if suppress_warnings:
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.get_logger().setLevel('ERROR')
logging.disable()
else:
warnings.simplefilter(action='default', category=Warning)
def testRandomShuffle2022Eager(self):
original = StructuredTensor.from_pyval([{
"x0": 0,
"y": {
"z": [[3, 13]]
}
}, {
"x0": 1,
"y": {
"z": [[3], [4, 13]]
}
}, {
"x0": 2,
"y": {
"z": [[3, 5], [4]]
}
}, {
"x0": 3,
"y": {
"z": [[3, 7, 1], [4]]
}
}, {
"x0": 4,
"y": {
"z": [[3], [4]]
}
}]) # pyformat: disable
expected = StructuredTensor.from_pyval([{
"x0": 1,
"y": {
"z": [[3], [4, 13]]
}
}, {
"x0": 0,
"y": {
"z": [[3, 13]]
}
}, {
"x0": 3,
"y": {
"z": [[3, 7, 1], [4]]
}
}, {
"x0": 4,
"y": {
"z": [[3], [4]]
}
}, {
"x0": 2,
"y": {
"z": [[3, 5], [4]]
}
}]) # pyformat: disable
random_seed.set_seed(1066)
result = structured_array_ops.random_shuffle(original, seed=2022)
self.assertAllEqual(result, expected)
def seed(s):
"""Sets the seed for the random number generator.
Uses `tf.set_random_seed`.
Args:
s: an integer.
"""
# TODO(wangpeng): make the signature the same as numpy
random_seed.set_seed(s)
def seed(s):
"""Sets the seed for the random number generator.
Uses `tf.set_random_seed`.
Args:
s: an integer.
"""
try:
s = int(s)
except TypeError:
# TODO(wangpeng): support this?
raise ValueError('np.seed currently only support integer arguments.')
random_seed.set_seed(s)
def seed(s):
"""Sets the seed for the random number generator.
Uses `tf.set_random_seed`.
Args:
s: an integer.
"""
try:
s = int(s)
except TypeError:
# TODO(wangpeng): support this?
raise ValueError(
f'Argument `s` got an invalid value {s}. Only integers are supported.')
random_seed.set_seed(s)
def _genParams(self, dtype=dtypes.float32):
batch_size = 1
image_height = 10
image_width = 10
channels = 1
image_shape = (batch_size, image_height, image_width, channels)
num_boxes = 3
boxes_shape = (num_boxes, 4)
random_seed.set_seed(123)
image = random_ops.random_normal(shape=image_shape, dtype=dtype)
boxes = random_ops.random_uniform(shape=boxes_shape, dtype=dtypes.float32)
box_indices = random_ops.random_uniform(
shape=(num_boxes,), minval=0, maxval=batch_size, dtype=dtypes.int32)
crop_size = constant_op.constant([3, 3], dtype=dtypes.int32)
return image, boxes, box_indices, crop_size
def _genParams(self,
use_cudnn=False,
data_format="NHWC",
dtype=dtypes.float32,
seed=123):
random_seed.set_seed(seed)
batch_size = 2 # no interaction over batch, so make small
if use_cudnn:
# When op-determinism is not enabled, one input channel, plus a
# cuDNN-supported filter size and number of output channels will result
# in cuDNN being used for both backprop-to-input and backprop-to-filter on
# cuDNN 7.6.3 and higher. When op-determnism is enabled, cuDNN is always
# used for backprop-to-filter.
input_channels = 1
else:
input_channels = 2 # no interaction over channels, so make small
input_height = 500
input_width = 1000
if data_format == "NHWC":
input_shape = (batch_size, input_height, input_width,
input_channels)
else: # "NCHW"
input_shape = (batch_size, input_channels, input_height,
input_width)
input_data = random_ops.random_normal(input_shape, dtype=dtype)
# The following filter size results in nondeterminism being exercised in
# cuDNN backprop (when determinism is not enabled) to both input and filter
# as well as in the specialized (non-cuDNN) depthwise backprop to filter.
filter_height = 7
filter_width = 7
channel_multiplier = 10
filter_shape = (filter_height, filter_width, input_channels,
channel_multiplier)
filter_data = random_ops.random_normal(filter_shape, dtype=dtype)
strides = [1, 1, 1, 1]
padding = "SAME"
output_height = input_height # because same padding
output_width = input_width # because same padding
output_channels = input_channels * channel_multiplier
if data_format == "NHWC":
output_shape = (batch_size, output_height, output_width,
output_channels)
else: # "NCHW"
output_shape = (batch_size, output_channels, output_height,
output_width)
return input_data, filter_data, strides, padding, output_shape
def testNextSentencePredictionExtractor(self,
sentences,
expected_segment_a,
expected_segment_b,
expected_labels,
random_next_sentence_threshold=0.5,
test_description=""):
sentences = ragged_factory_ops.constant(sentences)
# Set seed and rig the shuffle function to a deterministic reverse function
# instead. This is so that we have consistent and deterministic results.
random_seed.set_seed(1234)
nsp = segment_extractor_ops.NextSentencePredictionExtractor(
shuffle_fn=functools.partial(array_ops.reverse, axis=[-1]),
random_next_sentence_threshold=random_next_sentence_threshold,
)
results = nsp.get_segments(sentences)
actual_segment_a, actual_segment_b, actual_labels = results
self.assertAllEqual(expected_segment_a, actual_segment_a)
self.assertAllEqual(expected_segment_b, actual_segment_b)
self.assertAllEqual(expected_labels, actual_labels)
def benchmark_split_merge_from_logits_tokenizer(self):
random_seed.set_seed(5)
char_splits = self._get_char_level_splits().to_tensor()
if not context.executing_eagerly():
with session.Session() as sess:
sess.run(self.iterator.initializer)
char_splits = sess.run(char_splits)
logits = random_ops.random_uniform(char_splits.shape + (2, ),
minval=-6,
maxval=6,
dtype=dtypes.float32)
if not context.executing_eagerly():
# Evaluate logits computation to exclude these steps from op benchmarking
with session.Session() as sess:
logits = sess.run(logits)
tokenizer = text_ops.SplitMergeFromLogitsTokenizer()
self._run(tokenizer, {"logits": logits})
import import_ipynb
import tensorflow as tf
import pandas as pd
from tensorflow.python.framework import random_seed
from sklearn.preprocessing import StandardScaler
random_seed.set_seed(42)
class DataTest(tf.test.TestCase):
def setUp(self):
import Exercise4_04
super(DataTest, self).setUp()
self.exercise = Exercise4_04
self.df = pd.read_csv('../Datasets/qsar_androgen_receptor.csv', sep=';')
self.df.dropna(inplace=True)
def testTargetData(self):
target = self.df['positive'].apply(lambda x: 1 if x=='positive' else 0)
output = self.exercise.target
expected_output = target
self.assertAllEqual(expected_output, output)
def testFeatureData(self):
features = self.df.drop('positive', axis=1)
output = self.exercise.features
expected_output = features
self.assertAllEqual(expected_output, output)
# Libraries are loaded and the data is reshaped.
def create_xy(data):
'helper function to create x, c and y with proper shapes'
x = data.filter(like='x-', axis=1).values[:, :, np.newaxis]
c = data[top_cities[1:]].to_numpy()
y = data.Amsterdam.values[:, np.newaxis]
return x, c, y
# create correct shapes for tensorflow
x_train, c_train, y_train = create_xy(train)
x_test, c_test, y_test = create_xy(test)
# deterministic
set_seed(random_state)
# As before, I start out by a pure autoregressive model.
model = Sequential(layers=[GRU(cells), Dense(units=1, activation='linear')])
model.compile(optimizer='adam', loss='mae')
history = model.fit(x=x_train, y=y_train, epochs=epochs, batch_size=None,
shuffle=True,
validation_split=validation_split)
# The final test loss is;
def inverseAms(data):
return (ct.named_transformers_['Amsterdam']
.inverse_transform(data)
def train_k_lip_model(layer_type: type, layer_params: dict, batch_size: int,
steps_per_epoch: int, epochs: int, input_shape: tuple,
k_lip_model: float, k_lip_data: float, **kwargs):
"""
Create a generator, create a model, train it and return the results.
Args:
layer_type:
layer_params:
batch_size:
steps_per_epoch:
epochs:
input_shape:
k_lip_model:
k_lip_data:
**kwargs:
Returns:
the generator
"""
# clear session to avoid side effects from previous train
K.clear_session()
np.random.seed(42)
# create the keras model, defin opt, and compile it
model = generate_k_lip_model(layer_type, layer_params, input_shape,
k_lip_model)
optimizer = Adam(lr=0.001)
model.compile(optimizer=optimizer,
loss="mean_squared_error",
metrics=[metrics.mse])
# model.summary()
# create the synthetic data generator
output_shape = model.compute_output_shape((batch_size, ) + input_shape)[1:]
kernel = build_kernel(input_shape, output_shape, k_lip_data)
# define logging features
logdir = os.path.join("logs", "lip_layers", "%s" % layer_type.__name__)
hparams = dict(
layer_type=layer_type.__name__,
batch_size=batch_size,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
k_lip_data=k_lip_data,
k_lip_model=k_lip_model,
)
callback_list = [
callbacks.TensorBoard(logdir),
hp.KerasCallback(logdir, hparams)
]
if kwargs["callbacks"] is not None:
callback_list = callback_list + kwargs["callbacks"]
# train model
model.__getattribute__(FIT)(
linear_generator(batch_size, input_shape, kernel),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=0,
callbacks=callback_list,
)
# the seed is set to compare all models with the same data
x, y = linear_generator(batch_size, input_shape, kernel).send(None)
np.random.seed(42)
set_seed(42)
loss, mse = model.__getattribute__(EVALUATE)(
linear_generator(batch_size, input_shape, kernel),
steps=10,
)
empirical_lip_const = evaluate_lip_const(model=model, x=x, seed=42)
# save the model
model_checkpoint_path = os.path.join(logdir, "model.h5")
model.save(model_checkpoint_path, overwrite=True, save_format="h5")
del model
K.clear_session()
model = load_model(model_checkpoint_path)
np.random.seed(42)
set_seed(42)
from_disk_loss, from_disk_mse = model.__getattribute__(EVALUATE)(
linear_generator(batch_size, input_shape, kernel),
steps=10,
)
from_empirical_lip_const = evaluate_lip_const(model=model, x=x, seed=42)
# log metrics
file_writer = tf.summary.create_file_writer(os.path.join(
logdir, "metrics"))
file_writer.set_as_default()
tf.summary.scalar("lip_coef_estim", empirical_lip_const, step=epochs)
tf.summary.scalar("evaluation_mse", mse, step=epochs)
tf.summary.scalar("disk_load_evaluation_mse", from_disk_mse, step=epochs)
tf.summary.scalar("disk_load_lip_coef_estim",
from_empirical_lip_const,
step=epochs)
return (
mse,
empirical_lip_const.numpy(),
from_disk_mse,
from_empirical_lip_const.numpy(),
)
from tensorflow.python.framework import random_seed import numpy as np from abc import ABC as AbstractClass import languagemodels.bert from models.encoder import EncoderForFrozenLanguageModel from models.encoder import EncoderForLanguageModelTuning from models.components import VideoEncoder, TextEncoder from models.layers import GatedEmbeddingUnitReasoning, GatedEmbeddingModule from models.layers import TemporalAggregationLayer, NetVLAD from metrics.loss import bidirectional_max_margin_ranking_loss random_seed.set_seed(1) BATCH_SIZE = 2 FEATURE_SIZE = 15 OUTPUT_DIM = 10 NUM_EXPERTS = 3 EXPERT_ONE_SHAPE = (BATCH_SIZE, 20) EXPERT_TWO_SHAPE = (BATCH_SIZE, 5) EXPERT_THREE_SHAPE = (BATCH_SIZE, 3, 4) EXPERT_NETVLAD_CLUSTERS = 5 EXPERT_AGGREGATED_SIZE = OUTPUT_DIM MOCK_TEXT_EMBEDDING_SHAPE = (BATCH_SIZE, 5, 20) MOCK_MARGIN_PARAMETER = 0.05 MLP_LAYERS = 2
def test_consistent_random_seed_in_assert_all_equal(self):
random_seed.set_seed(1066)
index = random_ops.random_shuffle([0, 1, 2, 3, 4], seed=2021)
# This failed when `a` and `b` were evaluated in separate sessions.
self.assertAllEqual(index, index)
class Predictioner:
numpy.random.seed(1234)
set_seed(1234)
def __init__(self):
self.model = Sequential()
self.setup_default_model()
self.compile_model()
def save_model(self, path):
self.model.save(path)
def load_model(self, path):
self.model = keras.models.load_model(path)
def update_input(self, train_x, train_y):
self.push_train_sets(train_x, train_y)
self.y_scaler = MinMaxScaler()
self.x_scaler = MinMaxScaler()
self.reshape_train_sets()
self.adjust_scalers()
def push_train_sets(self, train_x, train_y):
self.train_x = train_x
self.train_y = train_y
def reshape_train_sets(self):
self.train_x = reshaper(self.train_x, self.train_x.shape[1])
self.train_y = reshaper(self.train_y, 1)
def adjust_scalers(self):
self.train_x = self.x_scaler.fit_transform(self.train_x)
self.train_y = self.y_scaler.fit_transform(self.train_y)
def setup_default_model(self):
self.model.add(Dense(30))
self.model.add(Dense(90, activation='relu'))
self.model.add(Dense(45, activation='relu'))
self.model.add(Dense(20, activation='relu'))
self.model.add(Dense(10, activation='relu'))
self.model.add(Dense(1))
def compile_model(self):
self.model.compile(
optimizer=keras.optimizers.Adam(),
loss=keras.losses.mean_squared_error,
metrics=[
keras.metrics.mean_squared_error,
keras.metrics.mean_squared_logarithmic_error,
keras.metrics.mean_absolute_percentage_error,
keras.metrics.mean_absolute_error,
]
)
def fit_model(self, verbose=0):
self.model.fit(
self.train_x, # [:int(len(self.train_x) * 0.66)],
self.train_y, # [:int(len(self.train_y) * 0.66)],
epochs=300,
batch_size=10,
verbose=verbose,
# validation_data=(self.train_y[int(len(self.train_x) * 0.66):],
# self.train_x[int(len(self.train_x) * 0.66):])
)
def evaluate(self, x_test, y_test):
return self.model.evaluate(x_test, y_test, batch_size=12, verbose=1)
def predict(self, prediction_interval_x):
prediction_interval_x = self.x_scaler.transform(prediction_interval_x)
predicted_y = self.model.predict(prediction_interval_x)
self.x_plot = self.x_scaler.inverse_transform(self.train_x)
self.y_plot = self.y_scaler.inverse_transform(self.train_y)
self.x_pred_plot = self.x_scaler.inverse_transform(prediction_interval_x)
self.y_pred_plot = self.y_scaler.inverse_transform(predicted_y)
return self.y_pred_plot
def visualize(self):
pyplot.scatter(self.x_pred_plot, self.y_pred_plot, label='Predicted')
pyplot.scatter(self.x_plot, self.y_plot, label='Actual')
pyplot.title('Input (x) versus Output (y)')
pyplot.xlabel('Input Variable (x)')
pyplot.ylabel('Output Variable (y)')
pyplot.legend()
pyplot.show()