def single_image_transform_model(cimage, simage):
sdim, cdim = simage.shape, cimage.shape
fcmodel = extract_features(cdim)
fcmodel.trainable = False
if cdim == sdim: fsmodel = fcmodel
else:
fsmodel = extract_features(sdim)
fsmodel.trainable = False
cimage = cimage.reshape(
(1, cimage.shape[0], cimage.shape[1], cimage.shape[2]))
cvals = fcmodel.predict(cimage)[0][0]
simage = simage.reshape(
(1, simage.shape[0], simage.shape[1], simage.shape[2]))
svals = fsmodel.predict(simage)[1][0]
dummy = Input(shape=cdim, dtype="float32", name="dummy_img")
from keras import initializers
timg = RawWeights(name='result',
activation=None,
initializer=initializers.uniform(0, 255))(dummy)
tfeats = fcmodel(timg)
cerr = Lambda(lambda x: x - cvals, name="ContentError")(tfeats[0])
serr = Lambda(lambda x: x - svals, name="StyleError")(tfeats[1])
model = Model(inputs=[dummy], outputs=[timg, cerr, serr])
return model
def on_status(self, status):
"""
Storing streaming tweets in the SQL database.
The returned status object has all the information about a tweet.
"""
# Exclude all retweets.
if status.retweeted or 'RT @' in status.text:
return True
id_str = status.id_str
created_at = status.created_at
text = status.text
user_location = status.user.location
tweet = {"text": text, "label": None}
tweet_processor = PreProcessTweets()
processed_tweet = tweet_processor.process_tweets([tweet])
polarity = classifier.classify(extract_features(processed_tweet[0]))
print(polarity)
print(processed_tweet)
print(extract_features(processed_tweet[0]))
insert_sql = """
INSERT INTO {}
(id_str, created_at, text, polarity, user_location)
VALUES (%s, %s, %s, %s, %s)
""".format(settings.table_name)
val = (id_str, created_at, text, polarity, user_location)
with Connection() as db:
db.create_table_if_not_existed(settings.table_name)
db.execute(insert_sql, val)
def task():
year, months = 2019, [10, 11, 12]
select = None
radius, aperture_size, incident_interval, time_step = '100', '6', '25', '1'
config = {
'year': year,
'months': months,
'radius': radius,
'aperture_size': aperture_size,
'incident_interval': incident_interval,
'time_step': time_step,
'select': select
}
month = months[0]
df = pd.read_pickle(
f'output/waze/{year}_{month}_{radius}_{aperture_size}_features.pkl')
for month in months[1:]:
temp_df = pd.read_pickle(
f'output/waze/{year}_{month}_{radius}_{aperture_size}_features.pkl'
)
df = df.append(temp_df, sort=False)
df.fillna(0, inplace=True)
# if select is not None:
# df = df[-1 * int(select):]
if os.path.exists(
f'output/{year}_{month}_{radius}_{aperture_size}_predict_proba.pkl'
):
x = pd.read_pickle(
f'output/{year}_{month}_{radius}_{aperture_size}_predict_proba.pkl'
)
else:
x = model.predict_proba(df, int(incident_interval), time_step)
x = model.extract_features(x, df)
x.to_pickle(
f'output/{year}_{month}_{radius}_{aperture_size}_predict_proba.pkl'
)
incident_df = load_incidents(aperture_size)
if os.path.exists(f'output/{year}_{month}_{radius}_{aperture_size}_y.pkl'):
y = pd.read_pickle(
f'output/{year}_{month}_{radius}_{aperture_size}_y.pkl')
else:
y = label_mapper(x, incident_df)
pd.Series(y).to_pickle(
f'output/{year}_{month}_{radius}_{aperture_size}_y.pkl')
models = [
'LogisticRegression', 'DecisionTreeClassifier',
'RandomForestClassifier'
]
for m in models:
print('model', m)
cv_results, y_pred = model.cross_validate(x, y, m)
for k, v in config.items():
print(k, ',', v)
for k, v in cv_results.items():
print(k, ',', np.average(v))
print()
def make_training_sample():
"""make_training_sample() -> (np.array of shape (8, 8, 13), (win bool, draw bool, loss bool))
Returns features and 3-tuple of bools of what the outcome is from a randomly selected board position.
You must first set the global variable `tablebase` to use this!
For example: train.tablebase = chess.syzygy.open_tablebases("./syzygy/")
"""
board = make_random_position()
result = tablebase.probe_wdl(board)
features = model.extract_features(board)
desired_output = (result > 0, result == 0, result < 0)
return features, desired_output
def predict(fen):
"""predict(fen: str) -> (win prob: float, draw prob: float, loss prob: float)"""
board = chess.Board(fen)
features = model.extract_features(board)
logits, = sess.run(net.flow,
feed_dict={
net.input_ph: [features],
net.is_training_ph: False,
})
assert logits.shape == (3, )
win, draw, loss = softmax(logits)
return win, draw, loss
def generic_data_pipeline(source_data,
target_data,
padded_length=default_padded_length):
source_data, target_data = model.extract_features(
source_data), model.extract_features(target_data)
source_data, target_data = model.pad_features(
source_data,
padded_length), model.pad_features(target_data, padded_length)
source_data, target_data = model.align(source_data, target_data)
source_data, target_data = source_data[:, 1:], target_data[:, 1:]
source_data, target_data = model.pad_features(
source_data,
padded_length), model.pad_features(target_data, padded_length)
source_data, target_data = model.apply_delta(
source_data), model.apply_delta(target_data)
return source_data, target_data
def optimize_style_model(tmodel, cimage,simage):
sdim, cdim = simage.shape, cimage.shape
fcmodel = extract_features(cdim)
fcmodel.trainable = False
if cdim==sdim: fsmodel = fcmodel
else:
fsmodel = extract_features(sdim)
fsmodel.trainable = False
cimage = cimage.reshape((1, cimage.shape[0], cimage.shape[1], cimage.shape[2]))
cvals = fcmodel.predict(cimage)[0][0]
simage = simage.reshape((1, simage.shape[0], simage.shape[1], simage.shape[2]))
svals = fsmodel.predict(simage)[1][0]
cinp = Input(shape=cimage.shape[1:],dtype="float32",name="content_img")
style_shape = tmodel.input_shape[1][1:]
dummy = Input(shape=style_shape,dtype="float32",name="dummy_style")
from keras import initializers
style = RawWeights(name='style',activation=None,
initializer=initializers.uniform(-0.01,0.01))(dummy)
timg = tmodel([cinp,style])
res = Lambda(lambda x:x,name="result")(timg)
tfeats = fcmodel(timg)
cerr = Lambda(lambda x: x-cvals, name="ContentError")(tfeats[0])
serr = Lambda(lambda x: x-svals, name="StyleError")(tfeats[1])
model = Model(inputs=[cinp,dummy],outputs=[res,cerr,serr])
return model
def cnn_model_fn(features, labels, mode):
# Make predictions
logits = extract_features(features, CLASSES, IMAGE_SIZE, mode)
predicted_classes = tf.argmax(input=logits, axis=1)
# Predict the model
if mode == tf.estimator.ModeKeys.PREDICT:
outputs = {
'class_ids': predicted_classes[:, tf.newaxis],
"predicted_label": predicted_classes,
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
return tf.estimator.EstimatorSpec(mode=mode, predictions=outputs)
# If not in mode.PREDICT, compute the loss
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
# Train the model
if mode == tf.estimator.ModeKeys.TRAIN:
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)
train_op = optimizer.minimize(
loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
# Eval the model
# Compute evaluation metrics.
accuracy = tf.metrics.accuracy(labels=labels,
predictions=predicted_classes,
name='acc_op')
metrics = {'accuracy': accuracy}
tf.summary.scalar('accuracy', accuracy[1])
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
eval_metric_ops=metrics)
elif base_model == 'inception_resnet':
_, x, _, is_train, features = model.inception_resnet_model(input_shape, label_shape)
feature_size = 1536
else:
AssertionError('Unknown base model present')
sess = tf.Session()
# Loading trained model checkpoint
saver = utils.load_checkpoint(os.path.join(model_dir,'model'), sess, 'model')
saver_unloaded = utils.load_checkpoint(os.path.join(model_dir,'model_unsaved'), sess, 'model_unsaved')
# Extracting feature dictionary for the provided dataset
data_pl = [x, is_train]
feature_dict = model.extract_features(sess, test_iter, test_data, data_pl, features)
print(f'size of feature dict : {len(feature_dict)}')
print('Feature_extraction Done')
# Saving feature dictionary into pickle file
with open(os.path.join(DATA_DIR, 'train_features.pickle'), 'wb') as handle:
pickle.dump(feature_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
# Loading metadata which will be later used to create
# temporal sequences
if base_data == 'train':
df = pd.read_csv(os.path.join(DATA_DIR, 'train_metadata.csv'), sep=',')
df_labels = pd.read_csv(os.path.join(DATA_DIR, 'reshaped_stage_2_train.csv'), sep=',')
df = pd.merge(df, df_labels, left_on='SOPInstanceUID', right_on='ImgID')
def analyze(source_path, target_path):
click.secho('Loading Audio Files: {} {} 🔍 '.format(source_path,
target_path),
fg='blue')
# Iterative Strategy
source_dataset, target_dataset = [], []
source_data, target_data = model.extract_features(
model.load_audio(source_path)), model.extract_features(
model.load_audio(target_path))
shorter = min([source_data.shape[0], target_data.shape[0]])
pad_length = max([source_data.shape[0], target_data.shape[0]]) + 200
for index in tqdm(range(int(shorter / default_padded_length))):
current_source_data = source_data[index * default_padded_length:index *
default_padded_length +
default_padded_length, :]
current_target_data = target_data[index * default_padded_length:index *
default_padded_length +
default_padded_length, :]
current_source_data, current_target_data = model.align(
current_source_data, current_target_data)
current_source_data, current_target_data = model.pad_features(
current_source_data,
pad_length=pad_length), model.pad_features(current_target_data,
pad_length=pad_length)
current_source_data, current_target_data = current_source_data[:,
1:], current_target_data[:,
1:]
current_source_data, current_target_data = model.apply_delta(
current_source_data), model.apply_delta(current_target_data)
source_dataset.append(current_source_data), target_dataset.append(
current_target_data)
source_dataset, target_dataset = numpy.asarray(
source_dataset), numpy.asarray(target_dataset)
joint_distribution = utilities.math.remove_zeros_frames(
model.get_joint_matrix(source_dataset, target_dataset))
# Convolution Strategy
# source_dataset, target_dataset = [], []
# source_data, target_data = model.extract_features( model.load_audio(source_path) ), model.extract_features(model.load_audio(target_path))
# source_data, target_data = model.align(source_data, target_data)
# shorter = min([ source_data.shape[0], target_data.shape[0] ])
# pad_length = max([ source_data.shape[0], target_data.shape[0] ]) + 200
# for index in tqdm( range( int( (shorter - default_padded_length) / default_skip_frames) - 1 ) ):
# current_source_data = source_data[index + (index * default_skip_frames):index + default_padded_length + (index * default_skip_frames), :]
# current_target_data = target_data[index + (index * default_skip_frames):index + default_padded_length + (index * default_skip_frames), :]
# current_source_data, current_target_data = model.pad_features(current_source_data, pad_length=pad_length), model.pad_features(current_target_data, pad_length=pad_length)
# current_source_data, current_target_data = current_source_data[:, 1:], current_target_data[:, 1:]
# current_source_data, current_target_data = model.apply_delta(current_source_data), model.apply_delta(current_target_data)
# source_dataset.append(current_source_data), target_dataset.append(current_target_data)
# source_dataset, target_dataset = numpy.asarray(source_dataset), numpy.asarray(target_dataset)
# joint_distribution = utilities.math.remove_zeros_frames( model.get_joint_matrix(source_dataset, target_dataset) )
# Single Process Strategy
# source_data = model.load_audio(source_path)
# target_data = model.load_audio(target_path)
# padded_length = max([ source_data.shape[0], target_data.shape[0] ]) + default_auto_pad_length
# source_data, target_data = generic_data_pipeline(source_data, target_data, padded_length)
# joint_distribution = utilities.math.remove_zeros_frames( model.get_joint_matrix(source_data, target_data) )
gaussian_mixture_model = model.create_model()
click.secho('Training Gaussian Mixture Model 🎓 ', fg='blue')
gaussian_mixture_model.fit(joint_distribution)
start_name = utilities.filesystem.extension(os.path.basename(source_path),
'')
end_name = utilities.filesystem.extension(os.path.basename(target_path),
'')
save_model_as(gaussian_mixture_model,
'{}-{}.pkl'.format(start_name, end_name))
click.secho('Training Finished on Gaussian Mixture Model ({}-{}) ✓'.format(
start_name, end_name),
fg='green')
