def load_model(self, filepath):
debug('[Model] Loading model from file %s' % filepath)
self.model = load(filepath)
self.session = tf.compat.v1.get_default_session()
self.model._make_predict_function()
self.graph = tf.compat.v1.get_default_graph()
self.graph.finalize()
def process_sents(model_path, sents, formal_path, informal_path,
max_seq_length):
formal = []
informal = []
sents_count = len(sents)
percent = 0.1
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)
tokenizer.fit_on_texts(sentences)
from keras.models import load_model as load
model = load(model_path)
print("Processing %s sents" % str(sents_count))
for i, sent in enumerate(sents):
sequences = tokenizer.texts_to_sequences([sent])
data = pad_sequences(sequences, maxlen=max_seq_length)
pred = model.predict(data, 1)
if pred > 0.9:
formal.append(sent)
elif pred < 0.1:
informal.append(sent)
if i / sents_count > percent:
print("Processed %d of sents ..." % int(100 * percent))
percent += 0.1
if formal:
with open(formal_path, 'w') as f:
f.writelines('\n'.join(formal))
if informal:
with open(informal_path, 'w') as f:
f.writelines('\n'.join(informal))
def fetch(name):
"""
Fetches an appropriate model to perform the prediction.
:param name: model's name
:return: a trained model
"""
K.clear_session()
try:
full_weights_path = path.join(path_prefix, *load_weights()[name])
if name == 'svm':
return SVMModel(joblib.load(full_weights_path))
elif name == 'cnn':
return CNNModel(load(full_weights_path))
elif name == 'mlp':
return MLPModel(load(full_weights_path))
except KeyError:
raise ModelNotFoundError(f'Model named {name} does not exist.')
def get_model(model_path, data_path, max_seq_length, max_num_words):
if not os.path.exists(model_path):
model = define_model(max_seq_length, max_num_words)
x_train, y_train, x_val, y_val = get_data(data_path, max_num_words, max_seq_length)
model.fit(x_train, y_train,
batch_size=128,
epochs=10,
validation_data=(x_val, y_val))
model.save(model_path)
else:
model = load(model_path)
return model
def __init__(self,
input_len: int,
num_features: int = 1,
num_layers: int = 2,
dropout: float = None,
load_model: bool = True,
filename: str = None):
self.filename = filename
if load_model and filename is not None and isfile(filename):
self.model = load(filename)
self.was_loaded = True
else:
self.model = self.build_network(num_layers, input_len, num_features, dropout)
self.was_loaded = False
self.model.summary()
def load_model(model_name):
loaded_model_summary = joblib.load('models/' + model_name + '.txt')
embedding_dimension = loaded_model_summary['embedding_dimension']
dropout_parameter = loaded_model_summary['dropout_parameter']
bidirectional = loaded_model_summary['bidirectional']
maxPooling = loaded_model_summary['maxPooling']
averagePooling = loaded_model_summary['averagePooling']
rnn_type = loaded_model_summary['rnn_type']
rnn_units = loaded_model_summary['rnn_units']
model = Model(embedding_dimension=embedding_dimension, dropout_parameter=dropout_parameter,
bidirectional=bidirectional, rnn_type=rnn_type,
maxPooling=maxPooling, averagePooling=averagePooling, rnn_units=rnn_units,
name=model_name)
model.model = load('models/' + model_name + '.h5')
model.accuracy = loaded_model_summary['accuracy']
model.precision = loaded_model_summary['precision']
model.recall = loaded_model_summary['recall']
model.f1 = loaded_model_summary['f1']
return model
#plt.imshow(fake_img_batch[0,])
return
GAN_mod, G, D, f = GAN.Define_GAN([28, 28, 1], 1, np.infty)
print('Importing target model')
def fn(correct, predicted):
return tf.nn.softmax_cross_entropy_with_logits(labels=correct,
logits=predicted)
target = load('./ModelC', custom_objects={'fn': fn})
print('Loading MNIST data')
(X, y), (X_test, y_test) = mnist.load_data()
X = np.divide(X, 255)
X = X.reshape(X.shape[0], 28, 28, 1)
X_test = np.divide(X_test, 255)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
y = to_categorical(y, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
print('Training')
train_for_n(batches=20000, batch_size=128, distillation='dynamic')
print('Saving models')
GAN_mod.save_weights(system_path + 'ModelC_AdvGAN_dynamicdist.hdf5')
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Utilise un modèle de machine learning pour concevoir un fonction is_affiliation(string)
"""
from keras.models import load_model as load
import pickle
model_filepath = "telecom_overtrained_model.h5"
tfidf_vect_filepath = "telecom_overtrained_model_tfidf.pickle"
model = load(model_filepath)
tfidf_vect = pickle.load(open(tfidf_vect_filepath, 'rb'))
def is_affiliation(string):
print(string)
return bool(
model.predict_classes(tfidf_vect.transform([string]).toarray())[0][0])
def lstm(df, data, months, var):
# if data=='iig_maitri' or data=='iig_bharati':
# df=pd.read_csv('datasets/'+data+'.csv', parse_dates=['obstime'], index_col='obstime')
# # print(df)
# elif data=='dcwis':
# df=pd.read_csv('datasets/'+data+'.csv', names=['obstime', 'tempr', 'ap', 'ws', 'rh', 'dew'], parse_dates=['obstime'], index_col='obstime')
data2['obstime'] = pd.to_datetime(data2['obstime'])
data2 = data2.set_index('obstime')
ds_temp = df[var].resample('M').mean()
if var == 'rh':
dsrh = df['rh']
dsrh = dsrh[dsrh > 10]
# plt.plot(dsrh)
ds_temp = dsrh.resample('M').mean()
if var == 'ws':
dsrh = df['ws']
dsrh = dsrh[dsrh >= 0]
# plt.plot(dsrh)
ds_temp = dsrh.resample('M').mean()
mean = ds_temp.mean()
if var == 'tempr':
mean = 0
if var == 'ap':
dsrh = df['ap']
dsrh = dsrh[dsrh > -10]
# plt.plot(dsrh)
ds_temp = dsrh.resample('M').mean()
ds_temp = ds_temp - mean
# df.tempr.plot()
ds_temp2 = ds_temp.dropna().values
ds_temp3 = ds_temp.values
# print(ds_temp)
# print(ds_temp2)
# model=Sequential()
# model.add(LSTM(300,input_shape=(12, 1),return_sequences=True))
# model.add(LSTM(300))
# model.add(Dense(1))
# model.compile(loss='mean_squared_error',optimizer='adam')
ds_temp2 = np.reshape(ds_temp2, (ds_temp2.shape[0], 1, 1))
def create_dataset(dataset, look_back=1):
dataX, dataY = [], []
for i in range(len(dataset)):
end_ix = i + look_back
if end_ix > len(dataset) - 1:
break
seqx, seqy = dataset[i:end_ix], dataset[end_ix]
dataX.append(seqx)
dataY.append(seqy)
return np.array(dataX), np.array(dataY)
# look_back=1
trainX, trainY = create_dataset(ds_temp2, 12)
# print(trainX.shape)
X = np.array(trainX).reshape(int(trainX.shape[0]), 12, 1)
# print(X.shape)
Y = np.array(trainY)
Y = np.reshape(Y, (Y.shape[0], 1))
# print(Y.shape)
# model.fit(X, Y, epochs=20, verbose=1, batch_size=1)
model = load('keras_models/' + data + '_lstm_' + var + '.h5')
# print(ds_temp2.size)
temp_values = ds_temp2
preds = ds_temp3
# testX=ds_temp[29598:]
# for i in range(60):
# x_input=temp_values[-24*10:]
# x_input=x_input.reshape(10,24,1)
# yhat=model.predict(x_input, verbose=1)
# temp_values=np.append(temp_values,yhat)
# months=24
for i in range(months):
x_input = temp_values[-12:]
x_input = x_input.reshape(1, 12, 1)
# print(i,' out of ',1000,': ')
yhat = model.predict(x_input, verbose=0)
temp_values = np.append(temp_values, yhat[0])
preds = np.append(preds, yhat[0])
dates = []
for year in range(2012, 2016 + int(months / 6)):
for month in range(1, 13):
dates.append(dt.datetime(year=year, month=month, day=28))
# preds, conf_int = pipe.predict(n_periods=months, return_conf_int=True)
# plt.subplot(211)
plt.plot(dates[:preds.size], preds + mean)
# plt.subplot(212)
plt.plot(dates[:ds_temp3.size], ds_temp3 + mean)
# plt.xlim(38000,39000)
plt.xlim(
str(2012 + int(preds.size / 12) - 6) + '-11',
str(2012 + int(preds.size / 12) + 1) + '-2')
# create the decoder model; "<": encoded(small) representation to big image
decoder = Model(encoded_input, decoder)
return encoder, decoder
print("Getting data...")
(X_train, X_test, X_validate), (y_train, y_test, y_validate) = get_data()
print("Done")
#####################################################################
print("AUTOENCODER")
#####################################################################
ckpt_loc = os.path.join(CUR_DIR, "ckpts", "conv-aue.hdf5")
if (os.path.isfile(ckpt_loc)):
print("Loading Autoencoder from directory %s..." % ckpt_loc)
autoencoder = load(ckpt_loc)
encoder, decoder = get_codec_from_aue(autoencoder)
else:
print("Training Autoencoder...")
autoencoder, encoder, decoder = build_conv_aue()
earlyStopping = EarlyStopping(monitor='val_loss',
patience=10,
verbose=1,
mode='min',
min_delta=0.0005)
mcp_save = ModelCheckpoint(ckpt_loc,
save_best_only=True,
verbose=1,
monitor='val_loss',
mode='min')
def load_model():
"""Loads a pretrained model."""
model = load('./model/model.h5')
return model
