def train():
# 读取训练数据和label数据
print("loading data.......")
imgs_train = get_data("F:\liuyang\MyNet\\U-net\\train_data.npy",(512,512))
imgs_label = get_data("F:\liuyang\MyNet\\U-net\\train_label.npy",(512,512))
# get Unet model
print("getting model...")
model = Unet(512,512,optimizer=Adam(1e-4),loss=dice_coef_loss,metrics=dice_coef)
# 保存权值
checkPoint = ModelCheckpoint('weighs.h5',monitor='val_loss',save_best_only=True)
# 开始训练
print("start training.......")
model.fit(imgs_train,imgs_label,batch_size=1,epochs=10,verbose=1, shuffle=True,callbacks=[checkPoint],validation_split=0.1)
def __init__(self) -> None:
""" Get data from helper -> the source data """
list_of_dataframes = get_data()
self.df_confirmed = list_of_dataframes['confirmed']
self.df_deaths = list_of_dataframes['deaths']
self.df_recovered = list_of_dataframes['recovered']
list_of_time_series = get_data(time_series=True)
self.df_time_series_confirmed = list_of_time_series['confirmed']
self.df_time_series_deaths = list_of_time_series['deaths']
self.df_time_series_recovered = list_of_time_series['recovered']
self.datetime_raw = self.df_confirmed['datetime'].unique().tolist()[0]
self.timestamp = datetime.strptime(self.datetime_raw, '%m/%d/%y').timestamp()
def test_model_from_keras(
subject,
batch_size=32,
folder='/scratch/sem19h24/EEGNet_reformat/EEGnet/initialModels/'):
"""
Import the keras model and test it
Parameters:
subject: Number between 1 and 9
batch_size: Batch Size to test the model
folder: Folder where the model files are found.
returns: loss, accuracy
"""
# Just use the default values and hope all dimensions are fine...
model = EEGNet(activation='elu', constrain_w=False, permuted_flatten=True)
model.load_model_params_from_keras(f"{folder}model{subject-1}_torch.npz")
if t.cuda.is_available():
model = model.cuda()
print_summary(model, None, None, None)
# get dataloader
test_samples, test_labels = get_data(subject, training=False)
test_loader = as_data_loader(test_samples,
test_labels,
batch_size=batch_size)
# prepare loss function, no optimizer necessary
loss_function = t.nn.CrossEntropyLoss()
return _test_net(model, test_loader, loss_function, train=False)
def get_all_book(url_book: list, rows: list):
"""[From a list of book urls, it finds all the information and adds them to rows]
Args:
url_book (list): [A list of book urls]
rows (list) : [A list of data:
product_page_url,
upc,
title,
price_including_tax,
price_excluding_tax,
number_available,
product_description,
category,
reviews_rating,
image_url,
filename]
"""
# loop from book url
for url in url_book:
soup = get_data(url)
# instence class BookFetcher
book_info = BookFetcher(url, soup)
book = book_info.get_book_info
# write each result to rows
rows.append(book)
def srap_books(url: str, rows: list):
"""[From a book category url, call url_book (), get_all_book () and get_next_page ()]
Args:
url (str): [url of a book category]
rows (list) : [A list of data:
product_page_url,
upc,
title,
price_including_tax,
price_excluding_tax,
number_available,
product_description,
category,
reviews_rating,
image_url,
filename]
"""
url_book = get_all_page(url)
get_all_book(url_book, rows)
# loop to get the next pages
while True:
soup = get_data(url)
url = get_next_page(soup, url)
if not url:
break
url_book = get_all_page(url)
get_all_book(url_book, rows)
del url_book[:]
def predict():
# get model
model = Unet(512,
512,
optimizer=Adam(1e-4),
loss=dice_coef_loss,
metrics=dice_coef)
# get test images
print("getting test images...")
testImagesPath = 'E:\ljs\\u-net\\train_data.npy'
imgs_test = get_data(testImagesPath, (512, 512))
# load weighs
print("loading weighs...")
model.load_weights('weighs.h5')
# predict
print("predicting ...")
imgs_predict = model.predict(imgs_test, batch_size=1, verbose=1)
# 把输出结果保存到一个文件夹里
print('saving output images...')
dir = get_randstr()
os.mkdir(dir)
for id, image in enumerate(imgs_predict):
image = (image[:, :, 0] * 255.).astype(np.uint8)
# plt 得到的结果默认是彩色存储
#plt.imsave(os.path.join(dir,str(id)+'.png'),image)
# skimage存灰度
imsave(os.path.join(dir, str(id) + '.png'), image)
def get_url_category(url: str) -> list:
"""[Find all urls for each category of books]
Args:
url_index (str): [url of the main book.toscrap page]
Returns:
list: [a list with the urls of each category]
"""
category_url = []
# request get
soup = get_data(url)
lis = soup.find('ul', class_='nav-list').find_all('li')
# loop over results
for li_ in lis:
category_link = li_.find('a')['href']
category_link = 'http://books.toscrape.com/' + \
category_link
category_url.append(category_link)
return category_url
def get_url_book(url: str) -> list:
"""[Find all book urls on a page]
Args:
url (str): [url of a book category page]
Returns:
list: [Returns a list with all the urls of the books on a page]
"""
book_url = []
# request get
soup = get_data(url)
articles = soup.find_all('article', class_='product_pod')
for article in articles:
link = article.find('a')['href']
# remove unwanted characters
link = 'http://books.toscrape.com/catalogue/' + \
link.strip('../../..')
book_url.append(link)
return book_url
def train_subject_specific_quant(subject,
epochs=500,
batch_size=32,
lr=0.001,
silent=False,
plot=True,
**kwargs):
"""
Trains a subject specific model for the given subject
Parameters:
- subject: Integer in the Range 1 <= subject <= 9
- epochs: Number of epochs to train
- batch_size: Batch Size
- lr: Learning Rate
- silent: bool, if True, hide all output including the progress bar
- plot: bool, if True, generate plots
- kwargs: Remaining arguments passed to the EEGnet model
Returns: (model, metrics)
- model: t.nn.Module, trained model
- metrics: t.tensor, size=[1, 4], accuracy, precision, recall, f1
"""
# load the data
train_samples, train_labels = get_data(subject, training=True)
test_samples, test_labels = get_data(subject, training=False)
train_loader = as_data_loader(train_samples,
train_labels,
batch_size=batch_size)
# test_loader = as_data_loader(test_samples, test_labels, batch_size=test_labels.shape[0])
test_loader = as_data_loader(test_samples,
test_labels,
batch_size=batch_size)
# prepare quantization configuration
qconfig = tq.QConfig(
activation=tq.MinMaxObserver.with_args(dtype=t.quint8),
weight=tq.MinMaxObserver.with_args(dtype=t.qint8))
# prepare the model
model = EEGNetQuant(T=train_samples.shape[2], qconfig=qconfig, **kwargs)
model.initialize_params()
if t.cuda.is_available():
model = model.cuda()
# prepare the quantization
tq.prepare_qat(model, inplace=True)
# prepare loss function and optimizer
loss_function = t.nn.CrossEntropyLoss()
optimizer = t.optim.Adam(model.parameters(), lr=lr, eps=1e-7)
scheduler = None
# print the training setup
print_summary(model, optimizer, loss_function, scheduler)
# prepare progress bar
with tqdm(desc=f"Subject {subject}",
total=epochs,
leave=False,
disable=silent,
unit='epoch',
ascii=True) as pbar:
# Early stopping is not allowed in this mode, because the testing data cannot be used for
# training!
model, metrics, _, history = _train_net(subject,
model,
train_loader,
test_loader,
loss_function,
optimizer,
scheduler=scheduler,
epochs=epochs,
early_stopping=False,
plot=plot,
pbar=pbar)
# convert the model into a quantized model
model = model.cpu()
tq.convert(model, inplace=True)
metrics = get_metrics_from_model(model, test_loader)
if not silent:
print(f"Subject {subject}: accuracy = {metrics[0, 0]}")
return model, metrics, history
def train_subject_specific_cv(subject,
n_splits=4,
epochs=500,
batch_size=32,
lr=0.001,
early_stopping=True,
silent=False,
plot=True,
**kwargs):
"""
Trains a subject specific model for the given subject, using K-Fold Cross Validation
Parameters:
- subject: Integer in the Range 1 <= subject <= 9
- n_splits: Number of splits for K-Fold CV
- epochs: Number of epochs to train
- batch_size: Batch Size
- lr: Learning Rate
- early_stopping: bool, approximate the number of epochs to train the network for the subject.
- silent: bool, if True, generate no output, including progress bar
- plot: bool, if True, generates plots
- kwargs: remaining parameters are passed to the EEGnet model
Returns: (models, metrics, epoch)
- models: List of t.nn.Module, size = [n_splits]
- metrics: t.tensor, size=[1, 4], accuracy, precision, recall, f1, averaged over all splits
- epoch: integer, number of epochs determined by early_stopping. If early_stopping is not
used, then this value will always be equal to the parameter epochs
Notes:
- Early Stopping: Uses early stopping to determine the best epoch to stop training for the
given subject by averaging over the stopping epoch of all splits.
"""
# load the raw data
samples, labels = get_data(subject, training=True)
samples, labels = as_tensor(samples, labels)
# prepare the models
models = [EEGNet(T=samples.shape[2], **kwargs) for _ in range(n_splits)]
metrics = t.zeros((n_splits, 4))
best_epoch = np.zeros(n_splits)
# prepare KFold
kfcv = KFoldCV(n_splits)
split = 0
# open the progress bar
with tqdm(desc=f"Subject {subject} CV, split 1",
total=n_splits * epochs,
leave=False,
unit='epoch',
ascii=True,
disable=silent) as pbar:
# loop over all splits
for split, indices in enumerate(kfcv.split(samples, labels)):
# set the progress bar title
pbar.set_description(f"Subject {subject} CV, split {split + 1}")
# generate dataset
train_idx, val_idx = indices
train_ds = t.utils.data.TensorDataset(samples[train_idx],
labels[train_idx])
val_ds = t.utils.data.TensorDataset(samples[val_idx],
labels[val_idx])
train_loader = t.utils.data.DataLoader(train_ds,
batch_size=batch_size,
shuffle=True)
val_loader = t.utils.data.DataLoader(val_ds,
batch_size=batch_size,
shuffle=True)
# prepare the model
model = models[split]
model.initialize_params()
if t.cuda.is_available():
model = model.cuda()
# prepare loss function and optimizer
loss_function = t.nn.CrossEntropyLoss()
optimizer = t.optim.Adam(model.parameters(), lr=lr)
model, split_metrics, split_epoch, _ = _train_net(
subject,
model,
train_loader,
val_loader,
loss_function,
optimizer,
epochs=epochs,
early_stopping=early_stopping,
plot=plot,
pbar=pbar)
metrics[split, :] = split_metrics[0, :]
best_epoch[split] = split_epoch
# average all metrics
metrics = metrics.mean(axis=0).reshape(1, 4)
# print the result
if not silent:
print(
f"Subje`ct {subject} CV: accuracy = {metrics[0, 0]}, at epoch {best_epoch.mean()} "
+ f"+- {best_epoch.std()}")
return models, metrics, int(best_epoch.mean().round())
def train_model(model, epoches=50, batch=32):
seq_len = 128
df, X_train, y_train, X_val, y_val, X_test, y_test, train_data_len, val_data_len, scaler = get_data(
)
CP = ModelCheckpoint(filepath='Transformer+TimeEmbedding.hdf5',
monitor='val_loss',
save_best_only=True,
verbose=1)
ES = EarlyStopping(monitor='val_loss', mode='min', patience=10)
history = model.fit(X_train,
y_train,
batch_size=batch,
epochs=epoches,
callbacks=[CP, ES],
verbose=1,
validation_data=(X_val, y_val))
model = load_model('Transformer+TimeEmbedding.hdf5',
custom_objects={
'Time2Vector': Time2Vector,
'SingleAttention': SingleAttention,
'MultiAttention': MultiAttention,
'TransformerEncoder': TransformerEncoder
})
#Print evaluation metrics for all datasets
train_eval = model.evaluate(X_train, y_train, verbose=0)
val_eval = model.evaluate(X_val, y_val, verbose=0)
test_eval = model.evaluate(X_test, y_test, verbose=0)
print('\nEvaluation metrics')
print('Training Data - Loss: {:.4f}, MAE: {:.4f}'.format(
train_eval[0], train_eval[1]))
print('Validation Data - Loss: {:.4f}, MAE: {:.4f}'.format(
val_eval[0], val_eval[1]))
print('Test Data - Loss: {:.4f}, MAE: {:.4f}'.format(
test_eval[0], test_eval[1]))
# Visualize Loss Vs. epochs
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(len(loss))
plt.figure()
plt.plot(epochs, loss, "b", label="Training loss")
plt.plot(epochs, val_loss, "r", label="Validation loss")
plt.title("Training and Validation Loss")
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.legend()
plt.show()
