def test(model: nn.Module, test_loader: DataLoader, loss_function, device, len_rotation=24) -> Dict:
model.eval()
test_loss = 0
outputs = []
targets = []
with torch.no_grad():
for idx, (x, y) in enumerate(test_loader):
x = torch.Tensor(np.vstack(x)).to(device)
y = torch.Tensor(y).to(device)
output = model(x)
test_loss += loss_function(output.view(-1), y.view(-1)).item()
outputs.append(output.cpu().numpy().reshape(-1))
targets.append(y.cpu().numpy().reshape(-1))
targets = np.concatenate(targets).reshape(-1)
outputs = np.concatenate(outputs).reshape(-1)
test_loss /= len(test_loader.dataset)*len_rotation
evaluation = {
'test_loss': test_loss,
'c_index': metrics.c_index(targets, outputs),
'RMSE': metrics.RMSE(targets, outputs),
'MAE': metrics.MAE(targets, outputs),
'SD': metrics.SD(targets, outputs),
'CORR': metrics.CORR(targets, outputs),
}
return evaluation
def test(model: nn.Module, test_loader, loss_function, device, show):
model.eval()
test_loss = 0
outputs = []
targets = []
with torch.no_grad():
for idx, (*x, y) in tqdm(enumerate(test_loader), disable=not show, total=len(test_loader)):
for i in range(len(x)):
x[i] = x[i].to(device)
y = y.to(device)
y_hat = model(*x)
test_loss += loss_function(y_hat.view(-1), y.view(-1)).item()
outputs.append(y_hat.cpu().numpy().reshape(-1))
targets.append(y.cpu().numpy().reshape(-1))
targets = np.concatenate(targets).reshape(-1)
outputs = np.concatenate(outputs).reshape(-1)
test_loss /= len(test_loader.dataset)
evaluation = {
'loss': test_loss,
'c_index': metrics.c_index(targets, outputs),
'RMSE': metrics.RMSE(targets, outputs),
'MAE': metrics.MAE(targets, outputs),
'SD': metrics.SD(targets, outputs),
'CORR': metrics.CORR(targets, outputs),
}
return evaluation
def cluster_evaluation(mask):
metric_list = []
name = mask[0:6]
sample_list = [1000, 3000, 5000, 7000, 9000, 11000, 14000]
for i in tqdm(sample_list):
try:
xtrain, xval, _, ytrain, yval, _ = dp.areal_model(length=i,
mask=mask)
m = gpm.multi_gp(xtrain, xval, ytrain, yval)
training_R2 = me.R2(m, xtrain, ytrain)
training_RMSE = me.RMSE(m, xtrain, ytrain)
val_R2 = me.R2(m, xval, yval)
val_RMSE = me.RMSE(m, xval, yval)
metric_list.append(
[i, training_R2, training_RMSE, val_R2, val_RMSE])
except Exception:
print(i + 100)
xtrain, xval, _, ytrain, yval, _ = dp.areal_model(length=i + 100,
mask=mask)
m = gpm.multi_gp(xtrain, xval, ytrain, yval)
training_R2 = me.R2(m, xtrain, ytrain)
training_RMSE = me.RMSE(m, xtrain, ytrain)
val_R2 = me.R2(m, xval, yval)
val_RMSE = me.RMSE(m, xval, yval)
metric_list.append(
[i, training_R2, training_RMSE, val_R2, val_RMSE])
df = pd.DataFrame(
metric_list,
columns=[
"samples", "training_R2", "training_RMSE", "val_R2", "val_RMSE"
],
)
df.to_csv(name + "-eval-2020-07-22.csv")
def multi_gp(xtrain, xval, ytrain, yval, save=False):
""" Returns simple GP model """
# model construction
k1 = gpflow.kernels.Periodic(
gpflow.kernels.RBF(lengthscales=1, variance=1, active_dims=[0]))
k1b = gpflow.kernels.RBF(lengthscales=2, variance=1, active_dims=[0])
k2 = gpflow.kernels.RBF(lengthscales=np.ones(len(xval[0]) - 1),
active_dims=np.arange(1, len(xval[0])))
# k3 = gpflow.kernels.White()
k = k1 * k1b + k2 # +k
# mean_function = gpflow.mean_functions.Linear(A=np.ones((len(xtrain[0]),
# 1)), b=[1])
# , mean_function=mean_function)
m = gpflow.models.GPR(data=(xtrain, ytrain.reshape(-1, 1)), kernel=k)
opt = gpflow.optimizers.Scipy()
# , options=dict(maxiter=1000)
opt.minimize(m.training_loss, m.trainable_variables)
# print_summary(m)
x_plot = np.concatenate((xtrain, xval))
y_gpr, y_std = m.predict_y(x_plot)
print(
" {0:.3f} | {1:.3f} | {2:.3f} | {3:.3f} | {4:.3f} | {5:.3f} |".format(
me.R2(m, xtrain, ytrain),
me.RMSE(m, xtrain, ytrain),
me.R2(m, xval, yval),
me.RMSE(m, xval, yval),
np.mean(y_gpr),
np.mean(y_std),
))
if save is not False:
filepath = save_model(m, xval, save)
print(filepath)
return m
def perform_analysis():
"""
:return:
"""
parser = parse_arg()
args = parser.parse_args()
# if len(sys.argv) == 1: # no arguments, so print help message
# print("""Usage: python script.py data_path program_input out_path""")
# return
#
# dir_in = os.getcwd()
# dir_out = os.getcwd()
#
# try:
# dir_in = sys.argv[1]
# dir_out = sys.argv[2]
# except:
# print("Parameters: path/to/simple/file input/folder output/folder")
# sys.exit(0)
#df = pd.read_csv(args.dir_in)
df = pd.read_csv(args.input)
(cal_df, tst_df) = separating_data_set(df)
(x_train, y_train) = splitting_dataset(cal_df)
(x_test, y_test) = splitting_dataset(tst_df)
print(x_train)
print(y_train)
print(x_test)
print(y_test)
model = build_fnn(x_train)
model.summary()
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)
random.seed(1)
model.fit(x_train, y_train, batch_size=args.batchSize, epochs=args.epochSize, validation_data=(x_test, y_test), callbacks=[early_stop])
fnn_losses = pd.DataFrame(model.history.history)
create_losses_plot(fnn_losses)
predictions = model.predict(x_test)
print("MSE:", metrics.MSE(y_test, predictions))
print("RMSE:", metrics.RMSE(y_test, predictions))
print("R-square:", metrics.R2(y_test, predictions))
print("RPD:", metrics.RPD(y_test, predictions))
create_prediction_plot(y_test, predictions)
def uib_evaluation(average=False):
metric_list = []
sample_list = [1000, 3000, 5000, 7000, 9000, 11000, 14000]
for i in tqdm(sample_list):
try:
xtrain, xval, _, ytrain, yval, _ = dp.areal_model(
length=i, EDA_average=average)
m = gpm.multi_gp(xtrain, xval, ytrain, yval)
training_R2 = me.R2(m, xtrain, ytrain)
training_RMSE = me.RMSE(m, xtrain, ytrain)
val_R2 = me.R2(m, xval, yval)
val_RMSE = me.RMSE(m, xval, yval)
metric_list.append(
[i, training_R2, training_RMSE, val_R2, val_RMSE])
except Exception:
print(i + 100)
xtrain, xval, _, ytrain, yval, _ = dp.areal_model(
length=i + 100, EDA_average=average)
m = gpm.multi_gp(xtrain, xval, ytrain, yval)
training_R2 = me.R2(m, xtrain, ytrain)
training_RMSE = me.RMSE(m, xtrain, ytrain)
val_R2 = me.R2(m, xval, yval)
val_RMSE = me.RMSE(m, xval, yval)
metric_list.append(
[i, training_R2, training_RMSE, val_R2, val_RMSE])
df = pd.DataFrame(
metric_list,
columns=[
"samples", "training_R2", "training_RMSE", "val_R2", "val_RMSE"
],
)
df.to_csv("uib-eval-2020-07-22.csv")
def hybrid_gp(xtrain, xval, ytrain, yval, save=False):
""" Returns whole basin or cluster GP model with hybrid kernel """
dimensions = len(xtrain[0])
k1 = gpflow.kernels.RBF(lengthscales=np.ones(dimensions),
active_dims=np.arange(0, dimensions))
k2 = gpflow.kernels.RBF(lengthscales=np.ones(dimensions),
active_dims=np.arange(0, dimensions))
alpha1 = hybrid_kernel(dimensions, 1)
alpha2 = hybrid_kernel(dimensions, 2)
k = alpha1 * k1 + alpha2 * k2
m = gpflow.models.GPR(data=(xtrain, ytrain.reshape(-1, 1)), kernel=k)
opt = gpflow.optimizers.Scipy()
opt.minimize(m.training_loss, m.trainable_variables)
# print_summary(m)
x_plot = np.concatenate((xtrain, xval))
y_gpr, y_std = m.predict_y(x_plot)
print(
" {0:.3f} | {1:.3f} | {2:.3f} | {3:.3f} | {4:.3f} | {5:.3f} |".format(
me.R2(m, xtrain, ytrain),
me.RMSE(m, xtrain, ytrain),
me.R2(m, xval, yval),
me.RMSE(m, xval, yval),
np.mean(y_gpr),
np.mean(y_std),
))
if save is True:
filepath = save_model(m, xval, "")
print(filepath)
return m
def predict(sess, XX, lagXX, YY, mYY, aYY, stepLen=params.decoder_length):
y_size = len(YY[0])
for yy in YY:
assert len(yy) == y_size
YY_pred = [[] for _ in range(len(YY))]
SS_pred = [[] for _ in range(len(YY))]
xItr = BatchGenerator(XX, batchSeqLen=params.sequence_length)
lagXItr = BatchGenerator(lagXX, batchSeqLen=params.sequence_length)
outputItr = BatchGenerator(YY, batchSeqLen=params.sequence_length)
currItr = xItr.iterFinished
total_loss = 0
while currItr == xItr.iterFinished:
b_x1, _, _, _, _, _ = xItr.nextBatch(batchSize=params.batch_size,
stepLen=stepLen)
b_lagx1, _, _, _, _, _ = lagXItr.nextBatch(
batchSize=params.batch_size, stepLen=stepLen)
b_y1, tsIndices_, _, _, _, mask = outputItr.nextBatch(
batchSize=params.batch_size, stepLen=stepLen)
#b_mYY = [mYY[tsInd] for tsInd in tsIndices_]
dictionary = {
inputPh: b_x1,
lagPh: b_lagx1,
outputPh: b_y1,
keep_prob: 1.00,
mode: 0.0,
tsIndicesPh: tsIndices_,
maskPh: mask
}
if params.is_probabilistic_pred:
loss_ret, pred_ret, sigma_ret = session.run(
[loss, pred, model_obj.sigma], feed_dict=dictionary)
else:
loss_ret, pred_ret = session.run([loss, pred],
feed_dict=dictionary)
sigma_ret = -1.0 * np.ones_like(pred_ret) # invalid sigma
total_loss += loss_ret
y_pred, sigma_pred = pred_ret, sigma_ret
for i, tsInd in enumerate(tsIndices_):
if mask[i] > 0:
YY_pred[tsInd] += y_pred[i].tolist()
SS_pred[tsInd] += sigma_pred[i].tolist()
YY_gt = np.squeeze(np.array(YY)[:, params.encoder_length:, :], axis=-1)
for yy, yy_pred in zip(YY_gt, YY_pred):
assert len(yy) == len(yy_pred)
YY_pred = np.array(YY_pred)
SS_pred = np.array(SS_pred)
assert YY_gt.shape[0] == YY_pred.shape[0]
assert YY_gt.shape[1] == YY_pred.shape[1]
if params.isNormalised:
YY_gt = YY_gt * np.array(mYY)
YY_pred = YY_pred * np.array(mYY)
if params.deep_ar_normalize_seq:
YY_gt = YY_gt * np.array(aYY)
YY_pred = YY_pred * np.array(aYY)
if params.isLogNormalised:
YY_gt = np.exp(YY_gt)
YY_pred = np.exp(YY_pred)
error = np.square(YY_gt - YY_pred)
test_len = YY_gt.shape[1]
score = [
metrics.RMSE(YY_gt, YY_pred),
metrics.RMSE(YY_gt[:, :1], YY_pred[:, :1]),
metrics.RMSE(YY_gt[:, :int(test_len / 2.0)],
YY_pred[:, :int(test_len / 2.0)]),
metrics.RMSE(YY_gt[:, int(test_len / 2.0):],
YY_pred[:, int(test_len / 2.0):])
]
print([
metrics.ND(YY_gt, YY_pred),
metrics.ND(YY_gt[:, :1], YY_pred[:, :1]),
metrics.ND(YY_gt[:, :int(test_len / 2.0)],
YY_pred[:, :int(test_len / 2.0)]),
metrics.ND(YY_gt[:, int(test_len / 2.0):],
YY_pred[:, int(test_len / 2.0):])
])
return total_loss, score, YY_gt, YY_pred, SS_pred
def perform_analysis():
"""
:return:
"""
parser = parse_arg()
args = parser.parse_args()
df = pd.read_csv(args.input)
# df = df.apply(lambda x: preProcessing.scaling_y_data(x) if x.name == 'OC' else x) # scaling OC data
(cal_df, tst_df) = separating_data_set(df)
(X_train, y_train) = splitting_dataset(cal_df)
(X_test, y_test) = splitting_dataset(tst_df)
# Scale the features
X_train = preProcessing.scaler_min_max_x_data(X_train)
X_test = preProcessing.scaler_min_max_x_data(X_test)
y_train = preProcessing.scaler_min_max_y_data(y_train)
y_test = preProcessing.scaler_min_max_y_data(y_test)
print(X_train)
print(y_train)
print(X_test)
print(y_test)
print(X_train.shape)
print(y_train.shape)
if args.hiddenLayers == 5:
model = build_fnn_5l(X_train)
elif args.hiddenLayers == 4:
model = build_fnn_4l(X_train)
elif args.hiddenLayers == 3:
model = build_fnn_3l(X_train)
elif args.hiddenLayers == 2:
model = build_fnn_2l(X_train)
else:
model = build_fnn_1l(X_train)
model.summary()
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)
random.seed(1)
model.fit(X_train, y_train, batch_size=args.batchSize, epochs=args.epochSize, validation_data=(X_test, y_test),
callbacks=[early_stop])
fnn_losses = pd.DataFrame(model.history.history)
create_losses_plot(fnn_losses)
trained_model_save(model, "trained_model.h5")
predictions = model.predict(X_test)
print("MSE:", metrics.MSE(y_test, predictions))
print("RMSE:", metrics.RMSE(y_test, predictions))
print("R-square:", metrics.R2(y_test, predictions))
print("RPD:", metrics.RPD(y_test, predictions))
create_prediction_plot(y_test, predictions)
def regression_report(y_true, y_pred):
print('--------------------------------')
print('MSE -', metrics.MSE(y_true, y_pred))
print('RMSE -', metrics.RMSE(y_true, y_pred))
print('MAN -', metrics.MAN(y_true, y_pred))
print('--------------------------------')
def runExperiment(experiment: experimentInfo, metricsResFileName,
clearMetricsFile):
dsPathsList_Test = experiment.dsList
outFileName = experiment.outName
test_building = experiment.building
meter_key = experiment.meter_key
pathOrigDS = experiment.pathOrigDS
meterTH = experiment.meterTH
print('House ', test_building)
# Load a "complete" dataset to have the test's timerange
test = DataSet(dsPathsList_Test[0])
test_elec = test.buildings[test_building].elec
testRef_meter = test_elec.submeters(
)[meter_key] # will be used as reference to align all meters based on this
# Align every test meter with testRef_meter as master
test_series_list = []
for path in dsPathsList_Test:
test = DataSet(path)
test_elec = test.buildings[test_building].elec
test_meter = test_elec.submeters()[meter_key]
# print('Stack test: ', test_meter.get_timeframe().start.date(), " - ", test_meter.get_timeframe().end.date())
aligned_meters = align_two_meters(testRef_meter, test_meter)
test_series_list.append(aligned_meters)
# Init vars for the output
MIN_CHUNK_LENGTH = 300 # Depends on the basemodels of the ensemble
timeframes = []
building_path = '/building{}'.format(test_meter.building())
mains_data_location = building_path + '/elec/meter1'
data_is_available = False
disag_filename = outFileName
output_datastore = HDFDataStore(disag_filename, 'w')
run = True
chunkDataForOutput = None
# -- Used to hold necessary data for saving the results using NILMTK (e.g. timeframes).
# -- (in case where chunks have different size (not in current implementation), must use the chunk whose windowsSize is the least (to have all the data))
while run:
try:
testX = []
columnInd = 0
# Get Next chunk of each series
for testXGen in test_series_list:
chunkALL = next(testXGen)
chunk = chunkALL[
'slave'] # slave is the meter needed (master is only for aligning)
chunk.fillna(0, inplace=True)
if (columnInd == 0):
chunkDataForOutput = chunk # Use 1st found chunk for it's metadata
if (testX == []):
testX = np.zeros(
[len(chunk), len(test_series_list)]
) # Initialize the array that will hold all of the series as columns
testX[:, columnInd] = chunk[:]
columnInd += 1
testX = scaler.transform(testX)
except:
run = False
break
if len(chunkDataForOutput) < MIN_CHUNK_LENGTH:
continue
# print("New sensible chunk: {}".format(len(chunk)))
startTime = chunkDataForOutput.index[0]
endTime = chunkDataForOutput.index[
-1] # chunkDataForOutput.shape[0] - 1
# print('Start:',startTime,'End:',endTime)
timeframes.append(TimeFrame(
startTime, endTime)) #info needed for output for use with NILMTK
measurement = ('power', 'active')
pred = clf.predict(testX)
column = pd.Series(pred, index=chunkDataForOutput.index, name=0)
appliance_powers_dict = {}
appliance_powers_dict[0] = column
appliance_power = pd.DataFrame(appliance_powers_dict)
appliance_power[appliance_power < 0] = 0
# Append prediction to output
data_is_available = True
cols = pd.MultiIndex.from_tuples([measurement])
meter_instance = test_meter.instance()
df = pd.DataFrame(appliance_power.values,
index=appliance_power.index,
columns=cols,
dtype="float32")
key = '{}/elec/meter{}'.format(building_path, meter_instance)
output_datastore.append(key, df)
# Append aggregate data to output
mains_df = pd.DataFrame(chunkDataForOutput,
columns=cols,
dtype="float32")
# Note (For later): not 100% right. Should be mains. But it won't be used anywhere, so it doesn't matter in this case
output_datastore.append(key=mains_data_location, value=mains_df)
# Save metadata to output
if data_is_available:
disagr = Disaggregator()
disagr.MODEL_NAME = 'Stacked model'
disagr._save_metadata_for_disaggregation(
output_datastore=output_datastore,
sample_period=sample_period,
measurement=measurement,
timeframes=timeframes,
building=test_meter.building(),
meters=[test_meter])
#======================== Calculate Metrics =====================================
testYDS = DataSet(pathOrigDS)
testYDS.set_window(start=test_meter.get_timeframe().start.date(),
end=test_meter.get_timeframe().end.date())
testY_elec = testYDS.buildings[test_building].elec
testY_meter = testY_elec.submeters()[meter_key]
test_mains = testY_elec.mains()
result = DataSet(disag_filename)
res_elec = result.buildings[test_building].elec
rpaf = metrics.recall_precision_accuracy_f1(res_elec[meter_key],
testY_meter, meterTH, meterTH)
relError = metrics.relative_error_total_energy(res_elec[meter_key],
testY_meter)
MAE = metrics.mean_absolute_error(res_elec[meter_key], testY_meter)
RMSE = metrics.RMSE(res_elec[meter_key], testY_meter)
print("============ Recall: {}".format(rpaf[0]))
print("============ Precision: {}".format(rpaf[1]))
print("============ Accuracy: {}".format(rpaf[2]))
print("============ F1 Score: {}".format(rpaf[3]))
print("============ Relative error in total energy: {}".format(relError))
print("============ Mean absolute error(in Watts): {}".format(MAE))
print("=== For docs: {:.4}\t{:.4}\t{:.4}\t{:.4}\t{:.4}\t{:.4}".format(
rpaf[0], rpaf[1], rpaf[2], rpaf[3], relError, MAE))
# print("============ RMSE: {}".format(RMSE))
# print("============ TECA: {}".format(metrics.TECA([res_elec[meter_key]],[testY_meter],test_mains)))
resDict = {
'model': 'TEST',
'building': test_building,
'Appliance': meter_key,
'Appliance_Type': 2,
'Recall': rpaf[0],
'Precision': rpaf[1],
'Accuracy': rpaf[2],
'F1': rpaf[3],
'relError': relError,
'MAE': MAE,
'RMSE': RMSE
}
metrics.writeResultsToCSV(resDict, metricsResFileName, clearMetricsFile)
if __name__ == "__main__":
data = pd.read_csv(FILENAME_READ)
data = data.drop('Unnamed: 0', axis=1)
columns = data.drop('Target', axis=1).columns
weights, y_trains, X_trains, y_tests, X_tests = lr.cross_validation(data)
data_write = pd.DataFrame(
columns=["", "T1", "T2", "T3", "T4", "T5", "E", "STD"])
for i in range(5):
y_pred_test = lr.predict(X_tests[i], weights[i])
y_pred_train = lr.predict(X_trains[i], weights[i])
y_test = np.array(y_tests[i])
y_train = np.array(y_trains[i])
r2_test = m.R2(y_test, y_pred_test)
r2_train = m.R2(y_train, y_pred_train)
rmse_test = m.RMSE(y_test, y_pred_test)
rmse_train = m.RMSE(y_train, y_pred_train)
data_write["T" + str(i + 1)] = [
r2_test, r2_train, rmse_test, rmse_train
] + list(weights[i].reshape(weights[i].shape[0], 1))
data_write["E"] = data_write[["T1", "T2", "T3", "T4", "T5"]].mean(axis=1)
data_write["STD"] = data_write[["T1", "T2", "T3", "T4", "T5"]].std(axis=1)
data_write.index = ["R^2_test", "R^2_train", "RMSE_test", "RMSE_train"
] + list(columns) + list(["1"])
data_write.to_csv("result.csv")
def single_loc_evaluation(location, perf_plot=False, hpar_plot=False):
metric_list = []
coord_list = sa.random_location_generator(location)
n = len(coord_list)
for i in tqdm(range(n)):
try:
xtrain, xval, _, ytrain, yval, _ = dp.point_model(
coords=list(coord_list[i]))
m = gpm.multi_gp(xtrain, xval, ytrain, yval)
training_R2 = me.R2(m, xtrain, ytrain)
training_RMSE = me.RMSE(m, xtrain, ytrain)
val_R2 = me.R2(m, xval, yval)
val_RMSE = me.RMSE(m, xval, yval)
time_kernel_lengthscale = float(
m.kernel.kernels[0].base_kernel.variance.value())
time_kernel_variance = float(
m.kernel.kernels[0].base_kernel.lengthscales.value())
time_kernel_periodicity = float(m.kernel.kernels[0].period.value())
N34_lengthscale = np.array(
m.kernel.kernels[1].lengthscales.value())[2]
d2m_lengthscale = np.array(
m.kernel.kernels[1].lengthscales.value())[0]
tcwv_lengthscale = np.array(
m.kernel.kernels[1].lengthscales.value())[1]
rbf_kernel_variance = float(m.kernel.kernels[1].variance.value())
metric_list.append([
coord_list[i, 0], coord_list[i, 1], training_R2, training_RMSE,
val_R2, val_RMSE, time_kernel_lengthscale,
time_kernel_variance, time_kernel_periodicity, N34_lengthscale,
d2m_lengthscale, tcwv_lengthscale, rbf_kernel_variance
])
except Exception:
pass
df = pd.DataFrame(
metric_list,
columns=[
"latitude", "longitude", "training_R2", "training_RMSE", "val_R2",
"val_RMSE", "time_kernel_lengthscale", "time_kernel_variance",
"time_kernel_periodicity", "N34_lengthscale", "d2m_lengthscale",
"tcwv_lengthscale", "rbf_kernel_variance"
],
)
now = datetime.datetime.now()
df.to_csv("_Data/single-locations-eval-" + now.strftime("%Y-%m-%d") +
".csv")
print(df.mean(axis=0))
df_prep = df.set_index(["latitude", "longitude"])
da = df_prep.to_xarray()
if perf_plot is True:
slm_perf_plots(da)
if hpar_plot is True:
slm_hpar_plots(da)
def run(config):
os.environ['DGLBACKEND'] = 'pytorch'
if config['model']['gpu'] != 'any':
os.environ['CUDA_VISIBLE_DEVICES'] = config['model']['gpu']
torch.backends.cudnn.benchmark = True
if not os.path.exists(config['model']['model_path']):
os.makedirs(config['model']['model_path'])
# ----------------------------------Prepare Dataset----------------------------------
print('Prepare Dataset')
sys.stdout.flush()
feature_map = {}
node_map = None
user_bought = None
if 'bipartite' in config['dataset'] and config['dataset']['bipartite']:
node_map = {}
user_bought = {}
num_features = 0
field_size = 0
train_dataset, valid_dataset, test_dataset = None, None, None
if config['dataset']['decoder'] == 'libfm':
for file in config['dataset']['paths'].values():
field_size = dataset.LibFMDataset.read_features(file, feature_map, field_size, node_map)
num_features = len(feature_map)
print("number of features:", num_features)
train_dataset = dataset.LibFMDataset(config['dataset']['paths']['train'], feature_map, node_map, user_bought)
if 'valid' in config['dataset']['paths']:
valid_dataset = dataset.LibFMDataset(config['dataset']['paths']['valid'], feature_map)
test_dataset = dataset.LibFMDataset(config['dataset']['paths']['test'], feature_map)
train_loader = DataLoader(train_dataset, drop_last=True,
batch_size=config['dataset']['batch_size'], shuffle=True,
num_workers=config['dataset']['num_workers'])
valid_loader = DataLoader(valid_dataset, batch_size=config['dataset']['batch_size'], shuffle=False, num_workers=0)
test_loader = DataLoader(test_dataset, batch_size=config['dataset']['batch_size'], shuffle=False, num_workers=0)
# ----------------------------------Load or Create Model----------------------------------
model = models.load_model(config['model']) # load
if model is None: # create
model = models.create_model(config['model'], num_features, field_size, node_map, user_bought)
assert model is not None
model.cuda()
# ----------------------------------Construct Optimizer----------------------------------
print('Construct Optimizer')
sys.stdout.flush()
optimizer = None
if config['model']['optimizer'] == 'Adagrad':
optimizer = optim.Adagrad(model.parameters(), lr=config['model']['learning_rate'],
initial_accumulator_value=1e-8)
elif config['model']['optimizer'] == 'Adam':
optimizer = optim.Adam(model.parameters(), lr=config['model']['learning_rate'])
elif config['model']['optimizer'] == 'SGD':
optimizer = optim.SGD(model.parameters(), lr=config['model']['learning_rate'])
elif config['model']['optimizer'] == 'Momentum':
optimizer = optim.SGD(model.parameters(), lr=config['model']['learning_rate'], momentum=0.95)
# ----------------------------------Construct Loss Function----------------------------------
print('Construct Loss Function')
sys.stdout.flush()
criterion = None
if config['model']['loss_type'] == 'square_loss':
criterion = nn.MSELoss(reduction='sum')
elif config['model']['loss_type'] == 'log_loss': # log_loss
criterion = nn.BCEWithLogitsLoss(reduction='sum')
# ---------Model Initial Check---------
if config['model']['evaluation']:
model.eval()
print("Model Initial Check: ", end='')
if config['task'] == 'rating':
train_result = metrics.RMSE(model, train_loader)
test_result = metrics.RMSE(model, test_loader)
if valid_dataset is not None:
valid_result = metrics.RMSE(model, valid_loader)
print("Train_RMSE: {:.3f},".format(train_result),
"Valid_RMSE: {:.3f}, Test_RMSE: {:.3f}".format(valid_result, test_result))
else:
print("Train_RMSE: {:.3f},".format(train_result), "Test_RMSE: {:.3f}".format(test_result))
elif config['task'] == 'ranking':
test_hr, test_ndcg = metrics.metrics(model, test_loader)
test_result = test_hr
print("Test_HR: {:.3f}, Test_NDCG: {:.3f}".format(test_hr, test_ndcg))
sys.stdout.flush()
# ----------------------------------Training----------------------------------
print('Training...')
if config['task'] == 'rating':
best_result = 100
elif config['task'] == 'ranking':
best_result = (0, 0)
else:
best_result = None
saved = False
start_time = time.time()
for epoch in range(config['model']['epochs']):
model.train()
loss = 0 # No effect, ignore this line
for i, (features, feature_values, label) in enumerate(train_loader):
features = features.cuda()
feature_values = feature_values.cuda()
label = label.cuda()
if config['model']['loss_type'] == 'log_loss':
label = label.clamp(min=0., max=1.)
model.zero_grad()
prediction = model(features, feature_values)
loss = criterion(prediction, label)
# ---------l2 regularization---------
if model.l2 is not None:
loss += model.l2_regularization()
loss.backward()
optimizer.step()
# ---------checkpoint---------
if i % config['model']['steps_per_checkpoint'] == 0:
print(
"Running Epoch {:03d}/{:03d}".format(epoch + 1, config['model']['epochs']),
"loss:{:.3f}".format(float(loss)),
"costs:", time.strftime("%H: %M: %S", time.gmtime(time.time() - start_time)))
start_time = time.time()
sys.stdout.flush()
# ----------------------------------Validation----------------------------------
if config['model']['evaluation']:
model.eval()
save_flag = False
if config['task'] == 'rating':
# train_result = metrics.RMSE(model, train_loader)
test_result = metrics.RMSE(model, test_loader)
if valid_dataset is not None:
# valid_result = metrics.RMSE(model, valid_loader)
print("\tRunning Epoch {:03d}/{:03d}".format(epoch + 1, config['model']['epochs']),
# "Train_RMSE: {:.3f},".format(train_result),
# "Valid_RMSE: {:.3f},".format(valid_result),
"Test_RMSE: {:.3f}".format(test_result))
else:
print("\tRunning Epoch {:03d}/{:03d}".format(epoch + 1, config['model']['epochs']),
# "Train_RMSE: {:.3f},".format(train_result),
"Test_RMSE: {:.3f}".format(test_result))
save_flag = test_result < best_result
elif config['task'] == 'ranking':
test_hr, test_ndcg = metrics.metrics(model, test_loader)
test_result = (test_hr, test_ndcg)
print("\tRunning Epoch {:03d}/{:03d}".format(epoch + 1, config['model']['epochs']),
"Test_HR: {:.3f}, Test_NDCG: {:.3f}".format(test_hr, test_ndcg))
save_flag = test_hr > best_result[0]
else:
test_result = best_result # No effect, ignore this line
if save_flag:
if config['model']['save']:
if 'tag' in config:
torch.save(model, os.path.join(config['model']['model_path'],
'{}_{}.pth'.format(config['model']['name'], config['tag'])))
saved = True
best_result = test_result
sys.stdout.flush()
# ----------------------------------Evaluation----------------------------------
if config['model']['evaluation']:
print('Evaluating...')
model.eval()
flag = False
if config['task'] == 'rating':
train_result = metrics.RMSE(model, train_loader)
test_result = metrics.RMSE(model, test_loader)
if valid_dataset is not None:
valid_result = metrics.RMSE(model, valid_loader)
print("Train_RMSE: {:.3f}, Valid_RMSE: {:.3f}, Test_RMSE: {:.3f}".format(train_result, valid_result,
test_result))
else:
print("Train_RMSE: {:.3f}, Test_RMSE: {:.3f}".format(train_result, test_result))
flag = test_result < best_result
elif config['task'] == 'ranking':
train_result = metrics.RMSE(model, train_loader)
test_hr, test_ndcg = metrics.metrics(model, test_loader)
test_result = (test_hr, test_ndcg)
print("Train_RMSE: {:.3f}, Test_HR: {:.3f}, Test_NDCG: {:.3f}".format(train_result, test_hr, test_ndcg))
flag = test_hr > best_result[0]
else:
test_result = best_result # No effect, ignore this line
if flag:
best_result = test_result
print('------Best Result: ', best_result, '------', sep='')
sys.stdout.flush()
if not saved and config['model']['save']:
if 'tag' in config:
torch.save(model, os.path.join(config['model']['model_path'],
'{}_{}.pth'.format(config['model']['name'], config['tag'])))
