def run_model5_mp(queue, col, trn, vld, tst, cat,
store_weather, store_data_max, \
model_param=1):
if cat == 0:
Y_hat2 = np.zeros((len(trn) + len(vld) + len(tst), 1))
else:
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat = build_model5(nm_trn,
nm_vld,
nm_tst,
store_weather,
column=col,
alpha_train=model_param)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn,
vld,
tst,
Y_hat,
store_data_max,
column=col)
queue.put((trn, vld, tst, Y_hat2, col, cat))
def run_model4_mp(queue, col, trn, vld, tst, cat,
store_weather, store_data_max, \
model_param=1):
if cat == 0:
Y_hat2 = np.zeros((len(trn) + len(vld) + len(tst), 1))
else:
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
_, fmat = sim(nm_trn, nm_vld, nm_tst, store_weather)
Y_hat = np.zeros((len(nm_trn) + len(nm_vld) + len(nm_tst), 1))
X = fmat[:len(nm_trn)]
Y = nm_trn[col].values[:, np.newaxis]
clf = linear_model.Ridge(alpha=model_param)
clf.fit(X, Y)
Y_hat[:] = clf.predict(fmat)
Y_hat2 = denormalize_store_data(trn,
vld,
tst,
Y_hat,
store_data_max,
column=col)
queue.put((trn, vld, tst, Y_hat2, col, cat))
def run_model1(store_data_file, store_weather_file, test_data_file, model_param, only_validate=False):
"""
the model uses the square error to measure the difference between Y_hat and
Y, and uses similarity to regulate Y_hat, that is, if one day's sale can be
reconstructed by similar day's sale. The performance of the model in this
task is not particularly good.
"""
print "start here"
# write header to test result
with open('test_result.csv', 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# develop training and validation set
train, valid = develop_valid_set2(store_data, store_weather, valid_size=100)
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set2(train, valid, test, store_weather)
# run prediction on testing data of each category
for n, trn, vld, tst in target_set:
print "%d, train(%d), valid(%d), test(%d)" % (n, len(trn), len(vld), len(tst))
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
v_init = None
Y_hat2 = None
for i in range(1):
# run prediction on all validation and testing data set
Y_hat = build_model1(nm_trn, nm_vld, nm_tst, store_weather, \
valid_init=v_init, alpha_train=model_param)
# save the code in case the model has stacking effect
#v_init=Y_hat[len(trn):]
# denormalize the sale
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2)
print "error at %d is: train(%f), valid(%f)" % (i, e1, e2)
# write results to test result
write_submission(trn, vld, tst, Y_hat2, 'test_result.csv', 'valid_result')
def run_model5(store_data_file, store_weather_file, test_data_file, \
model_param=1, validate_only=False, eval_err=None):
print "---------------------start here---------------------"
test_result_file ='test_result.csv'
# write header to test result
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set3(store_data, test, store_weather, store_data_max, valid_pct=0)
# run prediction on testing data of each category
for col, trn, vld, tst in target_set:
print "%s, train(%d), valid(%d), test(%d), model_param(%f)" % (col, len(trn), len(vld), len(tst), model_param)
if len(tst)==0: continue
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat=build_model5(nm_trn, nm_vld, nm_tst, store_weather, column=col, alpha_train=model_param)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max, column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error at item(%s) is: train(%f), valid(%f)" % (col, e1, e2)
if eval_err is not None:
eval_err.add_result(e1, len(trn), e2, len(vld))
# write results to test result
if not validate_only:
write_submission(trn, vld, tst, Y_hat2, test_result_file, 'valid_result', column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
if eval_err is not None:
e1, e2=eval_err.get_result()
logging.info("model5(p=%f) error is: train(%f), valid(%f)" % (model_param, e1, e2))
print "model5(p=%f) error is: train(%f), valid(%f)" % (model_param, e1, e2)
def run_model4v1(store_data_file, store_weather_file, test_data_file, \
model_param=1, validate_only=False, eval_err=None):
"""
ridge regression with log error term
"""
print "---------------------start here---------------------"
test_result_file ='test_result.csv'
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
store_data_max = store_data.groupby(level=1).max()
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set3(store_data, test, store_weather, store_data_max, columns=set(['1']))
for col, trn, vld, tst in target_set:
print "item(%s), train(%d), valid(%d), test(%d), model_param(%f)" % (col, len(trn), len(vld), len(tst), model_param)
if len(tst)==0: continue
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat, fmat_wegith=build_model_log_ridge(nm_trn, nm_vld, nm_tst, store_weather,col, alpha=model_param)
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat[:,np.newaxis], store_data_max, column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error at item(%s) is: train(%f), valid(%f)" % (col, e1, e2)
if eval_err is not None:
eval_err.add_result(e1, len(trn), e2, len(vld))
# write results to test result
if not validate_only:
write_submission(trn, vld, tst, Y_hat2, test_result_file, 'valid_result', column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
if eval_err is not None:
e1, e2=eval_err.get_result()
logging.info("model4v1(p=%f) error is: train(%f), valid(%f)" % (model_param, e1, e2))
print "model4v1(p=%f) error is: train(%f), valid(%f)" % (model_param, e1, e2)
def run_model5_mp(queue, col, trn, vld, tst, cat,
store_weather, store_data_max, \
model_param=1):
if cat==0:
Y_hat2=np.zeros((len(trn)+len(vld)+len(tst), 1))
else:
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat=build_model5(nm_trn, nm_vld, nm_tst, store_weather, column=col, alpha_train=model_param)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max, column=col)
queue.put((trn, vld, tst, Y_hat2, col, cat))
def run_model3(store_data_file, store_weather_file, test_data_file, model_param=1, validate_only=False):
print "start here"
test_result_file ='test_result.csv'
# write header to test result
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# develop training and validation set
train, valid = develop_valid_set2(store_data, store_weather, valid_size=100)
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set3(train, valid, test, store_weather, store_data_max)
# run prediction on testing data of each category
for col, trn, vld, tst in target_set:
print "%s, train(%d), valid(%d), test(%d)" % (col, len(trn), len(vld), len(tst))
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat=build_model3(nm_trn, nm_vld, nm_tst, store_weather, column=col, alpha_train=model_param)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max, column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error is: train(%f), valid(%f)" % (e1, e2)
# write results to test result
write_submission(trn, vld, tst, Y_hat2, test_result_file, 'valid_result', column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
def helper_model1v1(df, offset, train, m, Y_hat, store_data_max, v_init=None):
# construct feature matrix with one row more than training data
m2=np.zeros((len(train)+1,m.shape[1]))
# copy training feature matrix
m2[:len(train)]=m[:len(train)]
for i in range(len(df)):
m2[-1]=m[offset+i]
def l(i):
g=np.dot(m2, m2[i])
d=np.sum(g)
g=-g
g[i]=d+g[i]
return g
df0=df.iloc[i:i+1]
Y_hat0 = build_model(train, None, df0, l, v_init)
Y_hat1 = denormalize_store_data(train, None, df0, Y_hat0, store_data_max)
Y_hat[offset+i]=Y_hat1[-1]
def run_model2(store_data_file, store_weather_file, test_data_file):
print "start here"
# write header to test result
with open('test_result.csv', 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# develop training and validation set
train, valid = develop_valid_set2(store_data, store_weather, valid_size=0)
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set(train, valid, test, store_weather)
# run prediction on testing data of each category
for n, trn, vld, tst in target_set:
print "%d, train(%d), valid(%d), test(%d)" % (n, len(trn), len(vld),
len(tst))
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat, theta = build_model2(nm_trn, nm_vld, nm_tst, store_weather)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2)
print "error is: train(%f), valid(%f)" % (e1, e2)
# write results to test result
write_submission(trn, vld, tst, Y_hat2, 'test_result.csv')
def run_model2(store_data_file, store_weather_file, test_data_file):
print "start here"
# write header to test result
with open('test_result.csv', 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# develop training and validation set
train, valid = develop_valid_set2(store_data, store_weather, valid_size=0)
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set(train, valid, test, store_weather)
# run prediction on testing data of each category
for n, trn, vld, tst in target_set:
print "%d, train(%d), valid(%d), test(%d)" % (n, len(trn), len(vld), len(tst))
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat, theta=build_model2(nm_trn, nm_vld, nm_tst, store_weather)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2)
print "error is: train(%f), valid(%f)" % (e1, e2)
# write results to test result
write_submission(trn, vld, tst, Y_hat2, 'test_result.csv')
def run_model4_mp(queue, col, trn, vld, tst, cat,
store_weather, store_data_max, \
model_param=1):
if cat==0:
Y_hat2=np.zeros((len(trn)+len(vld)+len(tst), 1))
else:
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
_,fmat = sim(nm_trn, nm_vld, nm_tst, store_weather)
Y_hat = np.zeros((len(nm_trn) + len(nm_vld) + len(nm_tst), 1))
X = fmat[:len(nm_trn)]
Y = nm_trn[col].values[:,np.newaxis]
clf = linear_model.Ridge(alpha=model_param)
clf.fit(X, Y)
Y_hat[:] = clf.predict(fmat)
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max, column=col)
queue.put((trn, vld, tst, Y_hat2, col, cat))
def helper_model1v1(df, offset, train, m, Y_hat, store_data_max, v_init=None):
# construct feature matrix with one row more than training data
m2 = np.zeros((len(train) + 1, m.shape[1]))
# copy training feature matrix
m2[:len(train)] = m[:len(train)]
for i in range(len(df)):
m2[-1] = m[offset + i]
def l(i):
g = np.dot(m2, m2[i])
d = np.sum(g)
g = -g
g[i] = d + g[i]
return g
df0 = df.iloc[i:i + 1]
Y_hat0 = build_model(train, None, df0, l, v_init)
Y_hat1 = denormalize_store_data(train, None, df0, Y_hat0,
store_data_max)
Y_hat[offset + i] = Y_hat1[-1]
def run_model5(store_data_file, store_weather_file, test_data_file, \
model_param=1, validate_only=False, eval_err=None):
print "---------------------start here---------------------"
test_result_file = 'test_result.csv'
# write header to test result
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set3(store_data,
test,
store_weather,
store_data_max,
valid_pct=0)
# run prediction on testing data of each category
for col, trn, vld, tst in target_set:
print "%s, train(%d), valid(%d), test(%d), model_param(%f)" % (
col, len(trn), len(vld), len(tst), model_param)
if len(tst) == 0: continue
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat = build_model5(nm_trn,
nm_vld,
nm_tst,
store_weather,
column=col,
alpha_train=model_param)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn,
vld,
tst,
Y_hat,
store_data_max,
column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error at item(%s) is: train(%f), valid(%f)" % (col, e1, e2)
if eval_err is not None:
eval_err.add_result(e1, len(trn), e2, len(vld))
# write results to test result
if not validate_only:
write_submission(trn,
vld,
tst,
Y_hat2,
test_result_file,
'valid_result',
column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
if eval_err is not None:
e1, e2 = eval_err.get_result()
logging.info("model5(p=%f) error is: train(%f), valid(%f)" %
(model_param, e1, e2))
print "model5(p=%f) error is: train(%f), valid(%f)" % (model_param, e1,
e2)
def run_model4v1(store_data_file, store_weather_file, test_data_file, \
model_param=1, validate_only=False, eval_err=None):
"""
ridge regression with log error term
"""
print "---------------------start here---------------------"
test_result_file = 'test_result.csv'
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
store_data_max = store_data.groupby(level=1).max()
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set3(store_data,
test,
store_weather,
store_data_max,
columns=set(['1']))
for col, trn, vld, tst in target_set:
print "item(%s), train(%d), valid(%d), test(%d), model_param(%f)" % (
col, len(trn), len(vld), len(tst), model_param)
if len(tst) == 0: continue
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat, fmat_wegith = build_model_log_ridge(nm_trn,
nm_vld,
nm_tst,
store_weather,
col,
alpha=model_param)
Y_hat2 = denormalize_store_data(trn,
vld,
tst,
Y_hat[:, np.newaxis],
store_data_max,
column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error at item(%s) is: train(%f), valid(%f)" % (col, e1, e2)
if eval_err is not None:
eval_err.add_result(e1, len(trn), e2, len(vld))
# write results to test result
if not validate_only:
write_submission(trn,
vld,
tst,
Y_hat2,
test_result_file,
'valid_result',
column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
if eval_err is not None:
e1, e2 = eval_err.get_result()
logging.info("model4v1(p=%f) error is: train(%f), valid(%f)" %
(model_param, e1, e2))
print "model4v1(p=%f) error is: train(%f), valid(%f)" % (model_param,
e1, e2)
def run_model4(store_data_file, store_weather_file, test_data_file, \
model_param=1, validate_only=False, eval_err=None):
"""
ridge regression
"""
print "---------------------start here---------------------"
test_result_file = 'test_result.csv'
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
store_data_max = store_data.groupby(level=1).max()
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set4(store_data, test, store_weather,
store_data_max)
for col, trn, vld, tst, cat in target_set:
print "item(%s), train(%d), valid(%d), test(%d), model_param(%0.2f), cat(%d)" % \
(col, len(trn), len(vld), len(tst), model_param, cat)
if len(tst) == 0: continue
if cat == 0:
Y_hat2 = np.zeros((len(trn) + len(vld) + len(tst), 1))
else:
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
_, fmat = sim(nm_trn, nm_vld, nm_tst, store_weather)
Y_hat = np.zeros((len(nm_trn) + len(nm_vld) + len(nm_tst), 1))
X = fmat[:len(nm_trn)]
Y = nm_trn[col].values[:, np.newaxis]
clf = linear_model.Ridge(alpha=model_param)
clf.fit(X, Y)
Y_hat[:] = clf.predict(fmat)
Y_hat2 = denormalize_store_data(trn,
vld,
tst,
Y_hat,
store_data_max,
column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error at item(%s) is: train(%f), valid(%f)" % (col, e1, e2)
if eval_err is not None:
eval_err.add_result(e1, len(trn), e2, len(vld))
# write results to test result
if not validate_only:
write_submission(trn,
vld,
tst,
Y_hat2,
test_result_file,
'valid_result',
column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
if eval_err is not None:
e1, e2 = eval_err.get_result()
logging.info("model4(p=%f) error is: train(%f), valid(%f)" %
(model_param, e1, e2))
print "model4(p=%f) error is: train(%f), valid(%f)" % (model_param, e1,
e2)
def run_model3(store_data_file,
store_weather_file,
test_data_file,
model_param=1,
validate_only=False):
print "start here"
test_result_file = 'test_result.csv'
# write header to test result
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# develop training and validation set
train, valid = develop_valid_set2(store_data,
store_weather,
valid_size=100)
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set3(train, valid, test, store_weather,
store_data_max)
# run prediction on testing data of each category
for col, trn, vld, tst in target_set:
print "%s, train(%d), valid(%d), test(%d)" % (col, len(trn), len(vld),
len(tst))
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
Y_hat = build_model3(nm_trn,
nm_vld,
nm_tst,
store_weather,
column=col,
alpha_train=model_param)
# denormalize the sale
Y_hat2 = denormalize_store_data(trn,
vld,
tst,
Y_hat,
store_data_max,
column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error is: train(%f), valid(%f)" % (e1, e2)
# write results to test result
write_submission(trn,
vld,
tst,
Y_hat2,
test_result_file,
'valid_result',
column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
def run_model1(store_data_file,
store_weather_file,
test_data_file,
model_param,
only_validate=False):
"""
the model uses the square error to measure the difference between Y_hat and
Y, and uses similarity to regulate Y_hat, that is, if one day's sale can be
reconstructed by similar day's sale. The performance of the model in this
task is not particularly good.
"""
print "start here"
# write header to test result
with open('test_result.csv', 'w') as f:
f.write('id,units\n')
f.close()
# load data
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
# compute max item sales for each store as denominator
store_data_max = store_data.groupby(level=1).max()
# develop training and validation set
train, valid = develop_valid_set2(store_data,
store_weather,
valid_size=100)
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set2(train, valid, test, store_weather)
# run prediction on testing data of each category
for n, trn, vld, tst in target_set:
print "%d, train(%d), valid(%d), test(%d)" % (n, len(trn), len(vld),
len(tst))
# normalize training, validing and testing data set
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
v_init = None
Y_hat2 = None
for i in range(1):
# run prediction on all validation and testing data set
Y_hat = build_model1(nm_trn, nm_vld, nm_tst, store_weather, \
valid_init=v_init, alpha_train=model_param)
# save the code in case the model has stacking effect
#v_init=Y_hat[len(trn):]
# denormalize the sale
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat,
store_data_max)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2)
print "error at %d is: train(%f), valid(%f)" % (i, e1, e2)
# write results to test result
write_submission(trn, vld, tst, Y_hat2, 'test_result.csv',
'valid_result')
def run_model4(store_data_file, store_weather_file, test_data_file, \
model_param=1, validate_only=False, eval_err=None):
"""
ridge regression
"""
print "---------------------start here---------------------"
test_result_file ='test_result.csv'
with open(test_result_file, 'w') as f:
f.write('id,units\n')
f.close()
store_data, store_weather, test = load_data2(store_data_file, \
store_weather_file, test_data_file)
store_data_max = store_data.groupby(level=1).max()
# categorize testing data with a relevant but much smaller training set
target_set = build_target_set4(store_data, test, store_weather, store_data_max)
for col, trn, vld, tst, cat in target_set:
print "item(%s), train(%d), valid(%d), test(%d), model_param(%0.2f), cat(%d)" % \
(col, len(trn), len(vld), len(tst), model_param, cat)
if len(tst)==0: continue
if cat==0:
Y_hat2=np.zeros((len(trn)+len(vld)+len(tst), 1))
else:
nm_trn = normalize_store_data(trn, store_data_max)
nm_vld = normalize_store_data(vld, store_data_max)
nm_tst = normalize_store_data(tst, store_data_max)
_,fmat = sim(nm_trn, nm_vld, nm_tst, store_weather)
Y_hat = np.zeros((len(nm_trn) + len(nm_vld) + len(nm_tst), 1))
X = fmat[:len(nm_trn)]
Y = nm_trn[col].values[:,np.newaxis]
clf = linear_model.Ridge(alpha=model_param)
clf.fit(X, Y)
Y_hat[:] = clf.predict(fmat)
Y_hat2 = denormalize_store_data(trn, vld, tst, Y_hat, store_data_max, column=col)
# evaluate error in training and validation set
e1, e2 = eval_model(trn, vld, Y_hat2, column=col)
print "error at item(%s) is: train(%f), valid(%f)" % (col, e1, e2)
if eval_err is not None:
eval_err.add_result(e1, len(trn), e2, len(vld))
# write results to test result
if not validate_only:
write_submission(trn, vld, tst, Y_hat2, test_result_file, 'valid_result', column=col)
# write out zero estimation
if not validate_only:
write_submission_zero(test, store_data_max, test_result_file)
if eval_err is not None:
e1, e2=eval_err.get_result()
logging.info("model4(p=%f) error is: train(%f), valid(%f)" % (model_param, e1, e2))
print "model4(p=%f) error is: train(%f), valid(%f)" % (model_param, e1, e2)
