def ParallelSQN(f,
epoch,
P,
batch_size4SVRG,
stepsize=10**-5,
stepsize_type="fixed",
verbose=False,
optgap=10**(-30),
loss='logistic'):
'''
INPUT:
x : data
y : vector for label 1 or 0
K : number of outest iterations
P : number of parallel precossors
batch_size4SVRG :
batch_size4H
stepsize : default 10**-5
stepsize_type : fixed, decay 1/t, sqrt decay 1/sqrt( t )
OUTPUT:
'''
# global FILENAME
# FILENAME = outfile
# print FILENAME
# logging.basicConfig(level=logging.DEBUG, filename=FILENAME, filemode='w')
x, label = Util.readlibsvm(f)
#x = sklearn.preprocessing.normalize( x )
#label = (label == 1 )* 1
L = x.shape[0] // (batch_size4SVRG * P)
if loss == 'logistic':
loss = Loss.LogisticLoss_version2()
elif loss == 'svm':
loss = Loss.svm_quadratic()
elif loss == 'ridge':
loss = Loss.ridge_regression()
#regularizer = 1/x.shape[0]
regularizer = 10**(-3)
#loss = Loss.ridge_regression()
print("The number of cores : " + str(multiprocessing.cpu_count()))
print("The dataset : " + str(f))
print("The number of instances N : " + str(x.shape[0]))
print("The number of features p : " + str(x.shape[1]))
print('The number of processes P : ' + str(P))
print("The batch size for SVRG : " + str(batch_size4SVRG))
print("The step size : " + str(stepsize))
print("The epoch : " + str(epoch))
print("The loss type: " + str(loss))
print("The regularizer : " + str(regularizer))
# init shared mem variables
x_data = sharedmem.empty(len(x.data), dtype=x.data.dtype)
x_data = x.data
x_indices = sharedmem.empty(len(x.indices), dtype=x.indices.dtype)
x_indices = x.indices
x_indptr = sharedmem.empty(len(x.indptr), dtype=x.indptr.dtype)
x_indptr = x.indptr
y = sharedmem.empty(len(label), dtype=label.dtype)
y = label
x_shape = sharedmem.empty(len(x.shape), dtype=x.indices.dtype)
x_shape = x.shape
lock = Lock()
w = sharedmem.empty(x.shape[1], dtype=np.longdouble)
#w[:] = np.random.rand(x.shape[1],)#Array(c_double, np.random.rand(x.shape[0]), lock=False)
w[:] = np.zeros(x.shape[1], )
w_multi = sharedmem.empty(x.shape[1], dtype=np.longdouble)
w_multi[:] = np.copy(w) #multiprocessing.sharedctypes.copy(w)
u = sharedmem.empty(x.shape[1], dtype=np.longdouble)
u[:] = np.random.rand(x.shape[1], )
flag = sharedmem.empty(P, dtype=int)
flag[:] = np.zeros([
P,
])
# ----------------------------------------------
procs = []
# add master
procs.append(
Process(target=master_loop,
args=(loss, lock, L, P, x_data, x_indices, x_indptr, x_shape,
y, batch_size4SVRG, w, w_multi, u, epoch, flag, optgap,
stepsize_type, stepsize, regularizer)))
# add slaves
for proc_id in range(1, P):
t = Process(target=slave_loop,
args=(loss, lock, L, x_data, x_indices, x_indptr, x_shape,
y, batch_size4SVRG, w, w_multi, u, epoch, flag,
proc_id, stepsize_type, stepsize, regularizer))
procs.append(t)
# start all processes
for t in procs:
t.daemon = True
t.start()
# wait until all processes finish
for t in procs:
t.join()
print('Finish parallel ')
def sgd_ht(f,
regularizer,
epoch,
batch_size,
stepsize=10**-5,
stepsize_type="fixed",
verbose=False,
optgap=10**(-3),
loss='logistic',
ht_k=100,
log_interval=1,
output_folder='../logs/',
multi_class=False):
'''
INPUT:
x : data
y : vector for label 1 or 0
K : number of outest iterations
P : number of parallel precossors
batch_size4SVRG :
batch_size4H
stepsize : default 10**-5
stepsize_type : fixed, decay 1/t, sqrt decay 1/sqrt( t )
OUTPUT:
'''
# global FILENAME
# FILENAME = outfile
# print FILENAME
# logging.basicConfig(level=logging.DEBUG, filename=FILENAME, filemode='w')
if 'simulate' in f:
data = np.load(f)
x = data['x']
y = data['y']
else:
x, y = Util.readlibsvm(f)
if 'news20' in f:
y = y - 1 #x = sklearn.preprocessing.normalize( x )
index = np.arange(np.shape(x)[0])
np.random.shuffle(index)
x = x[index, :]
y = y[index]
#label = (label == 1 )* 1
folder_name = os.path.basename(f) + '_loss_' + str(
loss) + '_opt_sgd_' + 'stepsize_' + str(
stepsize) + '_batchsize_' + str(batch_size) + '_ht_k_' + str(
ht_k) + '_log_interval_' + str(log_interval) + '_epoch_' + str(
epoch)
logger = SummaryWriter(output_folder + folder_name)
file = open(output_folder + folder_name + str(".txt"), "w")
file.write("The dataset : " + str(f) + '\n')
file.write("The number of instances N : " + str(x.shape[0]) + '\n')
file.write("The number of features p : " + str(x.shape[1]) + '\n')
file.write("The step size : " + str(stepsize) + '\n')
file.write("The epoch : " + str(epoch) + '\n')
file.write("The loss type: " + str(loss) + '\n')
file.write("The regularizer : " + str(regularizer) + '\n')
file.write("The ht_k : " + str(ht_k) + '\n')
file.write("The batch_size : " + str(batch_size) + '\n')
file.write('num_epoch,num_IFO,num_HT,loss\n')
if loss == 'logistic':
loss = Loss.LogisticLoss_version2()
elif loss == 'svm':
loss = Loss.svm_quadratic()
elif loss == 'ridge':
loss = Loss.ridge_regression()
elif loss == 'multi_class_softmax_regression':
loss = Loss.multi_class_softmax_regression()
#regularizer = 1/x.shape[0]
#loss = Loss.ridge_regression()
print("The dataset : " + str(f))
print("The number of instances N : " + str(x.shape[0]))
print("The number of features p : " + str(x.shape[1]))
print("The batch size for SGD : " + str(batch_size))
print("The step size : " + str(stepsize))
print("The epoch : " + str(epoch))
print("The loss type: " + str(loss))
print("The regularizer : " + str(regularizer))
#
if multi_class:
w = np.zeros((x.shape[1], len(np.unique(y))))
else:
w = np.zeros(x.shape[1], )
# ---------------------------------------------
#store information
obj_list = list()
t0 = time.time()
iteration = 0
for k in range(epoch):
#print("master iteration ", k)
#setup stepsize
if stepsize_type == "fixed":
eta = stepsize
elif stepsize_type == "decay":
eta = 1. / (k + 1)
elif stepsize_type == "sqrtdecay":
#eta = 1/np.sqrt(k*L + t +1 )
eta = stepsize * 1. / np.sqrt(k + 1)
elif stepsize_type == "squaredecay":
eta = (1 / np.square(k + 1))
for t in range(int(x.shape[0] / batch_size)):
#print("master iteration ", k, t)
#lock.acquire()
sample_id = np.random.randint(x.shape[0] - batch_size, size=1)
sample_id = sample_id[0]
g1 = loss.grad(x[sample_id:sample_id + batch_size],
y[sample_id:sample_id + batch_size], w, regularizer)
#print( 'w_multi : ', np.square( np.linalg.norm( w_multi )))
Hv = -g1
#print( 'w : ', np.square( np.linalg.norm( w )))
#print(multiprocessing.current_process(), "Before Update w ", w[0:3], w[ -3:-1])
w = w + eta * Hv
if multi_class:
for j in range(w.shape[1]):
w[np.absolute(w[:, j]).argsort()[:-ht_k][::-1], j] = 0
else:
w[np.absolute(w).argsort()[:-ht_k][::-1]] = 0
#print(multiprocessing.current_process(), "After Update w ", w[0:3], w[ -3:-1])
iteration = iteration + 1
#--------------------------------------
if (t + 1) % log_interval == 0:
#obj_temp = loss.obj( x, y, w, regularizer )/loss.obj( x, y, np.zeros(x.shape[1],), regularizer )
obj_temp = loss.obj(x, y, w, regularizer)
#if k>0:
time_k = time.time()
#print( 'Epoch: '+ str(k+1) +', data passes : '+ str((k+1))+ ', time :' + str(time_k - t0 ))
#store informations
#if verbose:
#print( "objective value %.30f" % obj_temp)
#print( "Norm for w : ", np.square( np.linalg.norm( w )))
obj_list.append(obj_temp)
if k > 0 and np.abs(obj_list[-1] - obj_list[-2]) < optgap:
print("Optimality gap tolerance reached : optgap " +
str(optgap))
break
#return obj_list, datapasses_list, time_list
print('loss ' + str(obj_temp))
logger.add_scalar('loss_number-IFO', obj_temp,
k * x.shape[0] + t * batch_size)
logger.add_scalar('loss_number-HT', obj_temp, iteration)
file.write(
str((k * x.shape[0] + t * batch_size) / x.shape[0]) + ',' +
str(k * x.shape[0] + t * batch_size) + ',' +
str(iteration) + ',' + str(obj_temp) + '\n')
file.close()
