def _get_interp_idxs_weights_2d(x, xp, y, yp, x_log_spacing=False):
with tf.name_scope('get_interp_idxs_weights_2d'):
if x_log_spacing:
x = tf.log(x)
xp = tf.log(xp)
with tf.control_dependencies([yp]):
xp = tf.tile(xp, tf.shape(yp)) #yp.shape)
xyp = tf.expand_dims(tf.parallel_stack([xp, yp]), 1)
xy0 = tf.reshape(tf.parallel_stack([x[0], y[0]]), [2, 1, 1])
xy1 = tf.reshape(tf.parallel_stack([x[1], y[1]]), [2, 1, 1])
spacing = xy1 - xy0
ind_grid = (xyp - xy0) / spacing
ind = tf.cast(ind_grid, tf.int32) + [[[0], [1]]]
max_ind = [[[x.shape[0].value - 1]], [[y.shape[0].value - 1]]]
#max_ind = [[[x.shape[0] - 1]], [[y.shape[0] - 1]]]
ind = tf.minimum(ind, max_ind)
ind_float = tf.cast(ind, tf.float64)
xy_grid = ind_float * spacing + xy0
weight = tf.abs(xyp - xy_grid) / spacing
if x_log_spacing:
weight = tf.parallel_stack([tf.exp(weight[0]), weight[1]])
weight = 1. - weight
weight_sum = tf.reduce_sum(weight, axis=1, keepdims=True)
weight /= weight_sum
return ind, weight
def aggregate(self, gradients):
# Assertion
assert len(gradients) > 0, "Empty list of gradient to aggregate"
# Computation
reshape_gradients = gradients
shape = gradients[0].shape
if len(shape) == 2 and shape[1] == 10:
for i in range(len(gradients)):
reshape_gradients[i] = tf.reshape(gradients[i], [
54080,
])
else:
for i in range(len(gradients)):
reshape_gradients[i] = tf.reshape(gradients[i], [-1])
grad_avg = native.instantiate_op(type(self).co_name,
tf.parallel_stack(gradients),
f=self.__nbbyzwrks,
m=self.__nbselected)
# lib = tf.load_op_library('/home/starly/Desktop/Fed/autodist/autodist/native/op_krum.so')
# grad_avg = lib.Krum(gradients=tf.parallel_stack(gradients), f=self.__nbbyzwrks, m=self.__nbselected)
if len(shape) == 2 and shape[1] == 10:
#x = tf.placeholder(tf.float32, shape=[10,None], name="tmp")
#grad_avg = tf.reshape(grad_avg, shape=[tf.shape(x)[0],10])
grad_avg = tf.reshape(grad_avg, shape=[5408, 10])
else:
grad_avg = tf.reshape(grad_avg, shape)
return grad_avg
def aggregate(self, gradients):
# Assertion
assert len(gradients) > 0, "Empty list of gradient to aggregate"
# Computation
gradients = tf.parallel_stack(gradients)
return tf.py_func(self._aggregate, [gradients],
gradients.dtype,
stateful=False,
name="GAR_bulyan")
def aggregate(self, gradients):
with tf.name_scope("GAR_krum_tf"):
# Assertion
assert len(gradients) > 0, "Empty list of gradient to aggregate"
# Distance computations
distances = []
for i in range(self.__nbworkers - 1):
dists = list()
for j in range(i + 1, self.__nbworkers):
sqr_dst = tf.reduce_sum(
tf.squared_difference(gradients[i], gradients[j]))
dists.append(
tf.negative(
tf.where(tf.is_finite(sqr_dst), sqr_dst,
tf.constant(np.inf, dtype=sqr_dst.dtype)))
) # Use of 'negative' to get the smallest distances and score indexes in 'nn.top_k'
distances.append(dists)
# Score computations
scores = []
for i in range(self.__nbworkers):
dists = []
for j in range(self.__nbworkers):
if j == i:
continue
if j < i:
dists.append(distances[j][i - j - 1])
else:
dists.append(distances[i][j - i - 1])
dists = tf.parallel_stack(dists)
dists, _ = tf.nn.top_k(dists,
k=(self.__nbworkers - self.__nbbyzwrks -
2),
sorted=False)
scores.append(tf.reduce_sum(dists))
# Average of the 'nbselected' smallest scoring gradients
gradients = tf.parallel_stack(gradients)
scores = tf.parallel_stack(scores)
_, indexes = tf.nn.top_k(scores, k=self.__nbselected, sorted=False)
return tf.reduce_mean(tf.gather(gradients, indexes), axis=0)
def aggregate(self, gradients):
# Assertion
assert len(gradients) > 0, "Empty list of gradient to aggregate"
# Computation
reshape_gradients = gradients
shape = gradients[0].shape
if len(shape) == 2 and shape[1] == 10:
for i in range(len(gradients)):
reshape_gradients[i] = tf.reshape(gradients[i], [
54080,
])
else:
for i in range(len(gradients)):
reshape_gradients[i] = tf.reshape(gradients[i], [-1])
grad_avg = native.instantiate_op(type(self).co_name,
tf.parallel_stack(gradients),
f=self.__nbbyzwrks,
m=self.__multikrum)
if len(shape) == 2 and shape[1] == 10:
grad_avg = tf.reshape(grad_avg, shape=[5408, 10])
else:
grad_avg = tf.reshape(grad_avg, shape)
return grad_avg
def k_full(self,
input1,
w_input1,
input2=None,
w_input2=None): #k_type: data_data, data_test, test_test
input1 = self._input_layer_normalization(input1) #by row
if input2 is None:
input2 = input1
w_input2 = w_input1
else:
input2 = self._input_layer_normalization(input2)
#w_input2 = self._input_layer_normalization(w_input2)
with tf.name_scope("k_full"):
cov_init = tf.matmul(
input1, input2, transpose_b=True
) / input1.shape[1].value #100,100; den is a tensor
#w_cov = tf.matmul(w_input1, w_input2, transpose_b=True)/ w_input1.shape[1].value
self.k_diag(input1, w_input1) #110 vars
q_aa_init = self.layer_qaa_dict[0] #L个
q_ab = cov_init
q_ab = self.weight_var * q_ab + self.bias_var #100, 100
#corr = q_ab / q_aa_init[0]
corr = tf.cast(q_ab, tf.float32) / tf.cast(q_aa_init[0],
tf.float32) #100, 100
if 32 > 1:
batch_size, batch_count = self._get_batch_size_and_count(
input1, input2)
with tf.name_scope("q_ab"):
q_ab_all = []
for b_x in range(batch_count):
#tf.logging.info('computing kernel for batch:{}'.format(b_x))
with tf.name_scope("batch_%d" % b_x):
corr_flat_batch = corr[batch_size *
b_x:batch_size *
(b_x +
1), :] #batchsize, 100
corr_flat_batch = tf.reshape(
corr_flat_batch, [-1]) #batchsize*100
#w_cov_flat_batch = tf.reshape(w_cov[batch_size * b_x : batch_size * (b_x + 1), :], [-1])
for l in range(self.depth):
with tf.name_scope("layer_%d" % l):
q_aa = self.layer_qaa_dict[l]
q_ab = interp1.interp_lin_2d(
x=self.var_aa_grid,
y=self.corr_ab_grid,
z=self.qab_grid,
xp=q_aa,
yp=tf.cast(corr_flat_batch,
tf.float64)) #10000
q_ab = self.weight_var * q_ab + self.bias_var
corr_flat_batch = q_ab / self.layer_qaa_dict[
l + 1][0]
q_ab_all.append(q_ab)
q_ab_all = tf.parallel_stack(q_ab_all)
else:
with tf.name_scope("q_ab"):
corr_flat = tf.reshape(corr, [-1]) #10000
w_cov_flat = tf.reshape(w_cov, [-1])
for l in rangenp.reshape(scores_tr, [-1, 1])(self.depth):
with tf.name_scope("layer_%d" % l):
q_aa = self.layer_qaa_dict[l]
q_ab = interp1.interp_lin_2d(x=self.var_aa_grid,
y=self.corr_ab_grid,
z=self.qab_grid,
xp=q_aa,
yp=tf.cast(
corr_flat,
tf.float64))
if l == self.depth - 1:
q_ab = self.weight_var * q_ab + self.bias_var #+ self.weight_var * w_cov_flat)
else:
q_ab = self.weight_var * q_ab + self.bias_var
corr_flat = q_ab / self.layer_qaa_dict[l +
1][0]
q_ab_all = q_ab
return tf.reshape(q_ab_all, cov_init.shape, "qab")