def __merge(taob, tc, p, l):
tc[l] = 0
p[l - 1] = p[l - 1] + p[l]
p[l] = 0
if self.size_commtime_dict is not None:
tc[l - 1] = self.size_commtime_dict[l - 1]
else:
tc[l - 1] = utils.predict_allreduce_time_with_size(
alpha, beta, p[l - 1] * nbytes, num_of_workers)
def _generate_groups_mgwfbp(self):
num_of_workers = size()
p_alpha_beta_56Gbps = {
64: (0.00080632079996292579, 1.8 * 3.2713239529771973e-10),
32: (0.00040632079996292579, 1.5 * 3.2713239529771973e-10),
16: (0.00023583677659915685 * 3, 4.0594787739537565e-10),
8: (9.75367204301171e-05, 3.0568230536676206e-10),
4: (4.204298980348825e-05, 2.0589360830118177e-10),
2: (2.554691138304671e-06, 9.837548167872609e-11)
}
p_alpha_beta_10Gbps = {
64: (0.0023476410788581382 * 3, 9.643300782166769e-10),
32: (0.0013476410788581382 * 3, 8.643300782166769e-10),
16: (0.0009080981007148093, 7.395651186836712e-10),
8: (0.0005230272768511732, 8.570746975492128e-10),
4: (4.204298980348825e-05, 2.0589360830118177e-10),
2: (2.554691138304671e-06, 9.837548167872609e-11)
}
if self.alpha is not None:
alpha, beta = self.alpha, self.beta
else:
if self._rdma:
alpha, beta = p_alpha_beta_56Gbps[num_of_workers]
else:
alpha, beta = p_alpha_beta_10Gbps[num_of_workers]
nbytes = 2 if self._fp16 else 4
def __calculate_comm_start(tc, tb, taob, L):
taoc = [0] * L
taoc[L - 1] = taob[L - 1] + tb[L - 1]
for l in range(L - 1)[::-1]:
taoc[l] = max(taoc[l + 1] + tc[l + 1], taob[l] + tb[l])
return taoc
def __merge(taob, tc, p, l):
tc[l] = 0
p[l - 1] = p[l - 1] + p[l]
p[l] = 0
if self.size_commtime_dict is not None:
tc[l - 1] = self.size_commtime_dict[l - 1]
else:
tc[l - 1] = utils.predict_allreduce_time_with_size(
alpha, beta, p[l - 1] * nbytes, num_of_workers)
sizes = [
self._named_parameters[k].data.numel()
for k in self._seq_layernames
]
seq_layernames = self._seq_layernames
if not utils.check_unique(seq_layernames):
raise ValueError
self._sizes = sizes
p = sizes[:]
L = len(sizes)
if self.size_commtime_dict is not None:
tc = [self.size_commtime_dict[s] for s in sizes]
else:
tc = [
utils.predict_allreduce_time_with_size(alpha, beta, s * nbytes,
num_of_workers)
for s in sizes
]
tb = list(self._layerwise_times)
taob = [0] * L
for l in range(0, L - 1)[::-1]:
taob[l] = taob[l + 1] + tb[l + 1]
taoc = __calculate_comm_start(tc, tb, taob, L)
if rank() == 0:
#logger.debug('seq_layernames: %s', seq_layernames)
#logger.debug('tb: %s', tb)
#logger.debug('taob: %s', taob)
#logger.debug('sizes: %s', p)
#logger.warn('tc sum: %f', np.sum(tc))
pass
#logger.warn('tc: %s', tc)
#logger.warn('taoc: %s', taoc)
groups = []
group = []
idx = 0
key_groupidx_maps = {}
l = L - 1
key = seq_layernames[l]
key_groupidx_maps[key] = idx
for l in range(1, L)[::-1]:
key = seq_layernames[l]
group.append(key)
key_groupidx_maps[key] = idx
current_taob = taob[l - 1] + tb[l - 1]
merged = False
if current_taob < taoc[l] + tc[l]:
if taoc[l] > current_taob:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
merged = True
else:
t_wait = current_taob - taoc[l]
t_saved = alpha
if t_wait < t_saved:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
merged = True
#if not merged and (key.find('bn') >= 0 or key.find('bias') >= 0):
if not merged and p[l] < 8192:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
merged = True
if not merged:
idx += 1
groups.append(group)
group = []
#elif current_taob > taoc[l+1]+tc[l+1] and current_taob < taoc[l]+tc[l] and taoc[l]+alpha > current_taob:
# __merge(taob, tc, p, l)
# taoc = __calculate_comm_start(tc, tb, taob, L)
#else:
# idx += 1
# groups.append(group)
# group = []
l = 0
key = seq_layernames[l]
key_groupidx_maps[key] = idx
group.append(key)
if len(group) > 0:
groups.append(group)
if rank() == 0:
#logger.debug('seq_layernames: %s', seq_layernames)
#pass
#logger.info('Merged tc sum: %f', np.sum(tc))
print('Merged sizes: ', p[::-1])
print('# of parameters: ', np.sum(p[::-1]))
#logger.info('Merged tb: %s', tb[::-1])
#logger.info('Merged taob: %s', taob[::-1])
#logger.info('Merged tc: %s', tc[::-1])
#logger.info('Merged taoc: %s', taoc[::-1])
return groups, key_groupidx_maps
def _generate_groups_mgwfbp(self):
num_of_workers = size()
p_alpha_beta_56Gbps = {
16: (0.00023583677659915685, 4.0594787739537565e-10),
8: (9.75367204301171e-05, 3.0568230536676206e-10),
4: (4.204298980348825e-05, 2.0589360830118177e-10),
2: (2.554691138304671e-06, 9.837548167872609e-11)
}
p_alpha_beta_10Gbps = {
16: (0.0009080981007148093, 7.395651186836712e-10),
8: (0.0005230272768511732, 8.570746975492128e-10),
4: (4.204298980348825e-05, 2.0589360830118177e-10),
2: (2.554691138304671e-06, 9.837548167872609e-11)
}
if self.alpha is not None:
alpha, beta = self.alpha, self.beta
else:
if settings.CONNECTION == '10GbE':
alpha, beta = p_alpha_beta_10Gbps[num_of_workers]
else:
alpha, beta = p_alpha_beta_56Gbps[num_of_workers]
nbytes = 2 if settings.FP16 else 4
def __calculate_comm_start(tc, tb, taob, L):
taoc = [0] * L
taoc[L - 1] = taob[L - 1] + tb[L - 1]
for l in range(L - 1)[::-1]:
taoc[l] = max(taoc[l + 1] + tc[l + 1], taob[l] + tb[l])
return taoc
def __merge(taob, tc, p, l):
tc[l] = 0
p[l - 1] = p[l - 1] + p[l]
p[l] = 0
if self.size_commtime_dict is not None:
tc[l - 1] = self.size_commtime_dict[l - 1]
else:
tc[l - 1] = utils.predict_allreduce_time_with_size(
alpha, beta, p[l - 1] * nbytes, num_of_workers)
sizes = [
self._named_parameters[k].data.numel()
for k in self._seq_layernames
]
seq_layernames = self._seq_layernames
if not utils.check_unique(seq_layernames):
raise ValueError
self._sizes = sizes
p = sizes[:]
L = len(sizes)
if self.size_commtime_dict is not None:
tc = [self.size_commtime_dict[s] for s in sizes]
else:
tc = [
utils.predict_allreduce_time_with_size(alpha, beta, s * nbytes,
num_of_workers)
for s in sizes
]
tb = list(self._layerwise_times)
taob = [0] * L
for l in range(0, L - 1)[::-1]:
taob[l] = taob[l + 1] + tb[l + 1]
taoc = __calculate_comm_start(tc, tb, taob, L)
if rank() == 0:
logger.info('tc sum: %f', np.sum(tc))
groups = []
group = []
idx = 0
key_groupidx_maps = {}
l = L - 1
key = seq_layernames[l]
key_groupidx_maps[key] = idx
for l in range(1, L)[::-1]:
key = seq_layernames[l]
group.append(key)
key_groupidx_maps[key] = idx
current_taob = taob[l - 1] + tb[l - 1]
merged = False
if current_taob < taoc[l] + tc[l]:
if taoc[l] > current_taob:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
merged = True
else:
t_wait = current_taob - taoc[l]
t_saved = alpha
if t_wait < t_saved:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
merged = True
if not merged:
idx += 1
groups.append(group)
group = []
l = 0
key = seq_layernames[l]
key_groupidx_maps[key] = idx
group.append(key)
if len(group) > 0:
groups.append(group)
if rank() == 0:
logger.info('Predicted non-overlapped time: %f',
taoc[0] + tc[0] - (taob[0] + tb[0]))
logger.info('Predicted tb+tc= %f', taoc[0] + tc[0])
logger.info('Merged tc sum: %f', np.sum(tc))
return groups, key_groupidx_maps
def __merge(taob, tc, p, l):
tc[l] = 0
p[l - 1] = p[l - 1] + p[l]
p[l] = 0
tc[l - 1] = utils.predict_allreduce_time_with_size(
alpha, beta, p[l - 1] * 4, num_of_workers)
def _generate_groups_mgwfbp(self):
num_of_workers = size()
alpha = 9.618801111215886e-08
beta = 3.89407453e-13
def __calculate_comm_start(tc, tb, taob, L):
taoc = [0] * L
taoc[L - 1] = taob[L - 1] + tb[L - 1]
for l in range(L - 1)[::-1]:
taoc[l] = max(tc[l + 1] + tc[l + 1], taob[l] + tb[l])
return taoc
def __merge(taob, tc, p, l):
tc[l] = 0
p[l - 1] = p[l - 1] + p[l]
p[l] = 0
tc[l - 1] = utils.predict_allreduce_time_with_size(
alpha, beta, p[l - 1] * 4, num_of_workers)
sizes = [
self._named_parameters[k].data.numel()
for k in self._seq_layernames
][::-1]
seq_layernames = self._seq_layernames[::-1]
self._sizes = sizes
p = sizes[:]
L = len(sizes)
tc = [
utils.predict_allreduce_time_with_size(alpha, beta, s * 4,
num_of_workers)
for s in sizes
]
tb = list(self._layerwise_times[::-1])
taob = [0]
for t in tb[:-1]:
taob.append(t + taob[-1])
taob = taob[::-1]
taoc = __calculate_comm_start(tc, tb, taob, L)
if rank() == 0 and DEBUG:
logger.debug('seq_layernames: %s', seq_layernames)
logger.debug('tb: %s', tb)
logger.debug('taob: %s', taob)
logger.debug('sizes: %s', p)
logger.debug('tc: %s', tc)
logger.debug('taoc: %s', taoc)
groups = []
group = []
idx = 0
key_groupidx_maps = {}
l = L - 1
key = seq_layernames[l]
key_groupidx_maps[key] = idx
group.append(key)
for l in range(1, L - 1)[::-1]:
key = seq_layernames[l]
group.append(key)
key_groupidx_maps[key] = idx
current_taob = taob[l - 2] if l >= 2 else taob[0]
if current_taob - taoc[l] < alpha:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
else:
idx += 1
groups.append(group)
group = []
l = 0
key = seq_layernames[l]
key_groupidx_maps[key] = idx
group.append(key)
if len(group) > 0:
groups.append(group)
return groups, key_groupidx_maps
def _generate_groups_mgwfbp(self):
num_of_workers = size()
p_alpha_beta = {
16: (0.00010632079996292579, 1.5 * 3.2713239529771973e-10),
8: (9.75367204301171e-05, 3.0568230536676206e-10),
4: (4.204298980348825e-05, 2.0589360830118177e-10),
2: (2.554691138304671e-06, 9.837548167872609e-11)
}
alpha, beta = p_alpha_beta[num_of_workers]
def __calculate_comm_start(tc, tb, taob, L):
taoc = [0] * L
taoc[L - 1] = taob[L - 1] + tb[L - 1]
for l in range(L - 1)[::-1]:
taoc[l] = max(taoc[l + 1] + tc[l + 1], taob[l] + tb[l])
return taoc
def __merge(taob, tc, p, l):
tc[l] = 0
p[l - 1] = p[l - 1] + p[l]
p[l] = 0
tc[l - 1] = utils.predict_allreduce_time_with_size(
alpha, beta, p[l - 1] * 4, num_of_workers)
sizes = [
self._named_parameters[k].data.numel()
for k in self._seq_layernames
]
seq_layernames = self._seq_layernames
self._sizes = sizes
p = sizes[:]
L = len(sizes)
tc = [
utils.predict_allreduce_time_with_size(alpha, beta, s * 4,
num_of_workers)
for s in sizes
]
tb = list(self._layerwise_times)
taob = [0] * L
for l in range(0, L - 1)[::-1]:
taob[l] = taob[l + 1] + tb[l + 1]
taoc = __calculate_comm_start(tc, tb, taob, L)
if rank() == 0:
logger.warn('tc sum: %f', np.sum(tc))
logger.warn('tc: %s', tc)
logger.warn('taoc: %s', taoc)
groups = []
group = []
idx = 0
key_groupidx_maps = {}
l = L - 1
key = seq_layernames[l]
key_groupidx_maps[key] = idx
group.append(key)
for l in range(1, L - 1)[::-1]:
key = seq_layernames[l]
group.append(key)
key_groupidx_maps[key] = idx
current_taob = taob[l - 1] + tb[l - 1]
if current_taob < taoc[l + 1] + tc[l + 1]:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
elif current_taob > taoc[l + 1] + tc[l + 1] and current_taob < taoc[
l] + tc[l] and taoc[l] + alpha > current_taob:
__merge(taob, tc, p, l)
taoc = __calculate_comm_start(tc, tb, taob, L)
else:
idx += 1
groups.append(group)
group = []
l = 0
key = seq_layernames[l]
key_groupidx_maps[key] = idx
group.append(key)
logger.info('Predicted non-overlapped time: %f',
taoc[0] + tc[0] - (taob[0] + tb[0]))
logger.info('Predicted tb+tc= %f', taoc[0] + tc[0])
if len(group) > 0:
groups.append(group)
return groups, key_groupidx_maps