def hook(module, *unused): for name, p in module.named_parameters(): full_name = parent_name + '.' + name if self._use_timeline: # End forward computation timeline bf.timeline_end_activity(full_name) if not module.training: continue if p.requires_grad: if self._pushsum_delay[p] <= 0: raise AssertionError( "Unexpected behavior: forward computation were computed " "more than num_steps_per_communication times before call " "to step(). Adjust num_steps_per_communication to " "accumulate gradients locally.") self._pushsum_delay[p] -= 1 if self._pushsum_delay[p] == 0: ps_weights = self._named_ps_weights[full_name] extended_parameter = torch.cat( (p.data.view(-1), ps_weights), 0) self._named_extension_parameters[ name] = extended_parameter handle = bf.win_accumulate_nonblocking( tensor=extended_parameter, name=full_name, dst_weights=self.dst_weights, require_mutex=True) self._handles[p] = handle
def hook(module, *unused): for name, p in module.named_parameters(): if not module.training: continue if self._use_timeline: # End forward computation timeline bf.timeline_end_activity(parent_name + '.' + name) if p.requires_grad: if self._reduce_delay[p] <= 0: raise AssertionError( "Unexpected behavior: forward computation were computed " "more than num_steps_per_communication times before call " "to step(). Adjust num_steps_per_communication to " "accumulate gradients locally.") self._reduce_delay[p] -= 1 if self._reduce_delay[p] == 0: if self._reduce_method == 0: handle = self._allreduce_data_async(p) elif self._reduce_method == 1: handle = self._neighbor_allreduce_data_async(p) elif self._reduce_method == 2: handle = self._hierarchical_neighbor_allreduce_data_async( p) elif self._reduce_method == -1: handle = None else: raise ValueError( "Unknown reduce method. Do not change _reduce_method manually." ) self._handles[p] = handle
def test_timeline_with_python_interface(self): bf.timeline_start_activity("test_python_interface_x", "FAKE_ACTIVITY") time.sleep(0.1) bf.timeline_end_activity("test_python_interface_x") time.sleep(0.1) file_name = f"{self.temp_file}{bf.rank()}.json" with open(file_name, 'r') as tf: timeline_text = tf.read() assert 'FAKE_ACTIVITY' in timeline_text, timeline_text
def hook(module, *unused): for name, p in module.named_parameters(): if self._use_timeline: # End forward computation timeline bf.timeline_end_activity(parent_name + '.' + name) if not module.training: continue if p.requires_grad: if self._bluefog_delay[p] <= 0: raise AssertionError( "Unexpected behavior: forward computation were computed " "more than num_steps_per_communication times before call " "to step(). Adjust num_steps_per_communication to " "accumulate gradients locally.") self._bluefog_delay[p] -= 1 if self._bluefog_delay[p] == 0: handle = bf.win_get_nonblocking( name=parent_name + '.' + name, src_weights=self.src_weights, require_mutex=True) self._handles[p] = handle
def hook(*ignore): bf.timeline_end_activity(name)
def _timeline_forward_end_hook(module, *unused): for name, _ in module.named_parameters(): full_name = parent_name + '.' + name if parent_name else name bf.timeline_end_activity(full_name)
def f(): bf.timeline_start_activity("test_multi_thread", "THREAD") time.sleep(0.1) bf.timeline_end_activity("test_multi_thread")