def get_data_packet(self) -> Packet:
with self.lock:
self.data['time'] = time.time()
packet = Packet(self.data, callback=self.callback)
self.data = {}
self.callback = None
return packet
def flush(self):
data = {}
for key, value in self.indicators.items():
value = np.array(value)
step: np.ndarray = value[:, 0]
value: np.ndarray = value[:, 1]
while value.shape[0] > MAX_BUFFER_SIZE:
if value.shape[0] % 2 == 1:
value = np.concatenate((value, value[-1:]))
step = np.concatenate((step, step[-1:]))
n = value.shape[0] // 2
step = np.mean(step.reshape(n, 2), axis=-1)
value = np.mean(value.reshape(n, 2), axis=-1)
data[key] = {
'step': step.tolist(),
'value': value.tolist()
}
self.indicators = {}
self.api_caller.push(Packet({
'track': data,
'time': time.time()
}))
def get_data_packet(self) -> Packet:
with self.lock:
self.last_committed = time.time()
self.data['time'] = time.time()
packet = Packet(self.data)
self.data = {}
return packet
def get_data_packet(self) -> Packet:
with self.lock:
self.last_committed = time.time()
self.data['track'] = self.get_and_clear_indicators()
self.data['time'] = time.time()
packet = Packet(self.data)
self.data = {}
return packet
def start(self):
data = {
'name': self.name,
'comment': self.comment,
'time': time.time()
}
self.api_caller.push(Packet(data, callback=self._started))
from labml.internal.api.logs import API_LOGS
API_LOGS.set_api(self.api_caller, frequency=self.frequency)
def get_data_packet(self) -> Packet:
with self.lock:
self.last_committed = time.time()
self.data['time'] = time.time()
for type_ in ['stdout', 'logger']:
if type_ not in self.data:
continue
self.data[type_] = self._clean(self.data[type_])
packet = Packet(self.data)
self.data = {}
return packet
def flush(self):
data = {'time': time.time()}
if self.stdout is not None:
data['stdout'] = self.stdout
self.stdout = None
if self.stderr is not None:
data['stderr'] = self.stderr
self.stderr = None
if self.logger is not None:
data['logger'] = self.logger
self.logger = None
self.api_caller.push(Packet(data))
def status(self, rank: int, status: str, details: str, time_: float):
self.state = {
'rank': rank,
'status': status,
'details': details,
'time': time_
}
self.api_caller.push(
Packet({
'status': self.state,
'time': time.time()
}))
# TODO: Will have to fix this when there are other statuses that dont stop the experiment
# This will stop the thread after sending all the data
self.api_caller.stop()
def save_configs(self, configs: Dict[str, any]):
self.api_caller.push(Packet({'configs': configs}))
def save_indicators(self, dot_indicators: Dict[str, Indicator], indicators: Dict[str, Indicator]):
wildcards = {k: ind.to_dict() for k, ind in dot_indicators.items()}
inds = {k: ind.to_dict() for k, ind in indicators.items()}
self.api_caller.push(Packet({'wildcard_indicators': wildcards, 'indicators': inds}))