def _update_train_data_for_next_iteration(self, task_declaration: TaskDeclaration):
assert task_declaration.state == TaskDeclaration.State.VERIFY_IN_PROGRESS
task_declaration.current_iteration += 1
task_declaration.current_iteration_retry = 0
task_declaration.progress = (
task_declaration.current_iteration * task_declaration.epochs_in_iteration * 100
/ task_declaration.epochs)
count_ta = 0
for ta in task_declaration.get_task_assignments(states=(TaskAssignment.State.FINISHED,)):
train_data = ta.train_data
# share data to worker
train_data.set_encryption_key(ta.worker.enc_key)
train_data.save()
ta.state = TaskAssignment.State.TRAINING
ta.save()
count_ta += 1
assert task_declaration.workers_requested == count_ta
task_declaration.state = TaskDeclaration.State.EPOCH_IN_PROGRESS
task_declaration.save()
def _assign_initial_train_data(self, task_declaration: TaskDeclaration):
assert task_declaration.state == TaskDeclaration.State.DEPLOYMENT
# start of train
task_declaration.current_iteration += 1
task_declaration.current_iteration_retry = 0
accepted_task_assignment = task_declaration.get_task_assignments(states=(TaskAssignment.State.ACCEPTED,))
count_ta = 0
train_dirs_ipfs, files = Directory(multihash=task_declaration.dataset.train_dir_ipfs).ls()
test_dirs_ipfs, files = Directory(multihash=task_declaration.dataset.test_dir_ipfs).ls()
all_train_chunks_ipfs = self._chunk_it(
iterable=[x.multihash for x in train_dirs_ipfs],
count=task_declaration.workers_requested
)
assert len(all_train_chunks_ipfs) == task_declaration.workers_requested
all_test_chunks_ipfs = self._chunk_it(
iterable=[x.multihash for x in test_dirs_ipfs],
count=task_declaration.workers_requested
)
assert len(all_test_chunks_ipfs) == task_declaration.workers_requested
list_td_ta = []
with async_commit():
# create TrainData
for index, task_assignment in enumerate(accepted_task_assignment):
train_chunks_ipfs = all_train_chunks_ipfs[index]
test_chunks_ipfs = all_test_chunks_ipfs[index]
train_data = TrainData.create(
model_code_ipfs=task_declaration.train_model.code_ipfs,
train_chunks_ipfs=train_chunks_ipfs,
test_chunks_ipfs=test_chunks_ipfs,
data_index=index,
db=self.db,
encryption=self.encryption
)
list_td_ta.append((train_data, task_assignment))
logger.debug('Created {}, train chunks: {}, count:{}, test chunks: {}, count:{}'.format(
train_data, train_chunks_ipfs, len(train_chunks_ipfs), test_chunks_ipfs, len(test_chunks_ipfs)))
count_ta += 1
assert task_declaration.workers_requested == count_ta
with async_commit():
# share to worker
for train_data, task_assignment in list_td_ta:
train_data.task_assignment_id = task_assignment.asset_id
train_data.set_encryption_key(task_assignment.worker.enc_key)
train_data.save()
task_assignment.train_data_id = train_data.asset_id
task_assignment.state = TaskAssignment.State.TRAINING
task_assignment.save()
task_declaration.state = TaskDeclaration.State.EPOCH_IN_PROGRESS
task_declaration.save()