def _from_config(cls,
executor: dict,
config: Config,
additional_info: dict):
args = Executor.kwargs_for_interface(executor, config, suffix='valid')
model = Interface.from_config(args)
return cls(model)
def create_task(self, k: str, v: dict, name: str, info: dict):
task_type = TaskType.User.value
if v.get('task_type') == 'train' or \
Executor.is_trainable(v['type']):
task_type = TaskType.Train.value
gpu = str(v.get('gpu', '0'))
if '-' not in gpu:
gpu = int(gpu)
gpu_max = gpu
else:
gpu, gpu_max = map(int, gpu.split('-'))
if gpu == 0 and gpu_max > 0:
raise Exception(f'Executor = {k} Gpu_max can"t be>0 when gpu=0')
task = Task(name=name,
executor=k,
computer=self.info.get('computer'),
gpu=gpu,
gpu_max=gpu_max,
cpu=v.get('cpu', 1),
memory=v.get('memory', 0.1),
dag=self.dag.id,
debug=self.debug,
steps=int(v.get('steps', '1')),
type=task_type)
if self.layout_name and task_type == TaskType.Train.value:
if self.layout_name not in self.layouts:
raise Exception(f'Unknown report = {v["report"]}')
report_config = self.layouts[self.layout_name]
info['report_config'] = report_config
task.additional_info = yaml_dump(info)
self.provider.add(task, commit=False)
report = Report(config=yaml_dump(report_config),
name=task.name,
project=self.project,
layout=self.layout_name)
self.report_provider.add(report)
task.report = report.id
self.report_tasks_provider.add(
ReportTasks(report=report.id, task=task.id))
self.report_tasks_provider.add(
ReportTasks(report=self.dag_report_id, task=task.id))
self.provider.commit()
else:
task.additional_info = yaml_dump(self.additional_info)
self.provider.add(task)
return task.id
def create_executor(self):
self.info('create_executor')
additional_info = yaml_load(self.task.additional_info) \
if self.task.additional_info else dict()
self.executor = Executor.from_config(executor=self.task.executor,
config=self.config,
additional_info=additional_info,
session=self.session,
logger=self.logger)
def create_task(self,
k: str,
v: dict,
name: str,
info: dict,
cell: dict = None):
task_type = TaskType.User.value
v = deepcopy(v)
if v.get('task_type') == 'train' or \
Executor.is_trainable(v['type']):
task_type = TaskType.Train.value
gpu = str(v.get('gpu', '0'))
if '-' not in gpu:
gpu = int(gpu)
gpu_max = gpu
else:
gpu, gpu_max = map(int, gpu.split('-'))
if gpu == 0 and gpu_max > 0:
raise Exception(f'Executor = {k} Gpu_max can"t be>0 when gpu=0')
task = Task(name=name,
executor=k,
computer=self.info.get('computer') or v.get('computer'),
gpu=gpu,
gpu_max=gpu_max,
cpu=v.get('cpu', 1),
memory=v.get('memory', 0.1),
dag=self.dag.id,
debug=self.debug,
steps=int(v.get('steps', '1')),
type=task_type)
if cell is not None:
v.update(cell)
info['executor'] = v
task.additional_info = yaml_dump(info)
report = None
if self.layout_name and task_type == TaskType.Train.value:
if self.layout_name not in self.layouts:
raise Exception(f'Unknown report = {v["report"]}')
report_config = self.layouts[self.layout_name]
info['report_config'] = report_config
task.additional_info = yaml_dump(info)
report = Report(config=yaml_dump(report_config),
name=task.name,
project=self.project,
layout=self.layout_name)
return task, report
def create_executor(self):
self.info('create_executor')
os.environ['CUDA_VISIBLE_DEVICES'] = self.task.gpu_assigned or ''
additional_info = yaml_load(self.task.additional_info) \
if self.task.additional_info else dict()
self.executor = Executor.from_config(executor=self.task.executor,
config=self.config,
additional_info=additional_info,
session=self.session,
logger=self.logger)
def download(self):
self.info('download')
if not self.task.debug:
folder = self.storage.download(task=self.id)
else:
folder = os.getcwd()
os.chdir(folder)
libraries = self.library_provider.dag(self.task.dag)
executor_type = self.executor_type
self.info('download. folder changed')
mlcomp_executors_folder = join(dirname(abspath(__file__)), 'executors')
mlcomp_base_folder = os.path.abspath(
join(mlcomp_executors_folder, '../../../'))
imported, was_installation = self.storage.import_executor(
mlcomp_executors_folder, mlcomp_base_folder, executor_type)
if not imported:
imported, was_installation = self.storage.import_executor(
folder, folder, executor_type, libraries)
if not imported:
raise Exception(f'Executor = {executor_type} not found')
self.info('download. executor imported')
if was_installation and not self.task.debug:
if self.repeat_count > 0:
self.info('was installation. '
'set task status to Queued. '
'And resending the task to a queue')
self.task.status = TaskStatus.Queued.value
self.provider.commit()
try:
execute.apply_async((self.id, self.repeat_count - 1),
queue=self.queue_personal,
retry=False)
except Exception:
pass
finally:
sys.exit()
assert Executor.is_registered(executor_type), \
f'Executor {executor_type} was not found'
def download(self):
self.info('download')
if not self.task.debug:
folder = self.storage.download(task=self.id)
else:
folder = os.getcwd()
os.chdir(folder)
libraries = self.library_provider.dag(self.task.dag)
executor_type = self.executor_type
mlcomp_executors_folder = join(dirname(abspath(__file__)), 'executors')
mlcomp_base_folder = os.path.abspath(
join(mlcomp_executors_folder, '../../../'))
imported, was_installation = self.storage.import_executor(
mlcomp_executors_folder, mlcomp_base_folder, executor_type)
if not imported:
imported, was_installation = self.storage.import_executor(
folder, folder, executor_type, libraries)
if not imported:
raise Exception(f'Executor = {executor_type} not found')
if was_installation and not self.task.debug:
if self.repeat_count > 0:
try:
self.warning(traceback.format_exc())
execute.apply_async((self.id, self.repeat_count - 1),
queue=self.queue_personal)
except Exception:
pass
finally:
sys.exit()
assert Executor.is_registered(executor_type), \
f'Executor {executor_type} was not found'
