def __init__(self,
session: Session,
task: Task,
layout: str,
part: str = 'valid',
name: str = 'img_classify',
max_img_size: Tuple[int, int] = None,
main_metric: str = 'accuracy',
plot_count: int = 0):
self.session = session
self.task = task
self.layout = layout
self.part = part
self.name = name or 'img_classify'
self.max_img_size = max_img_size
self.main_metric = main_metric
self.plot_count = plot_count
self.dag_provider = DagProvider(session)
self.report_provider = ReportProvider(session)
self.layout_provider = ReportLayoutProvider(session)
self.task_provider = TaskProvider(session)
self.report_img_provider = ReportImgProvider(session)
self.report_task_provider = ReportTasksProvider(session)
self.report_series_provider = ReportSeriesProvider(session)
self.project = self.task_provider.project(task.id).id
self.layout = self.layout_provider.by_name(layout)
self.layout_dict = yaml_load(self.layout.content)
def create_providers(self):
self.log_info('create_providers')
self.dag_provider = DagProvider(self.session)
self.task_provider = TaskProvider(self.session)
self.file_provider = FileProvider(self.session)
self.dag_storage_provider = DagStorageProvider(self.session)
def create_base(self):
self.info('create_base')
self.provider = TaskProvider(self.session)
self.library_provider = DagLibraryProvider(self.session)
self.storage = Storage(self.session)
self.task = self.provider.by_id(
self.id, joinedload(Task.dag_rel, innerjoin=True))
if not self.task:
raise Exception(f'task with id = {self.id} is not found')
self.dag = self.task.dag_rel
self.executor = None
self.hostname = socket.gethostname()
self.docker_img = DOCKER_IMG
self.worker_index = os.getenv('WORKER_INDEX', -1)
self.queue_personal = f'{self.hostname}_{self.docker_img}_' \
f'{self.worker_index}'
self.config = Config.from_yaml(self.dag.config)
self.executor_type = self.config['executors'][
self.task.executor]['type']
def describe_task_names(dag: int):
pd.set_option('display.max_columns', None)
pd.set_option('display.expand_frame_repr', False)
pd.set_option('max_colwidth', -1)
provider = TaskProvider()
tasks = provider.by_dag(dag)
return pd.DataFrame([{'id': t.id, 'name': t.name} for t in tasks])
def create_providers(self):
self.provider = TaskProvider(self.session)
self.report_provider = ReportProvider(self.session)
self.report_tasks_provider = ReportTasksProvider(self.session)
self.report_layout_provider = ReportLayoutProvider(self.session)
self.project_provider = ProjectProvider(self.session)
self.storage = Storage(self.session)
self.dag_provider = DagProvider(self.session)
def create_providers(self):
self.log_info('create_providers')
self.provider = TaskProvider(self.session)
self.report_provider = ReportProvider(self.session)
self.report_tasks_provider = ReportTasksProvider(self.session)
self.report_layout_provider = ReportLayoutProvider(self.session)
self.project_provider = ProjectProvider(self.session)
self.storage = Storage(self.session,
logger=self.logger,
component=self.component)
self.dag_provider = DagProvider(self.session)
def __init__(self, session: Session, logger=None,
component: ComponentType = None,
max_file_size: int = 10 ** 5, max_count=10 ** 3):
self.file_provider = FileProvider(session)
self.provider = DagStorageProvider(session)
self.task_provider = TaskProvider(session)
self.library_provider = DagLibraryProvider(session)
self.dag_provider = DagProvider(session)
self.logger = logger
self.component = component
self.max_file_size = max_file_size
self.max_count = max_count
def task_before_update(mapper, connection, target):
target.last_activity = now()
if target.parent:
provider = TaskProvider(_session)
parent = provider.by_id(target.parent)
if parent is None:
return
parent.last_activity = target.last_activity
try:
provider.commit()
except StaleDataError:
pass
def stop_processes_not_exist(session: Session, logger):
provider = TaskProvider(session)
hostname = socket.gethostname()
tasks = provider.by_status(
TaskStatus.InProgress,
task_docker_assigned=DOCKER_IMG,
computer_assigned=hostname
)
# Kill processes which does not exist
hostname = socket.gethostname()
for t in tasks:
if not psutil.pid_exists(t.pid):
# tasks can retry the execution
if (now() - t.last_activity).total_seconds() < 30:
continue
os.system(f'kill -9 {t.pid}')
t.status = TaskStatus.Failed.value
logger.error(
f'process with pid = {t.pid} does not exist. '
f'Set task to failed state',
ComponentType.WorkerSupervisor, hostname, t.id
)
provider.commit()
additional_info = yaml_load(t.additional_info)
for p in additional_info.get('child_processes', []):
logger.info(f'killing child process = {p}')
os.system(f'kill -9 {p}')
# Kill processes which exist but should not
processes = get_pid('worker ')
ids = [p['PID'] for p in processes]
tasks = provider.by_ids(ids)
tasks = {t.pid: t for t in tasks}
for p in processes:
pid = p['PID']
if pid in tasks:
task = tasks[pid]
if task.status in [TaskStatus.Stopped.value,
TaskStatus.Failed.value,
TaskStatus.Skipped.value]:
logger.info(f'Kill processes which exist but should not. '
f'Pid = {pid}')
os.system(f'kill -9 {pid}')
def work(self):
project = ProjectProvider(self.session).by_id(self.project)
self.info(f'Task = {self.train_task} child_task: {self.child_task}')
model = Model(
created=now(),
name=self.name,
project=self.project,
equations='',
fold=self.fold
)
provider = ModelProvider(self.session)
if self.train_task:
task_provider = TaskProvider(self.session)
dag_provider = DagProvider(self.session)
task = task_provider.by_id(self.train_task)
dag = dag_provider.by_id(task.dag)
task_dir = join(TASK_FOLDER, str(self.child_task or task.id))
# get log directory
config = yaml_load(dag.config)
executor_config = config['executors'][task.executor]
catalyst_config_file = executor_config['args']['config']
catalyst_config_file = join(task_dir, catalyst_config_file)
catalyst_config = yaml_load(file=catalyst_config_file)
catalyst_logdir = catalyst_config['args']['logdir']
model.score_local = task.score
src_log = f'{task_dir}/{catalyst_logdir}'
models_dir = join(MODEL_FOLDER, project.name)
os.makedirs(models_dir, exist_ok=True)
model_path_tmp = f'{src_log}/traced.pth'
traced = trace_model_from_checkpoint(src_log, self, file=self.file)
model_path = f'{models_dir}/{model.name}.pth'
model_weight_path = f'{models_dir}/{model.name}_weight.pth'
torch.jit.save(traced, model_path_tmp)
shutil.copy(model_path_tmp, model_path)
file = self.file = 'best_full'
shutil.copy(f'{src_log}/checkpoints/{file}.pth',
model_weight_path)
provider.add(model)
def create_base(self):
self.info('create_base')
if app.current_task:
app.current_task.update_state(state=states.SUCCESS)
app.control.revoke(app.current_task.request.id, terminate=True)
self.provider = TaskProvider(self.session)
self.library_provider = DagLibraryProvider(self.session)
self.storage = Storage(self.session)
self.task = self.provider.by_id(
self.id, joinedload(Task.dag_rel, innerjoin=True))
if not self.task:
raise Exception(f'task with id = {self.id} is not found')
self.dag = self.task.dag_rel
self.executor = None
self.hostname = socket.gethostname()
self.docker_img = DOCKER_IMG
self.worker_index = os.getenv('WORKER_INDEX', -1)
self.queue_personal = f'{self.hostname}_{self.docker_img}_' \
f'{self.worker_index}'
self.config = Config.from_yaml(self.dag.config)
self.executor_type = self.config['executors'][
self.task.executor]['type']
executor = self.config['executors'][self.task.executor]
env = {'MKL_NUM_THREADS': 1, 'OMP_NUM_THREADS': 1}
env.update(executor.get('env', {}))
for k, v in env.items():
os.environ[k] = str(v)
self.info(f'Set env. {k} = {v}')