def dag_pipe(session: Session, config: dict, config_text: str = None):
assert 'interfaces' in config, 'interfaces missed'
assert 'pipes' in config, 'pipe missed'
info = config['info']
storage = Storage(session)
dag_provider = DagProvider(session)
folder = os.getcwd()
project = ProjectProvider(session).by_name(info['project']).id
dag = dag_provider.add(
Dag(
config=config_text,
project=project,
name=info['name'],
docker_img=info.get('docker_img'),
type=DagType.Pipe.value
)
)
storage.upload(folder, dag)
# Change model dags which have the same name
ModelProvider(session
).change_dag(project=project, name=info['name'], to=dag.id)
def project_edit():
data = request_data()
provider = ProjectProvider(_write_session)
res = provider.edit_project(data['name'], yaml_load(data['class_names']),
yaml_load(data['ignore_folders']))
return res
def work(self):
project = ProjectProvider(self.session).by_id(self.project)
self.info(f'Task = {self.train_task} child_task: {self.child_task}')
model = Model(created=now(),
name=self.name,
project=self.project,
equations='',
fold=self.fold)
provider = ModelProvider(self.session)
if self.train_task:
task_provider = TaskProvider(self.session)
task = task_provider.by_id(self.train_task)
model.score_local = task.score
task_dir = join(TASK_FOLDER, str(self.child_task or task.id))
src_log = f'{task_dir}/log'
models_dir = join(MODEL_FOLDER, project.name)
os.makedirs(models_dir, exist_ok=True)
model_path_tmp = f'{src_log}/traced.pth'
traced = trace_model_from_checkpoint(src_log, self, file=self.file)
model_path = f'{models_dir}/{model.name}.pth'
model_weight_path = f'{models_dir}/{model.name}_weight.pth'
torch.jit.save(traced, model_path_tmp)
shutil.copy(model_path_tmp, model_path)
file = self.file = 'best_full'
shutil.copy(f'{src_log}/checkpoints/{file}.pth', model_weight_path)
provider.add(model)
def sync(project: str, computer: str, only_from: bool, only_to: bool):
_create_computer()
computer = computer or socket.gethostname()
provider = ComputerProvider(_session)
project_provider = ProjectProvider(_session)
computer = provider.by_name(computer)
computers = provider.all()
folders_excluded = []
p = project_provider.by_name(project)
assert p, f'Project={project} is not found'
ignore = yaml_load(p.ignore_folders)
excluded = []
for f in ignore:
excluded.append(str(f))
folders_excluded.append([join('data', p.name), excluded])
folders_excluded.append([join('models', p.name), []])
for c in computers:
if c.name != computer.name:
if not only_from:
sync_directed(_session, computer, c, folders_excluded)
if not only_to:
sync_directed(_session, c, computer, folders_excluded)
def sync_manual(self, computer: Computer, provider: ComputerProvider):
"""
button sync was clicked manually
"""
if not computer.meta:
return
meta = yaml_load(computer.meta)
if 'manual_sync' not in meta:
return
manual_sync = meta['manual_sync']
project_provider = ProjectProvider(self.session)
docker_provider = DockerProvider(self.session)
dockers = docker_provider.get_online()
project = project_provider.by_id(manual_sync['project'])
for docker in dockers:
if docker.computer == computer.name:
continue
source = provider.by_name(docker.computer)
ignore_folders = [
[join('models', project.name), []]
]
sync_directed(self.session, target=computer, source=source,
ignore_folders=ignore_folders)
del meta['manual_sync']
computer.meta = yaml_dump(meta)
provider.update()
def projects():
data = request_data()
options = PaginatorOptions(**data['paginator'])
provider = ProjectProvider(_read_session)
res = provider.get(data, options)
return res
def project_add():
data = request_data()
provider = ProjectProvider(_write_session)
provider.add_project(
data['name'], yaml_load(data['class_names']),
yaml_load(data['ignore_folders'])
)
def create_providers(self):
self.provider = TaskProvider(self.session)
self.report_provider = ReportProvider(self.session)
self.report_tasks_provider = ReportTasksProvider(self.session)
self.report_layout_provider = ReportLayoutProvider(self.session)
self.project_provider = ProjectProvider(self.session)
self.storage = Storage(self.session)
self.dag_provider = DagProvider(self.session)
def create_providers(self):
self.log_info('create_providers')
self.provider = TaskProvider(self.session)
self.report_provider = ReportProvider(self.session)
self.report_tasks_provider = ReportTasksProvider(self.session)
self.report_layout_provider = ReportLayoutProvider(self.session)
self.project_provider = ProjectProvider(self.session)
self.storage = Storage(self.session,
logger=self.logger,
component=self.component)
self.dag_provider = DagProvider(self.session)
def sync_manual(self, computer: Computer, provider: ComputerProvider):
"""
button sync was clicked manually
"""
if not computer.meta:
return
meta = yaml_load(computer.meta)
if 'manual_sync' not in meta:
return
manual_sync = meta['manual_sync']
project_provider = ProjectProvider(self.session)
docker_provider = DockerProvider(self.session)
dockers = docker_provider.get_online()
project = project_provider.by_id(manual_sync['project'])
sync_folders = manual_sync['sync_folders']
ignore_folders = manual_sync['ignore_folders']
sync_folders = correct_folders(sync_folders, project.name)
ignore_folders = correct_folders(ignore_folders, project.name)
if not isinstance(sync_folders, list):
sync_folders = []
if not isinstance(ignore_folders, list):
ignore_folders = []
for docker in dockers:
if docker.computer == computer.name:
continue
source = provider.by_name(docker.computer)
folders = [[s, ignore_folders] for s in sync_folders]
computer.syncing_computer = source.name
provider.update()
try:
sync_directed(
self.session,
target=computer,
source=source,
folders=folders
)
except Exception as e:
self.process_error(e)
del meta['manual_sync']
computer.meta = yaml_dump(meta)
provider.update()
def work(self):
project = ProjectProvider(self.session).by_id(self.project)
self.info(f'Task = {self.train_task} child_task: {self.child_task}')
model = Model(
created=now(),
name=self.name,
project=self.project,
equations='',
fold=self.fold
)
provider = ModelProvider(self.session)
if self.train_task:
task_provider = TaskProvider(self.session)
dag_provider = DagProvider(self.session)
task = task_provider.by_id(self.train_task)
dag = dag_provider.by_id(task.dag)
task_dir = join(TASK_FOLDER, str(self.child_task or task.id))
# get log directory
config = yaml_load(dag.config)
executor_config = config['executors'][task.executor]
catalyst_config_file = executor_config['args']['config']
catalyst_config_file = join(task_dir, catalyst_config_file)
catalyst_config = yaml_load(file=catalyst_config_file)
catalyst_logdir = catalyst_config['args']['logdir']
model.score_local = task.score
src_log = f'{task_dir}/{catalyst_logdir}'
models_dir = join(MODEL_FOLDER, project.name)
os.makedirs(models_dir, exist_ok=True)
model_path_tmp = f'{src_log}/traced.pth'
traced = trace_model_from_checkpoint(src_log, self, file=self.file)
model_path = f'{models_dir}/{model.name}.pth'
model_weight_path = f'{models_dir}/{model.name}_weight.pth'
torch.jit.save(traced, model_path_tmp)
shutil.copy(model_path_tmp, model_path)
file = self.file = 'best_full'
shutil.copy(f'{src_log}/checkpoints/{file}.pth',
model_weight_path)
provider.add(model)
def dag_model_add(session: Session, data: dict):
if not data.get('task'):
model = Model(name=data['name'],
project=data['project'],
equations=data['equations'],
created=now())
ModelProvider(session).add(model)
return
task_provider = TaskProvider(session)
task = task_provider.by_id(data['task'],
options=joinedload(Task.dag_rel,
innerjoin=True))
child_tasks = task_provider.children(task.id)
computer = task.computer_assigned
child_task = None
if len(child_tasks) > 0:
child_task = child_tasks[0].id
computer = child_tasks[0].computer_assigned
project = ProjectProvider(session).by_id(task.dag_rel.project)
config = {
'info': {
'name': 'model_add',
'project': project.name,
'computer': computer
},
'executors': {
'model_add': {
'type': 'model_add',
'project': data['project'],
'task': data.get('task'),
'name': data['name'],
'file': data['file'],
'child_task': child_task,
'fold': data['fold']
}
}
}
dag_standard(session=session,
config=config,
debug=False,
upload_files=False)
def sync(project: str, computer: str, only_from: bool, only_to: bool,
online: bool):
"""
Syncs specified project on this computer with other computers
"""
check_statuses()
_create_computer()
_create_docker()
computer = computer or socket.gethostname()
provider = ComputerProvider(_session)
project_provider = ProjectProvider(_session)
computer = provider.by_name(computer)
computers = provider.all_with_last_activtiy()
p = project_provider.by_name(project)
assert p, f'Project={project} is not found'
sync_folders = yaml_load(p.sync_folders)
ignore_folders = yaml_load(p.ignore_folders)
sync_folders = correct_folders(sync_folders, p.name)
ignore_folders = correct_folders(ignore_folders, p.name)
if not isinstance(sync_folders, list):
sync_folders = []
if not isinstance(ignore_folders, list):
ignore_folders = []
folders = [[s, ignore_folders] for s in sync_folders]
for c in computers:
if c.name != computer.name:
if online and (now() - c.last_activity).total_seconds() > 100:
continue
if not only_from:
sync_directed(_session, computer, c, folders)
if not only_to:
sync_directed(_session, c, computer, folders)
def dag_model_add(session: Session, data: dict):
task_provider = TaskProvider(session)
task = task_provider.by_id(data['task'],
options=joinedload(Task.dag_rel,
innerjoin=True))
child_tasks = task_provider.children(task.id)
computer = task.computer_assigned
child_task = None
if len(child_tasks) > 0:
child_task = child_tasks[0].id
computer = child_tasks[0].computer_assigned
project = ProjectProvider(session).by_id(task.dag_rel.project)
interface_params = data.get('interface_params', '')
interface_params = yaml_load(interface_params)
config = {
'info': {
'name': 'model_add',
'project': project.name,
'computer': computer
},
'executors': {
'model_add': {
'type': 'model_add',
'dag': data['dag'],
'slot': data['slot'],
'interface': data['interface'],
'task': data.get('task'),
'name': data['name'],
'interface_params': interface_params,
'child_task': child_task
}
}
}
dag_standard(session=session,
config=config,
debug=False,
upload_files=False)
def project_remove():
id = request_data()['id']
ProjectProvider(_write_session).remove(id)
def report_add_start():
return {
'projects': ProjectProvider(_read_session).get()['data'],
'layouts': ReportLayoutProvider(_read_session).get()['data']
}
def test_add(self, session: Session):
provider = ProjectProvider(session)
project = provider.add_project(name='test')
assert provider.by_id(project.id)
def _configure(self, session):
provider = ProjectProvider(session)
provider.add_project(name='test')
return provider