def __init__(self,
session: Session,
task: Task,
layout: str,
part: str = 'valid',
name: str = 'img_segment',
max_img_size: Tuple[int, int] = None,
stack_type: str = 'vertical',
main_metric: str = 'dice',
plot_count: int = 0,
colors: List[Tuple] = None):
self.session = session
self.task = task
self.layout = layout
self.part = part
self.name = name or 'img_segment'
self.max_img_size = max_img_size
self.stack_type = stack_type
self.main_metric = main_metric
self.colors = colors
self.plot_count = plot_count
self.dag_provider = DagProvider(session)
self.report_provider = ReportProvider(session)
self.layout_provider = ReportLayoutProvider(session)
self.task_provider = TaskProvider(session)
self.report_img_provider = ReportImgProvider(session)
self.report_task_provider = ReportTasksProvider(session)
self.report_series_provider = ReportSeriesProvider(session)
self.project = self.task_provider.project(task.id).id
self.layout = self.layout_provider.by_name(layout)
self.layout_dict = yaml_load(self.layout.content)
self.create_base()
def report_update_layout_end():
data = request_data()
provider = ReportProvider(_write_session)
layout_provider = ReportLayoutProvider(_write_session)
provider.update_layout_end(data['id'], data['layout'],
layout_provider.all())
return provider.detail(data['id'])
def report_layouts():
data = request_data()
provider = ReportLayoutProvider(_read_session)
options = PaginatorOptions(**data['paginator'])
res = provider.get(data, options)
return res
def __init__(self,
session: Session,
task: Task,
layout: str,
part: str = 'valid',
name: str = 'img_classify',
max_img_size: Tuple[int, int] = None,
main_metric: str = 'accuracy',
plot_count: int = 0):
self.session = session
self.task = task
self.layout = layout
self.part = part
self.name = name or 'img_classify'
self.max_img_size = max_img_size
self.main_metric = main_metric
self.plot_count = plot_count
self.dag_provider = DagProvider(session)
self.report_provider = ReportProvider(session)
self.layout_provider = ReportLayoutProvider(session)
self.task_provider = TaskProvider(session)
self.report_img_provider = ReportImgProvider(session)
self.report_task_provider = ReportTasksProvider(session)
self.report_series_provider = ReportSeriesProvider(session)
self.project = self.task_provider.project(task.id).id
self.layout = self.layout_provider.by_name(layout)
self.layout_dict = yaml_load(self.layout.content)
def create_providers(self):
self.provider = TaskProvider(self.session)
self.report_provider = ReportProvider(self.session)
self.report_tasks_provider = ReportTasksProvider(self.session)
self.report_layout_provider = ReportLayoutProvider(self.session)
self.project_provider = ProjectProvider(self.session)
self.storage = Storage(self.session)
self.dag_provider = DagProvider(self.session)
def create_providers(self):
self.log_info('create_providers')
self.provider = TaskProvider(self.session)
self.report_provider = ReportProvider(self.session)
self.report_tasks_provider = ReportTasksProvider(self.session)
self.report_layout_provider = ReportLayoutProvider(self.session)
self.project_provider = ProjectProvider(self.session)
self.storage = Storage(self.session,
logger=self.logger,
component=self.component)
self.dag_provider = DagProvider(self.session)
def report_layout_edit():
data = request_data()
provider = ReportLayoutProvider(_write_session)
layout = provider.by_name(data['name'])
layout.last_modified = now()
if 'content' in data and data['content'] is not None:
data_loaded = yaml_load(data['content'])
ReportLayoutInfo(data_loaded)
layout.content = data['content']
if 'new_name' in data and data['new_name'] is not None:
layout.name = data['new_name']
provider.commit()
def report_add_end():
data = request_data()
provider = ReportProvider(_write_session)
layouts = ReportLayoutProvider(_write_session).all()
layout = layouts[data['layout']]
report = Report(name=data['name'],
project=data['project'],
config=yaml_dump(layout))
provider.add(report)
def upgrade(migrate_engine):
folder = os.path.dirname(__file__)
session = Session.create_session(connection_string=migrate_engine.url)
provider = ReportLayoutProvider(session)
try:
files = os.path.join(folder, '002', 'report_layout', '*.yml')
for path in glob(files):
name = str(os.path.basename(path).split('.')[0])
text = open(path).read()
provider.add(ReportLayout(name=name,
content=text,
last_modified=now()),
commit=False)
provider.commit()
except Exception:
provider.rollback()
raise
class SegmentationReportBuilder:
def __init__(self,
session: Session,
task: Task,
layout: str,
part: str = 'valid',
name: str = 'img_segment',
max_img_size: Tuple[int, int] = None,
stack_type: str = 'vertical',
main_metric: str = 'dice',
plot_count: int = 0,
colors: List[Tuple] = None):
self.session = session
self.task = task
self.layout = layout
self.part = part
self.name = name or 'img_segment'
self.max_img_size = max_img_size
self.stack_type = stack_type
self.main_metric = main_metric
self.colors = colors
self.plot_count = plot_count
self.dag_provider = DagProvider(session)
self.report_provider = ReportProvider(session)
self.layout_provider = ReportLayoutProvider(session)
self.task_provider = TaskProvider(session)
self.report_img_provider = ReportImgProvider(session)
self.report_task_provider = ReportTasksProvider(session)
self.report_series_provider = ReportSeriesProvider(session)
self.project = self.task_provider.project(task.id).id
self.layout = self.layout_provider.by_name(layout)
self.layout_dict = yaml_load(self.layout.content)
self.create_base()
def create_base(self):
report = Report(config=yaml_dump(self.layout_dict),
time=now(),
layout=self.layout.name,
project=self.project,
name=self.name)
self.report_provider.add(report)
self.report_task_provider.add(
ReportTasks(report=report.id, task=self.task.id))
self.task.report = report.id
self.task_provider.update()
def encode_pred(self, mask: np.array):
res = np.zeros((*mask.shape[1:], 3), dtype=np.uint8)
for i, c in enumerate(mask):
c = np.repeat(c[:, :, None], 3, axis=2)
color = self.colors[i] if self.colors is not None else (255, 255,
255)
res += (c * color).astype(np.uint8)
return res
def plot_mask(self, img: np.array, mask: np.array):
if len(img.shape) == 2:
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
img = img.astype(np.uint8)
mask = mask.astype(np.uint8)
for i, c in enumerate(mask):
contours, _ = cv2.findContours(c, cv2.RETR_LIST,
cv2.CHAIN_APPROX_NONE)
color = self.colors[i] if self.colors else (0, 255, 0)
for i in range(0, len(contours)):
cv2.polylines(img, contours[i], True, color, 2)
return img
def process_scores(self, scores):
for key, item in self.layout_dict['items'].items():
item['name'] = key
if item['type'] == 'series' and item['key'] in scores:
series = ReportSeries(name=item['name'],
value=scores[item['key']],
epoch=0,
time=now(),
task=self.task.id,
part='valid',
stage='stage1')
self.report_series_provider.add(series)
def process_pred(self,
imgs: np.array,
preds: dict,
targets: np.array = None,
attrs=None,
scores=None):
for key, item in self.layout_dict['items'].items():
item['name'] = key
if item['type'] != 'img_segment':
continue
report_imgs = []
dag = self.dag_provider.by_id(self.task.dag)
for i in range(len(imgs)):
if self.plot_count <= 0:
break
if targets is not None:
img = self.plot_mask(imgs[i], targets[i])
else:
img = imgs[i]
imgs_add = [img]
for key, value in preds.items():
imgs_add.append(self.encode_pred(value[i]))
for j in range(len(imgs_add)):
imgs_add[j] = resize_saving_ratio(imgs_add[j],
self.max_img_size)
if self.stack_type == 'horizontal':
img = np.hstack(imgs_add)
else:
img = np.vstack(imgs_add)
attr = attrs[i] if attrs else {}
score = None
if targets is not None:
score = scores[self.main_metric][i]
retval, buffer = cv2.imencode('.jpg', img)
report_img = ReportImg(group=item['name'],
epoch=0,
task=self.task.id,
img=buffer,
dag=self.task.dag,
part=self.part,
project=self.project,
score=score,
**attr)
self.plot_count -= 1
report_imgs.append(report_img)
dag.img_size += report_img.size
self.dag_provider.commit()
self.report_img_provider.bulk_save_objects(report_imgs)
def report_layout_remove():
data = request_data()
provider = ReportLayoutProvider(_write_session)
provider.remove(data['name'], key_column='name')
def report_layout_add():
data = request_data()
provider = ReportLayoutProvider(_write_session)
layout = ReportLayout(name=data['name'], content='', last_modified=now())
provider.add(layout)
def report_add_start():
return {
'projects': ProjectProvider(_read_session).get()['data'],
'layouts': ReportLayoutProvider(_read_session).get()['data']
}
class DagStandardBuilder:
def __init__(self,
session: Session,
config: dict,
debug: bool,
config_text: str = None,
upload_files: bool = True,
copy_files_from: int = None,
config_path: str = None,
control_reqs: bool = True,
logger=None,
component: ComponentType = None):
self.session = session
self.config = config
self.debug = debug
self.config_text = config_text
self.upload_files = upload_files
self.copy_files_from = copy_files_from
self.config_path = config_path
self.control_reqs = control_reqs
self.info = config['info']
self.layout_name = self.info.get('layout')
self.provider = None
self.report_provider = None
self.report_tasks_provider = None
self.report_layout_provider = None
self.storage = None
self.dag_provider = None
self.logger = logger
self.component = component
self.project = None
self.layouts = None
self.dag = None
self.dag_report_id = None
self.created = None
self.project_provider = None
def log_info(self, message: str):
if self.logger:
self.logger.info(message, self.component)
def create_providers(self):
self.log_info('create_providers')
self.provider = TaskProvider(self.session)
self.report_provider = ReportProvider(self.session)
self.report_tasks_provider = ReportTasksProvider(self.session)
self.report_layout_provider = ReportLayoutProvider(self.session)
self.project_provider = ProjectProvider(self.session)
self.storage = Storage(self.session,
logger=self.logger,
component=self.component)
self.dag_provider = DagProvider(self.session)
def load_base(self):
self.log_info('load_base')
project = self.project_provider.by_name(self.info['project'])
if project is None:
project = self.project_provider.add_project(self.info['project'])
self.project = project.id
self.layouts = self.report_layout_provider.all()
def create_report(self):
self.log_info('create_report')
self.dag_report_id = None
layout_name = self.layout_name
if layout_name:
if layout_name not in self.layouts:
raise Exception(f'Unknown layout = {layout_name}')
report = Report(config=yaml_dump(self.layouts[layout_name]),
name=self.info['name'],
project=self.project,
layout=layout_name)
self.report_provider.add(report)
self.dag_report_id = report.id
def create_dag(self):
self.log_info('create_dag')
dag = Dag(config=self.config_text or yaml_dump(self.config),
project=self.project,
name=self.info['name'],
docker_img=self.info.get('docker_img'),
type=DagType.Standard.value,
created=now(),
report=self.dag_report_id)
self.dag = self.dag_provider.add(dag)
def upload(self):
self.log_info('upload')
if self.upload_files:
folder = os.path.dirname(os.path.abspath(self.config_path))
if 'expdir' in self.config['info']:
path = os.path.dirname(os.path.abspath(self.config_path))
folder = os.path.abspath(
os.path.join(path, self.config['info']['expdir']))
self.storage.upload(folder,
self.dag,
control_reqs=self.control_reqs)
elif self.copy_files_from:
self.storage.copy_from(self.copy_files_from, self.dag)
def create_task(self, k: str, v: dict, name: str, info: dict):
task_type = TaskType.User.value
if v.get('task_type') == 'train' or \
Executor.is_trainable(v['type']):
task_type = TaskType.Train.value
gpu = str(v.get('gpu', '0'))
if '-' not in gpu:
gpu = int(gpu)
gpu_max = gpu
else:
gpu, gpu_max = map(int, gpu.split('-'))
if gpu == 0 and gpu_max > 0:
raise Exception(f'Executor = {k} Gpu_max can"t be>0 when gpu=0')
task = Task(name=name,
executor=k,
computer=self.info.get('computer') or v.get('computer'),
gpu=gpu,
gpu_max=gpu_max,
cpu=v.get('cpu', 1),
memory=v.get('memory', 0.1),
dag=self.dag.id,
debug=self.debug,
steps=int(v.get('steps', '1')),
type=task_type)
task.additional_info = yaml_dump(info)
report = None
if self.layout_name and task_type == TaskType.Train.value:
if self.layout_name not in self.layouts:
raise Exception(f'Unknown report = {v["report"]}')
report_config = self.layouts[self.layout_name]
info['report_config'] = report_config
task.additional_info = yaml_dump(info)
report = Report(config=yaml_dump(report_config),
name=task.name,
project=self.project,
layout=self.layout_name)
return task, report
def create_tasks(self):
self.log_info('create_tasks')
created = OrderedDict()
executors = self.config['executors']
tasks = []
dependencies = []
reports = []
while len(created) < len(executors):
for k, v in executors.items():
valid = True
if 'depends' in v:
depends = v['depends']
if not isinstance(depends, list):
depends = [depends]
for d in depends:
if d == k:
raise Exception(f'Executor {k} depends on itself')
if d not in executors:
raise Exception(f'Executor {k} depend on {d} '
f'which does not exist')
valid = valid and d in created
if valid:
names = []
infos = []
if 'grid' in v:
grid = v['grid']
cells = grid_cells(grid)
for i, (cell, cell_name) in enumerate(cells):
names.append(cell_name)
infos.append({'grid_cell': i})
else:
names.append(v.get('name', k))
infos.append({})
k_tasks = []
for name, info in zip(names, infos):
task, report = self.create_task(k,
v,
name=name,
info=info)
tasks.append(task)
k_tasks.append(task)
reports.append(report)
if 'depends' in v:
depends = v['depends']
if not isinstance(depends, list):
depends = [depends]
for d in depends:
for dd in created[d]:
dependencies.append((task, dd))
created[k] = k_tasks
not_empty_reports = [r for r in reports if r is not None]
if len(not_empty_reports) > 0:
self.provider.bulk_save_objects(not_empty_reports,
return_defaults=True)
for report, task in zip(reports, tasks):
if report is not None:
task.report = report.id
self.provider.bulk_save_objects(tasks, return_defaults=True)
if len(not_empty_reports) > 0:
report_tasks = []
for report, task in zip(reports, tasks):
if report is not None:
report_tasks.append(
ReportTasks(report=report.id, task=task.id))
self.report_tasks_provider.bulk_save_objects(report_tasks)
dependencies = [
TaskDependence(task_id=task.id, depend_id=dd.id)
for task, dd in dependencies
]
self.provider.bulk_save_objects(dependencies)
for k, v in created.items():
created[k] = [vv.id for vv in v]
self.created = created
def build(self):
self.create_providers()
self.load_base()
self.create_report()
self.create_dag()
self.upload()
self.create_tasks()
self.log_info('Done')
return self.created
class ClassificationReportBuilder:
def __init__(self,
session: Session,
task: Task,
layout: str,
part: str = 'valid',
name: str = 'img_classify',
max_img_size: Tuple[int, int] = None,
main_metric: str = 'accuracy',
plot_count: int = 0):
self.session = session
self.task = task
self.layout = layout
self.part = part
self.name = name or 'img_classify'
self.max_img_size = max_img_size
self.main_metric = main_metric
self.plot_count = plot_count
self.dag_provider = DagProvider(session)
self.report_provider = ReportProvider(session)
self.layout_provider = ReportLayoutProvider(session)
self.task_provider = TaskProvider(session)
self.report_img_provider = ReportImgProvider(session)
self.report_task_provider = ReportTasksProvider(session)
self.report_series_provider = ReportSeriesProvider(session)
self.project = self.task_provider.project(task.id).id
self.layout = self.layout_provider.by_name(layout)
self.layout_dict = yaml_load(self.layout.content)
def create_base(self):
report = Report(config=yaml_dump(self.layout_dict),
time=now(),
layout=self.layout.name,
project=self.project,
name=self.name)
self.report_provider.add(report)
self.report_task_provider.add(
ReportTasks(report=report.id, task=self.task.id))
self.task.report = report.id
self.task_provider.update()
def process_scores(self, scores):
for key, item in self.layout_dict['items'].items():
item['name'] = key
if item['type'] == 'series' and item['key'] in scores:
series = ReportSeries(name=item['name'],
value=float(scores[item['key']]),
epoch=0,
time=now(),
task=self.task.id,
part='valid',
stage='stage1')
self.report_series_provider.add(series)
def process_pred(self,
imgs: np.array,
preds: np.array,
targets: np.array = None,
attrs=None,
scores=None):
for key, item in self.layout_dict['items'].items():
item['name'] = key
if item['type'] != 'img_classify':
continue
report_imgs = []
dag = self.dag_provider.by_id(self.task.dag)
for i in range(len(imgs)):
if self.plot_count <= 0:
break
img = resize_saving_ratio(imgs[i], self.max_img_size)
pred = preds[i]
attr = attrs[i] if attrs else {}
y = None
score = None
if targets is not None:
y = targets[i]
score = float(scores[self.main_metric][i])
y_pred = pred.argmax()
retval, buffer = cv2.imencode('.jpg', img)
report_img = ReportImg(group=item['name'],
epoch=0,
task=self.task.id,
img=buffer,
dag=self.task.dag,
part=self.part,
project=self.project,
y_pred=y_pred,
y=y,
score=score,
**attr)
report_imgs.append(report_img)
dag.img_size += report_img.size
self.dag_provider.commit()
self.report_img_provider.bulk_save_objects(report_imgs)
if targets is not None and item.get('confusion_matrix'):
matrix = confusion_matrix(targets,
preds.argmax(axis=1),
labels=np.arange(preds.shape[1]))
matrix = np.array(matrix)
c = {'data': matrix}
obj = ReportImg(group=item['name'] + '_confusion',
epoch=0,
task=self.task.id,
img=pickle.dumps(c),
project=self.project,
dag=self.task.dag,
part=self.part)
self.report_img_provider.add(obj)
self.plot_count -= 1
def downgrade(migrate_engine):
session = Session.create_session(connection_string=migrate_engine.url)
provider = ReportLayoutProvider(session)
provider.session.query(ReportLayout).delete(synchronize_session=False)
provider.session.commit()