import os
import supervisely_lib as sly
WORKSPACE_ID = %%WORKSPACE_ID%%
src_project_name = '%%IN_PROJECT_NAME%%'
api = sly.Api(server_address=os.environ['SERVER_ADDRESS'], token=os.environ['API_TOKEN'])
#### End settings. ####
sly.logger.info('DOWNLOAD_PROJECT', extra={'title': src_project_name})
project = api.project.get_info_by_name(WORKSPACE_ID, src_project_name)
sly.download_project(
api, project.id, os.path.join(sly.TaskPaths.OUT_ARTIFACTS_DIR, src_project_name), log_progress=True)
sly.logger.info('Project {!r} has been successfully downloaded.'.format(src_project_name))
crops_per_image = %%crops_per_image:5%%
min_crop_side_fraction = %%min_crop_side_fraction:0.6%%
max_crop_side_fraction = %%max_crop_side_fraction:0.9%%
train_tag_name = '%%train_tag_name:train%%'
val_tag_name = '%%val_tag_name:val%%'
#### End settings. ####
# Download remote project
src_project_info = api.project.get_info_by_name(WORKSPACE_ID, src_project_name)
src_project_dir = os.path.join(sly.TaskPaths.DATA_DIR, src_project_name)
sly.logger.info('DOWNLOAD_PROJECT', extra={'title': src_project_name})
sly.download_project(api, src_project_info.id, src_project_dir, log_progress=True)
sly.logger.info('Project {!r} has been successfully downloaded. Starting to process.'.format(src_project_name))
src_project = sly.Project(directory=src_project_dir, mode=sly.OpenMode.READ)
dst_project_dir = os.path.join(sly.TaskPaths.OUT_PROJECTS_DIR, dst_project_name)
dst_project = sly.Project(directory=dst_project_dir, mode=sly.OpenMode.CREATE)
tag_meta_train = sly.TagMeta(train_tag_name, sly.TagValueType.NONE)
tag_meta_val = sly.TagMeta(val_tag_name, sly.TagValueType.NONE)
bbox_class_mapping = {
obj_class.name: (
obj_class if (obj_class.geometry_type == sly.Rectangle)
else sly.ObjClass(obj_class.name + '_bbox', sly.Rectangle, color=obj_class.color))
for obj_class in src_project.meta.obj_classes}
def export_as_masks(api: sly.Api, task_id, context, state, app_logger):
project_info = api.project.get_info_by_id(PROJECT_ID)
dataset_ids = [ds.id for ds in api.dataset.get_list(PROJECT_ID)]
sly.logger.info('DOWNLOAD_PROJECT', extra={'title': project_info.name})
dest_dir = os.path.join(my_app.data_dir,
f'{project_info.id}_{project_info.name}')
sly.download_project(api,
project_info.id,
dest_dir,
dataset_ids=dataset_ids,
log_progress=True)
sly.logger.info(
'Project {!r} has been successfully downloaded. Starting to render masks.'
.format(project_info.name))
if MACHINE_MASKS is True or HUMAN_MASKS is True:
project = sly.Project(directory=dest_dir, mode=sly.OpenMode.READ)
if MACHINE_MASKS:
machine_colors = {
obj_class.name: [idx, idx, idx]
for idx, obj_class in enumerate(project.meta.obj_classes,
start=1)
}
dump_json_file(machine_colors,
os.path.join(dest_dir,
'obj_class_to_machine_color.json'),
indent=2)
for dataset in project:
ds_progress = sly.Progress('Processing dataset: {!r}/{!r}'.format(
project.name, dataset.name),
total_cnt=len(dataset))
if HUMAN_MASKS is True:
human_masks_dir = os.path.join(dataset.directory,
'masks_human')
sly.fs.mkdir(human_masks_dir)
if MACHINE_MASKS is True:
machine_masks_dir = os.path.join(dataset.directory,
'masks_machine')
sly.fs.mkdir(machine_masks_dir)
for item_name in dataset:
item_paths = dataset.get_item_paths(item_name)
ann = sly.Annotation.load_json_file(item_paths.ann_path,
project.meta)
mask_img_name = os.path.splitext(item_name)[0] + '.png'
raw_img = sly.image.read(item_paths.img_path)
raw_img_rendered = raw_img.copy()
if HUMAN_MASKS is True:
for label in ann.labels:
label.geometry.draw(
raw_img_rendered,
color=label.obj_class.color,
config=label.obj_class.geometry_config,
thickness=THICKNESS)
raw_img_rendered = ((raw_img_rendered.astype(np.uint16) +
raw_img.astype(np.uint16)) /
2).astype(np.uint8)
sly.image.write(
os.path.join(human_masks_dir, mask_img_name),
np.concatenate([raw_img, raw_img_rendered], axis=1))
if MACHINE_MASKS is True:
machine_mask = np.zeros(shape=ann.img_size + (3, ),
dtype=np.uint8)
for label in ann.labels:
label.geometry.draw(
machine_mask,
color=machine_colors[label.obj_class.name],
thickness=THICKNESS)
sly.image.write(
os.path.join(machine_masks_dir, mask_img_name),
machine_mask)
ds_progress.iter_done_report()
sly.logger.info('Finished masks rendering.'.format(project_info.name))
full_archive_name = str(project_info.id) + '_' + project_info.name + '.tar'
result_archive = os.path.join(my_app.data_dir, full_archive_name)
sly.fs.archive_directory(dest_dir, result_archive)
app_logger.info("Result directory is archived")
upload_progress = []
remote_archive_path = "/Export-as-masks/{}_{}".format(
task_id, full_archive_name)
def _print_progress(monitor, upload_progress):
if len(upload_progress) == 0:
upload_progress.append(
sly.Progress(message="Upload {!r}".format(full_archive_name),
total_cnt=monitor.len,
ext_logger=app_logger,
is_size=True))
upload_progress[0].set_current_value(monitor.bytes_read)
file_info = api.file.upload(TEAM_ID, result_archive, remote_archive_path,
lambda m: _print_progress(m, upload_progress))
app_logger.info("Uploaded to Team-Files: {!r}".format(
file_info.full_storage_url))
api.task.set_output_archive(task_id,
file_info.id,
full_archive_name,
file_url=file_info.full_storage_url)
my_app.stop()
max_crop_side_fraction = float('%%max_crop_side_fraction:0.9%%')
train_tag_name = '%%train_tag_name:train%%'
val_tag_name = '%%val_tag_name:val%%'
# Set to empty string to disable adding a background object.
background_class_name = '%%background_class_name:bg%%'
#### End settings. ####
# Download remote project
src_project_info = api.project.get_info_by_name(WORKSPACE_ID, src_project_name)
src_project_dir = os.path.join(sly.TaskPaths.DATA_DIR, src_project_name)
sly.logger.info('DOWNLOAD_PROJECT', extra={'title': src_project_name})
sly.download_project(api, src_project_info.id, src_project_dir, dataset_ids=src_dataset_ids, log_progress=True)
sly.logger.info('Project {!r} has been successfully downloaded. Starting to process.'.format(src_project_name))
src_project = sly.Project(directory=src_project_dir, mode=sly.OpenMode.READ)
dst_project_dir = os.path.join(sly.TaskPaths.OUT_PROJECTS_DIR, dst_project_name)
dst_project = sly.Project(directory=dst_project_dir, mode=sly.OpenMode.CREATE)
tag_meta_train = sly.TagMeta(train_tag_name, sly.TagValueType.NONE)
tag_meta_val = sly.TagMeta(val_tag_name, sly.TagValueType.NONE)
class_bg = None
if background_class_name != '':
class_bg = sly.ObjClass(background_class_name, sly.Rectangle, color=[0, 0, 64])
dst_meta = src_project.meta.add_tag_metas([tag_meta_train, tag_meta_val]).add_obj_class(class_bg)
import os
import supervisely_lib as sly
WORKSPACE_ID = int('%%WORKSPACE_ID%%')
src_project_name = '%%IN_PROJECT_NAME%%'
src_dataset_ids = %%DATASET_IDS:None%%
api = sly.Api(server_address=os.environ['SERVER_ADDRESS'], token=os.environ['API_TOKEN'])
#### End settings. ####
sly.logger.info('DOWNLOAD_PROJECT', extra={'title': src_project_name})
project = api.project.get_info_by_name(WORKSPACE_ID, src_project_name)
download_dir = os.path.join(sly.TaskPaths.OUT_ARTIFACTS_DIR, src_project_name)
sly.download_project(api, project.id, download_dir, dataset_ids=src_dataset_ids, log_progress=True, batch_size=10)
sly.logger.info('Project {!r} has been successfully downloaded.'.format(src_project_name))
import os
import supervisely_lib as sly
WORKSPACE_ID = int('%%WORKSPACE_ID%%')
src_project_name = '%%IN_PROJECT_NAME%%'
src_dataset_ids = '%%DATASET_IDS:None%%'
api = sly.Api(server_address=os.environ['SERVER_ADDRESS'],
token=os.environ['API_TOKEN'])
#### End settings. ####
sly.logger.info('DOWNLOAD_PROJECT', extra={'title': src_project_name})
project = api.project.get_info_by_name(WORKSPACE_ID, src_project_name)
download_dir = os.path.join(sly.TaskPaths.OUT_ARTIFACTS_DIR, src_project_name)
sly.download_project(api,
project.id,
download_dir,
dataset_ids=src_dataset_ids,
log_progress=True)
sly.logger.info(
'Project {!r} has been successfully downloaded.'.format(src_project_name))
def train(api: sly.Api, task_id, context, state, app_logger):
try:
prepare_weights(state)
# prepare directory for original Supervisely project
project_dir = os.path.join(my_app.data_dir, "sly_project")
sly.fs.mkdir(project_dir, remove_content_if_exists=True
) # clean content for debug, has no effect in prod
# download and preprocess Sypervisely project (using cache)
download_progress = get_progress_cb("Download data (using cache)",
g.project_info.items_count * 2)
sly.download_project(api,
project_id,
project_dir,
cache=my_app.cache,
progress_cb=download_progress)
# preprocessing: transform labels to bboxes, filter classes, ...
sly.Project.to_detection_task(project_dir, inplace=True)
train_classes = state["selectedClasses"]
sly.Project.remove_classes_except(project_dir,
classes_to_keep=train_classes,
inplace=True)
if state["unlabeledImages"] == "ignore":
sly.Project.remove_items_without_objects(project_dir, inplace=True)
# split to train / validation sets (paths to images and annotations)
train_set, val_set = get_train_val_sets(project_dir, state)
verify_train_val_sets(train_set, val_set)
sly.logger.info(f"Train set: {len(train_set)} images")
sly.logger.info(f"Val set: {len(val_set)} images")
# prepare directory for data in YOLOv5 format (nn will use it for training)
train_data_dir = os.path.join(my_app.data_dir, "train_data")
sly.fs.mkdir(train_data_dir, remove_content_if_exists=True
) # clean content for debug, has no effect in prod
# convert Supervisely project to YOLOv5 format
progress_cb = get_progress_cb("Convert Supervisely to YOLOv5 format",
len(train_set) + len(val_set))
yolov5_format.transform(project_dir, train_data_dir, train_set,
val_set, progress_cb)
# init sys.argv for main training script
init_script_arguments(state, train_data_dir, g.project_info.name)
# start train script
api.app.set_field(task_id, "state.activeNames",
["labels", "train", "pred", "metrics"]) # "logs",
get_progress_cb("YOLOv5: Scanning data ", 1)(1)
train_yolov5.main()
# upload artifacts directory to Team Files
upload_artifacts(g.local_artifacts_dir, g.remote_artifacts_dir)
set_task_output()
except Exception as e:
msg = f"Oops! Something went wrong, please try again or contact tech support. Find more info in the app logs. Error: {repr(e)}"
sly.logger.error(msg, exc_info=True, extra={'exc_str': str(e)})
my_app.show_modal_window(msg, level="error")
api.app.set_field(task_id, "state.started", False)
# stop application
get_progress_cb("Finished, app is stopped automatically", 1)(1)
my_app.stop(wait=False)