def _load_ids(self):
if self._save_to_file_enabled and os.path.exists(self._ids_file):
self._ids_keys_map = json_load(self._ids_file, int_keys=True)
self._keys_ids_map = {
item['key']: id
for id, item in self._ids_keys_map.items()
}
def train(self):
completions = []
for f in self.iter_completions():
completions.append(json_load(f))
if self.ml_backends_connected:
for ml_backend in self.ml_backends:
ml_backend.train(completions, self)
def get(self, id):
if id in self.cache:
return self.cache[id]
else:
filename = os.path.join(self.path, str(id) + '.json')
if os.path.exists(filename):
data = json_load(filename)
self.cache[id] = data
return data
def train(self):
completions = []
for f in self.iter_completions():
completions.append(json_load(f))
train_status = False
if self.ml_backends_connected:
for ml_backend in self.ml_backends:
if ml_backend.connected:
ml_backend.train(completions, self)
train_status = True
return train_status
def __init__(self, **kwargs):
super(JSONStorage, self).__init__(**kwargs)
tasks = {}
if os.path.exists(self.path):
tasks = json_load(self.path, int_keys=True)
if len(tasks) == 0:
self.data = {}
elif isinstance(tasks, dict):
self.data = tasks
elif isinstance(self.data, list):
self.data = {int(task['id']): task for task in tasks}
self._save()
def __init__(self,
config_file,
checkpoint_file,
labels_file=None,
score_threshold=0.3,
device="cpu",
**kwargs):
"""
Load MMDetection model from config and checkpoint into memory.
(Check https://mmdetection.readthedocs.io/en/v1.2.0/GETTING_STARTED.html#high-level-apis-for-testing-images)
Optionally set mappings from COCO classes to target labels
:param config_file: Absolute path to MMDetection config file (e.g. /home/user/mmdetection/configs/faster_rcnn/faster_rcnn_r50_fpn_1x.py)
:param checkpoint_file: Absolute path MMDetection checkpoint file (e.g. /home/user/mmdetection/checkpoints/faster_rcnn_r50_fpn_1x_20181010-3d1b3351.pth)
:param labels_file: file with mappings from COCO labels to custom labels {"airplane": "Boeing"}
:param score_threshold: score threshold to wipe out noisy results
:param device: device (cpu, cuda:0, cuda:1, ...)
:param kwargs:
"""
super(MMDetection, self).__init__(**kwargs)
self.config_file = config_file
self.checkpoint_file = checkpoint_file
self.labels_file = labels_file
if self.labels_file and os.path.exists(self.labels_file):
self.label_map = json_load(self.labels_file)
else:
self.label_map = {}
(
self.from_name,
self.to_name,
self.value,
self.labels_in_config,
) = get_single_tag_keys(self.parsed_label_config, "RectangleLabels",
"Image")
schema = list(self.parsed_label_config.values())[0]
self.labels_in_config = set(self.labels_in_config)
# Collect label maps from `predicted_values="airplane,car"` attribute in <Label> tag
self.labels_attrs = schema.get("labels_attrs")
if self.labels_attrs:
for label_name, label_attrs in self.labels_attrs.items():
for predicted_value in label_attrs.get("predicted_values",
"").split(","):
self.label_map[predicted_value] = label_name
print("Load new model from: ", config_file, checkpoint_file)
self.model = init_detector(config_file, checkpoint_file, device=device)
self.score_thresh = score_threshold
def load_tasks(self):
self.tasks = {}
self.derived_input_schema = set()
tasks = json_load(self.config['input_path'])
if len(tasks) == 0:
logger.warning('No tasks loaded from ' + self.config['input_path'])
return
for task_id, task in tasks.items():
self.tasks[int(task_id)] = task
data_keys = set(task['data'].keys())
if not self.derived_input_schema:
self.derived_input_schema = data_keys
else:
self.derived_input_schema &= data_keys
print(str(len(self.tasks)) + ' tasks loaded from: ' + self.config['input_path'])
def create_project_dir(cls, project_name, args):
"""
Create project directory in args.root_dir/project_name, and initialize there all required files
If some files are missed, restore them from defaults.
If config files are specified by args, copy them in project directory
:param project_name:
:param args:
:return:
"""
dir = cls.get_project_dir(project_name, args)
if args.force:
delete_dir_content(dir)
os.makedirs(dir, exist_ok=True)
config = (json_load(args.config_path) if args.config_path else
json_load(find_file("default_config.json")))
def already_exists_error(what, path):
raise RuntimeError(
'{path} {what} already exists. Use "--force" option to recreate it.'
.format(path=path, what=what))
input_path = args.input_path or config.get("input_path")
# save label config
config_xml = "config.xml"
config_xml_path = os.path.join(dir, config_xml)
label_config_file = args.label_config or config.get("label_config")
if label_config_file:
copy2(label_config_file, config_xml_path)
print(label_config_file + " label config copied to " +
config_xml_path)
else:
if os.path.exists(config_xml_path) and not args.force:
already_exists_error("label config", config_xml_path)
if not input_path:
# create default config with polygons only if input data is not set
default_label_config = find_file(
"examples/image_polygons/config.xml")
copy2(default_label_config, config_xml_path)
print(default_label_config + " label config copied to " +
config_xml_path)
else:
with io.open(config_xml_path, mode="w") as fout:
fout.write("<View></View>")
print("Empty config has been created in " + config_xml_path)
config["label_config"] = config_xml
if args.source:
config["source"] = {
"type": args.source,
"path": args.source_path,
"params": args.source_params,
}
else:
# save tasks.json
tasks_json = "tasks.json"
tasks_json_path = os.path.join(dir, tasks_json)
if input_path:
tasks = cls._load_tasks(input_path, args, config_xml_path)
else:
tasks = {}
with io.open(tasks_json_path, mode="w") as fout:
json.dump(tasks, fout, indent=2)
config["input_path"] = tasks_json
config["source"] = {
"name": "Tasks",
"type": "tasks-json",
"path": os.path.abspath(tasks_json_path),
}
logger.debug(
"{tasks_json_path} input file with {n} tasks has been created from {input_path}"
.format(tasks_json_path=tasks_json_path,
n=len(tasks),
input_path=input_path))
if args.target:
config["target"] = {
"type": args.target,
"path": args.target_path,
"params": args.target_params,
}
else:
completions_dir = os.path.join(dir, "completions")
if os.path.exists(completions_dir) and not args.force:
already_exists_error("output dir", completions_dir)
if os.path.exists(completions_dir):
delete_dir_content(completions_dir)
print(completions_dir +
" output dir already exists. Clear it.")
else:
os.makedirs(completions_dir, exist_ok=True)
print(completions_dir + " output dir has been created.")
config["output_dir"] = "completions"
config["target"] = {
"name": "Completions",
"type": "completions-dir",
"path": os.path.abspath(completions_dir),
}
if "ml_backends" not in config or not isinstance(
config["ml_backends"], list):
config["ml_backends"] = []
if args.ml_backends:
for url in args.ml_backends:
config["ml_backends"].append(
cls._create_ml_backend_params(url, project_name))
if args.sampling:
config["sampling"] = args.sampling
if args.port:
config["port"] = args.port
if args.host:
config["host"] = args.host
if args.allow_serving_local_files:
config["allow_serving_local_files"] = True
if args.key_file and args.cert_file:
config["protocol"] = "https://"
config["cert"] = args.cert_file
config["key"] = args.key_file
if (hasattr(args, "web_gui_project_desc")
and args.web_gui_project_desc) or args.project_desc:
config[
"description"] = args.web_gui_project_desc or args.project_desc
# create config.json
config_json = "config.json"
config_json_path = os.path.join(dir, config_json)
if os.path.exists(config_json_path) and not args.force:
already_exists_error("config", config_json_path)
with io.open(config_json_path, mode="w") as f:
json.dump(config, f, indent=2)
print("")
print(
"Label Studio has been successfully initialized. Check project states in "
+ dir)
print("Start the server: label-studio start " + dir)
return dir
def create_project_dir(cls, project_name, args):
"""
Create project directory in args.root_dir/project_name, and initialize there all required files
If some files are missed, restore them from defaults.
If config files are specified by args, copy them in project directory
:param project_name:
:param args:
:return:
"""
dir = cls.get_project_dir(project_name, args)
os.makedirs(dir, exist_ok=True)
config = json_load(
args.config_path) if args.config_path else json_load(
find_file('default_config.json'))
def already_exists_error(what, path):
raise RuntimeError(
'{path} {what} already exists. Use "--force" option to recreate it.'
.format(path=path, what=what))
input_path = args.input_path or config.get('input_path')
# save label config
config_xml = 'config.xml'
config_xml_path = os.path.join(dir, config_xml)
label_config_file = args.label_config or config.get('label_config')
if label_config_file:
copy2(label_config_file, config_xml_path)
print(label_config_file + ' label config copied to ' +
config_xml_path)
else:
if os.path.exists(config_xml_path) and not args.force:
already_exists_error('label config', config_xml_path)
if not input_path:
# create default config with polygons only if input data is not set
default_label_config = find_file(
'examples/image_polygons/config.xml')
copy2(default_label_config, config_xml_path)
print(default_label_config + ' label config copied to ' +
config_xml_path)
else:
with io.open(config_xml_path, mode='w') as fout:
fout.write('<View></View>')
print('Empty config has been created in ' + config_xml_path)
config['label_config'] = config_xml
# save tasks.json
tasks_json = 'tasks.json'
tasks_json_path = os.path.join(dir, tasks_json)
if input_path:
tasks = cls._load_tasks(input_path, args, config_xml_path)
with io.open(tasks_json_path, mode='w') as fout:
json.dump(tasks, fout, indent=2)
print(tasks_json_path + ' input path has been created from ' +
input_path)
else:
if os.path.exists(tasks_json_path) and not args.force:
already_exists_error('input path', tasks_json_path)
with io.open(tasks_json_path, mode='w') as fout:
json.dump({}, fout)
print(tasks_json_path +
' input path has been created with empty tasks.')
config['input_path'] = tasks_json
# create completions dir
completions_dir = os.path.join(dir, 'completions')
if os.path.exists(completions_dir) and not args.force:
already_exists_error('output dir', completions_dir)
if os.path.exists(completions_dir):
delete_dir_content(completions_dir)
print(completions_dir + ' output dir already exists. Clear it.')
else:
os.makedirs(completions_dir, exist_ok=True)
print(completions_dir + ' output dir has been created.')
config['output_dir'] = 'completions'
if args.ml_backend_url:
if 'ml_backend' not in config or not isinstance(
config['ml_backend'], dict):
config['ml_backend'] = {}
config['ml_backend']['url'] = args.ml_backend_url
if args.ml_backend_name:
config['ml_backend']['name'] = args.ml_backend_name
else:
config['ml_backend']['name'] = str(uuid4())
# create config.json
config_json = 'config.json'
config_json_path = os.path.join(dir, config_json)
if os.path.exists(config_json_path) and not args.force:
already_exists_error('config', config_json_path)
with io.open(config_json_path, mode='w') as f:
json.dump(config, f, indent=2)
print('')
print(
'Label Studio has been successfully initialized. Check project states in '
+ dir)
print('Start the server: label-studio start ' + dir)
return dir
def get(self, id):
filename = os.path.join(self.path, str(id) + '.json')
if os.path.exists(filename):
return json_load(filename)
def _get_objects(self):
self.data = json_load(self.path, int_keys=True)
return (str(id) for id in self.data)
def items(self):
for key in self.ids():
filename = os.path.join(self.path, str(key) + '.json')
yield key, json_load(filename)
def load_tasks(self):
self.tasks = json_load(self.config['input_path'])
self.tasks = {int(k): v for k, v in self.tasks.items()}
print(str(len(self.tasks)) + ' tasks loaded from: ' + self.config['input_path'])
def create_project_dir(cls, project_name, args):
"""
Create project directory in args.root_dir/project_name, and initialize there all required files
If some files are missed, restore them from defaults.
If config files are specified by args, copy them in project directory
:param project_name:
:param args:
:return:
"""
dir = cls.get_project_dir(project_name, args)
if args.force:
delete_dir_content(dir)
os.makedirs(dir, exist_ok=True)
config = json_load(
args.config_path) if args.config_path else json_load(
find_file('default_config.json'))
def already_exists_error(what, path):
raise RuntimeError(
'{path} {what} already exists. Use "--force" option to recreate it.'
.format(path=path, what=what))
input_path = args.input_path or config.get('input_path')
# save label config
config_xml = 'config.xml'
config_xml_path = os.path.join(dir, config_xml)
label_config_file = args.label_config or config.get('label_config')
if label_config_file:
copy2(label_config_file, config_xml_path)
print(label_config_file + ' label config copied to ' +
config_xml_path)
else:
if os.path.exists(config_xml_path) and not args.force:
already_exists_error('label config', config_xml_path)
if not input_path:
# create default config with polygons only if input data is not set
default_label_config = find_file(
'examples/image_polygons/config.xml')
copy2(default_label_config, config_xml_path)
print(default_label_config + ' label config copied to ' +
config_xml_path)
else:
with io.open(config_xml_path, mode='w') as fout:
fout.write('<View></View>')
print('Empty config has been created in ' + config_xml_path)
config['label_config'] = config_xml
if args.source:
config['source'] = {
'type': args.source,
'path': args.source_path,
'params': args.source_params
}
else:
# save tasks.json
tasks_json = 'tasks.json'
tasks_json_path = os.path.join(dir, tasks_json)
if input_path:
tasks = cls._load_tasks(input_path, args, config_xml_path)
else:
tasks = {}
with io.open(tasks_json_path, mode='w') as fout:
json.dump(tasks, fout, indent=2)
config['input_path'] = tasks_json
config['source'] = {
'name': 'Tasks',
'type': 'tasks-json',
'path': os.path.abspath(tasks_json_path)
}
logger.debug(
'{tasks_json_path} input file with {n} tasks has been created from {input_path}'
.format(tasks_json_path=tasks_json_path,
n=len(tasks),
input_path=input_path))
if args.target:
config['target'] = {
'type': args.target,
'path': args.target_path,
'params': args.target_params
}
else:
completions_dir = os.path.join(dir, 'completions')
if os.path.exists(completions_dir) and not args.force:
already_exists_error('output dir', completions_dir)
if os.path.exists(completions_dir):
delete_dir_content(completions_dir)
print(completions_dir +
' output dir already exists. Clear it.')
else:
os.makedirs(completions_dir, exist_ok=True)
print(completions_dir + ' output dir has been created.')
config['output_dir'] = 'completions'
config['target'] = {
'name': 'Completions',
'type': 'completions-dir',
'path': os.path.abspath(completions_dir)
}
if 'ml_backends' not in config or not isinstance(
config['ml_backends'], list):
config['ml_backends'] = []
if args.ml_backends:
for url in args.ml_backends:
config['ml_backends'].append(
cls._create_ml_backend_params(url, project_name))
if args.sampling:
config['sampling'] = args.sampling
if args.port:
config['port'] = args.port
if args.host:
config['host'] = args.host
if args.allow_serving_local_files:
config['allow_serving_local_files'] = True
if args.key_file and args.cert_file:
config['protocol'] = 'https://'
config['cert'] = args.cert_file
config['key'] = args.key_file
# create config.json
config_json = 'config.json'
config_json_path = os.path.join(dir, config_json)
if os.path.exists(config_json_path) and not args.force:
already_exists_error('config', config_json_path)
with io.open(config_json_path, mode='w') as f:
json.dump(config, f, indent=2)
print('')
print(
'Label Studio has been successfully initialized. Check project states in '
+ dir)
print('Start the server: label-studio start ' + dir)
return dir
def items(self):
for id in self.ids():
filename = os.path.join(self.path, str(id) + '.json')
yield id, self.cache[id] if id in self.cache else json_load(filename)
def train(self):
completions = []
for f in self.iter_completions():
completions.append(json_load(f))
self.ml_backend.train(completions, self)
