def _load_ids(self): if self._save_to_file_enabled and os.path.exists(self._ids_file): self._ids_keys_map = json_load(self._ids_file, int_keys=True) self._keys_ids_map = { item['key']: id for id, item in self._ids_keys_map.items() }
def train(self): completions = [] for f in self.iter_completions(): completions.append(json_load(f)) if self.ml_backends_connected: for ml_backend in self.ml_backends: ml_backend.train(completions, self)
def get(self, id): if id in self.cache: return self.cache[id] else: filename = os.path.join(self.path, str(id) + '.json') if os.path.exists(filename): data = json_load(filename) self.cache[id] = data return data
def train(self): completions = [] for f in self.iter_completions(): completions.append(json_load(f)) train_status = False if self.ml_backends_connected: for ml_backend in self.ml_backends: if ml_backend.connected: ml_backend.train(completions, self) train_status = True return train_status
def __init__(self, **kwargs): super(JSONStorage, self).__init__(**kwargs) tasks = {} if os.path.exists(self.path): tasks = json_load(self.path, int_keys=True) if len(tasks) == 0: self.data = {} elif isinstance(tasks, dict): self.data = tasks elif isinstance(self.data, list): self.data = {int(task['id']): task for task in tasks} self._save()
def __init__(self, config_file, checkpoint_file, labels_file=None, score_threshold=0.3, device="cpu", **kwargs): """ Load MMDetection model from config and checkpoint into memory. (Check https://mmdetection.readthedocs.io/en/v1.2.0/GETTING_STARTED.html#high-level-apis-for-testing-images) Optionally set mappings from COCO classes to target labels :param config_file: Absolute path to MMDetection config file (e.g. /home/user/mmdetection/configs/faster_rcnn/faster_rcnn_r50_fpn_1x.py) :param checkpoint_file: Absolute path MMDetection checkpoint file (e.g. /home/user/mmdetection/checkpoints/faster_rcnn_r50_fpn_1x_20181010-3d1b3351.pth) :param labels_file: file with mappings from COCO labels to custom labels {"airplane": "Boeing"} :param score_threshold: score threshold to wipe out noisy results :param device: device (cpu, cuda:0, cuda:1, ...) :param kwargs: """ super(MMDetection, self).__init__(**kwargs) self.config_file = config_file self.checkpoint_file = checkpoint_file self.labels_file = labels_file if self.labels_file and os.path.exists(self.labels_file): self.label_map = json_load(self.labels_file) else: self.label_map = {} ( self.from_name, self.to_name, self.value, self.labels_in_config, ) = get_single_tag_keys(self.parsed_label_config, "RectangleLabels", "Image") schema = list(self.parsed_label_config.values())[0] self.labels_in_config = set(self.labels_in_config) # Collect label maps from `predicted_values="airplane,car"` attribute in <Label> tag self.labels_attrs = schema.get("labels_attrs") if self.labels_attrs: for label_name, label_attrs in self.labels_attrs.items(): for predicted_value in label_attrs.get("predicted_values", "").split(","): self.label_map[predicted_value] = label_name print("Load new model from: ", config_file, checkpoint_file) self.model = init_detector(config_file, checkpoint_file, device=device) self.score_thresh = score_threshold
def load_tasks(self): self.tasks = {} self.derived_input_schema = set() tasks = json_load(self.config['input_path']) if len(tasks) == 0: logger.warning('No tasks loaded from ' + self.config['input_path']) return for task_id, task in tasks.items(): self.tasks[int(task_id)] = task data_keys = set(task['data'].keys()) if not self.derived_input_schema: self.derived_input_schema = data_keys else: self.derived_input_schema &= data_keys print(str(len(self.tasks)) + ' tasks loaded from: ' + self.config['input_path'])
def create_project_dir(cls, project_name, args): """ Create project directory in args.root_dir/project_name, and initialize there all required files If some files are missed, restore them from defaults. If config files are specified by args, copy them in project directory :param project_name: :param args: :return: """ dir = cls.get_project_dir(project_name, args) if args.force: delete_dir_content(dir) os.makedirs(dir, exist_ok=True) config = (json_load(args.config_path) if args.config_path else json_load(find_file("default_config.json"))) def already_exists_error(what, path): raise RuntimeError( '{path} {what} already exists. Use "--force" option to recreate it.' .format(path=path, what=what)) input_path = args.input_path or config.get("input_path") # save label config config_xml = "config.xml" config_xml_path = os.path.join(dir, config_xml) label_config_file = args.label_config or config.get("label_config") if label_config_file: copy2(label_config_file, config_xml_path) print(label_config_file + " label config copied to " + config_xml_path) else: if os.path.exists(config_xml_path) and not args.force: already_exists_error("label config", config_xml_path) if not input_path: # create default config with polygons only if input data is not set default_label_config = find_file( "examples/image_polygons/config.xml") copy2(default_label_config, config_xml_path) print(default_label_config + " label config copied to " + config_xml_path) else: with io.open(config_xml_path, mode="w") as fout: fout.write("<View></View>") print("Empty config has been created in " + config_xml_path) config["label_config"] = config_xml if args.source: config["source"] = { "type": args.source, "path": args.source_path, "params": args.source_params, } else: # save tasks.json tasks_json = "tasks.json" tasks_json_path = os.path.join(dir, tasks_json) if input_path: tasks = cls._load_tasks(input_path, args, config_xml_path) else: tasks = {} with io.open(tasks_json_path, mode="w") as fout: json.dump(tasks, fout, indent=2) config["input_path"] = tasks_json config["source"] = { "name": "Tasks", "type": "tasks-json", "path": os.path.abspath(tasks_json_path), } logger.debug( "{tasks_json_path} input file with {n} tasks has been created from {input_path}" .format(tasks_json_path=tasks_json_path, n=len(tasks), input_path=input_path)) if args.target: config["target"] = { "type": args.target, "path": args.target_path, "params": args.target_params, } else: completions_dir = os.path.join(dir, "completions") if os.path.exists(completions_dir) and not args.force: already_exists_error("output dir", completions_dir) if os.path.exists(completions_dir): delete_dir_content(completions_dir) print(completions_dir + " output dir already exists. Clear it.") else: os.makedirs(completions_dir, exist_ok=True) print(completions_dir + " output dir has been created.") config["output_dir"] = "completions" config["target"] = { "name": "Completions", "type": "completions-dir", "path": os.path.abspath(completions_dir), } if "ml_backends" not in config or not isinstance( config["ml_backends"], list): config["ml_backends"] = [] if args.ml_backends: for url in args.ml_backends: config["ml_backends"].append( cls._create_ml_backend_params(url, project_name)) if args.sampling: config["sampling"] = args.sampling if args.port: config["port"] = args.port if args.host: config["host"] = args.host if args.allow_serving_local_files: config["allow_serving_local_files"] = True if args.key_file and args.cert_file: config["protocol"] = "https://" config["cert"] = args.cert_file config["key"] = args.key_file if (hasattr(args, "web_gui_project_desc") and args.web_gui_project_desc) or args.project_desc: config[ "description"] = args.web_gui_project_desc or args.project_desc # create config.json config_json = "config.json" config_json_path = os.path.join(dir, config_json) if os.path.exists(config_json_path) and not args.force: already_exists_error("config", config_json_path) with io.open(config_json_path, mode="w") as f: json.dump(config, f, indent=2) print("") print( "Label Studio has been successfully initialized. Check project states in " + dir) print("Start the server: label-studio start " + dir) return dir
def create_project_dir(cls, project_name, args): """ Create project directory in args.root_dir/project_name, and initialize there all required files If some files are missed, restore them from defaults. If config files are specified by args, copy them in project directory :param project_name: :param args: :return: """ dir = cls.get_project_dir(project_name, args) os.makedirs(dir, exist_ok=True) config = json_load( args.config_path) if args.config_path else json_load( find_file('default_config.json')) def already_exists_error(what, path): raise RuntimeError( '{path} {what} already exists. Use "--force" option to recreate it.' .format(path=path, what=what)) input_path = args.input_path or config.get('input_path') # save label config config_xml = 'config.xml' config_xml_path = os.path.join(dir, config_xml) label_config_file = args.label_config or config.get('label_config') if label_config_file: copy2(label_config_file, config_xml_path) print(label_config_file + ' label config copied to ' + config_xml_path) else: if os.path.exists(config_xml_path) and not args.force: already_exists_error('label config', config_xml_path) if not input_path: # create default config with polygons only if input data is not set default_label_config = find_file( 'examples/image_polygons/config.xml') copy2(default_label_config, config_xml_path) print(default_label_config + ' label config copied to ' + config_xml_path) else: with io.open(config_xml_path, mode='w') as fout: fout.write('<View></View>') print('Empty config has been created in ' + config_xml_path) config['label_config'] = config_xml # save tasks.json tasks_json = 'tasks.json' tasks_json_path = os.path.join(dir, tasks_json) if input_path: tasks = cls._load_tasks(input_path, args, config_xml_path) with io.open(tasks_json_path, mode='w') as fout: json.dump(tasks, fout, indent=2) print(tasks_json_path + ' input path has been created from ' + input_path) else: if os.path.exists(tasks_json_path) and not args.force: already_exists_error('input path', tasks_json_path) with io.open(tasks_json_path, mode='w') as fout: json.dump({}, fout) print(tasks_json_path + ' input path has been created with empty tasks.') config['input_path'] = tasks_json # create completions dir completions_dir = os.path.join(dir, 'completions') if os.path.exists(completions_dir) and not args.force: already_exists_error('output dir', completions_dir) if os.path.exists(completions_dir): delete_dir_content(completions_dir) print(completions_dir + ' output dir already exists. Clear it.') else: os.makedirs(completions_dir, exist_ok=True) print(completions_dir + ' output dir has been created.') config['output_dir'] = 'completions' if args.ml_backend_url: if 'ml_backend' not in config or not isinstance( config['ml_backend'], dict): config['ml_backend'] = {} config['ml_backend']['url'] = args.ml_backend_url if args.ml_backend_name: config['ml_backend']['name'] = args.ml_backend_name else: config['ml_backend']['name'] = str(uuid4()) # create config.json config_json = 'config.json' config_json_path = os.path.join(dir, config_json) if os.path.exists(config_json_path) and not args.force: already_exists_error('config', config_json_path) with io.open(config_json_path, mode='w') as f: json.dump(config, f, indent=2) print('') print( 'Label Studio has been successfully initialized. Check project states in ' + dir) print('Start the server: label-studio start ' + dir) return dir
def get(self, id): filename = os.path.join(self.path, str(id) + '.json') if os.path.exists(filename): return json_load(filename)
def _get_objects(self): self.data = json_load(self.path, int_keys=True) return (str(id) for id in self.data)
def items(self): for key in self.ids(): filename = os.path.join(self.path, str(key) + '.json') yield key, json_load(filename)
def load_tasks(self): self.tasks = json_load(self.config['input_path']) self.tasks = {int(k): v for k, v in self.tasks.items()} print(str(len(self.tasks)) + ' tasks loaded from: ' + self.config['input_path'])
def create_project_dir(cls, project_name, args): """ Create project directory in args.root_dir/project_name, and initialize there all required files If some files are missed, restore them from defaults. If config files are specified by args, copy them in project directory :param project_name: :param args: :return: """ dir = cls.get_project_dir(project_name, args) if args.force: delete_dir_content(dir) os.makedirs(dir, exist_ok=True) config = json_load( args.config_path) if args.config_path else json_load( find_file('default_config.json')) def already_exists_error(what, path): raise RuntimeError( '{path} {what} already exists. Use "--force" option to recreate it.' .format(path=path, what=what)) input_path = args.input_path or config.get('input_path') # save label config config_xml = 'config.xml' config_xml_path = os.path.join(dir, config_xml) label_config_file = args.label_config or config.get('label_config') if label_config_file: copy2(label_config_file, config_xml_path) print(label_config_file + ' label config copied to ' + config_xml_path) else: if os.path.exists(config_xml_path) and not args.force: already_exists_error('label config', config_xml_path) if not input_path: # create default config with polygons only if input data is not set default_label_config = find_file( 'examples/image_polygons/config.xml') copy2(default_label_config, config_xml_path) print(default_label_config + ' label config copied to ' + config_xml_path) else: with io.open(config_xml_path, mode='w') as fout: fout.write('<View></View>') print('Empty config has been created in ' + config_xml_path) config['label_config'] = config_xml if args.source: config['source'] = { 'type': args.source, 'path': args.source_path, 'params': args.source_params } else: # save tasks.json tasks_json = 'tasks.json' tasks_json_path = os.path.join(dir, tasks_json) if input_path: tasks = cls._load_tasks(input_path, args, config_xml_path) else: tasks = {} with io.open(tasks_json_path, mode='w') as fout: json.dump(tasks, fout, indent=2) config['input_path'] = tasks_json config['source'] = { 'name': 'Tasks', 'type': 'tasks-json', 'path': os.path.abspath(tasks_json_path) } logger.debug( '{tasks_json_path} input file with {n} tasks has been created from {input_path}' .format(tasks_json_path=tasks_json_path, n=len(tasks), input_path=input_path)) if args.target: config['target'] = { 'type': args.target, 'path': args.target_path, 'params': args.target_params } else: completions_dir = os.path.join(dir, 'completions') if os.path.exists(completions_dir) and not args.force: already_exists_error('output dir', completions_dir) if os.path.exists(completions_dir): delete_dir_content(completions_dir) print(completions_dir + ' output dir already exists. Clear it.') else: os.makedirs(completions_dir, exist_ok=True) print(completions_dir + ' output dir has been created.') config['output_dir'] = 'completions' config['target'] = { 'name': 'Completions', 'type': 'completions-dir', 'path': os.path.abspath(completions_dir) } if 'ml_backends' not in config or not isinstance( config['ml_backends'], list): config['ml_backends'] = [] if args.ml_backends: for url in args.ml_backends: config['ml_backends'].append( cls._create_ml_backend_params(url, project_name)) if args.sampling: config['sampling'] = args.sampling if args.port: config['port'] = args.port if args.host: config['host'] = args.host if args.allow_serving_local_files: config['allow_serving_local_files'] = True if args.key_file and args.cert_file: config['protocol'] = 'https://' config['cert'] = args.cert_file config['key'] = args.key_file # create config.json config_json = 'config.json' config_json_path = os.path.join(dir, config_json) if os.path.exists(config_json_path) and not args.force: already_exists_error('config', config_json_path) with io.open(config_json_path, mode='w') as f: json.dump(config, f, indent=2) print('') print( 'Label Studio has been successfully initialized. Check project states in ' + dir) print('Start the server: label-studio start ' + dir) return dir
def items(self): for id in self.ids(): filename = os.path.join(self.path, str(id) + '.json') yield id, self.cache[id] if id in self.cache else json_load(filename)
def train(self): completions = [] for f in self.iter_completions(): completions.append(json_load(f)) self.ml_backend.train(completions, self)