def merge_annotations(self, an1, an2): # I could also think of an implementation merging an1 and an2, and flattening the lists of lists # that are obtained assert (len(an1) == 1 and len(an2) == 1) d1 = an1[0] d2 = an2[0] if (d1['class'] != 'video'): raise NotImplemented( 'mergefiles: Currently, only annotation files from video can be merged.' ) if (d2['class'] != d1['class']): raise CommandError( "mergefiles: Both annotation files have to be of the same type (%s vs. %s)." % (d1['class'], d2['class'])) if (d1['filename'] != d2['filename']): raise CommandError( 'mergefiles: Both annotation files must annotate the same video file.' ) assert (d1['frames'] != None) assert (d2['frames'] != None) frames1 = d1['frames'] frames2 = d2['frames'] # collect list of nums frameNums1 = set() for frame in frames1: frameNums1.add(frame['num']) # make frames2 accessible by frame number frameNums2 = dict() for frame in frames2: frameNums2[frame['num']] = frame for frame in frames1: num = frame['num'] # look for frame with same timestamp in frames2 if num in frameNums2: # update annotations frame['annotations'].extend(frameNums2[num]['annotations']) # append frames with nums only in frames2 to frames1 numsOnlyIn2 = set(frameNums2.keys()) - frameNums1 for key in numsOnlyIn2: frames1.append(frameNums2[key]) from operator import itemgetter frames1.sort(key=itemgetter('num')) return an1
def handle(self, *args, **options): if len(args) < 2: raise CommandError("Expect at least 2 arguments.") self.labeltool.loadAnnotations(args[0]) for filename in args[1:]: rel_filename = filename try: if not os.path.isabs(filename): rel_filename = os.path.relpath(filename, os.path.dirname(args[0])) except: pass _, ext = os.path.splitext(rel_filename) if (not options['image'] and ext.lower() in self.video_extensions) or options['video']: logger.debug("Adding video file: %s" % rel_filename) item = self.labeltool.addVideoFile(rel_filename) else: logger.debug("Adding image file: %s" % rel_filename) item = self.labeltool.addImageFile(rel_filename) if options['unlabeled']: item.setUnlabeled(True) self.labeltool.saveAnnotations(args[0])
def handle(self, *args, **options): if len(args) != 1: raise CommandError( "python slothx.py setDefaultPersonBikeTypeToLightMotor <JSON>") import json json_file = args[0] self.labeltool.loadAnnotations(args[0], popupErrorMsg=False) annos = self.labeltool.annotations() for index, frmAnno in enumerate(annos): anno = frmAnno['annotations'] if not anno: continue for objindex, objanno in enumerate(anno): objclass = objanno.get('class', '') if objclass.lower() == 'personbike': objtype = objanno.get('Type', '') if objtype.lower() == '': objanno['Type'] = 'lightmotor' print "set ..." print "anno = {}".format(anno) frmAnno['annotations'] = anno self.labeltool._container.save(annos, "C:\\Users\\zx\\Desktop\\convert.json")
def handle(self, *args, **options): if len(args) != 3: raise CommandError("Usage: %s" % self.args) input1, input2, output = args[:] LOG.info("merging %s and %s into %s" % (input1, input2, output)) LOG.debug("loading annotations from %s" % input1) container1 = self.labeltool._container_factory.create(input1) an1 = container1.load(input1) LOG.debug("loading annotations from %s" % input2) container2 = self.labeltool._container_factory.create(input2) an2 = container2.load(input2) LOG.debug("merging annotations of %s and %s" % (input1, input2)) an3 = self.merge_annotations(an1, an2) nanno = [] try: fid = [] for i in xrange(len(an3)): temp = an3[i]["filename"] toks = temp.split("_")[-1].split(".")[0] fid.append(int(toks)) sort_index = np.argsort(fid) for i in xrange(len(sort_index)): nanno.append(an3[sort_index[i]]) except: fid = [] nanno = sorted(an3, key=lambda x: x["filename"]) LOG.debug("saving annotations to %s" % output) out_container = self.labeltool._container_factory.create(output) # out_container.save(an3, output) out_container.save(nanno, output)
def handle(self, *args, **options): if len(args) != 1: raise CommandError("python slothx.py showstats <JSON>") import json json_file = os.path.join(args[0], args[0] + ".json") f = open(json_file, "r") anno = json.load(f) anno = anno['labels'] from sloth.conf import config classes = config._class_name count = [] for c in classes: count.append(0) for i in xrange(len(anno)): print anno[i]["filename"] for an in anno[i]["annotations"]: if (an[config.METADATA_LABELCLASS_TOKEN] == "Ignore"): continue ind = classes.index(an[config.METADATA_LABELCLASS_TOKEN]) count[ind] = count[ind] + 1 import cv2 img_sz = cv2.imread(os.path.join(args[0], anno[i]["filename"])) print "Image width = ", img_sz.shape[1], " height = ", img_sz.shape[0] print "#Images in", args[0], " = ", len(anno) for i in xrange(len(classes)): print count[i], classes[i] print "Total", sum(count), "objects"
def handle(self, *args, **options): if len(args) != 2: raise CommandError("convert: Expecting exactly 2 arguments.") input, output = args[:] logger.info("converting from %s to %s" % (input, output)) logger.debug("loading annotations from %s" % input) self.labeltool.loadAnnotations(input) logger.debug("saving annotations to %s" % output) self.labeltool.saveAnnotations(output)
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Expect exactly 1 argument.") template_dir = os.path.join(sloth.__path__[0], 'conf') config_template = os.path.join(template_dir, 'default_config.py') target = args[0] if os.path.exists(target) and not options['force']: sys.stderr.write("Error: %s exists. Use -f to overwrite.\n" % target) return try: shutil.copy(config_template, target) _make_writeable(target) except OSError as e: sys.stderr.write("Notice: Couldn't set permission bits on %s.\n" % target)
def handle(self, *args, **options): if len(args) != 3: raise CommandError("Usage: %s" % self.args) input1, input2, output = args[:] logger.info("merging %s and %s into %s" % (input1, input2, output)) logger.debug("loading annotations from %s" % input1) container1 = self.labeltool._container_factory.create(input1) an1 = container1.load(input1) logger.debug("loading annotations from %s" % input2) container2 = self.labeltool._container_factory.create(input2) an2 = container2.load(input2) logger.debug("merging annotations of %s and %s" % (input1, input2)) an3 = self.merge_annotations(an1, an2) logger.debug("saving annotations to %s" % output) containerOut = self.labeltool._container_factory.create(output) containerOut.save(an3, output)
def handle(self, *args, **options): if len(args) != 1: raise CommandError("python slothx.py stat <JSON_FILES_DIR>") import json data_path = args[0] if not os.path.isdir(data_path): QMessageBox.critical( None, config.GUI_CRITIAL_ERROR_TEXT, config.GUI_INVALID_PATH_TEXT.format(data_path)) sys.exit(1) data_path = os.path.normpath(data_path) jsonFiles = [] for root, dirs, files in os.walk(data_path): files = [fi for fi in files if (fi.endswith(".json"))] for name in sorted(files): jsonFiles.append(name) statisticForOneJsonFile(name)
def handle(self, *args, **options): if len(args) != 1: raise CommandError("python slothx.py removeempty <JSON>") import json json_file = os.path.join(args[0], args[0] + ".json") f = open(json_file, "r") parse_json = json.load(f) anno = parse_json['labels'] for i in xrange(len(anno)): uanno = [] for an in anno[i]["annotations"]: if (config.METADATA_LABELCLASS_TOKEN in an): uanno.append(an) anno[i]["annotations"] = uanno n_anno = [] for i in xrange(len(anno)): if (len(anno[i]["annotations"]) > 0): n_anno.append(anno[i]) else: os.remove(os.path.join(args[0], anno[i]["filename"])) json_data = { 'version': config.VERSION, 'versiondesc': config.VERSIONDESC, 'date': str(datetime.datetime.now()) } json_data.append(n_anno) f.close() f = open(json_file, "w") json.dump(json_data, f, indent=4, separators=(',', ': '), sort_keys=True) f.write("\n") print "before remove", len(anno) print "after remove", len(n_anno)
def handle(self, *args, **options): if len(args) < 3: raise CommandError( "Expect at least 3 arguments. Commands: sloth.py append 10 video1 imag2..." ) self.labeltool.loadAnnotations(args[0], popupErrorMsg=False) present_filenames = { os.path.join(self.labeltool._imgDir, a[config.ANNOTATION_FILENAME_TOKEN]) for a in self.labeltool.annotations() } frmInterval = args[1] for filename in args[2:]: if filename in present_filenames: LOG.info("Not adding file again: %s" % filename) continue _, ext = os.path.splitext(filename) if (not options[config.ANNOTATION_IMAGE_TOKEN] and ext.lower() in self.video_extensions ) or options[config.ANNOTATION_VIDEO_TOKEN]: LOG.debug("Adding video file: %s" % filename) item, errorForFileHasBeenAdded = self.labeltool.addVideoFile( filename, frmInterval) else: LOG.debug("Adding image file: %s" % filename) item, errorForFileHasBeenAdded = self.labeltool.addImageFile( filename) if item: present_filenames.add(filename) if options['unlabeled']: item.setUnlabeled(True) self.labeltool.saveAnnotations(args[0]) return
def handle(self, *args, **options): if len(args) != 1: raise CommandError("python slothx.py reoder <JSON>") import json import numpy json_file = os.path.join(args[0], args[0] + ".json") f = open(json_file, "r") parse_json = json.load(f) anno = parse_json['labels'] fid = [] for i in xrange(len(anno)): temp = anno[i]["filename"] toks = temp.split("_")[-1].split(".")[0] fid.append(int(toks)) sort_index = numpy.argsort(fid) nanno = [] for i in xrange(len(sort_index)): nanno.append(anno[sort_index[i]]) json_data = { 'version': config.VERSION, 'versiondesc': config.VERSIONDESC, 'date': str(datetime.datetime.now()) } json_data.append(nanno) f.close() f = open(json_file, "w") json.dump(json_data, f, indent=4, separators=(',', ': '), sort_keys=True) f.write("\n") print "reorder done"
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Expect exactly 1 argument.") self.labeltool.loadAnnotations(args[0]) pprint(self.labeltool.annotations())
def handle(self, *args, **options): if (len(args) != 3) and (len(args) != 4): raise CommandError( "python slothx.py split <SRC_JSON> <DST_JSONS_BASEFILENAME> <PIC_NUM_IN_EACH_FILE> <CopyMediaFilesToSubFolder>" ) import json print "args = {}".format(args) input, output_fbasename, picNum = args[:3] enableCopyMediaFilesToSubFolder = int(args[3]) if (len(args) == 4) else 0 # enableCopyMediaFilesToSubFolder= 1 picNum = int(picNum) LOG.info( "split annotation data from %s to %s with filter and picNum %d" % (input, output_fbasename, picNum)) outtopdir = os.path.dirname(output_fbasename) if not os.path.exists(outtopdir): print "Error: output dir \'{}\' doesn't exist!".format(outtopdir) # print "loading annotations from {}".format(input) container1 = self.labeltool._container_factory.create(input) # get fields of "label" key anno_org = container1.load(input) mediaDir = container1._imgdir # print "anno_org = {}".format(anno_org) unique_data = {} mediaFileIndex = 0 fidx = 0 fnum = 0 imgsAnnoList = [] topAnnoDict = {} icnt = 0 outpartdir = None for i in xrange(len(anno_org)): if anno_org[i][ config. METADATA_LABELCLASS_TOKEN] == config.ANNOTATION_VIDEO_TOKEN: framesAnno = anno_org[i].get('frames', None) fnum = len(framesAnno) if framesAnno else 0 frms = {} for p in xrange(0, fnum, picNum): topAnnoDict = { k: v for k, v in anno_org[i].iteritems() if k != 'frames' } topAnnoDict['frames'] = framesAnno[p:p + picNum] outJsonFileName = output_fbasename + "_{}.json".format( fidx) out_container = self.labeltool._container_factory.create( outJsonFileName) out_container.save([topAnnoDict], outJsonFileName, mediaDir) fidx += 1 elif anno_org[i][ config. METADATA_LABELCLASS_TOKEN] == config.ANNOTATION_IMAGE_TOKEN: framesAnno = anno_org[i].get('annotations', None) fnum = len(framesAnno) if framesAnno else 0 icnt += 1 print "icnt = {} ... append".format(icnt) imgsAnnoList.append(anno_org[i]) outpartdir = output_fbasename # + "{}".format(fidx) outpartdir_abs = os.path.abspath(outpartdir) pos = outpartdir_abs.rfind("\\") if not pos: pos = outpartdir_abs.rfind("/") outpartparentname = os.path.abspath(outpartdir)[pos + 1:] outpartparentname = outpartparentname + "{}".format(fidx) if not os.path.exists(outpartdir): os.makedirs(outpartdir) if enableCopyMediaFilesToSubFolder: mediafilepath = os.path.join( mediaDir, anno_org[i].get('filename', None)) if not os.path.exists(mediafilepath): mediafilepath = os.path.join(os.path.dirname(input), mediafilepath) mediafilename = os.path.basename(mediafilepath) # copy media files to partdir outpart_mediafilepath = os.path.join( outpartdir, mediafilename) print "copy {} => {} ...".format(mediafilepath, outpart_mediafilepath) shutil.copy(mediafilepath, outpart_mediafilepath) if ((icnt % picNum) == 0): print "icnt = {} ... dump".format(icnt) """ mediaAbsDir = os.path.abspath(mediaDir) print "outpartdir = {} mediaDir = {} ".format(outpartdir, mediaAbsDir) try: outpart_imgdir = os.path.join(os.path.relpath(outpartdir, mediaAbsDir), mediaDir) except: outpart_imgdir = os.path.join(os.path.relpath(os.path.splitdrive(mediaAbsDir)[0], mediaAbsDir), "../", outpartdir) """ outpart_imgdir = "." outJsonFileName = os.path.join( outpartdir, outpartparentname + ".json".format(fidx)) out_container = self.labeltool._container_factory.create( outJsonFileName) out_container.save(imgsAnnoList, outJsonFileName, outpart_imgdir) imgsAnnoList = [] fidx += 1 # write left data if imgsAnnoList: outpart_imgdir = "." outJsonFileName = os.path.join( outpartdir, outpartparentname + ".json".format(fidx)) out_container = self.labeltool._container_factory.create( outJsonFileName) out_container.save(imgsAnnoList, outJsonFileName, outpart_imgdir) imgsAnnoList = [] fidx += 1 return
def handle(self, *args, **options): if len(args) != 2: raise CommandError( "python slothx.py cleanduplicate <SRC_JSON> <DST_JSON>") import json # print "args = {}".format(args) input, output = args[:] LOG.info("Cleaning duplicate data in %s to %s" % (input, output)) # print "loading annotations from {}".format(input) container1 = self.labeltool._container_factory.create(input) anno = container1.load(input) # print "anno = {}".format(anno) unique_data = {} mediaFileIndex = 0 for i in xrange(len(anno)): if anno[i][ config. METADATA_LABELCLASS_TOKEN] == config.ANNOTATION_VIDEO_TOKEN: key = "video, {}".format( anno[i][config.ANNOTATION_FILENAME_TOKEN]) # print "unique_data = {}".format(unique_data) if (key in unique_data.keys()): frames_record = unique_data[key][0] # print "frames_record = {}".format(frames_record) else: frames_record = {} for frame in anno[i]['frames']: frmanno = frame.get('annotations', None) frmidx = int(frame['frmidx']) if (key in unique_data.keys()) and ( frmidx in frames_record.keys()): if not frmanno: continue # print "frmidx = {}".format(frmidx) # print "frames_record[{}] = {}".format(frmidx, frames_record[frmidx]) frames_record[frmidx][0] += frame['annotations'] else: # note that, we use frameidx as frameRecordOder frames_record[frmidx] = [frame['annotations'], frmidx] unique_data[key] = [frames_record, mediaFileIndex] # print "mediaFileIndex = {} : after unique_data = {}".format(mediaFileIndex, unique_data) mediaFileIndex += 1 elif anno[i][ config. METADATA_LABELCLASS_TOKEN] == config.ANNOTATION_IMAGE_TOKEN: key = "image, {}".format( anno[i][config.ANNOTATION_FILENAME_TOKEN]) image_record = anno[i].get('annotations', None) # print "image_record = {}".format(image_record) if key in unique_data.keys(): if not image_record: continue unique_data[key][0] += image_record # print u"unique_data[{}] = {}".format(key, unique_data[key]) else: unique_data[key] = [image_record, mediaFileIndex] mediaFileIndex += 1 # ============================================================== # convert unique_data to annotation style data # ============================================================== # unique_data example1 (video): # uique_data = {'video, E:/colorAnno/sz2/192.168.1.62_01_20160311174825130_1/192.168.1.62_01_20160311174825130_1.mp4': # [{0: [[], 0], # 450: [[{u'subclass': [{u'y': 651.8255869054859, u'height': 151.58734579197346, u'width': 149.90304194984037, u'uppercolor0': u'#863C05', u'x': 1087.2181300968764, u'uppercolor0Tag': u'Brown', u'class': u'upper'}, {u'height': 189.4841822399668, u'width': 117.90126894931268, u'lowercolor0': u'#02107F', u'lowercolor0Tag': u'SapphireBlue', u'y': 805.939388460659, u'x': 1088.0602820179429, u'class': u'lower'}], u'height': 421.0759605332596, u'width': 165.06177652903784, u'y': 581.0848255358983, u'x': 1072.059395517679, u'class': u'pedestrain'}], 3], # 5700: [[], 0], # 300: [[{u'subclass': [{u'y': 377, u'height': 77, u'width': 177, u'uppercolor0': u'#ff0000', u'x': 277, u'uppercolor0Tag': u'Red', u'class': u'upper'}], u'height': 55, u'width': 177.0, u'y': 377, u'x': 277, u'class': u'pedestrain'}, {u'subclass': [{u'y': 497.52663138185187, u'height': 134.98321571704503, u'width': 119.45417320092486, u'x': 547.1001132602356, u'uppercolor0': u'#0432FF', u'uppercolor1': u'#000000', u'uppercolor0Tag': u'Blue', u'uppercolor1Tag': u'Black', u'class': u'upper'}, {u'height': 187.5430519254519, u'width': 81.2288377766289, u'lowercolor0': u'#505050', u'lowercolor0Tag': u'DarkGray', u'y': 634.8989305629154, u'x': 568.6018644364021, u'class': u'lower'}], u'height': 382.850625108964, u'width': 122.44052753094797, u'y': 445.5640660394496, u'x': 546.502842394231, u'class': u'pedestrain'}], 2], # 500: [[], 4], # 150: [[{u'subclass': [{u'y': 388, u'height': 88, u'width': 188, u'uppercolor0': u'#ff0000', u'x': 288, u'uppercolor0Tag': u'Red', u'class': u'upper'}], u'height': 66, u'width': 166.0, u'y': 366, u'x': 266, u'class': u'pedestrain'}, {u'subclass': [{u'y': 488, u'height': 288, u'width': 288, u'uppercolor0': u'#ff0000', u'x': 388, u'uppercolor0Tag': u'Red', u'class': u'upper'}], u'height': 266, u'width': 266.0, u'y': 466, u'x': 366, u'class': u'pedestrain'}], 1], # 5850: [[{u'subclass': [{u'y': 155.31914893617042, u'height': 83.68794326241144, u'width': 48.936170212766, u'uppercolor0': u'#C4AE85', u'x': 817.7304964539018, u'uppercolor0Tag': u'Khaki', u'class': u'upper'}, {u'height': 119.85815602836894, u'width': 81.5602836879433, u'lowercolor0': u'#000000', u'lowercolor0Tag': u'Black', u'y': 241.13475177304994, u'x': 793.6170212765968, u'class': u'lower'}], u'height': 249.6453900709223, u'width': 100.00000000000011, u'y': 114.18439716312072, u'x': 778.7234042553201, u'class': u'pedestrain'}], 1], # 1500: [[{u'subclass': [{u'y': 627.0000000000002, u'height': 149.0000000000001, u'width': 114.00000000000011, u'uppercolor0': u'#000000', u'x': 815.0000000000003, u'uppercolor0Tag': u'Black', u'class': u'upper'}, {u'height': 216.0000000000001, u'width': 89.00000000000011, u'lowercolor0': u'#000000', u'lowercolor0Tag': u'Black', u'y': 777.0000000000003, u'x': 831.0000000000003, u'class': u'lower'}], u'height': 444.0000000000002, u'width': 123.00000000000011, u'y': 555.0000000000002, u'x': 812.0000000000003, u'class': u'pedestrain'}], 5] # }, 1]} # ============================================================== fileitemsList = [] # valuelist format: [[mediaFileAnnotations, mediaFileDisplayOrder, mediaFilePathWithPrefix] ...] valuelist = [[value[0], value[1], key] for key, value in unique_data.iteritems()] valuelist = sorted(valuelist, key=lambda d: d[1], reverse=0) for index, item in enumerate(valuelist): mediatype = item[2][0:5] mediafilename = item[2][7:] mediaFileAnnotations = item[0] if mediatype == config.ANNOTATION_VIDEO_TOKEN: fileitem = { config.ANNOTATION_FILENAME_TOKEN: mediafilename, config.METADATA_LABELCLASS_TOKEN: config.ANNOTATION_VIDEO_TOKEN, config.ANNOTATION_FRAMES_TOKEN: [], } # fitemlist format: [[frameAnnotations,frameRecordOrder, frmIdx] ...] fitemlist = [[ value[0], value[1], key ] for key, value in mediaFileAnnotations.iteritems()] fitemlist = sorted(fitemlist, key=lambda d: d[1], reverse=0) fileitem['frames'] = [{ 'annotations': fitem[0], config.ANNOTATION_VIDEO_FILE_FRAME_IDX_TOKEN: fitem[2], config.ANNOTATION_VIDEO_FILE_FRAME_TIMESTAMP_TOKEN: 0, config.METADATA_LABELCLASS_TOKEN: config.ANNOTATION_FRAME_TOKEN } for index, fitem in enumerate(fitemlist)] elif mediatype == config.ANNOTATION_IMAGE_TOKEN: fileitem = { config.ANNOTATION_FILENAME_TOKEN: mediafilename, config.METADATA_LABELCLASS_TOKEN: config.ANNOTATION_IMAGE_TOKEN, 'annotations': mediaFileAnnotations } fileitemsList.append(fileitem) # ============================================================== # dump annotation_style data to file # ============================================================== # print "saving annotations to {}".format(output) out_container = self.labeltool._container_factory.create(output) out_container.save(fileitemsList, output) return
def handle(self, *args, **options): if len(args) != 1: raise CommandError("python slothx.py create <IMG_DIR>") import json data_path = args[0] if not os.path.isdir(data_path): QMessageBox.critical( None, config.GUI_CRITIAL_ERROR_TEXT, config.GUI_INVALID_PATH_TEXT.format(data_path)) sys.exit(1) data_path = os.path.normpath(data_path) imgdir = "." # os.path.normpath(data_path) # Note that, it is better use relative path to json file as imagedir filebasename = os.path.splitext(os.path.basename(data_path))[0] # print filebasename out_json = os.path.join(data_path, "{}.json".format(filebasename)) # print out_json lines = [] for root, dirs, files in os.walk(data_path): files = [ fi for fi in files if (fi.endswith(".jpg") or fi.endswith(".png") or fi.endswith(".bmp") or fi.endswith(".png")) ] for name in sorted(files): lines.append(name) annotations = [] # save to current dir for i in xrange(len(lines)): record = { config.METADATA_LABELCLASS_TOKEN: config.ANNOTATION_IMAGE_TOKEN, config.ANNOTATION_FILENAME_TOKEN: lines[i], 'annotations': [] } # print "{}: filename {}".format(i, lines[i]) annotations.append(record) json_data = { 'version': config.VERSION, 'versiondesc': config.VERSIONDESC, 'date': str(datetime.datetime.now()), 'imgdir': imgdir, 'labels': annotations } f = open(out_json, "w") json.dump(json_data, f, indent=4, separators=(',', ': '), sort_keys=True, ensure_ascii=True, encoding='gbk') f.write("\n") msg = config.GUI_CREATE_JSON_FILE_SUCCESS_TEXT.format( imgdir.decode('gbk'), len(lines), out_json.decode('gbk'), out_json.decode('gbk')) QMessageBox.critical(None, config.GUI_SUCCESS_TEXT, msg)