def copyAnnotation(self, token, name):
""" copy an annotation with a given name from a given token """
if token.hasAnnotation(name) == True and self.hasAnnotation(name) == False:
annotation = token.annotations.get(name)
newAnnotation = Annotation(name)
newAnnotation.copy(annotation)
self.annotations.add(newAnnotation)
def query_article(self, article):
escaped = self.escape_sparql(article)
ret = []
query = """
SELECT DISTINCT ?author ?author_fullname ?author_email
?date ?label ?type ?body_s ?body_p ?body_o ?body_l
?target_start ?target_startoffset ?target_endoffset
WHERE {
?annotation rdf:type oa:Annotation ;
oa:annotatedAt ?date ;
oa:annotatedBy ?author .
OPTIONAL { ?author foaf:name ?author_fullname }
OPTIONAL { ?author schema:email ?author_email }
OPTIONAL { ?annotation rdfs:label ?label }
OPTIONAL { ?annotation ao:type ?type }
OPTIONAL { ?annotation oa:hasBody ?body }
OPTIONAL { ?body rdf:subject ?body_s }
OPTIONAL { ?body rdf:predicate ?body_p }
OPTIONAL { ?body rdf:object ?body_o }
OPTIONAL { ?body rdfs:label ?body_l }
{ ?annotation oa:hasTarget ao:""" + escaped + """ }
UNION
{ ?annotation oa:hasTarget ?bnode .
?bnode rdf:type oa:SpecificResource ;
oa:hasSource ao:""" + escaped + """ ;
oa:hasSelector ?selector .
?selector rdf:type oa:FragmentSelector ;
rdf:value ?target_start ;
oa:start ?target_startoffset ;
oa:end ?target_endoffset }
}
"""
for row in self.sparql.query(query, initNs=initNS):
annotation = Annotation()
annotation.parse_rdf({
'target': article,
'author': row[0].encode('utf-8'),
'author_fullname': row[1].encode('utf-8') if row[1] is not None else None,
'author_email': row[2].encode('utf-8') if row[2] is not None else None,
'created': row[3].encode('utf-8') if row[3] is not None else None,
'label': row[4].encode('utf-8') if row[4] is not None else None,
'type': row[5].encode('utf-8') if row[5] is not None else None,
'subject': row[6].encode('utf-8') if row[6] is not None else None,
'predicate': row[7].encode('utf-8') if row[7] is not None else None,
'object': row[8].encode('utf-8') if row[8] is not None else None,
'obj_label': row[9].encode('utf-8') if row[9] is not None else None,
'target_start': row[10].encode('utf-8') if row[10] is not None else None,
'target_startoff': int(row[11]) if row[11] is not None else None,
'target_endoff': int(row[12]) if row[12] is not None else None
})
ret.append(annotation.to_dict())
return ret
def __init__(self, name, pnts, channels, scatters=None, notes=None):
"""
fcmdata(name, pnts, channels, scatters=None)
name: name of corresponding FCS file minus extension
pnts: array of data points
channels: a list of which markers/scatters are on which column of
the array.
scatters: a list of which indexes in channels are scatters
"""
self.name = name
# if type(pnts) != type(array([])):
# raise BadFCMPointDataTypeError(pnts, "pnts isn't a numpy.array")
self.tree = Tree(pnts, channels)
#self.pnts = pnts
#self.channels = channels
#TODO add some default intelegence for determining scatters if None
self.scatters = scatters
self.markers = []
if self.scatters is not None:
for chan in range(len(channels)):
if chan in self.scatters:
pass
elif self.tree.root.channels[chan] in self.scatters:
pass
else:
self.markers.append(chan)
if notes == None:
notes = Annotation()
self.notes = notes
def loadLabels(self):
filename = self.getLabelFilename()
if not filename:
self.clearAnnotation()
return
# If we have everything and the filename did not change, then we are good
if self.annotation and filename == self.currentLabelFile:
return
# Clear the current labels first
self.clearAnnotation()
try:
self.annotation = Annotation(self.gtType)
self.annotation.fromJsonFile(filename)
except IOError as e:
# This is the error if the file does not exist
message = "Error parsing labels in {0}. Message: {1}".format( filename, e.strerror )
self.statusBar().showMessage(message)
# Remember the filename loaded
self.currentLabelFile = filename
# Remeber the status bar message to restore it later
restoreMessage = self.statusBar().currentMessage()
# Restore the message
self.statusBar().showMessage( restoreMessage )
def object_hook(d):
"""
Usage
-----
>>> import simplejson as json
>>> with open('file.json', 'r') as f:
... json.load(f, object_hook=object_hook)
"""
from segment import Segment
from timeline import Timeline
from annotation import Annotation
from transcription import Transcription
if PYANNOTE_JSON_SEGMENT in d:
return Segment.from_json(d)
if PYANNOTE_JSON_TIMELINE in d:
return Timeline.from_json(d)
if PYANNOTE_JSON_ANNOTATION in d:
return Annotation.from_json(d)
if PYANNOTE_JSON_TRANSCRIPTION in d:
return Transcription.from_json(d)
return d
def object_hook(d):
"""
Usage
-----
>>> import simplejson as json
>>> with open('file.json', 'r') as f:
... json.load(f, object_hook=object_hook)
"""
from segment import Segment
from timeline import Timeline
from annotation import Annotation
from transcription import Transcription
if PYANNOTE_JSON_SEGMENT in d:
return Segment.from_json(d)
if PYANNOTE_JSON_TIMELINE in d:
return Timeline.from_json(d)
if PYANNOTE_JSON_ANNOTATION in d:
return Annotation.from_json(d)
if PYANNOTE_JSON_TRANSCRIPTION in d:
return Transcription.from_json(d)
return d
def geneNormalization(documents):
print("Gene Normalization:", file=sys.stderr)
for document in tqdm.tqdm(documents):
ids_by_pmid = getGeneByPMID(document['id'])
for passage in document['passages']:
for ann in passage['annotations']:
ids_by_mention = getGeneByMention(ann['text'])
flag = False
for id_by_mention in ids_by_mention:
if id_by_mention in ids_by_pmid:
flag = True
Annotation.setNCBIID(ann, id_by_mention)
break
if not flag:
Annotation.setNCBIID(
ann, ids_by_mention[0]
if len(ids_by_mention) > 0 else 'TBD')
def __init__(self, dandelion_raw_list):
self.dandelion_raw_list = dandelion_raw_list
self.good_annotations = []
if self.dandelion_raw_list:
for annotation_dict in self.dandelion_raw_list.get(
"annotations", []):
my_annotation = Annotation(annotation_dict)
if not my_annotation.suppress:
for top_entity in self.dandelion_raw_list.get(
"topEntities", []):
if my_annotation.uri == top_entity["uri"]:
my_annotation.is_top_entity = True
my_annotation.top_entity_score = top_entity[
"score"]
self.good_annotations.append(my_annotation)
def serialize(dic):
if 'annotations' in dic:
annotations = []
for ann in dic['annotations']:
annotations.append(Annotation.serialize(ann))
return AnnotateResponse(annotations)
else:
return dic
def _preload_annotations(self):
annotation_dir = self._annotation_dir
jsons = [
f for f in os.listdir(annotation_dir) if splitext(f)[1] == ".json"
]
self._annotations = {
splitext(f)[0]: Annotation.load(os.path.join(annotation_dir, f))
for f in jsons
}
def loclabel_gen(ano_path, loc_path, out_path):
pids = list(map(lambda x: x.strip('.json'), os.listdir(ano_path)))
annotations = {}
for pid in pids:
pid_json_path = os.path.join(ano_path, pid + '.json')
anno = Annotation()
anno.from_json(pid_json_path)
annotations[pid] = anno
coords = []
infile = open(loc_path)
for i, line in enumerate(infile):
pid, x_center, y_center = line.strip('\n').split(',')
coords.append((pid, x_center, y_center))
infile.close()
num_sample = len(coords)
print(f"{out_path} Total sample: {num_sample}")
outfile = open(out_path,'w')
for index in range(num_sample):
pid, x_center, y_center = coords[index]
x_center = int(x_center)
y_center = int(y_center)
x_top_left = int(x_center - IMG_SIZE / 2)
y_top_left = int(y_center - IMG_SIZE / 2)
label=[]
for x_idx in range(3):
for y_idx in range(3):
# (x, y) is the center of each patch
x = x_top_left + int((x_idx + 0.5) * SUB_SIZE)
y = y_top_left + int((y_idx + 0.5) * SUB_SIZE)
# get label information according to annotation
if annotations[pid].inside_polygons((x, y), True):
label.append(1)
else:
label.append(0)
# write output
outfile.write(f"{pid.lower()}, {x_center}, {y_center}, {str(label)[1:-1]}\n")
outfile.close()
def get_annotations():
user_map = get_user_map()
assignments = get_assignments(user_map)
annotations = {
video: Annotation(assignment, video)
for video, assignment in assignments.items()
}
return annotations
def targets(self, field, keypoint_sets):
assert self.keypoints is not None
assert self.skeleton is not None
annotations = [
Annotation(keypoints=self.keypoints, skeleton=self.skeleton).set(kps, fixed_score=None)
for kps in keypoint_sets
]
self._confidences(field[0])
self._regressions(field[1], field[2], field[3], field[4], annotations=annotations)
def parse_json(self, filename: str, classes: list) -> List[Annotation]:
with open(filename, 'r') as json_file:
json_data = json.load(json_file)
images = json_data["images"]
categories = json_data["categories"]
annotations = []
for anno in json_data["annotations"]:
image_id = anno["image_id"]
cls_id = anno["category_id"]
for info in images:
if info["id"] == image_id:
annotation = Annotation(
info["file_name"].split(".")[0])
annotation.image_size()
for category in categories:
if category["id"] == cls_id:
annotation.class_id = category["name"]
bndbox = {
"xmin": anno["bbox"][0],
"ymin": anno["bbox"][1],
"xmax": anno["bbox"][2] + anno["bbox"][0],
"ymax": anno["bbox"][3] + anno["bbox"][1]
}
annotation.bbox = (bndbox["xmin"], bndbox["ymin"],
bndbox["xmax"], bndbox["ymax"])
annotations.append(annotation)
return annotations
def dict_to_tf_examples(data):
global DATA_DIR
img_name = os.path.join(data['name'] + 'leftImg8bit.png')
img_path = os.path.join(DATA_DIR, 'leftImg8bit', data['relpath'], img_name)
annotation = Annotation()
annotation.fromJsonFile(data['json_path'])
instanceImg = createInstanceImage(annotation, "trainIds")
instanceImg.Format = 'PNG'
with tf.gfile.GFile(img_path, 'rb') as fid:
image_file = fid.read()
image_io = io.BytesIO(image_file)
image = Image.open(image_io)
splits_divider = FLAGS.splits_divider
split_width = int(np.ceil(image.width / splits_divider))
split_width_half = int(np.ceil(split_width / 2))
split_positions = [i * split_width for i in range(splits_divider - 1)]
split_positions.append(image.width - split_width)
split_positions += [
split_width_half + i * split_width for i in range(splits_divider - 1)
]
# split_positions += [random.randint(10,(image.width-split_width-10)) for i in range(FLAGS.splits_add)]
examples = []
for i, pos in enumerate(split_positions):
box = (pos, 0, pos + split_width, image.height)
sub_image = image.crop(box)
sub_instanceImg = instanceImg.crop(box)
examples.append(
sub_img_to_tf_example(img_name + '[#' + str(i) + ']', sub_image,
sub_instanceImg))
return (examples)
def build_gold_annotations(self):
""" Merge annotations, save as gold annotations.
Take union of all extractions, discard mismatched attributions
"""
# Merge annotations from annotators
self.gold_fic_annotations = {}
for fandom_fname in sorted(self.fandom_fnames):
self.build_fic_gold_annotations(
fandom_fname
) # saves to self.gold_fic_annotations[fandom_fname]
# Save out
for fandom_fname, annotations in sorted(
self.gold_fic_annotations.items()):
if self.span_type == 'coref':
gold_annotations = Annotation(self.annotations_dirpath,
fandom_fname,
file_ext='_entity_clusters.csv')
elif self.span_type == 'quotes':
gold_annotations = Annotation(
self.annotations_dirpath,
fandom_fname,
file_ext='_quote_attribution.csv')
gold_annotations.save_annotated_spans(annotations)
def __init__(self,
parent=None,
with_filename=True,
with_slider=True,
cache_capacity=500,
max_fps=0):
super(VideoWidget, self).__init__(parent)
self.with_filename = with_filename
self.with_slider = with_slider
self.video = Video(cache_capacity=cache_capacity, max_fps=max_fps)
self.annotation = Annotation()
self.tube_id = 0
self.tracker = None
self.sim_thr = 0.9
self.init_ui()
self.installEventFilter(self)
if self.with_slider:
self.slider.sliderReleased.connect(self.on_slider_released)
self.label_frame.bbox_added.connect(self.set_tracker)
self.label_frame.bbox_deleted.connect(self.del_tracker)
self.video.frame_updated.connect(self.update_frame)
self.video.export_progress_updated.connect(self.update_export_progress)
def modify_coref_tokens(self, coref_annotations_dirpath,
coref_annotations_ext):
""" Changes coref tokens to gold annotations in self.token_data.
Saves out to {token_output_dirpath}_gold_coref/token_fpath
"""
# Load gold mentions
gold = Annotation(coref_annotations_dirpath,
self.fandom_fname,
file_ext=coref_annotations_ext,
fic_csv_dirpath=self.fic_csv_dirpath)
gold.extract_annotated_spans()
# Build character name to id dictionary for gold characters (arbitrary)
self.char_name2id = defaultdict(lambda: len(self.char_name2id))
#self.char_name2id = {charname: len(self.char_name2id) for charname in sorted(gold.annotations_set)}
# Clear existing character coref annotations
self.token_data['characterId'] = -1
# Modify original tokens file
for span in gold.annotations:
self.modify_coref_span(span)
# Renumber BookNLP's own token IDs for re-running on modified output
self.renumber_token_ids()
# Save out
self.modified_token_output_dirpath = self.modified_token_output_dirpath.rstrip(
'/') + '_gold_coref'
if not os.path.exists(self.modified_token_output_dirpath):
os.mkdir(self.modified_token_output_dirpath)
self.modified_token_fpath = os.path.join(
self.modified_token_output_dirpath,
f'{self.fandom_fname}{self.token_file_ext}')
self.token_data.to_csv(self.modified_token_fpath,
sep='\t',
quoting=csv.QUOTE_NONE,
index=False)
def __init__(self, name, fcms=None, notes=None):
"""
Initialize with fcm collection and notes.
"""
# - how is this done in fcmdata?
self.fcmdict = {}
self.name = name
if fcms is not None:
for fcm in fcms:
self.fcmdict[fcm.name] = fcm
if notes is not None:
self.notes = Annotation()
else:
self.notes = notes
def targets(self, field, *, annotation_dicts):
assert self.keypoints is not None
assert self.skeleton is not None
annotations = [
Annotation(keypoints=self.keypoints,
skeleton=self.skeleton).set(ann['keypoints'],
fixed_score=None,
fixed_bbox=ann['bbox'])
for ann in annotation_dicts
]
self._confidences(field[0])
self._regressions(field[1], field[2], annotations=annotations)
def parse_xml(self, filename: str) -> Annotation:
name = os.path.splitext(filename)[0]
annotation = Annotation(name)
annotation.init_size()
with open(filename, 'r') as xml_file:
tree = ET.parse(xml_file)
root = tree.getroot()
labels = list()
for obj in root.iter('object'):
difficult = obj.find('difficult').text
cls = obj.find('name').text
if cls in self.classes.keys() and int(difficult) == 0:
cls_id = str(annotation.classes.get(cls))
xml_box = obj.find('bndbox')
bbox = (float(xml_box.find('xmin').text),
float(xml_box.find('xmax').text),
float(xml_box.find('ymin').text),
float(xml_box.find('ymax').text))
label = (cls_id, bbox)
print(label)
labels.append(label)
annotation.labels = labels
return annotation
def evaluate_coref(self, fandom_fname, fic_representation, save=True):
""" Evaluate coref for a fic.
Args:
save: save AnnotatedSpan objects in a pickled file in a tmp directory
"""
# Load gold mentions
gold = Annotation(self.coref_settings.gold_dirpath, fandom_fname,
file_ext=self.coref_settings.gold_ext,
fic_csv_dirpath=self.fic_csv_dirpath)
gold.extract_annotated_spans()
# Load predicted mentions
fic_representation.extract_character_mentions(
save_dirpath=self.coref_settings.preds_outpath)
# Get scores
coref_scores = scorer.coref_scores(
fic_representation.character_mentions, gold.annotations, exact_match=True)
print('\tCoref results:')
for key in ['lea_f1', 'lea_precision', 'lea_recall']:
print(f'\t\t{key}: {coref_scores[key]: .2%}')
print()
return coref_scores
def parse_txt(self, filename: str) -> Annotation:
name = os.path.splitext(filename)[0]
annotation = Annotation(name)
annotation.init_size()
labels = []
with open(filename, "r") as txt_file:
lines = txt_file.readlines()
for line in lines:
line = line.strip()
words = line.split()
class_id = words[0]
w, h = annotation.size
bbox_width = float(words[3]) * w
bbox_height = float(words[4]) * h
center_x = float(words[1]) * w
center_y = float(words[2]) * h
bbox = []
bbox.append(center_x - (bbox_width / 2))
bbox.append(center_y - (bbox_height / 2))
bbox.append(center_x + (bbox_width / 2))
bbox.append(center_y + (bbox_height / 2))
labels.append((class_id, bbox))
annotation.labels = labels
return annotation
def load_fic_annotations(self, fandom_fname, span_type):
# Load annotations
if span_type == 'coref':
file_ext = 'entity_clusters'
elif span_type == 'quotes':
file_ext = 'quote_attribution'
self.fic_annotations[fandom_fname] = {}
for annotator in self.annotators:
self.fic_annotations[fandom_fname][annotator] = Annotation(
self.annotations_dirpath,
fandom_fname,
file_ext=f'_{file_ext}_{annotator}.csv',
fic_csv_dirpath=self.fic_csv_dirpath)
self.fic_annotations[fandom_fname][
annotator].extract_annotated_spans()
def _ner_document_process(document, tokenizer):
text_li = []
tag_li = []
def f(text):
# str -> List[str]
x = tokenizer.convert_ids_to_tokens(tokenizer(text)['input_ids'])[1:-1]
tokens = []
for token in x:
if token[:2] == '##':
tokens[-1] += token[2:]
else:
tokens.append(token)
return tokens
for passage in document['passages']:
anns = passage['annotations']
anns = Annotation.sortAnns(anns, TBDFilter= False)
text = passage['text']
offset_p = passage['offset']
index = 0
if len(anns) == 0:
tokens = f(text)
text_li.extend(tokens)
tag_li.extend(['O'] * len(tokens))
else:
for ann in anns:
for i, location in enumerate(ann['locations']): # unnecessary currently because of filter in `Annotation.sortAnns`
if i > 0:
print("WARNING: PMID:{}, Ann id:{} Text:{}".format(
document['id'], ann['id'], ann['text']))
offset = location['offset']
length = location['length']
tokens = f(text[index:offset-offset_p])
text_li.extend(tokens)
tag_li.extend(['O'] * len(tokens))
if i == len(ann['locations']) - 1:
mention = text[offset-offset_p: offset-offset_p+length]
tokens = f(mention)
assert mention == ann['text'], mention + '\t' + ann['text'] +'\t'+ document['id']
assert len(tokens) > 0
tag_li.extend(['B'] + ['I']*(len(tokens) - 1))
text_li.extend(tokens)
index = max(offset - offset_p + length, index)
tokens = f(text[index:])
text_li.extend(tokens)
tag_li.extend(['O']*len(tokens))
assert len(text_li) == len(tag_li)
return text_li, tag_li
def move_mouse(self, event):
"""
Handles the drawing of the arrow when deciding where to annotate
"""
if WorldState.Instance().draw_plot:
if WorldState.Instance().session_dict['click_one']:
WorldState.Instance(
).session_dict['temp_annotation'] = Annotation(
WorldState.Instance()._ARROW,
(WorldState.Instance().session_dict['click_one_x'],
WorldState.Instance().session_dict['click_one_y']),
(event.xdata, event.ydata))
if WorldState.Instance().session_dict['annotate']:
WorldState.Instance().session_dict['redraw_legend'] = False
WorldState.Instance().draw_plot.plot()
WorldState.Instance().session_dict['redraw_legend'] = True
def wrap_annotations(sentences):
annotations = []
tid = 0
for sid, labels in enumerate(sentences):
for idx, label in enumerate(labels):
for ann in label.split('#'):
type = ann[2:]
if 'B-' in ann:
annotations.append(Annotation(type, sid, tid))
elif 'I-' in ann:
for _ann in reversed(annotations):
if type == _ann.annotation:
_ann.add_id(tid)
break
tid += 1
return annotations
def to_annotation(self, threshold=-np.inf, posterior=False):
"""
Parameters
----------
threshold : float, optional
Each track is annotated with the label with the highest score.
Yet, if the latter is smaller than `threshold`, label is replaced
with an `Unknown` instance.
posterior : bool, optional
If True, scores are posterior probabilities in open-set
identification. If top model posterior is higher than unknown
posterior, it is selected. Otherwise, label is replaced with an
`Unknown` instance.
"""
annotation = Annotation(uri=self.uri, modality=self.modality)
if not self:
return annotation
best = self.nbest(1, invert=False)
if posterior:
# compute unknown posterior
func = lambda p: 1. - np.nansum(p, axis=1)
Pu = self.apply(func, new_columns=['_'])
# threshold best target posterior
# with unknown posterior and threshold
for segment, track, label, value in best.itervalues():
if value < Pu[segment, track, '_'] or value < threshold:
label = Unknown()
annotation[segment, track] = label
else:
# threshold best target score with threshold
for segment, track, label, value in best.itervalues():
if value < threshold:
label = Unknown()
annotation[segment, track] = label
return annotation
def _load_annotations(self):
f = open(self.file_name, 'r')
for line in f:
line = line.strip()
if line != "":
annotations_list = [
Annotation(
int(begin_index) + 1, int(end_index), time_expression,
timex3)
for begin_index, end_index, time_expression, timex3 in map(
None, *([iter(line.split("\t"))] * 4))
]
else:
annotations_list = []
self.annotations.append(annotations_list)
f.close()
def __init__(self, parent=None, with_filename=True, with_slider=True,
cache_capacity=500, max_fps=0):
super(VideoWidget, self).__init__(parent)
self.with_filename = with_filename
self.with_slider = with_slider
self.video = Video(cache_capacity=cache_capacity, max_fps=max_fps)
self.annotation = Annotation()
self.tube_id = 0
self.tracker = None
self.sim_thr = 0.9
self.init_ui()
self.installEventFilter(self)
if self.with_slider:
self.slider.sliderReleased.connect(self.on_slider_released)
self.label_frame.bbox_added.connect(self.set_tracker)
self.label_frame.bbox_deleted.connect(self.del_tracker)
self.video.frame_updated.connect(self.update_frame)
self.video.export_progress_updated.connect(self.update_export_progress)
def test_get_annotated(self):
text = 'Lorem ipsum dolor sit amet. Consectetur adipiscing elit. Sed do eiusmod tempor incididunt.'
tagged_text = '<p><span>Lorem</span> ipsum dolor sit <span>amet</span>.</p><p>Consectetur adipiscing <span>elit</span>.</p><p>Sed do eiusmod tempor <span>incididunt</span>.</p>'
lst_annotations = [
Annotation('sentence', 0, 27),
Annotation('sentence', 28, 56),
Annotation('sentence', 57, 90),
Annotation('word', 22, 26),
Annotation('word', 51, 55),
Annotation('word', 79, 89),
Annotation('word', 0, 5)
]
tagged_text_test = get_annotated(
Document(text, 'test', lst_annotations))
self.assertEqual(tagged_text_test, tagged_text)
def count_annotated_objects_in_folder(folder:str, recursive:bool=True) -> int:
jsons = [
os.path.join(folder, f) for f in os.listdir(folder)
if os.path.splitext(f)[1] == ".json"
]
count = 0
for f in jsons:
ann = Annotation.load(f)
count += len(ann.objects) + len(ann.missing)
if recursive:
subfolders = [
os.path.join(folder, f)
for f in os.listdir(folder)
if os.path.isdir(os.path.join(folder, f))
]
count += sum(
count_annotated_objects_in_folder(subfolder, True)
for subfolder in subfolders
)
return count
def write_annotation(self, ann, annotations_csv):
unique_tuple = (ann[0], ann[1], ann[3]
) #description have not to be there
#check if annotation is already in dictionary
if unique_tuple in self.map_ann2id:
#assign current annotation id and increment count of this label
return self.map_ann2id[unique_tuple]
else:
ann_info = Annotation(ann[0], ann[1], ann[2], ann[3])
#add new annotation to dictionary and write row with annotation
self.annotation_id_cnt += 1
current_annotation_id = self.annotation_id_cnt
self.map_ann2id[unique_tuple] = current_annotation_id
self.id2counts[current_annotation_id] = {}
for i in self.back_sets_ids:
self.id2counts[current_annotation_id][i] = 0
ann_desc_id = self.ann_descriptions.get_annotation_description(
ann_info.attribute, ann_info.var_gene)
annotations_csv.writerow([
current_annotation_id, ann_info.label,
ann_info.description.replace('\\', '\\\\'), ann_desc_id
])
return current_annotation_id
def json2labelImg(inJson, outImg, encoding="trainIds"):
annotation = Annotation()
annotation.fromJsonFile(inJson)
labelImg = createLabelImage(annotation, encoding)
#labelImg = createLabelImage( annotation , encoding ,outline= name2label['hasInstances boundary'].trainId)
labelImg.save(outImg)
def json2instanceArr(inJson, encoding="ids", label_tochose='car'):
annotation = Annotation()
annotation.fromJsonFile(inJson)
instanceImg = getInstancewithLabel(annotation, encoding, label_tochose)
return instanceImg
class VideoWidget(QWidget):
frame_updated = pyqtSignal(int)
tube_annotated = pyqtSignal(dict)
annotation_loaded = pyqtSignal(list)
export_progress_updated = pyqtSignal(int)
def __init__(self, parent=None, with_filename=True, with_slider=True,
cache_capacity=500, max_fps=0):
super(VideoWidget, self).__init__(parent)
self.with_filename = with_filename
self.with_slider = with_slider
self.video = Video(cache_capacity=cache_capacity, max_fps=max_fps)
self.annotation = Annotation()
self.tube_id = 0
self.tracker = None
self.sim_thr = 0.9
self.init_ui()
self.installEventFilter(self)
if self.with_slider:
self.slider.sliderReleased.connect(self.on_slider_released)
self.label_frame.bbox_added.connect(self.set_tracker)
self.label_frame.bbox_deleted.connect(self.del_tracker)
self.video.frame_updated.connect(self.update_frame)
self.video.export_progress_updated.connect(self.update_export_progress)
def init_ui(self):
self.vbox_layout = QVBoxLayout()
if self.with_filename:
self.init_label_filename()
self.init_label_frame()
if self.with_slider:
self.init_slider()
self.setLayout(self.vbox_layout)
# self.setFocusPolicy(Qt.StrongFocus)
def init_label_filename(self):
self.label_filename = QLabel('filename')
self.label_filename.setAlignment(Qt.AlignCenter)
self.vbox_layout.addWidget(self.label_filename, 1)
def init_label_frame(self):
self.label_frame = ImageLabel('video')
self.label_frame.setAlignment(Qt.AlignCenter)
self.label_frame.setStyleSheet('border: 1px solid black')
self.vbox_layout.addWidget(self.label_frame, 10)
def init_slider(self):
self.slider = QSlider(Qt.Horizontal)
self.slider.setRange(0, 1000)
self.slider.setTickInterval(1)
self.slider.setValue(0)
self.slider.setEnabled(False)
self.vbox_layout.addWidget(self.slider, 1)
def eventFilter(self, object, event):
if event.type() == QEvent.KeyPress:
key = event.key()
if key == Qt.Key_D:
self.frame_forward()
return True
elif key == Qt.Key_A:
self.frame_backward()
return True
elif key == Qt.Key_S:
self.last_keyframe = self.cursor()
self.new_tube()
return True
elif key == Qt.Key_Left:
if self.status() == VideoStatus.play_backward:
self.pause()
elif self.video.status != VideoStatus.not_loaded:
self.play_backward()
return True
elif key == Qt.Key_Right:
if self.status() == VideoStatus.play_forward:
self.pause()
elif self.status() != VideoStatus.not_loaded:
self.play_forward()
return True
elif key == Qt.Key_Space:
self.pause()
return True
return False
def frame_forward(self):
if self.tracker is not None:
ori_hist = color_hist(self.tracker.init_region, 16)
if self.cursor() >= self.annotation.tube_end(self.tube_id):
self.last_keyframe = self.cursor()
cnt = 0
while cnt < 10:
frame = self.video.frame_forward()
self.update_frame(frame)
if (self.tracker is None or
self.cursor() < self.annotation.tube_end(self.tube_id)):
break
bbox = self.tracker.bbox
hist = color_hist(
frame.raw_img[bbox.left: bbox.right, bbox.top: bbox.bottom], 16)
if compare_hist(hist, ori_hist) < self.sim_thr:
break
cnt += 1
def frame_backward(self):
frame = self.video.frame_backward()
self.update_frame(frame)
def play_forward(self):
self.video.play_forward()
def play_backward(self):
self.video.play_backward()
def pause(self):
self.video.pause()
def jump_to_frame(self, frame_id):
self.clear_tracker()
frame = self.video.jump_to_frame(frame_id)
self.update_frame(frame)
def status(self):
return self.video.status
def cursor(self):
return self.video.cursor
def frame_cnt(self):
return self.video.frame_cnt
def current_frame(self):
return self.video.current_frame()
def new_tube(self):
label = self.label_frame.bbox_label
if label is not None:
self.tube_id = self.annotation.next_tube_id
self.annotation.add_tube(label, self.cursor())
self.label_frame.flash_reticle()
self.label_frame.toggle_reticle(True)
self.label_frame.is_new_tube = True
def clear_tracker(self):
self.tracker = None
def reset_tube_id(self):
self.tube_id = 0
def track(self, frame):
frame_rect = BoundingBox(None, 0, 0, 0,
self.video.width, self.video.height)
bbox, score = self.tracker.update(frame)
bbox = bbox.intersected(frame_rect)
return bbox
def adjust_track_bboxes(self, bbox):
self.annotation.interpolate(self.tube_id, bbox, self.last_keyframe,
self.cursor())
@pyqtSlot(VideoFrame)
def update_frame(self, frame):
# get bounding boxes of current tube and other tubes
bboxes = dict()
if (self.tracker is not None and self.video.is_forward() and
self.cursor() > self.annotation.tube_end(self.tube_id)):
bboxes['current_tube'] = self.track(frame)
self.annotation.set_bbox(self.tube_id, self.cursor(),
bboxes['current_tube'])
else:
bboxes['current_tube'] = self.annotation.get_bbox(
self.tube_id, self.cursor())
bboxes['other_tubes'] = self.annotation.get_bboxes(
self.cursor(), self.tube_id)
# show the frame and corresponding bounding boxes
self.label_frame.display(frame)
self.label_frame.update_bboxes(bboxes)
# update slider position
if self.with_slider:
self.slider.setValue(
int(self.slider.maximum() * frame.id / self.frame_cnt()))
# emit the frame id to the main window to update status bar
self.frame_updated.emit(frame.id)
@pyqtSlot(BoundingBox)
def set_tracker(self, bbox):
if self.tracker is not None and self.cursor() > self.last_keyframe + 1:
self.adjust_track_bboxes(bbox)
self.tracker = Tracker()
self.tracker.start_track(self.current_frame(), bbox)
self.annotation.set_bbox(self.tube_id, self.cursor(), bbox)
@pyqtSlot()
def del_tracker(self):
self.clear_tracker()
self.annotation.del_later_bboxes(self.tube_id, self.cursor())
self.annotation.save()
tube_info = self.annotation.tube(self.tube_id).to_dict(with_bboxes=False)
self.tube_annotated.emit(tube_info)
self.reset_tube_id()
@pyqtSlot()
def on_slider_released(self):
progress = self.slider.value() / self.slider.maximum()
frame_id = max(int(round(self.frame_cnt() * progress)), 1)
self.jump_to_frame(frame_id)
@pyqtSlot(int)
def jump_to_tube(self, tube_id):
self.tube_id = tube_id
self.jump_to_frame(self.annotation.tube_start(tube_id))
@pyqtSlot(int)
def del_tube(self, tube_id):
if self.tube_id == tube_id:
self.tube_id = 0
self.annotation.del_tube(tube_id)
self.annotation.save()
@pyqtSlot(int, str)
def change_tube_label(self, tube_id, label):
self.annotation.tubes[tube_id].label = label
self.annotation.save()
@pyqtSlot(str)
def update_bbox_label(self, label):
self.label_frame.update_bbox_label(label)
@pyqtSlot()
def open_file(self):
self.filename, _ = QFileDialog.getOpenFileName(
self, 'Load video', '/home/kchen/data/youtube/selected/',
'Videos (*.mp4 *.avi *.mkv *.flv *.m4v)')
if not self.filename:
return
if self.with_filename:
self.label_filename.setText(os.path.basename(self.filename))
if self.with_slider:
self.slider.setEnabled(True)
self.video.load(self.filename)
self.annotation.load(self.filename + '.annotation')
self.annotation_loaded.emit(self.annotation.get_brief_info())
self.jump_to_frame(1)
@pyqtSlot()
def export_video(self):
filename, _ = QFileDialog.getSaveFileName(
self, 'Export video', './', 'Videos (*.avi)')
t = threading.Thread(target=self.video.export,
kwargs=dict(out_file=filename,
annotation=self.annotation))
t.daemon = True
t.start()
@pyqtSlot(int)
def update_export_progress(self, progress):
self.export_progress_updated.emit(progress)
@pyqtSlot()
def save_annotation(self):
self.annotation.save()
def __init__(self, description, subcat_filter, max_sentences=None):
Annotation.__init__(self, description)
self.subcat_filter = subcat_filter
self.max_sentences = max_sentences
class CityscapesViewer(QtGui.QMainWindow):
#############################
## Construction / Destruction
#############################
# Constructor
def __init__(self):
# Construct base class
super(CityscapesViewer, self).__init__()
# This is the configuration.
# The filename of the image we currently working on
self.currentFile = ""
# The filename of the labels we currently working on
self.currentLabelFile = ""
# The path of the images of the currently loaded city
self.city = ""
# The name of the currently loaded city
self.cityName = ""
# Ground truth type
self.gtType = CsObjectType.POLY
# The path of the labels. In this folder we expect a folder for each city
# Within these city folders we expect the label with a filename matching
# the images, except for the extension
self.labelPath = ""
# The transparency of the labels over the image
self.transp = 0.5
# The zoom toggle
self.zoom = False
# The zoom factor
self.zoomFactor = 1.5
# The size of the zoom window. Currently there is no setter or getter for that
self.zoomSize = 400 #px
# The width that we actually use to show the image
self.w = 0
# The height that we actually use to show the image
self.h = 0
# The horizontal offset where we start drawing within the widget
self.xoff = 0
# The vertical offset where we start drawing withing the widget
self.yoff = 0
# A gap that we leave around the image as little border
self.bordergap = 20
# The scale that was used, ie
# self.w = self.scale * self.image.width()
# self.h = self.scale * self.image.height()
self.scale = 1.0
# Filenames of all images in current city
self.images = []
# Image extension
self.imageExt = "_leftImg8bit.png"
# Ground truth extension
self.gtExt = "_gt*.json"
# Current image as QImage
self.image = QtGui.QImage()
# Index of the current image within the city folder
self.idx = 0
# All annotated objects in current image, i.e. list of csPoly or csBbox
self.annotation = []
# The current object the mouse points to. It's index in self.labels
self.mouseObj = -1
# The object that is highlighted and its label. An object instance
self.highlightObj = None
self.highlightObjLabel = None
# The position of the mouse
self.mousePosOrig = None
# The position of the mouse scaled to label coordinates
self.mousePosScaled = None
# If the mouse is outside of the image
self.mouseOutsideImage = True
# The position of the mouse upon enabling the zoom window
self.mousePosOnZoom = None
# A list of toolbar actions that need an image
self.actImage = []
# A list of toolbar actions that need an image that is not the first
self.actImageNotFirst = []
# A list of toolbar actions that need an image that is not the last
self.actImageNotLast = []
# Toggle status of the play icon
self.playState = False
# Enable disparity visu in general
self.enableDisparity = True
# Show disparities instead of labels
self.showDisparity = False
# The filename of the disparity map we currently working on
self.currentDispFile = ""
# The disparity image
self.dispImg = None
# As overlay
self.dispOverlay = None
# The disparity search path
self.dispPath = None
# Disparity extension
self.dispExt = "_disparity.png"
# Generate colormap
try:
norm = matplotlib.colors.Normalize(vmin=3,vmax=100)
cmap = matplotlib.cm.plasma
self.colormap = matplotlib.cm.ScalarMappable( norm=norm , cmap=cmap )
except:
self.enableDisparity = False
# check if pillow was imported, otherwise no disparity visu possible
if not 'PILLOW_VERSION' in globals():
self.enableDisparity = False
# Default label
self.defaultLabel = 'static'
if self.defaultLabel not in name2label:
print('The {0} label is missing in the internal label definitions.'.format(self.defaultLabel))
return
# Last selected label
self.lastLabel = self.defaultLabel
# Setup the GUI
self.initUI()
# If we already know a city from the saved config -> load it
self.loadCity()
self.imageChanged()
# Destructor
def __del__(self):
return
# Construct everything GUI related. Called by constructor
def initUI(self):
# Create a toolbar
self.toolbar = self.addToolBar('Tools')
# Add the tool buttons
iconDir = os.path.join( os.path.dirname(sys.argv[0]) , 'icons' )
# Loading a new city
loadAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'open.png' )), '&Tools', self)
loadAction.setShortcuts(['o'])
self.setTip( loadAction, 'Open city' )
loadAction.triggered.connect( self.getCityFromUser )
self.toolbar.addAction(loadAction)
# Open previous image
backAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'back.png')), '&Tools', self)
backAction.setShortcut('left')
backAction.setStatusTip('Previous image')
backAction.triggered.connect( self.prevImage )
self.toolbar.addAction(backAction)
self.actImageNotFirst.append(backAction)
# Open next image
nextAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'next.png')), '&Tools', self)
nextAction.setShortcut('right')
self.setTip( nextAction, 'Next image' )
nextAction.triggered.connect( self.nextImage )
self.toolbar.addAction(nextAction)
self.actImageNotLast.append(nextAction)
# Play
playAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'play.png')), '&Tools', self)
playAction.setShortcut(' ')
playAction.setCheckable(True)
playAction.setChecked(False)
self.setTip( playAction, 'Play all images' )
playAction.triggered.connect( self.playImages )
self.toolbar.addAction(playAction)
self.actImageNotLast.append(playAction)
self.playAction = playAction
# Select image
selImageAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'shuffle.png' )), '&Tools', self)
selImageAction.setShortcut('i')
self.setTip( selImageAction, 'Select image' )
selImageAction.triggered.connect( self.selectImage )
self.toolbar.addAction(selImageAction)
self.actImage.append(selImageAction)
# Enable/disable disparity visu. Toggle button
if self.enableDisparity:
dispAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'disp.png' )), '&Tools', self)
dispAction.setShortcuts(['d'])
dispAction.setCheckable(True)
dispAction.setChecked(self.showDisparity)
self.setTip( dispAction, 'Enable/disable depth visualization' )
dispAction.toggled.connect( self.dispToggle )
self.toolbar.addAction(dispAction)
self.actImage.append(dispAction)
# Enable/disable zoom. Toggle button
zoomAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'zoom.png' )), '&Tools', self)
zoomAction.setShortcuts(['z'])
zoomAction.setCheckable(True)
zoomAction.setChecked(self.zoom)
self.setTip( zoomAction, 'Enable/disable permanent zoom' )
zoomAction.toggled.connect( self.zoomToggle )
self.toolbar.addAction(zoomAction)
self.actImage.append(zoomAction)
# Decrease transparency
minusAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'minus.png' )), '&Tools', self)
minusAction.setShortcut('-')
self.setTip( minusAction, 'Decrease transparency' )
minusAction.triggered.connect( self.minus )
self.toolbar.addAction(minusAction)
# Increase transparency
plusAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'plus.png' )), '&Tools', self)
plusAction.setShortcut('+')
self.setTip( plusAction, 'Increase transparency' )
plusAction.triggered.connect( self.plus )
self.toolbar.addAction(plusAction)
# Display path to current image in message bar
displayFilepathAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'filepath.png' )), '&Tools', self)
displayFilepathAction.setShortcut('f')
self.setTip( displayFilepathAction, 'Show path to current image' )
displayFilepathAction.triggered.connect( self.displayFilepath )
self.toolbar.addAction(displayFilepathAction)
# Display help message
helpAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'help19.png' )), '&Tools', self)
helpAction.setShortcut('h')
self.setTip( helpAction, 'Help' )
helpAction.triggered.connect( self.displayHelpMessage )
self.toolbar.addAction(helpAction)
# Close the application
exitAction = QtGui.QAction(QtGui.QIcon( os.path.join( iconDir , 'exit.png' )), '&Tools', self)
exitAction.setShortcuts(['Esc'])
self.setTip( exitAction, 'Exit' )
exitAction.triggered.connect( self.close )
self.toolbar.addAction(exitAction)
# The default text for the status bar
self.defaultStatusbar = 'Ready'
# Create a statusbar. Init with default
self.statusBar().showMessage( self.defaultStatusbar )
# Enable mouse move events
self.setMouseTracking(True)
self.toolbar.setMouseTracking(True)
# Open in full screen
self.showFullScreen( )
# Set a title
self.applicationTitle = 'Cityscapes Viewer v1.0'
self.setWindowTitle(self.applicationTitle)
self.displayHelpMessage()
self.getCityFromUser()
# And show the application
self.show()
#############################
## Toolbar call-backs
#############################
# Switch to previous image in file list
# Load the image
# Load its labels
# Update the mouse selection
# View
def prevImage(self):
if not self.images:
return
if self.idx > 0:
self.idx -= 1
self.imageChanged()
else:
message = "Already at the first image"
self.statusBar().showMessage(message)
return
# Switch to next image in file list
# Load the image
# Load its labels
# Update the mouse selection
# View
def nextImage(self):
if not self.images:
return
if self.idx < len(self.images)-1:
self.idx += 1
self.imageChanged()
elif self.playState:
self.playState = False
self.playAction.setChecked(False)
else:
message = "Already at the last image"
self.statusBar().showMessage(message)
if self.playState:
QtCore.QTimer.singleShot(0, self.nextImage)
return
# Play images, i.e. auto-switch to next image
def playImages(self, status):
self.playState = status
if self.playState:
QtCore.QTimer.singleShot(0, self.nextImage)
# Switch to a selected image of the file list
# Ask the user for an image
# Load the image
# Load its labels
# Update the mouse selection
# View
def selectImage(self):
if not self.images:
return
dlgTitle = "Select image to load"
self.statusBar().showMessage(dlgTitle)
items = QtCore.QStringList( [ os.path.basename(i) for i in self.images ] )
(item, ok) = QtGui.QInputDialog.getItem(self, dlgTitle, "Image", items, self.idx, False)
if (ok and item):
idx = items.indexOf(item)
if idx != self.idx:
self.idx = idx
self.imageChanged()
else:
# Restore the message
self.statusBar().showMessage( self.defaultStatusbar )
# Toggle zoom
def zoomToggle(self, status):
self.zoom = status
if status :
self.mousePosOnZoom = self.mousePosOrig
self.update()
# Toggle disparity visu
def dispToggle(self, status):
self.showDisparity = status
self.imageChanged()
# Increase label transparency
def minus(self):
self.transp = max(self.transp-0.1,0.0)
self.update()
def displayFilepath(self):
self.statusBar().showMessage("Current image: {0}".format( self.currentFile ))
self.update()
def displayHelpMessage(self):
message = self.applicationTitle + "\n\n"
message += "INSTRUCTIONS\n"
message += " - select a city from drop-down menu\n"
message += " - browse images and labels using\n"
message += " the toolbar buttons or the controls below\n"
message += "\n"
message += "CONTROLS\n"
message += " - select city [o]\n"
message += " - highlight objects [move mouse]\n"
message += " - next image [left arrow]\n"
message += " - previous image [right arrow]\n"
message += " - toggle autoplay [space]\n"
message += " - increase/decrease label transparency\n"
message += " [ctrl+mousewheel] or [+ / -]\n"
if self.enableDisparity:
message += " - show disparity/depth overlay (if available) [d]\n"
message += " - open zoom window [z]\n"
message += " zoom in/out [mousewheel]\n"
message += " enlarge/shrink zoom window [shift+mousewheel]\n"
message += " - select a specific image [i]\n"
message += " - show path to image below [f]\n"
message += " - exit viewer [esc]\n"
QtGui.QMessageBox.about(self, "HELP!", message)
self.update()
# Decrease label transparency
def plus(self):
self.transp = min(self.transp+0.1,1.0)
self.update()
# Close the application
def closeEvent(self,event):
event.accept()
#############################
## Custom events
#############################
def imageChanged(self):
# Load the first image
self.loadImage()
# Load its labels if available
self.loadLabels()
# Load disparities if available
self.loadDisparities()
# Update the object the mouse points to
self.updateMouseObject()
# Update the GUI
self.update()
#############################
## File I/O
#############################
# Load the currently selected city if possible
def loadCity(self):
# Search for all *.pngs to get the image list
self.images = []
if os.path.isdir(self.city):
self.images = glob.glob( os.path.join( self.city , '*' + self.imageExt ) )
self.images.sort()
if self.currentFile in self.images:
self.idx = self.images.index(self.currentFile)
else:
self.idx = 0
# Load the currently selected image
# Does only load if not previously loaded
# Does not refresh the GUI
def loadImage(self):
success = False
message = self.defaultStatusbar
if self.images:
filename = self.images[self.idx]
filename = os.path.normpath( filename )
if not self.image.isNull() and filename == self.currentFile:
success = True
else:
self.image = QtGui.QImage(filename)
if self.image.isNull():
message = "Failed to read image: {0}".format( filename )
else:
message = "Read image: {0}".format( filename )
self.currentFile = filename
success = True
# Update toolbar actions that need an image
for act in self.actImage:
act.setEnabled(success)
for act in self.actImageNotFirst:
act.setEnabled(success and self.idx > 0)
for act in self.actImageNotLast:
act.setEnabled(success and self.idx < len(self.images)-1)
self.statusBar().showMessage(message)
# Load the labels from file
# Only loads if they exist
# Otherwise the filename is stored and that's it
def loadLabels(self):
filename = self.getLabelFilename()
if not filename:
self.clearAnnotation()
return
# If we have everything and the filename did not change, then we are good
if self.annotation and filename == self.currentLabelFile:
return
# Clear the current labels first
self.clearAnnotation()
try:
self.annotation = Annotation(self.gtType)
self.annotation.fromJsonFile(filename)
except IOError as e:
# This is the error if the file does not exist
message = "Error parsing labels in {0}. Message: {1}".format( filename, e.strerror )
self.statusBar().showMessage(message)
# Remember the filename loaded
self.currentLabelFile = filename
# Remeber the status bar message to restore it later
restoreMessage = self.statusBar().currentMessage()
# Restore the message
self.statusBar().showMessage( restoreMessage )
# Load the disparity map from file
# Only loads if they exist
def loadDisparities(self):
if not self.enableDisparity:
return
if not self.showDisparity:
return
filename = self.getDisparityFilename()
if not filename:
self.dispImg = None
return
# If we have everything and the filename did not change, then we are good
if self.dispImg and filename == self.currentDispFile:
return
# Clear the current labels first
self.dispImg = None
try:
self.dispImg = Image.open(filename)
except IOError as e:
# This is the error if the file does not exist
message = "Error parsing disparities in {0}. Message: {1}".format( filename, e.strerror )
self.statusBar().showMessage(message)
self.dispImg = None
if self.dispImg:
dispNp = np.array( self.dispImg )
dispNp /= 128
dispNp.round()
dispNp = np.array( dispNp , dtype=np.uint8 )
dispQt = QtGui.QImage( dispNp.data , dispNp.shape[1] , dispNp.shape[0] , QtGui.QImage.Format_Indexed8 )
colortable = []
for i in range(256):
color = self.colormap.to_rgba(i)
colorRgb = ( int(color[0]*255) , int(color[1]*255) , int(color[2]*255) )
colortable.append( QtGui.qRgb( *colorRgb ) )
dispQt.setColorTable( colortable )
dispQt = dispQt.convertToFormat( QtGui.QImage.Format_ARGB32_Premultiplied )
self.dispOverlay = dispQt
# Remember the filename loaded
self.currentDispFile = filename
# Remember the status bar message to restore it later
restoreMessage = self.statusBar().currentMessage()
# Restore the message
self.statusBar().showMessage( restoreMessage )
#############################
## Drawing
#############################
# This method is called when redrawing everything
# Can be manually triggered by self.update()
# Note that there must not be any other self.update within this method
# or any methods that are called within
def paintEvent(self, event):
# Create a QPainter that can perform draw actions within a widget or image
qp = QtGui.QPainter()
# Begin drawing in the application widget
qp.begin(self)
# Update scale
self.updateScale(qp)
# Determine the object ID to highlight
self.getHighlightedObject(qp)
# Draw the image first
self.drawImage(qp)
if self.enableDisparity and self.showDisparity:
# Draw the disparities on top
overlay = self.drawDisp(qp)
else:
# Draw the labels on top
if self.gtType == CsObjectType.POLY:
overlay = self.drawLabels(qp)
elif self.gtType == CsObjectType.BBOX:
overlay = self.drawBboxes(qp)
# Draw the label name next to the mouse
self.drawLabelAtMouse(qp)
# Draw the zoom
self.drawZoom(qp, overlay)
# Thats all drawing
qp.end()
# Forward the paint event
QtGui.QMainWindow.paintEvent(self,event)
# Update the scaling
def updateScale(self, qp):
if not self.image.width() or not self.image.height():
return
# Horizontal offset
self.xoff = self.bordergap
# Vertical offset
self.yoff = self.toolbar.height()+self.bordergap
# We want to make sure to keep the image aspect ratio and to make it fit within the widget
# Without keeping the aspect ratio, each side of the image is scaled (multiplied) with
sx = float(qp.device().width() - 2*self.xoff) / self.image.width()
sy = float(qp.device().height() - 2*self.yoff) / self.image.height()
# To keep the aspect ratio while making sure it fits, we use the minimum of both scales
# Remember the scale for later
self.scale = min( sx , sy )
# These are then the actual dimensions used
self.w = self.scale * self.image.width()
self.h = self.scale * self.image.height()
# Determine the highlighted object for drawing
def getHighlightedObject(self, qp):
# This variable we want to fill
self.highlightObj = None
# Without labels we cannot do so
if not self.annotation:
return
# If available its the selected object
highlightObjId = -1
# If not available but the polygon is empty or closed, its the mouse object
if highlightObjId < 0 and not self.mouseOutsideImage:
highlightObjId = self.mouseObj
# Get the actual object that is highlighted
if highlightObjId >= 0:
self.highlightObj = self.annotation.objects[highlightObjId]
self.highlightObjLabel = self.annotation.objects[highlightObjId].label
# Draw the image in the given QPainter qp
def drawImage(self, qp):
# Return if no image available
if self.image.isNull():
return
# Save the painters current setting to a stack
qp.save()
# Draw the image
qp.drawImage(QtCore.QRect( self.xoff, self.yoff, self.w, self.h ), self.image)
# Restore the saved setting from the stack
qp.restore()
def getPolygon(self, obj):
poly = QtGui.QPolygonF()
for pt in obj.polygon:
point = QtCore.QPointF(pt.x,pt.y)
poly.append( point )
return poly
# Draw the labels in the given QPainter qp
# optionally provide a list of labels to ignore
def drawLabels(self, qp, ignore = []):
if self.image.isNull() or self.w == 0 or self.h == 0:
return
if not self.annotation:
return
# The overlay is created in the viewing coordinates
# This way, the drawing is more dense and the polygon edges are nicer
# We create an image that is the overlay
# Within this image we draw using another QPainter
# Finally we use the real QPainter to overlay the overlay-image on what is drawn so far
# The image that is used to draw the overlays
overlay = QtGui.QImage( self.w, self.h, QtGui.QImage.Format_ARGB32_Premultiplied )
# Fill the image with the default color
defaultLabel = name2label[self.defaultLabel]
col = QtGui.QColor( *defaultLabel.color )
overlay.fill( col )
# Create a new QPainter that draws in the overlay image
qp2 = QtGui.QPainter()
qp2.begin(overlay)
# The color of the outlines
qp2.setPen(QtGui.QColor('white'))
# Draw all objects
for obj in self.annotation.objects:
# The label of the object
name = assureSingleInstanceName( obj.label )
# If we do not know a color for this label, warn the user
if name not in name2label:
print("The annotations contain unkown labels. This should not happen. Please inform the datasets authors. Thank you!")
print("Details: label '{}', file '{}'".format(name,self.currentLabelFile))
continue
poly = self.getPolygon(obj)
# Scale the polygon properly
polyToDraw = poly * QtGui.QTransform.fromScale(self.scale,self.scale)
# Default drawing
# Color from color table, solid brush
col = QtGui.QColor( *name2label[name].color )
brush = QtGui.QBrush( col, QtCore.Qt.SolidPattern )
qp2.setBrush(brush)
# Overwrite drawing if this is the highlighted object
if self.highlightObj and obj == self.highlightObj:
# First clear everything below of the polygon
qp2.setCompositionMode( QtGui.QPainter.CompositionMode_Clear )
qp2.drawPolygon( polyToDraw )
qp2.setCompositionMode( QtGui.QPainter.CompositionMode_SourceOver )
# Set the drawing to a special pattern
brush = QtGui.QBrush(col,QtCore.Qt.DiagCrossPattern)
qp2.setBrush(brush)
qp2.drawPolygon( polyToDraw )
# Draw outline of selected object dotted
if self.highlightObj:
brush = QtGui.QBrush(QtCore.Qt.NoBrush)
qp2.setBrush(brush)
qp2.setPen(QtCore.Qt.DashLine)
polyToDraw = self.getPolygon(self.highlightObj) * QtGui.QTransform.fromScale(self.scale,self.scale)
qp2.drawPolygon( polyToDraw )
# End the drawing of the overlay
qp2.end()
# Save QPainter settings to stack
qp.save()
# Define transparency
qp.setOpacity(self.transp)
# Draw the overlay image
qp.drawImage(self.xoff,self.yoff,overlay)
# Restore settings
qp.restore()
return overlay
def getBoundingBox(self, obj):
bbox = QtCore.QRectF(obj.bbox[0], obj.bbox[1], obj.bbox[2], obj.bbox[3])
bboxVis = QtCore.QRectF(obj.bboxVis[0], obj.bboxVis[1], obj.bboxVis[2], obj.bboxVis[3])
return bbox, bboxVis
def scaleBoundingBox(self, bbox):
bboxToDraw = copy.deepcopy(bbox)
x,y,w,h = bboxToDraw.getRect()
bboxToDraw.setTopLeft(QtCore.QPointF(x*self.scale, y*self.scale))
bboxToDraw.setSize(QtCore.QSizeF(w*self.scale, h*self.scale))
return bboxToDraw
# Draw the labels in the given QPainter qp
# optionally provide a list of labels to ignore
def drawBboxes(self, qp, ignore = []):
if self.image.isNull() or self.w == 0 or self.h == 0:
return
if not self.annotation:
return
# The overlay is created in the viewing coordinates
# This way, the drawing is more dense and the polygon edges are nicer
# We create an image that is the overlay
# Within this image we draw using another QPainter
# Finally we use the real QPainter to overlay the overlay-image on what is drawn so far
# The image that is used to draw the overlays
overlay = QtGui.QImage( self.w, self.h, QtGui.QImage.Format_ARGB32_Premultiplied )
# Fill the image
col = QtGui.QColor(0, 0, 0, 0)
overlay.fill( col )
# Create a new QPainter that draws in the overlay image
qp2 = QtGui.QPainter()
qp2.begin(overlay)
# Draw all objects
for obj in self.annotation.objects:
bbox, bboxVis = self.getBoundingBox(obj)
bboxToDraw = self.scaleBoundingBox(bbox)
bboxVisToDraw = self.scaleBoundingBox(bbox)
# The label of the object
name = obj.label
# If we do not know a color for this label, warn the user
if name not in name2labelCp:
print("The annotations contain unknown labels. This should not happen. Please inform the datasets authors. Thank you!")
print("Details: label '{}', file '{}'".format(name,self.currentLabelFile))
continue
# Reset brush for QPainter object
qp2.setBrush(QtGui.QBrush())
# Color from color table
col = QtGui.QColor( *name2labelCp[name].color )
if name2labelCp[name].hasInstances:
if self.highlightObj and obj == self.highlightObj:
pen = QtGui.QPen(QtGui.QBrush( col ), 5.0)
else:
pen = QtGui.QPen(QtGui.QBrush( col ), 3.0)
qp2.setPen(pen)
qp2.setOpacity(1.0)
qp2.drawRect( bboxToDraw )
if self.highlightObj and obj == self.highlightObj:
pen = QtGui.QPen(QtGui.QBrush( col ), 3.0, style=QtCore.Qt.DotLine)
qp2.setPen(pen)
qp2.setOpacity(1.0)
qp2.drawRect( bboxVisToDraw )
else:
pen = QtGui.QPen(QtGui.QBrush( col ), 1.0, style=QtCore.Qt.DashLine)
qp2.setPen(pen)
qp2.setOpacity(1.0)
qp2.drawRect( bboxVisToDraw )
qp2.setBrush(QtGui.QBrush( col, QtCore.Qt.SolidPattern ))
qp2.setOpacity(0.4)
qp2.drawRect( bboxVisToDraw )
else:
if self.highlightObj and obj == self.highlightObj:
pen = QtGui.QPen(QtGui.QBrush( col ), 3.0)
qp2.setPen(pen)
qp2.setBrush(QtGui.QBrush( col, QtCore.Qt.NoBrush ))
else:
pen = QtGui.QPen(QtGui.QBrush( col ), 1.0)
qp2.setPen(pen)
qp2.setBrush(QtGui.QBrush( col, QtCore.Qt.DiagCrossPattern ))
qp2.setOpacity(1.0)
qp2.drawRect( bboxToDraw )
# End the drawing of the overlay
qp2.end()
# Save QPainter settings to stack
qp.save()
# Define transparency
qp.setOpacity(self.transp)
# Draw the overlay image
qp.drawImage(self.xoff,self.yoff,overlay)
# Restore settings
qp.restore()
return overlay
# Draw the label name next to the mouse
def drawLabelAtMouse(self, qp):
# Nothing to do without a highlighted object
if not self.highlightObj:
return
# Nothing to without a mouse position
if not self.mousePosOrig:
return
# Save QPainter settings to stack
qp.save()
# That is the mouse positiong
mouse = self.mousePosOrig
# Will show zoom
showZoom = self.zoom and not self.image.isNull() and self.w and self.h
# The text that is written next to the mouse
mouseText = self.highlightObj.label
# Where to write the text
# Depends on the zoom (additional offset to mouse to make space for zoom?)
# The location in the image (if we are at the top we want to write below of the mouse)
off = 36
if showZoom:
off += self.zoomSize/2
if mouse.y()-off > self.toolbar.height():
top = mouse.y()-off
btm = mouse.y()
vAlign = QtCore.Qt.AlignTop
else:
# The height of the cursor
if not showZoom:
off += 20
top = mouse.y()
btm = mouse.y()+off
vAlign = QtCore.Qt.AlignBottom
# Here we can draw
rect = QtCore.QRect()
rect.setTopLeft(QtCore.QPoint(mouse.x()-200,top))
rect.setBottomRight(QtCore.QPoint(mouse.x()+200,btm))
# The color
qp.setPen(QtGui.QColor('white'))
# The font to use
font = QtGui.QFont("Helvetica",20,QtGui.QFont.Bold)
qp.setFont(font)
# Non-transparent
qp.setOpacity(1)
# Draw the text, horizontally centered
qp.drawText(rect,QtCore.Qt.AlignHCenter|vAlign,mouseText)
# Restore settings
qp.restore()
# Draw the zoom
def drawZoom(self,qp,overlay):
# Zoom disabled?
if not self.zoom:
return
# No image
if self.image.isNull() or not self.w or not self.h:
return
# No mouse
if not self.mousePosOrig:
return
# Abbrevation for the zoom window size
zoomSize = self.zoomSize
# Abbrevation for the mouse position
mouse = self.mousePosOrig
# The pixel that is the zoom center
pix = self.mousePosScaled
# The size of the part of the image that is drawn in the zoom window
selSize = zoomSize / ( self.zoomFactor * self.zoomFactor )
# The selection window for the image
sel = QtCore.QRectF(pix.x() -selSize/2 ,pix.y() -selSize/2 ,selSize,selSize )
# The selection window for the widget
view = QtCore.QRectF(mouse.x()-zoomSize/2,mouse.y()-zoomSize/2,zoomSize,zoomSize)
if overlay :
overlay_scaled = overlay.scaled(self.image.width(), self.image.height())
else :
overlay_scaled = QtGui.QImage( self.image.width(), self.image.height(), QtGui.QImage.Format_ARGB32_Premultiplied )
# Show the zoom image
qp.save()
qp.drawImage(view,self.image,sel)
qp.setOpacity(self.transp)
qp.drawImage(view,overlay_scaled,sel)
qp.restore()
# Draw disparities
def drawDisp( self , qp ):
if not self.dispOverlay:
return
# Save QPainter settings to stack
qp.save()
# Define transparency
qp.setOpacity(self.transp)
# Draw the overlay image
qp.drawImage(QtCore.QRect( self.xoff, self.yoff, self.w, self.h ),self.dispOverlay)
# Restore settings
qp.restore()
return self.dispOverlay
#############################
## Mouse/keyboard events
#############################
# Mouse moved
# Need to save the mouse position
# Need to drag a polygon point
# Need to update the mouse selected object
def mouseMoveEvent(self,event):
if self.image.isNull() or self.w == 0 or self.h == 0:
return
mousePosOrig = QtCore.QPointF( event.x() , event.y() )
mousePosScaled = QtCore.QPointF( float(mousePosOrig.x() - self.xoff) / self.scale , float(mousePosOrig.y() - self.yoff) / self.scale )
mouseOutsideImage = not self.image.rect().contains( mousePosScaled.toPoint() )
mousePosScaled.setX( max( mousePosScaled.x() , 0. ) )
mousePosScaled.setY( max( mousePosScaled.y() , 0. ) )
mousePosScaled.setX( min( mousePosScaled.x() , self.image.rect().right() ) )
mousePosScaled.setY( min( mousePosScaled.y() , self.image.rect().bottom() ) )
if not self.image.rect().contains( mousePosScaled.toPoint() ):
print(self.image.rect())
print(mousePosScaled.toPoint())
self.mousePosScaled = None
self.mousePosOrig = None
self.updateMouseObject()
self.update()
return
self.mousePosScaled = mousePosScaled
self.mousePosOrig = mousePosOrig
self.mouseOutsideImage = mouseOutsideImage
# Redraw
self.updateMouseObject()
self.update()
# Mouse left the widget
def leaveEvent(self, event):
self.mousePosOrig = None
self.mousePosScaled = None
self.mouseOutsideImage = True
# Mouse wheel scrolled
def wheelEvent(self, event):
ctrlPressed = event.modifiers() & QtCore.Qt.ControlModifier
deltaDegree = event.delta() / 8 # Rotation in degree
deltaSteps = deltaDegree / 15 # Usually one step on the mouse is 15 degrees
if ctrlPressed:
self.transp = max(min(self.transp+(deltaSteps*0.1),1.0),0.0)
self.update()
else:
if self.zoom:
# If shift is pressed, change zoom window size
if event.modifiers() and QtCore.Qt.Key_Shift:
self.zoomSize += deltaSteps * 10
self.zoomSize = max( self.zoomSize, 10 )
self.zoomSize = min( self.zoomSize, 1000 )
# Change zoom factor
else:
self.zoomFactor += deltaSteps * 0.05
self.zoomFactor = max( self.zoomFactor, 0.1 )
self.zoomFactor = min( self.zoomFactor, 10 )
self.update()
#############################
## Little helper methods
#############################
# Helper method that sets tooltip and statustip
# Provide an QAction and the tip text
# This text is appended with a hotkeys and then assigned
def setTip( self, action, tip ):
tip += " (Hotkeys: '" + "', '".join([str(s.toString()) for s in action.shortcuts()]) + "')"
action.setStatusTip(tip)
action.setToolTip(tip)
# Update the object that is selected by the current mouse curser
def updateMouseObject(self):
self.mouseObj = -1
if self.mousePosScaled is None:
return
for idx in reversed(range(len(self.annotation.objects))):
obj = self.annotation.objects[idx]
if obj.objectType == CsObjectType.POLY:
if self.getPolygon(obj).containsPoint(self.mousePosScaled, QtCore.Qt.OddEvenFill):
self.mouseObj = idx
break
elif obj.objectType == CsObjectType.BBOX:
bbox, _ = self.getBoundingBox(obj)
if bbox.contains(self.mousePosScaled):
self.mouseObj = idx
break
# Clear the current labels
def clearAnnotation(self):
self.annotation = None
self.currentLabelFile = ""
def getCityFromUser(self):
# Reset the status bar to this message when leaving
restoreMessage = self.statusBar().currentMessage()
if 'CITYSCAPES_DATASET' in os.environ:
csPath = os.environ['CITYSCAPES_DATASET']
else:
csPath = os.path.join(os.path.dirname(os.path.realpath(__file__)),'..','..')
availableCities = []
annotations = [ "gtFine" , "gtCoarse" , "gtBboxCityPersons" ]
splits = [ "train_extra" , "train" , "val" , "test" ]
for gt in annotations:
for split in splits:
cities = glob.glob(os.path.join(csPath, gt, split, '*'))
cities.sort()
availableCities.extend( [ (split,gt,os.path.basename(c)) for c in cities if os.listdir(c) ] )
# List of possible labels
items = [split + ", " + gt + ", " + city for (split,gt,city) in availableCities]
# Specify title
dlgTitle = "Select new city"
message = dlgTitle
question = dlgTitle
message = "Select city for viewing"
question = "Which city would you like to view?"
self.statusBar().showMessage(message)
if items:
# Create and wait for dialog
(item, ok) = QtGui.QInputDialog.getItem(self, dlgTitle, question,
items, 0, False)
# Restore message
self.statusBar().showMessage(restoreMessage)
if ok and item:
(split, gt, city) = [str(i) for i in item.split(', ')]
if split == 'test' and not self.showDisparity:
self.transp = 0.1
else:
self.transp = 0.5
self.city = os.path.normpath(os.path.join(csPath, "leftImg8bit", split, city))
self.labelPath = os.path.normpath(os.path.join(csPath, gt , split, city))
self.dispPath = os.path.normpath(os.path.join(csPath, "disparity" , split, city))
if gt in [ "gtFine", "gtCoarse" ]:
self.gtType = CsObjectType.POLY
elif gt in [ "gtBboxCityPersons" ]:
self.gtType = CsObjectType.BBOX
self.loadCity()
self.imageChanged()
else:
warning = ""
warning += "The data was not found. Please:\n\n"
warning += " - make sure the scripts folder is in the Cityscapes root folder\n"
warning += "or\n"
warning += " - set CITYSCAPES_DATASET to the Cityscapes root folder\n"
warning += " e.g. 'export CITYSCAPES_DATASET=<root_path>'\n"
reply = QtGui.QMessageBox.information(self, "ERROR!", warning,
QtGui.QMessageBox.Ok)
if reply == QtGui.QMessageBox.Ok:
sys.exit()
return
# Determine if the given candidate for a label path makes sense
def isLabelPathValid(self, labelPath):
return os.path.isdir(labelPath)
# Get the filename where to load labels
# Returns empty string if not possible
def getLabelFilename(self):
# And we need to have a directory where labels should be searched
if not self.labelPath:
return ""
# Without the name of the current images, there is also nothing we can do
if not self.currentFile:
return ""
# Check if the label directory is valid.
if not self.isLabelPathValid(self.labelPath):
return ""
# Generate the filename of the label file
filename = os.path.basename(self.currentFile)
filename = filename.replace(self.imageExt, self.gtExt)
filename = os.path.join(self.labelPath, filename)
search = glob.glob(filename)
if not search:
return ""
filename = os.path.normpath(search[0])
return filename
# Get the filename where to load disparities
# Returns empty string if not possible
def getDisparityFilename( self ):
# And we need to have a directory where disparities should be searched
if not self.dispPath:
return ""
# Without the name of the current images, there is also nothing we can do
if not self.currentFile:
return ""
# Check if the label directory is valid.
if not os.path.isdir(self.dispPath):
return ""
# Generate the filename of the label file
filename = os.path.basename( self.currentFile )
filename = filename.replace( self.imageExt , self.dispExt )
filename = os.path.join( self.dispPath , filename )
filename = os.path.normpath(filename)
return filename
# Disable the popup menu on right click
def createPopupMenu(self):
pass
def json2labelImg(inJson,outImg,encoding="ids"):
annotation = Annotation()
annotation.fromJsonFile(inJson)
labelImg = createLabelImage( annotation , encoding )
labelImg.save( outImg )
def store_annotations(self, annotations):
for annotation in annotations:
ann = Annotation()
ann.parse_json(annotation)
ann.add_to_graph(self.sparql)
return
def json2instanceImg(inJson,outImg,encoding="ids"):
annotation = Annotation()
annotation.fromJsonFile(inJson)
instanceImg = createInstanceImage( annotation , encoding )
instanceImg.save( outImg )
def __init__(self, limit=10000, **options):
Annotation.__init__(self, **options)
self.limit = limit
def __init__(self, description, attributes, inner_attributes=None,
mode='split_workbooks', single_workbook_name=None,
min_tokens=2, max_tokens=3000, **options):
'''
Classify sentences based on some of their attributes, and write
them to XLS files. Lists of sentences will be taken from different
users, distributed as evenly as possible (cf. MixUsers).
sentences:
list of sentences (same as Filter.sentences). [Inherited
from Annotation base class]
description:
a textual description of the annotation. Will appear in a
'meta' sheet of each workbook. Mandatory.
attributes:
a single attribute, or a tuple of attributes, by which
the sentences will be classified into annotation files (the
exact behavior depends on the mode)
inner_attributes:
only applicable for the split workbooks mode. Combinations of
the inner attributes will form the names of sheets within
each workbook
mode:
'single_workbook': one workbook with different sheet for each
attribute combination
'split_workbooks': separate workbook for each attribute combination
'single_sheet': single workbook with a single sheet for all
attribute combinations (but combination will appear in a
dedicated spreadsheet column)
single_workbook_name:
name of single workbook, if applicable
min_tokens:
if an attribute combination has less than this number of occurences
discard this combination (do not write it to annotation files)
max_tokens:
stop writing sentences with any given attribute combination after
this amount of tokens of this combination have been written
Example:
ByAttributeAnnotation("30 to 40 dative verbs, topicalized",
sentences=sentences, attributes=['lemma', 'argument'])
'''
Annotation.__init__(self, description, **options)
self.attributes = attributes
if mode not in ('split_workbooks', 'single_workbook', 'single_sheet'):
raise ValueError('unknown mode "%s"' % mode)
if mode != 'split_workbooks' and inner_attributes:
raise ValueError('inner_attributes can only be set when' \
'mode is "split_workbooks"')
self.mode = mode
self.inner_attributes = inner_attributes
self.single_workbook_name = single_workbook_name
self.min_tokens = min_tokens
self.max_tokens = max_tokens
self.workbooks = {}
