def clean_tree(email, token, label_tree_id):
# Init API
api = Api(email, token)
# Get label tree info
label_tree_info = api.get('label-trees/{}'.format(label_tree_id)).json()['labels']
z_ids = []
for label_info in label_tree_info:
if label_info['name'].startswith('z_'):
z_ids.append(label_info['id'])
# Remove z kids
for label_info in label_tree_info:
if label_info['parent_id'] in z_ids:
try:
api.post('label-trees/{}/merge-labels'.format(label_tree_id), json={'remove': [label_info['id']]})
except:
print(label_info)
#print(api.get('labels/{}/annotations'.format(label_info['id'])).json())
#print(api.get('annotations/{}'.format('7875997')).json())
# Empty parents
for zid in z_ids:
parent = api.get('labels/{}/annotations'.format(zid)).json()
for annotation_id in parent:
api.delete('annotations/{}'.format(annotation_id))
# Remove parents
#api.post('label-trees/{}/merge-labels'.format(label_tree_id), json={'remove': z_ids})
print('{}'.format('\n'.join([(' '.join([l['name'], str(l['id']), str(l['parent_id'])])) for l in label_tree_info])))
# Create new parent
#api.post('label-trees/{}/labels'.format(label_tree_id), json={'color': "#0be6c8",
# "name": "HydroCorals"
# })
# Move HydroCorals
#api.put('labels/{}'.format(133260), json={"parent_id": 155169, "name": 'Branching'})
#api.put('labels/{}'.format(133261), json={"parent_id": 155169, "name": 'Encrusting'})
# Rename Hydroids
#api.put('labels/{}'.format(133262), json={"name": 'Colonial feather'})
#api.put('labels/{}'.format(133263), json={"name": 'Solitary'})
#api.put('labels/{}'.format(133259), json={"name": 'Matrix'})
#api.put('labels/{}'.format(133258), json={"name": 'Hydroids'})
print('\n\n---------- Finished ----------\n\n')
def fetch_incorrect_laser(email, token, survey_name, ofolder):
# Init API
api = Api(email, token)
# Get all surveys
surveys = api.get('volumes').json()
# Get info for survey of interest
survey_dict = get_survey(surveys, survey_name)
survey_id = survey_dict['id']
# Get the list of image IDs that belong to the survey of interest
image_ids = api.get('volumes/{}/images'.format(survey_id)).json()
print("\n\t... Found {} images.".format(len(image_ids)))
# Init result dict
dict = {'image_fname': [], 'image_id': [], 'n_laser': []}
print("\n... Looping across images.")
for image_id in image_ids:
# Get all annotations for the current image
annotation_info = api.get(
'images/{}/annotations'.format(image_id)).json()
# Get laser annotations for the given image
laser_annotations = get_label_annotations(annotation_info, LABEL_NAME)
# Check if nb laser annotations is wrong
if len(laser_annotations) != 2:
dict['image_id'].append(image_id)
dict['image_fname'].append(
api.get('images/{}'.format(image_id)).json()['filename'])
dict['n_laser'].append(len(laser_annotations))
# Output folder
if os.path.isdir(ofolder):
print('\nOutput folder already exists: {}.'.format(ofolder))
else:
print('\nCreating output folder: {}.'.format(ofolder))
os.makedirs(ofolder)
fname_out = os.path.join(ofolder, survey_name + '_incorrect_laser.csv')
df = pd.DataFrame.from_dict(dict)
df.to_csv(fname_out)
print('\nSaving results in: {}.'.format(fname_out))
print('\n\n---------- Finished ----------\n\n')
def convert_laser_circle_to_point(email, token, survey_name):
# Init API
api = Api(email, token)
# Get the available annotation shapes.
# https://biigle.de/doc/api/index.html#api-Shapes-IndexShapes
shapes = api.get('shapes').json()
shapes = {s['name']: s['id'] for s in shapes}
# Get all surveys
surveys = api.get('volumes').json()
# Get info for survey of interest
survey_dict = get_survey(surveys, survey_name)
survey_id = survey_dict['id']
# Get the list of image IDs that belong to the survey of interest
image_ids = api.get('volumes/{}/images'.format(survey_id)).json()
print("\n\t... Found {} images.".format(len(image_ids)))
print("\n... Looping across images.")
for image_id in image_ids:
# Check if laser point already exist
try:
laser_info = api.get(
'images/{}/laserpoints'.format(image_id)).json()
if laser_info:
print('\tAlready a laser point for image {}, skipping.'.format(
image_id))
continue
except:
# Get all annotations for the current image
annotation_info = api.get(
'images/{}/annotations'.format(image_id)).json()
# Get laser annotations for the given image
laser_annotations = get_label_annotations(annotation_info,
LABEL_NAME)
# Check if there is some laser annotations
if len(laser_annotations):
# Loop across laser points
for laser in laser_annotations:
# Check if laser shape is not a Point
if laser['shape_id'] != shapes['Point']:
# Check if laser shape is a circle
if laser['shape_id'] != 4:
print(
"NOT IMPLEMENTED YET: laser shape: {}.".format(
laser['shape_id']))
exit()
print(
'\t\tImage {} --> converting annotation {} to Point.'
.format(laser['image_id'], laser["id"]))
api.put('annotations/{}'.format(laser["id"]),
json={
'shape_id': shapes['Point'],
'points': laser['points'][:-1]
})
else:
print(
'\t\tCorrect laser annotation shape on image {}, annotation '
'{}.'.format(laser['image_id'], laser["id"]))
else:
print('\tNo {} found for image {}, skipping.'.format(
LABEL_NAME, image_id))
continue
print('\n\n---------- Finished ----------\n\n')
def remove_maia_duplicates(email,
token,
label_tree_id,
survey_name,
maia_date,
range_image=None,
label_id=None):
# Init API
api = Api(email, token)
# Get all surveys
surveys = api.get('volumes').json()
# Get info for survey of interest
survey_dict = get_survey(surveys, survey_name)
survey_id = survey_dict['id']
# Get all labels from label tree
if label_id is None:
label_list = api.get(
'label-trees/{}'.format(label_tree_id)).json()['labels']
else:
label_list = [{'id': i} for i in label_id]
cmpt_new = 0
# Loop through labels
for label in label_list:
# Get annotations
annotation_list = api.get(
'volumes/{}/annotations/filter/label/{}'.format(
survey_id, label['id'])).json()
if len(annotation_list):
print('\nProcessing Label ID: {}.'.format(label['id']))
annotation_id_list = list(annotation_list.keys())
for id_ in annotation_id_list:
annotation_info = api.get('annotations/{}'.format(id_)).json()
if annotation_info['created_at'].startswith(maia_date):
if range_image is not None and annotation_info['image_id'] in \
list(range(range_image[0], range_image[1]+1)):
labels_in_img = [
l for l in api.get('images/{}/annotations'.format(
annotation_info['image_id'])).json()
]
labels_in_img = [
l for l in labels_in_img if
int(l['labels'][0]['label_id']) == int(label['id'])
and not l['created_at'].startswith(maia_date)
]
if len(labels_in_img):
print('\tDeleting: ')
print(annotation_info)
api.delete('annotations/{}'.format(id_))
else:
print('\tSave: ')
print(annotation_info)
cmpt_new += 1
else:
cmpt_new += 1
print('\nNb of new annotations thanks of MAIA: {}.'.format(cmpt_new))
print('\n\n---------- Finished ----------\n\n')
def review_annotations(email,
token,
label_tree_id,
input_folder,
wnd_dims,
n_patches=None,
label_folder=[]):
# Init API
api = Api(email, token)
# subfolder list
if label_folder is None or len(label_folder) == 0:
taxa_list = [
f for f in os.listdir(input_folder)
if os.path.isdir(os.path.join(input_folder, f))
]
print('\nFound {} taxa.'.format(len(taxa_list)))
else:
taxa_list = []
for l in label_folder:
if os.path.isdir(os.path.join(input_folder, l)):
taxa_list.append(l)
print('\nReviewing {}.'.format(os.path.join(input_folder, l)))
else:
print('\nFolder not found: {}.'.format(
os.path.join(input_folder, l)))
exit()
# Get all labels from label tree
label_tree_info = api.get(
'label-trees/{}'.format(label_tree_id)).json()['labels']
label_dict = biigle_utils.get_folders_match_tree(label_tree_info)
# Log file
fname_log = os.path.join(
input_folder,
"logfile_" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ".csv")
dict_log = {"annotation_id": [], "from": [], "to": [], "who": []}
# Init GUI
gui = Tk()
gui.geometry(
str(wnd_dims[2]) + "x" + str(wnd_dims[3]) + "+" + str(550) + "+" +
str(0))
# Init main window
app = Window(master=gui,
height=wnd_dims[1],
width=wnd_dims[0],
list_labels=sorted(list(label_dict.keys())))
# Loop across taxa
for taxa in taxa_list:
print('\nReviewing: {} ...'.format(taxa))
taxa_folder = os.path.join(input_folder, taxa)
fname_list = [f for f in os.listdir(taxa_folder) if f.endswith('.jpg')]
if n_patches is not None and len(fname_list) > n_patches:
fname_list = random.sample(fname_list, n_patches)
fname_nested = [
fname_list[i:i + N_IMG_PER_WND]
for i in range(0, len(fname_list), N_IMG_PER_WND)
]
# Review per chuncks
for fname_sublist in tqdm(fname_nested, desc="Reviewing"):
path_list = [os.path.join(taxa_folder, f) for f in fname_sublist]
# Show image and wait
app.show_img(path_list, taxa)
app.wait_variable(app.accept_key)
# Close window
app.sample.destroy()
# Get changes
change_list = app.changes_list
# Reset
app.reset_accept()
# Loop across changes
for change in change_list:
annotation_id, annotation_folder = change
# Inform about change
image_id = api.get('image-annotations/{}'.format(
annotation_id)).json()['image_id']
image_info = api.get('images/{}'.format(image_id)).json()
print(
'\n\tChange annotation {} from {} to {} on image {} (image ID: {}).'
.format(annotation_id, taxa, annotation_folder,
image_info['filename'], image_info['id']))
# Change label
if annotation_folder != "NOT_VME":
api.post(
'image-annotations/{}/labels'.format(annotation_id),
json={
'label_id': label_dict[annotation_folder]['id'],
'confidence': 1
})
# Remove old label
old_label_id = [
ann['id']
for ann in api.get('image-annotations/{}/labels'.format(
annotation_id)).json()
if ann["label_id"] == label_dict[taxa]['id']
][0]
api.delete('image-annotation-labels/{}'.format(old_label_id))
# fill Log file
dict_log["annotation_id"].append(annotation_id)
dict_log["from"].append(taxa)
dict_log["to"].append(annotation_folder)
dict_log["who"].append(email)
# Move patch folder
ifname = os.path.join(input_folder, taxa,
str(annotation_id) + '.jpg')
ofname = os.path.join(input_folder, annotation_folder,
str(annotation_id) + '.jpg')
if not os.path.isdir(
os.path.join(input_folder, annotation_folder)):
os.makedirs(os.path.join(input_folder, annotation_folder))
shutil.move(ifname, ofname)
# Quit if asked
if app.quit_:
print('\nQuitting the review process.')
gui.destroy()
break
if app.quit_:
break
# Quit if not done yet
if not app.quit_:
gui.destroy()
# Save Log file
df = pd.DataFrame.from_dict(dict_log)
df.to_csv(fname_log, index=False)
print('\nSaving log file in: {}'.format(fname_log))
def move_annotations(email,
token,
input_folder,
output_folder,
label_tree_id,
src,
dest,
batch_size=100):
# Init API
api = Api(email, token)
# Get all labels from label tree
label_tree_info = api.get(
'label-trees/{}'.format(label_tree_id)).json()['labels']
label_dict = biigle_utils.get_folders_match_tree(label_tree_info)
# Check folders exist
src_folder = os.path.join(output_folder, src)
dest_folder = os.path.join(output_folder, dest)
for f in [input_folder, src_folder]:
if not os.path.isdir(f):
print('\nFolder not found: {}'.format(f))
exit()
if not os.path.isdir(dest_folder) and dest != "NOT_VME":
if dest in label_dict:
print('\nCreating destination folder: {}'.format(dest_folder))
os.makedirs(dest_folder)
else:
print('\nUnknown destination category: {}'.format(dest))
exit()
# Log file
fname_log = os.path.join(
output_folder,
"logfile_" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ".csv")
dict_log = {"annotation_id": [], "from": [], "to": [], "who": []}
# Get annotation fnames
input_fname_list = [
f for f in os.listdir(input_folder) if f.endswith('.jpg')
]
# Check if all annotations from input folder are coming from source folder
input_fname_list_new = []
for idx, f in enumerate(input_fname_list):
src_fname_cur = os.path.join(src_folder, f)
if not os.path.isfile(src_fname_cur):
print('\tWARNING: {} is not found in {}'.format(f, src_folder))
else:
input_fname_list_new.append(f)
print('\nFound {} annotations to move.'.format(len(input_fname_list_new)))
# Nested list
input_fname_nested = [
input_fname_list_new[i:i + batch_size]
for i in range(0, len(input_fname_list_new), batch_size)
]
# Move per batch
for fname_sublist in tqdm(input_fname_nested, desc="Reviewing"):
annotation_ids = [f.split('.jpg')[0] for f in fname_sublist]
annotation_infos = [
api.get('image-annotations/{}'.format(annotation_id)).json()
for annotation_id in annotation_ids
]
# Create batch
batch = []
for annotation_info in annotation_infos:
if dest != "NOT_VME":
annotation_info["label_id"] = label_dict[dest]['id']
annotation_info["confidence"] = 1.00
batch.append(copy.copy(annotation_info))
# Run batch
if dest != "NOT_VME":
api.post('image-annotations', json=batch)
# Cleanup
for batch_info in batch:
# Print info
image_info = api.get('images/{}'.format(
batch_info['image_id'])).json()
print(
'\n\tChange annotation {} from {} to {} on image {} (image ID: {}).'
.format(batch_info['id'], src, dest, image_info['filename'],
image_info['id']))
# Detach old label
old_label_id = [
ann['id']
for ann in api.get('image-annotations/{}/labels'.format(
batch_info['id'])).json()
if ann["label_id"] == label_dict[src]['id']
][0]
api.delete('image-annotation-labels/{}'.format(old_label_id))
# Move file
if dest != "NOT_VME":
shutil.move(
os.path.join(src_folder,
str(batch_info['id']) + '.jpg'),
os.path.join(dest_folder,
str(batch_info['id']) + '.jpg'))
else:
os.remove(
os.path.join(src_folder,
str(batch_info['id']) + '.jpg'))
# fill Log file
dict_log["annotation_id"].append(batch_info['id'])
dict_log["from"].append(src)
dict_log["to"].append(dest)
dict_log["who"].append(email)
# Save Log file
df = pd.DataFrame.from_dict(dict_log)
df.to_csv(fname_log, index=False)
print('\nSaving log file in: {}'.format(fname_log))
def pull_patches(email,
token,
survey_name,
label_tree_id,
output_folder,
range_image=[],
label_id=None,
skip=False):
# Init API
api = Api(email, token)
# Get all surveys
surveys = api.get('volumes').json()
# Get info for survey of interest
survey_dict = get_survey(surveys, survey_name)
survey_id = survey_dict['id']
# Output folder
if os.path.isdir(output_folder):
print('\nOutput folder already exists: {}.'.format(output_folder))
else:
print('\nCreating output folder: {}.'.format(output_folder))
os.makedirs(output_folder)
# Get label info
labels_info_list = api.get(
'volumes/{}/annotation-labels'.format(survey_id)).json()
label_tree_info = api.get('label-trees/{}'.format(label_tree_id)).json()
labels_info_list = add_parent_name(label_tree_info['labels'],
labels_info_list)
# Pick the label of interest
if label_id != None:
labels_info_list = [
label_info for label_info in labels_info_list
if label_info['id'] == label_id
]
# Init patch URL
patch_url = 'https://biigle.de/storage/largo-patches/{}/{}/{}/{}.jpg'
# List annotations of interest
if len(range_image):
list_img_annotations = []
images = api.get('volumes/{}/files'.format(survey_id)).json()
images_range = list(range(range_image[0], range_image[1] + 1))
images_of_interest = [ii for ii in images_range if ii in images]
for image_id in tqdm(images_of_interest,
desc="Fetching annotations of interest"):
try:
annotations_list = api.get(
'images/{}/annotations'.format(image_id)).json()
annotations_id_list = [str(a['id']) for a in annotations_list]
list_img_annotations = list_img_annotations + annotations_id_list
except:
pass
else:
list_img_annotations = None
for label_dict in labels_info_list:
annotations = api.get(
'volumes/{}/image-annotations/filter/label/{}'.format(
survey_id, label_dict['id'])).json()
if len(annotations) > 0:
# Label full name
full_name = label_dict['name'].replace(' ', '_')
if len(label_dict['parent_name']) > 0:
full_name = label_dict['parent_name'].replace(
' ', '_') + '-' + full_name
print('\tPulling patches for Label: {}.'.format(full_name))
print('\tFound {} annotations.'.format(len(annotations)))
# Label output folder
label_ofolder = os.path.join(output_folder, full_name)
if os.path.isdir(label_ofolder):
print('\tOutput folder already exists: {}.'.format(
label_ofolder))
if skip:
print('\t\tSkipping.')
continue
else:
print('\tCreating output folder: {}.'.format(label_ofolder))
os.makedirs(label_ofolder)
for annotation_id, image_uuid in tqdm(annotations.items(),
desc="Pulling"):
if list_img_annotations is None or (
len(list_img_annotations) > 0
and annotation_id in list_img_annotations):
url = patch_url.format(image_uuid[:2], image_uuid[2:4],
image_uuid, annotation_id)
patch = requests.get(url, stream=True)
if patch.ok != True:
print('Failed to fetch {}'.format(url))
continue
with open(
os.path.join(label_ofolder,
'{}.jpg'.format(annotation_id)),
'wb') as f:
patch.raw.decode_content = True
shutil.copyfileobj(patch.raw, f)
print('\n\n---------- Finished ----------\n\n')