def export_region_annotation_list(manifest_name): manifest_txt = open(export_annotation_root + manifest_name + '_slide_table.txt').readlines() result = [] for wsi in manifest_txt: info = wsi.split('\t') nuclei_annotation_root = "Data/nuclei_annotation_data/" annotation_project_root = nuclei_annotation_root + manifest_name + '/' if not os.path.exists(annotation_project_root + info[1] + '/'): continue tba_list_db = annotation_project_root + info[1] + '/' + 'tba_list.db' if not os.path.exists(tba_list_db): continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() if len(tba_result) == 0: continue wsi += "\t" + str(len(tba_result)) + "\t" + time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.stat(tba_list_db).st_mtime)) + "\t\t\t\t\t" temp = { "Slide ID": wsi.split("\t")[0], "UUID": wsi.split("\t")[1], "svs file": wsi.split("\t")[2], "a1": wsi.split("\t")[3], "a2": wsi.split("\t")[4], "a3": wsi.split("\t")[5], "a4": wsi.split("\t")[6], "a5": wsi.split("\t")[7], "a6": wsi.split("\t")[8], } result.append(temp) return result
def nuclei_annotation_add_sw(): slide_id = request.args.get('slide_id', default=1, type=int) annotator_id = request.args.get('annotator_id', default=1, type=int) annotation_project = request.args.get('project', default="None", type=str) slide_uuid = request.args.get('slide_uuid', default="", type=str) allowed_annotator = range(7) if annotator_id in allowed_annotator: x = request.args.get('x', 0, type=int) y = request.args.get('y', 0, type=int) annotation_root_folder = nuclei_annotation_data_root + annotation_project + '/' + slide_uuid + '/' if not os.path.exists(annotation_root_folder): os.mkdir(annotation_root_folder) tba_list_db = annotation_root_folder + 'tba_list.db' db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) db.incert_RegionCentre(-1, x, y) message = 'Successfully added diagnostic region' else: message = 'Only annotator ' + str( allowed_annotator) + ' is allowed to add diagnostic region' return jsonify(status=message, num_status=1)
def export_nuclei_annotation_list(manifest_name): manifest_txt = open(export_annotation_root + manifest_name + '_slide_table.txt').readlines() annotation_project_root = freehand_annotation_root + manifest_name + '/' annotator_no = 6 result = [] for wsi in manifest_txt: info = wsi.split('\t') flag = 0 nuclei_annotation_root = "Data/nuclei_annotation_data/" annotation_project_root = nuclei_annotation_root + manifest_name + '/' if not os.path.exists(annotation_project_root + info[1] + '/'): continue tba_list_db = annotation_project_root + info[1] + '/' + 'tba_list.db' if not os.path.exists(tba_list_db): continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() if len(tba_result) == 0: continue for annotator_id in range(annotator_no): annotated = 0 st_time_list = [] for item in tba_result: if os.path.exists(annotation_project_root + info[1] + '/' + 'a' + str(annotator_id) + '_r' + str(item[0]) + "_annotation.txt") and \ sum(numpy.loadtxt(annotation_project_root + info[1] + '/' + 'a' + str(annotator_id) + '_r' + str(item[0]) + "_annotation.txt")) > 0: annotated += 1 st_time_list.append( os.stat(annotation_project_root + info[1] + '/' + 'a' + str(annotator_id) + '_r' + str(item[0]) + "_annotation.txt").st_mtime) if len(st_time_list) > 0: wsi += time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(max(st_time_list))) \ + ' (' + str(annotated) + '/' + str(len(tba_result)) + ') ' + '\t' flag = 1 else: wsi += "No Annotation" + '\t' if flag: temp = { "Slide ID": wsi.split("\t")[0], "UUID": wsi.split("\t")[1], "svs file": wsi.split("\t")[2], "a1": wsi.split("\t")[3], "a2": wsi.split("\t")[4], "a3": wsi.split("\t")[5], "a4": wsi.split("\t")[6], "a5": wsi.split("\t")[7], "a6": wsi.split("\t")[8], "Download Link": "<a target='_blank' href='/export_nuclei_annotation_single?" \ "manifest_file=Data/annotation_project_manifest/" + manifest_name + \ ".txt&slide_id=" + wsi.split("\t")[1] + "'> Export </a>", } result.append(temp) return result
def nuclei_annotation_v2_rm_sw(): slide_id = request.args.get('slide_id', default=1, type=int) annotator_id = request.args.get('annotator_id', default=1, type=int) annotation_project = request.args.get('project', default="None", type=str) slide_uuid = request.args.get('slide_uuid', default="", type=str) allowed_annotator = range(7) if annotator_id in allowed_annotator: sw_id = request.args.get('sw_id', 0, type=int) annotation_root_folder = nuclei_annotation_data_root + annotation_project + '/' + slide_uuid + '/' tba_list_db = annotation_root_folder + 'tba_list.db' db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) db.delete_RegionCentre(sw_id) original_pic_url = annotation_root_folder + 'r' + str( sw_id) + '.png' if os.path.exists(original_pic_url): os.remove(original_pic_url) region_image_url = annotation_root_folder + 'r' + str( sw_id) + '.txt' if os.path.exists(region_image_url): os.remove(region_image_url) for annotator_id in range(7): annotator_data_url = annotation_root_folder + 'a' + str( annotator_id) + '_r' + str(sw_id) + '.txt' if os.path.exists(annotator_data_url): os.remove(annotator_data_url) result_pic_url = annotation_root_folder + 'a' + str( annotator_id) + '_r' + str(sw_id) + '.png' if os.path.exists(result_pic_url): os.remove(result_pic_url) points_file_name = annotation_root_folder + 'a' + str(annotator_id) + '_r' + \ str(sw_id) + '_points' + '.txt' if os.path.exists(points_file_name): os.remove(points_file_name) grades_file_name = annotation_root_folder + 'a' + str(annotator_id) + '_r' + \ str(sw_id) + '_grades' + '.txt' if os.path.exists(grades_file_name): os.remove(grades_file_name) message = 'Successfully removed diagnostic region' else: message = message = 'Only annotator ' + str( allowed_annotator) + ' is allowed to remove diagnostic region' return jsonify(status=message, num_status=1)
def refresh_nuclei_annotation_progress(): global annotation_project_table filename = manifest_root + 'table.npy' if not os.path.exists(filename): refresh_npy() else: result = annotation_project_table.copy() # numpy.load(filename) for i in range(len(result)): manifest_txt = open(manifest_root + result[i][0] + '.txt').readlines() annotation_project_root = nuclei_annotation_root + result[i][ 0] + '/' annotator_no = 6 region_no = 0 finish_region_no = numpy.zeros(annotator_no) for wsi in manifest_txt: info = wsi.split('\t') if not os.path.exists(annotation_project_root + info[0] + '/'): continue tba_list_db = annotation_project_root + info[ 0] + '/' + 'tba_list.db' if not os.path.exists(tba_list_db): continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() region_no = region_no + len(tba_result) for item in tba_result: for annotator_id in range(annotator_no): if os.path.exists(annotation_project_root + info[0] + '/' + 'a' + str(annotator_id + 1) + '_r' + str(item[0]) + "_annotation.txt") and \ sum(numpy.loadtxt(annotation_project_root + info[0] + '/' + 'a' + str(annotator_id + 1) + '_r' + str( item[0]) + "_annotation.txt")) > 0: finish_region_no[annotator_id] += 1 result_str = "" # 'Total: ' + str(region_no) + ', <br/>' for annotator_id in range(annotator_no): result_str += str(annotator_id + 1) + ': ' + \ str(int(finish_region_no[annotator_id])) + ' /' + str(region_no) + ', ' if annotator_id % 2: result_str += '<br/>' result[i][4] = result_str filename = manifest_root + 'table.npy' result = numpy.array(result) annotation_project_table = result.copy() numpy.save(filename, result) # 保存为.npy格式 return
def nuclei_annotation_update_tb_list(): slide_id = request.args.get('slide_id', default=1, type=int) annotator_id = request.args.get('annotator_id', default=1, type=int) annotation_project = request.args.get('project', default="None", type=str) slide_uuid = request.args.get('slide_uuid', default="", type=str) annotation_root_folder = nuclei_annotation_data_root + annotation_project + '/' + slide_uuid + '/' if not os.path.exists(annotation_root_folder): os.mkdir(annotation_root_folder) tba_list_db = annotation_root_folder + 'tba_list.db' db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() return jsonify(max_region=len(tba_result), reg_list=tba_result)
def make_region_back(manifest_name, region_size, slide_ids=None): original_data_root = 'static/data/Original_data/' nuclei_annotation_data_root = "static/data/nuclei_annotation_data/" export_folder = "region/" + manifest_name + "_" + str(region_size) + "/" if os.path.exists(export_folder): shutil.rmtree(export_folder) os.mkdir(export_folder) manifest_file = open(export_folder + "manifest.txt", "w") if slide_ids == None: slide_ids = os.listdir(nuclei_annotation_data_root + manifest_name + '/') for slide_ID in slide_ids: try: print(slide_ID) wsi = manifest_controller.get_info_by_uuid(slide_ID) svs_file_path = original_data_root + wsi[1] + '/' + wsi[2] tba_list_db = nuclei_annotation_data_root + manifest_name + '/' + wsi[ 1] + '/' + 'tba_list.db' if not os.path.exists(tba_list_db): continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() if len(tba_result) > 0: if not os.path.exists(export_folder + wsi[2]): os.mkdir(export_folder + wsi[2]) oslide = openslide.OpenSlide(svs_file_path) manifest_file.writelines(wsi[1] + '\t' + wsi[2] + '\t' + 'None' + os.linesep) for item in tba_result: patch = oslide.read_region( (item[1] - 256 - int(region_size / 2), item[2] - 256 - int(region_size / 2)), 0, (region_size, region_size)) patch.save(export_folder + wsi[2] + '/' + str(item[0]) + '_' + str(item[1]) + '_' + str(item[2]) + '_' + str(region_size) + '.png') oslide.close() except Exception as e: print("ERROR: " + slide_ID) print(e) manifest_file.close() make_archive_threadsafe( "export/" + manifest_name + "_" + str(region_size) + 'zip', export_folder)
def export_region_annotation_data(manifest_file_url, region_size, manifest_txt=None, export_file=None): annotation_project_root = nuclei_annotation_root + manifest_file_url.rsplit( "/", 1)[1][:-4] + '/' if manifest_txt != None: export_annotation_root_temp = "static/cache/" + str(uuid.uuid4()) + "/" if export_file == None: export_file = export_annotation_root_temp.replace("/cache", "")[:-1] + '.zip' print(export_file) if os.path.exists(export_file): os.remove(export_file) export_path = export_annotation_root_temp if os.path.exists(export_path): shutil.rmtree(export_path) os.mkdir(export_path) else: manifest_txt = open(manifest_file_url).readlines() if not (not (manifest_file_url.rsplit("/", 1)[1][:-4] == "") and not (manifest_file_url.rsplit("/", 1)[1][:-4] is None)): return export_file = export_annotation_root + manifest_file_url.rsplit( "/", 1)[1][:-4] + '_region_annotation.zip' if os.path.exists(export_file): os.remove(export_file) export_path = export_annotation_root + manifest_file_url.rsplit( "/", 1)[1][:-4] + '/region_annotation/' if os.path.exists(export_path): shutil.rmtree(export_path) elif not os.path.exists(export_annotation_root + manifest_file_url.rsplit("/", 1)[1][:-4]): os.mkdir(export_annotation_root + manifest_file_url.rsplit("/", 1)[1][:-4]) os.mkdir(export_path) wsi_count = 0 for wsi in manifest_txt: print(wsi) wsi_count += 1 print( time.asctime(time.localtime(time.time())), "start export nuclei annotation:" + str(wsi_count) + '/' + str(len(manifest_txt))) info = wsi.split('\t') try: svs_file_path = original_data_root + info[0] + '/' + info[1] tba_list_db = annotation_project_root + '/' + info[ 0] + '/' + 'tba_list.db' print(svs_file_path) if not os.path.exists(tba_list_db): continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() if len(tba_result) > 0: if not os.path.exists(export_path + info[0]): os.mkdir(export_path + info[0]) oslide = openslide.OpenSlide(svs_file_path) for item in tba_result: patch = oslide.read_region( (item[1] - 256 - int(int(region_size) / 2), item[2] - 256 - int(int(region_size) / 2)), 0, (int(region_size), int(region_size))) patch.save(export_path + info[0] + '/' + str(item[0]) + '_' + str(item[1]) + '_' + str(item[2]) + '_' + str(int(region_size)) + '.png') oslide.close() except Exception as e: print("ERROR: ", wsi) print(e) if os.listdir(export_path): make_archive_threadsafe(export_file, export_path) return export_file
def export_nuclei_annotation_data(manifest_file_url, manifest_txt=None, export_file=None): annotation_project_root = nuclei_annotation_root + manifest_file_url.rsplit( "/", 1)[1][:-4] + '/' if manifest_txt != None: export_annotation_root_temp = "static/cache/" + str(uuid.uuid4()) + "/" if export_file == None: export_file = export_annotation_root_temp.replace("/cache", "")[:-1] + '.zip' print(export_file) if os.path.exists(export_file): os.remove(export_file) export_path = export_annotation_root_temp if os.path.exists(export_path): shutil.rmtree(export_path) os.mkdir(export_path) else: manifest_txt = open(manifest_file_url).readlines() if not (not (manifest_file_url.rsplit("/", 1)[1][:-4] == "") and not (manifest_file_url.rsplit("/", 1)[1][:-4] is None)): return export_file = export_annotation_root + manifest_file_url.rsplit( "/", 1)[1][:-4] + '_nuclei_annotation.zip' if os.path.exists(export_file): os.remove(export_file) export_path = export_annotation_root + manifest_file_url.rsplit( "/", 1)[1][:-4] + '/nuclei_annotation/' if os.path.exists(export_path): shutil.rmtree(export_path) elif not os.path.exists(export_annotation_root + manifest_file_url.rsplit("/", 1)[1][:-4]): os.mkdir(export_annotation_root + manifest_file_url.rsplit("/", 1)[1][:-4]) os.mkdir(export_path) colour = [[255, 0, 209], [0, 255, 255], [0, 0, 255], [0, 0, 255], [255, 191, 0], [0, 0, 0], [0, 0, 0]] # colour = [tuple([124, 252, 0]), tuple([0, 255, 255]), tuple([137, 43, 224]), # tuple([255 * 0.82, 255 * 0.41, 255 * 0.12]), tuple([255, 0, 0]), tuple([0, 128, 255])] # color_scheme = [ # [0.49, 0.99, 0], [0, 1, 1], [0.54, 0.17, 0.88], [0.82, 0.41, 0.12], # [1, 0, 0], [0, 0.5, 1] # ] annotator_no = 6 for annotator_id in range(annotator_no): os.mkdir(export_path + 'a' + str(annotator_id + 1) + '/') wsi_count = 0 for wsi in manifest_txt: print(wsi) wsi_count += 1 print( time.asctime(time.localtime(time.time())), "start export nuclei annotation:" + str(wsi_count) + '/' + str(len(manifest_txt))) info = wsi.split('\t') if not os.path.exists(annotation_project_root + info[0] + '/'): continue tba_list_db = annotation_project_root + info[0] + '/' + 'tba_list.db' if not os.path.exists(tba_list_db): continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() for item in tba_result: original_pic_url = annotation_project_root + info[ 0] + '/' + 'r' + str(item[0]) + '.png' original_pic = cv2.imread(original_pic_url) for annotator_id in range(annotator_no): if os.path.exists(annotation_project_root + info[0] + '/' + 'a' + str(annotator_id + 1) + '_r' + str(item[0]) + "_annotation.txt") and \ sum(numpy.loadtxt(annotation_project_root + info[0] + '/' + 'a' + str(annotator_id + 1) + '_r' + str(item[0]) + "_annotation.txt")) > 0: region_image_url = annotation_project_root + info[0] + '/' + 'a' + str(annotator_id + 1) + \ '_r' + str(item[0]) + "_boundary.txt" region_image = numpy.loadtxt(region_image_url, delimiter=",", dtype=int) write_path = export_path + 'a' + str( annotator_id + 1) + '/' + info[0] + '/' if not os.path.exists(write_path): os.mkdir(write_path) annotator_data_url = annotation_project_root + info[0] + '/' + \ 'a' + str(annotator_id + 1) + '_' + 'r' + str(item[0]) + "_annotation.txt" annotator_data = numpy.loadtxt(annotator_data_url, delimiter=",", dtype=int) write_url = export_path + 'a' + str(annotator_id + 1) + '/' + info[0] + '/' \ + '_' + 'r' + str(item[0]) + '_original.png' cv2.imwrite(write_url, original_pic) mask = numpy.zeros(original_pic.shape) mask_original = numpy.zeros(original_pic.shape) for i, val in enumerate(annotator_data): if i != 1 and val != 0: mask[region_image == i] = colour[int(val) - 1] mask_original[region_image == i] = ( original_pic[region_image == i] * 2.7 + colour[val - 1]) / 3.3 else: mask_original[region_image == i] = original_pic[ region_image == i] write_url = export_path + 'a' + str(annotator_id + 1) + '/' + info[0] + '/' \ + '_' + 'r' + str(item[0]) + '_mask.png' cv2.imwrite(write_url, mask) mask_original[region_image == -1] = tuple([0, 0, 0]) write_url = export_path + 'a' + str(annotator_id + 1) + '/' + info[0] + '/' \ + '_' + 'r' + str(item[0]) + '_mask_original.png' cv2.imwrite(write_url, mask_original) write_url = export_path + 'a' + str(annotator_id + 1) + '/' + info[0] + '/' \ + '_' + 'r' + str(item[0]) + '_annotator_data.txt' numpy.savetxt(write_url, annotator_data, fmt="%d", delimiter=",") write_url = export_path + 'a' + str(annotator_id + 1) + '/' + info[0] + '/' \ + '_' + 'r' + str(item[0]) + '_boundary.txt' numpy.savetxt(write_url, region_image, fmt="%d", delimiter=",") annotator_no = 6 for annotator_id in range(annotator_no): if not os.listdir(export_path + 'a' + str(annotator_id + 1)): shutil.rmtree(export_path + 'a' + str(annotator_id + 1)) if os.listdir(export_path): make_archive_threadsafe(export_file, export_path) return export_file
def available_slide(): annotator_id = current_user.get_id() annotation_type = request.args.get('annotation_type', default="", type=str) project = request.args.get('project', default="", type=str) if project == "": return jsonify(manifest_controller.get_available_slide_id()) else: data = [] count = 0 if os.path.exists('export/' + project + '_slide_table.txt'): for wsi in open('export/' + project + '_slide_table.txt').readlines(): slide_id = int(wsi.split('\t')[0]) temp = {"id": slide_id, "text": wsi.replace('\t', ' ')} data.append(temp) def takeSecond(elem): return elem["id"] if annotation_type == "nuclei": for index in range(len(data)): nuclei_annotation_root = "Data/nuclei_annotation_data/" annotation_project_root = nuclei_annotation_root + project + '/' if not os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/'): continue tba_list_db = annotation_project_root + data[index][ "text"].split(" ")[1] + '/' + 'tba_list.db' if not os.path.exists(tba_list_db): continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() annotated = 0 for item in tba_result: if os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + annotator_id + '_r' + str(item[0]) + "_annotation.txt") and \ sum(numpy.loadtxt(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + annotator_id + '_r' + str( item[0]) + "_annotation.txt")) > 0: annotated += 1 if len(tba_result) > 0: data[index]["text"] = "(" + str(annotated) + '/' + str( len(tba_result)) + ") " + data[index]["text"] elif annotation_type == "freehand" or annotation_type == "freehand_stomache": for index in range(len(data)): annotation_root = "Data/" + annotation_type + "_annotation_data/" annotation_project_root = annotation_root + project + '/' if not os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/'): continue if os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + annotator_id + '.db') and \ check_freehand_annotation(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + annotator_id + '.db') > 0: data[index]["text"] = "*" + data[index]["text"] data.sort(key=takeSecond) for index in range(len(data)): count += 1 data[index]["text"] = "【" + str(count) + "】" + data[index]["text"] return jsonify(data)
def print_logs(project=""): # project = request.args.get('project', default="", type=str) if project == "": abort(404) else: data = [] count = 0 if os.path.exists('export/' + project + '_slide_table.txt'): for wsi in open('export/' + project + '_slide_table.txt').readlines(): slide_id = int(wsi.split('\t')[0]) temp = { "id": slide_id, "text": wsi.replace('\t', ' ').replace('\n', '') } data.append(temp) def takeSecond(elem): return elem["id"] for index in range(len(data)): nuclei_annotation_root = "Data/nuclei_annotation_data/" annotation_project_root = nuclei_annotation_root + project + '/' # if not os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/'): # continue tba_list_db = annotation_project_root + data[index]["text"].split( " ")[1] + '/' + 'tba_list.db' if not os.path.exists(tba_list_db): data[index]["text"] += " 0 0 0 0 0 0 0" continue db = nuclei_annotation_sqlite.SqliteConnector(tba_list_db) tba_result = db.get_RegionID_Centre() data[index]["text"] += " " + str(len(tba_result)) for annotator_id in range(1, 7): annotated = 0 for item in tba_result: if os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + str(annotator_id) + '_r' + str(item[0]) + "_annotation.txt") and \ sum(numpy.loadtxt(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + str(annotator_id) + '_r' + str(item[0]) + "_annotation.txt")) > 0: annotated += 1 data[index]["text"] += " " + str(annotated) for index in range(len(data)): for annotator_id in range(1, 7): nuclei_annotation_root = "Data/freehand_annotation_data/" annotation_project_root = nuclei_annotation_root + project + '/' if not os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/'): data[index]["text"] += " 0" continue if os.path.exists(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + str(annotator_id) + '.db') and \ len(freehand_annotation_sqlite.SqliteConnector(annotation_project_root + data[index]["text"].split(" ")[1] + '/' + 'a' + str(annotator_id) + '.db').get_lines()) > 0: data[index]["text"] += " 1" else: data[index]["text"] += " 0" data.sort(key=takeSecond) for index in range(len(data)): count += 1 data[index]["text"] = str(count) + " " + data[index]["text"] output = "ID SlideID UUID SVS_File " \ "NucleiTotal Nuclei1 Nuclei2 Nuclei3 Nuclei4 Nuclei5 Nuclei6 " \ "Freehand1 Freehand2 Freehand3 Freehand4 Freehand5 Freehand6\n" for index in range(len(data)): output += data[index]["text"] + '\n' return Response(output, mimetype='text/plain')