def task9(): """ Усредняющий и медианный филтр к шумам :return: """ img = IO.read_image("data/MODEL.jpg") r_n, sp_n = task8() filetered_r_n = m.normalize(PF.averaging_filter(r_n)) filetered_sp_n = m.normalize(PF.averaging_filter(sp_n)) filetered_med_r_n = m.normalize(PF.median_filter(r_n)) filetered_med_sp_n = m.normalize(PF.median_filter(sp_n, 7)) plt.imshow(r_n, cmap="gray") plt.figure() plt.imshow(filetered_r_n, cmap="gray") plt.figure() plt.imshow(filetered_med_r_n, cmap="gray") plt.show() plt.imshow(img, cmap='gray') plt.figure() plt.imshow(sp_n, cmap="gray") plt.figure() plt.imshow(filetered_sp_n, cmap="gray") plt.figure() plt.imshow(filetered_med_sp_n, cmap="gray") plt.show()
def task14(): """ 1 оригонал (оригинал + шум (sp + noise + both)) grad (sobel) laplasian :return: """ img = IO.read_image(r"data/MODEL.jpg") img2 = PF.gradient_sobel(img, "h") img3 = PF.gradient_sobel(img, "v") img4 = PF.gradient_sobel(img, "a") img5 = m.normalize(np.absolute(img2)) +\ m.normalize(np.absolute(img3)) IO.show_images([ img, m.normalize(img2), m.normalize(img3), m.normalize(img4), img5, img - img5 ], [ "Изначальное изображение", "Маска собела горизонт", "Маска собела верт", "Сумма нескольких масок", "Применение горизонтальной и вертикальной масок", "Суммируем ради красоты" ], fig_name="Маски Собела") img2 = PF.laplassian(img, "a") img3 = PF.laplassian(img, "b") IO.show_images([ m.normalize(img2), m.normalize(img3), img + m.normalize(img2), img + m.normalize(img3) ], [ "Лаплассиан, маска1", "Лапл, маска 2", "Сумма с маск 1", "Сумма с маской 2" ], fig_name="Лаплассиан")
def handle(env, start_response, handler, m=None): path = get_path(env) if len(path) > 0 and not path[-1] == '/': start_response('302 Redirect', [('Location', model.normalize(path) + "/")]) return path = model.normalize(path) try: d = model.describe(path) qs = urlparse.parse_qs(env['QUERY_STRING']) if not m and 'cache' in d: if 'r' in qs and qs['r'][0] == '1': if d['cache'] == 'normal': model.invalidate_cache(path) if d['cache'] == 'parent': model.invalidate_cache(model.parent(path)) if d and d['type'] == 'render' and env['REQUEST_METHOD'] == 'GET': parameters = {} if 'parameters' in d: parameters = d['parameters'] content, mime = render_handlers[d['function']](path, parameters, env) start_response('200 OK', [('Content-Type', mime), ('Content-Length', str(len(content)))]) return content if not m: m = model.load(path) content, redirect = handler(path, d, m, env) if redirect: redirect = model.normalize(redirect) close = '' if 'c' in qs: if qs['c'][0] == '0': close = '?c=1' if qs['c'][0] == '1': close = '?c=2' start_response('302 Redirect', [('Location', redirect + "/" + close)]) return else: start_response('200 OK', [('Content-Type', 'text/html;charset=utf-8'), ('Content-Length', str(len(content)))]) return content except model.NotFoundException as e: import traceback traceback.print_exc() raise restlite.Status, '404 Not Found' except model.ParseException as e: d = {} d.update({'e': e.errors}) d.update(e.attributes) start_response('302 Redirect', [('Location', model.normalize(path) + "/?" + urllib.urlencode(d, True))]) return
def parse(self, attributes): def getValues(string): return map(float, string.split(" ")) if "rotation" in attributes.keys(): values = getValues(attributes["rotation"]) vector, angle = model.normalize(values[0:3]), values[3] vector = model.normalize(vector) self.transform.rotate(vector, angle) if "scale" in attributes.keys(): self.transform.scale(getValues(attributes["scale"])) if "translation" in attributes.keys(): self.transform.translate(getValues(attributes["translation"]))
def read(self, stream, string): key = "([+e\d\-\.]+)" result = re.search("translation\s+" + "\s+".join([key] * 3), string, re.I | re.S) if result is not None: self.transform.translate(map(lambda x: float(result.group(x + 1)), range(0, 3))) result = re.search("rotation\s+" + "\s+".join([key] * 4), string, re.I | re.S) if result is not None: values = map(lambda x: float(result.group(x + 1)), range(0, 3)) vector, angle = model.normalize(values), float(result.group(4)) vector = model.normalize(vector) self.transform.rotate(vector, angle) result = re.search("scale\s+" + "\s+".join([key] * 3), string, re.I | re.S) if result is not None: self.transform.scale(map(lambda x: float(result.group(x + 1)), range(0, 3)))
def read(self, stream, string): key = '([+e\d\-\.]+)' result = re.search('translation\s+' + '\s+'.join([key] * 3), string, re.I | re.S) if result is not None: self.transform.translate([float(result.group(x)) for x in range(1, 4)]) result = re.search('rotation\s+' + '\s+'.join([key] * 4), string, re.I | re.S) if result is not None: values = [float(result.group(x)) for x in range(1, 4)] vector, angle = model.normalize(values), float(result.group(4)) vector = model.normalize(vector) self.transform.rotate(vector, angle) result = re.search('scale\s+' + '\s+'.join([key] * 3), string, re.I | re.S) if result is not None: self.transform.scale([float(result.group(x)) for x in range(1, 4)])
def task8(not_show=True): """ model_.jpg ??????????? - изображение 1. ????????? ??????????? гистограмма 2. ????????? ???????? - ?????? ?? ???? ??????? ? ????????? ??????(???????? ? ????) случаный шум - идем по строкам и делам + какой то рандом * S . потом скайл 3. ??????? ???? ? ????? шум - соль перец 4. зашумит спайками (для каждой строки + и -) впрос про scale - как его делать - обрезая или /max ? """ img = IO.read_image("data/MODEL.jpg") plt.imshow(img, cmap="gray") plt.show() if not not_show: plt.figure() plt.imshow(img, cmap="gray") plt.figure() plt.hist(img.flatten(), 255) plt.show() img_random_noise = np.zeros(img.shape) prob_of_empty = 0.3 shum_len = 100 for i in range(len(img)): img_random_noise[i] = img[i] + \ np.random.choice( [i for i in range(shum_len)], p=[prob_of_empty]+[(1-prob_of_empty)/(shum_len-1) for _ in range(shum_len-1)], size=img.shape[1]) img_random_noise = m.normalize(img_random_noise) img_SP_noise = np.zeros(img.shape) for i in range(len(img)): img_SP_noise[i] = m.add_pikes(img[i].copy(), len(img[i]) // 6, len(img[i]) // 5, 100, 200) img_SP_noise += img img_SP_noise = m.normalize(img_SP_noise) if not not_show: plt.imshow(img_random_noise, cmap="gray") plt.show() plt.imshow(img_SP_noise, cmap="gray") plt.show() return img_random_noise, img_SP_noise
def handle(env, start_response, handler, m=None): path = get_path(env) if len(path) > 0 and not path[-1] == '/': start_response('302 Redirect', [('Location', model.normalize(path)+"/")]) return path = model.normalize(path) try: d = model.describe(path) qs = urlparse.parse_qs(env['QUERY_STRING']) if not m and 'cache' in d: if 'r' in qs and qs['r'][0]=='1': if d['cache'] == 'normal': model.invalidate_cache(path) if d['cache'] == 'parent': model.invalidate_cache(model.parent(path)) if d and d['type'] == 'render' and env['REQUEST_METHOD'] == 'GET': parameters = {} if 'parameters' in d: parameters = d['parameters'] content, mime = render_handlers[d['function']](path, parameters, env) start_response('200 OK', [('Content-Type', mime), ('Content-Length', str(len(content)))]) return content if not m: m = model.load(path) content, redirect = handler(path, d, m, env) if redirect: redirect = model.normalize(redirect) close='' if 'c' in qs: if qs['c'][0]=='0': close='?c=1' if qs['c'][0]=='1': close='?c=2' start_response('302 Redirect', [('Location', redirect+"/"+close)]) return else: start_response('200 OK', [('Content-Type', 'text/html;charset=utf-8'), ('Content-Length', str(len(content)))]) return content except model.NotFoundException as e: import traceback traceback.print_exc() raise restlite.Status, '404 Not Found' except model.ParseException as e: d = {} d.update( { 'e': e.errors } ) d.update(e.attributes) start_response('302 Redirect', [('Location', model.normalize(path)+"/?"+urllib.urlencode(d, True))]) return
def show_project(path, parameters, env): m=re.match(r"^.*projects/[^/]+/+([^/]+)$", path) project=m.group(1) # Find people planned on this project persons = set() model.traverse( '/', lambda p : add_if_working_on(model.normalize(p[0]), project, persons) ) dates, slots, s = prepare_people_tasks( persons, project ) return visualize.render(dates, slots, sorted(s), variables={'qs':urlparse.parse_qs(env['QUERY_STRING']), 'context':'/', 'path':path, 'sum':True, 'url': path+'/', 'refreshable': False }), "text/html"
def task13(): """ Выделить контуры объекта MODEL (должны быть как бинарное изображение): ФНЧ (фильтр низких чатсот расфокусирует изображение), вычитаем из исходного применяем пороговое преобразование. ФВЧ (фильтр высоких частот к тому же изображению построчно или по столбцам), аналогично 1. Попробовать с шумами """ img = IO.read_image(r"data/MODEL.jpg") lpf = m.create_low_pass_filter(0.05, 8, 1) img2 = m.convolutional_2d(img, lpf) img3 = img - img2 img3 = m.normalize(m.threshold_filter_2d(img3, 240)) hpf = m.create_hight_pass_filter(0.2, 8, 1) img4 = m.convolutional_2d(img, hpf) img4 = m.normalize(m.threshold_filter_2d(img4, 100)) IO.show_images([img, img2, img3, img4], ["start", "conv res", "low filter", "hight filter"], fig_name="Контур с помощью фильров") img_n = m.add_random_noise_2d(img) img_n = m.add_salt_pepper_noise_2d(img_n) img_n = PF.median_filter(img_n) img_n = PF.averaging_filter(img_n) lpf = m.create_low_pass_filter(0.05, 8, 1) img2 = m.convolutional_2d(img_n, lpf) img3 = img - img2 # img3 = m.normalize(img3, 255) # print(img3.min(), img3.max()) img3 = m.normalize(m.threshold_filter_2d(img3, 78)) hpf = m.create_hight_pass_filter(0.2, 4, 1) img4 = m.convolutional_2d(img_n, hpf) # img4 = m.normalize(img4, 255) # print(img4.min(), img4.max()) # img4 = m.threshold_filter_2d(img4, 0) IO.show_images([img_n, img2, img3, m.normalize(img4)], ["start", "conv res", "low filter", "hight filter"], fig_name="Контур филтрами зашумленного изображения")
def parse(self, attributes): def getValues(string): return [float(x) for x in string.split(' ')] if 'rotation' in attributes: values = getValues(attributes['rotation']) vector, angle = model.normalize(values[0:3]), values[3] vector = model.normalize(vector) self.transform.rotate(vector, angle) debug('{:s}Rotation: {:.3f} about [{:.3f}, {:.3f}, {:.3f}]'.format( ' ' * (self.level + 1), values[3], *values[0:3])) if 'scale' in attributes: values = getValues(attributes['scale']) self.transform.scale(values) debug('{:s}Scale: [{:.3f}, {:.3f}, {:.3f}]'.format( ' ' * (self.level + 1), *values)) if 'translation' in attributes: values = getValues(attributes['translation']) self.transform.translate(values) debug('{:s}Translation: [{:.3f}, {:.3f}, {:.3f}]'.format( ' ' * (self.level + 1), *values))
def task15(): """ 1.oroginal 2. original + noise Контуры а)erosion b)silatation :return: """ img = IO.read_image(r"data/Dots.jpg") img = to_one_chanel(img) print(img.max(), img.mean(), img.min()) img = 255 - img print(set(img.flatten())) img = m.threshold_filter_2d(img, 100) print(set(img.flatten())) img2 = PF.erosion(img, 17) img3 = PF.dilatation(img2, 17) IO.show_images([m.normalize(img), m.normalize(img2), m.normalize(img3)], fig_name="Эрозия, пример") # with model img = IO.read_image(r"data/MODEL.jpg") img2 = m.threshold_filter_2d(img, 100) img3 = PF.erosion(img2, 10) img4 = img2 - img3 img5 = PF.dilatation(img2, 10) img6 = img5 - img2 IO.show_images([m.normalize(img2), m.normalize(img4), m.normalize(img6)]["Изначальное бинаризированное", "Эрозия", "Дилатация"], fig_name="Эрозия, детектирования края")
def generateMeshNormals(meshes): blueMaterial = model.Material() blueMaterial.color.diffuse = numpy.array([0.0, 0.0, 1.0]) normalLength = 0.2 urchin = model.LineArray(name='Normals') urchin.appearance().material = blueMaterial for mesh in meshes: if not mesh.appearance().normals: continue geoVertices, geoPolygons = mesh.geometry() smooth = mesh.appearance().smooth vertices = geoVertices if mesh.transform is None else [mesh.transform.apply(v) for v in geoVertices] def getNormal(points): return model.normalize(numpy.cross( vertices[points[1]] - vertices[points[0]], vertices[points[2]] - vertices[points[0]])) if smooth: normals = [numpy.zeros(3) for i in range(0, len(vertices))] for poly in geoPolygons: normal = getNormal(poly) for vertex in poly: normals[vertex] += normal normals = [model.normalize(vector) for vector in normals] for i in range(0, len(vertices)): lastIndex = len(urchin.geoVertices) urchin.geoVertices.append(vertices[i]) urchin.geoVertices.append(vertices[i] + normals[i] * normalLength) urchin.geoPolygons.append([lastIndex, lastIndex + 1]) else: normals = [getNormal(gp) for gp in geoPolygons] for i in range(0, len(geoPolygons)): gp = geoPolygons[i] position = numpy.zeros(3) for vertex in gp: position += vertices[vertex] position /= float(len(gp)) lastIndex = len(urchin.geoVertices) urchin.geoVertices.append(position) urchin.geoVertices.append(position + normals[i] * normalLength) urchin.geoPolygons.append([lastIndex, lastIndex + 1]) return [] if len(urchin.geoPolygons) == 0 else [urchin]
def __init__(self, radius, depth=1): if radius <= 0. or depth < 1: raise Exception(); model.Mesh.__init__(self) r = (1. + math.sqrt(5.)) / 4. vertices = [] vertices.append(numpy.array([-.5, r, 0.])) vertices.append(numpy.array([ .5, r, 0.])) vertices.append(numpy.array([-.5, -r, 0.])) vertices.append(numpy.array([ .5, -r, 0.])) vertices.append(numpy.array([ 0., -.5, r])) vertices.append(numpy.array([ 0., .5, r])) vertices.append(numpy.array([ 0., -.5, -r])) vertices.append(numpy.array([ 0., .5, -r])) vertices.append(numpy.array([ r, 0., -.5])) vertices.append(numpy.array([ r, 0., .5])) vertices.append(numpy.array([ -r, 0., -.5])) vertices.append(numpy.array([ -r, 0., .5])) for i in range(0, len(vertices)): vertices[i] = model.normalize(vertices[i]) polygons = [] polygons.extend([[ 0, 11, 5], [ 0, 5, 1], [ 0, 1, 7], [ 0, 7, 10], [ 0, 10, 11]]) polygons.extend([[ 1, 5, 9], [ 5, 11, 4], [11, 10, 2], [10, 7, 6], [ 7, 1, 8]]) polygons.extend([[ 3, 9, 4], [ 3, 4, 2], [ 3, 2, 6], [ 3, 6, 8], [ 3, 8, 9]]) polygons.extend([[ 4, 9, 5], [ 2, 4, 11], [ 6, 2, 10], [ 8, 6, 7], [ 9, 8, 1]]) def getMiddlePoint(v1, v2): return model.normalize(v1 + (v2 - v1) / 2) for i in range(0, depth): pNext = [] for face in polygons: index = len(vertices) vertices.append(getMiddlePoint(vertices[face[0]], vertices[face[1]])) vertices.append(getMiddlePoint(vertices[face[1]], vertices[face[2]])) vertices.append(getMiddlePoint(vertices[face[2]], vertices[face[0]])) pNext.append([face[0], index + 0, index + 2]) pNext.append([face[1], index + 1, index + 0]) pNext.append([face[2], index + 2, index + 1]) pNext.append([index + 0, index + 1, index + 2]) polygons = pNext self.geoVertices = vertices self.geoPolygons = polygons
def rotate(self, hrot, vrot): self.camera -= self.pov if hrot != 0.: horizRotationMatrix = numpy.matrix([ [math.cos(hrot), -math.sin(hrot), 0., 0.], [math.sin(hrot), math.cos(hrot), 0., 0.], [ 0., 0., 1., 0.], [ 0., 0., 0., 1.]]) self.camera = horizRotationMatrix * self.camera self.axis = horizRotationMatrix * self.axis if vrot != 0.: normal = model.normal(self.camera.getA()[:,0], self.axis.getA()[:,0]) normal = model.normalize(normal) vertRotationMatrix = model.rotationMatrix(normal, vrot) self.camera = vertRotationMatrix * self.camera self.axis = vertRotationMatrix * self.axis self.camera += self.pov
def move(self, x, y): self.camera -= self.pov axis = numpy.array(self.axis)[:,0][0:3] camera = numpy.array(self.camera)[:,0][0:3] normal = model.normalize(model.normal(axis, camera)) width = 2. * math.tan(self.fov / 2.) * numpy.linalg.norm(camera) offset = normal * (-x / width) + axis * (-y / width) rotationMatrix = numpy.matrix([ [1., 0., 0., offset[0]], [0., 1., 0., offset[1]], [0., 0., 1., offset[2]], [0., 0., 0., 1.]]) self.camera += self.pov self.camera = rotationMatrix * self.camera self.pov = rotationMatrix * self.pov
def collide(self, position, height, world): """ Checks to see if the player at the given `position` and `height` is colliding with any blocks in the world. Parameters ---------- position : tuple of len 3 The (x, y, z) position to check for collisions at. height : int or float The height of the player. Returns ------- position : tuple of len 3 The new position of the player taking into account collisions. """ # How much overlap with a dimension of a surrounding block you need to # have to count as a collision. If 0, touching terrain at all counts as # a collision. If .49, you sink into the ground, as if walking through # tall grass. If >= .5, you'll fall through the ground. pad = 0.25 p = list(position) np = model.normalize(position) for face in model.FACES: # check all surrounding blocks for i in xrange(3): # check each dimension independently if not face[i]: continue # How much overlap you have with this dimension. d = (p[i] - np[i]) * face[i] if d < pad: continue for dy in xrange(height): # check each height op = list(np) op[1] -= dy op[i] += face[i] if tuple(op) not in world.world: continue p[i] -= (d - pad) * face[i] if face == (0, -1, 0) or face == (0, 1, 0): # You are colliding with the ground or ceiling, so stop # falling / rising. self.dy = 0 break return tuple(p)
def compute_embedding(opt, model, data_mat): segment_length = int((opt.min_segment_length + opt.max_segment_length) / 2) segment_shift = int(opt.segment_shift_rate * segment_length) input_data = None seq_data = 0 length = data_mat.size(0) #print('data_mat ', data_mat.shape) for x in range(0, length, segment_shift): end = x + segment_length if end < length: feature_mat = data_mat[x:end, :, :] else: if x == 0: input_data = data_mat seq_data += 1 break seq_data += 1 if input_data is None: input_data = feature_mat else: input_data = torch.cat((input_data, feature_mat), 1) input_data = input_data.to(opt.device) seq_data = torch.LongTensor([seq_data]).to(opt.device) #print('input_data ', input_data.shape, seq_data) if opt.seq_training == 'true': if opt.model_type == 'cnn': input_data = input_data.transpose(0, 1).unsqueeze(1).to(opt.device) out_data, w, b, _ = model(input_data, seq_data) else: if opt.model_type == 'lstm' or opt.model_type == 'blstmp': out_data, w, b = model(input_data) elif opt.model_type == 'cnn': input_data = input_data.transpose(0, 1).unsqueeze(1).to(opt.device) out_data, w, b = model(input_data) out = torch.mean(out_data, 0) #print('out_data ', out_data.shape, out.shape) out = normalize(out) out = out.detach().cpu().numpy() out = out[np.newaxis, :] return out_data.detach().cpu().numpy(), out, w, b
def show_project(path, parameters, env): m = re.match(r"^.*projects/[^/]+/+([^/]+)$", path) project = m.group(1) # Find people planned on this project persons = set() model.traverse( '/', lambda p: add_if_working_on(model.normalize(p[0]), project, persons)) dates, slots, s = prepare_people_tasks(persons, project) return visualize.render(dates, slots, sorted(s), variables={ 'qs': urlparse.parse_qs(env['QUERY_STRING']), 'context': '/', 'path': path, 'sum': True, 'url': path + '/', 'refreshable': False }), "text/html"
def getTangent(points, texels): return model.normalize(model.tangent( vertices[points[1]] - vertices[points[0]], vertices[points[2]] - vertices[points[0]], texVertices[texels[1]] - texVertices[texels[0]], texVertices[texels[2]] - texVertices[texels[0]]))
def __init__(self, meshes, appearance=None, transform=None): super().__init__() self.appearance = appearance self.transform = transform self.parts = [] textured = meshes[0].isTextured() started = time.time() primitives = [0, 0] for mesh in meshes: for poly in mesh.geometry()[1]: count = len(poly) if count < 3 or count > 4: raise Exception() primitives[len(poly) - 3] += 1 triangles, quads = primitives[0] * 3, primitives[1] * 4 length = triangles + quads self.vertices = numpy.zeros(length * 3, dtype=numpy.float32) self.normals = numpy.zeros(length * 3, dtype=numpy.float32) self.texels = numpy.zeros(length * 2, dtype=numpy.float32) if textured else None self.tangents = numpy.zeros(length * 3, dtype=numpy.float32) if textured else None if triangles > 0: self.parts.append(RenderMesh.Faceset(GL_TRIANGLES, 0, triangles)) if quads > 0: self.parts.append(RenderMesh.Faceset(GL_QUADS, triangles, quads)) try: smooth = appearance.smooth except: smooth = False # Initial positions index = [0, triangles] for mesh in meshes: geoVertices, geoPolygons = mesh.geometry() texVertices, texPolygons = mesh.texture() vertices = geoVertices if mesh.transform is None else [mesh.transform.apply(v) for v in geoVertices] def getNormal(points): return model.normalize(numpy.cross( vertices[points[1]] - vertices[points[0]], vertices[points[2]] - vertices[points[0]])) def getTangent(points, texels): return model.normalize(model.tangent( vertices[points[1]] - vertices[points[0]], vertices[points[2]] - vertices[points[0]], texVertices[texels[1]] - texVertices[texels[0]], texVertices[texels[2]] - texVertices[texels[0]])) if smooth: normals = [numpy.zeros(3) for i in range(0, len(geoVertices))] for poly in geoPolygons: normal = getNormal(poly) for vertex in poly: normals[vertex] += normal normals = [model.normalize(vector) for vector in normals] if textured: tangents = [numpy.zeros(3) for i in range(0, len(geoVertices))] for gp, tp in zip(geoPolygons, texPolygons): tangent = getTangent(gp, tp) for vertex in gp: tangents[vertex] += tangent tangents = [model.normalize(vector) for vector in tangents] else: normals = [getNormal(gp) for gp in geoPolygons] if textured: tangents = [getTangent(gp, tp) for gp, tp in zip(geoPolygons, texPolygons)] for i in range(0, len(geoPolygons)): gp = geoPolygons[i] if textured: tp = texPolygons[i] count = len(gp) indexGroup = count - 3 offset = index[indexGroup] index[indexGroup] += count for vertex in range(0, count): geoStart, geoEnd, texStart, texEnd = 3 * offset, 3 * (offset + 1), 2 * offset, 2 * (offset + 1) self.vertices[geoStart:geoEnd] = vertices[gp[vertex]][0:3] self.normals[geoStart:geoEnd] = normals[gp[vertex]] if smooth else normals[i] if textured: self.texels[texStart:texEnd] = texVertices[tp[vertex]] self.tangents[geoStart:geoEnd] = tangents[gp[vertex]] if smooth else tangents[i] offset += 1 self.vao = glGenVertexArrays(1) glBindVertexArray(self.vao) self.verticesVBO = glGenBuffers(1) glBindBuffer(GL_ARRAY_BUFFER, self.verticesVBO) glBufferData(GL_ARRAY_BUFFER, self.vertices, GL_STATIC_DRAW) glEnableVertexAttribArray(0); glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, None) self.normalsVBO = glGenBuffers(1) glBindBuffer(GL_ARRAY_BUFFER, self.normalsVBO) glBufferData(GL_ARRAY_BUFFER, self.normals, GL_STATIC_DRAW) glEnableVertexAttribArray(1); glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 0, None) if textured: self.texelsVBO = glGenBuffers(1) glBindBuffer(GL_ARRAY_BUFFER, self.texelsVBO) glBufferData(GL_ARRAY_BUFFER, self.texels, GL_STATIC_DRAW) #glBufferData(GL_ARRAY_BUFFER, self.texels, GL_DYNAMIC_DRAW) glEnableVertexAttribArray(2); glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 0, None) self.tangentVBO = glGenBuffers(1) glBindBuffer(GL_ARRAY_BUFFER, self.tangentVBO) glBufferData(GL_ARRAY_BUFFER, self.tangents, GL_STATIC_DRAW) glEnableVertexAttribArray(3); glVertexAttribPointer(3, 3, GL_FLOAT, GL_FALSE, 0, None) glBindVertexArray(0) debug('Mesh created in {:f}, id {:s}, triangles {:d}, quads {:d}, vertices {:d}'.format( time.time() - started, self.ident, int(triangles / 3), int(quads / 4), length))
def writeTransform(root, mesh): started = time.time() if mesh.transform is None: translation = numpy.array([0.0, 0.0, 0.0]) rotation = numpy.array([1.0, 0.0, 0.0, 0.0]) scale = numpy.array([1.0, 1.0, 1.0]) else: translation = mesh.transform.matrix.getA()[:,3][0:3] translationMatrix = numpy.matrix([ [1.0, 0.0, 0.0, -translation[0]], [0.0, 1.0, 0.0, -translation[1]], [0.0, 0.0, 1.0, -translation[2]], [0.0, 0.0, 0.0, 1.0]]) translated = translationMatrix * mesh.transform.matrix scale = numpy.array([numpy.linalg.norm(translated.getA()[:,column][0:3]) for column in [0, 1, 2]]) scaleMatrix = numpy.matrix([ [1.0 / scale[0], 0.0, 0.0, 0.0], [ 0.0, 1.0 / scale[1], 0.0, 0.0], [ 0.0, 0.0, 1.0 / scale[2], 0.0], [ 0.0, 0.0, 0.0, 1.0]]) scaled = translated * scaleMatrix # Conversion from rotation matrix form to axis-angle form angle = math.acos(((scaled.trace() - 1.0) - 1.0) / 2.0) if angle == 0.0: rotation = numpy.array([1.0, 0.0, 0.0, 0.0]) else: skew = (scaled - scaled.transpose()).getA() vector = numpy.array([skew[2][1], skew[0][2], skew[1][0]]) vector = (1.0 / (2.0 * math.sin(angle))) * vector vector = model.normalize(vector) if abs(angle) < math.pi: rotation = numpy.array(vector.tolist() + [angle]) else: tensor = numpy.tensordot(vector, vector, 0) values = numpy.array([tensor[2][1], tensor[0][2], tensor[1][0]]) vector = numpy.diag(tensor) vector = model.normalize(vector) posIndices, negIndices = [], [] for i in range(0, 3): if values[i] < 0.0: negIndices.append(i) elif values[i] > 0.0: posIndices.append(i) if len(posIndices) == 1 and len(negIndices) == 2: vector[posIndices[0]] *= -1.0 elif len(posIndices) == 0 and len(negIndices) == 1: vector[negIndices[0]] *= -1.0 rotation = numpy.array(vector.tolist() + [angle]) debug('Transform {:s}: translation {:s}, rotation {:s}, scale {:s}'.format( mesh.ident, str(translation), str(rotation), str(scale))) transform = etree.SubElement(root, 'Transform') transform.attrib['DEF'] = 'OB_{:s}'.format(mesh.ident) transform.attrib['translation'] = '{:g} {:g} {:g}'.format(*translation) transform.attrib['scale'] = '{:g} {:g} {:g}'.format(*scale) transform.attrib['rotation'] = '{:g} {:g} {:g} {:g}'.format(*rotation) parent = mesh if mesh.parent is None else mesh.parent writeGroup(transform, parent) debug('Mesh exported in {:f}, name {:s}'.format(time.time() - started, mesh.ident))
def loft(path, shape, rotation=None, scaling=None): if len(path) < 2: raise Exception() if rotation is None: rotation = lambda t: numpy.zeros(3) if scaling is None: scaling = lambda t: numpy.ones(3) up = numpy.array([0.0, 0.0, 1.0]) # First pass to estimate v0 nonZeroProduct = None products = [] for i in range(0, len(path) - 1): v2 = model.normalize(path[i + 1][0:3] - path[i][0:3]) v0 = model.normalize(numpy.cross(up, v2)) if numpy.linalg.norm(v0) == 0.0: v0 = None elif nonZeroProduct is None: nonZeroProduct = v0 products.append((v0, v2)) # Second pass segments = [] for product in products: if product[0] is not None: nonZeroProduct = product[0] v0 = product[0] elif nonZeroProduct is not None: v0 = nonZeroProduct else: v0 = numpy.array([1.0, 0.0, 0.0]) v2 = product[1] v1 = model.normalize(numpy.cross(v2, v0)) m = numpy.matrix([ [v0[0], v1[0], v2[0], 0.0], [v0[1], v1[1], v2[1], 0.0], [v0[2], v1[2], v2[2], 0.0], [ 0.0, 0.0, 0.0, 1.0]]) segments.append(model.Transform(matrix=m).quaternion()) # Make slices slices = [] count = len(segments) current = segments[0] for i in range(0, count + 1): if i > 0 and i < len(segments): q = model.slerp(segments[i - 1], segments[i], 0.5) current = segments[i] else: q = current t = float(i) / count scaleTransform = model.Transform() scaleTransform.scale(scaling(t)) scaledShape = [scaleTransform.apply(x) for x in shape] transform = model.Transform(quaternion=q) transform.matrix *= model.rpyToMatrix(rotation(t)) transform.translate(path[i]) slices.append([transform.apply(x) for x in scaledShape]) return slices
prob_x, prob_z, learning_rate, var_list=d_vars) cost, zg_accuracy, train_G = model.minimize_generative_model(prob_z, learning_rate, var_list=g_vars) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) step = 0 while True: for _ in xrange(k_dg): step += 1 noise_z = model.sample_noise(batch_size, noise_size) batch_x, batch_y = mnist.train.next_batch(batch_size) batch_x = model.normalize(batch_x) d_result = sess.run([score, x_accuracy, z_accuracy, train_D], feed_dict={ z: noise_z, x: batch_x, keep_prob_g: 1.0, keep_prob_d: 0.8 }) noise_z = model.sample_noise(batch_size, noise_size) g_result = sess.run([cost, zg_accuracy, train_G], feed_dict={ z: noise_z, x: batch_x, keep_prob_g: 1.0, keep_prob_d: 0.8
def encodeTransform(mesh, level=0): output = '' started = time.time() if mesh.transform is None: translation = numpy.array([0.0, 0.0, 0.0]) rotation = numpy.array([1.0, 0.0, 0.0, 0.0]) scale = numpy.array([1.0, 1.0, 1.0]) else: translation = mesh.transform.matrix.getA()[:,3][0:3] translationMatrix = numpy.matrix([ [1.0, 0.0, 0.0, -translation[0]], [0.0, 1.0, 0.0, -translation[1]], [0.0, 0.0, 1.0, -translation[2]], [0.0, 0.0, 0.0, 1.0]]) translated = translationMatrix * mesh.transform.matrix scale = numpy.array([numpy.linalg.norm(translated.getA()[:,column][0:3]) for column in [0, 1, 2]]) scaleMatrix = numpy.matrix([ [1.0 / scale[0], 0.0, 0.0, 0.0], [ 0.0, 1.0 / scale[1], 0.0, 0.0], [ 0.0, 0.0, 1.0 / scale[2], 0.0], [ 0.0, 0.0, 0.0, 1.0]]) scaled = translated * scaleMatrix # Conversion from rotation matrix form to axis-angle form angle = math.acos(((scaled.trace() - 1.0) - 1.0) / 2.0) if angle == 0.: rotation = numpy.array([1.0, 0.0, 0.0, 0.0]) else: skew = (scaled - scaled.transpose()).getA() vector = numpy.array([skew[2][1], skew[0][2], skew[1][0]]) vector = (1.0 / (2.0 * math.sin(angle))) * vector vector = model.normalize(vector) if abs(angle) < math.pi: rotation = numpy.array(vector.tolist() + [angle]) else: tensor = numpy.tensordot(vector, vector, 0) values = numpy.array([tensor[2][1], tensor[0][2], tensor[1][0]]) vector = numpy.diag(tensor) vector = model.normalize(vector) posIndices, negIndices = [], [] for i in range(0, 3): if values[i] < 0.0: negIndices.append(i) elif values[i] > 0.0: posIndices.append(i) if len(posIndices) == 1 and len(negIndices) == 2: vector[posIndices[0]] *= -1.0 elif len(posIndices) == 0 and len(negIndices) == 1: vector[negIndices[0]] *= -1.0 rotation = numpy.array(vector.tolist() + [angle]) debug('Transform {:s}: translation {:s}, rotation {:s}, scale {:s}'.format( mesh.ident, str(translation), str(rotation), str(scale))) output += indent(level) + 'DEF OB_{:s} Transform {{\n'.format(mesh.ident) output += indent(level + 1) + 'translation {:g} {:g} {:g}\n'.format(*translation) output += indent(level + 1) + 'rotation {:g} {:g} {:g} {:g}\n'.format(*rotation) output += indent(level + 1) + 'scale {:g} {:g} {:g}\n'.format(*scale) output += indent(level + 1) + 'children [\n' parent = mesh if mesh.parent is None else mesh.parent output += encodeGroup(parent, level + 2) output += indent(level + 1) + ']\n' output += indent(level) + '}\n' debug('Mesh exported in {:f}, name {:s}'.format(time.time() - started, mesh.ident)) return output
def getMiddlePoint(v1, v2): return model.normalize(v1 + (v2 - v1) / 2)
def writeTransform(root, mesh): started = time.time() if mesh.transform is None: translation = numpy.array([0., 0., 0.]) rotation = numpy.array([1., 0., 0., 0.]) scale = numpy.array([1., 1., 1.]) else: translation = mesh.transform.value.getA()[:,3][0:3] translationMatrix = numpy.matrix([ [1., 0., 0., -translation[0]], [0., 1., 0., -translation[1]], [0., 0., 1., -translation[2]], [0., 0., 0., 1.]]) translated = translationMatrix * mesh.transform.value scale = numpy.array([numpy.linalg.norm(translated.getA()[:,column][0:3]) for column in [0, 1, 2]]) scaleMatrix = numpy.matrix([ [1. / scale[0], 0., 0., 0.], [ 0., 1. / scale[1], 0., 0.], [ 0., 0., 1. / scale[2], 0.], [ 0., 0., 0., 1.]]) scaled = translated * scaleMatrix #Conversion from rotation matrix form to axis-angle form angle = math.acos(((scaled.trace() - 1.) - 1.) / 2.) if angle == 0.: rotation = numpy.array([1., 0., 0., 0.]) else: skew = (scaled - scaled.transpose()).getA() vector = numpy.array([skew[2][1], skew[0][2], skew[1][0]]) vector = (1. / (2. * math.sin(angle))) * vector vector = model.normalize(vector) if abs(angle) < math.pi: rotation = numpy.array(vector.tolist() + [angle]) else: tensor = numpy.tensordot(vector, vector, 0) values = numpy.array([tensor[2][1], tensor[0][2], tensor[1][0]]) vector = numpy.diag(tensor) vector = model.normalize(vector) posIndices, negIndices = [], [] for i in range(0, 3): if values[i] < 0.: negIndices.append(i) elif values[i] > 0.: posIndices.append(i) if len(posIndices) == 1 and len(negIndices) == 2: vector[posIndices[0]] *= -1. elif len(posIndices) == 0 and len(negIndices) == 1: vector[negIndices[0]] *= -1. rotation = numpy.array(vector.tolist() + [angle]) debug("Transform %s: translation %s, rotation %s, scale %s" % (mesh.ident, str(translation), str(rotation), str(scale))) transform = etree.SubElement(root, "Transform") transform.attrib["DEF"] = "OB_%s" % mesh.ident transform.attrib["translation"] = "%f %f %f" % tuple(translation) transform.attrib["scale"] = "%f %f %f" % tuple(scale) transform.attrib["rotation"] = "%f %f %f %f" % tuple(rotation) parent = mesh if mesh.parent is None else mesh.parent writeGroup(transform, parent) debug("Mesh exported in %f, name %s" % (time.time() - started, mesh.ident))
def run_uda_training(log_dir, images_sd_tr, labels_sd_tr, images_sd_vl, labels_sd_vl, images_td_tr, images_td_vl): # ================================================================ # reset the graph built so far and build a new TF graph # ================================================================ tf.reset_default_graph() with tf.Graph().as_default(): # ============================ # set random seed for reproducibility # ============================ tf.random.set_random_seed(exp_config.run_num_uda) np.random.seed(exp_config.run_num_uda) # ================================================================ # create placeholders - segmentation net # ================================================================ images_sd_pl = tf.placeholder(tf.float32, shape=[exp_config.batch_size] + list(exp_config.image_size) + [1], name='images_sd') images_td_pl = tf.placeholder(tf.float32, shape=[exp_config.batch_size] + list(exp_config.image_size) + [1], name='images_td') labels_sd_pl = tf.placeholder(tf.uint8, shape=[exp_config.batch_size] + list(exp_config.image_size), name='labels_sd') training_pl = tf.placeholder(tf.bool, shape=[], name='training_or_testing') # ================================================================ # insert a normalization module in front of the segmentation network # ================================================================ images_sd_normalized, _ = model.normalize(images_sd_pl, exp_config, training_pl, scope_reuse=False) images_td_normalized, _ = model.normalize(images_td_pl, exp_config, training_pl, scope_reuse=True) # ================================================================ # get logit predictions from the segmentation network # ================================================================ predicted_seg_sd_logits, _, _ = model.predict_i2l(images_sd_normalized, exp_config, training_pl, scope_reuse=False) # ================================================================ # get all features from the segmentation network # ================================================================ seg_features_sd = model.get_all_features(images_sd_normalized, exp_config, scope_reuse=True) seg_features_td = model.get_all_features(images_td_normalized, exp_config, scope_reuse=True) # ================================================================ # resize all features to the same size # ================================================================ images_sd_features_resized = model.resize_features( [images_sd_normalized] + list(seg_features_sd), (64, 64), 'resize_sd_xfeat') images_td_features_resized = model.resize_features( [images_td_normalized] + list(seg_features_td), (64, 64), 'resize_td_xfeat') # ================================================================ # discriminator on features # ================================================================ d_logits_sd = model.discriminator(images_sd_features_resized, exp_config, training_pl, scope_name='discriminator', scope_reuse=False) d_logits_td = model.discriminator(images_td_features_resized, exp_config, training_pl, scope_name='discriminator', scope_reuse=True) # ================================================================ # add ops for calculation of the discriminator loss # ================================================================ d_loss_sd = tf.nn.sigmoid_cross_entropy_with_logits( labels=tf.ones_like(d_logits_sd), logits=d_logits_sd) d_loss_td = tf.nn.sigmoid_cross_entropy_with_logits( labels=tf.zeros_like(d_logits_td), logits=d_logits_td) loss_d_op = tf.reduce_mean(d_loss_sd + d_loss_td) tf.summary.scalar('tr_losses/loss_discriminator', loss_d_op) # ================================================================ # add ops for calculation of the adversarial loss that tries to get domain invariant features in the normalized image space # ================================================================ loss_g_op = model.loss_invariance(d_logits_td) tf.summary.scalar('tr_losses/loss_invariant_features', loss_g_op) # ================================================================ # add ops for calculation of the supervised segmentation loss # ================================================================ loss_seg_op = model.loss(predicted_seg_sd_logits, labels_sd_pl, nlabels=exp_config.nlabels, loss_type=exp_config.loss_type_i2l) tf.summary.scalar('tr_losses/loss_segmentation', loss_seg_op) # ================================================================ # total training loss for uda # ================================================================ loss_total_op = loss_seg_op + exp_config.lambda_uda * loss_g_op tf.summary.scalar('tr_losses/loss_total_uda', loss_total_op) # ================================================================ # merge all summaries # ================================================================ summary_scalars = tf.summary.merge_all() # ================================================================ # divide the vars into segmentation network, normalization network and the discriminator network # ================================================================ i2l_vars = [] normalization_vars = [] discriminator_vars = [] for v in tf.global_variables(): var_name = v.name if 'image_normalizer' in var_name: normalization_vars.append(v) i2l_vars.append( v ) # the normalization vars also need to be restored from the pre-trained i2l mapper elif 'i2l_mapper' in var_name: i2l_vars.append(v) elif 'discriminator' in var_name: discriminator_vars.append(v) # ================================================================ # add optimization ops # ================================================================ train_i2l_op = model.training_step( loss_total_op, i2l_vars, exp_config.optimizer_handle, learning_rate=exp_config.learning_rate) train_discriminator_op = model.training_step( loss_d_op, discriminator_vars, exp_config.optimizer_handle, learning_rate=exp_config.learning_rate) # ================================================================ # add ops for model evaluation # ================================================================ eval_loss = model.evaluation_i2l_uda_invariant_features( predicted_seg_sd_logits, labels_sd_pl, images_sd_pl, d_logits_td, nlabels=exp_config.nlabels, loss_type=exp_config.loss_type_i2l) # ================================================================ # add ops for adding image summary to tensorboard # ================================================================ summary_images = model.write_image_summary_uda_invariant_features( predicted_seg_sd_logits, labels_sd_pl, images_sd_pl, exp_config.nlabels) # ================================================================ # build the summary Tensor based on the TF collection of Summaries. # ================================================================ if exp_config.debug: print('creating summary op...') # ================================================================ # add init ops # ================================================================ init_ops = tf.global_variables_initializer() # ================================================================ # find if any vars are uninitialized # ================================================================ if exp_config.debug: logging.info( 'Adding the op to get a list of initialized variables...') uninit_vars = tf.report_uninitialized_variables() # ================================================================ # create session # ================================================================ sess = tf.Session() # ================================================================ # create a file writer object # This writes Summary protocol buffers to event files. # https://github.com/tensorflow/docs/blob/r1.12/site/en/api_docs/python/tf/summary/FileWriter.md # The FileWriter class provides a mechanism to create an event file in a given directory and add summaries and events to it. # The class updates the file contents asynchronously. # This allows a training program to call methods to add data to the file directly from the training loop, without slowing down training. # ================================================================ summary_writer = tf.summary.FileWriter(log_dir, sess.graph) # ================================================================ # create savers # ================================================================ saver = tf.train.Saver(var_list=i2l_vars) saver_lowest_loss = tf.train.Saver(var_list=i2l_vars, max_to_keep=3) # ================================================================ # summaries of the validation errors # ================================================================ vl_error_seg = tf.placeholder(tf.float32, shape=[], name='vl_error_seg') vl_error_seg_summary = tf.summary.scalar('validation/loss_seg', vl_error_seg) vl_dice = tf.placeholder(tf.float32, shape=[], name='vl_dice') vl_dice_summary = tf.summary.scalar('validation/dice', vl_dice) vl_error_invariance = tf.placeholder(tf.float32, shape=[], name='vl_error_invariance') vl_error_invariance_summary = tf.summary.scalar( 'validation/loss_invariance', vl_error_invariance) vl_error_total = tf.placeholder(tf.float32, shape=[], name='vl_error_total') vl_error_total_summary = tf.summary.scalar('validation/loss_total', vl_error_total) vl_summary = tf.summary.merge([ vl_error_seg_summary, vl_dice_summary, vl_error_invariance_summary, vl_error_total_summary ]) # ================================================================ # summaries of the training errors # ================================================================ tr_error_seg = tf.placeholder(tf.float32, shape=[], name='tr_error_seg') tr_error_seg_summary = tf.summary.scalar('training/loss_seg', tr_error_seg) tr_dice = tf.placeholder(tf.float32, shape=[], name='tr_dice') tr_dice_summary = tf.summary.scalar('training/dice', tr_dice) tr_error_invariance = tf.placeholder(tf.float32, shape=[], name='tr_error_invariance') tr_error_invariance_summary = tf.summary.scalar( 'training/loss_invariance', tr_error_invariance) tr_error_total = tf.placeholder(tf.float32, shape=[], name='tr_error_total') tr_error_total_summary = tf.summary.scalar('training/loss_total', tr_error_total) tr_summary = tf.summary.merge([ tr_error_seg_summary, tr_dice_summary, tr_error_invariance_summary, tr_error_total_summary ]) # ================================================================ # freeze the graph before execution # ================================================================ if exp_config.debug: logging.info( '============================================================') logging.info('Freezing the graph now!') tf.get_default_graph().finalize() # ================================================================ # Run the Op to initialize the variables. # ================================================================ if exp_config.debug: logging.info( '============================================================') logging.info('initializing all variables...') sess.run(init_ops) # ================================================================ # print names of uninitialized variables # ================================================================ uninit_variables = sess.run(uninit_vars) if exp_config.debug: logging.info( '============================================================') logging.info('This is the list of uninitialized variables:') for v in uninit_variables: print(v) # ================================================================ # Restore the segmentation network parameters and the pre-trained i2i mapper parameters # ================================================================ if exp_config.train_from_scratch is False: logging.info( '============================================================') path_to_model = sys_config.log_root + exp_config.expname_i2l + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_dice.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_lowest_loss.restore(sess, checkpoint_path) # ================================================================ # run training steps # ================================================================ step = 0 lowest_loss = 10000.0 validation_total_loss_list = [] while (step < exp_config.max_steps): # ================================================ # batches # ================================================ for batch in iterate_minibatches(images_sd=images_sd_tr, labels_sd=labels_sd_tr, images_td=images_td_tr, batch_size=exp_config.batch_size): x_sd, y_sd, x_td = batch # =========================== # define feed dict for this iteration # =========================== feed_dict = { images_sd_pl: x_sd, labels_sd_pl: y_sd, images_td_pl: x_td, training_pl: True } # ================================================ # update i2l and D successively # ================================================ sess.run(train_i2l_op, feed_dict=feed_dict) sess.run(train_discriminator_op, feed_dict=feed_dict) # =========================== # write the summaries and print an overview fairly often # =========================== if (step + 1) % exp_config.summary_writing_frequency == 0: logging.info( '============== Updating summary at step %d ' % step) summary_writer.add_summary( sess.run(summary_scalars, feed_dict=feed_dict), step) summary_writer.flush() # =========================== # Compute the loss on the entire training set # =========================== if step % exp_config.train_eval_frequency == 0: logging.info('============== Training Data Eval:') train_loss_seg, train_dice, train_loss_invariance = do_eval( sess, eval_loss, images_sd_pl, labels_sd_pl, images_td_pl, training_pl, images_sd_tr, labels_sd_tr, images_td_tr, exp_config.batch_size) # total training loss train_total_loss = train_loss_seg + exp_config.lambda_uda * train_loss_invariance # =========================== # update tensorboard summary of scalars # =========================== tr_summary_msg = sess.run(tr_summary, feed_dict={ tr_error_seg: train_loss_seg, tr_dice: train_dice, tr_error_invariance: train_loss_invariance, tr_error_total: train_total_loss }) summary_writer.add_summary(tr_summary_msg, step) # =========================== # Save a checkpoint periodically # =========================== if step % exp_config.save_frequency == 0: logging.info( '============== Periodically saving checkpoint:') checkpoint_file = os.path.join(log_dir, 'models/model.ckpt') saver.save(sess, checkpoint_file, global_step=step) # =========================== # Evaluate the model periodically on a validation set # =========================== if step % exp_config.val_eval_frequency == 0: logging.info('============== Validation Data Eval:') val_loss_seg, val_dice, val_loss_invariance = do_eval( sess, eval_loss, images_sd_pl, labels_sd_pl, images_td_pl, training_pl, images_sd_vl, labels_sd_vl, images_td_vl, exp_config.batch_size) # total val loss val_total_loss = val_loss_seg + exp_config.lambda_uda * val_loss_invariance validation_total_loss_list.append(val_total_loss) # =========================== # update tensorboard summary of scalars # =========================== vl_summary_msg = sess.run(vl_summary, feed_dict={ vl_error_seg: val_loss_seg, vl_dice: val_dice, vl_error_invariance: val_loss_invariance, vl_error_total: val_total_loss }) summary_writer.add_summary(vl_summary_msg, step) # =========================== # update tensorboard summary of images # =========================== summary_writer.add_summary( sess.run(summary_images, feed_dict={ images_sd_pl: x_sd, labels_sd_pl: y_sd, training_pl: False }), step) summary_writer.flush() # =========================== # save model if the val dice is the best yet # =========================== window_length = 5 if len(validation_total_loss_list) < window_length + 1: expo_moving_avg_loss_value = validation_total_loss_list[ -1] else: expo_moving_avg_loss_value = utils.exponential_moving_average( validation_total_loss_list, window=window_length)[-1] if expo_moving_avg_loss_value < lowest_loss: lowest_loss = val_total_loss lowest_loss_file = os.path.join( log_dir, 'models/lowest_loss.ckpt') saver_lowest_loss.save(sess, lowest_loss_file, global_step=step) logging.info( '******* SAVED MODEL at NEW BEST AVERAGE LOSS on VALIDATION SET at step %d ********' % step) # ================================================ # increment step # ================================================ step += 1 # ================================================================ # close tf session # ================================================================ sess.close() return 0
def predict_segmentation(subject_name, image, normalize=True, post_process=False): # ================================================================ # build the TF graph # ================================================================ with tf.Graph().as_default(): # ================================================================ # create placeholders # ================================================================ images_pl = tf.placeholder(tf.float32, shape=[None] + list(exp_config.image_size) + [1], name='images') # ================================================================ # insert a normalization module in front of the segmentation network # the normalization module is trained for each test image # ================================================================ images_normalized, added_residual = model.normalize( images_pl, exp_config, training_pl=tf.constant(False, dtype=tf.bool)) # ================================================================ # build the graph that computes predictions from the inference model # ================================================================ predicted_seg_logits, predicted_seg_softmax, predicted_seg = model.predict_i2l( images_normalized, exp_config, training_pl=tf.constant(False, dtype=tf.bool)) # ================================================================ # 3d prior # ================================================================ labels_3d_shape = [1] + list(exp_config.image_size_downsampled) # predict the current segmentation for the entire volume, downsample it and pass it through this placeholder predicted_seg_3d_pl = tf.placeholder(tf.uint8, shape=labels_3d_shape, name='true_labels_3d') predicted_seg_1hot_3d_pl = tf.one_hot(predicted_seg_3d_pl, depth=exp_config.nlabels) # denoise the noisy segmentation _, pred_seg_softmax_3d_noisy_autoencoded_softmax, _ = model.predict_l2l( predicted_seg_1hot_3d_pl, exp_config, training_pl=tf.constant(False, dtype=tf.bool)) # ================================================================ # divide the vars into segmentation network and normalization network # ================================================================ i2l_vars = [] l2l_vars = [] normalization_vars = [] for v in tf.global_variables(): var_name = v.name if 'image_normalizer' in var_name: normalization_vars.append(v) i2l_vars.append( v ) # the normalization vars also need to be restored from the pre-trained i2l mapper elif 'i2l_mapper' in var_name: i2l_vars.append(v) elif 'l2l_mapper' in var_name: l2l_vars.append(v) # ================================================================ # add init ops # ================================================================ init_ops = tf.global_variables_initializer() # ================================================================ # create session # ================================================================ sess = tf.Session() # ================================================================ # create saver # ================================================================ saver_i2l = tf.train.Saver(var_list=i2l_vars) saver_normalizer = tf.train.Saver(var_list=normalization_vars) saver_l2l = tf.train.Saver(var_list=l2l_vars) # ================================================================ # freeze the graph before execution # ================================================================ tf.get_default_graph().finalize() # ================================================================ # Run the Op to initialize the variables. # ================================================================ sess.run(init_ops) # ================================================================ # Restore the segmentation network parameters # ================================================================ logging.info( '============================================================') path_to_model = sys_config.log_root + 'i2l_mapper/' + exp_config.expname_i2l + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_dice.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_i2l.restore(sess, checkpoint_path) # ================================================================ # Restore the prior network parameters # ================================================================ logging.info( '============================================================') path_to_model = sys_config.log_root + 'l2l_mapper/' + exp_config.expname_l2l + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_dice.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_l2l.restore(sess, checkpoint_path) # ================================================================ # Make predictions for the image at the resolution of the image after pre-processing # ================================================================ mask_predicted = [] mask_predicted_soft = [] img_normalized = [] for b_i in range(0, image.shape[0], 1): X = np.expand_dims(image[b_i:b_i + 1, ...], axis=-1) mask_predicted.append( sess.run(predicted_seg, feed_dict={images_pl: X})) mask_predicted_soft.append( sess.run(predicted_seg_softmax, feed_dict={images_pl: X})) img_normalized.append( sess.run(images_normalized, feed_dict={images_pl: X})) mask_predicted = np.squeeze(np.array(mask_predicted)).astype(float) mask_predicted_soft = np.squeeze( np.array(mask_predicted_soft)).astype(float) img_normalized = np.squeeze(np.array(img_normalized)).astype(float) # ================================================================ # downsample predicted mask and pass it through the DAE # ================================================================ if post_process is True: # downsample mask_predicted_soft mask_predicted_soft_downsampled = rescale(mask_predicted_soft, [ 1 / exp_config.downsampling_factor_x, 1 / exp_config.downsampling_factor_y, 1 / exp_config.downsampling_factor_z ], order=1, preserve_range=True, multichannel=True, mode='constant') mask_predicted_downsampled = np.argmax( mask_predicted_soft_downsampled, axis=-1) # pass the downsampled prediction through the DAE mask_predicted_denoised = mask_predicted_downsampled for _ in range(exp_config.dae_post_process_runs): feed_dict = { predicted_seg_3d_pl: np.expand_dims(mask_predicted_denoised, axis=0) } y_pred_noisy_denoised_softmax = np.squeeze( sess.run(pred_seg_softmax_3d_noisy_autoencoded_softmax, feed_dict=feed_dict)).astype(np.float16) mask_predicted_denoised = np.argmax( y_pred_noisy_denoised_softmax, axis=-1) # upsample the denoised prediction mask_predicted = rescale(mask_predicted_denoised, [ exp_config.downsampling_factor_x, exp_config.downsampling_factor_y, exp_config.downsampling_factor_z ], order=0, preserve_range=True, multichannel=False, mode='constant').astype(np.uint8) sess.close() return mask_predicted, img_normalized
def gradient_sobel(_img, mask="h"): """ :param _img: :param mask: h for gorizontal, v for vertical :return: """ if mask != "a": if mask == "h": mask = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]]) else: mask = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]) side = len(mask) // 2 img = np.append([_img[0]], _img, axis=0) img = np.append(img, [_img[-1]], axis=0) img2 = np.zeros((img.shape[0], img.shape[1]+2)) img2[:, 1:-1] = img img2[:, 1] = img[:,0] img2[:, -1] = img[:, -1] img = np.zeros(_img.shape) for i in range(img.shape[0]): for j in range(img.shape[1]): i1 = i + 1 j1 = j + 1 img[i, j] = np.average(img2[i1-side:i1+side+1, j1-side:j1+side+1] * mask) return img else: mask = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]]) side = len(mask) // 2 img = np.append([_img[0]], _img, axis=0) img = np.append(img, [_img[-1]], axis=0) img2 = np.zeros((img.shape[0], img.shape[1] + 2)) img2[:, 1:-1] = img img2[:, 1] = img[:, 0] img2[:, -1] = img[:, -1] img = np.zeros(_img.shape) for i in range(img.shape[0]): for j in range(img.shape[1]): i1 = i + 1 j1 = j + 1 img[i, j] = np.average(img2[i1 - side:i1 + side + 1, j1 - side:j1 + side + 1] * mask) img_1 = img mask = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]) img = np.append([_img[0]], _img, axis=0) img = np.append(img, [img[-1]], axis=0) img2 = np.zeros((img.shape[0], img.shape[1] + 2)) img2[:, 1:-1] = img img2[:, 1] = img[:, 0] img2[:, -1] = img[:, -1] img = np.zeros(_img.shape) for i in range(img.shape[0]): for j in range(img.shape[1]): i1 = i + 1 j1 = j + 1 img[i, j] = np.average(img2[i1 - side:i1 + side + 1, j1 - side:j1 + side + 1] * mask) img_2 = img mask = np.array([[1, 2, 1], [0, 0, 0], [-1, -2, -1]]) side = len(mask) // 2 img = np.append([img[0]], img, axis=0) img = np.append(img, [img[-1]], axis=0) img2 = np.zeros((img.shape[0], img.shape[1] + 2)) img2[:, 1:-1] = img img2[:, 1] = img[:, 0] img2[:, -1] = img[:, -1] img = np.zeros(_img.shape) for i in range(img.shape[0]): for j in range(img.shape[1]): i1 = i + 1 j1 = j + 1 img[i, j] = np.average(img2[i1 - side:i1 + side + 1, j1 - side:j1 + side + 1] * mask) img_3 = img mask = np.array([[1, 0, -1], [2, 0, -2], [1, 0, -1]]) side = len(mask) // 2 img = np.append([img[0]], img, axis=0) img = np.append(img, [img[-1]], axis=0) img2 = np.zeros((img.shape[0], img.shape[1] + 2)) img2[:, 1:-1] = img img2[:, 1] = img[:, 0] img2[:, -1] = img[:, -1] img = np.zeros(_img.shape) for i in range(img.shape[0]): for j in range(img.shape[1]): i1 = i + 1 j1 = j + 1 img[i, j] = np.average(img2[i1 - side:i1 + side + 1, j1 - side:j1 + side + 1] * mask) img_4 = img print(img_1.min(), img_1.max()) print(img_2.min(), img_2.max()) print(img_3.min(), img_3.max()) print(img_4.min(), img_4.max()) # return m.threshold_filter_2d(img_1,img_1.mean()-img_1.std(),255)+\ # m.threshold_filter_2d(img_2,img_2.mean()-img_2.std(),255)+\ # m.threshold_filter_2d(img_3,img_3.mean()-img_3.std(),255)+\ # m.threshold_filter_2d(img_4,img_4.mean()-img_4.std(),255) return m.normalize(np.absolute(img_1))+ \ m.normalize(np.absolute(img_2))+ \ m.normalize(np.absolute(img_3))+ \ m.normalize(np.absolute(img_4))
def getNormal(points): return model.normalize(numpy.cross( vertices[points[1]] - vertices[points[0]], vertices[points[2]] - vertices[points[0]]))
def writeTransform(spec, stream, mesh, level=0): started = time.time() if mesh.transform is None: translation = numpy.array([0., 0., 0.]) rotation = numpy.array([1., 0., 0., 0.]) scale = numpy.array([1., 1., 1.]) else: translation = mesh.transform.value.getA()[:,3][0:3] translationMatrix = numpy.matrix([ [1., 0., 0., -translation[0]], [0., 1., 0., -translation[1]], [0., 0., 1., -translation[2]], [0., 0., 0., 1.]]) translated = translationMatrix * mesh.transform.value scale = numpy.array([numpy.linalg.norm(translated.getA()[:,column][0:3]) for column in [0, 1, 2]]) scaleMatrix = numpy.matrix([ [1. / scale[0], 0., 0., 0.], [ 0., 1. / scale[1], 0., 0.], [ 0., 0., 1. / scale[2], 0.], [ 0., 0., 0., 1.]]) scaled = translated * scaleMatrix #Conversion from rotation matrix form to axis-angle form angle = math.acos(((scaled.trace() - 1.) - 1.) / 2.) if angle == 0.: rotation = numpy.array([1., 0., 0., 0.]) else: skew = (scaled - scaled.transpose()).getA() vector = numpy.array([skew[2][1], skew[0][2], skew[1][0]]) vector = (1. / (2. * math.sin(angle))) * vector vector = model.normalize(vector) if abs(angle) < math.pi: rotation = numpy.array(vector.tolist() + [angle]) else: tensor = numpy.tensordot(vector, vector, 0) values = numpy.array([tensor[2][1], tensor[0][2], tensor[1][0]]) vector = numpy.diag(tensor) vector = model.normalize(vector) posIndices, negIndices = [], [] for i in range(0, 3): if values[i] < 0.: negIndices.append(i) elif values[i] > 0.: posIndices.append(i) if len(posIndices) == 1 and len(negIndices) == 2: vector[posIndices[0]] *= -1. elif len(posIndices) == 0 and len(negIndices) == 1: vector[negIndices[0]] *= -1. rotation = numpy.array(vector.tolist() + [angle]) debug("Transform %s: translation %s, rotation %s, scale %s" % (mesh.ident, str(translation), str(rotation), str(scale))) stream.write("%sDEF OB_%s Transform {\n" % ("\t" * level, mesh.ident)) stream.write("%stranslation %f %f %f\n" % tuple(["\t" * (level + 1)] + translation.tolist())) stream.write("%srotation %f %f %f %f\n" % tuple(["\t" * (level + 1)] + rotation.tolist())) stream.write("%sscale %f %f %f\n" % tuple(["\t" * (level + 1)] + scale.tolist())) stream.write("%schildren [\n" % ("\t" * (level + 1))) parent = mesh if mesh.parent is None else mesh.parent writeGroup(spec, stream, parent, level + 2) stream.write("%s]\n" % ("\t" * (level + 1))) stream.write("%s}\n" % ("\t" * level)) debug("Mesh exported in %f, name %s" % (time.time() - started, mesh.ident))
def run_training(log_dir, image, label, atlas, continue_run, log_dir_first_TD_subject=''): # ============================ # down sample the atlas - the losses will be evaluated in the downsampled space # ============================ atlas_downsampled = rescale(atlas, [ 1 / exp_config.downsampling_factor_x, 1 / exp_config.downsampling_factor_y, 1 / exp_config.downsampling_factor_z ], order=1, preserve_range=True, multichannel=True, mode='constant') atlas_downsampled = utils.crop_or_pad_volume_to_size_along_x_1hot( atlas_downsampled, int(256 / exp_config.downsampling_factor_x)) label_onehot = utils.make_onehot(label, exp_config.nlabels) label_onehot_downsampled = rescale(label_onehot, [ 1 / exp_config.downsampling_factor_x, 1 / exp_config.downsampling_factor_y, 1 / exp_config.downsampling_factor_z ], order=1, preserve_range=True, multichannel=True, mode='constant') label_onehot_downsampled = utils.crop_or_pad_volume_to_size_along_x_1hot( label_onehot_downsampled, int(256 / exp_config.downsampling_factor_x)) # ============================ # Initialize step number - this is number of mini-batch runs # ============================ init_step = 0 # ============================ # if continue_run is set to True, load the model parameters saved earlier # else start training from scratch # ============================ if continue_run: logging.info( '============================================================') logging.info('Continuing previous run') try: init_checkpoint_path = utils.get_latest_model_checkpoint_path( log_dir, 'models/model.ckpt') logging.info('Checkpoint path: %s' % init_checkpoint_path) init_step = int(init_checkpoint_path.split('/')[-1].split('-')[-1]) logging.info('Latest step was: %d' % init_step) except: logging.warning( 'Did not find init checkpoint. Maybe first run failed. Disabling continue mode...' ) continue_run = False init_step = 0 logging.info( '============================================================') # ================================================================ # reset the graph built so far and build a new TF graph # ================================================================ tf.reset_default_graph() with tf.Graph().as_default(): # ============================ # set random seed for reproducibility # ============================ tf.random.set_random_seed(exp_config.run_number) np.random.seed(exp_config.run_number) # ================================================================ # create placeholders - segmentation net # ================================================================ images_pl = tf.placeholder(tf.float32, shape=[exp_config.batch_size] + list(exp_config.image_size) + [1], name='images') learning_rate_pl = tf.placeholder(tf.float32, shape=[], name='learning_rate') training_pl = tf.placeholder(tf.bool, shape=[], name='training_or_testing') # ================================================================ # insert a normalization module in front of the segmentation network # the normalization module is trained for each test image # ================================================================ images_normalized, added_residual = model.normalize( images_pl, exp_config, training_pl) # ================================================================ # build the graph that computes predictions from the inference model # By setting the 'training_pl' to false directly, the update ops for the moments in the BN layer are not created at all. # This allows grouping the update ops together with the optimizer training, while training the normalizer - in case the normalizer has BN. # ================================================================ predicted_seg_logits, predicted_seg_softmax, predicted_seg = model.predict_i2l( images_normalized, exp_config, training_pl=tf.constant(False, dtype=tf.bool)) # ================================================================ # 3d prior # ================================================================ labels_3d_1hot_shape = [1] + list( exp_config.image_size_downsampled) + [exp_config.nlabels] # predict the current segmentation for the entire volume, downsample it and pass it through this placeholder predicted_seg_1hot_3d_pl = tf.placeholder(tf.float32, shape=labels_3d_1hot_shape, name='predicted_labels_3d') # denoise the noisy segmentation _, predicted_seg_softmax_3d_noisy_autoencoded_softmax, _ = model.predict_l2l( predicted_seg_1hot_3d_pl, exp_config, training_pl=tf.constant(False, dtype=tf.bool)) # ================================================================ # divide the vars into segmentation network and normalization network # ================================================================ i2l_vars = [] l2l_vars = [] normalization_vars = [] for v in tf.global_variables(): var_name = v.name if 'image_normalizer' in var_name: normalization_vars.append(v) i2l_vars.append( v ) # the normalization vars also need to be restored from the pre-trained i2l mapper elif 'i2l_mapper' in var_name: i2l_vars.append(v) elif 'l2l_mapper' in var_name: l2l_vars.append(v) # ================================================================ # Make a list of trainable i2l vars. This will be used to compute gradients. # The other list contains trainable as well as non-trainable parameters. This is required for saving and loading all parameters, but runs into trouble when gradients are asked to be computed for non-trainable parameters # ================================================================ i2l_vars_trainable = [] for v in i2l_vars: if v.trainable is True: i2l_vars_trainable.append(v) # ================================================================ # add ops for calculation of the prior loss - wrt an atlas or the outputs of the DAE # ================================================================ prior_label_1hot_pl = tf.placeholder( tf.float32, shape=[exp_config.batch_size_downsampled] + list( (exp_config.image_size_downsampled[1], exp_config.image_size_downsampled[2])) + [exp_config.nlabels], name='labels_prior') # down sample the predicted logits predicted_seg_logits_expanded = tf.expand_dims(predicted_seg_logits, axis=0) # the 'upsample' function will actually downsample the predictions, as the scaling factors have been set appropriately predicted_seg_logits_downsampled = layers.bilinear_upsample3D_( predicted_seg_logits_expanded, name='downsampled_predictions', factor_x=1 / exp_config.downsampling_factor_x, factor_y=1 / exp_config.downsampling_factor_y, factor_z=1 / exp_config.downsampling_factor_z) predicted_seg_logits_downsampled = tf.squeeze( predicted_seg_logits_downsampled ) # the first axis was added only for the downsampling in 3d # compute the dice between the predictions and the prior in the downsampled space loss_op = model.loss(logits=predicted_seg_logits_downsampled, labels=prior_label_1hot_pl, nlabels=exp_config.nlabels, loss_type=exp_config.loss_type_prior, mask_for_loss_within_mask=None, are_labels_1hot=True) tf.summary.scalar('tr_losses/loss', loss_op) # ================================================================ # one of the two prior losses will be used in the following manner: # the atlas prior will be used when the current prediction is deemed to be very far away from a reasonable solution # once a reasonable solution is reached, the dae prior will be used. # these 3d computations will be done outside the graph and will be passed via placeholders for logging in tensorboard # ================================================================ lambda_prior_atlas_pl = tf.placeholder(tf.float32, shape=[], name='lambda_prior_atlas') lambda_prior_dae_pl = tf.placeholder(tf.float32, shape=[], name='lambda_prior_dae') tf.summary.scalar('lambdas/prior_atlas', lambda_prior_atlas_pl) tf.summary.scalar('lambdas/prior_dae', lambda_prior_dae_pl) dice3d_prior_atlas_pl = tf.placeholder(tf.float32, shape=[], name='dice3d_prior_atlas') dice3d_prior_dae_pl = tf.placeholder(tf.float32, shape=[], name='dice3d_prior_dae') dice3d_gt_pl = tf.placeholder(tf.float32, shape=[], name='dice3d_gt') tf.summary.scalar('dice3d/prior_atlas', dice3d_prior_atlas_pl) tf.summary.scalar('dice3d/prior_dae', dice3d_prior_dae_pl) tf.summary.scalar('dice3d/gt', dice3d_gt_pl) # ================================================================ # add optimization ops # ================================================================ if exp_config.debug: print('creating training op...') # create an instance of the required optimizer optimizer = exp_config.optimizer_handle(learning_rate=learning_rate_pl) # initialize variable holding the accumlated gradients and create a zero-initialisation op accumulated_gradients = [ tf.Variable(tf.zeros_like(var.initialized_value()), trainable=False) for var in i2l_vars_trainable ] # accumulated gradients init op accumulated_gradients_zero_op = [ ac.assign(tf.zeros_like(ac)) for ac in accumulated_gradients ] # calculate gradients and define accumulation op update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): gradients = optimizer.compute_gradients( loss_op, var_list=i2l_vars_trainable) # compute_gradients return a list of (gradient, variable) pairs. accumulate_gradients_op = [ ac.assign_add(gg[0]) for ac, gg in zip(accumulated_gradients, gradients) ] # define the gradient mean op num_accumulation_steps_pl = tf.placeholder( dtype=tf.float32, name='num_accumulation_steps') accumulated_gradients_mean_op = [ ag.assign(tf.divide(ag, num_accumulation_steps_pl)) for ag in accumulated_gradients ] # reassemble the gradients in the [value, var] format and do define train op final_gradients = [(ag, gg[1]) for ag, gg in zip(accumulated_gradients, gradients)] train_op = optimizer.apply_gradients(final_gradients) # ================================================================ # sequence of running opt ops: # 1. at the start of each epoch, run accumulated_gradients_zero_op (no need to provide values for any placeholders) # 2. in each training iteration, run accumulate_gradients_op with regular feed dict of inputs and outputs # 3. at the end of the epoch (after all batches of the volume have been passed), run accumulated_gradients_mean_op, with a value for the placeholder num_accumulation_steps_pl # 4. finally, run the train_op. this also requires input output placeholders, as compute_gradients will be called again, but the returned gradient values will be replaced by the mean gradients. # ================================================================ # ================================================================ # previous train_op without accumulation of gradients # ================================================================ # train_op = model.training_step(loss_op, normalization_vars, exp_config.optimizer_handle, learning_rate_pl, update_bn_nontrainable_vars = True) # ================================================================ # build the summary Tensor based on the TF collection of Summaries. # ================================================================ if exp_config.debug: print('creating summary op...') # ================================================================ # add init ops # ================================================================ init_ops = tf.global_variables_initializer() # ================================================================ # find if any vars are uninitialized # ================================================================ if exp_config.debug: logging.info( 'Adding the op to get a list of initialized variables...') uninit_vars = tf.report_uninitialized_variables() # ================================================================ # create session # ================================================================ sess = tf.Session() # ================================================================ # create a summary writer # ================================================================ summary_writer = tf.summary.FileWriter(log_dir, sess.graph) # ================================================================ # summaries of the training errors # ================================================================ prior_dae_dice = tf.placeholder(tf.float32, shape=[], name='prior_dae_dice') prior_dae_dice_summary = tf.summary.scalar('test_img/prior_dae_dice', prior_dae_dice) prior_dae_output_dice_wrt_gt = tf.placeholder( tf.float32, shape=[], name='prior_dae_output_dice_wrt_gt') prior_dae_output_dice_wrt_gt_summary = tf.summary.scalar( 'test_img/prior_dae_output_dice_wrt_gt', prior_dae_output_dice_wrt_gt) prior_atlas_dice = tf.placeholder(tf.float32, shape=[], name='prior_atlas_dice') prior_atlas_dice_summary = tf.summary.scalar( 'test_img/prior_atlas_dice', prior_atlas_dice) prior_dae_atlas_dice_ratio = tf.placeholder( tf.float32, shape=[], name='prior_dae_atlas_dice_ratio') prior_dae_atlas_dice_ratio_summary = tf.summary.scalar( 'test_img/prior_dae_atlas_dice_ratio', prior_dae_atlas_dice_ratio) prior_dice = tf.placeholder(tf.float32, shape=[], name='prior_dice') prior_dice_summary = tf.summary.scalar('test_img/prior_dice', prior_dice) gt_dice = tf.placeholder(tf.float32, shape=[], name='gt_dice') gt_dice_summary = tf.summary.scalar('test_img/gt_dice', gt_dice) # ================================================================ # create savers # ================================================================ saver_i2l = tf.train.Saver(var_list=i2l_vars) saver_l2l = tf.train.Saver(var_list=l2l_vars) saver_test_data = tf.train.Saver(var_list=i2l_vars, max_to_keep=3) saver_best_loss = tf.train.Saver(var_list=i2l_vars, max_to_keep=3) # ================================================================ # add operations to compute dice between two 3d volumes # ================================================================ pred_3d_1hot_pl = tf.placeholder( tf.float32, shape=list(exp_config.image_size_downsampled) + [exp_config.nlabels], name='pred_3d') labl_3d_1hot_pl = tf.placeholder( tf.float32, shape=list(exp_config.image_size_downsampled) + [exp_config.nlabels], name='labl_3d') atls_3d_1hot_pl = tf.placeholder( tf.float32, shape=list(exp_config.image_size_downsampled) + [exp_config.nlabels], name='atls_3d') dice_3d_op_dae = losses.compute_dice_3d_without_batch_axis( prediction=pred_3d_1hot_pl, labels=labl_3d_1hot_pl) dice_3d_op_atlas = losses.compute_dice_3d_without_batch_axis( prediction=pred_3d_1hot_pl, labels=atls_3d_1hot_pl) # ================================================================ # freeze the graph before execution # ================================================================ if exp_config.debug: logging.info( '============================================================') logging.info('Freezing the graph now!') tf.get_default_graph().finalize() # ================================================================ # Run the Op to initialize the variables. # ================================================================ if exp_config.debug: logging.info( '============================================================') logging.info('initializing all variables...') sess.run(init_ops) # ================================================================ # print names of uninitialized variables # ================================================================ uninit_variables = sess.run(uninit_vars) if exp_config.debug: logging.info( '============================================================') logging.info('This is the list of uninitialized variables:') for v in uninit_variables: print(v) # ================================================================ # Restore the segmentation network parameters and the pre-trained i2i mapper parameters # After the adaptation for the 1st TD subject is done, start the adaptation for the subsequent subjects with those parameters # ================================================================ logging.info( '============================================================') path_to_model = sys_config.log_root + 'i2l_mapper/' + exp_config.expname_i2l + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_dice.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_i2l.restore(sess, checkpoint_path) # ================================================================ # Restore the prior network parameters # ================================================================ logging.info( '============================================================') path_to_model = sys_config.log_root + 'l2l_mapper/' + exp_config.expname_l2l + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_dice.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_l2l.restore(sess, checkpoint_path) # ================================================================ # After the adaptation for the 1st TD subject is done, start the adaptation for the subsequent subjects with those parameters # ================================================================ if log_dir_first_TD_subject is not '': logging.info( '============================================================') path_to_model = log_dir_first_TD_subject + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_score.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_test_data.restore(sess, checkpoint_path) max_steps_tta = exp_config.max_steps else: max_steps_tta = 5 * exp_config.max_steps # run the adaptation of the 1st TD subject for longer # ================================================================ # continue run from a saved checkpoint # ================================================================ if continue_run: # Restore session logging.info( '============================================================') logging.info('Restroring normalization module from: %s' % init_checkpoint_path) saver_test_data.restore(sess, init_checkpoint_path) # ================================================================ # run training epochs # ================================================================ step = init_step best_score = 0.0 while (step < max_steps_tta): # ================================================ # After every some epochs, # Get the prediction for the entire volume, evaluate it using the DAE. # Now, decide whether to use the DAE output or the atlas as the ground truth for the next update. # ================================================ if (step == init_step) or (step % exp_config.check_ood_frequency is 0): # ================== # 1. compute the current 3d segmentation prediction # ================== y_pred_soft = [] for batch in iterate_minibatches_images( image, batch_size=exp_config.batch_size): y_pred_soft.append( sess.run(predicted_seg_softmax, feed_dict={ images_pl: batch, training_pl: False })) y_pred_soft = np.squeeze(np.array(y_pred_soft)).astype(float) y_pred_soft = np.reshape(y_pred_soft, [ -1, y_pred_soft.shape[2], y_pred_soft.shape[3], y_pred_soft.shape[4] ]) # ================== # 2. downsample it. Let's call this guy 'A' # ================== y_pred_soft_downsampled = rescale(y_pred_soft, [ 1 / exp_config.downsampling_factor_x, 1 / exp_config.downsampling_factor_y, 1 / exp_config.downsampling_factor_z ], order=1, preserve_range=True, multichannel=True, mode='constant').astype( np.float32) # ================== # 3. pass the downsampled prediction through the DAE and get its output 'B' # ================== feed_dict = { predicted_seg_1hot_3d_pl: np.expand_dims(y_pred_soft_downsampled, axis=0) } y_pred_noisy_denoised_softmax = np.squeeze( sess.run( predicted_seg_softmax_3d_noisy_autoencoded_softmax, feed_dict=feed_dict)).astype(np.float16) y_pred_noisy_denoised = np.argmax( y_pred_noisy_denoised_softmax, axis=-1) # ================== # 4. compute the dice between: # a. 'A' (seg network prediction downsampled) and 'B' (dae network output) # b. 'B' (dae network output) and downsampled gt labels (for debugging, to see if the dae output is close to the gt.) # c. 'A' (seg network prediction downsampled) and 'C' (downsampled atlas) # ================== dAB = sess.run(dice_3d_op_dae, feed_dict={ pred_3d_1hot_pl: y_pred_soft_downsampled, labl_3d_1hot_pl: y_pred_noisy_denoised_softmax }) dBgt = sess.run(dice_3d_op_dae, feed_dict={ pred_3d_1hot_pl: y_pred_noisy_denoised_softmax, labl_3d_1hot_pl: label_onehot_downsampled }) dAC = sess.run(dice_3d_op_atlas, feed_dict={ pred_3d_1hot_pl: y_pred_soft_downsampled, atls_3d_1hot_pl: atlas_downsampled }) # ================== # 5. compute the ratio dice(AB) / dice(AC). pass the ratio through a threshold and decide whether to use the DAE or the atlas as the prior # ================== ratio_dice = dAB / (dAC + 1e-5) if exp_config.use_gt_for_tta is True: target_labels_for_this_epoch = label_onehot_downsampled prr = dBgt elif (ratio_dice > exp_config.dae_atlas_ratio_threshold) and ( dAC > exp_config.min_atlas_dice): target_labels_for_this_epoch = y_pred_noisy_denoised_softmax prr = dAB else: target_labels_for_this_epoch = atlas_downsampled prr = dAC # ================== # update losses on tensorboard # ================== summary_writer.add_summary( sess.run(prior_dae_dice_summary, feed_dict={prior_dae_dice: dAB}), step) summary_writer.add_summary( sess.run(prior_dae_output_dice_wrt_gt_summary, feed_dict={prior_dae_output_dice_wrt_gt: dBgt}), step) summary_writer.add_summary( sess.run(prior_atlas_dice_summary, feed_dict={prior_atlas_dice: dAC}), step) summary_writer.add_summary( sess.run( prior_dae_atlas_dice_ratio_summary, feed_dict={prior_dae_atlas_dice_ratio: ratio_dice}), step) summary_writer.add_summary( sess.run(prior_dice_summary, feed_dict={prior_dice: prr}), step) # ================== # save best model so far # ================== if best_score < prr: best_score = prr best_file = os.path.join(log_dir, 'models/best_score.ckpt') saver_best_loss.save(sess, best_file, global_step=step) logging.info( 'Found new best score (%f) at step %d - Saving model.' % (best_score, step)) # ================== # dice wrt gt # ================== y_pred = [] for batch in iterate_minibatches_images( image, batch_size=exp_config.batch_size): y_pred.append( sess.run(predicted_seg, feed_dict={ images_pl: batch, training_pl: False })) y_pred = np.squeeze(np.array(y_pred)).astype(float) y_pred = np.reshape(y_pred, [-1, y_pred.shape[2], y_pred.shape[3]]) dice_wrt_gt = met.f1_score(label.flatten(), y_pred.flatten(), average=None) summary_writer.add_summary( sess.run(gt_dice_summary, feed_dict={gt_dice: np.mean(dice_wrt_gt[1:])}), step) # ================== # visualize results # ================== if step % exp_config.vis_frequency is 0: # =========================== # save checkpoint # =========================== logging.info( '=============== Saving checkkpoint at step %d ... ' % step) checkpoint_file = os.path.join(log_dir, 'models/model.ckpt') saver_test_data.save(sess, checkpoint_file, global_step=step) y_pred_noisy_denoised_upscaled = utils.crop_or_pad_volume_to_size_along_x( rescale(y_pred_noisy_denoised, [ exp_config.downsampling_factor_x, exp_config.downsampling_factor_y, exp_config.downsampling_factor_z ], order=0, preserve_range=True, multichannel=False, mode='constant'), image.shape[0]).astype(np.uint8) x_norm = [] for batch in iterate_minibatches_images( image, batch_size=exp_config.batch_size): x = batch x_norm.append( sess.run(images_normalized, feed_dict={ images_pl: x, training_pl: False })) x_norm = np.squeeze(np.array(x_norm)).astype(float) x_norm = np.reshape(x_norm, [-1, x_norm.shape[2], x_norm.shape[3]]) utils_vis.save_sample_results( x=image, x_norm=x_norm, x_diff=x_norm - image, y=y_pred, y_pred_dae=y_pred_noisy_denoised_upscaled, at=np.argmax(atlas, axis=-1), gt=label, savepath=log_dir + '/results/visualize_images/step' + str(step) + '.png') # ================================================ # Part of training ops sequence: # 1. At the start of each epoch, run accumulated_gradients_zero_op (no need to provide values for any placeholders) # ================================================ sess.run(accumulated_gradients_zero_op) num_accumulation_steps = 0 # ================================================ # batches # ================================================ for batch in iterate_minibatches_images_and_downsampled_labels( images=image, batch_size=exp_config.batch_size, labels_downsampled=target_labels_for_this_epoch, batch_size_downsampled=exp_config.batch_size_downsampled): x, y = batch # =========================== # define feed dict for this iteration # =========================== feed_dict = { images_pl: x, prior_label_1hot_pl: y, learning_rate_pl: exp_config.learning_rate, training_pl: True } # ================================================ # Part of training ops sequence: # 2. in each training iteration, run accumulate_gradients_op with regular feed dict of inputs and outputs # ================================================ sess.run(accumulate_gradients_op, feed_dict=feed_dict) num_accumulation_steps = num_accumulation_steps + 1 step += 1 # ================================================ # Part of training ops sequence: # 3. At the end of the epoch (after all batches of the volume have been passed), run accumulated_gradients_mean_op, with a value for the placeholder num_accumulation_steps_pl # ================================================ sess.run( accumulated_gradients_mean_op, feed_dict={num_accumulation_steps_pl: num_accumulation_steps}) # ================================================================ # sequence of running opt ops: # 4. finally, run the train_op. this also requires input output placeholders, as compute_gradients will be called again, but the returned gradient values will be replaced by the mean gradients. # ================================================================ sess.run(train_op, feed_dict=feed_dict) # ================================================================ # ================================================================ sess.close() # ================================================================ # ================================================================ gc.collect() return 0
def predict_segmentation(subject_name, image, normalize=True): # ================================================================ # build the TF graph # ================================================================ with tf.Graph().as_default(): # ================================================================ # create placeholders # ================================================================ images_pl = tf.placeholder(tf.float32, shape=[None] + list(exp_config.image_size) + [1], name='images') # ================================================================ # insert a normalization module in front of the segmentation network # the normalization module is trained for each test image # ================================================================ images_normalized, added_residual = model.normalize( images_pl, exp_config, training_pl=tf.constant(False, dtype=tf.bool)) # ================================================================ # build the graph that computes predictions from the inference model # ================================================================ logits, softmax, preds = model.predict_i2l(images_normalized, exp_config, training_pl=tf.constant( False, dtype=tf.bool)) # ================================================================ # divide the vars into segmentation network and normalization network # ================================================================ i2l_vars = [] normalization_vars = [] for v in tf.global_variables(): var_name = v.name i2l_vars.append(v) if 'image_normalizer' in var_name: normalization_vars.append(v) # ================================================================ # add init ops # ================================================================ init_ops = tf.global_variables_initializer() # ================================================================ # create session # ================================================================ sess = tf.Session() # ================================================================ # create saver # ================================================================ saver_i2l = tf.train.Saver(var_list=i2l_vars) saver_normalizer = tf.train.Saver(var_list=normalization_vars) # ================================================================ # freeze the graph before execution # ================================================================ tf.get_default_graph().finalize() # ================================================================ # Run the Op to initialize the variables. # ================================================================ sess.run(init_ops) # ================================================================ # Restore the segmentation network parameters # ================================================================ logging.info( '============================================================') path_to_model = sys_config.log_root + 'i2l_mapper/' + exp_config.expname_i2l + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_dice.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_i2l.restore(sess, checkpoint_path) # ================================================================ # Restore the normalization network parameters # ================================================================ if normalize is True: logging.info( '============================================================') path_to_model = os.path.join( sys_config.log_root, exp_config.expname_normalizer ) + '/subject_' + subject_name + '/models/' checkpoint_path = utils.get_latest_model_checkpoint_path( path_to_model, 'best_score.ckpt') logging.info('Restoring the trained parameters from %s...' % checkpoint_path) saver_normalizer.restore(sess, checkpoint_path) logging.info( '============================================================') # ================================================================ # Make predictions for the image at the resolution of the image after pre-processing # ================================================================ mask_predicted = [] img_normalized = [] for b_i in range(0, image.shape[0], 1): X = np.expand_dims(image[b_i:b_i + 1, ...], axis=-1) mask_predicted.append(sess.run(preds, feed_dict={images_pl: X})) img_normalized.append( sess.run(images_normalized, feed_dict={images_pl: X})) mask_predicted = np.squeeze(np.array(mask_predicted)).astype(float) img_normalized = np.squeeze(np.array(img_normalized)).astype(float) sess.close() return mask_predicted, img_normalized
if __name__ == "__main__": train_images_idx3_ubyte_file = '.\\minist\\train-images-idx3-ubyte\\train-images.idx3-ubyte' train_label_idx1_ubyte_file = '.\\minist\\train-labels-idx1-ubyte\\train-labels.idx1-ubyte' test_images_idx3_ubyte_file = '.\\minist\\t10k-images-idx3-ubyte\\t10k-images.idx3-ubyte' test_label_idx1_ubyte_file = '.\\minist\\t10k-labels-idx1-ubyte\\t10k-labels.idx1-ubyte' #test_data = model.normalize(np.array(model.decode_idx3_ubyte(test_images_idx3_ubyte_file))) #test_label = np.array(model.decode_idx1_ubyte(test_label_idx1_ubyte_file)) #btSVM = BTreeSVM() #btSVM.LoadModel("D:\\SVM\\BTSVM\\test") # train_data = model.normalize(model.decode_idx3_ubyte(train_images_idx3_ubyte_file)) # train_label = model.decode_idx1_ubyte(train_label_idx1_ubyte_file) test_data = model.normalize(np.array(model.decode_idx3_ubyte(test_images_idx3_ubyte_file))) test_label = np.array(model.decode_idx1_ubyte(test_label_idx1_ubyte_file)) # print(len(test_data)) # print(len(test_label)) btSVM = BTreeSVM() # btSVM.SetData(train_data, train_label, test_data, test_label, 10) # btSVM.BuildTree() # btSVM.PrintTree() # btSVM.Save("D:\\SVM\\BTSVM\\m1") btSVM.LoadModel("D:\\SVM\\BTSVM\\m1") right = 0 for i in range(len(test_data)): #for i in range(10): p = btSVM.Predict(test_data[i]) if p == int(test_label[i]): right += 1
def run_training(continue_run): # ============================ # log experiment details # ============================ logging.info( '============================================================') logging.info('EXPERIMENT NAME: %s' % exp_config.experiment_name_i2l) # ============================ # Initialize step number - this is number of mini-batch runs # ============================ init_step = 0 # ============================ # if continue_run is set to True, load the model parameters saved earlier # else start training from scratch # ============================ if continue_run: logging.info( '============================================================') logging.info('Continuing previous run') try: init_checkpoint_path = utils.get_latest_model_checkpoint_path( log_dir, 'models/model.ckpt') logging.info('Checkpoint path: %s' % init_checkpoint_path) init_step = int( init_checkpoint_path.split('/')[-1].split('-') [-1]) + 1 # plus 1 as otherwise starts with eval logging.info('Latest step was: %d' % init_step) except: logging.warning( 'Did not find init checkpoint. Maybe first run failed. Disabling continue mode...' ) continue_run = False init_step = 0 logging.info( '============================================================') # ============================ # Load data # ============================ logging.info( '============================================================') logging.info('Loading data...') if exp_config.train_dataset is 'HCPT1': logging.info('Reading HCPT1 images...') logging.info('Data root directory: ' + sys_config.orig_data_root_hcp) data_brain_train = data_hcp.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_hcp, preprocessing_folder=sys_config.preproc_folder_hcp, idx_start=0, idx_end=1040, protocol='T1', size=exp_config.image_size, depth=exp_config.image_depth_hcp, target_resolution=exp_config.target_resolution_brain) imtr, gttr = [data_brain_train['images'], data_brain_train['labels']] data_brain_val = data_hcp.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_hcp, preprocessing_folder=sys_config.preproc_folder_hcp, idx_start=20, idx_end=25, protocol='T1', size=exp_config.image_size, depth=exp_config.image_depth_hcp, target_resolution=exp_config.target_resolution_brain) imvl, gtvl = [data_brain_val['images'], data_brain_val['labels']] if exp_config.train_dataset is 'HCPT2': logging.info('Reading HCPT2 images...') logging.info('Data root directory: ' + sys_config.orig_data_root_hcp) data_brain_train = data_hcp.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_hcp, preprocessing_folder=sys_config.preproc_folder_hcp, idx_start=0, idx_end=20, protocol='T2', size=exp_config.image_size, depth=exp_config.image_depth_hcp, target_resolution=exp_config.target_resolution_brain) imtr, gttr = [data_brain_train['images'], data_brain_train['labels']] data_brain_val = data_hcp.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_hcp, preprocessing_folder=sys_config.preproc_folder_hcp, idx_start=20, idx_end=25, protocol='T2', size=exp_config.image_size, depth=exp_config.image_depth_hcp, target_resolution=exp_config.target_resolution_brain) imvl, gtvl = [data_brain_val['images'], data_brain_val['labels']] elif exp_config.train_dataset is 'CALTECH': logging.info('Reading CALTECH images...') logging.info('Data root directory: ' + sys_config.orig_data_root_abide + 'CALTECH/') data_brain_train = data_abide.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_abide, preprocessing_folder=sys_config.preproc_folder_abide, site_name='CALTECH', idx_start=0, idx_end=10, protocol='T1', size=exp_config.image_size, depth=exp_config.image_depth_caltech, target_resolution=exp_config.target_resolution_brain) imtr, gttr = [data_brain_train['images'], data_brain_train['labels']] data_brain_val = data_abide.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_abide, preprocessing_folder=sys_config.preproc_folder_abide, site_name='CALTECH', idx_start=10, idx_end=15, protocol='T1', size=exp_config.image_size, depth=exp_config.image_depth_caltech, target_resolution=exp_config.target_resolution_brain) imvl, gtvl = [data_brain_val['images'], data_brain_val['labels']] elif exp_config.train_dataset is 'STANFORD': logging.info('Reading STANFORD images...') logging.info('Data root directory: ' + sys_config.orig_data_root_abide + 'STANFORD/') data_brain_train = data_abide.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_abide, preprocessing_folder=sys_config.preproc_folder_abide, site_name='STANFORD', idx_start=0, idx_end=10, protocol='T1', size=exp_config.image_size, depth=exp_config.image_depth_stanford, target_resolution=exp_config.target_resolution_brain) imtr, gttr = [data_brain_train['images'], data_brain_train['labels']] data_brain_val = data_abide.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_abide, preprocessing_folder=sys_config.preproc_folder_abide, site_name='STANFORD', idx_start=10, idx_end=15, protocol='T1', size=exp_config.image_size, depth=exp_config.image_depth_stanford, target_resolution=exp_config.target_resolution_brain) imvl, gtvl = [data_brain_val['images'], data_brain_val['labels']] elif exp_config.train_dataset is 'IXI': logging.info('Reading IXI images...') logging.info('Data root directory: ' + sys_config.orig_data_root_ixi) data_brain_train = data_ixi.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_ixi, preprocessing_folder=sys_config.preproc_folder_ixi, idx_start=0, idx_end=12, protocol='T2', size=exp_config.image_size, depth=exp_config.image_depth_ixi, target_resolution=exp_config.target_resolution_brain) imtr, gttr = [data_brain_train['images'], data_brain_train['labels']] data_brain_val = data_ixi.load_and_maybe_process_data( input_folder=sys_config.orig_data_root_ixi, preprocessing_folder=sys_config.preproc_folder_ixi, idx_start=12, idx_end=17, protocol='T2', size=exp_config.image_size, depth=exp_config.image_depth_ixi, target_resolution=exp_config.target_resolution_brain) imvl, gtvl = [data_brain_val['images'], data_brain_val['labels']] logging.info( 'Training Images: %s' % str(imtr.shape)) # expected: [num_slices, img_size_x, img_size_y] logging.info( 'Training Labels: %s' % str(gttr.shape)) # expected: [num_slices, img_size_x, img_size_y] logging.info('Validation Images: %s' % str(imvl.shape)) logging.info('Validation Labels: %s' % str(gtvl.shape)) logging.info( '============================================================') # ================================================================ # build the TF graph # ================================================================ with tf.Graph().as_default(): # ============================ # set random seed for reproducibility # ============================ tf.random.set_random_seed(exp_config.run_number) np.random.seed(exp_config.run_number) # ================================================================ # create placeholders # ================================================================ logging.info('Creating placeholders...') image_tensor_shape = [exp_config.batch_size] + list( exp_config.image_size) + [1] mask_tensor_shape = [exp_config.batch_size] + list( exp_config.image_size) images_pl = tf.placeholder(tf.float32, shape=image_tensor_shape, name='images') labels_pl = tf.placeholder(tf.uint8, shape=mask_tensor_shape, name='labels') learning_rate_pl = tf.placeholder(tf.float32, shape=[], name='learning_rate') training_pl = tf.placeholder(tf.bool, shape=[], name='training_or_testing') # ================================================================ # insert a normalization module in front of the segmentation network # the normalization module will be adapted for each test image # ================================================================ images_normalized, _ = model.normalize(images_pl, exp_config, training_pl) # ================================================================ # build the graph that computes predictions from the inference model # ================================================================ logits, _, _ = model.predict_i2l(images_normalized, exp_config, training_pl=training_pl) print('shape of inputs: ', images_pl.shape) # (batch_size, 256, 256, 1) print('shape of logits: ', logits.shape) # (batch_size, 256, 256, nlabels) # ================================================================ # create a list of all vars that must be optimized wrt # ================================================================ i2l_vars = [] for v in tf.trainable_variables(): i2l_vars.append(v) # ================================================================ # add ops for calculation of the supervised training loss # ================================================================ loss_op = model.loss(logits, labels_pl, nlabels=exp_config.nlabels, loss_type=exp_config.loss_type_i2l) tf.summary.scalar('loss', loss_op) # ================================================================ # add optimization ops. # Create different ops according to the variables that must be trained # ================================================================ print('creating training op...') train_op = model.training_step(loss_op, i2l_vars, exp_config.optimizer_handle, learning_rate_pl, update_bn_nontrainable_vars=True) # ================================================================ # add ops for model evaluation # ================================================================ print('creating eval op...') eval_loss = model.evaluation_i2l(logits, labels_pl, images_pl, nlabels=exp_config.nlabels, loss_type=exp_config.loss_type_i2l) # ================================================================ # build the summary Tensor based on the TF collection of Summaries. # ================================================================ print('creating summary op...') summary = tf.summary.merge_all() # ================================================================ # add init ops # ================================================================ init_ops = tf.global_variables_initializer() # ================================================================ # find if any vars are uninitialized # ================================================================ logging.info('Adding the op to get a list of initialized variables...') uninit_vars = tf.report_uninitialized_variables() # ================================================================ # create saver # ================================================================ saver = tf.train.Saver(max_to_keep=10) saver_best_dice = tf.train.Saver(max_to_keep=3) # ================================================================ # create session # ================================================================ sess = tf.Session() # ================================================================ # create a summary writer # ================================================================ summary_writer = tf.summary.FileWriter(log_dir, sess.graph) # ================================================================ # summaries of the validation errors # ================================================================ vl_error = tf.placeholder(tf.float32, shape=[], name='vl_error') vl_error_summary = tf.summary.scalar('validation/loss', vl_error) vl_dice = tf.placeholder(tf.float32, shape=[], name='vl_dice') vl_dice_summary = tf.summary.scalar('validation/dice', vl_dice) vl_summary = tf.summary.merge([vl_error_summary, vl_dice_summary]) # ================================================================ # summaries of the training errors # ================================================================ tr_error = tf.placeholder(tf.float32, shape=[], name='tr_error') tr_error_summary = tf.summary.scalar('training/loss', tr_error) tr_dice = tf.placeholder(tf.float32, shape=[], name='tr_dice') tr_dice_summary = tf.summary.scalar('training/dice', tr_dice) tr_summary = tf.summary.merge([tr_error_summary, tr_dice_summary]) # ================================================================ # freeze the graph before execution # ================================================================ logging.info('Freezing the graph now!') tf.get_default_graph().finalize() # ================================================================ # Run the Op to initialize the variables. # ================================================================ logging.info( '============================================================') logging.info('initializing all variables...') sess.run(init_ops) # ================================================================ # print names of all variables # ================================================================ logging.info( '============================================================') logging.info('This is the list of all variables:') for v in tf.trainable_variables(): print(v.name) # ================================================================ # print names of uninitialized variables # ================================================================ logging.info( '============================================================') logging.info('This is the list of uninitialized variables:') uninit_variables = sess.run(uninit_vars) for v in uninit_variables: print(v) # ================================================================ # continue run from a saved checkpoint # ================================================================ if continue_run: # Restore session logging.info( '============================================================') logging.info('Restroring session from: %s' % init_checkpoint_path) saver.restore(sess, init_checkpoint_path) # ================================================================ # ================================================================ step = init_step curr_lr = exp_config.learning_rate best_dice = 0 # ================================================================ # run training epochs # ================================================================ while (step < exp_config.max_steps): if step % 1000 is 0: logging.info( '============================================================' ) logging.info('step %d' % step) # ================================================ # batches # ================================================ for batch in iterate_minibatches(imtr, gttr, batch_size=exp_config.batch_size, train_or_eval='train'): curr_lr = exp_config.learning_rate start_time = time.time() x, y = batch # =========================== # avoid incomplete batches # =========================== if y.shape[0] < exp_config.batch_size: step += 1 continue # =========================== # create the feed dict for this training iteration # =========================== feed_dict = { images_pl: x, labels_pl: y, learning_rate_pl: curr_lr, training_pl: True } # =========================== # opt step # =========================== _, loss = sess.run([train_op, loss_op], feed_dict=feed_dict) # =========================== # compute the time for this mini-batch computation # =========================== duration = time.time() - start_time # =========================== # write the summaries and print an overview fairly often # =========================== if (step + 1) % exp_config.summary_writing_frequency == 0: logging.info( 'Step %d: loss = %.3f (%.3f sec for the last step)' % (step + 1, loss, duration)) # =========================== # Update the events file # =========================== summary_str = sess.run(summary, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) summary_writer.flush() # =========================== # Compute the loss on the entire training set # =========================== if step % exp_config.train_eval_frequency == 0: logging.info('Training Data Eval:') train_loss, train_dice = do_eval(sess, eval_loss, images_pl, labels_pl, training_pl, imtr, gttr, exp_config.batch_size) tr_summary_msg = sess.run(tr_summary, feed_dict={ tr_error: train_loss, tr_dice: train_dice }) summary_writer.add_summary(tr_summary_msg, step) # =========================== # Save a checkpoint periodically # =========================== if step % exp_config.save_frequency == 0: checkpoint_file = os.path.join(log_dir, 'models/model.ckpt') saver.save(sess, checkpoint_file, global_step=step) # =========================== # Evaluate the model periodically on a validation set # =========================== if step % exp_config.val_eval_frequency == 0: logging.info('Validation Data Eval:') val_loss, val_dice = do_eval(sess, eval_loss, images_pl, labels_pl, training_pl, imvl, gtvl, exp_config.batch_size) vl_summary_msg = sess.run(vl_summary, feed_dict={ vl_error: val_loss, vl_dice: val_dice }) summary_writer.add_summary(vl_summary_msg, step) # =========================== # save model if the val dice is the best yet # =========================== if val_dice > best_dice: best_dice = val_dice best_file = os.path.join(log_dir, 'models/best_dice.ckpt') saver_best_dice.save(sess, best_file, global_step=step) logging.info( 'Found new average best dice on validation sets! - %f - Saving model.' % val_dice) step += 1 sess.close()
def calc_content_loss(y4, y5, c4, c5): loss = nn.MSELoss()(normalize(y4), normalize(c4)) loss += nn.MSELoss()(normalize(y5), normalize(c5)) return loss
import librosa import numpy as np from pydub import AudioSegment from cons_notes_data.extract import extract_features_loaded from model import normalize, to_librosa name = '/home/moby/PycharmProjects/data/gpt/wav_split/1/bending_up_down_half/bending_up_down_half_1_1.wav' name_idmt = '/home/moby/PycharmProjects/data/IDMT-SMT-32/dataset2/audio/AR_A_fret_0-20.wav' audio = AudioSegment.from_file(name, format="wav")[:3000] y, sr = librosa.load(name, sr=44100) normalized = normalize(to_librosa(audio)) print(len(normalized)) print(len(y)) print(normalized) print(y[3000:6000]) logmel, mfcc = extract_features_loaded(normalized, sr) print(len(np.ndarray.flatten(logmel)))