def extractBrainPart(imgDir, n_divisions=3, x_part=0, y_part=0, z_part=0): imgPath = os.path.join(imgDir, "*") # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "brainpart_" + str(n_divisions) + "_" + str(x_part) + "_" + str(y_part) + "_" + str(z_part) + "_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') data = pickle.load(save) save.close() return data allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) data = [] n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = img.get_data() # Resize to a <scale> proportion of original size imgData_resized = sp.ndimage.interpolation.zoom( imgData_original, scale) imgData_flipped = np.flip(imgData_resized, 0) data.append([similarity]) printProgress(i + 1, n_samples) print "\nStoring the features in " + outputFileName output = open(outputFileName, "wb") pickle.dump(data, output) output.close() print "Done" return data
def extractCompleteBrain(imgDir): imgPath = os.path.join(imgDir, "*") # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "complete_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') data = pickle.load(save) save.close() return data allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) data = [] n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = img.get_data() single_brain = imgData.flatten().tolist() data.append(single_brain) printProgress(i + 1, n_samples) print "\n!!!!!NOT Storing the features !!!!" #output = open(outputFileName,"wb") #pickle.dump(data,output) #output.close() #print "Done" return data
def extractColorPercentage(imgDir, upperDark, upperGray, firstColor, secondColor): imgPath = os.path.join(imgDir, "*") # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "2ColorPercentage_" + str(upperDark) + "_" + str(upperGray) + "_" + str(firstColor) + "_" + str(secondColor) + "_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') zoneAverages = pickle.load(save) save.close() return zoneAverages # Fetch all directory listings of set_train and sort them on the image number allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" printProgress(0, n_samples) allPercentages = [] for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = img.get_data() imgDataSlice = imgData[:, :, imgData.shape[2] / 2, 0] COUNTS = [] voxelsCounted = 0.0 blackCounted = 0 grayCounted = 0 whiteCounted = 0 for j in range(0, imgData.shape[0]): for k in range(0, imgData.shape[1]): value = imgDataSlice[j][k] if value > 0: voxelsCounted += 1 if value <= upperDark: blackCounted += 1 if value <= upperGray and value > upperDark: grayCounted += 1 if value > upperGray: whiteCounted += 1 if (voxelsCounted > 0): COUNTS.append(blackCounted) COUNTS.append(grayCounted) COUNTS.append(whiteCounted) allPercentages.append(COUNTS[firstColor] / COUNTS[secondColor]) printProgress(i + 1, n_samples) print "\nStoring the features in " + outputFileName output = open(outputFileName, "wb") pickle.dump(allPercentages, output) output.close() print "Done" return allPercentages
def output_components(path, client): i = 0 l = 0 offset = 3000 directory = client.get_json( "query/components?q=SELECT component_name__v, checksum__v, component_type__v, LONGTEXT(mdl_definition__v), LONGTEXT(json_definition__v) FROM vault_component__v WHERE status__v='active__v' ORDER BY component_type__v LIMIT {0}" .format(offset)) l = directory["responseDetails"]["size"] printProgress.printProgress(i, l, prefix='Progress:', suffix='Complete', barLength=50) csv = [] Helpers.append_line(csv, 'Component Type,Name,Checksum') for component in directory["data"]: name = component["component_name__v"] type = component["component_type__v"] checksum = component["checksum__v"] mdl = component["mdl_definition__v"] json_def = component["json_definition__v"] Helpers.append_line( csv, '{0},{1},{2}'.format(type, type + "." + name, checksum)) type_folder = (path + "/%s") % (type) if not os.path.exists(type_folder): os.makedirs(type_folder) #json_folder = (type_folder + "/json/") #if not os.path.exists(json_folder): # os.makedirs(json_folder) mdl_folder = (type_folder) # + "/mdl") if not os.path.exists(mdl_folder): os.makedirs(mdl_folder) #Helpers.dump_json_file(name, json_def, json_folder) with open(mdl_folder + "/" + name + ".txt", "w") as f: if (type == "Workflow"): Helpers.dump_json_file(name + ".json", json_def, mdl_folder + "/") else: f.write(mdl.encode('utf-8')) i += 1 printProgress.printProgress(i, l, prefix='Progress:', suffix='Complete' + " - " + name, barLength=50) return csv
def extractColoredZone(imgDir, minColor, maxColor, nPartitions=1): imgPath = os.path.join(imgDir, "*") allColoredZones = [] # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "coloredzones2d_" + str(nPartitions) + "_" + str(minColor) + "_" + str(maxColor) + "_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') zoneAverages = pickle.load(save) save.close() return zoneAverages # Fetch all directory listings of set_train and sort them on the image number allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" printProgress(0, n_samples) for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = img.get_data() imgDataDisected = imgData[20:148, 35:163, 88, 0] colZones = np.asarray([[[0] * nPartitions] * nPartitions] * nPartitions) # Size should be same for all dimensions, imgData should # have same dimensions for x, y, z all such that they can be # divided by nPartitions for x in range(imgDataDisected.shape[0]): for y in range(imgDataDisected.shape[1]): #for z in range(imgDataDisected.shape[2]): val = imgDataDisected[x][y] #[z] partX = int((x * nPartitions) / imgDataDisected.shape[0]) partY = int((y * nPartitions) / imgDataDisected.shape[1]) #partZ = int((z*nPartitions)/imgDataDisected.shape[2]) if val <= maxColor and val >= minColor: colZones[partX][partY] += 1 allColoredZones.append(colZones.flatten().tolist()) printProgress(i + 1, n_samples) print "\nStoring the features in " + outputFileName output = open(outputFileName, "wb") pickle.dump(allColoredZones, output) output.close() print "Done" return allColoredZones
def extractHistograms(imgDir, maxValue=4000, nBins=-1, nPartitions=1): if nBins == -1: nBins = maxValue # The number of different intensities per point of the histogram binSize = math.ceil((maxValue * 1.) / nBins) imgPath = os.path.join(imgDir, "*") # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "histograms_" + str(nBins) + "-" + str(maxValue) + "_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') histograms = pickle.load(save) save.close() return histograms # Fetch all directory listings of set_train and sort them on the image number allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) histograms = [] n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" printProgress(0, n_samples) for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = img.get_data() # Count occurances of each intensity below the maxValue single_brain = np.array([[[[0] * nBins] * nPartitions] * nPartitions] * nPartitions) for x in range(imgData.shape[0]): for y in range(imgData.shape[1]): for z in range(imgData.shape[2]): val = imgData[x][y][z][0] partX = int((x * nPartitions) / imgData.shape[0]) partY = int((y * nPartitions) / imgData.shape[1]) partZ = int((z * nPartitions) / imgData.shape[2]) if val < maxValue and val > 0: c = int(val / binSize) single_brain[partX][partY][partZ][c] += 1 histograms.append(single_brain.flatten().tolist()) printProgress(i + 1, n_samples) print "\nStoring the features in " + outputFileName output = open(outputFileName, "wb") pickle.dump(histograms, output) output.close() print "Done" return histograms
def output_components(path, client, includeWorkflow): comps = VaultService.get_component_types(client) i = 0 l = 0 # build list of names first for component_json in comps: l += len(component_json["names"]) printProgress.printProgress(i, l, prefix='Progress:', suffix='Complete', barLength=50) for component_json in comps: component_type = component_json["type"] type_folder = (path + "/%s") % (component_type) for component_name in component_json["names"]: if not os.path.exists(type_folder): os.makedirs(type_folder) name = component_type + "." + component_name if includeWorkflow and component_type == "Workflow": wf = VaultService.get_workflow(client, component_name) Helpers.dump_json_file(component_name, wf, type_folder + "/") else: with open(type_folder + "/" + component_name + ".mdl", "w") as f: mdl = VaultService.get_component(client, name) f.write(mdl.encode('utf-8')) i += 1 printProgress.printProgress(i, l, prefix='Progress:', suffix='Complete' + " - " + name, barLength=50)
def extractFlipSim(imgDir, nPartitions=8, exponent=50): imgPath = os.path.join(imgDir, "*") # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "flipsim_" + str(nPartitions) + "-" + str(exponent) + "_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') data = pickle.load(save) save.close() return data # Fetch all directory listings of set_train and sort them on the image number allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" data = [] printProgress(0, n_samples) for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = np.asarray(img.get_data()[:, :, :, 0]) imgData_flipped = np.flip(imgData, 0) part_size_x = int(round(float(imgData.shape[0]) / nPartitions)) part_size_y = int(round(float(imgData.shape[1]) / nPartitions)) part_size_z = int(round(float(imgData.shape[2]) / nPartitions)) partsSim = [] for x in range(nPartitions): for y in range(nPartitions): for z in range(nPartitions): x_start = x * part_size_x if (x == nPartitions - 1): x_stop = imgData.shape[0] else: x_stop = (x + 1) * part_size_x y_start = y * part_size_y if (y == nPartitions - 1): y_stop = imgData.shape[1] else: y_stop = (y + 1) * part_size_y z_start = z * part_size_z if (z == nPartitions - 1): z_stop = imgData.shape[2] else: z_stop = (z + 1) * part_size_z imgPart = imgData[x_start:x_stop, y_start:y_stop, z_start:z_stop] imgPart_flipped = imgData_flipped[x_start:x_stop, y_start:y_stop, z_start:z_stop] #ssim_val = ssim(imgPart,imgPart_flipped) mse_val = mse(imgPart, imgPart_flipped) #similarity = [ssim_val**exponent,mse_val] partsSim.append(mse_val) data.append(partsSim) printProgress(i + 1, n_samples) print "\nStoring the features in " + outputFileName output = open(outputFileName, "wb") pickle.dump(data, output) output.close() print "Done" return data
def extractThreeColors(imgDir, darkColor, grayColor, whiteColor, nPartitions=1): imgPath = os.path.join(imgDir, "*") # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "threeColors_" + str(nPartitions) + "_" + str(darkColor) + "_" + str(grayColor) + "_" + str(whiteColor) + "_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') zoneAverages = pickle.load(save) save.close() return zoneAverages # Fetch all directory listings of set_train and sort them on the image number allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" printProgress(0, n_samples) allPercentages = [] for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = img.get_data() imgDataDisected = imgData[:, :, :, 0] percentages = [] voxelsCounted = 0.0 blackCounted = 0 grayCounted = 0 whiteCounted = 0 for j in range(0, imgData.shape[0]): for k in range(0, imgData.shape[1]): for l in range(0, imgData.shape[2]): value = imgDataDisected[j][k][l] if value > 0: voxelsCounted += 1 if value <= darkColor: blackCounted += 1 if value <= grayColor and value > darkColor: grayCounted += 1 if value <= whiteColor and value > grayColor: whiteCounted += 1 if (voxelsCounted > 0): percentages.append(blackCounted / voxelsCounted) percentages.append(grayCounted / voxelsCounted) percentages.append(whiteCounted / voxelsCounted) allPercentages.append(percentages) printProgress(i + 1, n_samples) print "\nStoring the features in " + outputFileName output = open(outputFileName, "wb") pickle.dump(allPercentages, output) output.close() print "Done" return allPercentages
def extractZoneAverages(imgDir, nPartitions=1): imgPath = os.path.join(imgDir, "*") allZoneAverages = [] # This is the cache for the feature, used to make sure we do the heavy computations more often than neccesary outputFileName = os.path.join( featuresDir, "zoneavg_" + str(nPartitions) + "_" + imgDir.replace(os.sep, "-") + ".feature") if os.path.isfile(outputFileName): save = open(outputFileName, 'rb') zoneAverages = pickle.load(save) save.close() return zoneAverages # Fetch all directory listings of set_train and sort them on the image number allImageSrc = sorted(glob.glob(imgPath), key=extractImgNumber) n_samples = len(allImageSrc) print "Found " + str(n_samples) + " images!" print "Preparing the data" printProgress(0, n_samples) for i in range(0, n_samples): img = nib.load(allImageSrc[i]) imgData = img.get_data() imgDataDisected = imgData[20:148, 35:163, 30:158, 0] # Size should be same for all dimensions, imgData should # have same dimensions for x, y, z all such that they can be # divided by nPartitions size = imgDataDisected.shape[0] / nPartitions ''' zoneAverages = [] for j in range(0,nPartitions): for k in range(0, nPartitions): for l in range(0, nPartitions): xStart = j*size yStart = k*size zStart = l*size totalSum = 0 totalVoxels = size*size*size for x in range(xStart, xStart+size): for y in range(yStart, yStart+size): for z in range(zStart, zStart+size): totalSum += imgDataDisected[x][y][z] mean = totalSum/totalVoxels zoneAverages.append(mean) #print "brain " +str(i)+" zone " + str(j) + ", " + str(k) + ", " + str(l) + " with mean " +str(mean) ''' zoneAverages = np.array([[[0] * nPartitions] * nPartitions] * nPartitions) totalVoxels = size * size * size for x in range(imgDataDisected.shape[0]): for y in range(imgDataDisected.shape[1]): for z in range(imgDataDisected.shape[2]): val = imgDataDisected[x][y][z] partX = int((x * nPartitions) / imgDataDisected.shape[0]) partY = int((y * nPartitions) / imgDataDisected.shape[1]) partZ = int((z * nPartitions) / imgDataDisected.shape[2]) if val > 0: zoneAverages[partX][partY][partZ] += val for j in range(zoneAverages.shape[0]): for k in range(zoneAverages.shape[1]): for l in range(zoneAverages.shape[1]): zoneAverages[j][k][l] = float( zoneAverages[j][k][l]) / totalVoxels allZoneAverages.append(zoneAverages.flatten().tolist()) printProgress(i + 1, n_samples) print "\nStoring the features in " + outputFileName output = open(outputFileName, "wb") pickle.dump(allZoneAverages, output) output.close() print "Done" return allZoneAverages
def main(): print(""" ▓█████ ██▀███ ██▀███ ▒█████ ██▀███ ██████ ▓█ ▀ ▓██ ▒ ██▒▓██ ▒ ██▒▒██▒ ██▒▓██ ▒ ██▒▒██ ▒ ▒███ ▓██ ░▄█ ▒▓██ ░▄█ ▒▒██░ ██▒▓██ ░▄█ ▒░ ▓██▄ ▒▓█ ▄ ▒██▀▀█▄ ▒██▀▀█▄ ▒██ ██░▒██▀▀█▄ ▒ ██▒ ░▒████▒░██▓ ▒██▒░██▓ ▒██▒░ ████▓▒░░██▓ ▒██▒▒██████▒▒ ░░ ▒░ ░░ ▒▓ ░▒▓░░ ▒▓ ░▒▓░░ ▒░▒░▒░ ░ ▒▓ ░▒▓░▒ ▒▓▒ ▒ ░ ░ ░ ░ ░▒ ░ ▒░ ░▒ ░ ▒░ ░ ▒ ▒░ ░▒ ░ ▒░░ ░▒ ░ ░ ░ ░░ ░ ░░ ░ ░ ░ ░ ▒ ░░ ░ ░ ░ ░ ░ ░ ░ ░ ░ ░ ░ ░ """) i = 0 l = 0 use_cache = True client = VaultService.get_client() instance_name = datetime.datetime.now() components = mdl.get_component_type_names(client, use_cache) l = len(components) printProgress.printProgress(i, l, prefix='Progress:', suffix='Complete', barLength=50) first = True for component_type in components: error_url = 'configuration/{0}/errors'.format(component_type) try: errors_resp = client.get_json(error_url) if first: create_markdown_file('common-errors', errors_resp, True) header = [ 'Type', 'Code', 'Reason', 'Type', 'Developer message', 'End user message\n' ] Helpers.save_as_file('errors', ','.join(header), 'output/errors/csv2', 'csv') #export_csv('errors', errors_resp, True) i += 1 first = False create_markdown_file('{0}-errors'.format(component_type), errors_resp) #export_csv('errors', errors_resp) printProgress.printProgress(i, l, prefix='Progress:', suffix='Complete' + " - " + component_type, barLength=50) except ApiClient.ApiException: print('Failed to get errors for {0} from the API'.format( component_type)) i += 1 print('Done!')